Add recovery notes and fuller context to tickets
Lint / JS (eslint) (push) Successful in 7s
Lint / Python (flake8) (push) Successful in 1m26s
Lint / Notify on failure (push) Skipped
Lint / Deploy (push) Skipped
Security / Python Security (bandit) (push) Successful in 1m30s
Lint / Python (flake8) (pull_request) Successful in 52s
Test / Python Tests (pytest) (push) Successful in 1m4s
Lint / JS (eslint) (pull_request) Successful in 7s
Lint / Notify on failure (pull_request) Skipped
Lint / Deploy (pull_request) Skipped
Security / Python Security (bandit) (pull_request) Successful in 1m28s
Test / Python Tests (pytest) (pull_request) Successful in 2m0s

When an alert that opened a ticket clears, the monitor now comments on the
ticket (when it cleared, how long it lasted, failed checks, a link back).
It never closes or changes a ticket's status. Failed notes retry each
cycle for 24 hours; a 401/403 from Tinker stops them until restart.

Tickets are now Markdown with a "What to check" list and, where known:
the switch port the device plugs into (kept from the last time it was
seen, so a dead host still shows it), cutsheet cables and their path,
how often the alert fired in the last 7/30 days, other open alerts on the
device, and links into Gandalf (gandalf.public_url). Titles are unchanged
so Tinker's duplicate detection still works.

network_events gains recovery_notified, added at monitor startup; rows
that cleared before the upgrade count as already notified.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
jaredandClaude Opus 5.5 committed 2026-11-10 19:44:50 -05:00
1 parent ff0094c8ff
commit 437f7a415f
8 files changed
+886 -133

No files matched your search

+75
View File
@@ -348,6 +348,81 @@ def cleanup_expired_suppressions() -> int:
return n
def ensure_events_schema() -> None:
"""Add columns newer than the original events table, so deploys need no manual migration."""
with get_conn() as conn:
with conn.cursor() as cur:
cur.execute("SHOW COLUMNS FROM network_events LIKE 'recovery_notified'")
if cur.fetchone():
return
cur.execute(
'ALTER TABLE network_events ADD COLUMN recovery_notified TINYINT(1) NOT NULL DEFAULT 0'
)
# Events that cleared before this existed are history, not owed a comment
cur.execute(
'UPDATE network_events SET recovery_notified=1, last_seen=last_seen WHERE resolved_at IS NOT NULL'
)
def get_pending_recovery_notes(hours: int = 24, exclude_types: tuple = ()) -> list:
"""Resolved events with a ticket whose recovery comment hasn't been posted yet."""
sql = """SELECT id, event_type, target_name, target_detail, ticket_id,
first_seen, resolved_at, consecutive_failures
FROM network_events
WHERE resolved_at IS NOT NULL AND ticket_id IS NOT NULL AND recovery_notified=0
AND resolved_at > DATE_SUB(NOW(), INTERVAL %s HOUR)"""
params: list = [hours]
if exclude_types:
sql += f' AND event_type NOT IN ({", ".join(["%s"] * len(exclude_types))})' # nosec B608 - placeholders only
params.extend(exclude_types)
with get_conn() as conn:
with conn.cursor() as cur:
cur.execute(sql + ' ORDER BY resolved_at', params)
return list(cur.fetchall())
def mark_recovery_notified(event_id: int) -> None:
with get_conn() as conn:
with conn.cursor() as cur:
# last_seen updates itself on any write; keep it as the last failing check
cur.execute(
'UPDATE network_events SET recovery_notified=1, last_seen=last_seen WHERE id=%s',
(event_id,),
)
def event_stats(event_type: str, target_name: str, target_detail: str = '') -> dict:
"""How often one alert has fired before, for a ticket's history section.
Returns {'last_7d', 'last_30d', 'last_first_seen', 'last_resolved_at'} over resolved events.
"""
key = (event_type, target_name, target_detail or '')
with get_conn() as conn:
with conn.cursor() as cur:
cur.execute(
"""SELECT COALESCE(SUM(first_seen > DATE_SUB(NOW(), INTERVAL 7 DAY)), 0) AS last_7d,
COUNT(*) AS last_30d
FROM network_events
WHERE event_type=%s AND target_name=%s AND target_detail=%s
AND resolved_at IS NOT NULL AND first_seen > DATE_SUB(NOW(), INTERVAL 30 DAY)""",
key,
)
counts = cur.fetchone() or {}
cur.execute(
"""SELECT first_seen, resolved_at FROM network_events
WHERE event_type=%s AND target_name=%s AND target_detail=%s
AND resolved_at IS NOT NULL ORDER BY resolved_at DESC LIMIT 1""",
key,
)
last = cur.fetchone() or {}
return {
'last_7d': int(counts.get('last_7d') or 0),
'last_30d': int(counts.get('last_30d') or 0),
'last_first_seen': last.get('first_seen'),
'last_resolved_at': last.get('resolved_at'),
}
def purge_old_resolved_events(days: int = 90) -> int:
"""Delete resolved events older than `days` days. Returns count deleted."""
with get_conn() as conn: