Add recovery notes and fuller context to tickets
Lint / JS (eslint) (push) Successful in 7s
Lint / Python (flake8) (push) Successful in 1m26s
Lint / Notify on failure (push) Skipped
Lint / Deploy (push) Skipped
Security / Python Security (bandit) (push) Successful in 1m30s
Lint / Python (flake8) (pull_request) Successful in 52s
Test / Python Tests (pytest) (push) Successful in 1m4s
Lint / JS (eslint) (pull_request) Successful in 7s
Lint / Notify on failure (pull_request) Skipped
Lint / Deploy (pull_request) Skipped
Security / Python Security (bandit) (pull_request) Successful in 1m28s
Test / Python Tests (pytest) (pull_request) Successful in 2m0s
Lint / JS (eslint) (push) Successful in 7s
Lint / Python (flake8) (push) Successful in 1m26s
Lint / Notify on failure (push) Skipped
Lint / Deploy (push) Skipped
Security / Python Security (bandit) (push) Successful in 1m30s
Lint / Python (flake8) (pull_request) Successful in 52s
Test / Python Tests (pytest) (push) Successful in 1m4s
Lint / JS (eslint) (pull_request) Successful in 7s
Lint / Notify on failure (pull_request) Skipped
Lint / Deploy (pull_request) Skipped
Security / Python Security (bandit) (pull_request) Successful in 1m28s
Test / Python Tests (pytest) (pull_request) Successful in 2m0s
When an alert that opened a ticket clears, the monitor now comments on the ticket (when it cleared, how long it lasted, failed checks, a link back). It never closes or changes a ticket's status. Failed notes retry each cycle for 24 hours; a 401/403 from Tinker stops them until restart. Tickets are now Markdown with a "What to check" list and, where known: the switch port the device plugs into (kept from the last time it was seen, so a dead host still shows it), cutsheet cables and their path, how often the alert fired in the last 7/30 days, other open alerts on the device, and links into Gandalf (gandalf.public_url). Titles are unchanged so Tinker's duplicate detection still works. network_events gains recovery_notified, added at monitor startup; rows that cleared before the upgrade count as already notified. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
ff0094c8ff
commit
437f7a415f
8 files changed
+886
-133
No files matched your search
@@ -348,6 +348,81 @@ def cleanup_expired_suppressions() -> int:
|
||||
return n
|
||||
|
||||
|
||||
def ensure_events_schema() -> None:
|
||||
"""Add columns newer than the original events table, so deploys need no manual migration."""
|
||||
with get_conn() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("SHOW COLUMNS FROM network_events LIKE 'recovery_notified'")
|
||||
if cur.fetchone():
|
||||
return
|
||||
cur.execute(
|
||||
'ALTER TABLE network_events ADD COLUMN recovery_notified TINYINT(1) NOT NULL DEFAULT 0'
|
||||
)
|
||||
# Events that cleared before this existed are history, not owed a comment
|
||||
cur.execute(
|
||||
'UPDATE network_events SET recovery_notified=1, last_seen=last_seen WHERE resolved_at IS NOT NULL'
|
||||
)
|
||||
|
||||
|
||||
def get_pending_recovery_notes(hours: int = 24, exclude_types: tuple = ()) -> list:
|
||||
"""Resolved events with a ticket whose recovery comment hasn't been posted yet."""
|
||||
sql = """SELECT id, event_type, target_name, target_detail, ticket_id,
|
||||
first_seen, resolved_at, consecutive_failures
|
||||
FROM network_events
|
||||
WHERE resolved_at IS NOT NULL AND ticket_id IS NOT NULL AND recovery_notified=0
|
||||
AND resolved_at > DATE_SUB(NOW(), INTERVAL %s HOUR)"""
|
||||
params: list = [hours]
|
||||
if exclude_types:
|
||||
sql += f' AND event_type NOT IN ({", ".join(["%s"] * len(exclude_types))})' # nosec B608 - placeholders only
|
||||
params.extend(exclude_types)
|
||||
with get_conn() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(sql + ' ORDER BY resolved_at', params)
|
||||
return list(cur.fetchall())
|
||||
|
||||
|
||||
def mark_recovery_notified(event_id: int) -> None:
|
||||
with get_conn() as conn:
|
||||
with conn.cursor() as cur:
|
||||
# last_seen updates itself on any write; keep it as the last failing check
|
||||
cur.execute(
|
||||
'UPDATE network_events SET recovery_notified=1, last_seen=last_seen WHERE id=%s',
|
||||
(event_id,),
|
||||
)
|
||||
|
||||
|
||||
def event_stats(event_type: str, target_name: str, target_detail: str = '') -> dict:
|
||||
"""How often one alert has fired before, for a ticket's history section.
|
||||
|
||||
Returns {'last_7d', 'last_30d', 'last_first_seen', 'last_resolved_at'} over resolved events.
|
||||
"""
|
||||
key = (event_type, target_name, target_detail or '')
|
||||
with get_conn() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"""SELECT COALESCE(SUM(first_seen > DATE_SUB(NOW(), INTERVAL 7 DAY)), 0) AS last_7d,
|
||||
COUNT(*) AS last_30d
|
||||
FROM network_events
|
||||
WHERE event_type=%s AND target_name=%s AND target_detail=%s
|
||||
AND resolved_at IS NOT NULL AND first_seen > DATE_SUB(NOW(), INTERVAL 30 DAY)""",
|
||||
key,
|
||||
)
|
||||
counts = cur.fetchone() or {}
|
||||
cur.execute(
|
||||
"""SELECT first_seen, resolved_at FROM network_events
|
||||
WHERE event_type=%s AND target_name=%s AND target_detail=%s
|
||||
AND resolved_at IS NOT NULL ORDER BY resolved_at DESC LIMIT 1""",
|
||||
key,
|
||||
)
|
||||
last = cur.fetchone() or {}
|
||||
return {
|
||||
'last_7d': int(counts.get('last_7d') or 0),
|
||||
'last_30d': int(counts.get('last_30d') or 0),
|
||||
'last_first_seen': last.get('first_seen'),
|
||||
'last_resolved_at': last.get('resolved_at'),
|
||||
}
|
||||
|
||||
|
||||
def purge_old_resolved_events(days: int = 90) -> int:
|
||||
"""Delete resolved events older than `days` days. Returns count deleted."""
|
||||
with get_conn() as conn:
|
||||
|
||||
Reference in new issue
Block a user