Dashboard: flag stale alerts and tickets left open
Lint / JS (eslint) (push) Successful in 19s
Lint / Python (flake8) (push) Successful in 1m7s
Lint / Notify on failure (push) Skipped
Lint / Deploy (push) Skipped
Security / Python Security (bandit) (push) Successful in 52s
Test / Python Tests (pytest) (push) Successful in 44s
Lint / JS (eslint) (pull_request) Successful in 7s
Lint / Python (flake8) (pull_request) Successful in 50s
Lint / Notify on failure (pull_request) Skipped
Lint / Deploy (pull_request) Skipped
Security / Python Security (bandit) (pull_request) Successful in 52s
Test / Python Tests (pytest) (pull_request) Successful in 1m35s

Gandalf only comments on tickets, so a ticket can stay open long after its
alert cleared (pbs's ticket sat open for a month), and an alert can stay
active for days once nobody is looking.

- The monitor reads Tinker's open ticket list (tickets_api.php, read scope)
  every 15 minutes and saves it. A failed read keeps the previous list.
- Active alerts get a stale badge after monitor.stale_alert_days (3), and a
  "ticket closed" badge when their ticket was closed while still active.
- A Needs Follow-up panel lists open tickets whose every alert cleared at
  least monitor.stale_ticket_hours (24) ago.
- A "TO FOLLOW UP" status chip counts both. If the open ticket list is over
  2 hours old, the ticket flags are left off rather than guessed.
- Replaced an — in the alert count notice.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-11-14 15:57:58 -05:00
co-authored by Claude Opus 5.5
parent 7b80633088
commit 0921e82c54
10 changed files with 462 additions and 4 deletions
+71
View File
@@ -274,6 +274,72 @@ def _annotate_suppressions(events: list, suppressions: list) -> None:
ev['is_suppressed'] = db.check_suppressed(suppressions, 'host', name, detail)
# The monitor refreshes the open ticket list every 15 minutes; older than this and
# it has stopped working, so nothing is flagged rather than flagging on stale data.
_OPEN_TICKETS_MAX_AGE_S = 2 * 3600
def _stale_settings() -> tuple:
"""(days an alert can stay active before it counts as stale,
hours a ticket can stay open after its alert cleared)."""
mon = _config().get('monitor', {})
try:
days = float(mon.get('stale_alert_days', 3))
hours = int(mon.get('stale_ticket_hours', 24))
except (TypeError, ValueError):
days, hours = 3.0, 24
return days, hours
def _open_tickets() -> Optional[dict]:
"""{ticket_id: status} of open Tinker tickets, or None when the list is missing or out of date."""
try:
raw = db.get_state('open_tickets')
data = json.loads(raw) if raw else None
if not data or time.time() - float(data.get('checked_at', 0)) > _OPEN_TICKETS_MAX_AGE_S:
return None
return {str(k): v for k, v in (data.get('tickets') or {}).items()}
except Exception as e:
logger.error(f'Reading open ticket list failed: {e}')
return None
def _annotate_staleness(events: list, open_tickets: Optional[dict]) -> None:
"""Add stale_days (set when an alert has been active too long) and ticket_closed
(its ticket was closed in Tinker while the alert is still active) to each event."""
days, _ = _stale_settings()
now = datetime.now(timezone.utc)
for ev in events:
ev['stale_days'] = None
ev['ticket_closed'] = False
try:
first = datetime.fromisoformat(str(ev.get('first_seen', '')).replace('Z', '+00:00'))
if first.tzinfo is None:
first = first.replace(tzinfo=timezone.utc)
age = (now - first).total_seconds() / 86400
if age >= days:
ev['stale_days'] = int(age)
except ValueError:
pass
tid = ev.get('ticket_id')
if tid and open_tickets is not None and str(tid) not in open_tickets:
ev['ticket_closed'] = True
def _stale_view(events: list) -> list:
"""Flag stale active alerts in place; return tickets left open after their alerts cleared."""
open_tickets = _open_tickets()
_annotate_staleness(events, open_tickets)
if open_tickets is None:
return []
_, hours = _stale_settings()
try:
return db.get_cleared_tickets(list(open_tickets), older_than_hours=hours)
except Exception as e:
logger.error(f'Finding tickets left open failed: {e}')
return []
def _state_json(key: str) -> dict:
"""Return a monitor_state JSON blob as a dict, or {} if missing or unreadable."""
raw = db.get_state(key)
@@ -307,6 +373,7 @@ def index():
snapshot = {}
suppressions = db.get_active_suppressions()
_annotate_suppressions(events, suppressions)
stale_tickets = _stale_view(events)
recent_resolved = db.get_recent_resolved(hours=24, limit=10)
return render_template(
'index.html',
@@ -318,6 +385,8 @@ def index():
last_check=last_check,
suppressions=suppressions,
recent_resolved=recent_resolved,
stale_tickets=stale_tickets,
stale_alerts=sum(1 for e in events if e.get('stale_days') is not None or e.get('ticket_closed')),
daemon_ok=_daemon_ok(last_check),
)
@@ -419,11 +488,13 @@ def api_status():
active = db.get_active_events(limit=_PAGE_LIMIT)
suppressions = db.get_active_suppressions()
_annotate_suppressions(active, suppressions)
stale_tickets = _stale_view(active)
last_check = db.get_state('last_check', 'Never')
return jsonify({
'summary': db.get_status_summary(),
'last_check': last_check,
'events': active,
'stale_tickets': stale_tickets,
'total_active': db.count_active_events(),
'daemon_ok': _daemon_ok(last_check),
})