Dashboard: flag stale alerts and tickets left open
Lint / JS (eslint) (push) Successful in 19s
Lint / Python (flake8) (push) Successful in 1m7s
Lint / Notify on failure (push) Skipped
Lint / Deploy (push) Skipped
Security / Python Security (bandit) (push) Successful in 52s
Test / Python Tests (pytest) (push) Successful in 44s
Lint / JS (eslint) (pull_request) Successful in 7s
Lint / Python (flake8) (pull_request) Successful in 50s
Lint / Notify on failure (pull_request) Skipped
Lint / Deploy (pull_request) Skipped
Security / Python Security (bandit) (pull_request) Successful in 52s
Test / Python Tests (pytest) (pull_request) Successful in 1m35s
Lint / JS (eslint) (push) Successful in 19s
Lint / Python (flake8) (push) Successful in 1m7s
Lint / Notify on failure (push) Skipped
Lint / Deploy (push) Skipped
Security / Python Security (bandit) (push) Successful in 52s
Test / Python Tests (pytest) (push) Successful in 44s
Lint / JS (eslint) (pull_request) Successful in 7s
Lint / Python (flake8) (pull_request) Successful in 50s
Lint / Notify on failure (pull_request) Skipped
Lint / Deploy (pull_request) Skipped
Security / Python Security (bandit) (pull_request) Successful in 52s
Test / Python Tests (pytest) (pull_request) Successful in 1m35s
Gandalf only comments on tickets, so a ticket can stay open long after its alert cleared (pbs's ticket sat open for a month), and an alert can stay active for days once nobody is looking. - The monitor reads Tinker's open ticket list (tickets_api.php, read scope) every 15 minutes and saves it. A failed read keeps the previous list. - Active alerts get a stale badge after monitor.stale_alert_days (3), and a "ticket closed" badge when their ticket was closed while still active. - A Needs Follow-up panel lists open tickets whose every alert cleared at least monitor.stale_ticket_hours (24) ago. - A "TO FOLLOW UP" status chip counts both. If the open ticket list is over 2 hours old, the ticket flags are left off rather than guessed. - Replaced an — in the alert count notice. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -274,6 +274,72 @@ def _annotate_suppressions(events: list, suppressions: list) -> None:
|
||||
ev['is_suppressed'] = db.check_suppressed(suppressions, 'host', name, detail)
|
||||
|
||||
|
||||
# The monitor refreshes the open ticket list every 15 minutes; older than this and
|
||||
# it has stopped working, so nothing is flagged rather than flagging on stale data.
|
||||
_OPEN_TICKETS_MAX_AGE_S = 2 * 3600
|
||||
|
||||
|
||||
def _stale_settings() -> tuple:
|
||||
"""(days an alert can stay active before it counts as stale,
|
||||
hours a ticket can stay open after its alert cleared)."""
|
||||
mon = _config().get('monitor', {})
|
||||
try:
|
||||
days = float(mon.get('stale_alert_days', 3))
|
||||
hours = int(mon.get('stale_ticket_hours', 24))
|
||||
except (TypeError, ValueError):
|
||||
days, hours = 3.0, 24
|
||||
return days, hours
|
||||
|
||||
|
||||
def _open_tickets() -> Optional[dict]:
|
||||
"""{ticket_id: status} of open Tinker tickets, or None when the list is missing or out of date."""
|
||||
try:
|
||||
raw = db.get_state('open_tickets')
|
||||
data = json.loads(raw) if raw else None
|
||||
if not data or time.time() - float(data.get('checked_at', 0)) > _OPEN_TICKETS_MAX_AGE_S:
|
||||
return None
|
||||
return {str(k): v for k, v in (data.get('tickets') or {}).items()}
|
||||
except Exception as e:
|
||||
logger.error(f'Reading open ticket list failed: {e}')
|
||||
return None
|
||||
|
||||
|
||||
def _annotate_staleness(events: list, open_tickets: Optional[dict]) -> None:
|
||||
"""Add stale_days (set when an alert has been active too long) and ticket_closed
|
||||
(its ticket was closed in Tinker while the alert is still active) to each event."""
|
||||
days, _ = _stale_settings()
|
||||
now = datetime.now(timezone.utc)
|
||||
for ev in events:
|
||||
ev['stale_days'] = None
|
||||
ev['ticket_closed'] = False
|
||||
try:
|
||||
first = datetime.fromisoformat(str(ev.get('first_seen', '')).replace('Z', '+00:00'))
|
||||
if first.tzinfo is None:
|
||||
first = first.replace(tzinfo=timezone.utc)
|
||||
age = (now - first).total_seconds() / 86400
|
||||
if age >= days:
|
||||
ev['stale_days'] = int(age)
|
||||
except ValueError:
|
||||
pass
|
||||
tid = ev.get('ticket_id')
|
||||
if tid and open_tickets is not None and str(tid) not in open_tickets:
|
||||
ev['ticket_closed'] = True
|
||||
|
||||
|
||||
def _stale_view(events: list) -> list:
|
||||
"""Flag stale active alerts in place; return tickets left open after their alerts cleared."""
|
||||
open_tickets = _open_tickets()
|
||||
_annotate_staleness(events, open_tickets)
|
||||
if open_tickets is None:
|
||||
return []
|
||||
_, hours = _stale_settings()
|
||||
try:
|
||||
return db.get_cleared_tickets(list(open_tickets), older_than_hours=hours)
|
||||
except Exception as e:
|
||||
logger.error(f'Finding tickets left open failed: {e}')
|
||||
return []
|
||||
|
||||
|
||||
def _state_json(key: str) -> dict:
|
||||
"""Return a monitor_state JSON blob as a dict, or {} if missing or unreadable."""
|
||||
raw = db.get_state(key)
|
||||
@@ -307,6 +373,7 @@ def index():
|
||||
snapshot = {}
|
||||
suppressions = db.get_active_suppressions()
|
||||
_annotate_suppressions(events, suppressions)
|
||||
stale_tickets = _stale_view(events)
|
||||
recent_resolved = db.get_recent_resolved(hours=24, limit=10)
|
||||
return render_template(
|
||||
'index.html',
|
||||
@@ -318,6 +385,8 @@ def index():
|
||||
last_check=last_check,
|
||||
suppressions=suppressions,
|
||||
recent_resolved=recent_resolved,
|
||||
stale_tickets=stale_tickets,
|
||||
stale_alerts=sum(1 for e in events if e.get('stale_days') is not None or e.get('ticket_closed')),
|
||||
daemon_ok=_daemon_ok(last_check),
|
||||
)
|
||||
|
||||
@@ -419,11 +488,13 @@ def api_status():
|
||||
active = db.get_active_events(limit=_PAGE_LIMIT)
|
||||
suppressions = db.get_active_suppressions()
|
||||
_annotate_suppressions(active, suppressions)
|
||||
stale_tickets = _stale_view(active)
|
||||
last_check = db.get_state('last_check', 'Never')
|
||||
return jsonify({
|
||||
'summary': db.get_status_summary(),
|
||||
'last_check': last_check,
|
||||
'events': active,
|
||||
'stale_tickets': stale_tickets,
|
||||
'total_active': db.count_active_events(),
|
||||
'daemon_ok': _daemon_ok(last_check),
|
||||
})
|
||||
|
||||
Reference in New Issue
Block a user