Add an alert history page and tie /health staleness to the poll interval
Lint / JS (eslint) (push) Successful in 6s
Lint / Python (flake8) (push) Successful in 1m56s
Lint / Notify on failure (push) Skipped
Lint / Deploy (push) Skipped
Security / Python Security (bandit) (push) Successful in 2m3s
Test / Python Tests (pytest) (push) Successful in 1m0s
Lint / Python (flake8) (pull_request) Successful in 48s
Lint / JS (eslint) (pull_request) Successful in 6s
Lint / Notify on failure (pull_request) Skipped
Lint / Deploy (pull_request) Skipped
Security / Python Security (bandit) (pull_request) Successful in 2m7s
Test / Python Tests (pytest) (pull_request) Successful in 2m6s

/events lists every alert from the last 24 hours to 90 days, open or
cleared, with how long each lasted, failed checks and its ticket. Filters
(device, type, severity, state, range) live in the URL, so device pages
and tickets link straight to a device's history. A "Recurring" table
lists alerts that fired more than once on the same target. On phones the
rows become cards.

/api/events/history backs it (100 per page). The existing /api/events is
unchanged.

/health and the dashboard's monitor indicator now call the monitor
stalled after five missed polls (at least 10 minutes) instead of a fixed
20, and /health reports the monitor's age.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-11-10 20:29:09 -05:00
co-authored by Claude Opus 5.5
parent 9e2a2b987c
commit 9d561db703
11 changed files with 582 additions and 17 deletions
+80 -16
View File
@@ -17,6 +17,7 @@ import time
import uuid
from datetime import datetime, timezone
from functools import wraps
from typing import Optional
from flask import Flask, jsonify, make_response, redirect, render_template, request, send_file, url_for
@@ -104,15 +105,34 @@ def add_security_headers(response):
return response
def _daemon_ok(last_check: str) -> bool:
"""Return True if monitor last checked within 20 minutes."""
def _monitor_stale_after() -> int:
"""Seconds without a completed check before the monitor counts as stalled.
Five missed polls, and never under 10 minutes: one slow cycle (an SSH timeout
to a dead host) shouldn't trip it, but a hung monitor is caught in minutes.
"""
try:
poll = int(_config().get('monitor', {}).get('poll_interval', 120))
except (TypeError, ValueError):
poll = 120
return max(600, poll * 5)
def _monitor_age(last_check: str) -> Optional[float]:
"""Seconds since the monitor's last check, or None if it never ran or can't be read."""
if not last_check or last_check == 'Never':
return False
return None
try:
ts = datetime.strptime(last_check, '%Y-%m-%d %H:%M:%S UTC').replace(tzinfo=timezone.utc)
return (datetime.now(timezone.utc) - ts).total_seconds() < 1200
except Exception:
return False
except ValueError:
return None
return (datetime.now(timezone.utc) - ts).total_seconds()
def _daemon_ok(last_check: str) -> bool:
"""True if the monitor checked recently enough (see _monitor_stale_after)."""
age = _monitor_age(last_check)
return age is not None and age < _monitor_stale_after()
# ---------------------------------------------------------------------------
@@ -308,6 +328,50 @@ def inspector():
return redirect(url_for('index'))
@app.route('/events')
@require_auth
def events_page():
return render_template('events.html', user=_get_user())
_HISTORY_RANGES = {'24h': 24, '7d': 24 * 7, '30d': 24 * 30, '90d': 24 * 90}
_HISTORY_PAGE = 100
@app.route('/api/events/history')
@require_auth
def api_events_history():
"""Every alert open at some point in a time window, with summaries.
Query: range (24h|7d|30d|90d), type, target (substring of name or detail),
severity, status (all|active|resolved), offset.
"""
args = request.args
hours = _HISTORY_RANGES.get(args.get('range', '7d'))
if hours is None:
return jsonify({'error': f'range must be one of {", ".join(_HISTORY_RANGES)}'}), 400
severity = args.get('severity', '')
if severity not in ('', 'critical', 'warning', 'info'):
return jsonify({'error': 'severity must be critical, warning or info'}), 400
status = args.get('status', 'all')
if status not in ('all', 'active', 'resolved'):
return jsonify({'error': 'status must be all, active or resolved'}), 400
try:
offset = max(int(args.get('offset', 0)), 0)
except ValueError:
return jsonify({'error': 'offset must be an integer'}), 400
event_type = args.get('type', '')[:60]
target = args.get('target', '').strip()[:255]
result = db.get_events_history(hours, event_type, target, severity, status,
limit=_HISTORY_PAGE, offset=offset)
result['flappers'] = db.get_flappers(hours, event_type, target, severity)
result['types'] = db.get_event_types()
result['offset'] = offset
result['page_size'] = _HISTORY_PAGE
return jsonify(result)
@app.route('/suppressions')
@require_auth
@require_admin
@@ -1090,19 +1154,19 @@ def health():
checks['db'] = 'error'
overall = 'degraded'
# Monitor freshness: fail if last_check is older than 20 minutes
# Monitor freshness: a hung or crashed monitor stops updating last_check
try:
last_check = db.get_state('last_check', '')
if last_check:
ts = datetime.strptime(last_check, '%Y-%m-%d %H:%M:%S UTC').replace(tzinfo=timezone.utc)
age_s = (datetime.now(timezone.utc) - ts).total_seconds()
if age_s > 1200:
checks['monitor'] = 'stale'
overall = 'degraded'
else:
checks['monitor'] = 'ok'
else:
age_s = _monitor_age(last_check)
if not last_check:
checks['monitor'] = 'no data yet'
elif age_s is None or age_s > _monitor_stale_after():
checks['monitor'] = 'stale'
checks['monitor_age_s'] = int(age_s) if age_s is not None else None
overall = 'degraded'
else:
checks['monitor'] = 'ok'
checks['monitor_age_s'] = int(age_s)
except Exception as e:
logger.error(f'Health check monitor error: {e}')
checks['monitor'] = 'error'