Add an alert history page and tie /health staleness to the poll interval
Lint / JS (eslint) (push) Successful in 6s
Lint / Python (flake8) (push) Successful in 1m56s
Lint / Notify on failure (push) Skipped
Lint / Deploy (push) Skipped
Security / Python Security (bandit) (push) Successful in 2m3s
Test / Python Tests (pytest) (push) Successful in 1m0s
Lint / Python (flake8) (pull_request) Successful in 48s
Lint / JS (eslint) (pull_request) Successful in 6s
Lint / Notify on failure (pull_request) Skipped
Lint / Deploy (pull_request) Skipped
Security / Python Security (bandit) (pull_request) Successful in 2m7s
Test / Python Tests (pytest) (pull_request) Successful in 2m6s
Lint / JS (eslint) (push) Successful in 6s
Lint / Python (flake8) (push) Successful in 1m56s
Lint / Notify on failure (push) Skipped
Lint / Deploy (push) Skipped
Security / Python Security (bandit) (push) Successful in 2m3s
Test / Python Tests (pytest) (push) Successful in 1m0s
Lint / Python (flake8) (pull_request) Successful in 48s
Lint / JS (eslint) (pull_request) Successful in 6s
Lint / Notify on failure (pull_request) Skipped
Lint / Deploy (pull_request) Skipped
Security / Python Security (bandit) (pull_request) Successful in 2m7s
Test / Python Tests (pytest) (pull_request) Successful in 2m6s
/events lists every alert from the last 24 hours to 90 days, open or cleared, with how long each lasted, failed checks and its ticket. Filters (device, type, severity, state, range) live in the URL, so device pages and tickets link straight to a device's history. A "Recurring" table lists alerts that fired more than once on the same target. On phones the rows become cards. /api/events/history backs it (100 per page). The existing /api/events is unchanged. /health and the dashboard's monitor indicator now call the monitor stalled after five missed polls (at least 10 minutes) instead of a fixed 20, and /health reports the monitor's age. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -17,6 +17,7 @@ import time
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
from functools import wraps
|
||||
from typing import Optional
|
||||
|
||||
from flask import Flask, jsonify, make_response, redirect, render_template, request, send_file, url_for
|
||||
|
||||
@@ -104,15 +105,34 @@ def add_security_headers(response):
|
||||
return response
|
||||
|
||||
|
||||
def _daemon_ok(last_check: str) -> bool:
|
||||
"""Return True if monitor last checked within 20 minutes."""
|
||||
def _monitor_stale_after() -> int:
|
||||
"""Seconds without a completed check before the monitor counts as stalled.
|
||||
|
||||
Five missed polls, and never under 10 minutes: one slow cycle (an SSH timeout
|
||||
to a dead host) shouldn't trip it, but a hung monitor is caught in minutes.
|
||||
"""
|
||||
try:
|
||||
poll = int(_config().get('monitor', {}).get('poll_interval', 120))
|
||||
except (TypeError, ValueError):
|
||||
poll = 120
|
||||
return max(600, poll * 5)
|
||||
|
||||
|
||||
def _monitor_age(last_check: str) -> Optional[float]:
|
||||
"""Seconds since the monitor's last check, or None if it never ran or can't be read."""
|
||||
if not last_check or last_check == 'Never':
|
||||
return False
|
||||
return None
|
||||
try:
|
||||
ts = datetime.strptime(last_check, '%Y-%m-%d %H:%M:%S UTC').replace(tzinfo=timezone.utc)
|
||||
return (datetime.now(timezone.utc) - ts).total_seconds() < 1200
|
||||
except Exception:
|
||||
return False
|
||||
except ValueError:
|
||||
return None
|
||||
return (datetime.now(timezone.utc) - ts).total_seconds()
|
||||
|
||||
|
||||
def _daemon_ok(last_check: str) -> bool:
|
||||
"""True if the monitor checked recently enough (see _monitor_stale_after)."""
|
||||
age = _monitor_age(last_check)
|
||||
return age is not None and age < _monitor_stale_after()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -308,6 +328,50 @@ def inspector():
|
||||
return redirect(url_for('index'))
|
||||
|
||||
|
||||
@app.route('/events')
|
||||
@require_auth
|
||||
def events_page():
|
||||
return render_template('events.html', user=_get_user())
|
||||
|
||||
|
||||
_HISTORY_RANGES = {'24h': 24, '7d': 24 * 7, '30d': 24 * 30, '90d': 24 * 90}
|
||||
_HISTORY_PAGE = 100
|
||||
|
||||
|
||||
@app.route('/api/events/history')
|
||||
@require_auth
|
||||
def api_events_history():
|
||||
"""Every alert open at some point in a time window, with summaries.
|
||||
|
||||
Query: range (24h|7d|30d|90d), type, target (substring of name or detail),
|
||||
severity, status (all|active|resolved), offset.
|
||||
"""
|
||||
args = request.args
|
||||
hours = _HISTORY_RANGES.get(args.get('range', '7d'))
|
||||
if hours is None:
|
||||
return jsonify({'error': f'range must be one of {", ".join(_HISTORY_RANGES)}'}), 400
|
||||
severity = args.get('severity', '')
|
||||
if severity not in ('', 'critical', 'warning', 'info'):
|
||||
return jsonify({'error': 'severity must be critical, warning or info'}), 400
|
||||
status = args.get('status', 'all')
|
||||
if status not in ('all', 'active', 'resolved'):
|
||||
return jsonify({'error': 'status must be all, active or resolved'}), 400
|
||||
try:
|
||||
offset = max(int(args.get('offset', 0)), 0)
|
||||
except ValueError:
|
||||
return jsonify({'error': 'offset must be an integer'}), 400
|
||||
event_type = args.get('type', '')[:60]
|
||||
target = args.get('target', '').strip()[:255]
|
||||
|
||||
result = db.get_events_history(hours, event_type, target, severity, status,
|
||||
limit=_HISTORY_PAGE, offset=offset)
|
||||
result['flappers'] = db.get_flappers(hours, event_type, target, severity)
|
||||
result['types'] = db.get_event_types()
|
||||
result['offset'] = offset
|
||||
result['page_size'] = _HISTORY_PAGE
|
||||
return jsonify(result)
|
||||
|
||||
|
||||
@app.route('/suppressions')
|
||||
@require_auth
|
||||
@require_admin
|
||||
@@ -1090,19 +1154,19 @@ def health():
|
||||
checks['db'] = 'error'
|
||||
overall = 'degraded'
|
||||
|
||||
# Monitor freshness: fail if last_check is older than 20 minutes
|
||||
# Monitor freshness: a hung or crashed monitor stops updating last_check
|
||||
try:
|
||||
last_check = db.get_state('last_check', '')
|
||||
if last_check:
|
||||
ts = datetime.strptime(last_check, '%Y-%m-%d %H:%M:%S UTC').replace(tzinfo=timezone.utc)
|
||||
age_s = (datetime.now(timezone.utc) - ts).total_seconds()
|
||||
if age_s > 1200:
|
||||
checks['monitor'] = 'stale'
|
||||
overall = 'degraded'
|
||||
else:
|
||||
checks['monitor'] = 'ok'
|
||||
else:
|
||||
age_s = _monitor_age(last_check)
|
||||
if not last_check:
|
||||
checks['monitor'] = 'no data yet'
|
||||
elif age_s is None or age_s > _monitor_stale_after():
|
||||
checks['monitor'] = 'stale'
|
||||
checks['monitor_age_s'] = int(age_s) if age_s is not None else None
|
||||
overall = 'degraded'
|
||||
else:
|
||||
checks['monitor'] = 'ok'
|
||||
checks['monitor_age_s'] = int(age_s)
|
||||
except Exception as e:
|
||||
logger.error(f'Health check monitor error: {e}')
|
||||
checks['monitor'] = 'error'
|
||||
|
||||
Reference in New Issue
Block a user