From 9d561db703e33bedf2aefca9eeb850cfe842706f Mon Sep 17 00:00:00 2001 From: Jared Vititoe Date: Tue, 10 Nov 2026 20:29:09 -0500 Subject: [PATCH] Add an alert history page and tie /health staleness to the poll interval /events lists every alert from the last 24 hours to 90 days, open or cleared, with how long each lasted, failed checks and its ticket. Filters (device, type, severity, state, range) live in the URL, so device pages and tickets link straight to a device's history. A "Recurring" table lists alerts that fired more than once on the same target. On phones the rows become cards. /api/events/history backs it (100 per page). The existing /api/events is unchanged. /health and the dashboard's monitor indicator now call the monitor stalled after five missed polls (at least 10 minutes) instead of a fixed 20, and /health reports the monitor's age. Co-Authored-By: Claude Opus 5.5 --- README.md | 19 ++++ app.py | 96 +++++++++++++++---- db.py | 94 ++++++++++++++++++ monitor.py | 1 + static/device.js | 3 +- static/events.js | 218 ++++++++++++++++++++++++++++++++++++++++++ static/style.css | 29 ++++++ templates/base.html | 9 ++ templates/events.html | 66 +++++++++++++ tests/test_app.py | 60 ++++++++++++ ticket_text.py | 4 + 11 files changed, 582 insertions(+), 17 deletions(-) create mode 100644 static/events.js create mode 100644 templates/events.html diff --git a/README.md b/README.md index 4308436..ae32188 100644 --- a/README.md +++ b/README.md @@ -114,6 +114,23 @@ Ping-only (no node_exporter): **pbs** (10.10.10.3) - Quick-suppress modal: apply timed or manual suppression from any alert row - Auto-refreshes every 30 seconds via `/api/status` + `/api/network` +### Alert history (`/events`) +Every alert Gandalf raised in the last 24 hours to 90 days, including ones that cleared before anyone +saw them: when each started, how long it lasted, failed checks, and its ticket. +- Filter by device (name, port or interface), alert type, severity, and open or cleared. Filters are + kept in the URL, so `/events?target=large1&range=90d` is a shareable link; device pages and tickets + link to their own history this way. +- **Recurring** lists the same alert on the same target more than once in the range, worst first, since + a flapping cable or optic shows up here before anyone notices the tickets. +- API: `GET /api/events/history?range=24h|7d|30d|90d&type=&target=&severity=&status=all|active|resolved&offset=` + (100 per page). + +### Health check (`/health`) +No login needed. Returns 200 when the database answers and the monitor has completed a check recently, +otherwise 503 with the failing check: `{"status": "degraded", "checks": {"db": "ok", "monitor": "stale", +"monitor_age_s": 742}}`. The monitor counts as stalled after five missed polls (at least 10 minutes), +so point an uptime monitor at it to catch a hung `gandalf-monitor`, not just a dead web app. + ### Device pages (`/device/`) One page per host, switch, router, AP, PDU, or modem. Reach them from the search box on the dashboard (`/` focuses it), or from any device name. Search matches hostname, IP, MAC, model, type, @@ -611,12 +628,14 @@ gandalf/ │ ├── app.js # Dashboard JS (auto-refresh, events, suppress modal, search, modem panel) │ ├── device.js # Device page rendering │ ├── cutsheet.js # Cutsheet page +│ ├── events.js # Alert history page │ └── port-inspector.js # Switch front panel, port panel, path check, diagnostics └── templates/ ├── base.html # Shared layout (header, nav, footer) ├── index.html # Dashboard page ├── device.html # Per-device page ├── cutsheet.html # Cutsheet page and its editors + ├── events.html # Alert history page ├── _modem_panel.html # WAN / Fiber panel, shared by the dashboard and modem page └── suppressions.html # Suppression management page ``` diff --git a/app.py b/app.py index 5d2f390..56ab426 100644 --- a/app.py +++ b/app.py @@ -17,6 +17,7 @@ import time import uuid from datetime import datetime, timezone from functools import wraps +from typing import Optional from flask import Flask, jsonify, make_response, redirect, render_template, request, send_file, url_for @@ -104,15 +105,34 @@ def add_security_headers(response): return response -def _daemon_ok(last_check: str) -> bool: - """Return True if monitor last checked within 20 minutes.""" +def _monitor_stale_after() -> int: + """Seconds without a completed check before the monitor counts as stalled. + + Five missed polls, and never under 10 minutes: one slow cycle (an SSH timeout + to a dead host) shouldn't trip it, but a hung monitor is caught in minutes. + """ + try: + poll = int(_config().get('monitor', {}).get('poll_interval', 120)) + except (TypeError, ValueError): + poll = 120 + return max(600, poll * 5) + + +def _monitor_age(last_check: str) -> Optional[float]: + """Seconds since the monitor's last check, or None if it never ran or can't be read.""" if not last_check or last_check == 'Never': - return False + return None try: ts = datetime.strptime(last_check, '%Y-%m-%d %H:%M:%S UTC').replace(tzinfo=timezone.utc) - return (datetime.now(timezone.utc) - ts).total_seconds() < 1200 - except Exception: - return False + except ValueError: + return None + return (datetime.now(timezone.utc) - ts).total_seconds() + + +def _daemon_ok(last_check: str) -> bool: + """True if the monitor checked recently enough (see _monitor_stale_after).""" + age = _monitor_age(last_check) + return age is not None and age < _monitor_stale_after() # --------------------------------------------------------------------------- @@ -308,6 +328,50 @@ def inspector(): return redirect(url_for('index')) +@app.route('/events') +@require_auth +def events_page(): + return render_template('events.html', user=_get_user()) + + +_HISTORY_RANGES = {'24h': 24, '7d': 24 * 7, '30d': 24 * 30, '90d': 24 * 90} +_HISTORY_PAGE = 100 + + +@app.route('/api/events/history') +@require_auth +def api_events_history(): + """Every alert open at some point in a time window, with summaries. + + Query: range (24h|7d|30d|90d), type, target (substring of name or detail), + severity, status (all|active|resolved), offset. + """ + args = request.args + hours = _HISTORY_RANGES.get(args.get('range', '7d')) + if hours is None: + return jsonify({'error': f'range must be one of {", ".join(_HISTORY_RANGES)}'}), 400 + severity = args.get('severity', '') + if severity not in ('', 'critical', 'warning', 'info'): + return jsonify({'error': 'severity must be critical, warning or info'}), 400 + status = args.get('status', 'all') + if status not in ('all', 'active', 'resolved'): + return jsonify({'error': 'status must be all, active or resolved'}), 400 + try: + offset = max(int(args.get('offset', 0)), 0) + except ValueError: + return jsonify({'error': 'offset must be an integer'}), 400 + event_type = args.get('type', '')[:60] + target = args.get('target', '').strip()[:255] + + result = db.get_events_history(hours, event_type, target, severity, status, + limit=_HISTORY_PAGE, offset=offset) + result['flappers'] = db.get_flappers(hours, event_type, target, severity) + result['types'] = db.get_event_types() + result['offset'] = offset + result['page_size'] = _HISTORY_PAGE + return jsonify(result) + + @app.route('/suppressions') @require_auth @require_admin @@ -1090,19 +1154,19 @@ def health(): checks['db'] = 'error' overall = 'degraded' - # Monitor freshness: fail if last_check is older than 20 minutes + # Monitor freshness: a hung or crashed monitor stops updating last_check try: last_check = db.get_state('last_check', '') - if last_check: - ts = datetime.strptime(last_check, '%Y-%m-%d %H:%M:%S UTC').replace(tzinfo=timezone.utc) - age_s = (datetime.now(timezone.utc) - ts).total_seconds() - if age_s > 1200: - checks['monitor'] = 'stale' - overall = 'degraded' - else: - checks['monitor'] = 'ok' - else: + age_s = _monitor_age(last_check) + if not last_check: checks['monitor'] = 'no data yet' + elif age_s is None or age_s > _monitor_stale_after(): + checks['monitor'] = 'stale' + checks['monitor_age_s'] = int(age_s) if age_s is not None else None + overall = 'degraded' + else: + checks['monitor'] = 'ok' + checks['monitor_age_s'] = int(age_s) except Exception as e: logger.error(f'Health check monitor error: {e}') checks['monitor'] = 'error' diff --git a/db.py b/db.py index ce28214..ff64359 100644 --- a/db.py +++ b/db.py @@ -423,6 +423,100 @@ def event_stats(event_type: str, target_name: str, target_detail: str = '') -> d } +def _history_where(since_hours: int, event_type: str = '', target: str = '', + severity: str = '', status: str = 'all') -> tuple: + """WHERE clause shared by the history list and its summaries. + + An event is in range if it was open at any point in the window: it started + in it, or it was still open when the window began. + """ + clauses = ['(first_seen > DATE_SUB(NOW(), INTERVAL %s HOUR) OR ' + 'resolved_at IS NULL OR resolved_at > DATE_SUB(NOW(), INTERVAL %s HOUR))'] + params: list = [since_hours, since_hours] + if event_type: + clauses.append('event_type=%s') + params.append(event_type) + if severity: + clauses.append('severity=%s') + params.append(severity) + if target: + like = '%' + target.replace('\\', '\\\\').replace('%', '\\%').replace('_', '\\_') + '%' + clauses.append('(target_name LIKE %s OR target_detail LIKE %s)') + params.extend([like, like]) + if status == 'active': + clauses.append('resolved_at IS NULL') + elif status == 'resolved': + clauses.append('resolved_at IS NOT NULL') + return ' AND '.join(clauses), params + + +def _iso(rows: list, keys=('first_seen', 'last_seen', 'resolved_at')) -> list: + for r in rows: + for k in keys: + if r.get(k) and hasattr(r[k], 'isoformat'): + r[k] = r[k].isoformat() + 'Z' + return rows + + +def get_events_history(since_hours: int, event_type: str = '', target: str = '', severity: str = '', + status: str = 'all', limit: int = 100, offset: int = 0) -> dict: + """Events open at any time in the last `since_hours`, newest first, with totals. + + Returns {'events', 'total', 'by_type': {type: count}}. + """ + where, params = _history_where(since_hours, event_type, target, severity, status) + with get_conn() as conn: + with conn.cursor() as cur: + cur.execute( + f"""SELECT id, event_type, severity, source_type, target_name, target_detail, description, + first_seen, last_seen, resolved_at, consecutive_failures, ticket_id, + TIMESTAMPDIFF(SECOND, first_seen, COALESCE(resolved_at, NOW())) AS duration_s + FROM network_events WHERE {where} + ORDER BY first_seen DESC, id DESC LIMIT %s OFFSET %s""", # nosec B608 - fixed clauses + params + [limit, offset], + ) + events = _iso(list(cur.fetchall())) + cur.execute( + f"""SELECT event_type, COUNT(*) AS n + FROM network_events WHERE {where} GROUP BY event_type ORDER BY n DESC""", # nosec B608 + params, + ) + groups = cur.fetchall() + return { + 'events': events, + 'total': sum(int(g['n']) for g in groups), + 'by_type': {g['event_type']: int(g['n']) for g in groups}, + } + + +def get_flappers(since_hours: int, event_type: str = '', target: str = '', severity: str = '', + min_count: int = 2, limit: int = 10) -> list: + """The alerts that fired most often in the window: the ones worth a root cause.""" + where, params = _history_where(since_hours, event_type, target, severity) + with get_conn() as conn: + with conn.cursor() as cur: + cur.execute( + f"""SELECT event_type, target_name, target_detail, COUNT(*) AS n, + SUM(TIMESTAMPDIFF(SECOND, first_seen, COALESCE(resolved_at, NOW()))) AS total_s, + MAX(first_seen) AS last_seen, SUM(resolved_at IS NULL) AS open_now + FROM network_events WHERE {where} + GROUP BY event_type, target_name, target_detail + HAVING n >= %s ORDER BY n DESC, last_seen DESC LIMIT %s""", # nosec B608 - fixed clauses + params + [min_count, limit], + ) + rows = list(cur.fetchall()) + for r in rows: + r['n'], r['total_s'], r['open_now'] = int(r['n']), int(r['total_s'] or 0), int(r['open_now'] or 0) + return _iso(rows, ('last_seen',)) + + +def get_event_types() -> list: + with get_conn() as conn: + with conn.cursor() as cur: + cur.execute('SELECT DISTINCT event_type FROM network_events ORDER BY event_type') + return [r['event_type'] for r in cur.fetchall()] + + def purge_old_resolved_events(days: int = 90) -> int: """Delete resolved events older than `days` days. Returns count deleted.""" with get_conn() as conn: diff --git a/monitor.py b/monitor.py index 863d4f3..a069b40 100644 --- a/monitor.py +++ b/monitor.py @@ -1159,6 +1159,7 @@ class NetworkMonitor: lines = [] if device: lines.append(f'- {ticket_text.link(f"{device} in Gandalf", ticket_text.device_url(base, device))}') + lines.append(f'- {ticket_text.link(f"Alert history for {device}", ticket_text.history_url(base, device))}') lines.append(f'- {ticket_text.link("Gandalf dashboard", base + "/")}') return lines diff --git a/static/device.js b/static/device.js index b1f6bf0..93d2d8b 100644 --- a/static/device.js +++ b/static/device.js @@ -300,7 +300,8 @@ function renderEvents(events) { ${esc(lt.time.ago(_toIso(e.first_seen)))} ${e.resolved_at ? `resolved ${esc(lt.time.ago(_toIso(e.resolved_at)))}` : 'active'} `).join('')}`; - return section('Alerts', body); + const more = `

Full alert history for ${esc(DEVICE_NAME)} →

`; + return section('Alerts', body + more); } function currentRows() { diff --git a/static/events.js b/static/events.js new file mode 100644 index 0000000..aaae2e0 --- /dev/null +++ b/static/events.js @@ -0,0 +1,218 @@ +'use strict'; +/* global deviceUrl, _toIso */ +// Alert history page: renders /api/events/history with filters kept in the URL. + +const esc = s => lt.escHtml(s == null ? '' : String(s)); +const FILTERS = ['target', 'range', 'type', 'severity', 'status']; +const DEFAULTS = { target: '', range: '7d', type: '', severity: '', status: 'all' }; +const RANGE_TEXT = { '24h': '24 hours', '7d': '7 days', '30d': '30 days', '90d': '90 days' }; + +let _offset = 0; +let _loadSeq = 0; + +function typeText(t) { + return String(t || '').replace(/_/g, ' '); +} + +function duration(seconds) { + let s = Math.max(0, Math.floor(seconds || 0)); + if (s < 60) return `${s} s`; + let m = Math.floor(s / 60); + if (m < 60) return `${m} min`; + let h = Math.floor(m / 60); + m %= 60; + if (h < 48) return m ? `${h} h ${m} min` : `${h} h`; + const d = Math.floor(h / 24); + h %= 24; + return h ? `${d} d ${h} h` : `${d} d`; +} + +function when(iso) { + if (!iso) return '–'; + const d = new Date(_toIso(iso)); + if (isNaN(d)) return esc(iso); + const text = d.toLocaleString([], { month: 'short', day: 'numeric', hour: '2-digit', minute: '2-digit' }); + return `${esc(text)}`; +} + +function ticketLink(id) { + if (!id) return '–'; + const base = (typeof GANDALF_CONFIG !== 'undefined' && GANDALF_CONFIG.ticket_web_url) || ''; + return base ? `#${esc(id)}` : `#${esc(id)}`; +} + +function filters() { + const f = {}; + FILTERS.forEach(k => { f[k] = document.getElementById(`ev-${k}`).value.trim(); }); + return f; +} + +function readUrl() { + const q = new URLSearchParams(location.search); + FILTERS.forEach(k => { + const el = document.getElementById(`ev-${k}`); + const v = q.get(k); + if (v == null) return; + if (el.tagName === 'SELECT' && ![...el.options].some(o => o.value === v)) { + // An alert type from a link may not be in the list until data arrives + if (k === 'type') el.add(new Option(typeText(v), v)); + else return; + } + el.value = v; + }); +} + +function writeUrl(f) { + const q = new URLSearchParams(); + FILTERS.forEach(k => { if (f[k] && f[k] !== DEFAULTS[k]) q.set(k, f[k]); }); + const qs = q.toString(); + history.replaceState(null, '', qs ? `?${qs}` : location.pathname); + document.getElementById('ev-clear').hidden = !FILTERS.some(k => k !== 'range' && f[k] && f[k] !== DEFAULTS[k]); +} + +function setFilter(changes) { + Object.entries(changes).forEach(([k, v]) => { + const el = document.getElementById(`ev-${k}`); + if (el.tagName === 'SELECT' && v && ![...el.options].some(o => o.value === v)) el.add(new Option(typeText(v), v)); + el.value = v; + }); + _offset = 0; + load(); +} + +function fillTypes(types) { + const sel = document.getElementById('ev-type'); + const current = sel.value; + sel.innerHTML = '' + + types.map(t => ``).join(''); + if (current && !types.includes(current)) sel.add(new Option(typeText(current), current)); + sel.value = current; +} + +function renderSummary(data, f) { + const open = data.events.filter(e => !e.resolved_at).length; + const chips = [ + `${data.total} alert${data.total === 1 ? '' : 's'} in the last ${esc(RANGE_TEXT[f.range] || f.range)}`, + ]; + if (open && data.offset === 0) chips.push(`${open} open now`); + Object.entries(data.by_type || {}).forEach(([t, n]) => { + const on = f.type === t; + chips.push(``); + }); + document.getElementById('ev-summary').innerHTML = chips.join(''); +} + +function renderFlappers(rows) { + const sec = document.getElementById('ev-flappers-section'); + sec.hidden = !rows.length; + if (!rows.length) return; + document.getElementById('ev-flappers').innerHTML = `
+ + + ${rows.map(r => ` + + + + + + + + `).join('')}
Alerts that fired more than once
AlertTargetTimesTotal timeLast started
${esc(typeText(r.event_type))}${r.open_now ? ' open' : ''}${esc(r.target_name)} + ${r.target_detail ? `${esc(r.target_detail)}` : ''}${r.n}${esc(duration(r.total_s))}${when(r.last_seen)}
`; +} + +function renderTable(data) { + document.getElementById('ev-count').textContent = data.total; + const wrap = document.getElementById('ev-table'); + if (!data.events.length) { + wrap.innerHTML = '
✔
' + + '
No alerts match
'; + document.getElementById('ev-pager').innerHTML = ''; + return; + } + wrap.innerHTML = `
+ + + + ${data.events.map(e => { + const open = !e.resolved_at; + return ` + + + + + + + + + `; + }).join('')}
Alert history
StateTypeTargetStartedLastedChecksTicketDescription
${open ? 'open' : 'cleared'}${esc(e.target_name)} + ${e.target_detail ? `${esc(e.target_detail)}` : ''}${when(e.first_seen)}${esc(duration(e.duration_s))}${open ? ' so far' : ''}${esc(e.consecutive_failures)}${ticketLink(e.ticket_id)}${esc(e.description || '')}
`; + + const end = data.offset + data.events.length; + const prev = data.offset > 0; + const next = end < data.total; + document.getElementById('ev-pager').innerHTML = (prev || next) ? ` + + ${data.offset + 1}–${end} of ${data.total} + ` : ''; +} + +async function load() { + const f = filters(); + writeUrl(f); + const q = new URLSearchParams(); + FILTERS.forEach(k => { if (f[k]) q.set(k, f[k]); }); + if (_offset) q.set('offset', _offset); + const seq = ++_loadSeq; + try { + const data = await lt.api.get(`/api/events/history?${q}`); + if (seq !== _loadSeq) return; // a newer filter change is already loading + fillTypes(data.types || []); + renderSummary(data, f); + renderFlappers(data.flappers || []); + renderTable(data); + } catch (err) { + if (seq !== _loadSeq) return; + document.getElementById('ev-table').innerHTML = + `
Could not load alerts
+
${esc(err.message || err)}
`; + } +} + +let _typing = null; +document.getElementById('ev-target').addEventListener('input', () => { + clearTimeout(_typing); + _typing = setTimeout(() => { _offset = 0; load(); }, 300); +}); +document.getElementById('ev-filters').addEventListener('change', e => { + if (e.target.id === 'ev-target') return; + _offset = 0; + load(); +}); +document.getElementById('ev-filters').addEventListener('submit', e => { e.preventDefault(); _offset = 0; load(); }); +document.getElementById('ev-clear').addEventListener('click', () => { + setFilter({ target: '', type: '', severity: '', status: 'all' }); +}); +document.addEventListener('click', e => { + const typeChip = e.target.closest('.ev-type-chip'); + if (typeChip) return setFilter({ type: typeChip.dataset.type }); + const show = e.target.closest('[data-show-type]'); + if (show) { + const changes = { type: show.dataset.showType }; + if (show.dataset.showTarget) changes.target = show.dataset.showTarget; + return setFilter(changes); + } + const page = e.target.closest('[data-page]'); + if (page && !page.disabled) { + _offset = Math.max(0, parseInt(page.dataset.page, 10) || 0); + load(); + document.getElementById('ev-count').scrollIntoView({ block: 'nearest' }); + } +}); + +readUrl(); +load(); diff --git a/static/style.css b/static/style.css index 0573e06..dfc4cd7 100644 --- a/static/style.css +++ b/static/style.css @@ -1393,3 +1393,32 @@ polygon.fp-led-off { fill: #6b6c71; } .cs-label-card { break-inside: avoid; border-color: #000; color: #000; } .lt-table, .lt-table td, .lt-table th { color: #000 !important; } } + +/* ── Alert history (/events) ───────────────────────────────────────── */ +.ev-filters .lt-select { flex: 0 1 170px; } +.ev-type-chip { background: none; cursor: pointer; font: inherit; font-size: .82rem; } +.ev-type-chip:hover, .ev-type-chip:focus-visible { border-color: var(--accent-cyan); } +.ev-type-chip.active { border-color: var(--accent-cyan); background: var(--accent-cyan-dim); } +.ev-link { background: none; border: 0; padding: 0; color: inherit; font: inherit; cursor: pointer; + text-decoration: underline dotted; text-underline-offset: 3px; text-align: left; } +.ev-link:hover, .ev-link:focus-visible { color: var(--accent-cyan); } +.ev-table td { vertical-align: top; } +.ev-table .cs-port { display: block; } +.ev-pager { display: flex; gap: 12px; align-items: center; justify-content: center; margin-top: 12px; font-size: .85rem; } +.g-section-more { margin: 10px 12px 4px; font-size: .85rem; } +.ev-table .lt-badge, .ev-c-type { white-space: nowrap; } + +/* Phones: each alert becomes a card instead of a squeezed table row */ +@media (max-width: 760px) { + .ev-cards thead { display: none; } + .ev-cards, .ev-cards tbody { display: block; width: 100%; } + .ev-cards tr { display: flex; flex-wrap: wrap; align-items: baseline; gap: 4px 14px; padding: 10px 12px; + border-bottom: 1px solid var(--border-color); } + .ev-cards td { display: block; padding: 0; border: 0; max-width: none; white-space: normal; font-size: .82rem; } + .ev-cards td[data-label]::before { content: attr(data-label) " "; color: var(--text-muted); font-size: .72rem; + text-transform: uppercase; letter-spacing: .05em; } + .ev-cards .ev-c-target { flex: 1 1 100%; order: -1; font-size: .95rem; } + .ev-cards .ev-c-target .cs-port { display: inline; margin-left: 6px; } + .ev-cards .ev-c-desc { flex: 1 1 100%; color: var(--text-muted); white-space: normal; overflow: visible; } + .ev-cards .ev-c-action { flex: 1 1 100%; } +} diff --git a/templates/base.html b/templates/base.html index 43802af..e2c69f3 100644 --- a/templates/base.html +++ b/templates/base.html @@ -37,6 +37,9 @@ Dashboard + History Cutsheet @@ -82,6 +85,11 @@ {% if request.endpoint == 'index' %}aria-current="page"{% endif %}> Dashboard + + History + @@ -324,6 +332,7 @@ // Command palette lt.cmdPalette.init([ { id: 'nav-dashboard', group: 'Navigate', icon: '~', label: 'Dashboard', action: function() { window.location.href = '/'; } }, + { id: 'nav-history', group: 'Navigate', icon: '⏱', label: 'Alert History', action: function() { window.location.href = '/events'; } }, { id: 'nav-cutsheet', group: 'Navigate', icon: '🔌', label: 'Cutsheet', action: function() { window.location.href = '/cutsheet'; } }, { id: 'nav-suppressions', group: 'Navigate', icon: '🔕', label: 'Suppressions', action: function() { window.location.href = '/suppressions'; } }, { id: 'action-refresh', group: 'Actions', icon: '↻', label: 'Refresh Data', kbd: 'R', action: function() { lt.autoRefresh.now(); } }, diff --git a/templates/events.html b/templates/events.html new file mode 100644 index 0000000..27a2dba --- /dev/null +++ b/templates/events.html @@ -0,0 +1,66 @@ +{% extends "base.html" %} +{% block title %}Alert History – GANDALF{% endblock %} + +{% block content %} +
+
+

Alert History

+

Every alert Gandalf raised, including the ones that cleared on their own. Resolved alerts are kept 90 days.

+
+
+ + + +
+ + + +
+
+

Alerts

+ 0 +
+
+ ╚╝ +
Loading…
+
+
+
+{% endblock %} + +{% block scripts %} + +{% endblock %} diff --git a/tests/test_app.py b/tests/test_app.py index 2c8f52b..3ae3687 100644 --- a/tests/test_app.py +++ b/tests/test_app.py @@ -316,3 +316,63 @@ class TestCutsheetLivePorts: def test_admin_only(self, cs_client): client, _ = cs_client assert client.get('/api/cutsheet/live-ports', headers=USER).status_code == 403 + + +class TestAlertHistory: + HISTORY = {'events': [], 'total': 0, 'by_type': {}} + + def _db(self): + db = gandalf.db + db.get_events_history.return_value = dict(self.HISTORY) + db.get_flappers.return_value = [] + db.get_event_types.return_value = ['host_unreachable'] + return db + + def test_page_and_nav(self, client): + html = client.get('/events', headers=AUTH).get_data(as_text=True) + assert 'Alert History' in html and 'events.js' in html + assert 'href="/events"' in client.get('/', headers=AUTH).get_data(as_text=True) + + def test_defaults_to_seven_days(self, client): + db = self._db() + data = client.get('/api/events/history', headers=AUTH).get_json() + assert db.get_events_history.call_args.args == (168, '', '', '', 'all') + assert data['types'] == ['host_unreachable'] and data['page_size'] == 100 + + def test_filters_are_passed_through(self, client): + db = self._db() + client.get('/api/events/history?range=90d&type=interface_down&target=+large1+&severity=critical' + '&status=resolved&offset=200', headers=AUTH) + assert db.get_events_history.call_args.args == (2160, 'interface_down', 'large1', 'critical', 'resolved') + assert db.get_events_history.call_args.kwargs == {'limit': 100, 'offset': 200} + assert db.get_flappers.call_args.args == (2160, 'interface_down', 'large1', 'critical') + + @pytest.mark.parametrize('query', ['range=1y', 'severity=bad', 'status=open', 'offset=x']) + def test_bad_filters_are_rejected(self, client, query): + self._db() + assert client.get(f'/api/events/history?{query}', headers=AUTH).status_code == 400 + + def test_needs_login(self, client): + assert client.get('/api/events/history').status_code == 401 + + +class TestHealth: + def _health(self, client, age_s, poll=120): + from datetime import datetime, timedelta, timezone + stamp = (datetime.now(timezone.utc) - timedelta(seconds=age_s)).strftime('%Y-%m-%d %H:%M:%S UTC') + gandalf.db.get_state.side_effect = lambda key, default=None: stamp if key == 'last_check' else default + with patch.object(gandalf, '_cfg', dict(CONFIG, monitor={'poll_interval': poll})): + return client.get('/health') + + def test_recent_check_is_healthy(self, client): + resp = self._health(client, 150) + assert resp.status_code == 200 and resp.get_json()['checks']['monitor'] == 'ok' + + def test_stalled_monitor_fails_after_ten_minutes(self, client): + resp = self._health(client, 660) + assert resp.status_code == 503 + assert resp.get_json()['checks'] == {'db': 'ok', 'monitor': 'stale', 'monitor_age_s': 660} + + def test_limit_follows_a_long_poll_interval(self, client): + assert self._health(client, 1000, poll=300).status_code == 200 + assert self._health(client, 1600, poll=300).status_code == 503 diff --git a/ticket_text.py b/ticket_text.py index b98fd3a..3e8b1c1 100644 --- a/ticket_text.py +++ b/ticket_text.py @@ -49,6 +49,10 @@ def device_url(base: str, name: str, port_idx=None) -> str: return f'{url}#port-{port_idx}' if port_idx not in (None, '') else url +def history_url(base: str, target: str) -> str: + return f'{base}/events?target={quote(target, safe="")}&range=90d' if base and target else '' + + def cutsheet_url(base: str, cable_id) -> str: return f'{base}/cutsheet#cable-{cable_id}' if base else ''