Add an alert history page and tie /health staleness to the poll interval
Lint / JS (eslint) (push) Successful in 6s
Lint / Python (flake8) (push) Successful in 1m56s
Lint / Notify on failure (push) Skipped
Lint / Deploy (push) Skipped
Security / Python Security (bandit) (push) Successful in 2m3s
Test / Python Tests (pytest) (push) Successful in 1m0s
Lint / Python (flake8) (pull_request) Successful in 48s
Lint / JS (eslint) (pull_request) Successful in 6s
Lint / Notify on failure (pull_request) Skipped
Lint / Deploy (pull_request) Skipped
Security / Python Security (bandit) (pull_request) Successful in 2m7s
Test / Python Tests (pytest) (pull_request) Successful in 2m6s
Lint / JS (eslint) (push) Successful in 6s
Lint / Python (flake8) (push) Successful in 1m56s
Lint / Notify on failure (push) Skipped
Lint / Deploy (push) Skipped
Security / Python Security (bandit) (push) Successful in 2m3s
Test / Python Tests (pytest) (push) Successful in 1m0s
Lint / Python (flake8) (pull_request) Successful in 48s
Lint / JS (eslint) (pull_request) Successful in 6s
Lint / Notify on failure (pull_request) Skipped
Lint / Deploy (pull_request) Skipped
Security / Python Security (bandit) (pull_request) Successful in 2m7s
Test / Python Tests (pytest) (pull_request) Successful in 2m6s
/events lists every alert from the last 24 hours to 90 days, open or cleared, with how long each lasted, failed checks and its ticket. Filters (device, type, severity, state, range) live in the URL, so device pages and tickets link straight to a device's history. A "Recurring" table lists alerts that fired more than once on the same target. On phones the rows become cards. /api/events/history backs it (100 per page). The existing /api/events is unchanged. /health and the dashboard's monitor indicator now call the monitor stalled after five missed polls (at least 10 minutes) instead of a fixed 20, and /health reports the monitor's age. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -114,6 +114,23 @@ Ping-only (no node_exporter): **pbs** (10.10.10.3)
|
||||
- Quick-suppress modal: apply timed or manual suppression from any alert row
|
||||
- Auto-refreshes every 30 seconds via `/api/status` + `/api/network`
|
||||
|
||||
### Alert history (`/events`)
|
||||
Every alert Gandalf raised in the last 24 hours to 90 days, including ones that cleared before anyone
|
||||
saw them: when each started, how long it lasted, failed checks, and its ticket.
|
||||
- Filter by device (name, port or interface), alert type, severity, and open or cleared. Filters are
|
||||
kept in the URL, so `/events?target=large1&range=90d` is a shareable link; device pages and tickets
|
||||
link to their own history this way.
|
||||
- **Recurring** lists the same alert on the same target more than once in the range, worst first, since
|
||||
a flapping cable or optic shows up here before anyone notices the tickets.
|
||||
- API: `GET /api/events/history?range=24h|7d|30d|90d&type=&target=&severity=&status=all|active|resolved&offset=`
|
||||
(100 per page).
|
||||
|
||||
### Health check (`/health`)
|
||||
No login needed. Returns 200 when the database answers and the monitor has completed a check recently,
|
||||
otherwise 503 with the failing check: `{"status": "degraded", "checks": {"db": "ok", "monitor": "stale",
|
||||
"monitor_age_s": 742}}`. The monitor counts as stalled after five missed polls (at least 10 minutes),
|
||||
so point an uptime monitor at it to catch a hung `gandalf-monitor`, not just a dead web app.
|
||||
|
||||
### Device pages (`/device/<name>`)
|
||||
One page per host, switch, router, AP, PDU, or modem. Reach them from the search box on the
|
||||
dashboard (`/` focuses it), or from any device name. Search matches hostname, IP, MAC, model, type,
|
||||
@@ -611,12 +628,14 @@ gandalf/
|
||||
│ ├── app.js # Dashboard JS (auto-refresh, events, suppress modal, search, modem panel)
|
||||
│ ├── device.js # Device page rendering
|
||||
│ ├── cutsheet.js # Cutsheet page
|
||||
│ ├── events.js # Alert history page
|
||||
│ └── port-inspector.js # Switch front panel, port panel, path check, diagnostics
|
||||
└── templates/
|
||||
├── base.html # Shared layout (header, nav, footer)
|
||||
├── index.html # Dashboard page
|
||||
├── device.html # Per-device page
|
||||
├── cutsheet.html # Cutsheet page and its editors
|
||||
├── events.html # Alert history page
|
||||
├── _modem_panel.html # WAN / Fiber panel, shared by the dashboard and modem page
|
||||
└── suppressions.html # Suppression management page
|
||||
```
|
||||
|
||||
@@ -17,6 +17,7 @@ import time
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
from functools import wraps
|
||||
from typing import Optional
|
||||
|
||||
from flask import Flask, jsonify, make_response, redirect, render_template, request, send_file, url_for
|
||||
|
||||
@@ -104,15 +105,34 @@ def add_security_headers(response):
|
||||
return response
|
||||
|
||||
|
||||
def _daemon_ok(last_check: str) -> bool:
|
||||
"""Return True if monitor last checked within 20 minutes."""
|
||||
def _monitor_stale_after() -> int:
|
||||
"""Seconds without a completed check before the monitor counts as stalled.
|
||||
|
||||
Five missed polls, and never under 10 minutes: one slow cycle (an SSH timeout
|
||||
to a dead host) shouldn't trip it, but a hung monitor is caught in minutes.
|
||||
"""
|
||||
try:
|
||||
poll = int(_config().get('monitor', {}).get('poll_interval', 120))
|
||||
except (TypeError, ValueError):
|
||||
poll = 120
|
||||
return max(600, poll * 5)
|
||||
|
||||
|
||||
def _monitor_age(last_check: str) -> Optional[float]:
|
||||
"""Seconds since the monitor's last check, or None if it never ran or can't be read."""
|
||||
if not last_check or last_check == 'Never':
|
||||
return False
|
||||
return None
|
||||
try:
|
||||
ts = datetime.strptime(last_check, '%Y-%m-%d %H:%M:%S UTC').replace(tzinfo=timezone.utc)
|
||||
return (datetime.now(timezone.utc) - ts).total_seconds() < 1200
|
||||
except Exception:
|
||||
return False
|
||||
except ValueError:
|
||||
return None
|
||||
return (datetime.now(timezone.utc) - ts).total_seconds()
|
||||
|
||||
|
||||
def _daemon_ok(last_check: str) -> bool:
|
||||
"""True if the monitor checked recently enough (see _monitor_stale_after)."""
|
||||
age = _monitor_age(last_check)
|
||||
return age is not None and age < _monitor_stale_after()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -308,6 +328,50 @@ def inspector():
|
||||
return redirect(url_for('index'))
|
||||
|
||||
|
||||
@app.route('/events')
|
||||
@require_auth
|
||||
def events_page():
|
||||
return render_template('events.html', user=_get_user())
|
||||
|
||||
|
||||
_HISTORY_RANGES = {'24h': 24, '7d': 24 * 7, '30d': 24 * 30, '90d': 24 * 90}
|
||||
_HISTORY_PAGE = 100
|
||||
|
||||
|
||||
@app.route('/api/events/history')
|
||||
@require_auth
|
||||
def api_events_history():
|
||||
"""Every alert open at some point in a time window, with summaries.
|
||||
|
||||
Query: range (24h|7d|30d|90d), type, target (substring of name or detail),
|
||||
severity, status (all|active|resolved), offset.
|
||||
"""
|
||||
args = request.args
|
||||
hours = _HISTORY_RANGES.get(args.get('range', '7d'))
|
||||
if hours is None:
|
||||
return jsonify({'error': f'range must be one of {", ".join(_HISTORY_RANGES)}'}), 400
|
||||
severity = args.get('severity', '')
|
||||
if severity not in ('', 'critical', 'warning', 'info'):
|
||||
return jsonify({'error': 'severity must be critical, warning or info'}), 400
|
||||
status = args.get('status', 'all')
|
||||
if status not in ('all', 'active', 'resolved'):
|
||||
return jsonify({'error': 'status must be all, active or resolved'}), 400
|
||||
try:
|
||||
offset = max(int(args.get('offset', 0)), 0)
|
||||
except ValueError:
|
||||
return jsonify({'error': 'offset must be an integer'}), 400
|
||||
event_type = args.get('type', '')[:60]
|
||||
target = args.get('target', '').strip()[:255]
|
||||
|
||||
result = db.get_events_history(hours, event_type, target, severity, status,
|
||||
limit=_HISTORY_PAGE, offset=offset)
|
||||
result['flappers'] = db.get_flappers(hours, event_type, target, severity)
|
||||
result['types'] = db.get_event_types()
|
||||
result['offset'] = offset
|
||||
result['page_size'] = _HISTORY_PAGE
|
||||
return jsonify(result)
|
||||
|
||||
|
||||
@app.route('/suppressions')
|
||||
@require_auth
|
||||
@require_admin
|
||||
@@ -1090,19 +1154,19 @@ def health():
|
||||
checks['db'] = 'error'
|
||||
overall = 'degraded'
|
||||
|
||||
# Monitor freshness: fail if last_check is older than 20 minutes
|
||||
# Monitor freshness: a hung or crashed monitor stops updating last_check
|
||||
try:
|
||||
last_check = db.get_state('last_check', '')
|
||||
if last_check:
|
||||
ts = datetime.strptime(last_check, '%Y-%m-%d %H:%M:%S UTC').replace(tzinfo=timezone.utc)
|
||||
age_s = (datetime.now(timezone.utc) - ts).total_seconds()
|
||||
if age_s > 1200:
|
||||
checks['monitor'] = 'stale'
|
||||
overall = 'degraded'
|
||||
else:
|
||||
checks['monitor'] = 'ok'
|
||||
else:
|
||||
age_s = _monitor_age(last_check)
|
||||
if not last_check:
|
||||
checks['monitor'] = 'no data yet'
|
||||
elif age_s is None or age_s > _monitor_stale_after():
|
||||
checks['monitor'] = 'stale'
|
||||
checks['monitor_age_s'] = int(age_s) if age_s is not None else None
|
||||
overall = 'degraded'
|
||||
else:
|
||||
checks['monitor'] = 'ok'
|
||||
checks['monitor_age_s'] = int(age_s)
|
||||
except Exception as e:
|
||||
logger.error(f'Health check monitor error: {e}')
|
||||
checks['monitor'] = 'error'
|
||||
|
||||
@@ -423,6 +423,100 @@ def event_stats(event_type: str, target_name: str, target_detail: str = '') -> d
|
||||
}
|
||||
|
||||
|
||||
def _history_where(since_hours: int, event_type: str = '', target: str = '',
|
||||
severity: str = '', status: str = 'all') -> tuple:
|
||||
"""WHERE clause shared by the history list and its summaries.
|
||||
|
||||
An event is in range if it was open at any point in the window: it started
|
||||
in it, or it was still open when the window began.
|
||||
"""
|
||||
clauses = ['(first_seen > DATE_SUB(NOW(), INTERVAL %s HOUR) OR '
|
||||
'resolved_at IS NULL OR resolved_at > DATE_SUB(NOW(), INTERVAL %s HOUR))']
|
||||
params: list = [since_hours, since_hours]
|
||||
if event_type:
|
||||
clauses.append('event_type=%s')
|
||||
params.append(event_type)
|
||||
if severity:
|
||||
clauses.append('severity=%s')
|
||||
params.append(severity)
|
||||
if target:
|
||||
like = '%' + target.replace('\\', '\\\\').replace('%', '\\%').replace('_', '\\_') + '%'
|
||||
clauses.append('(target_name LIKE %s OR target_detail LIKE %s)')
|
||||
params.extend([like, like])
|
||||
if status == 'active':
|
||||
clauses.append('resolved_at IS NULL')
|
||||
elif status == 'resolved':
|
||||
clauses.append('resolved_at IS NOT NULL')
|
||||
return ' AND '.join(clauses), params
|
||||
|
||||
|
||||
def _iso(rows: list, keys=('first_seen', 'last_seen', 'resolved_at')) -> list:
|
||||
for r in rows:
|
||||
for k in keys:
|
||||
if r.get(k) and hasattr(r[k], 'isoformat'):
|
||||
r[k] = r[k].isoformat() + 'Z'
|
||||
return rows
|
||||
|
||||
|
||||
def get_events_history(since_hours: int, event_type: str = '', target: str = '', severity: str = '',
|
||||
status: str = 'all', limit: int = 100, offset: int = 0) -> dict:
|
||||
"""Events open at any time in the last `since_hours`, newest first, with totals.
|
||||
|
||||
Returns {'events', 'total', 'by_type': {type: count}}.
|
||||
"""
|
||||
where, params = _history_where(since_hours, event_type, target, severity, status)
|
||||
with get_conn() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
f"""SELECT id, event_type, severity, source_type, target_name, target_detail, description,
|
||||
first_seen, last_seen, resolved_at, consecutive_failures, ticket_id,
|
||||
TIMESTAMPDIFF(SECOND, first_seen, COALESCE(resolved_at, NOW())) AS duration_s
|
||||
FROM network_events WHERE {where}
|
||||
ORDER BY first_seen DESC, id DESC LIMIT %s OFFSET %s""", # nosec B608 - fixed clauses
|
||||
params + [limit, offset],
|
||||
)
|
||||
events = _iso(list(cur.fetchall()))
|
||||
cur.execute(
|
||||
f"""SELECT event_type, COUNT(*) AS n
|
||||
FROM network_events WHERE {where} GROUP BY event_type ORDER BY n DESC""", # nosec B608
|
||||
params,
|
||||
)
|
||||
groups = cur.fetchall()
|
||||
return {
|
||||
'events': events,
|
||||
'total': sum(int(g['n']) for g in groups),
|
||||
'by_type': {g['event_type']: int(g['n']) for g in groups},
|
||||
}
|
||||
|
||||
|
||||
def get_flappers(since_hours: int, event_type: str = '', target: str = '', severity: str = '',
|
||||
min_count: int = 2, limit: int = 10) -> list:
|
||||
"""The alerts that fired most often in the window: the ones worth a root cause."""
|
||||
where, params = _history_where(since_hours, event_type, target, severity)
|
||||
with get_conn() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
f"""SELECT event_type, target_name, target_detail, COUNT(*) AS n,
|
||||
SUM(TIMESTAMPDIFF(SECOND, first_seen, COALESCE(resolved_at, NOW()))) AS total_s,
|
||||
MAX(first_seen) AS last_seen, SUM(resolved_at IS NULL) AS open_now
|
||||
FROM network_events WHERE {where}
|
||||
GROUP BY event_type, target_name, target_detail
|
||||
HAVING n >= %s ORDER BY n DESC, last_seen DESC LIMIT %s""", # nosec B608 - fixed clauses
|
||||
params + [min_count, limit],
|
||||
)
|
||||
rows = list(cur.fetchall())
|
||||
for r in rows:
|
||||
r['n'], r['total_s'], r['open_now'] = int(r['n']), int(r['total_s'] or 0), int(r['open_now'] or 0)
|
||||
return _iso(rows, ('last_seen',))
|
||||
|
||||
|
||||
def get_event_types() -> list:
|
||||
with get_conn() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute('SELECT DISTINCT event_type FROM network_events ORDER BY event_type')
|
||||
return [r['event_type'] for r in cur.fetchall()]
|
||||
|
||||
|
||||
def purge_old_resolved_events(days: int = 90) -> int:
|
||||
"""Delete resolved events older than `days` days. Returns count deleted."""
|
||||
with get_conn() as conn:
|
||||
|
||||
@@ -1159,6 +1159,7 @@ class NetworkMonitor:
|
||||
lines = []
|
||||
if device:
|
||||
lines.append(f'- {ticket_text.link(f"{device} in Gandalf", ticket_text.device_url(base, device))}')
|
||||
lines.append(f'- {ticket_text.link(f"Alert history for {device}", ticket_text.history_url(base, device))}')
|
||||
lines.append(f'- {ticket_text.link("Gandalf dashboard", base + "/")}')
|
||||
return lines
|
||||
|
||||
|
||||
+2
-1
@@ -300,7 +300,8 @@ function renderEvents(events) {
|
||||
<td class="ts-cell" title="${esc(e.first_seen || '')}">${esc(lt.time.ago(_toIso(e.first_seen)))}</td>
|
||||
<td>${e.resolved_at ? `resolved ${esc(lt.time.ago(_toIso(e.resolved_at)))}` : '<strong>active</strong>'}</td>
|
||||
</tr>`).join('')}</tbody></table></div>`;
|
||||
return section('Alerts', body);
|
||||
const more = `<p class="g-section-more"><a href="/events?target=${encodeURIComponent(DEVICE_NAME)}&range=90d">Full alert history for ${esc(DEVICE_NAME)} →</a></p>`;
|
||||
return section('Alerts', body + more);
|
||||
}
|
||||
|
||||
function currentRows() {
|
||||
|
||||
@@ -0,0 +1,218 @@
|
||||
'use strict';
|
||||
/* global deviceUrl, _toIso */
|
||||
// Alert history page: renders /api/events/history with filters kept in the URL.
|
||||
|
||||
const esc = s => lt.escHtml(s == null ? '' : String(s));
|
||||
const FILTERS = ['target', 'range', 'type', 'severity', 'status'];
|
||||
const DEFAULTS = { target: '', range: '7d', type: '', severity: '', status: 'all' };
|
||||
const RANGE_TEXT = { '24h': '24 hours', '7d': '7 days', '30d': '30 days', '90d': '90 days' };
|
||||
|
||||
let _offset = 0;
|
||||
let _loadSeq = 0;
|
||||
|
||||
function typeText(t) {
|
||||
return String(t || '').replace(/_/g, ' ');
|
||||
}
|
||||
|
||||
function duration(seconds) {
|
||||
let s = Math.max(0, Math.floor(seconds || 0));
|
||||
if (s < 60) return `${s} s`;
|
||||
let m = Math.floor(s / 60);
|
||||
if (m < 60) return `${m} min`;
|
||||
let h = Math.floor(m / 60);
|
||||
m %= 60;
|
||||
if (h < 48) return m ? `${h} h ${m} min` : `${h} h`;
|
||||
const d = Math.floor(h / 24);
|
||||
h %= 24;
|
||||
return h ? `${d} d ${h} h` : `${d} d`;
|
||||
}
|
||||
|
||||
function when(iso) {
|
||||
if (!iso) return '–';
|
||||
const d = new Date(_toIso(iso));
|
||||
if (isNaN(d)) return esc(iso);
|
||||
const text = d.toLocaleString([], { month: 'short', day: 'numeric', hour: '2-digit', minute: '2-digit' });
|
||||
return `<span title="${esc(d.toLocaleString())}">${esc(text)}</span>`;
|
||||
}
|
||||
|
||||
function ticketLink(id) {
|
||||
if (!id) return '<span class="val-neutral">–</span>';
|
||||
const base = (typeof GANDALF_CONFIG !== 'undefined' && GANDALF_CONFIG.ticket_web_url) || '';
|
||||
return base ? `<a href="${esc(base + id)}" target="_blank" rel="noopener">#${esc(id)}</a>` : `#${esc(id)}`;
|
||||
}
|
||||
|
||||
function filters() {
|
||||
const f = {};
|
||||
FILTERS.forEach(k => { f[k] = document.getElementById(`ev-${k}`).value.trim(); });
|
||||
return f;
|
||||
}
|
||||
|
||||
function readUrl() {
|
||||
const q = new URLSearchParams(location.search);
|
||||
FILTERS.forEach(k => {
|
||||
const el = document.getElementById(`ev-${k}`);
|
||||
const v = q.get(k);
|
||||
if (v == null) return;
|
||||
if (el.tagName === 'SELECT' && ![...el.options].some(o => o.value === v)) {
|
||||
// An alert type from a link may not be in the list until data arrives
|
||||
if (k === 'type') el.add(new Option(typeText(v), v));
|
||||
else return;
|
||||
}
|
||||
el.value = v;
|
||||
});
|
||||
}
|
||||
|
||||
function writeUrl(f) {
|
||||
const q = new URLSearchParams();
|
||||
FILTERS.forEach(k => { if (f[k] && f[k] !== DEFAULTS[k]) q.set(k, f[k]); });
|
||||
const qs = q.toString();
|
||||
history.replaceState(null, '', qs ? `?${qs}` : location.pathname);
|
||||
document.getElementById('ev-clear').hidden = !FILTERS.some(k => k !== 'range' && f[k] && f[k] !== DEFAULTS[k]);
|
||||
}
|
||||
|
||||
function setFilter(changes) {
|
||||
Object.entries(changes).forEach(([k, v]) => {
|
||||
const el = document.getElementById(`ev-${k}`);
|
||||
if (el.tagName === 'SELECT' && v && ![...el.options].some(o => o.value === v)) el.add(new Option(typeText(v), v));
|
||||
el.value = v;
|
||||
});
|
||||
_offset = 0;
|
||||
load();
|
||||
}
|
||||
|
||||
function fillTypes(types) {
|
||||
const sel = document.getElementById('ev-type');
|
||||
const current = sel.value;
|
||||
sel.innerHTML = '<option value="">All alert types</option>' +
|
||||
types.map(t => `<option value="${esc(t)}">${esc(typeText(t))}</option>`).join('');
|
||||
if (current && !types.includes(current)) sel.add(new Option(typeText(current), current));
|
||||
sel.value = current;
|
||||
}
|
||||
|
||||
function renderSummary(data, f) {
|
||||
const open = data.events.filter(e => !e.resolved_at).length;
|
||||
const chips = [
|
||||
`<span class="cs-chip"><b>${data.total}</b> alert${data.total === 1 ? '' : 's'} in the last ${esc(RANGE_TEXT[f.range] || f.range)}</span>`,
|
||||
];
|
||||
if (open && data.offset === 0) chips.push(`<span class="cs-chip cs-chip-bad"><b>${open}</b> open now</span>`);
|
||||
Object.entries(data.by_type || {}).forEach(([t, n]) => {
|
||||
const on = f.type === t;
|
||||
chips.push(`<button type="button" class="cs-chip ev-type-chip${on ? ' active' : ''}" data-type="${esc(on ? '' : t)}"
|
||||
aria-pressed="${on}"><b>${n}</b> ${esc(typeText(t))}</button>`);
|
||||
});
|
||||
document.getElementById('ev-summary').innerHTML = chips.join('');
|
||||
}
|
||||
|
||||
function renderFlappers(rows) {
|
||||
const sec = document.getElementById('ev-flappers-section');
|
||||
sec.hidden = !rows.length;
|
||||
if (!rows.length) return;
|
||||
document.getElementById('ev-flappers').innerHTML = `<div class="lt-table-wrap"><table class="lt-table ev-cards">
|
||||
<caption class="lt-sr-only">Alerts that fired more than once</caption>
|
||||
<thead><tr><th>Alert</th><th>Target</th><th>Times</th><th>Total time</th><th>Last started</th><th></th></tr></thead>
|
||||
<tbody>${rows.map(r => `
|
||||
<tr>
|
||||
<td class="ev-c-type">${esc(typeText(r.event_type))}${r.open_now ? ' <span class="lt-badge badge-critical">open</span>' : ''}</td>
|
||||
<td class="ev-c-target"><a class="device-link" href="${esc(deviceUrl(r.target_name))}">${esc(r.target_name)}</a>
|
||||
${r.target_detail ? `<span class="cs-port">${esc(r.target_detail)}</span>` : ''}</td>
|
||||
<td data-label="Times"><strong>${r.n}</strong></td>
|
||||
<td data-label="Total">${esc(duration(r.total_s))}</td>
|
||||
<td class="ts-cell" data-label="Last">${when(r.last_seen)}</td>
|
||||
<td class="ev-c-action"><button type="button" class="lt-btn lt-btn-secondary lt-btn-sm" data-show-type="${esc(r.event_type)}"
|
||||
data-show-target="${esc(r.target_name)}">Show these</button></td>
|
||||
</tr>`).join('')}</tbody></table></div>`;
|
||||
}
|
||||
|
||||
function renderTable(data) {
|
||||
document.getElementById('ev-count').textContent = data.total;
|
||||
const wrap = document.getElementById('ev-table');
|
||||
if (!data.events.length) {
|
||||
wrap.innerHTML = '<div class="lt-empty-state lt-empty-state--sm"><div class="lt-empty-state-icon">✔</div>' +
|
||||
'<div class="lt-empty-state-title">No alerts match</div></div>';
|
||||
document.getElementById('ev-pager').innerHTML = '';
|
||||
return;
|
||||
}
|
||||
wrap.innerHTML = `<div class="lt-table-wrap"><table class="lt-table ev-table ev-cards">
|
||||
<caption class="lt-sr-only">Alert history</caption>
|
||||
<thead><tr><th>State</th><th>Type</th><th>Target</th><th>Started</th><th>Lasted</th>
|
||||
<th title="Consecutive failed checks">Checks</th><th>Ticket</th><th>Description</th></tr></thead>
|
||||
<tbody>${data.events.map(e => {
|
||||
const open = !e.resolved_at;
|
||||
return `<tr class="${open ? `row-${esc(e.severity)}` : ''}">
|
||||
<td class="ev-c-state"><span class="lt-badge badge-${open ? esc(e.severity) : 'neutral'}">${open ? 'open' : 'cleared'}</span></td>
|
||||
<td class="ev-c-type"><button type="button" class="ev-link" data-show-type="${esc(e.event_type)}"
|
||||
title="Only show ${esc(typeText(e.event_type))}">${esc(typeText(e.event_type))}</button></td>
|
||||
<td class="ev-c-target"><a class="device-link" href="${esc(deviceUrl(e.target_name))}">${esc(e.target_name)}</a>
|
||||
${e.target_detail ? `<span class="cs-port">${esc(e.target_detail)}</span>` : ''}</td>
|
||||
<td class="ts-cell" data-label="Started">${when(e.first_seen)}</td>
|
||||
<td data-label="Lasted">${esc(duration(e.duration_s))}${open ? ' <span class="val-neutral">so far</span>' : ''}</td>
|
||||
<td data-label="Checks">${esc(e.consecutive_failures)}</td>
|
||||
<td data-label="Ticket">${ticketLink(e.ticket_id)}</td>
|
||||
<td class="desc-cell ev-c-desc" title="${esc(e.description || '')}">${esc(e.description || '')}</td>
|
||||
</tr>`;
|
||||
}).join('')}</tbody></table></div>`;
|
||||
|
||||
const end = data.offset + data.events.length;
|
||||
const prev = data.offset > 0;
|
||||
const next = end < data.total;
|
||||
document.getElementById('ev-pager').innerHTML = (prev || next) ? `
|
||||
<button type="button" class="lt-btn lt-btn-secondary lt-btn-sm" data-page="${data.offset - data.page_size}" ${prev ? '' : 'disabled'}>← Newer</button>
|
||||
<span>${data.offset + 1}–${end} of ${data.total}</span>
|
||||
<button type="button" class="lt-btn lt-btn-secondary lt-btn-sm" data-page="${end}" ${next ? '' : 'disabled'}>Older →</button>` : '';
|
||||
}
|
||||
|
||||
async function load() {
|
||||
const f = filters();
|
||||
writeUrl(f);
|
||||
const q = new URLSearchParams();
|
||||
FILTERS.forEach(k => { if (f[k]) q.set(k, f[k]); });
|
||||
if (_offset) q.set('offset', _offset);
|
||||
const seq = ++_loadSeq;
|
||||
try {
|
||||
const data = await lt.api.get(`/api/events/history?${q}`);
|
||||
if (seq !== _loadSeq) return; // a newer filter change is already loading
|
||||
fillTypes(data.types || []);
|
||||
renderSummary(data, f);
|
||||
renderFlappers(data.flappers || []);
|
||||
renderTable(data);
|
||||
} catch (err) {
|
||||
if (seq !== _loadSeq) return;
|
||||
document.getElementById('ev-table').innerHTML =
|
||||
`<div class="lt-empty-state lt-empty-state--sm"><div class="lt-empty-state-title">Could not load alerts</div>
|
||||
<div class="lt-empty-state-text">${esc(err.message || err)}</div></div>`;
|
||||
}
|
||||
}
|
||||
|
||||
let _typing = null;
|
||||
document.getElementById('ev-target').addEventListener('input', () => {
|
||||
clearTimeout(_typing);
|
||||
_typing = setTimeout(() => { _offset = 0; load(); }, 300);
|
||||
});
|
||||
document.getElementById('ev-filters').addEventListener('change', e => {
|
||||
if (e.target.id === 'ev-target') return;
|
||||
_offset = 0;
|
||||
load();
|
||||
});
|
||||
document.getElementById('ev-filters').addEventListener('submit', e => { e.preventDefault(); _offset = 0; load(); });
|
||||
document.getElementById('ev-clear').addEventListener('click', () => {
|
||||
setFilter({ target: '', type: '', severity: '', status: 'all' });
|
||||
});
|
||||
document.addEventListener('click', e => {
|
||||
const typeChip = e.target.closest('.ev-type-chip');
|
||||
if (typeChip) return setFilter({ type: typeChip.dataset.type });
|
||||
const show = e.target.closest('[data-show-type]');
|
||||
if (show) {
|
||||
const changes = { type: show.dataset.showType };
|
||||
if (show.dataset.showTarget) changes.target = show.dataset.showTarget;
|
||||
return setFilter(changes);
|
||||
}
|
||||
const page = e.target.closest('[data-page]');
|
||||
if (page && !page.disabled) {
|
||||
_offset = Math.max(0, parseInt(page.dataset.page, 10) || 0);
|
||||
load();
|
||||
document.getElementById('ev-count').scrollIntoView({ block: 'nearest' });
|
||||
}
|
||||
});
|
||||
|
||||
readUrl();
|
||||
load();
|
||||
@@ -1393,3 +1393,32 @@ polygon.fp-led-off { fill: #6b6c71; }
|
||||
.cs-label-card { break-inside: avoid; border-color: #000; color: #000; }
|
||||
.lt-table, .lt-table td, .lt-table th { color: #000 !important; }
|
||||
}
|
||||
|
||||
/* ── Alert history (/events) ───────────────────────────────────────── */
|
||||
.ev-filters .lt-select { flex: 0 1 170px; }
|
||||
.ev-type-chip { background: none; cursor: pointer; font: inherit; font-size: .82rem; }
|
||||
.ev-type-chip:hover, .ev-type-chip:focus-visible { border-color: var(--accent-cyan); }
|
||||
.ev-type-chip.active { border-color: var(--accent-cyan); background: var(--accent-cyan-dim); }
|
||||
.ev-link { background: none; border: 0; padding: 0; color: inherit; font: inherit; cursor: pointer;
|
||||
text-decoration: underline dotted; text-underline-offset: 3px; text-align: left; }
|
||||
.ev-link:hover, .ev-link:focus-visible { color: var(--accent-cyan); }
|
||||
.ev-table td { vertical-align: top; }
|
||||
.ev-table .cs-port { display: block; }
|
||||
.ev-pager { display: flex; gap: 12px; align-items: center; justify-content: center; margin-top: 12px; font-size: .85rem; }
|
||||
.g-section-more { margin: 10px 12px 4px; font-size: .85rem; }
|
||||
.ev-table .lt-badge, .ev-c-type { white-space: nowrap; }
|
||||
|
||||
/* Phones: each alert becomes a card instead of a squeezed table row */
|
||||
@media (max-width: 760px) {
|
||||
.ev-cards thead { display: none; }
|
||||
.ev-cards, .ev-cards tbody { display: block; width: 100%; }
|
||||
.ev-cards tr { display: flex; flex-wrap: wrap; align-items: baseline; gap: 4px 14px; padding: 10px 12px;
|
||||
border-bottom: 1px solid var(--border-color); }
|
||||
.ev-cards td { display: block; padding: 0; border: 0; max-width: none; white-space: normal; font-size: .82rem; }
|
||||
.ev-cards td[data-label]::before { content: attr(data-label) " "; color: var(--text-muted); font-size: .72rem;
|
||||
text-transform: uppercase; letter-spacing: .05em; }
|
||||
.ev-cards .ev-c-target { flex: 1 1 100%; order: -1; font-size: .95rem; }
|
||||
.ev-cards .ev-c-target .cs-port { display: inline; margin-left: 6px; }
|
||||
.ev-cards .ev-c-desc { flex: 1 1 100%; color: var(--text-muted); white-space: normal; overflow: visible; }
|
||||
.ev-cards .ev-c-action { flex: 1 1 100%; }
|
||||
}
|
||||
|
||||
@@ -37,6 +37,9 @@
|
||||
<a href="{{ url_for('index') }}"
|
||||
class="lt-nav-drawer-link{% if request.endpoint == 'index' %} active{% endif %}"
|
||||
{% if request.endpoint == 'index' %}aria-current="page"{% endif %}>Dashboard</a>
|
||||
<a href="{{ url_for('events_page') }}"
|
||||
class="lt-nav-drawer-link{% if request.endpoint == 'events_page' %} active{% endif %}"
|
||||
{% if request.endpoint == 'events_page' %}aria-current="page"{% endif %}>History</a>
|
||||
<a href="{{ url_for('cutsheet_page') }}"
|
||||
class="lt-nav-drawer-link{% if request.endpoint == 'cutsheet_page' %} active{% endif %}"
|
||||
{% if request.endpoint == 'cutsheet_page' %}aria-current="page"{% endif %}>Cutsheet</a>
|
||||
@@ -82,6 +85,11 @@
|
||||
{% if request.endpoint == 'index' %}aria-current="page"{% endif %}>
|
||||
Dashboard
|
||||
</a>
|
||||
<a href="{{ url_for('events_page') }}"
|
||||
class="lt-nav-link{% if request.endpoint == 'events_page' %} active{% endif %}"
|
||||
{% if request.endpoint == 'events_page' %}aria-current="page"{% endif %}>
|
||||
History
|
||||
</a>
|
||||
<a href="{{ url_for('cutsheet_page') }}"
|
||||
class="lt-nav-link{% if request.endpoint == 'cutsheet_page' %} active{% endif %}"
|
||||
{% if request.endpoint == 'cutsheet_page' %}aria-current="page"{% endif %}>
|
||||
@@ -324,6 +332,7 @@
|
||||
// Command palette
|
||||
lt.cmdPalette.init([
|
||||
{ id: 'nav-dashboard', group: 'Navigate', icon: '~', label: 'Dashboard', action: function() { window.location.href = '/'; } },
|
||||
{ id: 'nav-history', group: 'Navigate', icon: '⏱', label: 'Alert History', action: function() { window.location.href = '/events'; } },
|
||||
{ id: 'nav-cutsheet', group: 'Navigate', icon: '🔌', label: 'Cutsheet', action: function() { window.location.href = '/cutsheet'; } },
|
||||
{ id: 'nav-suppressions', group: 'Navigate', icon: '🔕', label: 'Suppressions', action: function() { window.location.href = '/suppressions'; } },
|
||||
{ id: 'action-refresh', group: 'Actions', icon: '↻', label: 'Refresh Data', kbd: 'R', action: function() { lt.autoRefresh.now(); } },
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
{% extends "base.html" %}
|
||||
{% block title %}Alert History – GANDALF{% endblock %}
|
||||
|
||||
{% block content %}
|
||||
<div class="lt-page-header">
|
||||
<div>
|
||||
<h1 class="lt-page-title">Alert History</h1>
|
||||
<p class="g-page-sub">Every alert Gandalf raised, including the ones that cleared on their own. Resolved alerts are kept 90 days.</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<form class="cs-filters ev-filters" id="ev-filters" role="search" aria-label="Filter alert history">
|
||||
<input type="search" class="lt-input" id="ev-target" name="target" placeholder="Device, host, port or interface"
|
||||
aria-label="Filter by device" autocomplete="off">
|
||||
<select class="lt-select" id="ev-range" name="range" aria-label="Time range">
|
||||
<option value="24h">Last 24 hours</option>
|
||||
<option value="7d" selected>Last 7 days</option>
|
||||
<option value="30d">Last 30 days</option>
|
||||
<option value="90d">Last 90 days</option>
|
||||
</select>
|
||||
<select class="lt-select" id="ev-type" name="type" aria-label="Alert type">
|
||||
<option value="">All alert types</option>
|
||||
</select>
|
||||
<select class="lt-select" id="ev-severity" name="severity" aria-label="Severity">
|
||||
<option value="">All severities</option>
|
||||
<option value="critical">Critical</option>
|
||||
<option value="warning">Warning</option>
|
||||
<option value="info">Info</option>
|
||||
</select>
|
||||
<select class="lt-select" id="ev-status" name="status" aria-label="State">
|
||||
<option value="all">Open and cleared</option>
|
||||
<option value="active">Open now</option>
|
||||
<option value="resolved">Cleared</option>
|
||||
</select>
|
||||
<button type="button" class="lt-btn lt-btn-secondary lt-btn-sm" id="ev-clear" hidden>Clear filters</button>
|
||||
</form>
|
||||
|
||||
<div id="ev-summary" class="cs-summary" aria-live="polite"></div>
|
||||
|
||||
<section class="g-section" id="ev-flappers-section" hidden>
|
||||
<div class="g-section-header">
|
||||
<h2 class="g-section-title">Recurring</h2>
|
||||
</div>
|
||||
<p class="cs-hint">The same alert on the same target more than once in this range. Something that keeps coming back usually has one cause worth finding.</p>
|
||||
<div class="lt-frame">
|
||||
<span class="lt-frame-bl">╚</span><span class="lt-frame-br">╝</span>
|
||||
<div id="ev-flappers"></div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="g-section">
|
||||
<div class="g-section-header">
|
||||
<h2 class="g-section-title">Alerts</h2>
|
||||
<span class="g-section-badge" id="ev-count">0</span>
|
||||
</div>
|
||||
<div class="lt-frame">
|
||||
<span class="lt-frame-bl">╚</span><span class="lt-frame-br">╝</span>
|
||||
<div id="ev-table"><div class="lt-empty-state lt-empty-state--sm"><div class="lt-empty-state-title">Loading…</div></div></div>
|
||||
</div>
|
||||
<div class="ev-pager" id="ev-pager"></div>
|
||||
</section>
|
||||
{% endblock %}
|
||||
|
||||
{% block scripts %}
|
||||
<script src="{{ url_for('static', filename='events.js') }}"></script>
|
||||
{% endblock %}
|
||||
@@ -316,3 +316,63 @@ class TestCutsheetLivePorts:
|
||||
def test_admin_only(self, cs_client):
|
||||
client, _ = cs_client
|
||||
assert client.get('/api/cutsheet/live-ports', headers=USER).status_code == 403
|
||||
|
||||
|
||||
class TestAlertHistory:
|
||||
HISTORY = {'events': [], 'total': 0, 'by_type': {}}
|
||||
|
||||
def _db(self):
|
||||
db = gandalf.db
|
||||
db.get_events_history.return_value = dict(self.HISTORY)
|
||||
db.get_flappers.return_value = []
|
||||
db.get_event_types.return_value = ['host_unreachable']
|
||||
return db
|
||||
|
||||
def test_page_and_nav(self, client):
|
||||
html = client.get('/events', headers=AUTH).get_data(as_text=True)
|
||||
assert 'Alert History' in html and 'events.js' in html
|
||||
assert 'href="/events"' in client.get('/', headers=AUTH).get_data(as_text=True)
|
||||
|
||||
def test_defaults_to_seven_days(self, client):
|
||||
db = self._db()
|
||||
data = client.get('/api/events/history', headers=AUTH).get_json()
|
||||
assert db.get_events_history.call_args.args == (168, '', '', '', 'all')
|
||||
assert data['types'] == ['host_unreachable'] and data['page_size'] == 100
|
||||
|
||||
def test_filters_are_passed_through(self, client):
|
||||
db = self._db()
|
||||
client.get('/api/events/history?range=90d&type=interface_down&target=+large1+&severity=critical'
|
||||
'&status=resolved&offset=200', headers=AUTH)
|
||||
assert db.get_events_history.call_args.args == (2160, 'interface_down', 'large1', 'critical', 'resolved')
|
||||
assert db.get_events_history.call_args.kwargs == {'limit': 100, 'offset': 200}
|
||||
assert db.get_flappers.call_args.args == (2160, 'interface_down', 'large1', 'critical')
|
||||
|
||||
@pytest.mark.parametrize('query', ['range=1y', 'severity=bad', 'status=open', 'offset=x'])
|
||||
def test_bad_filters_are_rejected(self, client, query):
|
||||
self._db()
|
||||
assert client.get(f'/api/events/history?{query}', headers=AUTH).status_code == 400
|
||||
|
||||
def test_needs_login(self, client):
|
||||
assert client.get('/api/events/history').status_code == 401
|
||||
|
||||
|
||||
class TestHealth:
|
||||
def _health(self, client, age_s, poll=120):
|
||||
from datetime import datetime, timedelta, timezone
|
||||
stamp = (datetime.now(timezone.utc) - timedelta(seconds=age_s)).strftime('%Y-%m-%d %H:%M:%S UTC')
|
||||
gandalf.db.get_state.side_effect = lambda key, default=None: stamp if key == 'last_check' else default
|
||||
with patch.object(gandalf, '_cfg', dict(CONFIG, monitor={'poll_interval': poll})):
|
||||
return client.get('/health')
|
||||
|
||||
def test_recent_check_is_healthy(self, client):
|
||||
resp = self._health(client, 150)
|
||||
assert resp.status_code == 200 and resp.get_json()['checks']['monitor'] == 'ok'
|
||||
|
||||
def test_stalled_monitor_fails_after_ten_minutes(self, client):
|
||||
resp = self._health(client, 660)
|
||||
assert resp.status_code == 503
|
||||
assert resp.get_json()['checks'] == {'db': 'ok', 'monitor': 'stale', 'monitor_age_s': 660}
|
||||
|
||||
def test_limit_follows_a_long_poll_interval(self, client):
|
||||
assert self._health(client, 1000, poll=300).status_code == 200
|
||||
assert self._health(client, 1600, poll=300).status_code == 503
|
||||
|
||||
@@ -49,6 +49,10 @@ def device_url(base: str, name: str, port_idx=None) -> str:
|
||||
return f'{url}#port-{port_idx}' if port_idx not in (None, '') else url
|
||||
|
||||
|
||||
def history_url(base: str, target: str) -> str:
|
||||
return f'{base}/events?target={quote(target, safe="")}&range=90d' if base and target else ''
|
||||
|
||||
|
||||
def cutsheet_url(base: str, cable_id) -> str:
|
||||
return f'{base}/cutsheet#cable-{cable_id}' if base else ''
|
||||
|
||||
|
||||
Reference in New Issue
Block a user