diff --git a/README.md b/README.md index 31a518b..21a351a 100644 --- a/README.md +++ b/README.md @@ -309,6 +309,12 @@ can view it; only the `admin` group can edit, and every change is logged with wh - Suppression history (last 50) - Available targets reference grid (all known hosts + interfaces) +**Maintenance**: admins get a **Maintenance** button in the header. It starts a global suppression for +30 min, 1, 2 or 4 hours with a reason (by default "Maintenance"). While it runs, every page shows a banner +saying when tickets resume, with **End now** for admins. Gandalf keeps checking and recording alerts, and +anything still failing when maintenance ends gets a ticket as usual. Use it before a UniFi update, a switch +swap or recabling. + --- ## Alert Logic @@ -386,7 +392,23 @@ Outages can be over between polls: two real ones lasted about two minutes agains So WAN outages and modem restarts also come from the UniFi system log, and a drop in the modem's uptime records a restart even if nothing logged it. A single short outage only appears in the panel's history. -To silence modem alerts during maintenance, create a `host` suppression with the modem's `name`. +**One ticket per incident.** The modem and WAN reports from one check cycle open a single ticket, led by +the most urgent (a P2 "WAN modem unreachable" leads, and an RPDU power-cycle seen in the same cycle is listed +under "Also seen in this incident"). A report that comes while the modem has an open alert, or one that +ended under 30 minutes ago, becomes a comment on that alert's ticket instead of a new ticket. If the +comment can't be posted, the report gets its own ticket as before. + +**UniFi updates.** UniFi logs `NETWORK_UPDATED` (Network app) and `FIRMWARE_UPDATED_V2` (device firmware) +once an update has finished. Updating the Network app can drop the UDM's WAN, and Smart Power may then +power-cycle a modem that was fine. When Gandalf reads one of these entries it: + +- comments on modem and WAN tickets first seen from 20 minutes before the update to 15 minutes after, + naming the update (for example "UniFi Network 11.0.97 to 11.0.99") as the likely cause; +- opens no new modem or WAN tickets for 15 minutes after the update. A problem still there after that gets + its ticket as usual; a power-cycle report during the hold only goes into the event history. + +To silence modem alerts during maintenance, create a `host` suppression with the modem's `name`, or use +**Maintenance** (below) to pause every ticket. ### Baseline Tracking diff --git a/app.py b/app.py index 683a760..92a2754 100644 --- a/app.py +++ b/app.py @@ -60,6 +60,19 @@ def inject_config(): } +@app.context_processor +def inject_maintenance(): + """The active global suppression (maintenance), shown as a banner on every page.""" + try: + active = [s for s in db.get_active_suppressions() if s.get('target_type') == 'all'] + except Exception as e: + logger.error(f'Reading maintenance state failed: {e}') + active = [] + # Of several, the one that ends last decides when tickets resume; no expiry beats any + active.sort(key=lambda s: (s.get('expires_at') is None, s.get('expires_at') or '')) + return {'maintenance': active[-1] if active else None} + + # In-memory diagnostic job store { job_id: { status, result, created_at } } _diag_jobs: dict = {} _diag_lock = threading.Lock() diff --git a/db.py b/db.py index 8f80984..a9f207f 100644 --- a/db.py +++ b/db.py @@ -4,7 +4,7 @@ import logging import threading from contextlib import contextmanager from datetime import datetime, timedelta, timezone -from typing import Optional +from typing import List, Optional import pymysql import pymysql.cursors @@ -198,6 +198,35 @@ def set_ticket_id(event_id: int, ticket_id: str) -> None: ) +def get_incident_ticket(target_name: str, within_s: int) -> Optional[str]: + """The ticket of *target_name*'s newest event that is still open or ended within *within_s*.""" + with get_conn() as conn: + with conn.cursor() as cur: + cur.execute( + """SELECT ticket_id FROM network_events + WHERE target_name=%s AND ticket_id IS NOT NULL AND ticket_id <> '' + AND (resolved_at IS NULL OR resolved_at >= NOW() - INTERVAL %s SECOND) + ORDER BY first_seen DESC, id DESC LIMIT 1""", + (target_name, int(within_s)), + ) + row = cur.fetchone() + return row['ticket_id'] if row else None + + +def get_ticket_ids_between(target_name: str, start: datetime, end: datetime) -> List[str]: + """Tickets of *target_name*'s events first seen between *start* and *end* (UTC), oldest first.""" + with get_conn() as conn: + with conn.cursor() as cur: + cur.execute( + """SELECT ticket_id, MIN(first_seen) AS first FROM network_events + WHERE target_name=%s AND ticket_id IS NOT NULL AND ticket_id <> '' + AND first_seen BETWEEN %s AND %s + GROUP BY ticket_id ORDER BY first""", + (target_name, start, end), + ) + return [r['ticket_id'] for r in cur.fetchall()] + + def get_active_events(limit: int = 200, offset: int = 0) -> list: with get_conn() as conn: with conn.cursor() as cur: diff --git a/modem.py b/modem.py index 21cd2e6..0498f1e 100644 --- a/modem.py +++ b/modem.py @@ -242,6 +242,31 @@ LOG_WAN_RESTORED = 'NETWORK_WAN_RESTORED_2' LOG_WAN_UNSTABLE = 'NETWORK_WAN_FAILED_MULTIPLE_TIMES_2' LOG_MODEM_RESTARTED = 'SMART_POWER_MODEM_RESTARTED' WAN_LOG_KEYS = (LOG_WAN_FAILED, LOG_WAN_RESTORED, LOG_WAN_UNSTABLE, LOG_MODEM_RESTARTED) +# An app or firmware update finishing. The UDM often drops its WAN while the Network app +# updates, and Smart Power may then power-cycle a modem that was fine. +LOG_NETWORK_UPDATED = 'NETWORK_UPDATED' +LOG_FIRMWARE_UPDATED = 'FIRMWARE_UPDATED_V2' +_UPDATE_KEY_RE = re.compile(r'[A-Z_]+_(?:UPDATED|UPGRADED)(?:_V\d+)?') + + +def is_update_key(key: str) -> bool: + """True for log keys that mean an update finished (other app names follow the same pattern).""" + return key in (LOG_NETWORK_UPDATED, LOG_FIRMWARE_UPDATED) or ( + bool(_UPDATE_KEY_RE.fullmatch(key or '')) and 'AVAILABLE' not in key + and not key.startswith('MADE_CHANGES')) + + +def wanted_log_key(key: str) -> bool: + return key in WAN_LOG_KEYS or is_update_key(key) + + +def update_label(event: dict) -> str: + """'UniFi Network 11.0.97 to 11.0.99' from an update log event.""" + what = event.get('app') or event.get('device') or 'UniFi' + before, after = event.get('previous') or '', event.get('version') or '' + if before and after: + return f'{what} {before} to {after}' + return f'{what} {after}'.strip() _HISTORY_LIMIT = 50 # The RPDU's log entry and the gateway's calculated boot time can differ by this much. diff --git a/monitor.py b/monitor.py index c19d7b5..83af575 100644 --- a/monitor.py +++ b/monitor.py @@ -13,7 +13,8 @@ import re import shlex import statistics import time -from datetime import datetime, timezone +from contextlib import contextmanager +from datetime import datetime, timedelta, timezone from typing import Dict, List, Optional from urllib.parse import urljoin, urlparse @@ -192,8 +193,12 @@ class UnifiClient: logger.error(f'UniFi client list error: {e}') return [] - def get_system_events(self, since_ms: int, keys: tuple) -> Optional[List[dict]]: - """Return system log entries with the given keys, or None if the log is unavailable.""" + def get_system_events(self, since_ms: int, keys) -> Optional[List[dict]]: + """Return system log entries with the given keys, or None if the log is unavailable. + + keys: a tuple of log keys, or a function that takes a key and says whether to keep it. + """ + wanted = keys if callable(keys) else keys.__contains__ url = f'{self.base_url}/proxy/network/v2/api/site/{self.site_id}/system-log/all' found: List[dict] = [] try: @@ -210,19 +215,29 @@ class UnifiClient: if not items: break for e in items: - if e.get('key') in keys and e.get('timestamp', 0) >= since_ms: + if wanted(e.get('key') or '') and e.get('timestamp', 0) >= since_ms: params = e.get('parameters') or {} found.append({ 'key': e['key'], 'ts': e['timestamp'] / 1000, 'device': (params.get('DEVICE') or {}).get('name', ''), 'wan': (params.get('WAN_ID') or {}).get('name', '') or str((params.get('WAN_ID') or {}).get('id', '')), + **self._update_fields(params), }) return found except Exception as e: logger.error(f'UniFi system log error: {e}') return None + @staticmethod + def _update_fields(params: dict) -> dict: + """The app and versions of an update entry; nothing for other entries.""" + def text(key): + value = params.get(key) + return str(value.get('name') or value.get('id') or '') if isinstance(value, dict) else str(value or '') + fields = {'app': text('APPLICATION'), 'version': text('VERSION'), 'previous': text('PREVIOUS_VERSION')} + return {k: v for k, v in fields.items() if v} + def get_devices(self) -> Optional[List[dict]]: """Return list of UniFi devices, or None if the controller is unreachable.""" try: @@ -965,6 +980,13 @@ CLUSTER_NAME = 'proxmox-cluster' DEFAULT_PUBLIC_URL = 'https://gandalf.lotusguild.org' # Reports of something already over: the ticket is the whole story, no recovery note ONE_SHOT_EVENTS = ('modem_power_cycled', 'wan_unstable') +# Modem and WAN reports this close together are one incident, so they share one ticket +WAN_INCIDENT_WINDOW_S = 1800 +# UniFi logs an update when it finishes, after any WAN trouble it caused. So tickets opened +# this long before the entry get a note, and new modem/WAN tickets wait this long after it. +UPDATE_LOOKBACK_S = 1200 +UPDATE_HOLD_S = 900 +UNIFI_UPDATE_STATE = 'unifi_update' # Ports a host or device was last seen on, kept so a ticket for a dead host # (whose link, and so its LLDP entry, is gone) can still say where it plugs in KNOWN_UPLINKS_STATE = 'known_uplinks' @@ -1543,14 +1565,95 @@ class NetworkMonitor: event_type, severity, 'modem', name, detail, f'{description} ({_now_utc()})', ) if not sup and consec >= self.modem_fail_thresh and not ticket_id: - self._open_ticket( - event_id, event_type, name, detail, - title=f'[{name}][auto][production][issue][network][cluster-wide] {heading}', - heading='WAN Modem Alert', - facts=[('Modem', f'{name} ({self.modem.base_url})'), ('Detected', _now_utc()), - ('Consecutive check failures', str(consec))], - summary=description, checks=[advice], priority=priority, - ) + self._queue_wan_ticket({ + 'event_id': event_id, 'event_type': event_type, 'detail': detail, + 'title': f'[{name}][auto][production][issue][network][cluster-wide] {heading}', + 'name': heading, 'heading': 'WAN Modem Alert', + 'facts': [('Modem', f'{name} ({self.modem.base_url})'), ('Detected', _now_utc()), + ('Consecutive check failures', str(consec))], + 'summary': description, 'checks': [advice], 'priority': priority, 'recovers': True, + }) + + # ------------------------------------------------------------------ + # Modem and WAN tickets: one per incident, held around UniFi updates + # ------------------------------------------------------------------ + @contextmanager + def _wan_batch(self): + """Collect the cycle's modem and WAN tickets so related ones are opened together.""" + self._wan_pending = [] + try: + yield + finally: + pending, self._wan_pending = self._wan_pending, None + if pending: + self._open_wan_tickets(pending) + + def _queue_wan_ticket(self, item: dict) -> None: + held = self._update_hold() + if held: + logger.info(f'Not opening a ticket for {item["event_type"]} yet: {held} just finished') + return + if getattr(self, '_wan_pending', None) is not None: + self._wan_pending.append(item) + else: + self._open_wan_tickets([item]) + + def _update_hold(self) -> Optional[str]: + """The update that modem and WAN tickets are waiting out, if any.""" + raw = db.get_state(UNIFI_UPDATE_STATE) + if not raw: + return None + try: + update = json.loads(raw) + except ValueError: + return None + if update.get('hold_until', 0) <= time.time(): + return None + return update.get('label') or 'a UniFi update' + + def _open_wan_tickets(self, items: List[dict]) -> None: + """Comment on the incident's ticket if one is recent, else open one ticket for all of *items*.""" + # Most urgent first; at equal priority an ongoing problem leads over a past event + items = sorted(items, key=lambda i: (int(i['priority']), not i['recovers'])) + incident = db.get_incident_ticket(self.modem_name, WAN_INCIDENT_WINDOW_S) + if incident: + left = [] + for item in items: + text = ticket_text.incident_comment(item['name'], item['summary'], item['checks'], _now_utc()) + if self.tickets.comment(incident, text): + db.set_ticket_id(item['event_id'], incident) + else: + left.append(item) # comments failing must not lose the report + items = left + if not items: + return + first, others = items[0], items[1:] + sections = [] + if first['recovers']: + sections = self._ticket_sections(first['event_id'], first['event_type'], self.modem_name, first['detail']) + if others: + sections.append(('Also seen in this incident', [f'- **{o["name"]}:** {o["summary"]}' for o in others])) + desc = ticket_text.description(first['heading'], first['facts'], first['summary'], first['checks'], + sections, recovers=any(i['recovers'] for i in items)) + tid = self.tickets.create(first['title'], desc, priority=first['priority']) + if tid: + for item in items: + db.set_ticket_id(item['event_id'], tid) + + def _note_unifi_update(self, event: dict) -> None: + """Hold new modem/WAN tickets for a while and explain the ones opened just before.""" + label = modem_api.update_label(event) + when = datetime.fromtimestamp(event['ts'], timezone.utc) + logger.info(f'UniFi update finished: {label} at {when:%Y-%m-%d %H:%M:%S} UTC') + if event['ts'] + UPDATE_HOLD_S > time.time(): + db.set_state(UNIFI_UPDATE_STATE, {'label': label, 'at': event['ts'], + 'hold_until': event['ts'] + UPDATE_HOLD_S}) + naive = when.replace(tzinfo=None) + tickets = db.get_ticket_ids_between( + self.modem_name, naive - timedelta(seconds=UPDATE_LOOKBACK_S), naive + timedelta(seconds=UPDATE_HOLD_S)) + for tid in tickets: + self.tickets.comment(tid, ticket_text.update_comment( + label, f'{when:%Y-%m-%d %H:%M} UTC', UPDATE_LOOKBACK_S // 60, UPDATE_HOLD_S // 60)) def _process_modem_failure(self, suppressions: list, error: Exception) -> None: """Handle a failed poll. Link and optical events stay as they are: we can't see them.""" @@ -1690,14 +1793,13 @@ class NetworkMonitor: name = self.modem_name event_id, _, _, ticket_id = db.upsert_event(event_type, 'warning', 'unifi', name, '', description) if not db.check_suppressed(suppressions, 'host', name) and not ticket_id: - tid = self.tickets.create( - f'[{name}][auto][production][issue][network][cluster-wide] {heading}', - ticket_text.description('WAN Event', [('Modem', name), ('Reported', _now_utc())], - description, [advice], recovers=False), - priority=priority, - ) - if tid: - db.set_ticket_id(event_id, tid) + self._queue_wan_ticket({ + 'event_id': event_id, 'event_type': event_type, 'detail': '', + 'title': f'[{name}][auto][production][issue][network][cluster-wide] {heading}', + 'name': heading, 'heading': 'WAN Event', + 'facts': [('Modem', name), ('Reported', _now_utc())], + 'summary': description, 'checks': [advice], 'priority': priority, 'recovers': False, + }) db.resolve_event(event_type, name, '') def _process_wan_log(self, suppressions: list) -> None: @@ -1713,12 +1815,21 @@ class NetworkMonitor: return # After long monitor downtime, don't open tickets for stale events. since_ms = max(int(raw_cursor) + 1, int((now - 86400) * 1000)) - events = self.unifi.get_system_events(since_ms, modem_api.WAN_LOG_KEYS) + events = self.unifi.get_system_events(since_ms, modem_api.wanted_log_key) if not events: return # None = log unavailable, so the cursor stays put and it's retried + # Updates first: one in this batch explains (and holds) the WAN events beside it + updates = [e for e in events if modem_api.is_update_key(e['key'])] + for e in sorted(updates, key=lambda ev: ev['ts']): + try: + self._note_unifi_update(e) + except Exception as ex: + logger.error(f'Noting UniFi update failed: {ex}', exc_info=True) + history = self._load_wan_history() - actionable = modem_api.apply_wan_events(history, events) + actionable = modem_api.apply_wan_events( + history, [e for e in events if not modem_api.is_update_key(e['key'])]) # UniFi repeats "failed multiple times" per failure, so ticket it once a day at most. unstable = [e for e in actionable if e['key'] == modem_api.LOG_WAN_UNSTABLE] @@ -1927,9 +2038,11 @@ class NetworkMonitor: # WAN modem: isolated so a failure here can't stop the rest of the cycle if self.modem: try: - # Log first, so a restart it reports can explain an uptime reset below - self._process_wan_log(suppressions) - self._collect_modem(suppressions) + # Log first, so a restart it reports can explain an uptime reset below. + # Tickets open when the batch ends, so one incident gets one ticket. + with self._wan_batch(): + self._process_wan_log(suppressions) + self._collect_modem(suppressions) except Exception as e: logger.error(f'Modem collection failed: {e}', exc_info=True) diff --git a/static/app.js b/static/app.js index b586c4f..1c9c995 100644 --- a/static/app.js +++ b/static/app.js @@ -648,3 +648,65 @@ document.addEventListener('click', e => { ); } }); + +// ── Maintenance: pause every ticket for a while (a global suppression with an expiry) ── +(function () { + const until = document.querySelector('.maint-banner .maint-until'); + if (until) { + const t = new Date(until.getAttribute('datetime')); + if (!isNaN(t)) { + const sameDay = t.toDateString() === new Date().toDateString(); + until.textContent = sameDay + ? t.toLocaleTimeString([], { hour: 'numeric', minute: '2-digit' }) + : t.toLocaleString([], { weekday: 'short', hour: 'numeric', minute: '2-digit' }); + until.title = t.toLocaleString(); + } + } + + let minutes = 60; + document.addEventListener('click', async e => { + if (e.target.closest('[data-action="start-maintenance"]')) { + lt.modal.open('maint-modal'); + document.getElementById('maint-reason')?.focus(); + return; + } + const pill = e.target.closest('.maint-pill'); + if (pill) { + minutes = parseInt(pill.dataset.minutes, 10); + document.querySelectorAll('.maint-pill').forEach(p => { + const on = p === pill; + p.classList.toggle('active', on); + p.setAttribute('aria-pressed', String(on)); + }); + return; + } + const end = e.target.closest('[data-action="end-maintenance"]'); + if (end) { + end.disabled = true; + try { + await lt.api.delete(`/api/suppressions/${encodeURIComponent(end.dataset.supId)}`); + window.location.reload(); + } catch (err) { + end.disabled = false; + showToast(err.message || 'Could not end maintenance', 'error'); + } + } + }); + + document.getElementById('maint-form')?.addEventListener('submit', async e => { + e.preventDefault(); + const reason = (document.getElementById('maint-reason').value || '').trim(); + if (!reason) { showToast('Reason is required', 'error'); return; } + const submit = e.target.querySelector('[type="submit"]'); + submit.disabled = true; + try { + await lt.api.post('/api/suppressions', { + target_type: 'all', target_name: '', reason, expires_minutes: minutes, + }); + window.location.reload(); + } catch (err) { + submit.disabled = false; + showToast(err.message || 'Could not start maintenance', 'error'); + } + }); +})(); diff --git a/static/style.css b/static/style.css index 7b78ab9..9732b88 100644 --- a/static/style.css +++ b/static/style.css @@ -383,6 +383,22 @@ .pill:hover { border-color: var(--green); color: var(--green); background: var(--green-dim); } .pill.active, .pill-manual.active { border-color: var(--amber); color: var(--amber); background: var(--amber-dim); text-shadow: var(--glow-amber); } +/* Maintenance modal pills: same look, own class so the suppress modal's handler skips them */ +.maint-pill { padding: 3px 10px; border: 1px solid var(--border-color); background: transparent; + font-family: var(--font); font-size: .72em; font-weight: bold; cursor: pointer; + color: var(--text-muted); transition: all .15s; letter-spacing: .04em; } +.maint-pill:hover { border-color: var(--green); color: var(--green); background: var(--green-dim); } +.maint-pill.active { border-color: var(--amber); color: var(--amber); background: var(--amber-dim); text-shadow: var(--glow-amber); } +.maint-intro { margin: 0 0 12px; } + +/* Maintenance banner: every page while a global suppression is active */ +.maint-banner { + display: flex; align-items: center; justify-content: space-between; gap: 12px; flex-wrap: wrap; + margin-bottom: 16px; padding: 8px 14px; border: 1px solid var(--amber); background: var(--amber-dim); + color: var(--amber); font-size: .82em; +} +.maint-banner strong { letter-spacing: .06em; text-transform: uppercase; } +.maint-reason { color: var(--text-muted); } /* ── Available targets grid (suppressions page) ───────────────────── */ .targets-grid { display: grid; grid-template-columns: repeat(auto-fill, minmax(180px, 1fr)); gap: 10px; } diff --git a/templates/base.html b/templates/base.html index 2d9f9d4..ac8a75f 100644 --- a/templates/base.html +++ b/templates/base.html @@ -125,6 +125,10 @@ {{ _uname }} {% if user.groups and 'admin' in user.groups %} ADMIN + {% if not maintenance %} + + {% endif %} {% endif %} @@ -184,6 +188,20 @@
+ {% if maintenance %} +
+ + Maintenance: Gandalf is not opening tickets + {% if maintenance.expires_at %}until {% else %}until it is ended{% endif %}. + Alerts are still recorded, and anything still failing afterwards gets a ticket. + {{ maintenance.reason }}{% if maintenance.suppressed_by %} ({{ maintenance.suppressed_by }}){% endif %} + + {% if user.groups and 'admin' in user.groups %} + + {% endif %} +
+ {% endif %} {% block content %}{% endblock %}
@@ -203,9 +221,47 @@ | - GANDALF — TDS v1.2 + GANDALF | TDS v1.2 + {% if user.groups and 'admin' in user.groups and not maintenance %} + + + {% endif %} +