WAN tickets: one per incident, notice UniFi updates, maintenance button
Lint / JS (eslint) (push) Successful in 42s
Lint / Python (flake8) (push) Successful in 1m35s
Lint / Notify on failure (push) Skipped
Lint / Deploy (push) Skipped
Security / Python Security (bandit) (push) Successful in 1m3s
Test / Python Tests (pytest) (push) Successful in 1m5s
Lint / Python (flake8) (pull_request) Successful in 58s
Lint / JS (eslint) (pull_request) Successful in 8s
Lint / Notify on failure (pull_request) Skipped
Lint / Deploy (pull_request) Skipped
Security / Python Security (bandit) (pull_request) Successful in 1m1s
Test / Python Tests (pytest) (pull_request) Successful in 1m10s

A UniFi Network update on 2026-11-12 dropped the UDM's WAN, Smart Power
power-cycled the modem, and Gandalf opened two tickets for the one event.

- Modem and WAN reports from one check cycle open a single ticket, led by
  the most urgent, with the rest listed under "Also seen in this incident".
  A report while the modem has an open alert, or one that ended under 30
  minutes ago, becomes a comment on that ticket instead.
- UniFi update log entries (NETWORK_UPDATED, FIRMWARE_UPDATED_V2 and the
  like) add a note to modem/WAN tickets from 20 minutes before the update
  to 15 after, and hold new modem/WAN tickets for 15 minutes.
- A Maintenance button for admins starts a global suppression for 30 min
  to 4 hours, with a banner on every page and End now.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-11-12 12:53:22 -05:00
co-authored by Claude Opus 5.5
parent 6b03fe7b44
commit 379e3be365
11 changed files with 541 additions and 28 deletions
+138 -25
View File
@@ -13,7 +13,8 @@ import re
import shlex
import statistics
import time
from datetime import datetime, timezone
from contextlib import contextmanager
from datetime import datetime, timedelta, timezone
from typing import Dict, List, Optional
from urllib.parse import urljoin, urlparse
@@ -192,8 +193,12 @@ class UnifiClient:
logger.error(f'UniFi client list error: {e}')
return []
def get_system_events(self, since_ms: int, keys: tuple) -> Optional[List[dict]]:
"""Return system log entries with the given keys, or None if the log is unavailable."""
def get_system_events(self, since_ms: int, keys) -> Optional[List[dict]]:
"""Return system log entries with the given keys, or None if the log is unavailable.
keys: a tuple of log keys, or a function that takes a key and says whether to keep it.
"""
wanted = keys if callable(keys) else keys.__contains__
url = f'{self.base_url}/proxy/network/v2/api/site/{self.site_id}/system-log/all'
found: List[dict] = []
try:
@@ -210,19 +215,29 @@ class UnifiClient:
if not items:
break
for e in items:
if e.get('key') in keys and e.get('timestamp', 0) >= since_ms:
if wanted(e.get('key') or '') and e.get('timestamp', 0) >= since_ms:
params = e.get('parameters') or {}
found.append({
'key': e['key'], 'ts': e['timestamp'] / 1000,
'device': (params.get('DEVICE') or {}).get('name', ''),
'wan': (params.get('WAN_ID') or {}).get('name', '')
or str((params.get('WAN_ID') or {}).get('id', '')),
**self._update_fields(params),
})
return found
except Exception as e:
logger.error(f'UniFi system log error: {e}')
return None
@staticmethod
def _update_fields(params: dict) -> dict:
"""The app and versions of an update entry; nothing for other entries."""
def text(key):
value = params.get(key)
return str(value.get('name') or value.get('id') or '') if isinstance(value, dict) else str(value or '')
fields = {'app': text('APPLICATION'), 'version': text('VERSION'), 'previous': text('PREVIOUS_VERSION')}
return {k: v for k, v in fields.items() if v}
def get_devices(self) -> Optional[List[dict]]:
"""Return list of UniFi devices, or None if the controller is unreachable."""
try:
@@ -965,6 +980,13 @@ CLUSTER_NAME = 'proxmox-cluster'
DEFAULT_PUBLIC_URL = 'https://gandalf.lotusguild.org'
# Reports of something already over: the ticket is the whole story, no recovery note
ONE_SHOT_EVENTS = ('modem_power_cycled', 'wan_unstable')
# Modem and WAN reports this close together are one incident, so they share one ticket
WAN_INCIDENT_WINDOW_S = 1800
# UniFi logs an update when it finishes, after any WAN trouble it caused. So tickets opened
# this long before the entry get a note, and new modem/WAN tickets wait this long after it.
UPDATE_LOOKBACK_S = 1200
UPDATE_HOLD_S = 900
UNIFI_UPDATE_STATE = 'unifi_update'
# Ports a host or device was last seen on, kept so a ticket for a dead host
# (whose link, and so its LLDP entry, is gone) can still say where it plugs in
KNOWN_UPLINKS_STATE = 'known_uplinks'
@@ -1543,14 +1565,95 @@ class NetworkMonitor:
event_type, severity, 'modem', name, detail, f'{description} ({_now_utc()})',
)
if not sup and consec >= self.modem_fail_thresh and not ticket_id:
self._open_ticket(
event_id, event_type, name, detail,
title=f'[{name}][auto][production][issue][network][cluster-wide] {heading}',
heading='WAN Modem Alert',
facts=[('Modem', f'{name} ({self.modem.base_url})'), ('Detected', _now_utc()),
('Consecutive check failures', str(consec))],
summary=description, checks=[advice], priority=priority,
)
self._queue_wan_ticket({
'event_id': event_id, 'event_type': event_type, 'detail': detail,
'title': f'[{name}][auto][production][issue][network][cluster-wide] {heading}',
'name': heading, 'heading': 'WAN Modem Alert',
'facts': [('Modem', f'{name} ({self.modem.base_url})'), ('Detected', _now_utc()),
('Consecutive check failures', str(consec))],
'summary': description, 'checks': [advice], 'priority': priority, 'recovers': True,
})
# ------------------------------------------------------------------
# Modem and WAN tickets: one per incident, held around UniFi updates
# ------------------------------------------------------------------
@contextmanager
def _wan_batch(self):
"""Collect the cycle's modem and WAN tickets so related ones are opened together."""
self._wan_pending = []
try:
yield
finally:
pending, self._wan_pending = self._wan_pending, None
if pending:
self._open_wan_tickets(pending)
def _queue_wan_ticket(self, item: dict) -> None:
held = self._update_hold()
if held:
logger.info(f'Not opening a ticket for {item["event_type"]} yet: {held} just finished')
return
if getattr(self, '_wan_pending', None) is not None:
self._wan_pending.append(item)
else:
self._open_wan_tickets([item])
def _update_hold(self) -> Optional[str]:
"""The update that modem and WAN tickets are waiting out, if any."""
raw = db.get_state(UNIFI_UPDATE_STATE)
if not raw:
return None
try:
update = json.loads(raw)
except ValueError:
return None
if update.get('hold_until', 0) <= time.time():
return None
return update.get('label') or 'a UniFi update'
def _open_wan_tickets(self, items: List[dict]) -> None:
"""Comment on the incident's ticket if one is recent, else open one ticket for all of *items*."""
# Most urgent first; at equal priority an ongoing problem leads over a past event
items = sorted(items, key=lambda i: (int(i['priority']), not i['recovers']))
incident = db.get_incident_ticket(self.modem_name, WAN_INCIDENT_WINDOW_S)
if incident:
left = []
for item in items:
text = ticket_text.incident_comment(item['name'], item['summary'], item['checks'], _now_utc())
if self.tickets.comment(incident, text):
db.set_ticket_id(item['event_id'], incident)
else:
left.append(item) # comments failing must not lose the report
items = left
if not items:
return
first, others = items[0], items[1:]
sections = []
if first['recovers']:
sections = self._ticket_sections(first['event_id'], first['event_type'], self.modem_name, first['detail'])
if others:
sections.append(('Also seen in this incident', [f'- **{o["name"]}:** {o["summary"]}' for o in others]))
desc = ticket_text.description(first['heading'], first['facts'], first['summary'], first['checks'],
sections, recovers=any(i['recovers'] for i in items))
tid = self.tickets.create(first['title'], desc, priority=first['priority'])
if tid:
for item in items:
db.set_ticket_id(item['event_id'], tid)
def _note_unifi_update(self, event: dict) -> None:
"""Hold new modem/WAN tickets for a while and explain the ones opened just before."""
label = modem_api.update_label(event)
when = datetime.fromtimestamp(event['ts'], timezone.utc)
logger.info(f'UniFi update finished: {label} at {when:%Y-%m-%d %H:%M:%S} UTC')
if event['ts'] + UPDATE_HOLD_S > time.time():
db.set_state(UNIFI_UPDATE_STATE, {'label': label, 'at': event['ts'],
'hold_until': event['ts'] + UPDATE_HOLD_S})
naive = when.replace(tzinfo=None)
tickets = db.get_ticket_ids_between(
self.modem_name, naive - timedelta(seconds=UPDATE_LOOKBACK_S), naive + timedelta(seconds=UPDATE_HOLD_S))
for tid in tickets:
self.tickets.comment(tid, ticket_text.update_comment(
label, f'{when:%Y-%m-%d %H:%M} UTC', UPDATE_LOOKBACK_S // 60, UPDATE_HOLD_S // 60))
def _process_modem_failure(self, suppressions: list, error: Exception) -> None:
"""Handle a failed poll. Link and optical events stay as they are: we can't see them."""
@@ -1690,14 +1793,13 @@ class NetworkMonitor:
name = self.modem_name
event_id, _, _, ticket_id = db.upsert_event(event_type, 'warning', 'unifi', name, '', description)
if not db.check_suppressed(suppressions, 'host', name) and not ticket_id:
tid = self.tickets.create(
f'[{name}][auto][production][issue][network][cluster-wide] {heading}',
ticket_text.description('WAN Event', [('Modem', name), ('Reported', _now_utc())],
description, [advice], recovers=False),
priority=priority,
)
if tid:
db.set_ticket_id(event_id, tid)
self._queue_wan_ticket({
'event_id': event_id, 'event_type': event_type, 'detail': '',
'title': f'[{name}][auto][production][issue][network][cluster-wide] {heading}',
'name': heading, 'heading': 'WAN Event',
'facts': [('Modem', name), ('Reported', _now_utc())],
'summary': description, 'checks': [advice], 'priority': priority, 'recovers': False,
})
db.resolve_event(event_type, name, '')
def _process_wan_log(self, suppressions: list) -> None:
@@ -1713,12 +1815,21 @@ class NetworkMonitor:
return
# After long monitor downtime, don't open tickets for stale events.
since_ms = max(int(raw_cursor) + 1, int((now - 86400) * 1000))
events = self.unifi.get_system_events(since_ms, modem_api.WAN_LOG_KEYS)
events = self.unifi.get_system_events(since_ms, modem_api.wanted_log_key)
if not events:
return # None = log unavailable, so the cursor stays put and it's retried
# Updates first: one in this batch explains (and holds) the WAN events beside it
updates = [e for e in events if modem_api.is_update_key(e['key'])]
for e in sorted(updates, key=lambda ev: ev['ts']):
try:
self._note_unifi_update(e)
except Exception as ex:
logger.error(f'Noting UniFi update failed: {ex}', exc_info=True)
history = self._load_wan_history()
actionable = modem_api.apply_wan_events(history, events)
actionable = modem_api.apply_wan_events(
history, [e for e in events if not modem_api.is_update_key(e['key'])])
# UniFi repeats "failed multiple times" per failure, so ticket it once a day at most.
unstable = [e for e in actionable if e['key'] == modem_api.LOG_WAN_UNSTABLE]
@@ -1927,9 +2038,11 @@ class NetworkMonitor:
# WAN modem: isolated so a failure here can't stop the rest of the cycle
if self.modem:
try:
# Log first, so a restart it reports can explain an uptime reset below
self._process_wan_log(suppressions)
self._collect_modem(suppressions)
# Log first, so a restart it reports can explain an uptime reset below.
# Tickets open when the batch ends, so one incident gets one ticket.
with self._wan_batch():
self._process_wan_log(suppressions)
self._collect_modem(suppressions)
except Exception as e:
logger.error(f'Modem collection failed: {e}', exc_info=True)