WAN tickets: one per incident, notice UniFi updates, maintenance button
Lint / JS (eslint) (push) Successful in 42s
Lint / Python (flake8) (push) Successful in 1m35s
Lint / Notify on failure (push) Skipped
Lint / Deploy (push) Skipped
Security / Python Security (bandit) (push) Successful in 1m3s
Test / Python Tests (pytest) (push) Successful in 1m5s
Lint / Python (flake8) (pull_request) Successful in 58s
Lint / JS (eslint) (pull_request) Successful in 8s
Lint / Notify on failure (pull_request) Skipped
Lint / Deploy (pull_request) Skipped
Security / Python Security (bandit) (pull_request) Successful in 1m1s
Test / Python Tests (pytest) (pull_request) Successful in 1m10s
Lint / JS (eslint) (push) Successful in 42s
Lint / Python (flake8) (push) Successful in 1m35s
Lint / Notify on failure (push) Skipped
Lint / Deploy (push) Skipped
Security / Python Security (bandit) (push) Successful in 1m3s
Test / Python Tests (pytest) (push) Successful in 1m5s
Lint / Python (flake8) (pull_request) Successful in 58s
Lint / JS (eslint) (pull_request) Successful in 8s
Lint / Notify on failure (pull_request) Skipped
Lint / Deploy (pull_request) Skipped
Security / Python Security (bandit) (pull_request) Successful in 1m1s
Test / Python Tests (pytest) (pull_request) Successful in 1m10s
A UniFi Network update on 2026-11-12 dropped the UDM's WAN, Smart Power power-cycled the modem, and Gandalf opened two tickets for the one event. - Modem and WAN reports from one check cycle open a single ticket, led by the most urgent, with the rest listed under "Also seen in this incident". A report while the modem has an open alert, or one that ended under 30 minutes ago, becomes a comment on that ticket instead. - UniFi update log entries (NETWORK_UPDATED, FIRMWARE_UPDATED_V2 and the like) add a note to modem/WAN tickets from 20 minutes before the update to 15 after, and hold new modem/WAN tickets for 15 minutes. - A Maintenance button for admins starts a global suppression for 30 min to 4 hours, with a banner on every page and End now. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
+138
-25
@@ -13,7 +13,8 @@ import re
|
||||
import shlex
|
||||
import statistics
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
from contextlib import contextmanager
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Dict, List, Optional
|
||||
from urllib.parse import urljoin, urlparse
|
||||
|
||||
@@ -192,8 +193,12 @@ class UnifiClient:
|
||||
logger.error(f'UniFi client list error: {e}')
|
||||
return []
|
||||
|
||||
def get_system_events(self, since_ms: int, keys: tuple) -> Optional[List[dict]]:
|
||||
"""Return system log entries with the given keys, or None if the log is unavailable."""
|
||||
def get_system_events(self, since_ms: int, keys) -> Optional[List[dict]]:
|
||||
"""Return system log entries with the given keys, or None if the log is unavailable.
|
||||
|
||||
keys: a tuple of log keys, or a function that takes a key and says whether to keep it.
|
||||
"""
|
||||
wanted = keys if callable(keys) else keys.__contains__
|
||||
url = f'{self.base_url}/proxy/network/v2/api/site/{self.site_id}/system-log/all'
|
||||
found: List[dict] = []
|
||||
try:
|
||||
@@ -210,19 +215,29 @@ class UnifiClient:
|
||||
if not items:
|
||||
break
|
||||
for e in items:
|
||||
if e.get('key') in keys and e.get('timestamp', 0) >= since_ms:
|
||||
if wanted(e.get('key') or '') and e.get('timestamp', 0) >= since_ms:
|
||||
params = e.get('parameters') or {}
|
||||
found.append({
|
||||
'key': e['key'], 'ts': e['timestamp'] / 1000,
|
||||
'device': (params.get('DEVICE') or {}).get('name', ''),
|
||||
'wan': (params.get('WAN_ID') or {}).get('name', '')
|
||||
or str((params.get('WAN_ID') or {}).get('id', '')),
|
||||
**self._update_fields(params),
|
||||
})
|
||||
return found
|
||||
except Exception as e:
|
||||
logger.error(f'UniFi system log error: {e}')
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _update_fields(params: dict) -> dict:
|
||||
"""The app and versions of an update entry; nothing for other entries."""
|
||||
def text(key):
|
||||
value = params.get(key)
|
||||
return str(value.get('name') or value.get('id') or '') if isinstance(value, dict) else str(value or '')
|
||||
fields = {'app': text('APPLICATION'), 'version': text('VERSION'), 'previous': text('PREVIOUS_VERSION')}
|
||||
return {k: v for k, v in fields.items() if v}
|
||||
|
||||
def get_devices(self) -> Optional[List[dict]]:
|
||||
"""Return list of UniFi devices, or None if the controller is unreachable."""
|
||||
try:
|
||||
@@ -965,6 +980,13 @@ CLUSTER_NAME = 'proxmox-cluster'
|
||||
DEFAULT_PUBLIC_URL = 'https://gandalf.lotusguild.org'
|
||||
# Reports of something already over: the ticket is the whole story, no recovery note
|
||||
ONE_SHOT_EVENTS = ('modem_power_cycled', 'wan_unstable')
|
||||
# Modem and WAN reports this close together are one incident, so they share one ticket
|
||||
WAN_INCIDENT_WINDOW_S = 1800
|
||||
# UniFi logs an update when it finishes, after any WAN trouble it caused. So tickets opened
|
||||
# this long before the entry get a note, and new modem/WAN tickets wait this long after it.
|
||||
UPDATE_LOOKBACK_S = 1200
|
||||
UPDATE_HOLD_S = 900
|
||||
UNIFI_UPDATE_STATE = 'unifi_update'
|
||||
# Ports a host or device was last seen on, kept so a ticket for a dead host
|
||||
# (whose link, and so its LLDP entry, is gone) can still say where it plugs in
|
||||
KNOWN_UPLINKS_STATE = 'known_uplinks'
|
||||
@@ -1543,14 +1565,95 @@ class NetworkMonitor:
|
||||
event_type, severity, 'modem', name, detail, f'{description} ({_now_utc()})',
|
||||
)
|
||||
if not sup and consec >= self.modem_fail_thresh and not ticket_id:
|
||||
self._open_ticket(
|
||||
event_id, event_type, name, detail,
|
||||
title=f'[{name}][auto][production][issue][network][cluster-wide] {heading}',
|
||||
heading='WAN Modem Alert',
|
||||
facts=[('Modem', f'{name} ({self.modem.base_url})'), ('Detected', _now_utc()),
|
||||
('Consecutive check failures', str(consec))],
|
||||
summary=description, checks=[advice], priority=priority,
|
||||
)
|
||||
self._queue_wan_ticket({
|
||||
'event_id': event_id, 'event_type': event_type, 'detail': detail,
|
||||
'title': f'[{name}][auto][production][issue][network][cluster-wide] {heading}',
|
||||
'name': heading, 'heading': 'WAN Modem Alert',
|
||||
'facts': [('Modem', f'{name} ({self.modem.base_url})'), ('Detected', _now_utc()),
|
||||
('Consecutive check failures', str(consec))],
|
||||
'summary': description, 'checks': [advice], 'priority': priority, 'recovers': True,
|
||||
})
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Modem and WAN tickets: one per incident, held around UniFi updates
|
||||
# ------------------------------------------------------------------
|
||||
@contextmanager
|
||||
def _wan_batch(self):
|
||||
"""Collect the cycle's modem and WAN tickets so related ones are opened together."""
|
||||
self._wan_pending = []
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
pending, self._wan_pending = self._wan_pending, None
|
||||
if pending:
|
||||
self._open_wan_tickets(pending)
|
||||
|
||||
def _queue_wan_ticket(self, item: dict) -> None:
|
||||
held = self._update_hold()
|
||||
if held:
|
||||
logger.info(f'Not opening a ticket for {item["event_type"]} yet: {held} just finished')
|
||||
return
|
||||
if getattr(self, '_wan_pending', None) is not None:
|
||||
self._wan_pending.append(item)
|
||||
else:
|
||||
self._open_wan_tickets([item])
|
||||
|
||||
def _update_hold(self) -> Optional[str]:
|
||||
"""The update that modem and WAN tickets are waiting out, if any."""
|
||||
raw = db.get_state(UNIFI_UPDATE_STATE)
|
||||
if not raw:
|
||||
return None
|
||||
try:
|
||||
update = json.loads(raw)
|
||||
except ValueError:
|
||||
return None
|
||||
if update.get('hold_until', 0) <= time.time():
|
||||
return None
|
||||
return update.get('label') or 'a UniFi update'
|
||||
|
||||
def _open_wan_tickets(self, items: List[dict]) -> None:
|
||||
"""Comment on the incident's ticket if one is recent, else open one ticket for all of *items*."""
|
||||
# Most urgent first; at equal priority an ongoing problem leads over a past event
|
||||
items = sorted(items, key=lambda i: (int(i['priority']), not i['recovers']))
|
||||
incident = db.get_incident_ticket(self.modem_name, WAN_INCIDENT_WINDOW_S)
|
||||
if incident:
|
||||
left = []
|
||||
for item in items:
|
||||
text = ticket_text.incident_comment(item['name'], item['summary'], item['checks'], _now_utc())
|
||||
if self.tickets.comment(incident, text):
|
||||
db.set_ticket_id(item['event_id'], incident)
|
||||
else:
|
||||
left.append(item) # comments failing must not lose the report
|
||||
items = left
|
||||
if not items:
|
||||
return
|
||||
first, others = items[0], items[1:]
|
||||
sections = []
|
||||
if first['recovers']:
|
||||
sections = self._ticket_sections(first['event_id'], first['event_type'], self.modem_name, first['detail'])
|
||||
if others:
|
||||
sections.append(('Also seen in this incident', [f'- **{o["name"]}:** {o["summary"]}' for o in others]))
|
||||
desc = ticket_text.description(first['heading'], first['facts'], first['summary'], first['checks'],
|
||||
sections, recovers=any(i['recovers'] for i in items))
|
||||
tid = self.tickets.create(first['title'], desc, priority=first['priority'])
|
||||
if tid:
|
||||
for item in items:
|
||||
db.set_ticket_id(item['event_id'], tid)
|
||||
|
||||
def _note_unifi_update(self, event: dict) -> None:
|
||||
"""Hold new modem/WAN tickets for a while and explain the ones opened just before."""
|
||||
label = modem_api.update_label(event)
|
||||
when = datetime.fromtimestamp(event['ts'], timezone.utc)
|
||||
logger.info(f'UniFi update finished: {label} at {when:%Y-%m-%d %H:%M:%S} UTC')
|
||||
if event['ts'] + UPDATE_HOLD_S > time.time():
|
||||
db.set_state(UNIFI_UPDATE_STATE, {'label': label, 'at': event['ts'],
|
||||
'hold_until': event['ts'] + UPDATE_HOLD_S})
|
||||
naive = when.replace(tzinfo=None)
|
||||
tickets = db.get_ticket_ids_between(
|
||||
self.modem_name, naive - timedelta(seconds=UPDATE_LOOKBACK_S), naive + timedelta(seconds=UPDATE_HOLD_S))
|
||||
for tid in tickets:
|
||||
self.tickets.comment(tid, ticket_text.update_comment(
|
||||
label, f'{when:%Y-%m-%d %H:%M} UTC', UPDATE_LOOKBACK_S // 60, UPDATE_HOLD_S // 60))
|
||||
|
||||
def _process_modem_failure(self, suppressions: list, error: Exception) -> None:
|
||||
"""Handle a failed poll. Link and optical events stay as they are: we can't see them."""
|
||||
@@ -1690,14 +1793,13 @@ class NetworkMonitor:
|
||||
name = self.modem_name
|
||||
event_id, _, _, ticket_id = db.upsert_event(event_type, 'warning', 'unifi', name, '', description)
|
||||
if not db.check_suppressed(suppressions, 'host', name) and not ticket_id:
|
||||
tid = self.tickets.create(
|
||||
f'[{name}][auto][production][issue][network][cluster-wide] {heading}',
|
||||
ticket_text.description('WAN Event', [('Modem', name), ('Reported', _now_utc())],
|
||||
description, [advice], recovers=False),
|
||||
priority=priority,
|
||||
)
|
||||
if tid:
|
||||
db.set_ticket_id(event_id, tid)
|
||||
self._queue_wan_ticket({
|
||||
'event_id': event_id, 'event_type': event_type, 'detail': '',
|
||||
'title': f'[{name}][auto][production][issue][network][cluster-wide] {heading}',
|
||||
'name': heading, 'heading': 'WAN Event',
|
||||
'facts': [('Modem', name), ('Reported', _now_utc())],
|
||||
'summary': description, 'checks': [advice], 'priority': priority, 'recovers': False,
|
||||
})
|
||||
db.resolve_event(event_type, name, '')
|
||||
|
||||
def _process_wan_log(self, suppressions: list) -> None:
|
||||
@@ -1713,12 +1815,21 @@ class NetworkMonitor:
|
||||
return
|
||||
# After long monitor downtime, don't open tickets for stale events.
|
||||
since_ms = max(int(raw_cursor) + 1, int((now - 86400) * 1000))
|
||||
events = self.unifi.get_system_events(since_ms, modem_api.WAN_LOG_KEYS)
|
||||
events = self.unifi.get_system_events(since_ms, modem_api.wanted_log_key)
|
||||
if not events:
|
||||
return # None = log unavailable, so the cursor stays put and it's retried
|
||||
|
||||
# Updates first: one in this batch explains (and holds) the WAN events beside it
|
||||
updates = [e for e in events if modem_api.is_update_key(e['key'])]
|
||||
for e in sorted(updates, key=lambda ev: ev['ts']):
|
||||
try:
|
||||
self._note_unifi_update(e)
|
||||
except Exception as ex:
|
||||
logger.error(f'Noting UniFi update failed: {ex}', exc_info=True)
|
||||
|
||||
history = self._load_wan_history()
|
||||
actionable = modem_api.apply_wan_events(history, events)
|
||||
actionable = modem_api.apply_wan_events(
|
||||
history, [e for e in events if not modem_api.is_update_key(e['key'])])
|
||||
|
||||
# UniFi repeats "failed multiple times" per failure, so ticket it once a day at most.
|
||||
unstable = [e for e in actionable if e['key'] == modem_api.LOG_WAN_UNSTABLE]
|
||||
@@ -1927,9 +2038,11 @@ class NetworkMonitor:
|
||||
# WAN modem: isolated so a failure here can't stop the rest of the cycle
|
||||
if self.modem:
|
||||
try:
|
||||
# Log first, so a restart it reports can explain an uptime reset below
|
||||
self._process_wan_log(suppressions)
|
||||
self._collect_modem(suppressions)
|
||||
# Log first, so a restart it reports can explain an uptime reset below.
|
||||
# Tickets open when the batch ends, so one incident gets one ticket.
|
||||
with self._wan_batch():
|
||||
self._process_wan_log(suppressions)
|
||||
self._collect_modem(suppressions)
|
||||
except Exception as e:
|
||||
logger.error(f'Modem collection failed: {e}', exc_info=True)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user