Add AT&T modem monitoring and device pages; replace Link Debug
Lint / Python (flake8) (pull_request) Successful in 2m6s
Lint / JS (eslint) (pull_request) Successful in 18s
Lint / Notify on failure (pull_request) Skipped
Lint / Deploy (pull_request) Skipped
Security / Python Security (bandit) (pull_request) Successful in 2m11s
Test / Python Tests (pytest) (pull_request) Successful in 2m20s
Lint / JS (eslint) (push) Successful in 10s
Lint / Python (flake8) (push) Successful in 1m58s
Lint / Notify on failure (push) Skipped
Security / Python Security (bandit) (push) Successful in 1m58s
Lint / Deploy (push) Successful in 4s
Test / Python Tests (pytest) (push) Successful in 58s

Modem and WAN:
- Poll the BGW320 status pages read-only for light levels, PON and WAN
  state, with a WAN / Fiber panel, 7-day Rx trend, and alerts on WAN down,
  optical limits, and Rx drift
- Read WAN outages and Smart Power restarts from the UniFi system log,
  since outages can be over between 5 minute polls

Device pages:
- Add device search to the dashboard and a page per host, switch, router,
  AP, PDU, or modem
- Move all Link Debug data into expandable rows there; /links redirects
- Collect UDM ports and UniFi device details (firmware, load, WAN, outlets)
- Reorder the dashboard

Monitoring fixes:
- Alert when a configured host stops reporting: its series disappear
  rather than reporting down, so a dead host never raised an alert
- Alert on Prometheus outages instead of blanking the dashboard
- Treat a failed Pulse ping as unknown rather than host unreachable
- Stop new switch ports reporting lifetime bytes as one interval's rate
- Run database sessions in UTC: other server time zones shifted displayed
  ages and timed suppression expiries

Also log every avatar 404 reason, add a favicon, and correct the WAN link
labels (1G on Port 9, not 10G SFP+).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit was merged in pull request #4.
This commit is contained in:
jaredandClaude Opus 5.5 committed 2026-11-04 23:40:17 -05:00
1 parent e1716d5228
commit e32499b7f6
25 files changed
+3120 -821

No files matched your search

+542 -30
View File
@@ -11,14 +11,17 @@ import json
import logging
import re
import shlex
import statistics
import time
from datetime import datetime, timezone
from typing import Dict, List, Optional
from urllib.parse import urlparse
import requests
from urllib3.exceptions import InsecureRequestWarning
import db
import modem as modem_api
logging.basicConfig(
@@ -53,7 +56,8 @@ class PrometheusClient:
def __init__(self, url: str):
self.url = url.rstrip('/')
def query(self, promql: str) -> list:
def _fetch(self, promql: str) -> Optional[list]:
"""Return query results, or None if the query itself failed."""
try:
resp = requests.get(
f'{self.url}/api/v1/query',
@@ -64,13 +68,23 @@ class PrometheusClient:
data = resp.json()
if data.get('status') == 'success':
return data['data']['result']
logger.error(f'Prometheus query unsuccessful ({promql!r}): {data}')
except Exception as e:
logger.error(f'Prometheus query failed ({promql!r}): {e}')
return []
return None
def get_interface_states(self) -> Dict[str, Dict[str, bool]]:
"""Return {instance: {device: is_up}} for physical interfaces."""
results = self.query('node_network_up')
def query(self, promql: str) -> list:
return self._fetch(promql) or []
def get_interface_states(self) -> Optional[Dict[str, Dict[str, bool]]]:
"""Return {instance: {device: is_up}} for physical interfaces.
Returns None when Prometheus can't answer or has no series at all, so
callers can tell "we can't see anything" apart from "this host is gone".
"""
results = self._fetch('node_network_up')
if not results:
return None
hosts: Dict[str, Dict[str, bool]] = {}
for r in results:
instance = r['metric'].get('instance', '')
@@ -85,6 +99,10 @@ class PrometheusClient:
# UniFi client
# --------------------------------------------------------------------------
class UnifiClient:
# Devices whose port tables are collected: switches and the gateway/router
_PORT_DEVICE_TYPES = ('usw', 'udm', 'ugw', 'uxg')
_SYSTEM_LOG_MAX_PAGES = 10
def __init__(self, cfg: dict):
self.base_url = cfg['controller']
self.site_id = cfg.get('site_id', 'default')
@@ -97,6 +115,96 @@ class UnifiClient:
'Accept': 'application/json',
}
def get_device_details(self) -> Optional[Dict[str, dict]]:
"""Return firmware, load, WAN, and outlet details per device name.
Uses the v1 stat API, the only one that includes these fields.
"""
try:
url = f'{self.base_url}/proxy/network/api/s/{self.site_id}/stat/device'
resp = self.session.get(url, headers=self.headers, timeout=15)
resp.raise_for_status()
return {
d.get('name') or d.get('mac', 'unknown'): self._device_detail(d)
for d in resp.json().get('data', [])
}
except Exception as e:
logger.error(f'UniFi device detail error: {e}')
return None
@staticmethod
def _device_detail(d: dict) -> dict:
def number(raw) -> Optional[float]:
try:
return float(raw)
except (TypeError, ValueError):
return None
stats = d.get('system-stats') or {}
wan = {}
for key in ('wan1', 'wan2'):
w = d.get(key) or {}
if w:
wan[key] = {
'up': w.get('up'), 'speed_mbps': w.get('speed'),
'latency_ms': w.get('latency'), 'availability': w.get('availability'),
}
return {
'firmware': d.get('version', ''),
'uptime_s': d.get('uptime'),
'cpu_pct': number(stats.get('cpu')),
'mem_pct': number(stats.get('mem')),
'clients': d.get('num_sta'),
'temps': [
{'name': t.get('name', ''), 'value_c': number(t.get('value'))}
for t in d.get('temperatures') or []
],
'wan': wan,
'power_w': number(d.get('outlet_ac_power_consumption')),
'outlets': [
{
'index': o.get('index'),
'name': o.get('name', ''),
'on': o.get('relay_state'),
'auto_power_cycle': o.get('cycle_enabled'),
'power_w': number(o.get('outlet_power')),
'current_a': number(o.get('outlet_current')),
}
for o in d.get('outlet_table') or []
],
}
def get_system_events(self, since_ms: int, keys: tuple) -> Optional[List[dict]]:
"""Return system log entries with the given keys, or None if the log is unavailable."""
url = f'{self.base_url}/proxy/network/v2/api/site/{self.site_id}/system-log/all'
found: List[dict] = []
try:
for page in range(self._SYSTEM_LOG_MAX_PAGES):
resp = self.session.post(
url, headers=self.headers, timeout=20,
json={
'timestampFrom': since_ms, 'timestampTo': int(time.time() * 1000),
'pageNumber': page, 'pageSize': 200,
},
)
resp.raise_for_status()
items = resp.json().get('data') or []
if not items:
break
for e in items:
if e.get('key') in keys and e.get('timestamp', 0) >= since_ms:
params = e.get('parameters') or {}
found.append({
'key': e['key'], 'ts': e['timestamp'] / 1000,
'device': (params.get('DEVICE') or {}).get('name', ''),
'wan': (params.get('WAN_ID') or {}).get('name', '')
or str((params.get('WAN_ID') or {}).get('id', '')),
})
return found
except Exception as e:
logger.error(f'UniFi system log error: {e}')
return None
def get_devices(self) -> Optional[List[dict]]:
"""Return list of UniFi devices, or None if the controller is unreachable."""
try:
@@ -122,7 +230,7 @@ class UnifiClient:
return None
def get_switch_ports(self) -> Optional[Dict[str, dict]]:
"""Return per-port stats for all UniFi switches, keyed by switch name.
"""Return per-port stats for UniFi switches and routers, keyed by device name.
Uses the v1 stat API which includes full port_table data.
Returns {switch_name: {'ip': str, 'model': str, 'ports': {port_name: {...}}}}.
@@ -134,7 +242,7 @@ class UnifiClient:
devices = resp.json().get('data', [])
result: Dict[str, dict] = {}
for dev in devices:
if dev.get('type', '').lower() != 'usw':
if dev.get('type', '').lower() not in self._PORT_DEVICE_TYPES:
continue
sw_name = dev.get('name') or dev.get('mac', 'unknown')
sw_ip = dev.get('ip', '')
@@ -321,13 +429,19 @@ class PulseClient:
return self.run_command(command, _retry=False)
return None
def ping(self, ip: str, count: int = 3, timeout: int = 2) -> bool:
"""Ping *ip* via the Pulse worker. Returns True if host responds."""
def ping(self, ip: str, count: int = 3, timeout: int = 2) -> Optional[bool]:
"""Ping *ip* via the Pulse worker.
Returns True/False for the host's reachability, or None if Pulse itself
couldn't run the check, which says nothing about the host.
"""
ip_q = shlex.quote(ip)
output = self.run_command(
f'ping -c {count} -W {timeout} {ip_q} >/dev/null 2>&1 && echo REACHABLE || echo UNREACHABLE'
)
return output is not None and output.strip() == 'REACHABLE'
if output is None:
return None
return output.strip() == 'REACHABLE'
# --------------------------------------------------------------------------
@@ -615,9 +729,12 @@ class LinkStatsCollector:
except Exception as e:
logger.warning(f'UniFi switch port collection failed: {e}')
unifi_devices = (self.unifi.get_device_details() or {}) if self.unifi else {}
return {
'hosts': result_hosts,
'unifi_switches': unifi_switches,
'unifi_devices': unifi_devices,
'updated': datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M:%S UTC'),
}
@@ -625,8 +742,10 @@ class LinkStatsCollector:
"""Compute per-port byte/error rates from delta against previous snapshot."""
dt = now - self._prev_unifi_time if self._prev_unifi_time > 0 else 0
def rate(new_val: int, old_val: int) -> Optional[float]:
if dt <= 0:
def rate(new_val: int, old_val: Optional[int]) -> Optional[float]:
# No earlier reading (first poll, or a new/renamed port) means no rate,
# not "all bytes since boot arrived in this one interval".
if dt <= 0 or old_val is None:
return None
return max(0.0, (new_val - old_val) / dt)
@@ -637,12 +756,12 @@ class LinkStatsCollector:
for pname, d in sw_data['ports'].items():
entry = dict(d)
prev = prev_ports.get(pname, {})
entry['tx_bytes_rate'] = rate(d['tx_bytes'], prev.get('tx_bytes', 0))
entry['rx_bytes_rate'] = rate(d['rx_bytes'], prev.get('rx_bytes', 0))
entry['tx_errs_rate'] = rate(d['tx_errors'], prev.get('tx_errors', 0))
entry['rx_errs_rate'] = rate(d['rx_errors'], prev.get('rx_errors', 0))
entry['tx_drops_rate'] = rate(d['tx_dropped'], prev.get('tx_dropped', 0))
entry['rx_drops_rate'] = rate(d['rx_dropped'], prev.get('rx_dropped', 0))
entry['tx_bytes_rate'] = rate(d['tx_bytes'], prev.get('tx_bytes'))
entry['rx_bytes_rate'] = rate(d['rx_bytes'], prev.get('rx_bytes'))
entry['tx_errs_rate'] = rate(d['tx_errors'], prev.get('tx_errors'))
entry['rx_errs_rate'] = rate(d['rx_errors'], prev.get('rx_errors'))
entry['tx_drops_rate'] = rate(d['tx_dropped'], prev.get('tx_dropped'))
entry['rx_drops_rate'] = rate(d['rx_dropped'], prev.get('rx_dropped'))
merged_ports[pname] = entry
result[sw_name] = {'ip': sw_data['ip'], 'model': sw_data['model'],
'ports': merged_ports}
@@ -685,6 +804,27 @@ class NetworkMonitor:
self.cluster_thresh = mon.get('cluster_threshold', 3)
self.cluster_name = mon.get('cluster_name', CLUSTER_NAME)
# Optional WAN modem (AT&T BGW-series); see modem.py
modem_cfg = self.cfg.get('modem') or {}
self.modem = modem_api.ModemClient(modem_cfg) if modem_cfg.get('url') else None
self.modem_name = modem_cfg.get('name', 'att-modem')
# Modem reboots (including AT&T's automatic firmware updates) take a few
# minutes, so this can be raised independently of failure_threshold.
self.modem_fail_thresh = modem_cfg.get('failure_threshold', self.fail_thresh)
self.rx_drift_db = modem_cfg.get('rx_drift_db', 3.0)
self.rx_drift_min_samples = modem_cfg.get('rx_drift_min_samples', 288)
self.modem_lan_ip = urlparse(modem_cfg['url']).hostname if modem_cfg.get('url') else ''
self._prev_modem_counters: Optional[dict] = None
self._prev_modem_time = 0.0
if self.modem:
try:
db.ensure_modem_schema()
except Exception as e:
logger.error(f'Could not create modem history table: {e}')
# Last good Prometheus result, shown (as unknown) during a Prometheus outage
self._last_iface_states: Dict[str, Dict[str, bool]] = {}
# Build Prometheus instance → hostname lookup
self._instance_map: Dict[str, str] = {
h['prometheus_instance']: h['name']
@@ -744,6 +884,22 @@ class NetworkMonitor:
if host_has_regression:
hosts_with_regression.append(host)
# A dead host doesn't report its interfaces as down: its series just
# disappear, so a configured host that's absent needs its own check.
for instance, host in self._instance_map.items():
if instance in states:
db.resolve_event('host_metrics_missing', host, instance)
continue
hosts_with_regression.append(host)
sup = db.check_suppressed(suppressions, 'host', host)
event_id, is_new, consec, ticket_id = db.upsert_event(
'host_metrics_missing', 'critical', 'prometheus',
host, instance,
f'No metrics from {host} ({instance}): host or node_exporter is down ({_now_utc()})',
)
if not sup and consec >= self.fail_thresh and not ticket_id:
self._ticket_metrics_missing(event_id, host, instance, consec)
db.set_baseline(new_baseline)
# Cluster-wide check – only genuine regressions count
@@ -752,7 +908,7 @@ class NetworkMonitor:
event_id, is_new, consec, ticket_id = db.upsert_event(
'cluster_network_issue', 'critical', 'prometheus',
self.cluster_name, '',
f'{len(hosts_with_regression)} hosts reporting simultaneous interface failures: '
f'{len(hosts_with_regression)} hosts with simultaneous interface or metrics failures: '
f'{", ".join(hosts_with_regression)}',
)
if not sup and (is_new or not ticket_id):
@@ -765,7 +921,8 @@ class NetworkMonitor:
f'Affected hosts: {", ".join(hosts_with_regression)}\n'
f'Detected: {_now_utc()}\n\n'
f'{len(hosts_with_regression)} Proxmox hosts simultaneously reported '
f'interface regressions (link-down on interfaces previously known UP).\n'
f'interface regressions (link-down on interfaces previously known UP) '
f'or stopped reporting metrics.\n'
f'This likely indicates a switch or upstream network failure.\n\n'
f'Please check the core and management switches immediately.'
)
@@ -775,6 +932,54 @@ class NetworkMonitor:
else:
db.resolve_event('cluster_network_issue', self.cluster_name, '')
def _ticket_metrics_missing(
self, event_id: int, host: str, instance: str, consec: int
) -> None:
title = (
f'[{host}][auto][production][issue][network][single-node] '
f'No metrics from host'
)
desc = (
f'Host Metrics Alert\n{"=" * 40}\n\n'
f'Host: {host}\n'
f'Prometheus instance: {instance}\n'
f'Detected: {_now_utc()}\n'
f'Consecutive check failures: {consec}\n\n'
f'Prometheus is working but has no node_network_up series for {host}.\n'
f'Either the host is down or node_exporter has stopped on it.\n\n'
f'Please check the host power and network connectivity, then '
f'prometheus-node-exporter on the host.'
)
tid = self.tickets.create(title, desc, priority='2')
if tid:
db.set_ticket_id(event_id, tid)
def _process_prometheus_outage(self, suppressions: list) -> None:
"""Interface and host alerts can't fire without Prometheus, so say so."""
sup = db.check_suppressed(suppressions, 'host', 'prometheus')
event_id, is_new, consec, ticket_id = db.upsert_event(
'monitor_source_unavailable', 'critical', 'prometheus',
'prometheus', '',
f'Prometheus returned no interface data; host and interface alerting is paused ({_now_utc()})',
)
if not sup and consec >= self.fail_thresh and not ticket_id:
title = (
'[prometheus][auto][production][issue][network][cluster-wide] '
'Gandalf has no Prometheus data'
)
desc = (
f'Monitoring Source Alert\n{"=" * 40}\n\n'
f'Prometheus URL: {self.prom.url}\n'
f'Detected: {_now_utc()}\n'
f'Consecutive check failures: {consec}\n\n'
f'Gandalf could not get node_network_up data from Prometheus.\n'
f'While this lasts, host and interface failures will NOT be detected.\n\n'
f'Please check the Prometheus service and its scrape targets.'
)
tid = self.tickets.create(title, desc, priority='2')
if tid:
db.set_ticket_id(event_id, tid)
def _ticket_interface(
self, event_id: int, host: str, iface: str, consec: int
) -> None:
@@ -842,10 +1047,15 @@ class NetworkMonitor:
# ------------------------------------------------------------------
# Ping-only hosts (no node_exporter)
# ------------------------------------------------------------------
def _process_ping_hosts(self, suppressions: list, ping_states: Dict[str, bool]) -> None:
def _process_ping_hosts(self, suppressions: list, ping_states: Dict[str, Optional[bool]]) -> None:
for h in self.cfg.get('monitor', {}).get('ping_hosts', []):
name, ip = h['name'], h['ip']
reachable = ping_states.get(name, False)
reachable = ping_states.get(name)
if reachable is None:
# Pulse couldn't run the check. Neither alert nor resolve on no evidence.
logger.warning(f'Ping check for {name} ({ip}) unavailable this cycle')
continue
if not reachable:
sup = db.check_suppressed(suppressions, 'host', name)
@@ -881,26 +1091,301 @@ class NetworkMonitor:
if tid:
db.set_ticket_id(event_id, tid)
# ------------------------------------------------------------------
# WAN modem monitoring
# ------------------------------------------------------------------
_MODEM_METRIC_LABELS = {
'rx_power': 'Rx power', 'tx_power': 'Tx power', 'temperature': 'Temperature',
'vcc': 'Supply voltage', 'tx_bias': 'Tx bias',
}
_MODEM_METRIC_UNITS = {'rx_power': ' dBm', 'tx_power': ' dBm', 'temperature': ' C'}
def _modem_event(
self, suppressions: list, event_type: str, severity: str, detail: str,
description: str, heading: str, priority: str, advice: str,
) -> None:
name = self.modem_name
sup = db.check_suppressed(suppressions, 'host', name)
event_id, is_new, consec, ticket_id = db.upsert_event(
event_type, severity, 'modem', name, detail, f'{description} ({_now_utc()})',
)
if not sup and consec >= self.modem_fail_thresh and not ticket_id:
title = f'[{name}][auto][production][issue][network][cluster-wide] {heading}'
desc = (
f'WAN Modem Alert\n{"=" * 40}\n\n'
f'Modem: {name} ({self.modem.base_url})\n'
f'Detected: {_now_utc()}\n'
f'Consecutive check failures: {consec}\n\n'
f'{description}\n\n{advice}'
)
tid = self.tickets.create(title, desc, priority=priority)
if tid:
db.set_ticket_id(event_id, tid)
def _process_modem_failure(self, suppressions: list, error: Exception) -> None:
"""Handle a failed poll. Link and optical events stay as they are: we can't see them."""
name = self.modem_name
if isinstance(error, modem_api.ModemUnreachable):
db.resolve_event('modem_data_invalid', name, '')
self._modem_event(
suppressions, 'modem_unreachable', 'critical', '',
f'WAN modem is not responding: {error}',
'WAN modem unreachable', '2',
'Internet access is probably affected too. Check modem power and the '
'cable between the modem and the UDM-Pro.',
)
else:
db.resolve_event('modem_unreachable', name, '')
self._modem_event(
suppressions, 'modem_data_invalid', 'warning', '',
f'WAN modem status could not be read: {error}',
'WAN modem status unreadable', '3',
'The modem answered but its pages could not be parsed, so fiber alerting is '
'paused. A firmware update may have changed them; see modem.py.',
)
def _process_modem(self, data: dict, suppressions: list) -> None:
"""Raise and resolve alerts from a successful modem poll."""
name = self.modem_name
db.resolve_event('modem_unreachable', name, '')
db.resolve_event('modem_data_invalid', name, '')
problems = modem_api.wan_problems(data)
if problems:
self._modem_event(
suppressions, 'wan_down', 'critical', '',
f'WAN link is down: {"; ".join(problems)}.',
'WAN link down', '1',
'Check the fiber at the modem and the AT&T service status. Rx power at or below '
'the alarm limit points to a fiber fault rather than a provider outage.',
)
else:
db.resolve_event('wan_down', name, '')
for key, metric in data.get('optics', {}).items():
status = modem_api.metric_status(metric)
if status == 'alarm' and problems and key == 'rx_power':
# wan_down already covers this, and a cut fiber would otherwise open two tickets.
status = 'ok'
label = self._MODEM_METRIC_LABELS.get(key, key)
value = metric.get('value')
reading = f' at {value}{self._MODEM_METRIC_UNITS.get(key, "")}' if value is not None else ''
limits = (
f' (warning {metric.get("warn_low")} to {metric.get("warn_high")}, '
f'alarm {metric.get("alarm_low")} to {metric.get("alarm_high")})'
) if value is not None else ''
advice = (
'Inspect and clean the fiber connector at the modem and look for tight bends. '
'If that does not help, contact AT&T: the reading is what the modem measures '
'on the provider side of the connection.'
)
if status == 'alarm':
db.resolve_event('fiber_signal_warning', name, key)
self._modem_event(
suppressions, 'fiber_signal_alarm', 'critical', key,
f'{label} is in the alarm range{reading}{limits}.',
f'Fiber {label} alarm', '2', advice,
)
elif status == 'warning':
db.resolve_event('fiber_signal_alarm', name, key)
self._modem_event(
suppressions, 'fiber_signal_warning', 'warning', key,
f'{label} is in the warning range{reading}{limits}.',
f'Fiber {label} warning', '3', advice,
)
else:
db.resolve_event('fiber_signal_alarm', name, key)
db.resolve_event('fiber_signal_warning', name, key)
self._check_rx_drift(data, suppressions)
def _check_rx_drift(self, data: dict, suppressions: list) -> None:
"""Alert on a sustained Rx drop relative to this link's own recent normal.
The modem's limits are very wide (warning at -27.9 dBm on a link that sits
around -17), so a damaged connector could lose many dB before they trip.
"""
name = self.modem_name
rx = (data.get('optics', {}).get('rx_power') or {}).get('value')
if rx is None:
return
history = db.get_modem_rx_values(days=7)
if len(history) < self.rx_drift_min_samples:
return # not enough history yet to know what normal looks like
baseline = statistics.median(history)
drop = round(baseline - rx, 1)
if drop >= self.rx_drift_db:
self._modem_event(
suppressions, 'fiber_rx_drift', 'warning', 'rx_power',
f'Rx power is {rx} dBm, {drop} dB below its 7-day median of {baseline:.1f} dBm.',
'Fiber Rx power dropped', '3',
'The signal is still in range but has changed. Look for a recently disturbed '
'or bent fiber and check the connector at the modem.',
)
else:
db.resolve_event('fiber_rx_drift', name, 'rx_power')
def _load_wan_history(self) -> dict:
raw = db.get_state('wan_history')
if raw:
try:
return json.loads(raw)
except ValueError:
logger.error('Failed to parse wan_history JSON; starting over')
return {'outages': [], 'restarts': []}
def _note_modem_restart(self, data: dict) -> None:
uptime = data.get('uptime_s')
if uptime is None:
return
previous = None
raw = db.get_state('modem_stats')
if raw:
try:
previous = json.loads(raw).get('uptime_s')
except ValueError:
logger.error('Failed to parse modem_stats JSON')
if previous is not None and uptime < previous:
logger.warning(f'Modem restarted: uptime went from {previous}s to {uptime}s')
history = self._load_wan_history()
modem_api.record_uptime_reset(history, time.time() - uptime)
db.set_state('wan_history', history)
def _one_shot_ticket(
self, suppressions: list, event_type: str, description: str,
heading: str, priority: str, advice: str,
) -> None:
"""Report something that has already happened: open a ticket and resolve the event."""
name = self.modem_name
event_id, _, _, ticket_id = db.upsert_event(event_type, 'warning', 'unifi', name, '', description)
if not db.check_suppressed(suppressions, 'host', name) and not ticket_id:
tid = self.tickets.create(
f'[{name}][auto][production][issue][network][cluster-wide] {heading}',
f'WAN Event\n{"=" * 40}\n\nModem: {name}\nReported: {_now_utc()}\n\n{description}\n\n{advice}',
priority=priority,
)
if tid:
db.set_ticket_id(event_id, tid)
db.resolve_event(event_type, name, '')
def _process_wan_log(self, suppressions: list) -> None:
"""Read WAN outages and modem restarts from the UniFi log.
These can be over in two minutes, so a 5 minute poll of the modem may never see them.
"""
now = time.time()
raw_cursor = db.get_state('unifi_log_cursor')
if raw_cursor is None:
# First run: start from now rather than ticketing old history.
db.set_state('unifi_log_cursor', str(int(now * 1000)))
return
# After long monitor downtime, don't open tickets for stale events.
since_ms = max(int(raw_cursor) + 1, int((now - 86400) * 1000))
events = self.unifi.get_system_events(since_ms, modem_api.WAN_LOG_KEYS)
if not events:
return # None = log unavailable, so the cursor stays put and it's retried
history = self._load_wan_history()
actionable = modem_api.apply_wan_events(history, events)
# UniFi repeats "failed multiple times" per failure, so ticket it once a day at most.
unstable = [e for e in actionable if e['key'] == modem_api.LOG_WAN_UNSTABLE]
ticket_unstable = bool(unstable) and now - history.get('unstable_ticketed_at', 0) >= 86400
if ticket_unstable:
history['unstable_ticketed_at'] = now
# Save first: a failed ticket call shouldn't replay these events next cycle.
db.set_state('wan_history', history)
db.set_state('unifi_log_cursor', str(int(max(e['ts'] for e in events) * 1000)))
for e in actionable:
if e['key'] == modem_api.LOG_MODEM_RESTARTED:
when = datetime.fromtimestamp(e['ts'], timezone.utc).strftime('%Y-%m-%d %H:%M:%S UTC')
self._one_shot_ticket(
suppressions, 'modem_power_cycled',
f'{e.get("device") or "Smart Power"} power-cycled the modem at {when} '
f'to try to restore the internet connection.',
'Modem auto power-cycled after internet loss', '3',
'The internet connection failed badly enough to trigger auto power-cycle. '
'Check the WAN / Fiber panel for outages around this time and Rx power for a drop.',
)
if ticket_unstable:
self._one_shot_ticket(
suppressions, 'wan_unstable',
'UniFi reports the WAN connection went down multiple times in the last 24 hours.',
'WAN connection unstable', '2',
'See the WAN history on the modem page. Repeated drops with healthy Rx power '
'point toward the provider or the modem rather than the fiber.',
)
def _collect_modem(self, suppressions: list) -> None:
"""Poll the modem, then store its status and raise any alerts."""
try:
data = self.modem.poll()
except (modem_api.ModemUnreachable, modem_api.ModemDataInvalid) as e:
logger.warning(f'Modem poll failed: {e}')
self._process_modem_failure(suppressions, e)
db.set_state('modem_stats', {
'reachable': False, 'error': str(e), 'name': self.modem_name,
'lan_ip': self.modem_lan_ip, 'updated': _now_utc(),
})
return
# Uptime going backwards between polls means the gateway restarted, even if
# it came back too quickly for any poll to fail. Read before it's overwritten.
self._note_modem_restart(data)
now = time.monotonic()
data['rates'] = modem_api.counter_rates(
data['counters'], self._prev_modem_counters, now - self._prev_modem_time,
)
self._prev_modem_counters = data['counters']
self._prev_modem_time = now
data['reachable'] = True
data['name'] = self.modem_name
data['lan_ip'] = self.modem_lan_ip
data['wan_problems'] = modem_api.wan_problems(data)
data['optics_status'] = {k: modem_api.metric_status(m) for k, m in data['optics'].items()}
db.set_state('modem_stats', data)
# Alert before recording, so a sudden drop isn't compared against itself.
self._process_modem(data, suppressions)
optics = data['optics']
db.record_modem_sample(
(optics.get('rx_power') or {}).get('value'),
(optics.get('tx_power') or {}).get('value'),
(optics.get('temperature') or {}).get('value'),
)
# ------------------------------------------------------------------
# Snapshot collection (for dashboard)
# ------------------------------------------------------------------
def _collect_snapshot(
self, iface_states: Dict[str, Dict[str, bool]],
self, iface_states: Optional[Dict[str, Dict[str, bool]]],
unifi_devices: Optional[List[dict]] = None,
ping_states: Optional[Dict[str, bool]] = None,
ping_states: Optional[Dict[str, Optional[bool]]] = None,
) -> dict:
# Accept pre-fetched devices; fall back to empty list if unavailable
display_unifi = unifi_devices if unifi_devices is not None else []
# During a Prometheus outage keep showing the hosts, with their state unknown,
# rather than emptying the dashboard or implying that they're down.
prom_ok = iface_states is not None
shown_states = iface_states if prom_ok else self._last_iface_states
hosts = {}
for instance, ifaces in iface_states.items():
for instance, ifaces in shown_states.items():
if instance not in self._instance_map:
continue # skip Prometheus instances not in config (e.g. LXC app servers)
host = self._hostname(instance)
phys = {k: v for k, v in ifaces.items()}
up_count = sum(1 for v in phys.values() if v)
total = len(phys)
if total == 0 or up_count == total:
if not prom_ok:
status = 'unknown'
elif total == 0 or up_count == total:
status = 'up'
elif up_count == 0:
status = 'down'
@@ -914,13 +1399,23 @@ class NetworkMonitor:
'source': 'prometheus',
}
# Configured hosts with no data: down if Prometheus is fine, otherwise unknown
for instance, host in self._instance_map.items():
if instance not in shown_states:
hosts[host] = {
'ip': instance.split(':')[0],
'interfaces': {},
'status': 'down' if prom_ok else 'unknown',
'source': 'prometheus',
}
for h in self.cfg.get('monitor', {}).get('ping_hosts', []):
name, ip = h['name'], h['ip']
reachable = (ping_states or {}).get(name, False)
reachable = (ping_states or {}).get(name)
hosts[name] = {
'ip': ip,
'interfaces': {},
'status': 'up' if reachable else 'down',
'status': 'unknown' if reachable is None else ('up' if reachable else 'down'),
'source': 'ping',
}
@@ -949,13 +1444,15 @@ class NetworkMonitor:
unifi_devices = self.unifi.get_devices()
# 3a. Ping-only hosts once — shared by snapshot and alert processing
ping_states: Dict[str, bool] = {
ping_states: Dict[str, Optional[bool]] = {
h['name']: self.pulse.ping(h['ip'])
for h in self.cfg.get('monitor', {}).get('ping_hosts', [])
}
# 3b. Collect and store snapshot for dashboard
snapshot = self._collect_snapshot(iface_states, unifi_devices, ping_states)
if iface_states is not None:
self._last_iface_states = iface_states
db.set_state('network_snapshot', snapshot)
db.set_state('last_check', _now_utc())
@@ -968,14 +1465,29 @@ class NetworkMonitor:
# 5. Process alerts using already-fetched data
suppressions = db.get_active_suppressions()
self._process_interfaces(iface_states, suppressions)
if iface_states is None:
self._process_prometheus_outage(suppressions)
else:
db.resolve_event('monitor_source_unavailable', 'prometheus', '')
self._process_interfaces(iface_states, suppressions)
self._process_unifi(unifi_devices, suppressions)
self._process_ping_hosts(suppressions, ping_states)
# WAN modem: isolated so a failure here can't stop the rest of the cycle
if self.modem:
try:
# Log first, so a restart it reports can explain an uptime reset below
self._process_wan_log(suppressions)
self._collect_modem(suppressions)
except Exception as e:
logger.error(f'Modem collection failed: {e}', exc_info=True)
# Housekeeping: deactivate expired suppressions and purge old resolved events
db.cleanup_expired_suppressions()
db.purge_old_resolved_events(days=90)
if self.modem:
db.purge_old_modem_samples(days=90)
logger.info('Network check cycle complete')