Add AT&T modem monitoring and device pages; replace Link Debug
Lint / Python (flake8) (pull_request) Successful in 2m6s
Lint / JS (eslint) (pull_request) Successful in 18s
Lint / Notify on failure (pull_request) Skipped
Lint / Deploy (pull_request) Skipped
Security / Python Security (bandit) (pull_request) Successful in 2m11s
Test / Python Tests (pytest) (pull_request) Successful in 2m20s
Lint / JS (eslint) (push) Successful in 10s
Lint / Python (flake8) (push) Successful in 1m58s
Lint / Notify on failure (push) Skipped
Security / Python Security (bandit) (push) Successful in 1m58s
Lint / Deploy (push) Successful in 4s
Test / Python Tests (pytest) (push) Successful in 58s
Lint / Python (flake8) (pull_request) Successful in 2m6s
Lint / JS (eslint) (pull_request) Successful in 18s
Lint / Notify on failure (pull_request) Skipped
Lint / Deploy (pull_request) Skipped
Security / Python Security (bandit) (pull_request) Successful in 2m11s
Test / Python Tests (pytest) (pull_request) Successful in 2m20s
Lint / JS (eslint) (push) Successful in 10s
Lint / Python (flake8) (push) Successful in 1m58s
Lint / Notify on failure (push) Skipped
Security / Python Security (bandit) (push) Successful in 1m58s
Lint / Deploy (push) Successful in 4s
Test / Python Tests (pytest) (push) Successful in 58s
Modem and WAN: - Poll the BGW320 status pages read-only for light levels, PON and WAN state, with a WAN / Fiber panel, 7-day Rx trend, and alerts on WAN down, optical limits, and Rx drift - Read WAN outages and Smart Power restarts from the UniFi system log, since outages can be over between 5 minute polls Device pages: - Add device search to the dashboard and a page per host, switch, router, AP, PDU, or modem - Move all Link Debug data into expandable rows there; /links redirects - Collect UDM ports and UniFi device details (firmware, load, WAN, outlets) - Reorder the dashboard Monitoring fixes: - Alert when a configured host stops reporting: its series disappear rather than reporting down, so a dead host never raised an alert - Alert on Prometheus outages instead of blanking the dashboard - Treat a failed Pulse ping as unknown rather than host unreachable - Stop new switch ports reporting lifetime bytes as one interval's rate - Run database sessions in UTC: other server time zones shifted displayed ages and timed suppression expiries Also log every avatar 404 reason, add a favicon, and correct the WAN link labels (1G on Port 9, not 10G SFP+). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit was merged in pull request #4.
This commit is contained in:
1 parent
e1716d5228
commit
e32499b7f6
25 files changed
+3120
-821
No files matched your search
+542
-30
@@ -11,14 +11,17 @@ import json
|
||||
import logging
|
||||
import re
|
||||
import shlex
|
||||
import statistics
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
from typing import Dict, List, Optional
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import requests
|
||||
from urllib3.exceptions import InsecureRequestWarning
|
||||
|
||||
import db
|
||||
import modem as modem_api
|
||||
|
||||
|
||||
logging.basicConfig(
|
||||
@@ -53,7 +56,8 @@ class PrometheusClient:
|
||||
def __init__(self, url: str):
|
||||
self.url = url.rstrip('/')
|
||||
|
||||
def query(self, promql: str) -> list:
|
||||
def _fetch(self, promql: str) -> Optional[list]:
|
||||
"""Return query results, or None if the query itself failed."""
|
||||
try:
|
||||
resp = requests.get(
|
||||
f'{self.url}/api/v1/query',
|
||||
@@ -64,13 +68,23 @@ class PrometheusClient:
|
||||
data = resp.json()
|
||||
if data.get('status') == 'success':
|
||||
return data['data']['result']
|
||||
logger.error(f'Prometheus query unsuccessful ({promql!r}): {data}')
|
||||
except Exception as e:
|
||||
logger.error(f'Prometheus query failed ({promql!r}): {e}')
|
||||
return []
|
||||
return None
|
||||
|
||||
def get_interface_states(self) -> Dict[str, Dict[str, bool]]:
|
||||
"""Return {instance: {device: is_up}} for physical interfaces."""
|
||||
results = self.query('node_network_up')
|
||||
def query(self, promql: str) -> list:
|
||||
return self._fetch(promql) or []
|
||||
|
||||
def get_interface_states(self) -> Optional[Dict[str, Dict[str, bool]]]:
|
||||
"""Return {instance: {device: is_up}} for physical interfaces.
|
||||
|
||||
Returns None when Prometheus can't answer or has no series at all, so
|
||||
callers can tell "we can't see anything" apart from "this host is gone".
|
||||
"""
|
||||
results = self._fetch('node_network_up')
|
||||
if not results:
|
||||
return None
|
||||
hosts: Dict[str, Dict[str, bool]] = {}
|
||||
for r in results:
|
||||
instance = r['metric'].get('instance', '')
|
||||
@@ -85,6 +99,10 @@ class PrometheusClient:
|
||||
# UniFi client
|
||||
# --------------------------------------------------------------------------
|
||||
class UnifiClient:
|
||||
# Devices whose port tables are collected: switches and the gateway/router
|
||||
_PORT_DEVICE_TYPES = ('usw', 'udm', 'ugw', 'uxg')
|
||||
_SYSTEM_LOG_MAX_PAGES = 10
|
||||
|
||||
def __init__(self, cfg: dict):
|
||||
self.base_url = cfg['controller']
|
||||
self.site_id = cfg.get('site_id', 'default')
|
||||
@@ -97,6 +115,96 @@ class UnifiClient:
|
||||
'Accept': 'application/json',
|
||||
}
|
||||
|
||||
def get_device_details(self) -> Optional[Dict[str, dict]]:
|
||||
"""Return firmware, load, WAN, and outlet details per device name.
|
||||
|
||||
Uses the v1 stat API, the only one that includes these fields.
|
||||
"""
|
||||
try:
|
||||
url = f'{self.base_url}/proxy/network/api/s/{self.site_id}/stat/device'
|
||||
resp = self.session.get(url, headers=self.headers, timeout=15)
|
||||
resp.raise_for_status()
|
||||
return {
|
||||
d.get('name') or d.get('mac', 'unknown'): self._device_detail(d)
|
||||
for d in resp.json().get('data', [])
|
||||
}
|
||||
except Exception as e:
|
||||
logger.error(f'UniFi device detail error: {e}')
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _device_detail(d: dict) -> dict:
|
||||
def number(raw) -> Optional[float]:
|
||||
try:
|
||||
return float(raw)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
stats = d.get('system-stats') or {}
|
||||
wan = {}
|
||||
for key in ('wan1', 'wan2'):
|
||||
w = d.get(key) or {}
|
||||
if w:
|
||||
wan[key] = {
|
||||
'up': w.get('up'), 'speed_mbps': w.get('speed'),
|
||||
'latency_ms': w.get('latency'), 'availability': w.get('availability'),
|
||||
}
|
||||
return {
|
||||
'firmware': d.get('version', ''),
|
||||
'uptime_s': d.get('uptime'),
|
||||
'cpu_pct': number(stats.get('cpu')),
|
||||
'mem_pct': number(stats.get('mem')),
|
||||
'clients': d.get('num_sta'),
|
||||
'temps': [
|
||||
{'name': t.get('name', ''), 'value_c': number(t.get('value'))}
|
||||
for t in d.get('temperatures') or []
|
||||
],
|
||||
'wan': wan,
|
||||
'power_w': number(d.get('outlet_ac_power_consumption')),
|
||||
'outlets': [
|
||||
{
|
||||
'index': o.get('index'),
|
||||
'name': o.get('name', ''),
|
||||
'on': o.get('relay_state'),
|
||||
'auto_power_cycle': o.get('cycle_enabled'),
|
||||
'power_w': number(o.get('outlet_power')),
|
||||
'current_a': number(o.get('outlet_current')),
|
||||
}
|
||||
for o in d.get('outlet_table') or []
|
||||
],
|
||||
}
|
||||
|
||||
def get_system_events(self, since_ms: int, keys: tuple) -> Optional[List[dict]]:
|
||||
"""Return system log entries with the given keys, or None if the log is unavailable."""
|
||||
url = f'{self.base_url}/proxy/network/v2/api/site/{self.site_id}/system-log/all'
|
||||
found: List[dict] = []
|
||||
try:
|
||||
for page in range(self._SYSTEM_LOG_MAX_PAGES):
|
||||
resp = self.session.post(
|
||||
url, headers=self.headers, timeout=20,
|
||||
json={
|
||||
'timestampFrom': since_ms, 'timestampTo': int(time.time() * 1000),
|
||||
'pageNumber': page, 'pageSize': 200,
|
||||
},
|
||||
)
|
||||
resp.raise_for_status()
|
||||
items = resp.json().get('data') or []
|
||||
if not items:
|
||||
break
|
||||
for e in items:
|
||||
if e.get('key') in keys and e.get('timestamp', 0) >= since_ms:
|
||||
params = e.get('parameters') or {}
|
||||
found.append({
|
||||
'key': e['key'], 'ts': e['timestamp'] / 1000,
|
||||
'device': (params.get('DEVICE') or {}).get('name', ''),
|
||||
'wan': (params.get('WAN_ID') or {}).get('name', '')
|
||||
or str((params.get('WAN_ID') or {}).get('id', '')),
|
||||
})
|
||||
return found
|
||||
except Exception as e:
|
||||
logger.error(f'UniFi system log error: {e}')
|
||||
return None
|
||||
|
||||
def get_devices(self) -> Optional[List[dict]]:
|
||||
"""Return list of UniFi devices, or None if the controller is unreachable."""
|
||||
try:
|
||||
@@ -122,7 +230,7 @@ class UnifiClient:
|
||||
return None
|
||||
|
||||
def get_switch_ports(self) -> Optional[Dict[str, dict]]:
|
||||
"""Return per-port stats for all UniFi switches, keyed by switch name.
|
||||
"""Return per-port stats for UniFi switches and routers, keyed by device name.
|
||||
|
||||
Uses the v1 stat API which includes full port_table data.
|
||||
Returns {switch_name: {'ip': str, 'model': str, 'ports': {port_name: {...}}}}.
|
||||
@@ -134,7 +242,7 @@ class UnifiClient:
|
||||
devices = resp.json().get('data', [])
|
||||
result: Dict[str, dict] = {}
|
||||
for dev in devices:
|
||||
if dev.get('type', '').lower() != 'usw':
|
||||
if dev.get('type', '').lower() not in self._PORT_DEVICE_TYPES:
|
||||
continue
|
||||
sw_name = dev.get('name') or dev.get('mac', 'unknown')
|
||||
sw_ip = dev.get('ip', '')
|
||||
@@ -321,13 +429,19 @@ class PulseClient:
|
||||
return self.run_command(command, _retry=False)
|
||||
return None
|
||||
|
||||
def ping(self, ip: str, count: int = 3, timeout: int = 2) -> bool:
|
||||
"""Ping *ip* via the Pulse worker. Returns True if host responds."""
|
||||
def ping(self, ip: str, count: int = 3, timeout: int = 2) -> Optional[bool]:
|
||||
"""Ping *ip* via the Pulse worker.
|
||||
|
||||
Returns True/False for the host's reachability, or None if Pulse itself
|
||||
couldn't run the check, which says nothing about the host.
|
||||
"""
|
||||
ip_q = shlex.quote(ip)
|
||||
output = self.run_command(
|
||||
f'ping -c {count} -W {timeout} {ip_q} >/dev/null 2>&1 && echo REACHABLE || echo UNREACHABLE'
|
||||
)
|
||||
return output is not None and output.strip() == 'REACHABLE'
|
||||
if output is None:
|
||||
return None
|
||||
return output.strip() == 'REACHABLE'
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
@@ -615,9 +729,12 @@ class LinkStatsCollector:
|
||||
except Exception as e:
|
||||
logger.warning(f'UniFi switch port collection failed: {e}')
|
||||
|
||||
unifi_devices = (self.unifi.get_device_details() or {}) if self.unifi else {}
|
||||
|
||||
return {
|
||||
'hosts': result_hosts,
|
||||
'unifi_switches': unifi_switches,
|
||||
'unifi_devices': unifi_devices,
|
||||
'updated': datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M:%S UTC'),
|
||||
}
|
||||
|
||||
@@ -625,8 +742,10 @@ class LinkStatsCollector:
|
||||
"""Compute per-port byte/error rates from delta against previous snapshot."""
|
||||
dt = now - self._prev_unifi_time if self._prev_unifi_time > 0 else 0
|
||||
|
||||
def rate(new_val: int, old_val: int) -> Optional[float]:
|
||||
if dt <= 0:
|
||||
def rate(new_val: int, old_val: Optional[int]) -> Optional[float]:
|
||||
# No earlier reading (first poll, or a new/renamed port) means no rate,
|
||||
# not "all bytes since boot arrived in this one interval".
|
||||
if dt <= 0 or old_val is None:
|
||||
return None
|
||||
return max(0.0, (new_val - old_val) / dt)
|
||||
|
||||
@@ -637,12 +756,12 @@ class LinkStatsCollector:
|
||||
for pname, d in sw_data['ports'].items():
|
||||
entry = dict(d)
|
||||
prev = prev_ports.get(pname, {})
|
||||
entry['tx_bytes_rate'] = rate(d['tx_bytes'], prev.get('tx_bytes', 0))
|
||||
entry['rx_bytes_rate'] = rate(d['rx_bytes'], prev.get('rx_bytes', 0))
|
||||
entry['tx_errs_rate'] = rate(d['tx_errors'], prev.get('tx_errors', 0))
|
||||
entry['rx_errs_rate'] = rate(d['rx_errors'], prev.get('rx_errors', 0))
|
||||
entry['tx_drops_rate'] = rate(d['tx_dropped'], prev.get('tx_dropped', 0))
|
||||
entry['rx_drops_rate'] = rate(d['rx_dropped'], prev.get('rx_dropped', 0))
|
||||
entry['tx_bytes_rate'] = rate(d['tx_bytes'], prev.get('tx_bytes'))
|
||||
entry['rx_bytes_rate'] = rate(d['rx_bytes'], prev.get('rx_bytes'))
|
||||
entry['tx_errs_rate'] = rate(d['tx_errors'], prev.get('tx_errors'))
|
||||
entry['rx_errs_rate'] = rate(d['rx_errors'], prev.get('rx_errors'))
|
||||
entry['tx_drops_rate'] = rate(d['tx_dropped'], prev.get('tx_dropped'))
|
||||
entry['rx_drops_rate'] = rate(d['rx_dropped'], prev.get('rx_dropped'))
|
||||
merged_ports[pname] = entry
|
||||
result[sw_name] = {'ip': sw_data['ip'], 'model': sw_data['model'],
|
||||
'ports': merged_ports}
|
||||
@@ -685,6 +804,27 @@ class NetworkMonitor:
|
||||
self.cluster_thresh = mon.get('cluster_threshold', 3)
|
||||
self.cluster_name = mon.get('cluster_name', CLUSTER_NAME)
|
||||
|
||||
# Optional WAN modem (AT&T BGW-series); see modem.py
|
||||
modem_cfg = self.cfg.get('modem') or {}
|
||||
self.modem = modem_api.ModemClient(modem_cfg) if modem_cfg.get('url') else None
|
||||
self.modem_name = modem_cfg.get('name', 'att-modem')
|
||||
# Modem reboots (including AT&T's automatic firmware updates) take a few
|
||||
# minutes, so this can be raised independently of failure_threshold.
|
||||
self.modem_fail_thresh = modem_cfg.get('failure_threshold', self.fail_thresh)
|
||||
self.rx_drift_db = modem_cfg.get('rx_drift_db', 3.0)
|
||||
self.rx_drift_min_samples = modem_cfg.get('rx_drift_min_samples', 288)
|
||||
self.modem_lan_ip = urlparse(modem_cfg['url']).hostname if modem_cfg.get('url') else ''
|
||||
self._prev_modem_counters: Optional[dict] = None
|
||||
self._prev_modem_time = 0.0
|
||||
if self.modem:
|
||||
try:
|
||||
db.ensure_modem_schema()
|
||||
except Exception as e:
|
||||
logger.error(f'Could not create modem history table: {e}')
|
||||
|
||||
# Last good Prometheus result, shown (as unknown) during a Prometheus outage
|
||||
self._last_iface_states: Dict[str, Dict[str, bool]] = {}
|
||||
|
||||
# Build Prometheus instance → hostname lookup
|
||||
self._instance_map: Dict[str, str] = {
|
||||
h['prometheus_instance']: h['name']
|
||||
@@ -744,6 +884,22 @@ class NetworkMonitor:
|
||||
if host_has_regression:
|
||||
hosts_with_regression.append(host)
|
||||
|
||||
# A dead host doesn't report its interfaces as down: its series just
|
||||
# disappear, so a configured host that's absent needs its own check.
|
||||
for instance, host in self._instance_map.items():
|
||||
if instance in states:
|
||||
db.resolve_event('host_metrics_missing', host, instance)
|
||||
continue
|
||||
hosts_with_regression.append(host)
|
||||
sup = db.check_suppressed(suppressions, 'host', host)
|
||||
event_id, is_new, consec, ticket_id = db.upsert_event(
|
||||
'host_metrics_missing', 'critical', 'prometheus',
|
||||
host, instance,
|
||||
f'No metrics from {host} ({instance}): host or node_exporter is down ({_now_utc()})',
|
||||
)
|
||||
if not sup and consec >= self.fail_thresh and not ticket_id:
|
||||
self._ticket_metrics_missing(event_id, host, instance, consec)
|
||||
|
||||
db.set_baseline(new_baseline)
|
||||
|
||||
# Cluster-wide check – only genuine regressions count
|
||||
@@ -752,7 +908,7 @@ class NetworkMonitor:
|
||||
event_id, is_new, consec, ticket_id = db.upsert_event(
|
||||
'cluster_network_issue', 'critical', 'prometheus',
|
||||
self.cluster_name, '',
|
||||
f'{len(hosts_with_regression)} hosts reporting simultaneous interface failures: '
|
||||
f'{len(hosts_with_regression)} hosts with simultaneous interface or metrics failures: '
|
||||
f'{", ".join(hosts_with_regression)}',
|
||||
)
|
||||
if not sup and (is_new or not ticket_id):
|
||||
@@ -765,7 +921,8 @@ class NetworkMonitor:
|
||||
f'Affected hosts: {", ".join(hosts_with_regression)}\n'
|
||||
f'Detected: {_now_utc()}\n\n'
|
||||
f'{len(hosts_with_regression)} Proxmox hosts simultaneously reported '
|
||||
f'interface regressions (link-down on interfaces previously known UP).\n'
|
||||
f'interface regressions (link-down on interfaces previously known UP) '
|
||||
f'or stopped reporting metrics.\n'
|
||||
f'This likely indicates a switch or upstream network failure.\n\n'
|
||||
f'Please check the core and management switches immediately.'
|
||||
)
|
||||
@@ -775,6 +932,54 @@ class NetworkMonitor:
|
||||
else:
|
||||
db.resolve_event('cluster_network_issue', self.cluster_name, '')
|
||||
|
||||
def _ticket_metrics_missing(
|
||||
self, event_id: int, host: str, instance: str, consec: int
|
||||
) -> None:
|
||||
title = (
|
||||
f'[{host}][auto][production][issue][network][single-node] '
|
||||
f'No metrics from host'
|
||||
)
|
||||
desc = (
|
||||
f'Host Metrics Alert\n{"=" * 40}\n\n'
|
||||
f'Host: {host}\n'
|
||||
f'Prometheus instance: {instance}\n'
|
||||
f'Detected: {_now_utc()}\n'
|
||||
f'Consecutive check failures: {consec}\n\n'
|
||||
f'Prometheus is working but has no node_network_up series for {host}.\n'
|
||||
f'Either the host is down or node_exporter has stopped on it.\n\n'
|
||||
f'Please check the host power and network connectivity, then '
|
||||
f'prometheus-node-exporter on the host.'
|
||||
)
|
||||
tid = self.tickets.create(title, desc, priority='2')
|
||||
if tid:
|
||||
db.set_ticket_id(event_id, tid)
|
||||
|
||||
def _process_prometheus_outage(self, suppressions: list) -> None:
|
||||
"""Interface and host alerts can't fire without Prometheus, so say so."""
|
||||
sup = db.check_suppressed(suppressions, 'host', 'prometheus')
|
||||
event_id, is_new, consec, ticket_id = db.upsert_event(
|
||||
'monitor_source_unavailable', 'critical', 'prometheus',
|
||||
'prometheus', '',
|
||||
f'Prometheus returned no interface data; host and interface alerting is paused ({_now_utc()})',
|
||||
)
|
||||
if not sup and consec >= self.fail_thresh and not ticket_id:
|
||||
title = (
|
||||
'[prometheus][auto][production][issue][network][cluster-wide] '
|
||||
'Gandalf has no Prometheus data'
|
||||
)
|
||||
desc = (
|
||||
f'Monitoring Source Alert\n{"=" * 40}\n\n'
|
||||
f'Prometheus URL: {self.prom.url}\n'
|
||||
f'Detected: {_now_utc()}\n'
|
||||
f'Consecutive check failures: {consec}\n\n'
|
||||
f'Gandalf could not get node_network_up data from Prometheus.\n'
|
||||
f'While this lasts, host and interface failures will NOT be detected.\n\n'
|
||||
f'Please check the Prometheus service and its scrape targets.'
|
||||
)
|
||||
tid = self.tickets.create(title, desc, priority='2')
|
||||
if tid:
|
||||
db.set_ticket_id(event_id, tid)
|
||||
|
||||
def _ticket_interface(
|
||||
self, event_id: int, host: str, iface: str, consec: int
|
||||
) -> None:
|
||||
@@ -842,10 +1047,15 @@ class NetworkMonitor:
|
||||
# ------------------------------------------------------------------
|
||||
# Ping-only hosts (no node_exporter)
|
||||
# ------------------------------------------------------------------
|
||||
def _process_ping_hosts(self, suppressions: list, ping_states: Dict[str, bool]) -> None:
|
||||
def _process_ping_hosts(self, suppressions: list, ping_states: Dict[str, Optional[bool]]) -> None:
|
||||
for h in self.cfg.get('monitor', {}).get('ping_hosts', []):
|
||||
name, ip = h['name'], h['ip']
|
||||
reachable = ping_states.get(name, False)
|
||||
reachable = ping_states.get(name)
|
||||
|
||||
if reachable is None:
|
||||
# Pulse couldn't run the check. Neither alert nor resolve on no evidence.
|
||||
logger.warning(f'Ping check for {name} ({ip}) unavailable this cycle')
|
||||
continue
|
||||
|
||||
if not reachable:
|
||||
sup = db.check_suppressed(suppressions, 'host', name)
|
||||
@@ -881,26 +1091,301 @@ class NetworkMonitor:
|
||||
if tid:
|
||||
db.set_ticket_id(event_id, tid)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# WAN modem monitoring
|
||||
# ------------------------------------------------------------------
|
||||
_MODEM_METRIC_LABELS = {
|
||||
'rx_power': 'Rx power', 'tx_power': 'Tx power', 'temperature': 'Temperature',
|
||||
'vcc': 'Supply voltage', 'tx_bias': 'Tx bias',
|
||||
}
|
||||
_MODEM_METRIC_UNITS = {'rx_power': ' dBm', 'tx_power': ' dBm', 'temperature': ' C'}
|
||||
|
||||
def _modem_event(
|
||||
self, suppressions: list, event_type: str, severity: str, detail: str,
|
||||
description: str, heading: str, priority: str, advice: str,
|
||||
) -> None:
|
||||
name = self.modem_name
|
||||
sup = db.check_suppressed(suppressions, 'host', name)
|
||||
event_id, is_new, consec, ticket_id = db.upsert_event(
|
||||
event_type, severity, 'modem', name, detail, f'{description} ({_now_utc()})',
|
||||
)
|
||||
if not sup and consec >= self.modem_fail_thresh and not ticket_id:
|
||||
title = f'[{name}][auto][production][issue][network][cluster-wide] {heading}'
|
||||
desc = (
|
||||
f'WAN Modem Alert\n{"=" * 40}\n\n'
|
||||
f'Modem: {name} ({self.modem.base_url})\n'
|
||||
f'Detected: {_now_utc()}\n'
|
||||
f'Consecutive check failures: {consec}\n\n'
|
||||
f'{description}\n\n{advice}'
|
||||
)
|
||||
tid = self.tickets.create(title, desc, priority=priority)
|
||||
if tid:
|
||||
db.set_ticket_id(event_id, tid)
|
||||
|
||||
def _process_modem_failure(self, suppressions: list, error: Exception) -> None:
|
||||
"""Handle a failed poll. Link and optical events stay as they are: we can't see them."""
|
||||
name = self.modem_name
|
||||
if isinstance(error, modem_api.ModemUnreachable):
|
||||
db.resolve_event('modem_data_invalid', name, '')
|
||||
self._modem_event(
|
||||
suppressions, 'modem_unreachable', 'critical', '',
|
||||
f'WAN modem is not responding: {error}',
|
||||
'WAN modem unreachable', '2',
|
||||
'Internet access is probably affected too. Check modem power and the '
|
||||
'cable between the modem and the UDM-Pro.',
|
||||
)
|
||||
else:
|
||||
db.resolve_event('modem_unreachable', name, '')
|
||||
self._modem_event(
|
||||
suppressions, 'modem_data_invalid', 'warning', '',
|
||||
f'WAN modem status could not be read: {error}',
|
||||
'WAN modem status unreadable', '3',
|
||||
'The modem answered but its pages could not be parsed, so fiber alerting is '
|
||||
'paused. A firmware update may have changed them; see modem.py.',
|
||||
)
|
||||
|
||||
def _process_modem(self, data: dict, suppressions: list) -> None:
|
||||
"""Raise and resolve alerts from a successful modem poll."""
|
||||
name = self.modem_name
|
||||
db.resolve_event('modem_unreachable', name, '')
|
||||
db.resolve_event('modem_data_invalid', name, '')
|
||||
|
||||
problems = modem_api.wan_problems(data)
|
||||
if problems:
|
||||
self._modem_event(
|
||||
suppressions, 'wan_down', 'critical', '',
|
||||
f'WAN link is down: {"; ".join(problems)}.',
|
||||
'WAN link down', '1',
|
||||
'Check the fiber at the modem and the AT&T service status. Rx power at or below '
|
||||
'the alarm limit points to a fiber fault rather than a provider outage.',
|
||||
)
|
||||
else:
|
||||
db.resolve_event('wan_down', name, '')
|
||||
|
||||
for key, metric in data.get('optics', {}).items():
|
||||
status = modem_api.metric_status(metric)
|
||||
if status == 'alarm' and problems and key == 'rx_power':
|
||||
# wan_down already covers this, and a cut fiber would otherwise open two tickets.
|
||||
status = 'ok'
|
||||
label = self._MODEM_METRIC_LABELS.get(key, key)
|
||||
value = metric.get('value')
|
||||
reading = f' at {value}{self._MODEM_METRIC_UNITS.get(key, "")}' if value is not None else ''
|
||||
limits = (
|
||||
f' (warning {metric.get("warn_low")} to {metric.get("warn_high")}, '
|
||||
f'alarm {metric.get("alarm_low")} to {metric.get("alarm_high")})'
|
||||
) if value is not None else ''
|
||||
advice = (
|
||||
'Inspect and clean the fiber connector at the modem and look for tight bends. '
|
||||
'If that does not help, contact AT&T: the reading is what the modem measures '
|
||||
'on the provider side of the connection.'
|
||||
)
|
||||
|
||||
if status == 'alarm':
|
||||
db.resolve_event('fiber_signal_warning', name, key)
|
||||
self._modem_event(
|
||||
suppressions, 'fiber_signal_alarm', 'critical', key,
|
||||
f'{label} is in the alarm range{reading}{limits}.',
|
||||
f'Fiber {label} alarm', '2', advice,
|
||||
)
|
||||
elif status == 'warning':
|
||||
db.resolve_event('fiber_signal_alarm', name, key)
|
||||
self._modem_event(
|
||||
suppressions, 'fiber_signal_warning', 'warning', key,
|
||||
f'{label} is in the warning range{reading}{limits}.',
|
||||
f'Fiber {label} warning', '3', advice,
|
||||
)
|
||||
else:
|
||||
db.resolve_event('fiber_signal_alarm', name, key)
|
||||
db.resolve_event('fiber_signal_warning', name, key)
|
||||
|
||||
self._check_rx_drift(data, suppressions)
|
||||
|
||||
def _check_rx_drift(self, data: dict, suppressions: list) -> None:
|
||||
"""Alert on a sustained Rx drop relative to this link's own recent normal.
|
||||
|
||||
The modem's limits are very wide (warning at -27.9 dBm on a link that sits
|
||||
around -17), so a damaged connector could lose many dB before they trip.
|
||||
"""
|
||||
name = self.modem_name
|
||||
rx = (data.get('optics', {}).get('rx_power') or {}).get('value')
|
||||
if rx is None:
|
||||
return
|
||||
history = db.get_modem_rx_values(days=7)
|
||||
if len(history) < self.rx_drift_min_samples:
|
||||
return # not enough history yet to know what normal looks like
|
||||
baseline = statistics.median(history)
|
||||
drop = round(baseline - rx, 1)
|
||||
if drop >= self.rx_drift_db:
|
||||
self._modem_event(
|
||||
suppressions, 'fiber_rx_drift', 'warning', 'rx_power',
|
||||
f'Rx power is {rx} dBm, {drop} dB below its 7-day median of {baseline:.1f} dBm.',
|
||||
'Fiber Rx power dropped', '3',
|
||||
'The signal is still in range but has changed. Look for a recently disturbed '
|
||||
'or bent fiber and check the connector at the modem.',
|
||||
)
|
||||
else:
|
||||
db.resolve_event('fiber_rx_drift', name, 'rx_power')
|
||||
|
||||
def _load_wan_history(self) -> dict:
|
||||
raw = db.get_state('wan_history')
|
||||
if raw:
|
||||
try:
|
||||
return json.loads(raw)
|
||||
except ValueError:
|
||||
logger.error('Failed to parse wan_history JSON; starting over')
|
||||
return {'outages': [], 'restarts': []}
|
||||
|
||||
def _note_modem_restart(self, data: dict) -> None:
|
||||
uptime = data.get('uptime_s')
|
||||
if uptime is None:
|
||||
return
|
||||
previous = None
|
||||
raw = db.get_state('modem_stats')
|
||||
if raw:
|
||||
try:
|
||||
previous = json.loads(raw).get('uptime_s')
|
||||
except ValueError:
|
||||
logger.error('Failed to parse modem_stats JSON')
|
||||
if previous is not None and uptime < previous:
|
||||
logger.warning(f'Modem restarted: uptime went from {previous}s to {uptime}s')
|
||||
history = self._load_wan_history()
|
||||
modem_api.record_uptime_reset(history, time.time() - uptime)
|
||||
db.set_state('wan_history', history)
|
||||
|
||||
def _one_shot_ticket(
|
||||
self, suppressions: list, event_type: str, description: str,
|
||||
heading: str, priority: str, advice: str,
|
||||
) -> None:
|
||||
"""Report something that has already happened: open a ticket and resolve the event."""
|
||||
name = self.modem_name
|
||||
event_id, _, _, ticket_id = db.upsert_event(event_type, 'warning', 'unifi', name, '', description)
|
||||
if not db.check_suppressed(suppressions, 'host', name) and not ticket_id:
|
||||
tid = self.tickets.create(
|
||||
f'[{name}][auto][production][issue][network][cluster-wide] {heading}',
|
||||
f'WAN Event\n{"=" * 40}\n\nModem: {name}\nReported: {_now_utc()}\n\n{description}\n\n{advice}',
|
||||
priority=priority,
|
||||
)
|
||||
if tid:
|
||||
db.set_ticket_id(event_id, tid)
|
||||
db.resolve_event(event_type, name, '')
|
||||
|
||||
def _process_wan_log(self, suppressions: list) -> None:
|
||||
"""Read WAN outages and modem restarts from the UniFi log.
|
||||
|
||||
These can be over in two minutes, so a 5 minute poll of the modem may never see them.
|
||||
"""
|
||||
now = time.time()
|
||||
raw_cursor = db.get_state('unifi_log_cursor')
|
||||
if raw_cursor is None:
|
||||
# First run: start from now rather than ticketing old history.
|
||||
db.set_state('unifi_log_cursor', str(int(now * 1000)))
|
||||
return
|
||||
# After long monitor downtime, don't open tickets for stale events.
|
||||
since_ms = max(int(raw_cursor) + 1, int((now - 86400) * 1000))
|
||||
events = self.unifi.get_system_events(since_ms, modem_api.WAN_LOG_KEYS)
|
||||
if not events:
|
||||
return # None = log unavailable, so the cursor stays put and it's retried
|
||||
|
||||
history = self._load_wan_history()
|
||||
actionable = modem_api.apply_wan_events(history, events)
|
||||
|
||||
# UniFi repeats "failed multiple times" per failure, so ticket it once a day at most.
|
||||
unstable = [e for e in actionable if e['key'] == modem_api.LOG_WAN_UNSTABLE]
|
||||
ticket_unstable = bool(unstable) and now - history.get('unstable_ticketed_at', 0) >= 86400
|
||||
if ticket_unstable:
|
||||
history['unstable_ticketed_at'] = now
|
||||
|
||||
# Save first: a failed ticket call shouldn't replay these events next cycle.
|
||||
db.set_state('wan_history', history)
|
||||
db.set_state('unifi_log_cursor', str(int(max(e['ts'] for e in events) * 1000)))
|
||||
|
||||
for e in actionable:
|
||||
if e['key'] == modem_api.LOG_MODEM_RESTARTED:
|
||||
when = datetime.fromtimestamp(e['ts'], timezone.utc).strftime('%Y-%m-%d %H:%M:%S UTC')
|
||||
self._one_shot_ticket(
|
||||
suppressions, 'modem_power_cycled',
|
||||
f'{e.get("device") or "Smart Power"} power-cycled the modem at {when} '
|
||||
f'to try to restore the internet connection.',
|
||||
'Modem auto power-cycled after internet loss', '3',
|
||||
'The internet connection failed badly enough to trigger auto power-cycle. '
|
||||
'Check the WAN / Fiber panel for outages around this time and Rx power for a drop.',
|
||||
)
|
||||
if ticket_unstable:
|
||||
self._one_shot_ticket(
|
||||
suppressions, 'wan_unstable',
|
||||
'UniFi reports the WAN connection went down multiple times in the last 24 hours.',
|
||||
'WAN connection unstable', '2',
|
||||
'See the WAN history on the modem page. Repeated drops with healthy Rx power '
|
||||
'point toward the provider or the modem rather than the fiber.',
|
||||
)
|
||||
|
||||
def _collect_modem(self, suppressions: list) -> None:
|
||||
"""Poll the modem, then store its status and raise any alerts."""
|
||||
try:
|
||||
data = self.modem.poll()
|
||||
except (modem_api.ModemUnreachable, modem_api.ModemDataInvalid) as e:
|
||||
logger.warning(f'Modem poll failed: {e}')
|
||||
self._process_modem_failure(suppressions, e)
|
||||
db.set_state('modem_stats', {
|
||||
'reachable': False, 'error': str(e), 'name': self.modem_name,
|
||||
'lan_ip': self.modem_lan_ip, 'updated': _now_utc(),
|
||||
})
|
||||
return
|
||||
|
||||
# Uptime going backwards between polls means the gateway restarted, even if
|
||||
# it came back too quickly for any poll to fail. Read before it's overwritten.
|
||||
self._note_modem_restart(data)
|
||||
|
||||
now = time.monotonic()
|
||||
data['rates'] = modem_api.counter_rates(
|
||||
data['counters'], self._prev_modem_counters, now - self._prev_modem_time,
|
||||
)
|
||||
self._prev_modem_counters = data['counters']
|
||||
self._prev_modem_time = now
|
||||
data['reachable'] = True
|
||||
data['name'] = self.modem_name
|
||||
data['lan_ip'] = self.modem_lan_ip
|
||||
data['wan_problems'] = modem_api.wan_problems(data)
|
||||
data['optics_status'] = {k: modem_api.metric_status(m) for k, m in data['optics'].items()}
|
||||
db.set_state('modem_stats', data)
|
||||
|
||||
# Alert before recording, so a sudden drop isn't compared against itself.
|
||||
self._process_modem(data, suppressions)
|
||||
|
||||
optics = data['optics']
|
||||
db.record_modem_sample(
|
||||
(optics.get('rx_power') or {}).get('value'),
|
||||
(optics.get('tx_power') or {}).get('value'),
|
||||
(optics.get('temperature') or {}).get('value'),
|
||||
)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Snapshot collection (for dashboard)
|
||||
# ------------------------------------------------------------------
|
||||
def _collect_snapshot(
|
||||
self, iface_states: Dict[str, Dict[str, bool]],
|
||||
self, iface_states: Optional[Dict[str, Dict[str, bool]]],
|
||||
unifi_devices: Optional[List[dict]] = None,
|
||||
ping_states: Optional[Dict[str, bool]] = None,
|
||||
ping_states: Optional[Dict[str, Optional[bool]]] = None,
|
||||
) -> dict:
|
||||
# Accept pre-fetched devices; fall back to empty list if unavailable
|
||||
display_unifi = unifi_devices if unifi_devices is not None else []
|
||||
|
||||
# During a Prometheus outage keep showing the hosts, with their state unknown,
|
||||
# rather than emptying the dashboard or implying that they're down.
|
||||
prom_ok = iface_states is not None
|
||||
shown_states = iface_states if prom_ok else self._last_iface_states
|
||||
|
||||
hosts = {}
|
||||
for instance, ifaces in iface_states.items():
|
||||
for instance, ifaces in shown_states.items():
|
||||
if instance not in self._instance_map:
|
||||
continue # skip Prometheus instances not in config (e.g. LXC app servers)
|
||||
host = self._hostname(instance)
|
||||
phys = {k: v for k, v in ifaces.items()}
|
||||
up_count = sum(1 for v in phys.values() if v)
|
||||
total = len(phys)
|
||||
if total == 0 or up_count == total:
|
||||
if not prom_ok:
|
||||
status = 'unknown'
|
||||
elif total == 0 or up_count == total:
|
||||
status = 'up'
|
||||
elif up_count == 0:
|
||||
status = 'down'
|
||||
@@ -914,13 +1399,23 @@ class NetworkMonitor:
|
||||
'source': 'prometheus',
|
||||
}
|
||||
|
||||
# Configured hosts with no data: down if Prometheus is fine, otherwise unknown
|
||||
for instance, host in self._instance_map.items():
|
||||
if instance not in shown_states:
|
||||
hosts[host] = {
|
||||
'ip': instance.split(':')[0],
|
||||
'interfaces': {},
|
||||
'status': 'down' if prom_ok else 'unknown',
|
||||
'source': 'prometheus',
|
||||
}
|
||||
|
||||
for h in self.cfg.get('monitor', {}).get('ping_hosts', []):
|
||||
name, ip = h['name'], h['ip']
|
||||
reachable = (ping_states or {}).get(name, False)
|
||||
reachable = (ping_states or {}).get(name)
|
||||
hosts[name] = {
|
||||
'ip': ip,
|
||||
'interfaces': {},
|
||||
'status': 'up' if reachable else 'down',
|
||||
'status': 'unknown' if reachable is None else ('up' if reachable else 'down'),
|
||||
'source': 'ping',
|
||||
}
|
||||
|
||||
@@ -949,13 +1444,15 @@ class NetworkMonitor:
|
||||
unifi_devices = self.unifi.get_devices()
|
||||
|
||||
# 3a. Ping-only hosts once — shared by snapshot and alert processing
|
||||
ping_states: Dict[str, bool] = {
|
||||
ping_states: Dict[str, Optional[bool]] = {
|
||||
h['name']: self.pulse.ping(h['ip'])
|
||||
for h in self.cfg.get('monitor', {}).get('ping_hosts', [])
|
||||
}
|
||||
|
||||
# 3b. Collect and store snapshot for dashboard
|
||||
snapshot = self._collect_snapshot(iface_states, unifi_devices, ping_states)
|
||||
if iface_states is not None:
|
||||
self._last_iface_states = iface_states
|
||||
db.set_state('network_snapshot', snapshot)
|
||||
db.set_state('last_check', _now_utc())
|
||||
|
||||
@@ -968,14 +1465,29 @@ class NetworkMonitor:
|
||||
|
||||
# 5. Process alerts using already-fetched data
|
||||
suppressions = db.get_active_suppressions()
|
||||
self._process_interfaces(iface_states, suppressions)
|
||||
if iface_states is None:
|
||||
self._process_prometheus_outage(suppressions)
|
||||
else:
|
||||
db.resolve_event('monitor_source_unavailable', 'prometheus', '')
|
||||
self._process_interfaces(iface_states, suppressions)
|
||||
self._process_unifi(unifi_devices, suppressions)
|
||||
|
||||
self._process_ping_hosts(suppressions, ping_states)
|
||||
|
||||
# WAN modem: isolated so a failure here can't stop the rest of the cycle
|
||||
if self.modem:
|
||||
try:
|
||||
# Log first, so a restart it reports can explain an uptime reset below
|
||||
self._process_wan_log(suppressions)
|
||||
self._collect_modem(suppressions)
|
||||
except Exception as e:
|
||||
logger.error(f'Modem collection failed: {e}', exc_info=True)
|
||||
|
||||
# Housekeeping: deactivate expired suppressions and purge old resolved events
|
||||
db.cleanup_expired_suppressions()
|
||||
db.purge_old_resolved_events(days=90)
|
||||
if self.modem:
|
||||
db.purge_old_modem_samples(days=90)
|
||||
|
||||
logger.info('Network check cycle complete')
|
||||
|
||||
|
||||
Reference in new issue
Block a user