Lint / JS (eslint) (push) Successful in 1m11s
Lint / Python (flake8) (push) Successful in 1m55s
Lint / Notify on failure (push) Skipped
Lint / Deploy (push) Skipped
Security / Python Security (bandit) (push) Successful in 54s
Lint / Python (flake8) (pull_request) Successful in 53s
Test / Python Tests (pytest) (push) Successful in 1m5s
Lint / JS (eslint) (pull_request) Successful in 11s
Lint / Notify on failure (pull_request) Skipped
Lint / Deploy (pull_request) Skipped
Security / Python Security (bandit) (pull_request) Successful in 55s
Test / Python Tests (pytest) (pull_request) Successful in 1m3s
With modem.access_code in config.json, the monitor logs in to the BGW320 (md5 of code + nonce, as its own page does) for two read-only checks, stored in monitor_state.modem_checks: - NAT sessions in use every 30 minutes; Needs attention warns at 80%. - While UniFi says the internet is slow, a ping from the modem itself to 1.1.1.1 (at most every 10 minutes). Compared with UniFi's latency through the modem, it says whether the delay is past the modem (the provider), in the modem, or gone. The Internet line in Needs attention and the WAN / Fiber panel show the result. A refused code is reported and not retried for 30 minutes; other failures pause the checks for 10. The unauthenticated status polling is unchanged. Tested against the real gateway and a fake one. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
216 lines
11 KiB
Python
216 lines
11 KiB
Python
"""What needs looking at, gathered from every page into one list for the dashboard.
|
|
|
|
Two groups: 'broken' is something not working right now; 'tidy' is housekeeping, where
|
|
the network works but the records or names don't match it. Each entry links to the page
|
|
where it is fixed. Pure functions, so the rules are easy to test.
|
|
"""
|
|
import time
|
|
from typing import Dict, List, Optional
|
|
|
|
import modem
|
|
|
|
GROUPS = ('broken', 'tidy')
|
|
|
|
|
|
def _plural(n: int, one: str, many: Optional[str] = None) -> str:
|
|
return f'{n} {one if n == 1 else (many or one + "s")}'
|
|
|
|
|
|
def _entry(group: str, severity: str, text: str, href: str, detail: str = '') -> dict:
|
|
return {'group': group, 'severity': severity, 'text': text, 'href': href, 'detail': detail}
|
|
|
|
|
|
def _alerts(events: List[dict]) -> List[dict]:
|
|
live = [e for e in events if not e.get('is_suppressed') and e.get('severity') != 'info']
|
|
if not live:
|
|
return []
|
|
critical = sum(1 for e in live if e.get('severity') == 'critical')
|
|
targets = sorted({str(e.get('target_name') or '') for e in live if e.get('target_name')}, key=str.lower)
|
|
text = _plural(len(live), 'active alert')
|
|
if critical:
|
|
text += f' ({critical} critical)'
|
|
shown = ', '.join(targets[:4]) + (f' and {len(targets) - 4} more' if len(targets) > 4 else '')
|
|
return [_entry('broken', 'bad' if critical else 'warn', text, '#events-table-wrap', shown)]
|
|
|
|
|
|
def _power(power: List[dict]) -> List[dict]:
|
|
out = []
|
|
for src in power or []:
|
|
name = src.get('node') or ''
|
|
if src.get('stale_since'):
|
|
out.append(_entry('broken', 'warn', f'{name}: outlet watts are frozen', '/topology',
|
|
'UniFi keeps sending the same per-outlet readings; the total is still live. '
|
|
'Restarting the unit usually clears it.'))
|
|
for o in src.get('outlets') or []:
|
|
if not o.get('on') and o.get('target'):
|
|
out.append(_entry('broken', 'bad', f'{name} outlet {o.get("index")} is off, and it feeds {o["target"]}',
|
|
'/topology'))
|
|
idle = []
|
|
for src in power or []:
|
|
for o in src.get('outlets') or []:
|
|
# Drawing real power, but nothing in Gandalf says what is plugged in
|
|
if o.get('on') and not o.get('target') and o.get('match') != 'cleared' and (o.get('power_w') or 0) >= 5:
|
|
idle.append(f'{src.get("node")} {o.get("index")}')
|
|
if idle:
|
|
out.append(_entry('tidy', 'info', f'{_plural(len(idle), "powered outlet")} with no device assigned',
|
|
'/topology', ', '.join(idle[:6]) + (' ...' if len(idle) > 6 else '')))
|
|
return out
|
|
|
|
|
|
def _cutsheet(view: dict) -> List[dict]:
|
|
out = []
|
|
cables = [c for c in view.get('cables') or [] if c.get('status') != 'retired']
|
|
bad = [c for c in cables if c.get('problem')]
|
|
if bad:
|
|
names = ', '.join(c.get('label') or f'#{c["id"]}' for c in bad[:5])
|
|
out.append(_entry('broken', 'bad', f'{_plural(len(bad), "cable")} not where the cutsheet says',
|
|
'/cutsheet?problems=1#cs-cables-section', names))
|
|
|
|
names = view.get('port_names') or []
|
|
wrong = [r for r in names if r.get('kind') in ('wrong', 'moved')]
|
|
if wrong:
|
|
out.append(_entry('tidy', 'warn', f'{_plural(len(wrong), "UniFi port name")} pointing at the wrong device',
|
|
'/cutsheet#cs-portnames-section',
|
|
', '.join(f'{r["switch"]} port {r["port_idx"]} "{r["port_name"]}"' for r in wrong[:3])))
|
|
empty = [r for r in names if r.get('kind') == 'empty']
|
|
if empty:
|
|
out.append(_entry('tidy', 'info', f'{_plural(len(empty), "named port")} with nothing plugged in',
|
|
'/cutsheet#cs-portnames-section',
|
|
', '.join(f'"{r["port_name"]}"' for r in empty[:4])))
|
|
|
|
missing = view.get('suggestions') or []
|
|
if missing:
|
|
out.append(_entry('tidy', 'info', f'{_plural(len(missing), "live link")} not on the cutsheet',
|
|
'/cutsheet#cs-suggest-section'))
|
|
|
|
labels = view.get('labels') or {}
|
|
todo = len(labels.get('cables') or []) + len(labels.get('ports') or [])
|
|
if todo:
|
|
out.append(_entry('tidy', 'info', f'{_plural(todo, "label")} to print and stick on',
|
|
'/cutsheet#cs-labels-section'))
|
|
return out
|
|
|
|
|
|
def _rack(issues: Dict[str, dict]) -> List[dict]:
|
|
out = []
|
|
for rack, r in (issues or {}).items():
|
|
for o in r.get('overlaps') or []:
|
|
out.append(_entry('tidy', 'warn', f'{o["name"]} overlaps {", ".join(o["with"])} at {o["units"]}',
|
|
'/cutsheet#cs-items-section', rack))
|
|
for o in r.get('outside') or []:
|
|
out.append(_entry('tidy', 'warn', f'{o["name"]} at {o["units"]} is outside {rack}',
|
|
'/cutsheet#cs-items-section'))
|
|
return out
|
|
|
|
|
|
# UniFi's WAN timeline: judge the last hour, and ignore a copy the monitor stopped refreshing
|
|
_WAN_WINDOW_S = 3600
|
|
_WAN_STALE_S = 1800
|
|
_WAN_BAD_PERIODS = 3 # periods with any problem before the list mentions it
|
|
_WAN_LOSS_PERIODS = 2 # periods with packet loss before it counts as broken
|
|
|
|
|
|
def _ago(seconds: float) -> str:
|
|
minutes = max(1, round(seconds / 60))
|
|
return f'{minutes} min ago' if minutes < 60 else f'{round(minutes / 60)} h ago'
|
|
|
|
|
|
_NAT_FULL = 0.8 # share of the gateway's NAT sessions in use before it's worth a line
|
|
|
|
|
|
def _modem_ping_note(checks: Optional[dict], now: float) -> str:
|
|
"""One sentence on the latest ping from the modem itself, if it ran in the last hour."""
|
|
ping = (checks or {}).get('ping') or {}
|
|
if not ping or now - (ping.get('at') or 0) > _WAN_WINDOW_S:
|
|
return ''
|
|
when, avg, ours = _ago(now - ping['at']), ping.get('avg_ms'), ping.get('ours_ms')
|
|
theirs = (f'{round(avg)} ms' if avg is not None else 'no reply') + (
|
|
f', {ping["loss_pct"]}% lost' if ping.get('loss_pct') else '')
|
|
ours_txt = f'; ours through it {round(ours)} ms' if ours is not None else ''
|
|
verdict = {
|
|
'upstream': "so the delay is past the modem, on the provider's side",
|
|
'gateway': 'so the delay is in the modem, or between it and the router',
|
|
'clear': 'both fine by then',
|
|
}.get(ping.get('verdict'), '')
|
|
return f"Modem's own ping {when}: {theirs}{ours_txt}, {verdict}. "
|
|
|
|
|
|
def _modem_checks(checks: Optional[dict]) -> List[dict]:
|
|
out = []
|
|
nat = (checks or {}).get('nat') or {}
|
|
if nat.get('available') and nat.get('in_use', 0) / nat['available'] >= _NAT_FULL:
|
|
pct = round(100 * nat['in_use'] / nat['available'])
|
|
out.append(_entry('broken', 'warn', f'Modem NAT table {pct}% full',
|
|
'#modem-section', f'{nat["in_use"]:,} of {nat["available"]:,} sessions in use. '
|
|
'When it fills, new connections stall or fail.'))
|
|
error = (checks or {}).get('error') or {}
|
|
if error.get('login'):
|
|
out.append(_entry('tidy', 'warn', 'Modem login failed: check modem.access_code in config.json',
|
|
'#modem-section', 'Gandalf waits 30 minutes before trying again.'))
|
|
return out
|
|
|
|
|
|
def _wan(quality: Optional[dict], now: float, checks: Optional[dict] = None) -> List[dict]:
|
|
if not quality or now - (quality.get('updated') or 0) > _WAN_STALE_S:
|
|
return []
|
|
out = []
|
|
periods = quality.get('periods') or []
|
|
# The last hour is the newest 12 periods (each stamped with its start), so the count
|
|
# doesn't dip while UniFi is still filling in the current one
|
|
newest = periods[-1]['ts'] if periods else now
|
|
recent = modem.quality_counts(periods, newest - _WAN_WINDOW_S + modem.QUALITY_PERIOD_S)
|
|
total = sum(recent.values())
|
|
bad = total - recent['ok']
|
|
day = modem.quality_counts(periods, now - 86400)
|
|
day_bad = sum(day.values()) - day['ok']
|
|
latencies = quality.get('latencies') or {}
|
|
detail = (f'UniFi flagged {day_bad} of the last {sum(day.values())} five-minute periods. '
|
|
if day_bad else '')
|
|
if latencies:
|
|
detail += 'Now: ' + ', '.join(f'{k} {v} ms' for k, v in sorted(latencies.items())) + '. '
|
|
detail += _modem_ping_note(checks, now)
|
|
detail += 'See the 24 hour strip on the WAN / Fiber panel.'
|
|
lossy = recent['loss'] + recent['down']
|
|
if lossy >= _WAN_LOSS_PERIODS:
|
|
out.append(_entry('broken', 'bad', f'Internet: packet loss in {lossy} of the last {total} '
|
|
'five-minute periods', '#modem-section', detail))
|
|
elif bad >= _WAN_BAD_PERIODS:
|
|
out.append(_entry('broken', 'warn', f'Internet: high latency in {bad} of the last {total} '
|
|
'five-minute periods', '#modem-section', detail))
|
|
drops = [d for d in quality.get('downtime') or [] if d['ts'] >= now - _WAN_WINDOW_S]
|
|
if len(drops) == 1:
|
|
out.append(_entry('broken', 'warn', f'Internet dropped for {round(drops[0]["seconds"])} s, '
|
|
f'{_ago(now - drops[0]["ts"])}', '#modem-section'))
|
|
elif drops:
|
|
out.append(_entry('broken', 'warn', f'Internet dropped {len(drops)} times in the last hour, '
|
|
f'{round(sum(d["seconds"] for d in drops))} s in all',
|
|
'#modem-section'))
|
|
return out
|
|
|
|
|
|
def build(events: List[dict], power: List[dict], cutsheet_view: Optional[dict],
|
|
rack_issues: Optional[Dict[str, dict]] = None, wan_quality: Optional[dict] = None,
|
|
now: Optional[float] = None, modem_checks: Optional[dict] = None) -> dict:
|
|
"""{'broken': [entry], 'tidy': [entry]}, worst first within each group.
|
|
|
|
events: active alerts, annotated with 'is_suppressed'. power: topology.build_power output.
|
|
cutsheet_view: the cutsheet API view (cables with 'problem', suggestions, port_names,
|
|
labels), or None when the cutsheet couldn't be loaded. rack_issues: {rack: rack_layout()}.
|
|
wan_quality: modem.wan_quality() output as stored by the monitor. modem_checks: the
|
|
monitor's logged-in modem checks (NAT sessions, ping from the modem). now: for tests.
|
|
Entry: {group, severity: bad|warn|info, text, href, detail}.
|
|
"""
|
|
now = time.time() if now is None else now
|
|
entries = (_wan(wan_quality, now, modem_checks) + _modem_checks(modem_checks)
|
|
+ _alerts(events) + _power(power))
|
|
if cutsheet_view is not None:
|
|
entries += _cutsheet(cutsheet_view)
|
|
entries += _rack(rack_issues or {})
|
|
rank = {'bad': 0, 'warn': 1, 'info': 2}
|
|
out = {g: [] for g in GROUPS}
|
|
for e in entries:
|
|
out[e['group']].append(e)
|
|
for g in GROUPS:
|
|
out[g].sort(key=lambda e: rank[e['severity']])
|
|
return out
|