Files
gandalf/attention.py
jaredandClaude Opus 5.5 de75f71bf8
Lint / JS (eslint) (push) Successful in 1m11s
Lint / Python (flake8) (push) Successful in 1m55s
Lint / Notify on failure (push) Skipped
Lint / Deploy (push) Skipped
Security / Python Security (bandit) (push) Successful in 54s
Lint / Python (flake8) (pull_request) Successful in 53s
Test / Python Tests (pytest) (push) Successful in 1m5s
Lint / JS (eslint) (pull_request) Successful in 11s
Lint / Notify on failure (pull_request) Skipped
Lint / Deploy (pull_request) Skipped
Security / Python Security (bandit) (pull_request) Successful in 55s
Test / Python Tests (pytest) (pull_request) Successful in 1m3s
Modem access code: NAT sessions, and a ping from the modem when slow
With modem.access_code in config.json, the monitor logs in to the
BGW320 (md5 of code + nonce, as its own page does) for two read-only
checks, stored in monitor_state.modem_checks:

- NAT sessions in use every 30 minutes; Needs attention warns at 80%.
- While UniFi says the internet is slow, a ping from the modem itself
  to 1.1.1.1 (at most every 10 minutes). Compared with UniFi's latency
  through the modem, it says whether the delay is past the modem (the
  provider), in the modem, or gone. The Internet line in Needs
  attention and the WAN / Fiber panel show the result.

A refused code is reported and not retried for 30 minutes; other
failures pause the checks for 10. The unauthenticated status polling
is unchanged. Tested against the real gateway and a fake one.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-19 18:32:31 -05:00

216 lines
11 KiB
Python

"""What needs looking at, gathered from every page into one list for the dashboard.
Two groups: 'broken' is something not working right now; 'tidy' is housekeeping, where
the network works but the records or names don't match it. Each entry links to the page
where it is fixed. Pure functions, so the rules are easy to test.
"""
import time
from typing import Dict, List, Optional
import modem
GROUPS = ('broken', 'tidy')
def _plural(n: int, one: str, many: Optional[str] = None) -> str:
return f'{n} {one if n == 1 else (many or one + "s")}'
def _entry(group: str, severity: str, text: str, href: str, detail: str = '') -> dict:
return {'group': group, 'severity': severity, 'text': text, 'href': href, 'detail': detail}
def _alerts(events: List[dict]) -> List[dict]:
live = [e for e in events if not e.get('is_suppressed') and e.get('severity') != 'info']
if not live:
return []
critical = sum(1 for e in live if e.get('severity') == 'critical')
targets = sorted({str(e.get('target_name') or '') for e in live if e.get('target_name')}, key=str.lower)
text = _plural(len(live), 'active alert')
if critical:
text += f' ({critical} critical)'
shown = ', '.join(targets[:4]) + (f' and {len(targets) - 4} more' if len(targets) > 4 else '')
return [_entry('broken', 'bad' if critical else 'warn', text, '#events-table-wrap', shown)]
def _power(power: List[dict]) -> List[dict]:
out = []
for src in power or []:
name = src.get('node') or ''
if src.get('stale_since'):
out.append(_entry('broken', 'warn', f'{name}: outlet watts are frozen', '/topology',
'UniFi keeps sending the same per-outlet readings; the total is still live. '
'Restarting the unit usually clears it.'))
for o in src.get('outlets') or []:
if not o.get('on') and o.get('target'):
out.append(_entry('broken', 'bad', f'{name} outlet {o.get("index")} is off, and it feeds {o["target"]}',
'/topology'))
idle = []
for src in power or []:
for o in src.get('outlets') or []:
# Drawing real power, but nothing in Gandalf says what is plugged in
if o.get('on') and not o.get('target') and o.get('match') != 'cleared' and (o.get('power_w') or 0) >= 5:
idle.append(f'{src.get("node")} {o.get("index")}')
if idle:
out.append(_entry('tidy', 'info', f'{_plural(len(idle), "powered outlet")} with no device assigned',
'/topology', ', '.join(idle[:6]) + (' ...' if len(idle) > 6 else '')))
return out
def _cutsheet(view: dict) -> List[dict]:
out = []
cables = [c for c in view.get('cables') or [] if c.get('status') != 'retired']
bad = [c for c in cables if c.get('problem')]
if bad:
names = ', '.join(c.get('label') or f'#{c["id"]}' for c in bad[:5])
out.append(_entry('broken', 'bad', f'{_plural(len(bad), "cable")} not where the cutsheet says',
'/cutsheet?problems=1#cs-cables-section', names))
names = view.get('port_names') or []
wrong = [r for r in names if r.get('kind') in ('wrong', 'moved')]
if wrong:
out.append(_entry('tidy', 'warn', f'{_plural(len(wrong), "UniFi port name")} pointing at the wrong device',
'/cutsheet#cs-portnames-section',
', '.join(f'{r["switch"]} port {r["port_idx"]} "{r["port_name"]}"' for r in wrong[:3])))
empty = [r for r in names if r.get('kind') == 'empty']
if empty:
out.append(_entry('tidy', 'info', f'{_plural(len(empty), "named port")} with nothing plugged in',
'/cutsheet#cs-portnames-section',
', '.join(f'"{r["port_name"]}"' for r in empty[:4])))
missing = view.get('suggestions') or []
if missing:
out.append(_entry('tidy', 'info', f'{_plural(len(missing), "live link")} not on the cutsheet',
'/cutsheet#cs-suggest-section'))
labels = view.get('labels') or {}
todo = len(labels.get('cables') or []) + len(labels.get('ports') or [])
if todo:
out.append(_entry('tidy', 'info', f'{_plural(todo, "label")} to print and stick on',
'/cutsheet#cs-labels-section'))
return out
def _rack(issues: Dict[str, dict]) -> List[dict]:
out = []
for rack, r in (issues or {}).items():
for o in r.get('overlaps') or []:
out.append(_entry('tidy', 'warn', f'{o["name"]} overlaps {", ".join(o["with"])} at {o["units"]}',
'/cutsheet#cs-items-section', rack))
for o in r.get('outside') or []:
out.append(_entry('tidy', 'warn', f'{o["name"]} at {o["units"]} is outside {rack}',
'/cutsheet#cs-items-section'))
return out
# UniFi's WAN timeline: judge the last hour, and ignore a copy the monitor stopped refreshing
_WAN_WINDOW_S = 3600
_WAN_STALE_S = 1800
_WAN_BAD_PERIODS = 3 # periods with any problem before the list mentions it
_WAN_LOSS_PERIODS = 2 # periods with packet loss before it counts as broken
def _ago(seconds: float) -> str:
minutes = max(1, round(seconds / 60))
return f'{minutes} min ago' if minutes < 60 else f'{round(minutes / 60)} h ago'
_NAT_FULL = 0.8 # share of the gateway's NAT sessions in use before it's worth a line
def _modem_ping_note(checks: Optional[dict], now: float) -> str:
"""One sentence on the latest ping from the modem itself, if it ran in the last hour."""
ping = (checks or {}).get('ping') or {}
if not ping or now - (ping.get('at') or 0) > _WAN_WINDOW_S:
return ''
when, avg, ours = _ago(now - ping['at']), ping.get('avg_ms'), ping.get('ours_ms')
theirs = (f'{round(avg)} ms' if avg is not None else 'no reply') + (
f', {ping["loss_pct"]}% lost' if ping.get('loss_pct') else '')
ours_txt = f'; ours through it {round(ours)} ms' if ours is not None else ''
verdict = {
'upstream': "so the delay is past the modem, on the provider's side",
'gateway': 'so the delay is in the modem, or between it and the router',
'clear': 'both fine by then',
}.get(ping.get('verdict'), '')
return f"Modem's own ping {when}: {theirs}{ours_txt}, {verdict}. "
def _modem_checks(checks: Optional[dict]) -> List[dict]:
out = []
nat = (checks or {}).get('nat') or {}
if nat.get('available') and nat.get('in_use', 0) / nat['available'] >= _NAT_FULL:
pct = round(100 * nat['in_use'] / nat['available'])
out.append(_entry('broken', 'warn', f'Modem NAT table {pct}% full',
'#modem-section', f'{nat["in_use"]:,} of {nat["available"]:,} sessions in use. '
'When it fills, new connections stall or fail.'))
error = (checks or {}).get('error') or {}
if error.get('login'):
out.append(_entry('tidy', 'warn', 'Modem login failed: check modem.access_code in config.json',
'#modem-section', 'Gandalf waits 30 minutes before trying again.'))
return out
def _wan(quality: Optional[dict], now: float, checks: Optional[dict] = None) -> List[dict]:
if not quality or now - (quality.get('updated') or 0) > _WAN_STALE_S:
return []
out = []
periods = quality.get('periods') or []
# The last hour is the newest 12 periods (each stamped with its start), so the count
# doesn't dip while UniFi is still filling in the current one
newest = periods[-1]['ts'] if periods else now
recent = modem.quality_counts(periods, newest - _WAN_WINDOW_S + modem.QUALITY_PERIOD_S)
total = sum(recent.values())
bad = total - recent['ok']
day = modem.quality_counts(periods, now - 86400)
day_bad = sum(day.values()) - day['ok']
latencies = quality.get('latencies') or {}
detail = (f'UniFi flagged {day_bad} of the last {sum(day.values())} five-minute periods. '
if day_bad else '')
if latencies:
detail += 'Now: ' + ', '.join(f'{k} {v} ms' for k, v in sorted(latencies.items())) + '. '
detail += _modem_ping_note(checks, now)
detail += 'See the 24 hour strip on the WAN / Fiber panel.'
lossy = recent['loss'] + recent['down']
if lossy >= _WAN_LOSS_PERIODS:
out.append(_entry('broken', 'bad', f'Internet: packet loss in {lossy} of the last {total} '
'five-minute periods', '#modem-section', detail))
elif bad >= _WAN_BAD_PERIODS:
out.append(_entry('broken', 'warn', f'Internet: high latency in {bad} of the last {total} '
'five-minute periods', '#modem-section', detail))
drops = [d for d in quality.get('downtime') or [] if d['ts'] >= now - _WAN_WINDOW_S]
if len(drops) == 1:
out.append(_entry('broken', 'warn', f'Internet dropped for {round(drops[0]["seconds"])} s, '
f'{_ago(now - drops[0]["ts"])}', '#modem-section'))
elif drops:
out.append(_entry('broken', 'warn', f'Internet dropped {len(drops)} times in the last hour, '
f'{round(sum(d["seconds"] for d in drops))} s in all',
'#modem-section'))
return out
def build(events: List[dict], power: List[dict], cutsheet_view: Optional[dict],
rack_issues: Optional[Dict[str, dict]] = None, wan_quality: Optional[dict] = None,
now: Optional[float] = None, modem_checks: Optional[dict] = None) -> dict:
"""{'broken': [entry], 'tidy': [entry]}, worst first within each group.
events: active alerts, annotated with 'is_suppressed'. power: topology.build_power output.
cutsheet_view: the cutsheet API view (cables with 'problem', suggestions, port_names,
labels), or None when the cutsheet couldn't be loaded. rack_issues: {rack: rack_layout()}.
wan_quality: modem.wan_quality() output as stored by the monitor. modem_checks: the
monitor's logged-in modem checks (NAT sessions, ping from the modem). now: for tests.
Entry: {group, severity: bad|warn|info, text, href, detail}.
"""
now = time.time() if now is None else now
entries = (_wan(wan_quality, now, modem_checks) + _modem_checks(modem_checks)
+ _alerts(events) + _power(power))
if cutsheet_view is not None:
entries += _cutsheet(cutsheet_view)
entries += _rack(rack_issues or {})
rank = {'bad': 0, 'warn': 1, 'info': 2}
out = {g: [] for g in GROUPS}
for e in entries:
out[e['group']].append(e)
for g in GROUPS:
out[g].sort(key=lambda e: rank[e['severity']])
return out