2026-11-15 14:56:13 -05:00
|
|
|
"""What needs looking at, gathered from every page into one list for the dashboard.
|
|
|
|
|
|
|
|
|
|
Two groups: 'broken' is something not working right now; 'tidy' is housekeeping, where
|
|
|
|
|
the network works but the records or names don't match it. Each entry links to the page
|
|
|
|
|
where it is fixed. Pure functions, so the rules are easy to test.
|
|
|
|
|
"""
|
2026-11-16 12:38:39 -05:00
|
|
|
import time
|
2026-11-15 14:56:13 -05:00
|
|
|
from typing import Dict, List, Optional
|
|
|
|
|
|
2026-11-16 12:38:39 -05:00
|
|
|
import modem
|
|
|
|
|
|
2026-11-15 14:56:13 -05:00
|
|
|
GROUPS = ('broken', 'tidy')
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _plural(n: int, one: str, many: Optional[str] = None) -> str:
|
|
|
|
|
return f'{n} {one if n == 1 else (many or one + "s")}'
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _entry(group: str, severity: str, text: str, href: str, detail: str = '') -> dict:
|
|
|
|
|
return {'group': group, 'severity': severity, 'text': text, 'href': href, 'detail': detail}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _alerts(events: List[dict]) -> List[dict]:
|
|
|
|
|
live = [e for e in events if not e.get('is_suppressed') and e.get('severity') != 'info']
|
|
|
|
|
if not live:
|
|
|
|
|
return []
|
|
|
|
|
critical = sum(1 for e in live if e.get('severity') == 'critical')
|
|
|
|
|
targets = sorted({str(e.get('target_name') or '') for e in live if e.get('target_name')}, key=str.lower)
|
|
|
|
|
text = _plural(len(live), 'active alert')
|
|
|
|
|
if critical:
|
|
|
|
|
text += f' ({critical} critical)'
|
|
|
|
|
shown = ', '.join(targets[:4]) + (f' and {len(targets) - 4} more' if len(targets) > 4 else '')
|
|
|
|
|
return [_entry('broken', 'bad' if critical else 'warn', text, '#events-table-wrap', shown)]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _power(power: List[dict]) -> List[dict]:
|
|
|
|
|
out = []
|
|
|
|
|
for src in power or []:
|
|
|
|
|
name = src.get('node') or ''
|
|
|
|
|
if src.get('stale_since'):
|
|
|
|
|
out.append(_entry('broken', 'warn', f'{name}: outlet watts are frozen', '/topology',
|
|
|
|
|
'UniFi keeps sending the same per-outlet readings; the total is still live. '
|
|
|
|
|
'Restarting the unit usually clears it.'))
|
|
|
|
|
for o in src.get('outlets') or []:
|
|
|
|
|
if not o.get('on') and o.get('target'):
|
|
|
|
|
out.append(_entry('broken', 'bad', f'{name} outlet {o.get("index")} is off, and it feeds {o["target"]}',
|
|
|
|
|
'/topology'))
|
|
|
|
|
idle = []
|
|
|
|
|
for src in power or []:
|
|
|
|
|
for o in src.get('outlets') or []:
|
|
|
|
|
# Drawing real power, but nothing in Gandalf says what is plugged in
|
|
|
|
|
if o.get('on') and not o.get('target') and o.get('match') != 'cleared' and (o.get('power_w') or 0) >= 5:
|
|
|
|
|
idle.append(f'{src.get("node")} {o.get("index")}')
|
|
|
|
|
if idle:
|
|
|
|
|
out.append(_entry('tidy', 'info', f'{_plural(len(idle), "powered outlet")} with no device assigned',
|
|
|
|
|
'/topology', ', '.join(idle[:6]) + (' ...' if len(idle) > 6 else '')))
|
|
|
|
|
return out
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _cutsheet(view: dict) -> List[dict]:
|
|
|
|
|
out = []
|
|
|
|
|
cables = [c for c in view.get('cables') or [] if c.get('status') != 'retired']
|
|
|
|
|
bad = [c for c in cables if c.get('problem')]
|
|
|
|
|
if bad:
|
|
|
|
|
names = ', '.join(c.get('label') or f'#{c["id"]}' for c in bad[:5])
|
|
|
|
|
out.append(_entry('broken', 'bad', f'{_plural(len(bad), "cable")} not where the cutsheet says',
|
|
|
|
|
'/cutsheet?problems=1#cs-cables-section', names))
|
|
|
|
|
|
|
|
|
|
names = view.get('port_names') or []
|
|
|
|
|
wrong = [r for r in names if r.get('kind') in ('wrong', 'moved')]
|
|
|
|
|
if wrong:
|
|
|
|
|
out.append(_entry('tidy', 'warn', f'{_plural(len(wrong), "UniFi port name")} pointing at the wrong device',
|
|
|
|
|
'/cutsheet#cs-portnames-section',
|
|
|
|
|
', '.join(f'{r["switch"]} port {r["port_idx"]} "{r["port_name"]}"' for r in wrong[:3])))
|
|
|
|
|
empty = [r for r in names if r.get('kind') == 'empty']
|
|
|
|
|
if empty:
|
|
|
|
|
out.append(_entry('tidy', 'info', f'{_plural(len(empty), "named port")} with nothing plugged in',
|
|
|
|
|
'/cutsheet#cs-portnames-section',
|
|
|
|
|
', '.join(f'"{r["port_name"]}"' for r in empty[:4])))
|
|
|
|
|
|
|
|
|
|
missing = view.get('suggestions') or []
|
|
|
|
|
if missing:
|
|
|
|
|
out.append(_entry('tidy', 'info', f'{_plural(len(missing), "live link")} not on the cutsheet',
|
|
|
|
|
'/cutsheet#cs-suggest-section'))
|
|
|
|
|
|
|
|
|
|
labels = view.get('labels') or {}
|
|
|
|
|
todo = len(labels.get('cables') or []) + len(labels.get('ports') or [])
|
|
|
|
|
if todo:
|
|
|
|
|
out.append(_entry('tidy', 'info', f'{_plural(todo, "label")} to print and stick on',
|
|
|
|
|
'/cutsheet#cs-labels-section'))
|
|
|
|
|
return out
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _rack(issues: Dict[str, dict]) -> List[dict]:
|
|
|
|
|
out = []
|
|
|
|
|
for rack, r in (issues or {}).items():
|
|
|
|
|
for o in r.get('overlaps') or []:
|
|
|
|
|
out.append(_entry('tidy', 'warn', f'{o["name"]} overlaps {", ".join(o["with"])} at {o["units"]}',
|
|
|
|
|
'/cutsheet#cs-items-section', rack))
|
|
|
|
|
for o in r.get('outside') or []:
|
|
|
|
|
out.append(_entry('tidy', 'warn', f'{o["name"]} at {o["units"]} is outside {rack}',
|
|
|
|
|
'/cutsheet#cs-items-section'))
|
|
|
|
|
return out
|
|
|
|
|
|
|
|
|
|
|
2026-11-16 12:38:39 -05:00
|
|
|
# UniFi's WAN timeline: judge the last hour, and ignore a copy the monitor stopped refreshing
|
|
|
|
|
_WAN_WINDOW_S = 3600
|
|
|
|
|
_WAN_STALE_S = 1800
|
|
|
|
|
_WAN_BAD_PERIODS = 3 # periods with any problem before the list mentions it
|
|
|
|
|
_WAN_LOSS_PERIODS = 2 # periods with packet loss before it counts as broken
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _ago(seconds: float) -> str:
|
|
|
|
|
minutes = max(1, round(seconds / 60))
|
|
|
|
|
return f'{minutes} min ago' if minutes < 60 else f'{round(minutes / 60)} h ago'
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _wan(quality: Optional[dict], now: float) -> List[dict]:
|
|
|
|
|
if not quality or now - (quality.get('updated') or 0) > _WAN_STALE_S:
|
|
|
|
|
return []
|
|
|
|
|
out = []
|
|
|
|
|
periods = quality.get('periods') or []
|
|
|
|
|
# The last hour is the newest 12 periods (each stamped with its start), so the count
|
|
|
|
|
# doesn't dip while UniFi is still filling in the current one
|
|
|
|
|
newest = periods[-1]['ts'] if periods else now
|
|
|
|
|
recent = modem.quality_counts(periods, newest - _WAN_WINDOW_S + modem.QUALITY_PERIOD_S)
|
|
|
|
|
total = sum(recent.values())
|
|
|
|
|
bad = total - recent['ok']
|
|
|
|
|
day = modem.quality_counts(periods, now - 86400)
|
|
|
|
|
day_bad = sum(day.values()) - day['ok']
|
|
|
|
|
latencies = quality.get('latencies') or {}
|
|
|
|
|
detail = (f'UniFi flagged {day_bad} of the last {sum(day.values())} five-minute periods. '
|
|
|
|
|
if day_bad else '')
|
|
|
|
|
if latencies:
|
|
|
|
|
detail += 'Now: ' + ', '.join(f'{k} {v} ms' for k, v in sorted(latencies.items())) + '. '
|
|
|
|
|
detail += 'See the 24 hour strip on the WAN / Fiber panel.'
|
|
|
|
|
lossy = recent['loss'] + recent['down']
|
|
|
|
|
if lossy >= _WAN_LOSS_PERIODS:
|
|
|
|
|
out.append(_entry('broken', 'bad', f'Internet: packet loss in {lossy} of the last {total} '
|
|
|
|
|
'five-minute periods', '#modem-section', detail))
|
|
|
|
|
elif bad >= _WAN_BAD_PERIODS:
|
|
|
|
|
out.append(_entry('broken', 'warn', f'Internet: high latency in {bad} of the last {total} '
|
|
|
|
|
'five-minute periods', '#modem-section', detail))
|
|
|
|
|
drops = [d for d in quality.get('downtime') or [] if d['ts'] >= now - _WAN_WINDOW_S]
|
|
|
|
|
if len(drops) == 1:
|
|
|
|
|
out.append(_entry('broken', 'warn', f'Internet dropped for {round(drops[0]["seconds"])} s, '
|
|
|
|
|
f'{_ago(now - drops[0]["ts"])}', '#modem-section'))
|
|
|
|
|
elif drops:
|
|
|
|
|
out.append(_entry('broken', 'warn', f'Internet dropped {len(drops)} times in the last hour, '
|
|
|
|
|
f'{round(sum(d["seconds"] for d in drops))} s in all',
|
|
|
|
|
'#modem-section'))
|
|
|
|
|
return out
|
|
|
|
|
|
|
|
|
|
|
2026-11-15 14:56:13 -05:00
|
|
|
def build(events: List[dict], power: List[dict], cutsheet_view: Optional[dict],
|
2026-11-16 12:38:39 -05:00
|
|
|
rack_issues: Optional[Dict[str, dict]] = None, wan_quality: Optional[dict] = None,
|
|
|
|
|
now: Optional[float] = None) -> dict:
|
2026-11-15 14:56:13 -05:00
|
|
|
"""{'broken': [entry], 'tidy': [entry]}, worst first within each group.
|
|
|
|
|
|
|
|
|
|
events: active alerts, annotated with 'is_suppressed'. power: topology.build_power output.
|
|
|
|
|
cutsheet_view: the cutsheet API view (cables with 'problem', suggestions, port_names,
|
|
|
|
|
labels), or None when the cutsheet couldn't be loaded. rack_issues: {rack: rack_layout()}.
|
2026-11-16 12:38:39 -05:00
|
|
|
wan_quality: modem.wan_quality() output as stored by the monitor. now: for tests.
|
2026-11-15 14:56:13 -05:00
|
|
|
Entry: {group, severity: bad|warn|info, text, href, detail}.
|
|
|
|
|
"""
|
2026-11-16 12:38:39 -05:00
|
|
|
entries = _wan(wan_quality, time.time() if now is None else now) + _alerts(events) + _power(power)
|
2026-11-15 14:56:13 -05:00
|
|
|
if cutsheet_view is not None:
|
|
|
|
|
entries += _cutsheet(cutsheet_view)
|
|
|
|
|
entries += _rack(rack_issues or {})
|
|
|
|
|
rank = {'bad': 0, 'warn': 1, 'info': 2}
|
|
|
|
|
out = {g: [] for g in GROUPS}
|
|
|
|
|
for e in entries:
|
|
|
|
|
out[e['group']].append(e)
|
|
|
|
|
for g in GROUPS:
|
|
|
|
|
out[g].sort(key=lambda e: rank[e['severity']])
|
|
|
|
|
return out
|