Files

177 lines
8.5 KiB
Python
Raw Permalink Normal View History

"""What needs looking at, gathered from every page into one list for the dashboard.
Two groups: 'broken' is something not working right now; 'tidy' is housekeeping, where
the network works but the records or names don't match it. Each entry links to the page
where it is fixed. Pure functions, so the rules are easy to test.
"""
import time
from typing import Dict, List, Optional
import modem
GROUPS = ('broken', 'tidy')
def _plural(n: int, one: str, many: Optional[str] = None) -> str:
return f'{n} {one if n == 1 else (many or one + "s")}'
def _entry(group: str, severity: str, text: str, href: str, detail: str = '') -> dict:
return {'group': group, 'severity': severity, 'text': text, 'href': href, 'detail': detail}
def _alerts(events: List[dict]) -> List[dict]:
live = [e for e in events if not e.get('is_suppressed') and e.get('severity') != 'info']
if not live:
return []
critical = sum(1 for e in live if e.get('severity') == 'critical')
targets = sorted({str(e.get('target_name') or '') for e in live if e.get('target_name')}, key=str.lower)
text = _plural(len(live), 'active alert')
if critical:
text += f' ({critical} critical)'
shown = ', '.join(targets[:4]) + (f' and {len(targets) - 4} more' if len(targets) > 4 else '')
return [_entry('broken', 'bad' if critical else 'warn', text, '#events-table-wrap', shown)]
def _power(power: List[dict]) -> List[dict]:
out = []
for src in power or []:
name = src.get('node') or ''
if src.get('stale_since'):
out.append(_entry('broken', 'warn', f'{name}: outlet watts are frozen', '/topology',
'UniFi keeps sending the same per-outlet readings; the total is still live. '
'Restarting the unit usually clears it.'))
for o in src.get('outlets') or []:
if not o.get('on') and o.get('target'):
out.append(_entry('broken', 'bad', f'{name} outlet {o.get("index")} is off, and it feeds {o["target"]}',
'/topology'))
idle = []
for src in power or []:
for o in src.get('outlets') or []:
# Drawing real power, but nothing in Gandalf says what is plugged in
if o.get('on') and not o.get('target') and o.get('match') != 'cleared' and (o.get('power_w') or 0) >= 5:
idle.append(f'{src.get("node")} {o.get("index")}')
if idle:
out.append(_entry('tidy', 'info', f'{_plural(len(idle), "powered outlet")} with no device assigned',
'/topology', ', '.join(idle[:6]) + (' ...' if len(idle) > 6 else '')))
return out
def _cutsheet(view: dict) -> List[dict]:
out = []
cables = [c for c in view.get('cables') or [] if c.get('status') != 'retired']
bad = [c for c in cables if c.get('problem')]
if bad:
names = ', '.join(c.get('label') or f'#{c["id"]}' for c in bad[:5])
out.append(_entry('broken', 'bad', f'{_plural(len(bad), "cable")} not where the cutsheet says',
'/cutsheet?problems=1#cs-cables-section', names))
names = view.get('port_names') or []
wrong = [r for r in names if r.get('kind') in ('wrong', 'moved')]
if wrong:
out.append(_entry('tidy', 'warn', f'{_plural(len(wrong), "UniFi port name")} pointing at the wrong device',
'/cutsheet#cs-portnames-section',
', '.join(f'{r["switch"]} port {r["port_idx"]} "{r["port_name"]}"' for r in wrong[:3])))
empty = [r for r in names if r.get('kind') == 'empty']
if empty:
out.append(_entry('tidy', 'info', f'{_plural(len(empty), "named port")} with nothing plugged in',
'/cutsheet#cs-portnames-section',
', '.join(f'"{r["port_name"]}"' for r in empty[:4])))
missing = view.get('suggestions') or []
if missing:
out.append(_entry('tidy', 'info', f'{_plural(len(missing), "live link")} not on the cutsheet',
'/cutsheet#cs-suggest-section'))
labels = view.get('labels') or {}
todo = len(labels.get('cables') or []) + len(labels.get('ports') or [])
if todo:
out.append(_entry('tidy', 'info', f'{_plural(todo, "label")} to print and stick on',
'/cutsheet#cs-labels-section'))
return out
def _rack(issues: Dict[str, dict]) -> List[dict]:
out = []
for rack, r in (issues or {}).items():
for o in r.get('overlaps') or []:
out.append(_entry('tidy', 'warn', f'{o["name"]} overlaps {", ".join(o["with"])} at {o["units"]}',
'/cutsheet#cs-items-section', rack))
for o in r.get('outside') or []:
out.append(_entry('tidy', 'warn', f'{o["name"]} at {o["units"]} is outside {rack}',
'/cutsheet#cs-items-section'))
return out
# UniFi's WAN timeline: judge the last hour, and ignore a copy the monitor stopped refreshing
_WAN_WINDOW_S = 3600
_WAN_STALE_S = 1800
_WAN_BAD_PERIODS = 3 # periods with any problem before the list mentions it
_WAN_LOSS_PERIODS = 2 # periods with packet loss before it counts as broken
def _ago(seconds: float) -> str:
minutes = max(1, round(seconds / 60))
return f'{minutes} min ago' if minutes < 60 else f'{round(minutes / 60)} h ago'
def _wan(quality: Optional[dict], now: float) -> List[dict]:
if not quality or now - (quality.get('updated') or 0) > _WAN_STALE_S:
return []
out = []
periods = quality.get('periods') or []
# The last hour is the newest 12 periods (each stamped with its start), so the count
# doesn't dip while UniFi is still filling in the current one
newest = periods[-1]['ts'] if periods else now
recent = modem.quality_counts(periods, newest - _WAN_WINDOW_S + modem.QUALITY_PERIOD_S)
total = sum(recent.values())
bad = total - recent['ok']
day = modem.quality_counts(periods, now - 86400)
day_bad = sum(day.values()) - day['ok']
latencies = quality.get('latencies') or {}
detail = (f'UniFi flagged {day_bad} of the last {sum(day.values())} five-minute periods. '
if day_bad else '')
if latencies:
detail += 'Now: ' + ', '.join(f'{k} {v} ms' for k, v in sorted(latencies.items())) + '. '
detail += 'See the 24 hour strip on the WAN / Fiber panel.'
lossy = recent['loss'] + recent['down']
if lossy >= _WAN_LOSS_PERIODS:
out.append(_entry('broken', 'bad', f'Internet: packet loss in {lossy} of the last {total} '
'five-minute periods', '#modem-section', detail))
elif bad >= _WAN_BAD_PERIODS:
out.append(_entry('broken', 'warn', f'Internet: high latency in {bad} of the last {total} '
'five-minute periods', '#modem-section', detail))
drops = [d for d in quality.get('downtime') or [] if d['ts'] >= now - _WAN_WINDOW_S]
if len(drops) == 1:
out.append(_entry('broken', 'warn', f'Internet dropped for {round(drops[0]["seconds"])} s, '
f'{_ago(now - drops[0]["ts"])}', '#modem-section'))
elif drops:
out.append(_entry('broken', 'warn', f'Internet dropped {len(drops)} times in the last hour, '
f'{round(sum(d["seconds"] for d in drops))} s in all',
'#modem-section'))
return out
def build(events: List[dict], power: List[dict], cutsheet_view: Optional[dict],
rack_issues: Optional[Dict[str, dict]] = None, wan_quality: Optional[dict] = None,
now: Optional[float] = None) -> dict:
"""{'broken': [entry], 'tidy': [entry]}, worst first within each group.
events: active alerts, annotated with 'is_suppressed'. power: topology.build_power output.
cutsheet_view: the cutsheet API view (cables with 'problem', suggestions, port_names,
labels), or None when the cutsheet couldn't be loaded. rack_issues: {rack: rack_layout()}.
wan_quality: modem.wan_quality() output as stored by the monitor. now: for tests.
Entry: {group, severity: bad|warn|info, text, href, detail}.
"""
entries = _wan(wan_quality, time.time() if now is None else now) + _alerts(events) + _power(power)
if cutsheet_view is not None:
entries += _cutsheet(cutsheet_view)
entries += _rack(rack_issues or {})
rank = {'bad': 0, 'warn': 1, 'info': 2}
out = {g: [] for g in GROUPS}
for e in entries:
out[e['group']].append(e)
for g in GROUPS:
out[g].sort(key=lambda e: rank[e['severity']])
return out