From 6a717af3f2ec2d4db065cc723719fbf54010e0be Mon Sep 17 00:00:00 2001 From: Jared Vititoe Date: Mon, 16 Nov 2026 12:38:39 -0500 Subject: [PATCH] Internet health from UniFi: Needs attention line and a 24 hour strip UniFi already rates the WAN every 5 minutes (fine, high latency, packet loss, down), the same timeline its Network app charts. The monitor now reads the last day of it from the dashboard API each cycle, read only, instead of Gandalf pinging on its own. - modem.wan_quality() turns the dashboard data into periods, drops and current service latencies; stored as monitor_state.wan_quality. - Needs attention: "Internet: packet loss in N of the last 12 five-minute periods" (2+ lossy periods, red), "high latency" (3+ bad periods, amber), and drops in the last hour. Stale copies are ignored. - WAN / Fiber panel: a 24 hour strip with a legend and latency now, also shown while the modem itself can't be read. Co-Authored-By: Claude Opus 5.5 --- README.md | 20 ++++++++++---- app.py | 3 ++- attention.py | 58 ++++++++++++++++++++++++++++++++++++++-- modem.py | 50 ++++++++++++++++++++++++++++++++++ monitor.py | 26 ++++++++++++++++++ static/app.js | 35 ++++++++++++++++++++++++ static/style.css | 12 +++++++++ tests/test_app.py | 13 +++++++++ tests/test_attention.py | 59 +++++++++++++++++++++++++++++++++++++++++ tests/test_modem.py | 46 ++++++++++++++++++++++++++++++++ tests/test_monitor.py | 30 +++++++++++++++++++++ 11 files changed, 344 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index b4ef7ca..6429aa2 100644 --- a/README.md +++ b/README.md @@ -83,7 +83,7 @@ Two processes share a MariaDB database: | Source | What it provides | |---|---| | **Prometheus** (`10.10.10.48:9090`) | Physical NIC link state + traffic/error rates via `node_exporter` | -| **UniFi API** (`https://10.10.10.1`) | Switch port stats, device status, LLDP neighbor table, PoE data | +| **UniFi API** (`https://10.10.10.1`) | Switch port stats, device status, LLDP neighbor table, PoE data, the WAN health timeline (latency, loss, drops) | | **Pulse Worker** | SSH relay: runs `ethtool` + SFP DOM queries on each Proxmox host | | **Ping** | Reachability for hosts without `node_exporter` (e.g. PBS) | | **AT&T BGW320** (`http://192.168.1.254`) | Fiber light levels, PON and WAN state, WAN counters, scraped read-only from the gateway's status pages (no login) | @@ -108,8 +108,9 @@ Ping-only (no node_exporter): **pbs** (10.10.10.3) ### Dashboard (`/`) - **Needs attention** at the top gathers what every page would flag into two lists, each line linking to where it is fixed: - - *Broken now*: active alerts (one line, with the devices), an outlet that is off but feeds a device, - frozen outlet readings, and cables not where the cutsheet says. + - *Broken now*: internet trouble in the last hour (see below), active alerts (one line, with the + devices), an outlet that is off but feeds a device, frozen outlet readings, and cables not where the + cutsheet says. - *Tidy up*: UniFi port names pointing at the wrong device or at an empty port, live links not on the cutsheet, labels to print, powered outlets with no device assigned, and rack items that overlap or sit outside the rack. @@ -117,7 +118,7 @@ Ping-only (no node_exporter): **pbs** (10.10.10.3) Served by `GET /api/attention` (`attention.py`) and refreshed with the rest of the page. - Device search (see Device pages below) - Active alerts table with severity, target, consecutive failures, ticket link -- WAN / Fiber panel: WAN and PON state, Rx/Tx power, optics temperature, throughput, errors, gateway uptime and last restart, 7-day Rx trend, recent WAN outages +- WAN / Fiber panel: WAN and PON state, Rx/Tx power, optics temperature, throughput, errors, gateway uptime and last restart, a 24 hour internet health strip from UniFi, 7-day Rx trend, recent WAN outages - Hosts with per-NIC link state; NICs marked unused fold into a "+N unused" line - UniFi devices as a compact list (name and model; offline ones turn red with a suppress button) - Recently resolved alerts, folded until opened @@ -501,6 +502,15 @@ Outages can be over between polls: two real ones lasted about two minutes agains So WAN outages and modem restarts also come from the UniFi system log, and a drop in the modem's uptime records a restart even if nothing logged it. A single short outage only appears in the panel's history. +Internet quality (slow or lossy rather than down) comes from UniFi too, not from Gandalf's own pings: the +UDM rates every 5 minutes as fine, high latency, packet loss or down, which is the timeline its Network app +charts. Each cycle the monitor reads the last day of it from the dashboard API (read only) into +`monitor_state.wan_quality`. The panel draws it as a 24 hour strip with the current latency to Cloudflare, +Google and Microsoft. Needs attention says *Internet: packet loss* (red) when 2 or more of the last 12 +periods had loss or downtime, *high latency* (amber) when 3 or more had any problem, and lists drops from +the last hour. These are dashboard lines, not tickets, and they go once the hour is clean. A copy older +than 30 minutes is ignored. + **One ticket per incident.** The modem and WAN reports from one check cycle open a single ticket, led by the most urgent (a P2 "WAN modem unreachable" leads, and an RPDU power-cycle seen in the same cycle is listed under "Also seen in this incident"). A report that comes while the modem has an open alert, or one that @@ -910,7 +920,7 @@ Take LED colors from the model's quick start guide. See the comment at the top o `recovery_notified` once the recovery note is on its ticket (column added automatically at monitor startup) - `suppression_rules`: `active=FALSE` when removed; `expires_at` checked at query time - `monitor_state`: key/value store; `interface_baseline`, `link_stats`, and `modem_stats` are JSON blobs -- `monitor_state` also holds `wan_history` (WAN outages, modem restarts) and `unifi_log_cursor` (how far the UniFi log has been read) +- `monitor_state` also holds `wan_history` (WAN outages, modem restarts), `wan_quality` (UniFi's 5 minute WAN health timeline) and `unifi_log_cursor` (how far the UniFi log has been read) - Database sessions run in UTC (`db.py`), whatever the server's time zone - `modem_samples`: optical readings per poll, kept 90 days; created automatically at monitor startup - `cutsheet_items`, `cutsheet_cables`, `cutsheet_port_labels`, `cutsheet_history`: cutsheet records and diff --git a/app.py b/app.py index 3421f8a..d35d4df 100644 --- a/app.py +++ b/app.py @@ -555,6 +555,7 @@ def api_modem(): logger.error(f'Failed to load modem history: {e}') data['history'] = [] data['wan_history'] = _state_json('wan_history') + data['wan_quality'] = _state_json('wan_quality') or None data['configured'] = True return jsonify(data) @@ -1359,7 +1360,7 @@ def _attention() -> dict: logger.error(f'Cutsheet unavailable for the attention list: {e}') sheet = {'items': (view or {}).get('items') or [], 'cables': (view or {}).get('cables') or []} power = _topology_graph(events, sheet, feeds).get('power') or [] - return attention.build(events, power, view, racks) + return attention.build(events, power, view, racks, _state_json('wan_quality')) def _topology_graph(events: list, sheet: dict, feeds: list) -> dict: diff --git a/attention.py b/attention.py index fc00f65..fee10ff 100644 --- a/attention.py +++ b/attention.py @@ -4,8 +4,11 @@ Two groups: 'broken' is something not working right now; 'tidy' is housekeeping, the network works but the records or names don't match it. Each entry links to the page where it is fixed. Pure functions, so the rules are easy to test. """ +import time from typing import Dict, List, Optional +import modem + GROUPS = ('broken', 'tidy') @@ -100,16 +103,67 @@ def _rack(issues: Dict[str, dict]) -> List[dict]: return out +# UniFi's WAN timeline: judge the last hour, and ignore a copy the monitor stopped refreshing +_WAN_WINDOW_S = 3600 +_WAN_STALE_S = 1800 +_WAN_BAD_PERIODS = 3 # periods with any problem before the list mentions it +_WAN_LOSS_PERIODS = 2 # periods with packet loss before it counts as broken + + +def _ago(seconds: float) -> str: + minutes = max(1, round(seconds / 60)) + return f'{minutes} min ago' if minutes < 60 else f'{round(minutes / 60)} h ago' + + +def _wan(quality: Optional[dict], now: float) -> List[dict]: + if not quality or now - (quality.get('updated') or 0) > _WAN_STALE_S: + return [] + out = [] + periods = quality.get('periods') or [] + # The last hour is the newest 12 periods (each stamped with its start), so the count + # doesn't dip while UniFi is still filling in the current one + newest = periods[-1]['ts'] if periods else now + recent = modem.quality_counts(periods, newest - _WAN_WINDOW_S + modem.QUALITY_PERIOD_S) + total = sum(recent.values()) + bad = total - recent['ok'] + day = modem.quality_counts(periods, now - 86400) + day_bad = sum(day.values()) - day['ok'] + latencies = quality.get('latencies') or {} + detail = (f'UniFi flagged {day_bad} of the last {sum(day.values())} five-minute periods. ' + if day_bad else '') + if latencies: + detail += 'Now: ' + ', '.join(f'{k} {v} ms' for k, v in sorted(latencies.items())) + '. ' + detail += 'See the 24 hour strip on the WAN / Fiber panel.' + lossy = recent['loss'] + recent['down'] + if lossy >= _WAN_LOSS_PERIODS: + out.append(_entry('broken', 'bad', f'Internet: packet loss in {lossy} of the last {total} ' + 'five-minute periods', '#modem-section', detail)) + elif bad >= _WAN_BAD_PERIODS: + out.append(_entry('broken', 'warn', f'Internet: high latency in {bad} of the last {total} ' + 'five-minute periods', '#modem-section', detail)) + drops = [d for d in quality.get('downtime') or [] if d['ts'] >= now - _WAN_WINDOW_S] + if len(drops) == 1: + out.append(_entry('broken', 'warn', f'Internet dropped for {round(drops[0]["seconds"])} s, ' + f'{_ago(now - drops[0]["ts"])}', '#modem-section')) + elif drops: + out.append(_entry('broken', 'warn', f'Internet dropped {len(drops)} times in the last hour, ' + f'{round(sum(d["seconds"] for d in drops))} s in all', + '#modem-section')) + return out + + def build(events: List[dict], power: List[dict], cutsheet_view: Optional[dict], - rack_issues: Optional[Dict[str, dict]] = None) -> dict: + rack_issues: Optional[Dict[str, dict]] = None, wan_quality: Optional[dict] = None, + now: Optional[float] = None) -> dict: """{'broken': [entry], 'tidy': [entry]}, worst first within each group. events: active alerts, annotated with 'is_suppressed'. power: topology.build_power output. cutsheet_view: the cutsheet API view (cables with 'problem', suggestions, port_names, labels), or None when the cutsheet couldn't be loaded. rack_issues: {rack: rack_layout()}. + wan_quality: modem.wan_quality() output as stored by the monitor. now: for tests. Entry: {group, severity: bad|warn|info, text, href, detail}. """ - entries = _alerts(events) + _power(power) + entries = _wan(wan_quality, time.time() if now is None else now) + _alerts(events) + _power(power) if cutsheet_view is not None: entries += _cutsheet(cutsheet_view) entries += _rack(rack_issues or {}) diff --git a/modem.py b/modem.py index 0498f1e..e0b399f 100644 --- a/modem.py +++ b/modem.py @@ -310,3 +310,53 @@ def record_uptime_reset(history: dict, boot_ts: float) -> None: return restarts.append({'at': boot_ts, 'by': None, 'confirmed': True}) del restarts[:-_HISTORY_LIMIT] + + +# --------------------------------------------------------------------------- +# WAN quality: UniFi's own 5 minute health timeline (high latency, packet loss, down), +# the same data its Network app charts. Read only; Gandalf doesn't measure it again. +# --------------------------------------------------------------------------- +QUALITY_STATES = ('ok', 'latency', 'loss', 'down') # mildest to worst +QUALITY_PERIOD_S = 300 + + +def wan_quality(dashboard: dict, now: float) -> Optional[dict]: + """The primary WAN's health timeline from UniFi's aggregated dashboard, or None. + + Returns {'wan', 'periods': [{'ts', 'state'}], 'downtime': [{'ts', 'seconds'}], + 'latencies': {service: ms}, 'updated'}. Each period is 5 minutes; its state is the + worst flag UniFi set for it. + """ + details = ((dashboard or {}).get('wan_history') or {}).get('wan_history_details') or [] + if not details: + return None + primary = next((d for d in details if d.get('network_group') == 'WAN'), details[0]) + periods = [] + for h in primary.get('health_history') or []: + if not isinstance(h.get('timestamp'), (int, float)): + continue + state = ('down' if h.get('wan_downtime') else 'loss' if h.get('packet_loss') + else 'latency' if h.get('high_latency') else 'ok') + periods.append({'ts': h['timestamp'] / 1000, 'state': state}) + periods.sort(key=lambda p: p['ts']) + downtime = sorted( + ({'ts': d['timestamp'] / 1000, 'seconds': d.get('downtime_in_seconds') or 0} + for d in primary.get('downtime_history') or [] if isinstance(d.get('timestamp'), (int, float))), + key=lambda d: d['ts']) + latencies = {} + for w in ((dashboard.get('wan') or {}).get('wan_details') or []): + if w.get('network_group') == primary.get('network_group'): + for s in (w.get('stats') or {}).get('service_latencies') or []: + if s.get('service_name') and isinstance(s.get('latency'), (int, float)): + latencies[s['service_name']] = s['latency'] + return {'wan': primary.get('network_group') or '', 'periods': periods, 'downtime': downtime, + 'latencies': latencies, 'updated': now} + + +def quality_counts(periods: List[dict], since: float) -> Dict[str, int]: + """How many periods starting at or after *since* were in each state.""" + counts = {s: 0 for s in QUALITY_STATES} + for p in periods: + if p['ts'] >= since and p['state'] in counts: + counts[p['state']] += 1 + return counts diff --git a/monitor.py b/monitor.py index 3d50550..f4465dc 100644 --- a/monitor.py +++ b/monitor.py @@ -229,6 +229,19 @@ class UnifiClient: logger.error(f'UniFi system log error: {e}') return None + def get_wan_dashboard(self) -> Optional[dict]: + """UniFi's aggregated dashboard for the last day (WAN health timeline and service + latencies, as the Network app charts them), or None if unavailable. Read only.""" + url = f'{self.base_url}/proxy/network/v2/api/site/{self.site_id}/aggregated-dashboard' + try: + resp = self.session.get(url, headers=self.headers, timeout=20, params={'historySeconds': 86400}) + resp.raise_for_status() + data = resp.json() + return data if isinstance(data, dict) else None + except Exception as e: + logger.error(f'UniFi dashboard error: {e}') + return None + @staticmethod def _update_fields(params: dict) -> dict: """The app and versions of an update entry; nothing for other entries.""" @@ -1950,6 +1963,13 @@ class NetworkMonitor: 'point toward the provider or the modem rather than the fiber.', ) + def _collect_wan_quality(self) -> None: + """Store UniFi's WAN health timeline. If UniFi can't be read, the last copy stays + and goes stale, and the dashboard stops drawing conclusions from it.""" + quality = modem_api.wan_quality(self.unifi.get_wan_dashboard(), time.time()) + if quality is not None: + db.set_state('wan_quality', quality) + def _collect_modem(self, suppressions: list) -> None: """Poll the modem, then store its status and raise any alerts.""" try: @@ -2135,6 +2155,12 @@ class NetworkMonitor: except Exception as e: logger.error(f'Modem collection failed: {e}', exc_info=True) + # Internet quality as UniFi measures it (latency, loss, drops): for the dashboard + try: + self._collect_wan_quality() + except Exception as e: + logger.error(f'WAN quality collection failed: {e}', exc_info=True) + # Note on the tickets of anything that cleared this cycle (or failed to post before) try: self._post_recovery_notes() diff --git a/static/app.js b/static/app.js index 86a6fe1..1429f86 100644 --- a/static/app.js +++ b/static/app.js @@ -385,6 +385,39 @@ function _modemOutages(history) { `; } +const _QUALITY_LABELS = { ok: 'fine', latency: 'high latency', loss: 'packet loss', down: 'down' }; + +// UniFi's own internet health timeline, one block per 5 minutes, worst flag wins +function _modemQuality(q) { + const periods = (q && q.periods) || []; + if (!periods.length) return ''; + const counts = { ok: 0, latency: 0, loss: 0, down: 0 }; + periods.forEach(p => { if (p.state in counts) counts[p.state] += 1; }); + const t0 = periods[0].ts, span = Math.max(periods[periods.length - 1].ts + 300 - t0, 300); + const blocks = periods.map(p => { + const x = (p.ts - t0) / span * 1000, w = Math.max(300 / span * 1000, 1); + const tip = `${_modemWhen(p.ts)}: ${_QUALITY_LABELS[p.state] || p.state}`; + return `` + + `${lt.escHtml(tip)}`; + }).join(''); + const flagged = ['latency', 'loss', 'down'].filter(s => counts[s]) + .map(s => `${_QUALITY_LABELS[s]} ${counts[s]}`).join(', '); + const summary = flagged ? `${flagged} of ${periods.length} periods` : 'no problems'; + const lat = Object.entries(q.latencies || {}).sort() + .map(([k, v]) => `${lt.escHtml(k)} ${lt.escHtml(String(v))} ms`).join(', '); + const legend = ['latency', 'loss', 'down'] + .map(s => `${_QUALITY_LABELS[s]}`).join(''); + return ` +
+
Internet, last 24 hours (from UniFi) ${lt.escHtml(summary)}
+ ${blocks} +
${lt.escHtml(_modemWhen(t0))} + ${legend}now
+ ${lat ? `
Latency now: ${lat}
` : ''} +
`; +} + function updateModem(data) { const section = document.getElementById('modem-section'); const body = document.getElementById('modem-body'); @@ -405,6 +438,7 @@ function updateModem(data) { body.innerHTML = `
Modem status unavailable: ${lt.escHtml(data.error || 'unknown error')} Last attempt ${lt.escHtml(data.updated || '-')}
+ ${_modemQuality(data.wan_quality)} ${_modemChart(data.history)}`; return; } @@ -435,6 +469,7 @@ function updateModem(data) { ${_modemStat('Errors / drops', `${errs} / ${drops}`, errs ? 'lt-text-amber' : '', 'since modem reboot')} ${_modemStat('Uptime', _modemUptime(data.uptime_s), '', _modemLastRestart(data.wan_history))} + ${_modemQuality(data.wan_quality)} ${_modemChart(data.history)} ${_modemOutages(data.wan_history)}
Updated ${lt.escHtml(data.updated || '-')}
`; diff --git a/static/style.css b/static/style.css index 8c26cbd..c1866ed 100644 --- a/static/style.css +++ b/static/style.css @@ -844,6 +844,18 @@ .modem-chart svg { display: block; width: 100%; height: 70px; border: 1px solid var(--border-color); } .modem-chart svg polyline { stroke: var(--terminal-cyan, #0ff); stroke-width: 1.5; } .modem-chart-empty { padding: 0 12px 8px; font-size: .72em; color: var(--text-muted); } +.modem-quality svg { height: 20px; } +.wq-ok { fill: var(--accent-green-dim, rgba(0, 255, 0, .25)); } +.wq-latency { fill: var(--accent-amber); } +.wq-loss { fill: var(--accent-red); } +.wq-down { fill: var(--text-primary, #fff); } +.wq-foot { display: flex; justify-content: space-between; gap: 8px; flex-wrap: wrap; + font-size: .68em; color: var(--text-muted); margin-top: 3px; } +.wq-legend { display: inline-flex; gap: 10px; flex-wrap: wrap; } +.wq-key { display: inline-block; width: 9px; height: 9px; margin-right: 4px; vertical-align: -1px; } +.wq-key.wq-latency { background: var(--accent-amber); } +.wq-key.wq-loss { background: var(--accent-red); } +.wq-key.wq-down { background: var(--text-primary, #fff); } .modem-updated { padding: 0 12px 10px; } /* ── Device search (home page) ────────────────────────────────────── */ diff --git a/tests/test_app.py b/tests/test_app.py index e7eae05..f4a3676 100644 --- a/tests/test_app.py +++ b/tests/test_app.py @@ -820,6 +820,19 @@ class TestAttention: assert resp.status_code == 200 assert set(resp.get_json()) == {'broken', 'tidy'} + def test_internet_quality_from_unifi(self, client): + now = time.time() + quality = {'periods': [{'ts': now - (12 - i) * 300, 'state': 'loss'} for i in range(12)], + 'downtime': [], 'latencies': {}, 'updated': now} + state = {'wan_quality': json.dumps(quality), 'modem_stats': json.dumps({'reachable': True})} + with patch.object(gandalf.db, 'get_state', side_effect=lambda k, d=None: state.get(k, d)), \ + patch.object(gandalf.db, 'get_modem_history', return_value=[]), \ + patch.object(gandalf.db, 'cutsheet_load', side_effect=RuntimeError('db down')): + data = client.get('/api/attention', headers=AUTH).get_json() + modem = client.get('/api/modem', headers=AUTH).get_json() + assert data['broken'][0]['text'] == 'Internet: packet loss in 12 of the last 12 five-minute periods' + assert len(modem['wan_quality']['periods']) == 12 + def test_dashboard_has_the_list_and_says_what_gandalf_stands_for(self, client): html = client.get('/', headers=AUTH).get_data(as_text=True) assert 'id="attention-body"' in html diff --git a/tests/test_attention.py b/tests/test_attention.py index 5703e06..db72519 100644 --- a/tests/test_attention.py +++ b/tests/test_attention.py @@ -84,3 +84,62 @@ class TestRack: def test_nothing_to_report(self): assert attention.build([], [], {}, {}) == {'broken': [], 'tidy': []} + + +NOW = 1_000_000.0 + + +def _quality(states, downtime=(), latencies=None, updated=NOW): + """states: newest last, one per 5 minutes ending at NOW.""" + n = len(states) + return {'periods': [{'ts': NOW - (n - i) * 300, 'state': s} for i, s in enumerate(states)], + 'downtime': [{'ts': NOW - ago, 'seconds': s} for ago, s in downtime], + 'latencies': latencies or {}, 'updated': updated} + + +def _wan(q): + return [e for e in attention.build([], [], None, wan_quality=q, now=NOW)['broken'] + if e['text'].startswith('Internet')] + + +class TestWan: + def test_healthy_says_nothing(self): + assert _wan(_quality(['ok'] * 288)) == [] + + def test_latency_in_the_last_hour(self): + [e] = _wan(_quality(['ok'] * 276 + ['latency', 'ok'] * 3 + ['latency'] * 3 + ['ok'] * 3, + latencies={'google': 26, 'cloudflare': 12})) + assert e['severity'] == 'warn' and e['href'] == '#modem-section' + assert e['text'] == 'Internet: high latency in 6 of the last 12 five-minute periods' + assert 'UniFi flagged 6 of the last 288' in e['detail'] + assert 'Now: cloudflare 12 ms, google 26 ms' in e['detail'] + + def test_a_couple_of_slow_periods_are_not_worth_a_line(self): + assert _wan(_quality(['ok'] * 10 + ['latency'] * 2)) == [] + + def test_old_trouble_has_cleared(self): + assert _wan(_quality(['latency'] * 100 + ['ok'] * 12)) == [] + + def test_packet_loss_is_broken(self): + [e] = _wan(_quality(['ok'] * 9 + ['loss', 'latency', 'down'])) + assert e['severity'] == 'bad' + assert e['text'] == 'Internet: packet loss in 2 of the last 12 five-minute periods' + + def test_a_drop_in_the_last_hour(self): + [e] = _wan(_quality(['ok'] * 12, downtime=[(900, 90), (7200, 30)])) + assert e['text'] == 'Internet dropped for 90 s, 15 min ago' + + def test_several_drops(self): + [e] = _wan(_quality(['ok'] * 12, downtime=[(900, 90), (1800, 30)])) + assert e['text'] == 'Internet dropped 2 times in the last hour, 120 s in all' + + def test_stale_copy_is_ignored(self): + assert _wan(_quality(['loss'] * 12, updated=NOW - 3600)) == [] + + def test_none_is_fine(self): + assert _wan(None) == [] and _wan({}) == [] + + def test_internet_comes_before_alerts_of_the_same_severity(self): + out = attention.build([_alert('x', 'warning')], [], None, + wan_quality=_quality(['latency'] * 12), now=NOW)['broken'] + assert out[0]['text'].startswith('Internet') diff --git a/tests/test_modem.py b/tests/test_modem.py index 5dfbbe8..ec6cfa9 100644 --- a/tests/test_modem.py +++ b/tests/test_modem.py @@ -255,3 +255,49 @@ class TestWanHistory: h = {} modem.record_uptime_reset(h, 5000) assert h['restarts'] == [{'at': 5000, 'by': None, 'confirmed': True}] + + +def _dashboard(health, downtime=(), latencies=None, group='WAN'): + return { + 'wan_history': {'wan_history_details': [{ + 'network_group': group, + 'health_history': [dict({'timestamp': ts * 1000, 'high_latency': False, 'packet_loss': False, + 'wan_downtime': False}, **flags) for ts, flags in health], + 'downtime_history': [{'timestamp': ts * 1000, 'downtime_in_seconds': s} for ts, s in downtime], + }]}, + 'wan': {'wan_details': [{'network_group': group, 'stats': {'service_latencies': [ + {'service_name': k, 'latency': v} for k, v in (latencies or {}).items()]}}]}, + } + + +class TestWanQuality: + def test_worst_flag_wins_and_periods_are_in_time_order(self): + q = modem.wan_quality(_dashboard([ + (600, {'high_latency': True, 'packet_loss': True}), + (0, {}), + (300, {'high_latency': True}), + (900, {'wan_downtime': True, 'packet_loss': True}), + ]), now=1000) + assert [p['state'] for p in q['periods']] == ['ok', 'latency', 'loss', 'down'] + assert [p['ts'] for p in q['periods']] == [0, 300, 600, 900] + assert q['updated'] == 1000 and q['wan'] == 'WAN' + + def test_downtime_and_latencies(self): + q = modem.wan_quality(_dashboard([(0, {})], downtime=[(500, 90)], + latencies={'cloudflare': 12, 'google': 26}), now=1) + assert q['downtime'] == [{'ts': 500, 'seconds': 90}] + assert q['latencies'] == {'cloudflare': 12, 'google': 26} + + def test_prefers_the_primary_wan(self): + d = _dashboard([(0, {'high_latency': True})], group='WAN2') + d['wan_history']['wan_history_details'].append( + _dashboard([(0, {})])['wan_history']['wan_history_details'][0]) + assert modem.wan_quality(d, 0)['periods'][0]['state'] == 'ok' + + @pytest.mark.parametrize('dashboard', [None, {}, {'wan_history': {}}, {'wan_history': {'wan_history_details': []}}]) + def test_missing_history_is_none(self, dashboard): + assert modem.wan_quality(dashboard, 0) is None + + def test_counts_since(self): + periods = [{'ts': 0, 'state': 'latency'}, {'ts': 300, 'state': 'ok'}, {'ts': 600, 'state': 'loss'}] + assert modem.quality_counts(periods, 300) == {'ok': 1, 'latency': 0, 'loss': 1, 'down': 0} diff --git a/tests/test_monitor.py b/tests/test_monitor.py index 118e347..d6aa737 100644 --- a/tests/test_monitor.py +++ b/tests/test_monitor.py @@ -1018,3 +1018,33 @@ class TestSyncOpenTickets: mon._open_tickets_synced -= monitor.OPEN_TICKETS_SYNC_S + 1 mon._sync_open_tickets() assert mon.tickets.open_tickets.call_count == 2 + + +class TestWanQualityCollection: + def test_stores_the_timeline(self, mon, db): + mon.unifi = MagicMock() + mon.unifi.get_wan_dashboard.return_value = {'wan_history': {'wan_history_details': [ + {'network_group': 'WAN', 'health_history': [{'timestamp': 1000, 'high_latency': True}]}]}} + mon._collect_wan_quality() + key, value = db.set_state.call_args.args + assert key == 'wan_quality' + assert value['periods'] == [{'ts': 1.0, 'state': 'latency'}] + + def test_unavailable_keeps_the_last_copy(self, mon, db): + mon.unifi = MagicMock() + mon.unifi.get_wan_dashboard.return_value = None + mon._collect_wan_quality() + assert not db.set_state.called + + def test_client_reads_the_day(self): + c = monitor.UnifiClient({'controller': 'https://udm.test', 'api_key': 'k'}) + c.session.get = MagicMock(return_value=MagicMock(json=lambda: {'wan_history': {}})) + assert c.get_wan_dashboard() == {'wan_history': {}} + url = c.session.get.call_args.args[0] + assert url.endswith('/proxy/network/v2/api/site/default/aggregated-dashboard') + assert c.session.get.call_args.kwargs['params'] == {'historySeconds': 86400} + + def test_client_error_is_none(self): + c = monitor.UnifiClient({'controller': 'https://udm.test', 'api_key': 'k'}) + c.session.get = MagicMock(side_effect=RuntimeError('boom')) + assert c.get_wan_dashboard() is None -- 2.47.3