Internet health from UniFi: Needs attention line and a 24 hour strip
Lint / JS (eslint) (push) Successful in 1m10s
Lint / Python (flake8) (push) Successful in 2m31s
Lint / Notify on failure (push) Skipped
Lint / Deploy (push) Skipped
Security / Python Security (bandit) (push) Successful in 1m26s
Lint / Python (flake8) (pull_request) Successful in 1m4s
Test / Python Tests (pytest) (push) Successful in 1m13s
Lint / JS (eslint) (pull_request) Successful in 7s
Lint / Notify on failure (pull_request) Skipped
Lint / Deploy (pull_request) Skipped
Security / Python Security (bandit) (pull_request) Successful in 1m0s
Test / Python Tests (pytest) (pull_request) Successful in 1m3s
Lint / JS (eslint) (push) Successful in 1m10s
Lint / Python (flake8) (push) Successful in 2m31s
Lint / Notify on failure (push) Skipped
Lint / Deploy (push) Skipped
Security / Python Security (bandit) (push) Successful in 1m26s
Lint / Python (flake8) (pull_request) Successful in 1m4s
Test / Python Tests (pytest) (push) Successful in 1m13s
Lint / JS (eslint) (pull_request) Successful in 7s
Lint / Notify on failure (pull_request) Skipped
Lint / Deploy (pull_request) Skipped
Security / Python Security (bandit) (pull_request) Successful in 1m0s
Test / Python Tests (pytest) (pull_request) Successful in 1m3s
UniFi already rates the WAN every 5 minutes (fine, high latency, packet loss, down), the same timeline its Network app charts. The monitor now reads the last day of it from the dashboard API each cycle, read only, instead of Gandalf pinging on its own. - modem.wan_quality() turns the dashboard data into periods, drops and current service latencies; stored as monitor_state.wan_quality. - Needs attention: "Internet: packet loss in N of the last 12 five-minute periods" (2+ lossy periods, red), "high latency" (3+ bad periods, amber), and drops in the last hour. Stale copies are ignored. - WAN / Fiber panel: a 24 hour strip with a legend and latency now, also shown while the modem itself can't be read. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
f1355c4bd8
commit
6a717af3f2
11 files changed
+344
-8
No files matched your search
@@ -820,6 +820,19 @@ class TestAttention:
|
||||
assert resp.status_code == 200
|
||||
assert set(resp.get_json()) == {'broken', 'tidy'}
|
||||
|
||||
def test_internet_quality_from_unifi(self, client):
|
||||
now = time.time()
|
||||
quality = {'periods': [{'ts': now - (12 - i) * 300, 'state': 'loss'} for i in range(12)],
|
||||
'downtime': [], 'latencies': {}, 'updated': now}
|
||||
state = {'wan_quality': json.dumps(quality), 'modem_stats': json.dumps({'reachable': True})}
|
||||
with patch.object(gandalf.db, 'get_state', side_effect=lambda k, d=None: state.get(k, d)), \
|
||||
patch.object(gandalf.db, 'get_modem_history', return_value=[]), \
|
||||
patch.object(gandalf.db, 'cutsheet_load', side_effect=RuntimeError('db down')):
|
||||
data = client.get('/api/attention', headers=AUTH).get_json()
|
||||
modem = client.get('/api/modem', headers=AUTH).get_json()
|
||||
assert data['broken'][0]['text'] == 'Internet: packet loss in 12 of the last 12 five-minute periods'
|
||||
assert len(modem['wan_quality']['periods']) == 12
|
||||
|
||||
def test_dashboard_has_the_list_and_says_what_gandalf_stands_for(self, client):
|
||||
html = client.get('/', headers=AUTH).get_data(as_text=True)
|
||||
assert 'id="attention-body"' in html
|
||||
|
||||
@@ -84,3 +84,62 @@ class TestRack:
|
||||
|
||||
def test_nothing_to_report(self):
|
||||
assert attention.build([], [], {}, {}) == {'broken': [], 'tidy': []}
|
||||
|
||||
|
||||
NOW = 1_000_000.0
|
||||
|
||||
|
||||
def _quality(states, downtime=(), latencies=None, updated=NOW):
|
||||
"""states: newest last, one per 5 minutes ending at NOW."""
|
||||
n = len(states)
|
||||
return {'periods': [{'ts': NOW - (n - i) * 300, 'state': s} for i, s in enumerate(states)],
|
||||
'downtime': [{'ts': NOW - ago, 'seconds': s} for ago, s in downtime],
|
||||
'latencies': latencies or {}, 'updated': updated}
|
||||
|
||||
|
||||
def _wan(q):
|
||||
return [e for e in attention.build([], [], None, wan_quality=q, now=NOW)['broken']
|
||||
if e['text'].startswith('Internet')]
|
||||
|
||||
|
||||
class TestWan:
|
||||
def test_healthy_says_nothing(self):
|
||||
assert _wan(_quality(['ok'] * 288)) == []
|
||||
|
||||
def test_latency_in_the_last_hour(self):
|
||||
[e] = _wan(_quality(['ok'] * 276 + ['latency', 'ok'] * 3 + ['latency'] * 3 + ['ok'] * 3,
|
||||
latencies={'google': 26, 'cloudflare': 12}))
|
||||
assert e['severity'] == 'warn' and e['href'] == '#modem-section'
|
||||
assert e['text'] == 'Internet: high latency in 6 of the last 12 five-minute periods'
|
||||
assert 'UniFi flagged 6 of the last 288' in e['detail']
|
||||
assert 'Now: cloudflare 12 ms, google 26 ms' in e['detail']
|
||||
|
||||
def test_a_couple_of_slow_periods_are_not_worth_a_line(self):
|
||||
assert _wan(_quality(['ok'] * 10 + ['latency'] * 2)) == []
|
||||
|
||||
def test_old_trouble_has_cleared(self):
|
||||
assert _wan(_quality(['latency'] * 100 + ['ok'] * 12)) == []
|
||||
|
||||
def test_packet_loss_is_broken(self):
|
||||
[e] = _wan(_quality(['ok'] * 9 + ['loss', 'latency', 'down']))
|
||||
assert e['severity'] == 'bad'
|
||||
assert e['text'] == 'Internet: packet loss in 2 of the last 12 five-minute periods'
|
||||
|
||||
def test_a_drop_in_the_last_hour(self):
|
||||
[e] = _wan(_quality(['ok'] * 12, downtime=[(900, 90), (7200, 30)]))
|
||||
assert e['text'] == 'Internet dropped for 90 s, 15 min ago'
|
||||
|
||||
def test_several_drops(self):
|
||||
[e] = _wan(_quality(['ok'] * 12, downtime=[(900, 90), (1800, 30)]))
|
||||
assert e['text'] == 'Internet dropped 2 times in the last hour, 120 s in all'
|
||||
|
||||
def test_stale_copy_is_ignored(self):
|
||||
assert _wan(_quality(['loss'] * 12, updated=NOW - 3600)) == []
|
||||
|
||||
def test_none_is_fine(self):
|
||||
assert _wan(None) == [] and _wan({}) == []
|
||||
|
||||
def test_internet_comes_before_alerts_of_the_same_severity(self):
|
||||
out = attention.build([_alert('x', 'warning')], [], None,
|
||||
wan_quality=_quality(['latency'] * 12), now=NOW)['broken']
|
||||
assert out[0]['text'].startswith('Internet')
|
||||
@@ -255,3 +255,49 @@ class TestWanHistory:
|
||||
h = {}
|
||||
modem.record_uptime_reset(h, 5000)
|
||||
assert h['restarts'] == [{'at': 5000, 'by': None, 'confirmed': True}]
|
||||
|
||||
|
||||
def _dashboard(health, downtime=(), latencies=None, group='WAN'):
|
||||
return {
|
||||
'wan_history': {'wan_history_details': [{
|
||||
'network_group': group,
|
||||
'health_history': [dict({'timestamp': ts * 1000, 'high_latency': False, 'packet_loss': False,
|
||||
'wan_downtime': False}, **flags) for ts, flags in health],
|
||||
'downtime_history': [{'timestamp': ts * 1000, 'downtime_in_seconds': s} for ts, s in downtime],
|
||||
}]},
|
||||
'wan': {'wan_details': [{'network_group': group, 'stats': {'service_latencies': [
|
||||
{'service_name': k, 'latency': v} for k, v in (latencies or {}).items()]}}]},
|
||||
}
|
||||
|
||||
|
||||
class TestWanQuality:
|
||||
def test_worst_flag_wins_and_periods_are_in_time_order(self):
|
||||
q = modem.wan_quality(_dashboard([
|
||||
(600, {'high_latency': True, 'packet_loss': True}),
|
||||
(0, {}),
|
||||
(300, {'high_latency': True}),
|
||||
(900, {'wan_downtime': True, 'packet_loss': True}),
|
||||
]), now=1000)
|
||||
assert [p['state'] for p in q['periods']] == ['ok', 'latency', 'loss', 'down']
|
||||
assert [p['ts'] for p in q['periods']] == [0, 300, 600, 900]
|
||||
assert q['updated'] == 1000 and q['wan'] == 'WAN'
|
||||
|
||||
def test_downtime_and_latencies(self):
|
||||
q = modem.wan_quality(_dashboard([(0, {})], downtime=[(500, 90)],
|
||||
latencies={'cloudflare': 12, 'google': 26}), now=1)
|
||||
assert q['downtime'] == [{'ts': 500, 'seconds': 90}]
|
||||
assert q['latencies'] == {'cloudflare': 12, 'google': 26}
|
||||
|
||||
def test_prefers_the_primary_wan(self):
|
||||
d = _dashboard([(0, {'high_latency': True})], group='WAN2')
|
||||
d['wan_history']['wan_history_details'].append(
|
||||
_dashboard([(0, {})])['wan_history']['wan_history_details'][0])
|
||||
assert modem.wan_quality(d, 0)['periods'][0]['state'] == 'ok'
|
||||
|
||||
@pytest.mark.parametrize('dashboard', [None, {}, {'wan_history': {}}, {'wan_history': {'wan_history_details': []}}])
|
||||
def test_missing_history_is_none(self, dashboard):
|
||||
assert modem.wan_quality(dashboard, 0) is None
|
||||
|
||||
def test_counts_since(self):
|
||||
periods = [{'ts': 0, 'state': 'latency'}, {'ts': 300, 'state': 'ok'}, {'ts': 600, 'state': 'loss'}]
|
||||
assert modem.quality_counts(periods, 300) == {'ok': 1, 'latency': 0, 'loss': 1, 'down': 0}
|
||||
@@ -1018,3 +1018,33 @@ class TestSyncOpenTickets:
|
||||
mon._open_tickets_synced -= monitor.OPEN_TICKETS_SYNC_S + 1
|
||||
mon._sync_open_tickets()
|
||||
assert mon.tickets.open_tickets.call_count == 2
|
||||
|
||||
|
||||
class TestWanQualityCollection:
|
||||
def test_stores_the_timeline(self, mon, db):
|
||||
mon.unifi = MagicMock()
|
||||
mon.unifi.get_wan_dashboard.return_value = {'wan_history': {'wan_history_details': [
|
||||
{'network_group': 'WAN', 'health_history': [{'timestamp': 1000, 'high_latency': True}]}]}}
|
||||
mon._collect_wan_quality()
|
||||
key, value = db.set_state.call_args.args
|
||||
assert key == 'wan_quality'
|
||||
assert value['periods'] == [{'ts': 1.0, 'state': 'latency'}]
|
||||
|
||||
def test_unavailable_keeps_the_last_copy(self, mon, db):
|
||||
mon.unifi = MagicMock()
|
||||
mon.unifi.get_wan_dashboard.return_value = None
|
||||
mon._collect_wan_quality()
|
||||
assert not db.set_state.called
|
||||
|
||||
def test_client_reads_the_day(self):
|
||||
c = monitor.UnifiClient({'controller': 'https://udm.test', 'api_key': 'k'})
|
||||
c.session.get = MagicMock(return_value=MagicMock(json=lambda: {'wan_history': {}}))
|
||||
assert c.get_wan_dashboard() == {'wan_history': {}}
|
||||
url = c.session.get.call_args.args[0]
|
||||
assert url.endswith('/proxy/network/v2/api/site/default/aggregated-dashboard')
|
||||
assert c.session.get.call_args.kwargs['params'] == {'historySeconds': 86400}
|
||||
|
||||
def test_client_error_is_none(self):
|
||||
c = monitor.UnifiClient({'controller': 'https://udm.test', 'api_key': 'k'})
|
||||
c.session.get = MagicMock(side_effect=RuntimeError('boom'))
|
||||
assert c.get_wan_dashboard() is None
|
||||
Reference in new issue
Block a user