WAN tickets: one per incident, notice UniFi updates, maintenance button
Lint / JS (eslint) (push) Successful in 42s
Lint / Python (flake8) (push) Successful in 1m35s
Lint / Notify on failure (push) Skipped
Lint / Deploy (push) Skipped
Security / Python Security (bandit) (push) Successful in 1m3s
Test / Python Tests (pytest) (push) Successful in 1m5s
Lint / Python (flake8) (pull_request) Successful in 58s
Lint / JS (eslint) (pull_request) Successful in 8s
Lint / Notify on failure (pull_request) Skipped
Lint / Deploy (pull_request) Skipped
Security / Python Security (bandit) (pull_request) Successful in 1m1s
Test / Python Tests (pytest) (pull_request) Successful in 1m10s
Lint / JS (eslint) (push) Successful in 42s
Lint / Python (flake8) (push) Successful in 1m35s
Lint / Notify on failure (push) Skipped
Lint / Deploy (push) Skipped
Security / Python Security (bandit) (push) Successful in 1m3s
Test / Python Tests (pytest) (push) Successful in 1m5s
Lint / Python (flake8) (pull_request) Successful in 58s
Lint / JS (eslint) (pull_request) Successful in 8s
Lint / Notify on failure (pull_request) Skipped
Lint / Deploy (pull_request) Skipped
Security / Python Security (bandit) (pull_request) Successful in 1m1s
Test / Python Tests (pytest) (pull_request) Successful in 1m10s
A UniFi Network update on 2026-11-12 dropped the UDM's WAN, Smart Power power-cycled the modem, and Gandalf opened two tickets for the one event. - Modem and WAN reports from one check cycle open a single ticket, led by the most urgent, with the rest listed under "Also seen in this incident". A report while the modem has an open alert, or one that ended under 30 minutes ago, becomes a comment on that ticket instead. - UniFi update log entries (NETWORK_UPDATED, FIRMWARE_UPDATED_V2 and the like) add a note to modem/WAN tickets from 20 minutes before the update to 15 after, and hold new modem/WAN tickets for 15 minutes. - A Maintenance button for admins starts a global suppression for 30 min to 4 hours, with a banner on every page and End now. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -610,3 +610,41 @@ class TestUnusedInterfaces:
|
||||
gandalf.db.get_unused_interfaces.return_value = {'large1': {'enp36s0': {'reason': 'spare'}}}
|
||||
data = client.get('/api/device/large1', headers=AUTH).get_json()
|
||||
assert data['unused_interfaces'] == {'enp36s0': {'reason': 'spare'}}
|
||||
|
||||
|
||||
class TestMaintenance:
|
||||
SUP = {'id': 7, 'target_type': 'all', 'target_name': '', 'reason': 'UniFi Network update',
|
||||
'suppressed_by': 'jared', 'expires_at': '2026-11-12T20:00:00Z'}
|
||||
|
||||
def test_no_banner_and_a_start_button_by_default(self, client):
|
||||
html = client.get('/', headers=AUTH).get_data(as_text=True)
|
||||
assert 'maint-banner' not in html
|
||||
assert 'data-action="start-maintenance"' in html and 'id="maint-modal"' in html
|
||||
|
||||
def test_banner_on_every_page_while_on(self, client):
|
||||
gandalf.db.get_active_suppressions.return_value = [
|
||||
{'id': 3, 'target_type': 'host', 'target_name': 'large1', 'reason': 'x', 'expires_at': None},
|
||||
self.SUP,
|
||||
]
|
||||
for page in ('/', '/events', '/rack'):
|
||||
html = client.get(page, headers=AUTH).get_data(as_text=True)
|
||||
assert 'class="maint-banner"' in html, page
|
||||
assert 'datetime="2026-11-12T20:00:00Z"' in html and 'UniFi Network update' in html
|
||||
assert 'data-sup-id="7"' in html and 'data-action="start-maintenance"' not in html
|
||||
|
||||
def test_open_ended_maintenance_wins(self, client):
|
||||
gandalf.db.get_active_suppressions.return_value = [
|
||||
self.SUP, dict(self.SUP, id=9, expires_at=None, reason='Rewiring')]
|
||||
html = client.get('/', headers=AUTH).get_data(as_text=True)
|
||||
assert 'until it is ended' in html and 'data-sup-id="9"' in html
|
||||
|
||||
def test_viewers_see_the_banner_without_buttons(self, client):
|
||||
gandalf.db.get_active_suppressions.return_value = [self.SUP]
|
||||
viewer = dict(AUTH, **{'Remote-Groups': 'staff'})
|
||||
with patch.object(gandalf, '_cfg', dict(CONFIG, auth=dict(CONFIG['auth'], allowed_groups=['admin', 'staff']))):
|
||||
html = client.get('/', headers=viewer).get_data(as_text=True)
|
||||
assert 'class="maint-banner"' in html and 'data-action="end-maintenance"' not in html
|
||||
|
||||
def test_a_db_error_means_no_banner(self, client):
|
||||
gandalf.db.get_active_suppressions.side_effect = RuntimeError('db down')
|
||||
assert gandalf.inject_maintenance() == {'maintenance': None}
|
||||
|
||||
@@ -31,6 +31,8 @@ def db():
|
||||
mock_db.cutsheet_load.return_value = {'items': [], 'cables': [], 'port_labels': []}
|
||||
mock_db.get_pending_recovery_notes.return_value = []
|
||||
mock_db.get_device_link.return_value = ''
|
||||
mock_db.get_incident_ticket.return_value = None
|
||||
mock_db.get_ticket_ids_between.return_value = []
|
||||
yield mock_db
|
||||
|
||||
|
||||
@@ -381,6 +383,47 @@ class TestWanLog:
|
||||
mon._process_wan_log([])
|
||||
assert not mon.tickets.create.called
|
||||
|
||||
def test_unifi_update_is_not_a_wan_event(self, mon, db):
|
||||
now = time.time()
|
||||
self._state(db, unifi_log_cursor=str(int(now * 1000)))
|
||||
self._mon(mon, [{'key': 'NETWORK_UPDATED', 'ts': now, 'device': '', 'wan': '',
|
||||
'app': 'UniFi Network', 'previous': '11.0.97', 'version': '11.0.99'}])
|
||||
mon._process_wan_log([])
|
||||
assert not mon.tickets.create.called
|
||||
saved = dict((c.args[0], c.args[1]) for c in db.set_state.call_args_list)
|
||||
assert saved[monitor.UNIFI_UPDATE_STATE]['label'] == 'UniFi Network 11.0.97 to 11.0.99'
|
||||
assert saved['wan_history'] == {'outages': [], 'restarts': []}
|
||||
|
||||
def test_update_comments_on_tickets_opened_just_before(self, mon, db):
|
||||
now = time.time()
|
||||
self._state(db, unifi_log_cursor=str(int(now * 1000)))
|
||||
db.get_ticket_ids_between.return_value = ['207521068']
|
||||
self._mon(mon, [{'key': 'NETWORK_UPDATED', 'ts': now, 'device': '', 'wan': '',
|
||||
'app': 'UniFi Network', 'previous': '11.0.97', 'version': '11.0.99'}])
|
||||
mon._process_wan_log([])
|
||||
start, end = db.get_ticket_ids_between.call_args.args[1:]
|
||||
assert (end - start).total_seconds() == monitor.UPDATE_LOOKBACK_S + monitor.UPDATE_HOLD_S
|
||||
tid, text = mon.tickets.comment.call_args.args
|
||||
assert tid == '207521068' and 'UniFi Network 11.0.97 to 11.0.99' in text
|
||||
|
||||
def test_update_in_the_same_read_holds_the_power_cycle_ticket(self, mon, db):
|
||||
now = time.time()
|
||||
state = {'unifi_log_cursor': str(int(now * 1000))}
|
||||
db.get_state.side_effect = lambda key, default=None: state.get(key, default)
|
||||
db.set_state.side_effect = lambda key, value: state.__setitem__(
|
||||
key, json.dumps(value) if isinstance(value, dict) else value)
|
||||
self._mon(mon, [
|
||||
{'key': modem_api.LOG_MODEM_RESTARTED, 'ts': now - 600, 'device': 'RPDU', 'wan': ''},
|
||||
{'key': 'NETWORK_UPDATED', 'ts': now, 'device': '', 'wan': '', 'app': 'UniFi Network'},
|
||||
])
|
||||
mon._process_wan_log([])
|
||||
assert not mon.tickets.create.called
|
||||
db.resolve_event.assert_called_with('modem_power_cycled', 'att-modem', '')
|
||||
|
||||
def test_hold_ends(self, mon, db):
|
||||
self._state(db, unifi_update=json.dumps({'label': 'x', 'hold_until': time.time() - 1}))
|
||||
assert mon._update_hold() is None
|
||||
|
||||
def test_uptime_going_backwards_records_a_restart(self, mon, db):
|
||||
self._state(db, modem_stats=json.dumps({'uptime_s': 5000}))
|
||||
mon._note_modem_restart({'uptime_s': 60})
|
||||
@@ -393,6 +436,55 @@ class TestWanLog:
|
||||
assert not db.set_state.called
|
||||
|
||||
|
||||
class TestWanIncident:
|
||||
"""Last night's sequence: the RPDU power-cycles the modem in the cycle the modem goes unreachable."""
|
||||
|
||||
def _cycle(self, mon, db):
|
||||
db.upsert_event.side_effect = [(30, True, 1, None), (29, False, 2, None)]
|
||||
mon.modem.poll.side_effect = modem_api.ModemUnreachable('Network is unreachable')
|
||||
with mon._wan_batch():
|
||||
mon._one_shot_ticket([], 'modem_power_cycled', 'RPDU power-cycled the modem.',
|
||||
'Modem auto power-cycled after internet loss', '3', 'Check the WAN panel.')
|
||||
mon._collect_modem([])
|
||||
|
||||
def test_one_ticket_led_by_the_most_urgent(self, mon, db):
|
||||
self._cycle(mon, db)
|
||||
assert mon.tickets.create.call_count == 1
|
||||
title, desc = mon.tickets.create.call_args.args
|
||||
assert 'WAN modem unreachable' in title
|
||||
assert mon.tickets.create.call_args.kwargs['priority'] == '2'
|
||||
assert '### Also seen in this incident' in desc and 'power-cycled' in desc
|
||||
assert sorted(c.args for c in db.set_ticket_id.call_args_list) == [(29, '123'), (30, '123')]
|
||||
|
||||
def test_recent_incident_gets_a_comment_instead(self, mon, db):
|
||||
db.get_incident_ticket.return_value = '555'
|
||||
mon.tickets.comment.return_value = True
|
||||
self._cycle(mon, db)
|
||||
assert not mon.tickets.create.called
|
||||
assert [c.args[0] for c in mon.tickets.comment.call_args_list] == ['555', '555']
|
||||
assert 'Also seen: WAN modem unreachable' in mon.tickets.comment.call_args_list[0].args[1]
|
||||
assert sorted(c.args for c in db.set_ticket_id.call_args_list) == [(29, '555'), (30, '555')]
|
||||
db.get_incident_ticket.assert_called_with('att-modem', monitor.WAN_INCIDENT_WINDOW_S)
|
||||
|
||||
def test_failed_comment_falls_back_to_a_ticket(self, mon, db):
|
||||
db.get_incident_ticket.return_value = '555'
|
||||
mon.tickets.comment.return_value = False
|
||||
self._cycle(mon, db)
|
||||
assert mon.tickets.create.call_count == 1
|
||||
|
||||
def test_outside_a_batch_tickets_open_at_once(self, mon, db):
|
||||
mon._one_shot_ticket([], 'wan_unstable', 'Unstable.', 'WAN connection unstable', '2', 'Look.')
|
||||
assert 'unstable' in mon.tickets.create.call_args.args[0]
|
||||
assert 'no recovery note' in mon.tickets.create.call_args.args[1]
|
||||
|
||||
def test_update_hold_skips_new_tickets(self, mon, db):
|
||||
db.get_state.side_effect = lambda key, default=None: (
|
||||
json.dumps({'label': 'UniFi Network 11.0.99', 'hold_until': time.time() + 60})
|
||||
if key == monitor.UNIFI_UPDATE_STATE else default)
|
||||
self._cycle(mon, db)
|
||||
assert not mon.tickets.create.called and not mon.tickets.comment.called
|
||||
|
||||
|
||||
class TestUnifiDeviceDetail:
|
||||
def test_parses_string_stats_and_outlets(self):
|
||||
d = monitor.UnifiClient._device_detail({
|
||||
@@ -421,6 +513,31 @@ class TestUnifiDeviceDetail:
|
||||
got = c.get_system_events(1000, modem_api.WAN_LOG_KEYS)
|
||||
assert got == [{'key': modem_api.LOG_MODEM_RESTARTED, 'ts': 2.0, 'device': 'RPDU', 'wan': ''}]
|
||||
|
||||
def test_system_log_reads_update_versions(self):
|
||||
c = monitor.UnifiClient({'controller': 'https://u.test', 'api_key': 'k'})
|
||||
page = MagicMock()
|
||||
page.json.side_effect = [
|
||||
{'data': [
|
||||
{'key': 'NETWORK_UPDATED', 'timestamp': 3000, 'parameters': {
|
||||
'APPLICATION': 'UniFi Network', 'PREVIOUS_VERSION': '11.0.97', 'VERSION': '11.0.99'}},
|
||||
{'key': 'FIRMWARE_UPDATE_AVAILABLE_V2', 'timestamp': 3000},
|
||||
{'key': 'MADE_CHANGES_ONE_SETTING_OBJECT', 'timestamp': 3000},
|
||||
]},
|
||||
{'data': []},
|
||||
]
|
||||
c.session.post = MagicMock(return_value=page)
|
||||
got = c.get_system_events(1000, modem_api.wanted_log_key)
|
||||
assert got == [{'key': 'NETWORK_UPDATED', 'ts': 3.0, 'device': '', 'wan': '',
|
||||
'app': 'UniFi Network', 'previous': '11.0.97', 'version': '11.0.99'}]
|
||||
|
||||
@pytest.mark.parametrize('key,expected', [
|
||||
('NETWORK_UPDATED', True), ('FIRMWARE_UPDATED_V2', True), ('PROTECT_UPDATED', True),
|
||||
('FIRMWARE_UPDATE_AVAILABLE_V2', False), ('MADE_CHANGES_ONE_SETTING_OBJECT', False),
|
||||
('NETWORK_WAN_FAILED_2', False),
|
||||
])
|
||||
def test_update_keys(self, key, expected):
|
||||
assert modem_api.is_update_key(key) is expected
|
||||
|
||||
def test_system_log_failure_is_none_not_empty(self):
|
||||
c = monitor.UnifiClient({'controller': 'https://u.test', 'api_key': 'k'})
|
||||
c.session.post = MagicMock(side_effect=OSError('boom'))
|
||||
|
||||
Reference in New Issue
Block a user