WAN tickets: one per incident, notice UniFi updates, maintenance button
Lint / JS (eslint) (push) Successful in 42s
Lint / Python (flake8) (push) Successful in 1m35s
Lint / Notify on failure (push) Skipped
Lint / Deploy (push) Skipped
Security / Python Security (bandit) (push) Successful in 1m3s
Test / Python Tests (pytest) (push) Successful in 1m5s
Lint / Python (flake8) (pull_request) Successful in 58s
Lint / JS (eslint) (pull_request) Successful in 8s
Lint / Notify on failure (pull_request) Skipped
Lint / Deploy (pull_request) Skipped
Security / Python Security (bandit) (pull_request) Successful in 1m1s
Test / Python Tests (pytest) (pull_request) Successful in 1m10s

A UniFi Network update on 2026-11-12 dropped the UDM's WAN, Smart Power
power-cycled the modem, and Gandalf opened two tickets for the one event.

- Modem and WAN reports from one check cycle open a single ticket, led by
  the most urgent, with the rest listed under "Also seen in this incident".
  A report while the modem has an open alert, or one that ended under 30
  minutes ago, becomes a comment on that ticket instead.
- UniFi update log entries (NETWORK_UPDATED, FIRMWARE_UPDATED_V2 and the
  like) add a note to modem/WAN tickets from 20 minutes before the update
  to 15 after, and hold new modem/WAN tickets for 15 minutes.
- A Maintenance button for admins starts a global suppression for 30 min
  to 4 hours, with a banner on every page and End now.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-11-12 12:53:22 -05:00
co-authored by Claude Opus 5.5
parent 6b03fe7b44
commit 379e3be365
11 changed files with 541 additions and 28 deletions
+38
View File
@@ -610,3 +610,41 @@ class TestUnusedInterfaces:
gandalf.db.get_unused_interfaces.return_value = {'large1': {'enp36s0': {'reason': 'spare'}}}
data = client.get('/api/device/large1', headers=AUTH).get_json()
assert data['unused_interfaces'] == {'enp36s0': {'reason': 'spare'}}
class TestMaintenance:
SUP = {'id': 7, 'target_type': 'all', 'target_name': '', 'reason': 'UniFi Network update',
'suppressed_by': 'jared', 'expires_at': '2026-11-12T20:00:00Z'}
def test_no_banner_and_a_start_button_by_default(self, client):
html = client.get('/', headers=AUTH).get_data(as_text=True)
assert 'maint-banner' not in html
assert 'data-action="start-maintenance"' in html and 'id="maint-modal"' in html
def test_banner_on_every_page_while_on(self, client):
gandalf.db.get_active_suppressions.return_value = [
{'id': 3, 'target_type': 'host', 'target_name': 'large1', 'reason': 'x', 'expires_at': None},
self.SUP,
]
for page in ('/', '/events', '/rack'):
html = client.get(page, headers=AUTH).get_data(as_text=True)
assert 'class="maint-banner"' in html, page
assert 'datetime="2026-11-12T20:00:00Z"' in html and 'UniFi Network update' in html
assert 'data-sup-id="7"' in html and 'data-action="start-maintenance"' not in html
def test_open_ended_maintenance_wins(self, client):
gandalf.db.get_active_suppressions.return_value = [
self.SUP, dict(self.SUP, id=9, expires_at=None, reason='Rewiring')]
html = client.get('/', headers=AUTH).get_data(as_text=True)
assert 'until it is ended' in html and 'data-sup-id="9"' in html
def test_viewers_see_the_banner_without_buttons(self, client):
gandalf.db.get_active_suppressions.return_value = [self.SUP]
viewer = dict(AUTH, **{'Remote-Groups': 'staff'})
with patch.object(gandalf, '_cfg', dict(CONFIG, auth=dict(CONFIG['auth'], allowed_groups=['admin', 'staff']))):
html = client.get('/', headers=viewer).get_data(as_text=True)
assert 'class="maint-banner"' in html and 'data-action="end-maintenance"' not in html
def test_a_db_error_means_no_banner(self, client):
gandalf.db.get_active_suppressions.side_effect = RuntimeError('db down')
assert gandalf.inject_maintenance() == {'maintenance': None}
+117
View File
@@ -31,6 +31,8 @@ def db():
mock_db.cutsheet_load.return_value = {'items': [], 'cables': [], 'port_labels': []}
mock_db.get_pending_recovery_notes.return_value = []
mock_db.get_device_link.return_value = ''
mock_db.get_incident_ticket.return_value = None
mock_db.get_ticket_ids_between.return_value = []
yield mock_db
@@ -381,6 +383,47 @@ class TestWanLog:
mon._process_wan_log([])
assert not mon.tickets.create.called
def test_unifi_update_is_not_a_wan_event(self, mon, db):
now = time.time()
self._state(db, unifi_log_cursor=str(int(now * 1000)))
self._mon(mon, [{'key': 'NETWORK_UPDATED', 'ts': now, 'device': '', 'wan': '',
'app': 'UniFi Network', 'previous': '11.0.97', 'version': '11.0.99'}])
mon._process_wan_log([])
assert not mon.tickets.create.called
saved = dict((c.args[0], c.args[1]) for c in db.set_state.call_args_list)
assert saved[monitor.UNIFI_UPDATE_STATE]['label'] == 'UniFi Network 11.0.97 to 11.0.99'
assert saved['wan_history'] == {'outages': [], 'restarts': []}
def test_update_comments_on_tickets_opened_just_before(self, mon, db):
now = time.time()
self._state(db, unifi_log_cursor=str(int(now * 1000)))
db.get_ticket_ids_between.return_value = ['207521068']
self._mon(mon, [{'key': 'NETWORK_UPDATED', 'ts': now, 'device': '', 'wan': '',
'app': 'UniFi Network', 'previous': '11.0.97', 'version': '11.0.99'}])
mon._process_wan_log([])
start, end = db.get_ticket_ids_between.call_args.args[1:]
assert (end - start).total_seconds() == monitor.UPDATE_LOOKBACK_S + monitor.UPDATE_HOLD_S
tid, text = mon.tickets.comment.call_args.args
assert tid == '207521068' and 'UniFi Network 11.0.97 to 11.0.99' in text
def test_update_in_the_same_read_holds_the_power_cycle_ticket(self, mon, db):
now = time.time()
state = {'unifi_log_cursor': str(int(now * 1000))}
db.get_state.side_effect = lambda key, default=None: state.get(key, default)
db.set_state.side_effect = lambda key, value: state.__setitem__(
key, json.dumps(value) if isinstance(value, dict) else value)
self._mon(mon, [
{'key': modem_api.LOG_MODEM_RESTARTED, 'ts': now - 600, 'device': 'RPDU', 'wan': ''},
{'key': 'NETWORK_UPDATED', 'ts': now, 'device': '', 'wan': '', 'app': 'UniFi Network'},
])
mon._process_wan_log([])
assert not mon.tickets.create.called
db.resolve_event.assert_called_with('modem_power_cycled', 'att-modem', '')
def test_hold_ends(self, mon, db):
self._state(db, unifi_update=json.dumps({'label': 'x', 'hold_until': time.time() - 1}))
assert mon._update_hold() is None
def test_uptime_going_backwards_records_a_restart(self, mon, db):
self._state(db, modem_stats=json.dumps({'uptime_s': 5000}))
mon._note_modem_restart({'uptime_s': 60})
@@ -393,6 +436,55 @@ class TestWanLog:
assert not db.set_state.called
class TestWanIncident:
"""Last night's sequence: the RPDU power-cycles the modem in the cycle the modem goes unreachable."""
def _cycle(self, mon, db):
db.upsert_event.side_effect = [(30, True, 1, None), (29, False, 2, None)]
mon.modem.poll.side_effect = modem_api.ModemUnreachable('Network is unreachable')
with mon._wan_batch():
mon._one_shot_ticket([], 'modem_power_cycled', 'RPDU power-cycled the modem.',
'Modem auto power-cycled after internet loss', '3', 'Check the WAN panel.')
mon._collect_modem([])
def test_one_ticket_led_by_the_most_urgent(self, mon, db):
self._cycle(mon, db)
assert mon.tickets.create.call_count == 1
title, desc = mon.tickets.create.call_args.args
assert 'WAN modem unreachable' in title
assert mon.tickets.create.call_args.kwargs['priority'] == '2'
assert '### Also seen in this incident' in desc and 'power-cycled' in desc
assert sorted(c.args for c in db.set_ticket_id.call_args_list) == [(29, '123'), (30, '123')]
def test_recent_incident_gets_a_comment_instead(self, mon, db):
db.get_incident_ticket.return_value = '555'
mon.tickets.comment.return_value = True
self._cycle(mon, db)
assert not mon.tickets.create.called
assert [c.args[0] for c in mon.tickets.comment.call_args_list] == ['555', '555']
assert 'Also seen: WAN modem unreachable' in mon.tickets.comment.call_args_list[0].args[1]
assert sorted(c.args for c in db.set_ticket_id.call_args_list) == [(29, '555'), (30, '555')]
db.get_incident_ticket.assert_called_with('att-modem', monitor.WAN_INCIDENT_WINDOW_S)
def test_failed_comment_falls_back_to_a_ticket(self, mon, db):
db.get_incident_ticket.return_value = '555'
mon.tickets.comment.return_value = False
self._cycle(mon, db)
assert mon.tickets.create.call_count == 1
def test_outside_a_batch_tickets_open_at_once(self, mon, db):
mon._one_shot_ticket([], 'wan_unstable', 'Unstable.', 'WAN connection unstable', '2', 'Look.')
assert 'unstable' in mon.tickets.create.call_args.args[0]
assert 'no recovery note' in mon.tickets.create.call_args.args[1]
def test_update_hold_skips_new_tickets(self, mon, db):
db.get_state.side_effect = lambda key, default=None: (
json.dumps({'label': 'UniFi Network 11.0.99', 'hold_until': time.time() + 60})
if key == monitor.UNIFI_UPDATE_STATE else default)
self._cycle(mon, db)
assert not mon.tickets.create.called and not mon.tickets.comment.called
class TestUnifiDeviceDetail:
def test_parses_string_stats_and_outlets(self):
d = monitor.UnifiClient._device_detail({
@@ -421,6 +513,31 @@ class TestUnifiDeviceDetail:
got = c.get_system_events(1000, modem_api.WAN_LOG_KEYS)
assert got == [{'key': modem_api.LOG_MODEM_RESTARTED, 'ts': 2.0, 'device': 'RPDU', 'wan': ''}]
def test_system_log_reads_update_versions(self):
c = monitor.UnifiClient({'controller': 'https://u.test', 'api_key': 'k'})
page = MagicMock()
page.json.side_effect = [
{'data': [
{'key': 'NETWORK_UPDATED', 'timestamp': 3000, 'parameters': {
'APPLICATION': 'UniFi Network', 'PREVIOUS_VERSION': '11.0.97', 'VERSION': '11.0.99'}},
{'key': 'FIRMWARE_UPDATE_AVAILABLE_V2', 'timestamp': 3000},
{'key': 'MADE_CHANGES_ONE_SETTING_OBJECT', 'timestamp': 3000},
]},
{'data': []},
]
c.session.post = MagicMock(return_value=page)
got = c.get_system_events(1000, modem_api.wanted_log_key)
assert got == [{'key': 'NETWORK_UPDATED', 'ts': 3.0, 'device': '', 'wan': '',
'app': 'UniFi Network', 'previous': '11.0.97', 'version': '11.0.99'}]
@pytest.mark.parametrize('key,expected', [
('NETWORK_UPDATED', True), ('FIRMWARE_UPDATED_V2', True), ('PROTECT_UPDATED', True),
('FIRMWARE_UPDATE_AVAILABLE_V2', False), ('MADE_CHANGES_ONE_SETTING_OBJECT', False),
('NETWORK_WAN_FAILED_2', False),
])
def test_update_keys(self, key, expected):
assert modem_api.is_update_key(key) is expected
def test_system_log_failure_is_none_not_empty(self):
c = monitor.UnifiClient({'controller': 'https://u.test', 'api_key': 'k'})
c.session.post = MagicMock(side_effect=OSError('boom'))