diff --git a/monitor.py b/monitor.py index 6894ece..99258cf 100644 --- a/monitor.py +++ b/monitor.py @@ -1195,6 +1195,10 @@ class NetworkMonitor: def _process_interfaces(self, states: Dict[str, Dict[str, bool]], suppressions: list) -> None: baseline = db.get_baseline() new_baseline = {k: dict(v) for k, v in baseline.items()} + # An interface with an open link-down alert was up before it went down, whatever the + # baseline says now: after a baseline reset it reads as "first seen down", which would + # leave the alert open forever without updating, ticketing or resolving it. + open_down = set(db.get_open_event_targets('interface_down')) # Only count hosts with genuine regressions (UP→DOWN) toward cluster threshold hosts_with_regression: List[str] = [] @@ -1207,6 +1211,8 @@ class NetworkMonitor: for iface, is_up in ifaces.items(): prev = baseline.get(host, {}).get(iface) # 'up', 'initial_down', or None + if (host, iface) in open_down: + prev = 'up' if is_up: new_baseline[host][iface] = 'up' diff --git a/tests/test_monitor.py b/tests/test_monitor.py index ad418c4..e68d1a8 100644 --- a/tests/test_monitor.py +++ b/tests/test_monitor.py @@ -31,6 +31,7 @@ def db(): mock_db.cutsheet_load.return_value = {'items': [], 'cables': [], 'port_labels': []} mock_db.get_pending_recovery_notes.return_value = [] mock_db.get_device_link.return_value = '' + mock_db.get_open_event_targets.return_value = [] yield mock_db @@ -117,6 +118,24 @@ class TestDeadHostDetection: mon._process_interfaces({HOST_A: {'eth0': True}}, []) assert ('cluster_network_issue', 'test-cluster', '') in upserted(db) + def test_open_alert_survives_a_baseline_reset(self, mon, db): + # The baseline lost eth1 (reset), but its link-down alert is still open: keep alerting + db.get_open_event_targets.return_value = [('alpha', 'eth1')] + db.upsert_event.return_value = (3, False, 2, None) + mon._process_interfaces({HOST_A: {'eth0': True, 'eth1': False}, HOST_B: {'eth0': True}}, []) + assert ('interface_down', 'alpha', 'eth1') in upserted(db) + assert 'Interface eth1 link-down' in mon.tickets.create.call_args.args[0] + + def test_open_alert_resolves_when_the_link_returns(self, mon, db): + db.get_open_event_targets.return_value = [('alpha', 'eth1')] + mon._process_interfaces({HOST_A: {'eth0': True, 'eth1': True}, HOST_B: {'eth0': True}}, []) + db.resolve_event.assert_any_call('interface_down', 'alpha', 'eth1') + + def test_first_seen_down_without_an_alert_still_stays_quiet(self, mon, db): + db.get_open_event_targets.return_value = [] + mon._process_interfaces({HOST_A: {'eth0': True, 'eth1': False}, HOST_B: {'eth0': True}}, []) + assert ('interface_down', 'alpha', 'eth1') not in upserted(db) + def test_dead_host_shows_as_down_on_dashboard(self, mon): snap = mon._collect_snapshot({HOST_A: {'eth0': True}}) assert snap['hosts']['alpha']['status'] == 'up'