From 178a03e898afec91f7dccf63783ab09771756825 Mon Sep 17 00:00:00 2001 From: Jared Vititoe Date: Fri, 6 Nov 2026 19:35:50 -0500 Subject: [PATCH] Give hosts longer before ticketing them as gone A host's node_exporter stops as soon as a shutdown begins and a normal reboot takes about 3-5 minutes, so the 2-check failure_threshold cut a ticket for planned reboots. Add monitor.host_failure_threshold, default 5, for hosts with no metrics and ping-only hosts. Interface link-down alerts keep the shorter threshold, and the dashboard still shows the host as down straight away. Closes #14 Co-Authored-By: Claude Opus 5.5 --- README.md | 6 ++++-- monitor.py | 11 ++++++++--- tests/test_monitor.py | 23 +++++++++++++++++++++++ 3 files changed, 35 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 4bcb7e0..9493f0b 100644 --- a/README.md +++ b/README.md @@ -200,8 +200,8 @@ entry in `FACEPLATES`. |---|---| | UniFi device offline (≥`unifi_failure_threshold` consecutive checks) | P2 High | | Proxmox host NIC link-down regression (≥2 consecutive checks) | P2 High | -| Host unreachable via ping (≥2 consecutive checks) | P2 High | -| Configured host has no metrics in Prometheus (host or `node_exporter` down, ≥2 consecutive checks) | P2 High | +| Host unreachable via ping (≥`host_failure_threshold` consecutive checks) | P2 High | +| Configured host has no metrics in Prometheus (host or `node_exporter` down, ≥`host_failure_threshold` consecutive checks) | P2 High | | Prometheus unavailable, so host and interface alerting is paused (≥2 consecutive checks) | P2 High | | ≥3 hosts simultaneously reporting interface or metrics failures | P1 Critical | | WAN down: broadband or optical status not Up, or PON not in O5 (≥`modem.failure_threshold`) | P1 Critical | @@ -293,6 +293,7 @@ Shared by both processes. Located in the working directory (`/var/www/html/prod/ "monitor": { "poll_interval": 120, "failure_threshold": 2, + "host_failure_threshold": 5, "unifi_failure_threshold": 5, "cluster_threshold": 3, "ping_hosts": [ @@ -323,6 +324,7 @@ Shared by both processes. Located in the working directory (`/var/www/html/prod/ | `hosts` | Maps Prometheus instance labels → display hostnames | | `monitor.poll_interval` | Seconds between full check cycles (default: 120) | | `monitor.failure_threshold` | Consecutive failures before creating ticket (default: 2) | +| `monitor.host_failure_threshold` | Consecutive failures before ticketing a host that has disappeared, whether it has no metrics or doesn't answer ping (default: 5). A normal reboot takes about 3-5 minutes, so the default rides it out: the host still shows as down on the dashboard, but a ticket opens only after about 8 minutes at the default poll interval. For longer maintenance, suppress the host first. | | `monitor.unifi_failure_threshold` | Consecutive failures before ticketing a UniFi device as offline (default: same as `failure_threshold`). Set higher than `failure_threshold` — UniFi switches/APs can take several minutes to rejoin after a firmware-update reboot, so a low threshold cuts tickets for momentary maintenance reboots instead of real failures. | | `monitor.cluster_threshold` | Hosts with failures to trigger cluster-wide P1 (default: 3) | | `monitor.ping_hosts` | Hosts checked only by ping (no node_exporter) | diff --git a/monitor.py b/monitor.py index 70eebb2..88ea2a8 100644 --- a/monitor.py +++ b/monitor.py @@ -844,6 +844,10 @@ class NetworkMonitor: # Use a separate, more forgiving threshold so those momentary reboots # don't cut tickets, while still catching genuine device failures. self.unifi_fail_thresh = mon.get('unifi_failure_threshold', self.fail_thresh) + # A host's exporter stops as soon as a shutdown begins and a normal reboot + # takes about 3-5 minutes, so wait longer than that before ticketing a host + # that has disappeared. A NIC going down on a running host stays on fail_thresh. + self.host_fail_thresh = mon.get('host_failure_threshold', 5) self.cluster_thresh = mon.get('cluster_threshold', 3) self.cluster_name = mon.get('cluster_name', CLUSTER_NAME) @@ -940,7 +944,7 @@ class NetworkMonitor: host, instance, f'No metrics from {host} ({instance}): host or node_exporter is down ({_now_utc()})', ) - if not sup and consec >= self.fail_thresh and not ticket_id: + if not sup and consec >= self.host_fail_thresh and not ticket_id: self._ticket_metrics_missing(event_id, host, instance, consec) db.set_baseline(new_baseline) @@ -1107,7 +1111,7 @@ class NetworkMonitor: name, ip, f'Host {name} ({ip}) unreachable via ping ({_now_utc()})', ) - if not sup and consec >= self.fail_thresh and not ticket_id: + if not sup and consec >= self.host_fail_thresh and not ticket_id: self._ticket_unreachable(event_id, name, ip, consec) else: db.resolve_event('host_unreachable', name, ip) @@ -1474,7 +1478,8 @@ class NetworkMonitor: def run(self) -> None: logger.info( f'Gandalf monitor started – poll_interval={self.poll_interval}s ' - f'fail_thresh={self.fail_thresh} unifi_fail_thresh={self.unifi_fail_thresh}' + f'fail_thresh={self.fail_thresh} host_fail_thresh={self.host_fail_thresh} ' + f'unifi_fail_thresh={self.unifi_fail_thresh}' ) while True: try: diff --git a/tests/test_monitor.py b/tests/test_monitor.py index fdd191b..44e8cb3 100644 --- a/tests/test_monitor.py +++ b/tests/test_monitor.py @@ -32,6 +32,7 @@ def mon(db): m = monitor.NetworkMonitor.__new__(monitor.NetworkMonitor) m.cfg = {} m.fail_thresh = 2 + m.host_fail_thresh = 5 m.cluster_thresh = 2 m.cluster_name = 'test-cluster' m.tickets = MagicMock() @@ -72,6 +73,7 @@ def healthy_modem(rx=-16.9, rx_status=None): class TestDeadHostDetection: def test_host_missing_from_prometheus_alerts_and_tickets(self, mon, db): + db.upsert_event.return_value = (1, False, 5, None) mon._process_interfaces({HOST_A: {'eth0': True}}, []) assert ('host_metrics_missing', 'beta', HOST_B) in upserted(db) assert 'No metrics' in mon.tickets.create.call_args.args[0] @@ -86,7 +88,19 @@ class TestDeadHostDetection: mon._process_interfaces({HOST_A: {'eth0': True}}, []) assert not mon.tickets.create.called + def test_reboot_length_outage_is_recorded_without_a_ticket(self, mon, db): + db.upsert_event.return_value = (1, False, 4, None) + mon._process_interfaces({HOST_A: {'eth0': True}}, []) + assert ('host_metrics_missing', 'beta', HOST_B) in upserted(db) + assert not mon.tickets.create.called + + def test_interface_down_keeps_the_shorter_threshold(self, mon, db): + db.get_baseline.return_value = {'alpha': {'eth0': 'up'}, 'beta': {'eth0': 'up'}} + mon._process_interfaces({HOST_A: {'eth0': False}, HOST_B: {'eth0': True}}, []) + assert 'Interface eth0 link-down' in mon.tickets.create.call_args.args[0] + def test_suppressed_host_gets_no_ticket(self, mon, db): + db.upsert_event.return_value = (1, False, 5, None) db.check_suppressed.return_value = True mon._process_interfaces({HOST_A: {'eth0': True}}, []) assert not mon.tickets.create.called @@ -150,6 +164,15 @@ class TestPingUnknown: mon._process_ping_hosts([], {'pbs': False}) assert ('host_unreachable', 'pbs', '10.0.0.3') in upserted(db) + def test_ping_host_ticket_waits_out_a_reboot(self, mon, db): + mon.cfg = {'monitor': {'ping_hosts': [{'name': 'pbs', 'ip': '10.0.0.3'}]}} + db.upsert_event.return_value = (1, False, 4, None) + mon._process_ping_hosts([], {'pbs': False}) + assert not mon.tickets.create.called + db.upsert_event.return_value = (1, False, 5, None) + mon._process_ping_hosts([], {'pbs': False}) + assert 'Host unreachable' in mon.tickets.create.call_args.args[0] + class TestUnifiRates: def _collector(self): -- 2.47.3