Merge pull request 'Give hosts longer before ticketing them as gone' (#15) from host-reboot-threshold into main

README.md

@@ -200,8 +200,8 @@ entry in `FACEPLATES`.
 |---|---|
 | UniFi device offline (≥`unifi_failure_threshold` consecutive checks) | P2 High |
 | Proxmox host NIC link-down regression (≥2 consecutive checks) | P2 High |
-| Host unreachable via ping (≥2 consecutive checks) | P2 High |
-| Configured host has no metrics in Prometheus (host or `node_exporter` down, ≥2 consecutive checks) | P2 High |
+| Host unreachable via ping (≥`host_failure_threshold` consecutive checks) | P2 High |
+| Configured host has no metrics in Prometheus (host or `node_exporter` down, ≥`host_failure_threshold` consecutive checks) | P2 High |
 | Prometheus unavailable, so host and interface alerting is paused (≥2 consecutive checks) | P2 High |
 | ≥3 hosts simultaneously reporting interface or metrics failures | P1 Critical |
 | WAN down: broadband or optical status not Up, or PON not in O5 (≥`modem.failure_threshold`) | P1 Critical |
@@ -293,6 +293,7 @@ Shared by both processes. Located in the working directory (`/var/www/html/prod/
   "monitor": {
     "poll_interval": 120,
     "failure_threshold": 2,
+    "host_failure_threshold": 5,
     "unifi_failure_threshold": 5,
     "cluster_threshold": 3,
     "ping_hosts": [
@@ -323,6 +324,7 @@ Shared by both processes. Located in the working directory (`/var/www/html/prod/
 | `hosts` | Maps Prometheus instance labels → display hostnames |
 | `monitor.poll_interval` | Seconds between full check cycles (default: 120) |
 | `monitor.failure_threshold` | Consecutive failures before creating ticket (default: 2) |
+| `monitor.host_failure_threshold` | Consecutive failures before ticketing a host that has disappeared, whether it has no metrics or doesn't answer ping (default: 5). A normal reboot takes about 3-5 minutes, so the default rides it out: the host still shows as down on the dashboard, but a ticket opens only after about 8 minutes at the default poll interval. For longer maintenance, suppress the host first. |
 | `monitor.unifi_failure_threshold` | Consecutive failures before ticketing a UniFi device as offline (default: same as `failure_threshold`). Set higher than `failure_threshold` — UniFi switches/APs can take several minutes to rejoin after a firmware-update reboot, so a low threshold cuts tickets for momentary maintenance reboots instead of real failures. |
 | `monitor.cluster_threshold` | Hosts with failures to trigger cluster-wide P1 (default: 3) |
 | `monitor.ping_hosts` | Hosts checked only by ping (no node_exporter) |

monitor.py

@@ -844,6 +844,10 @@ class NetworkMonitor:
         # Use a separate, more forgiving threshold so those momentary reboots
         # don't cut tickets, while still catching genuine device failures.
         self.unifi_fail_thresh = mon.get('unifi_failure_threshold', self.fail_thresh)
+        # A host's exporter stops as soon as a shutdown begins and a normal reboot
+        # takes about 3-5 minutes, so wait longer than that before ticketing a host
+        # that has disappeared. A NIC going down on a running host stays on fail_thresh.
+        self.host_fail_thresh = mon.get('host_failure_threshold', 5)
         self.cluster_thresh = mon.get('cluster_threshold', 3)
         self.cluster_name = mon.get('cluster_name', CLUSTER_NAME)
 
@@ -940,7 +944,7 @@ class NetworkMonitor:
                 host, instance,
                 f'No metrics from {host} ({instance}): host or node_exporter is down ({_now_utc()})',
             )
-            if not sup and consec >= self.fail_thresh and not ticket_id:
+            if not sup and consec >= self.host_fail_thresh and not ticket_id:
                 self._ticket_metrics_missing(event_id, host, instance, consec)
 
         db.set_baseline(new_baseline)
@@ -1107,7 +1111,7 @@ class NetworkMonitor:
                     name, ip,
                     f'Host {name} ({ip}) unreachable via ping ({_now_utc()})',
                 )
-                if not sup and consec >= self.fail_thresh and not ticket_id:
+                if not sup and consec >= self.host_fail_thresh and not ticket_id:
                     self._ticket_unreachable(event_id, name, ip, consec)
             else:
                 db.resolve_event('host_unreachable', name, ip)
@@ -1474,7 +1478,8 @@ class NetworkMonitor:
     def run(self) -> None:
         logger.info(
             f'Gandalf monitor started – poll_interval={self.poll_interval}s '
-            f'fail_thresh={self.fail_thresh} unifi_fail_thresh={self.unifi_fail_thresh}'
+            f'fail_thresh={self.fail_thresh} host_fail_thresh={self.host_fail_thresh} '
+            f'unifi_fail_thresh={self.unifi_fail_thresh}'
         )
         while True:
             try:

tests/test_monitor.py

@@ -32,6 +32,7 @@ def mon(db):
     m = monitor.NetworkMonitor.__new__(monitor.NetworkMonitor)
     m.cfg = {}
     m.fail_thresh = 2
+    m.host_fail_thresh = 5
     m.cluster_thresh = 2
     m.cluster_name = 'test-cluster'
     m.tickets = MagicMock()
@@ -72,6 +73,7 @@ def healthy_modem(rx=-16.9, rx_status=None):
 
 class TestDeadHostDetection:
     def test_host_missing_from_prometheus_alerts_and_tickets(self, mon, db):
+        db.upsert_event.return_value = (1, False, 5, None)
         mon._process_interfaces({HOST_A: {'eth0': True}}, [])
         assert ('host_metrics_missing', 'beta', HOST_B) in upserted(db)
         assert 'No metrics' in mon.tickets.create.call_args.args[0]
@@ -86,7 +88,19 @@ class TestDeadHostDetection:
         mon._process_interfaces({HOST_A: {'eth0': True}}, [])
         assert not mon.tickets.create.called
 
+    def test_reboot_length_outage_is_recorded_without_a_ticket(self, mon, db):
+        db.upsert_event.return_value = (1, False, 4, None)
+        mon._process_interfaces({HOST_A: {'eth0': True}}, [])
+        assert ('host_metrics_missing', 'beta', HOST_B) in upserted(db)
+        assert not mon.tickets.create.called
+
+    def test_interface_down_keeps_the_shorter_threshold(self, mon, db):
+        db.get_baseline.return_value = {'alpha': {'eth0': 'up'}, 'beta': {'eth0': 'up'}}
+        mon._process_interfaces({HOST_A: {'eth0': False}, HOST_B: {'eth0': True}}, [])
+        assert 'Interface eth0 link-down' in mon.tickets.create.call_args.args[0]
+
     def test_suppressed_host_gets_no_ticket(self, mon, db):
+        db.upsert_event.return_value = (1, False, 5, None)
         db.check_suppressed.return_value = True
         mon._process_interfaces({HOST_A: {'eth0': True}}, [])
         assert not mon.tickets.create.called
@@ -150,6 +164,15 @@ class TestPingUnknown:
         mon._process_ping_hosts([], {'pbs': False})
         assert ('host_unreachable', 'pbs', '10.0.0.3') in upserted(db)
 
+    def test_ping_host_ticket_waits_out_a_reboot(self, mon, db):
+        mon.cfg = {'monitor': {'ping_hosts': [{'name': 'pbs', 'ip': '10.0.0.3'}]}}
+        db.upsert_event.return_value = (1, False, 4, None)
+        mon._process_ping_hosts([], {'pbs': False})
+        assert not mon.tickets.create.called
+        db.upsert_event.return_value = (1, False, 5, None)
+        mon._process_ping_hosts([], {'pbs': False})
+        assert 'Host unreachable' in mon.tickets.create.call_args.args[0]
+
 
 class TestUnifiRates:
     def _collector(self):