Give hosts longer before ticketing them as gone
Lint / JS (eslint) (push) Successful in 16s
Lint / Python (flake8) (push) Successful in 1m23s
Lint / Notify on failure (push) Skipped
Lint / Deploy (push) Skipped
Security / Python Security (bandit) (push) Successful in 1m49s
Lint / Python (flake8) (pull_request) Successful in 54s
Test / Python Tests (pytest) (push) Successful in 1m35s
Lint / JS (eslint) (pull_request) Successful in 12s
Lint / Notify on failure (pull_request) Skipped
Lint / Deploy (pull_request) Skipped
Security / Python Security (bandit) (pull_request) Successful in 1m42s
Test / Python Tests (pytest) (pull_request) Successful in 2m3s

A host's node_exporter stops as soon as a shutdown begins and a normal
reboot takes about 3-5 minutes, so the 2-check failure_threshold cut a
ticket for planned reboots. Add monitor.host_failure_threshold, default 5,
for hosts with no metrics and ping-only hosts. Interface link-down alerts
keep the shorter threshold, and the dashboard still shows the host as down
straight away.

Closes #14

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
jaredandClaude Opus 5.5 committed 2026-11-06 19:35:50 -05:00
1 parent 7c917969d8
commit 178a03e898
3 files changed
+35 -5

No files matched your search

+23
View File
@@ -32,6 +32,7 @@ def mon(db):
m = monitor.NetworkMonitor.__new__(monitor.NetworkMonitor)
m.cfg = {}
m.fail_thresh = 2
m.host_fail_thresh = 5
m.cluster_thresh = 2
m.cluster_name = 'test-cluster'
m.tickets = MagicMock()
@@ -72,6 +73,7 @@ def healthy_modem(rx=-16.9, rx_status=None):
class TestDeadHostDetection:
def test_host_missing_from_prometheus_alerts_and_tickets(self, mon, db):
db.upsert_event.return_value = (1, False, 5, None)
mon._process_interfaces({HOST_A: {'eth0': True}}, [])
assert ('host_metrics_missing', 'beta', HOST_B) in upserted(db)
assert 'No metrics' in mon.tickets.create.call_args.args[0]
@@ -86,7 +88,19 @@ class TestDeadHostDetection:
mon._process_interfaces({HOST_A: {'eth0': True}}, [])
assert not mon.tickets.create.called
def test_reboot_length_outage_is_recorded_without_a_ticket(self, mon, db):
db.upsert_event.return_value = (1, False, 4, None)
mon._process_interfaces({HOST_A: {'eth0': True}}, [])
assert ('host_metrics_missing', 'beta', HOST_B) in upserted(db)
assert not mon.tickets.create.called
def test_interface_down_keeps_the_shorter_threshold(self, mon, db):
db.get_baseline.return_value = {'alpha': {'eth0': 'up'}, 'beta': {'eth0': 'up'}}
mon._process_interfaces({HOST_A: {'eth0': False}, HOST_B: {'eth0': True}}, [])
assert 'Interface eth0 link-down' in mon.tickets.create.call_args.args[0]
def test_suppressed_host_gets_no_ticket(self, mon, db):
db.upsert_event.return_value = (1, False, 5, None)
db.check_suppressed.return_value = True
mon._process_interfaces({HOST_A: {'eth0': True}}, [])
assert not mon.tickets.create.called
@@ -150,6 +164,15 @@ class TestPingUnknown:
mon._process_ping_hosts([], {'pbs': False})
assert ('host_unreachable', 'pbs', '10.0.0.3') in upserted(db)
def test_ping_host_ticket_waits_out_a_reboot(self, mon, db):
mon.cfg = {'monitor': {'ping_hosts': [{'name': 'pbs', 'ip': '10.0.0.3'}]}}
db.upsert_event.return_value = (1, False, 4, None)
mon._process_ping_hosts([], {'pbs': False})
assert not mon.tickets.create.called
db.upsert_event.return_value = (1, False, 5, None)
mon._process_ping_hosts([], {'pbs': False})
assert 'Host unreachable' in mon.tickets.create.call_args.args[0]
class TestUnifiRates:
def _collector(self):