Give hosts longer before ticketing them as gone
Lint / JS (eslint) (push) Successful in 16s
Lint / Python (flake8) (push) Successful in 1m23s
Lint / Notify on failure (push) Skipped
Lint / Deploy (push) Skipped
Security / Python Security (bandit) (push) Successful in 1m49s
Lint / Python (flake8) (pull_request) Successful in 54s
Test / Python Tests (pytest) (push) Successful in 1m35s
Lint / JS (eslint) (pull_request) Successful in 12s
Lint / Notify on failure (pull_request) Skipped
Lint / Deploy (pull_request) Skipped
Security / Python Security (bandit) (pull_request) Successful in 1m42s
Test / Python Tests (pytest) (pull_request) Successful in 2m3s
Lint / JS (eslint) (push) Successful in 16s
Lint / Python (flake8) (push) Successful in 1m23s
Lint / Notify on failure (push) Skipped
Lint / Deploy (push) Skipped
Security / Python Security (bandit) (push) Successful in 1m49s
Lint / Python (flake8) (pull_request) Successful in 54s
Test / Python Tests (pytest) (push) Successful in 1m35s
Lint / JS (eslint) (pull_request) Successful in 12s
Lint / Notify on failure (pull_request) Skipped
Lint / Deploy (pull_request) Skipped
Security / Python Security (bandit) (pull_request) Successful in 1m42s
Test / Python Tests (pytest) (pull_request) Successful in 2m3s
A host's node_exporter stops as soon as a shutdown begins and a normal reboot takes about 3-5 minutes, so the 2-check failure_threshold cut a ticket for planned reboots. Add monitor.host_failure_threshold, default 5, for hosts with no metrics and ping-only hosts. Interface link-down alerts keep the shorter threshold, and the dashboard still shows the host as down straight away. Closes #14 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -32,6 +32,7 @@ def mon(db):
|
||||
m = monitor.NetworkMonitor.__new__(monitor.NetworkMonitor)
|
||||
m.cfg = {}
|
||||
m.fail_thresh = 2
|
||||
m.host_fail_thresh = 5
|
||||
m.cluster_thresh = 2
|
||||
m.cluster_name = 'test-cluster'
|
||||
m.tickets = MagicMock()
|
||||
@@ -72,6 +73,7 @@ def healthy_modem(rx=-16.9, rx_status=None):
|
||||
|
||||
class TestDeadHostDetection:
|
||||
def test_host_missing_from_prometheus_alerts_and_tickets(self, mon, db):
|
||||
db.upsert_event.return_value = (1, False, 5, None)
|
||||
mon._process_interfaces({HOST_A: {'eth0': True}}, [])
|
||||
assert ('host_metrics_missing', 'beta', HOST_B) in upserted(db)
|
||||
assert 'No metrics' in mon.tickets.create.call_args.args[0]
|
||||
@@ -86,7 +88,19 @@ class TestDeadHostDetection:
|
||||
mon._process_interfaces({HOST_A: {'eth0': True}}, [])
|
||||
assert not mon.tickets.create.called
|
||||
|
||||
def test_reboot_length_outage_is_recorded_without_a_ticket(self, mon, db):
|
||||
db.upsert_event.return_value = (1, False, 4, None)
|
||||
mon._process_interfaces({HOST_A: {'eth0': True}}, [])
|
||||
assert ('host_metrics_missing', 'beta', HOST_B) in upserted(db)
|
||||
assert not mon.tickets.create.called
|
||||
|
||||
def test_interface_down_keeps_the_shorter_threshold(self, mon, db):
|
||||
db.get_baseline.return_value = {'alpha': {'eth0': 'up'}, 'beta': {'eth0': 'up'}}
|
||||
mon._process_interfaces({HOST_A: {'eth0': False}, HOST_B: {'eth0': True}}, [])
|
||||
assert 'Interface eth0 link-down' in mon.tickets.create.call_args.args[0]
|
||||
|
||||
def test_suppressed_host_gets_no_ticket(self, mon, db):
|
||||
db.upsert_event.return_value = (1, False, 5, None)
|
||||
db.check_suppressed.return_value = True
|
||||
mon._process_interfaces({HOST_A: {'eth0': True}}, [])
|
||||
assert not mon.tickets.create.called
|
||||
@@ -150,6 +164,15 @@ class TestPingUnknown:
|
||||
mon._process_ping_hosts([], {'pbs': False})
|
||||
assert ('host_unreachable', 'pbs', '10.0.0.3') in upserted(db)
|
||||
|
||||
def test_ping_host_ticket_waits_out_a_reboot(self, mon, db):
|
||||
mon.cfg = {'monitor': {'ping_hosts': [{'name': 'pbs', 'ip': '10.0.0.3'}]}}
|
||||
db.upsert_event.return_value = (1, False, 4, None)
|
||||
mon._process_ping_hosts([], {'pbs': False})
|
||||
assert not mon.tickets.create.called
|
||||
db.upsert_event.return_value = (1, False, 5, None)
|
||||
mon._process_ping_hosts([], {'pbs': False})
|
||||
assert 'Host unreachable' in mon.tickets.create.call_args.args[0]
|
||||
|
||||
|
||||
class TestUnifiRates:
|
||||
def _collector(self):
|
||||
|
||||
Reference in New Issue
Block a user