Add recovery notes and fuller context to tickets
Lint / JS (eslint) (push) Successful in 7s
Lint / Python (flake8) (push) Successful in 1m26s
Lint / Notify on failure (push) Skipped
Lint / Deploy (push) Skipped
Security / Python Security (bandit) (push) Successful in 1m30s
Lint / Python (flake8) (pull_request) Successful in 52s
Test / Python Tests (pytest) (push) Successful in 1m4s
Lint / JS (eslint) (pull_request) Successful in 7s
Lint / Notify on failure (pull_request) Skipped
Lint / Deploy (pull_request) Skipped
Security / Python Security (bandit) (pull_request) Successful in 1m28s
Test / Python Tests (pytest) (pull_request) Successful in 2m0s
Lint / JS (eslint) (push) Successful in 7s
Lint / Python (flake8) (push) Successful in 1m26s
Lint / Notify on failure (push) Skipped
Lint / Deploy (push) Skipped
Security / Python Security (bandit) (push) Successful in 1m30s
Lint / Python (flake8) (pull_request) Successful in 52s
Test / Python Tests (pytest) (push) Successful in 1m4s
Lint / JS (eslint) (pull_request) Successful in 7s
Lint / Notify on failure (pull_request) Skipped
Lint / Deploy (pull_request) Skipped
Security / Python Security (bandit) (pull_request) Successful in 1m28s
Test / Python Tests (pytest) (pull_request) Successful in 2m0s
When an alert that opened a ticket clears, the monitor now comments on the ticket (when it cleared, how long it lasted, failed checks, a link back). It never closes or changes a ticket's status. Failed notes retry each cycle for 24 hours; a 401/403 from Tinker stops them until restart. Tickets are now Markdown with a "What to check" list and, where known: the switch port the device plugs into (kept from the last time it was seen, so a dead host still shows it), cutsheet cables and their path, how often the alert fired in the last 7/30 days, other open alerts on the device, and links into Gandalf (gandalf.public_url). Titles are unchanged so Tinker's duplicate detection still works. network_events gains recovery_notified, added at monitor startup; rows that cleared before the upgrade count as already notified. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
ff0094c8ff
commit
437f7a415f
8 files changed
+886
-133
No files matched your search
@@ -3,6 +3,7 @@ import json
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
from datetime import timedelta
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
@@ -24,6 +25,11 @@ def db():
|
||||
mock_db.upsert_event.return_value = (1, False, 2, None)
|
||||
mock_db.get_modem_rx_values.return_value = []
|
||||
mock_db.get_state.return_value = None
|
||||
mock_db.event_stats.return_value = {'last_7d': 0, 'last_30d': 0,
|
||||
'last_first_seen': None, 'last_resolved_at': None}
|
||||
mock_db.get_events_for_target.return_value = []
|
||||
mock_db.cutsheet_load.return_value = {'items': [], 'cables': [], 'port_labels': []}
|
||||
mock_db.get_pending_recovery_notes.return_value = []
|
||||
yield mock_db
|
||||
|
||||
|
||||
@@ -560,3 +566,177 @@ class TestCutsheetMismatch:
|
||||
assert not db.upsert_event.called
|
||||
resolved = {c.args for c in db.resolve_event.call_args_list}
|
||||
assert ('cable_mismatch', 'sw', 'cable 7') in resolved and ('cable_mismatch', 'other', 'cable 9') in resolved
|
||||
|
||||
|
||||
class TestTicketContext:
|
||||
SWITCHES = {'agg': {'ports': {
|
||||
'Large1': {'port_idx': 7, 'up': True, 'speed_mbps': 10000,
|
||||
'lldp': {'system_name': 'large1', 'port_id': 'enp1s0f0'},
|
||||
'sfp': {'found': True, 'kind': 'dac', 'part': 'DAC-10G'}},
|
||||
'Port 3': {'port_idx': 3, 'up': False, 'speed_mbps': 0},
|
||||
}}}
|
||||
|
||||
def _mon(self, mon, switches=None):
|
||||
mon.public_url = 'https://gandalf.test'
|
||||
mon.poll_interval = 120
|
||||
mon._link_data = {'unifi_switches': switches if switches is not None else self.SWITCHES}
|
||||
return mon
|
||||
|
||||
def _desc(self, mon):
|
||||
return mon.tickets.create.call_args.args[1]
|
||||
|
||||
def test_interface_ticket_names_the_switch_port_and_links(self, mon, db):
|
||||
self._mon(mon)._ticket_interface(1, 'large1', 'enp1s0f0', 2)
|
||||
desc = self._desc(mon)
|
||||
assert '### Where it is connected' in desc
|
||||
assert '`enp1s0f0` → [agg port 7 "Large1": up at 10 Gbps, module: DAC DAC-10G]' in desc
|
||||
assert '(https://gandalf.test/device/agg#port-7)' in desc
|
||||
assert 'https://gandalf.test/device/large1' in desc
|
||||
assert 'Opened automatically by Gandalf' in desc
|
||||
|
||||
def test_dead_host_uses_the_last_known_port(self, mon, db):
|
||||
mon._known_uplinks = {'large1': {'seen': '2026-11-10 10:00:00 UTC', 'uplinks': [
|
||||
{'switch': 'agg', 'port': 'Port 3', 'port_idx': 3, 'host_interface': 'eno1'}]}}
|
||||
self._mon(mon)._ticket_metrics_missing(1, 'micro1', 'x:9100', 5)
|
||||
assert 'not seen this device on a switch port' in self._desc(mon)
|
||||
mon.tickets.create.reset_mock()
|
||||
unplugged = {'agg': {'ports': {'Port 3': {'port_idx': 3, 'up': False}}}}
|
||||
self._mon(mon, unplugged)._ticket_unreachable(1, 'large1', '10.0.0.9', 5)
|
||||
desc = self._desc(mon)
|
||||
assert 'agg port 3: down' in desc
|
||||
assert 'last seen linked there 2026-11-10 10:00:00 UTC' in desc
|
||||
assert 'Down for at least' in desc and '10 min' in desc
|
||||
|
||||
def test_cutsheet_runs_and_history(self, mon, db):
|
||||
db.cutsheet_load.return_value = {'items': [], 'port_labels': [], 'cables': [
|
||||
{'id': 4, 'label': 'C04', 'a_device': 'agg', 'a_port': '7', 'a_side': '',
|
||||
'b_device': 'large1', 'b_port': 'enp1s0f0', 'b_side': '', 'status': 'installed'}]}
|
||||
db.event_stats.return_value = {'last_7d': 3, 'last_30d': 5,
|
||||
'last_first_seen': monitor._utcnow(), 'last_resolved_at': monitor._utcnow()}
|
||||
self._mon(mon)._ticket_interface(1, 'large1', 'enp1s0f0', 2)
|
||||
desc = self._desc(mon)
|
||||
assert '[**C04**](https://gandalf.test/cutsheet#cable-4): agg port 7 → large1 enp1s0f0' in desc
|
||||
assert 'Happened 3 times in the last 7 days and 5 in the last 30' in desc
|
||||
assert 'This keeps coming back' in desc
|
||||
|
||||
def test_other_open_alerts_are_listed(self, mon, db):
|
||||
db.get_events_for_target.return_value = [
|
||||
{'id': 1, 'event_type': 'interface_down', 'target_name': 'large1', 'target_detail': 'eno1'},
|
||||
{'id': 2, 'event_type': 'interface_down', 'target_name': 'large1', 'target_detail': 'eno2',
|
||||
'first_seen': 'T', 'ticket_id': '55'},
|
||||
{'id': 3, 'event_type': 'host_unreachable', 'target_name': 'large1', 'resolved_at': 'R'},
|
||||
]
|
||||
self._mon(mon)._ticket_interface(1, 'large1', 'eno1', 2)
|
||||
desc = self._desc(mon)
|
||||
assert '- interface down on large1 eno2 since T (ticket #55)' in desc
|
||||
assert 'eno1 since' not in desc and 'host unreachable' not in desc
|
||||
|
||||
def test_a_failing_section_still_opens_the_ticket(self, mon, db):
|
||||
db.event_stats.side_effect = RuntimeError('db gone')
|
||||
self._mon(mon)._ticket_interface(1, 'large1', 'enp1s0f0', 2)
|
||||
desc = self._desc(mon)
|
||||
assert '### History' not in desc and '### Where it is connected' in desc
|
||||
db.set_ticket_id.assert_called_with(1, '123')
|
||||
|
||||
def test_cable_mismatch_shows_the_switch_port(self, mon, db):
|
||||
self._mon(mon)._ticket_cable_mismatch(9, {
|
||||
'switch': 'agg', 'port': '7', 'cable_id': 4, 'label': 'C04', 'key': 'cable 4',
|
||||
'detail': 'A different device is connected', 'path': 'agg port 7 → tv'}, 24)
|
||||
desc = self._desc(mon)
|
||||
assert '[C04](https://gandalf.test/cutsheet#cable-4)' in desc
|
||||
assert '[agg port 7 "Large1": up at 10 Gbps' in desc
|
||||
|
||||
def test_titles_keep_the_tag_format(self, mon, db):
|
||||
mon.unifi_fail_thresh = 3
|
||||
self._mon(mon)._ticket_unifi(1, {'name': 'ap1', 'type': 'uap', 'model': 'U6', 'ip': '1.2.3.4'})
|
||||
assert mon.tickets.create.call_args.args[0] == \
|
||||
'[ap1][auto][production][issue][network][single-node] UniFi device offline'
|
||||
|
||||
def test_remember_uplinks_saves_only_on_change(self, mon, db):
|
||||
self._mon(mon)
|
||||
mon._known_uplinks = None
|
||||
mon._remember_uplinks({'unifi_switches': self.SWITCHES})
|
||||
saved = db.set_state.call_args.args
|
||||
assert saved[0] == 'known_uplinks'
|
||||
assert saved[1]['large1']['uplinks'] == [
|
||||
{'switch': 'agg', 'port': 'Large1', 'port_idx': 7, 'host_interface': 'enp1s0f0'}]
|
||||
db.set_state.reset_mock()
|
||||
mon._remember_uplinks({'unifi_switches': self.SWITCHES})
|
||||
assert not db.set_state.called
|
||||
|
||||
|
||||
class TestRecoveryNotes:
|
||||
def _event(self, **kw):
|
||||
now = monitor._utcnow()
|
||||
e = {'id': 8, 'event_type': 'host_metrics_missing', 'target_name': 'micro1', 'target_detail': 'x:9100',
|
||||
'ticket_id': '993', 'first_seen': now - timedelta(minutes=12), 'resolved_at': now,
|
||||
'consecutive_failures': 6}
|
||||
e.update(kw)
|
||||
return e
|
||||
|
||||
def test_cleared_alert_comments_and_marks(self, mon, db):
|
||||
mon.public_url = 'https://gandalf.test'
|
||||
mon.tickets.comments_denied = False
|
||||
mon.tickets.comment.return_value = True
|
||||
db.get_pending_recovery_notes.return_value = [self._event()]
|
||||
mon._post_recovery_notes()
|
||||
ticket_id, text = mon.tickets.comment.call_args.args
|
||||
assert ticket_id == '993'
|
||||
assert 'Cleared' in text and '12 min' in text and 'Failed checks:** 6' in text
|
||||
assert 'https://gandalf.test/device/micro1' in text
|
||||
db.mark_recovery_notified.assert_called_once_with(8)
|
||||
assert db.get_pending_recovery_notes.call_args.kwargs['exclude_types'] == monitor.ONE_SHOT_EVENTS
|
||||
|
||||
def test_failed_comment_is_left_for_next_cycle(self, mon, db):
|
||||
mon.tickets.comments_denied = False
|
||||
mon.tickets.comment.return_value = False
|
||||
db.get_pending_recovery_notes.return_value = [self._event(), self._event(id=9)]
|
||||
mon._post_recovery_notes()
|
||||
assert not db.mark_recovery_notified.called
|
||||
assert mon.tickets.comment.call_count == 2
|
||||
|
||||
def test_denied_key_stops_trying(self, mon, db):
|
||||
mon.tickets.comments_denied = True
|
||||
mon._post_recovery_notes()
|
||||
assert not db.get_pending_recovery_notes.called
|
||||
|
||||
def test_cable_mismatch_links_the_cutsheet(self, mon, db):
|
||||
mon.public_url = 'https://gandalf.test'
|
||||
url = mon._event_url({'event_type': 'cable_mismatch', 'target_name': 'sw', 'target_detail': 'cable 7'})
|
||||
assert url == 'https://gandalf.test/cutsheet#cable-7'
|
||||
|
||||
|
||||
class TestTicketClientComment:
|
||||
def _client(self):
|
||||
return monitor.TicketClient({'url': 'http://t.test/create_ticket_api.php', 'api_key': 'k'})
|
||||
|
||||
def test_comment_endpoint_sits_beside_create(self):
|
||||
assert self._client().comment_url == 'http://t.test/api/ticket_comment_api.php'
|
||||
sub = monitor.TicketClient({'url': 'http://t.test/tinker/create_ticket_api.php', 'api_key': 'k'})
|
||||
assert sub.comment_url == 'http://t.test/tinker/api/ticket_comment_api.php'
|
||||
|
||||
def test_payload(self):
|
||||
c = self._client()
|
||||
resp = MagicMock(status_code=200)
|
||||
resp.json.return_value = {'success': True, 'comment_id': 1}
|
||||
with patch.object(monitor.requests, 'post', return_value=resp) as post:
|
||||
assert c.comment('42', 'hi') is True
|
||||
assert post.call_args.kwargs['json'] == {'ticket_id': '42', 'comment_text': 'hi', 'markdown_enabled': True}
|
||||
assert post.call_args.kwargs['headers'] == {'Authorization': 'Bearer k'}
|
||||
|
||||
def test_forbidden_disables_comments(self):
|
||||
c = self._client()
|
||||
with patch.object(monitor.requests, 'post', return_value=MagicMock(status_code=403)) as post:
|
||||
assert c.comment('42', 'hi') is False
|
||||
assert c.comment('43', 'hi') is False
|
||||
assert post.call_count == 1 and c.comments_denied
|
||||
|
||||
def test_missing_ticket_counts_as_done(self):
|
||||
with patch.object(monitor.requests, 'post', return_value=MagicMock(status_code=404)):
|
||||
assert self._client().comment('42', 'hi') is True
|
||||
|
||||
def test_network_error_is_retryable(self):
|
||||
c = self._client()
|
||||
with patch.object(monitor.requests, 'post', side_effect=OSError('down')):
|
||||
assert c.comment('42', 'hi') is False
|
||||
assert not c.comments_denied
|
||||
Reference in new issue
Block a user