Add recovery notes and fuller context to tickets
Lint / JS (eslint) (push) Successful in 7s
Lint / Python (flake8) (push) Successful in 1m26s
Lint / Notify on failure (push) Skipped
Lint / Deploy (push) Skipped
Security / Python Security (bandit) (push) Successful in 1m30s
Lint / Python (flake8) (pull_request) Successful in 52s
Test / Python Tests (pytest) (push) Successful in 1m4s
Lint / JS (eslint) (pull_request) Successful in 7s
Lint / Notify on failure (pull_request) Skipped
Lint / Deploy (pull_request) Skipped
Security / Python Security (bandit) (pull_request) Successful in 1m28s
Test / Python Tests (pytest) (pull_request) Successful in 2m0s

When an alert that opened a ticket clears, the monitor now comments on the
ticket (when it cleared, how long it lasted, failed checks, a link back).
It never closes or changes a ticket's status. Failed notes retry each
cycle for 24 hours; a 401/403 from Tinker stops them until restart.

Tickets are now Markdown with a "What to check" list and, where known:
the switch port the device plugs into (kept from the last time it was
seen, so a dead host still shows it), cutsheet cables and their path,
how often the alert fired in the last 7/30 days, other open alerts on the
device, and links into Gandalf (gandalf.public_url). Titles are unchanged
so Tinker's duplicate detection still works.

network_events gains recovery_notified, added at monitor startup; rows
that cleared before the upgrade count as already notified.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
jaredandClaude Opus 5.5 committed 2026-11-10 19:44:50 -05:00
1 parent ff0094c8ff
commit 437f7a415f
8 files changed
+886 -133

No files matched your search

+180
View File
@@ -3,6 +3,7 @@ import json
import os
import sys
import time
from datetime import timedelta
from unittest.mock import MagicMock, patch
import pytest
@@ -24,6 +25,11 @@ def db():
mock_db.upsert_event.return_value = (1, False, 2, None)
mock_db.get_modem_rx_values.return_value = []
mock_db.get_state.return_value = None
mock_db.event_stats.return_value = {'last_7d': 0, 'last_30d': 0,
'last_first_seen': None, 'last_resolved_at': None}
mock_db.get_events_for_target.return_value = []
mock_db.cutsheet_load.return_value = {'items': [], 'cables': [], 'port_labels': []}
mock_db.get_pending_recovery_notes.return_value = []
yield mock_db
@@ -560,3 +566,177 @@ class TestCutsheetMismatch:
assert not db.upsert_event.called
resolved = {c.args for c in db.resolve_event.call_args_list}
assert ('cable_mismatch', 'sw', 'cable 7') in resolved and ('cable_mismatch', 'other', 'cable 9') in resolved
class TestTicketContext:
SWITCHES = {'agg': {'ports': {
'Large1': {'port_idx': 7, 'up': True, 'speed_mbps': 10000,
'lldp': {'system_name': 'large1', 'port_id': 'enp1s0f0'},
'sfp': {'found': True, 'kind': 'dac', 'part': 'DAC-10G'}},
'Port 3': {'port_idx': 3, 'up': False, 'speed_mbps': 0},
}}}
def _mon(self, mon, switches=None):
mon.public_url = 'https://gandalf.test'
mon.poll_interval = 120
mon._link_data = {'unifi_switches': switches if switches is not None else self.SWITCHES}
return mon
def _desc(self, mon):
return mon.tickets.create.call_args.args[1]
def test_interface_ticket_names_the_switch_port_and_links(self, mon, db):
self._mon(mon)._ticket_interface(1, 'large1', 'enp1s0f0', 2)
desc = self._desc(mon)
assert '### Where it is connected' in desc
assert '`enp1s0f0` → [agg port 7 "Large1": up at 10 Gbps, module: DAC DAC-10G]' in desc
assert '(https://gandalf.test/device/agg#port-7)' in desc
assert 'https://gandalf.test/device/large1' in desc
assert 'Opened automatically by Gandalf' in desc
def test_dead_host_uses_the_last_known_port(self, mon, db):
mon._known_uplinks = {'large1': {'seen': '2026-11-10 10:00:00 UTC', 'uplinks': [
{'switch': 'agg', 'port': 'Port 3', 'port_idx': 3, 'host_interface': 'eno1'}]}}
self._mon(mon)._ticket_metrics_missing(1, 'micro1', 'x:9100', 5)
assert 'not seen this device on a switch port' in self._desc(mon)
mon.tickets.create.reset_mock()
unplugged = {'agg': {'ports': {'Port 3': {'port_idx': 3, 'up': False}}}}
self._mon(mon, unplugged)._ticket_unreachable(1, 'large1', '10.0.0.9', 5)
desc = self._desc(mon)
assert 'agg port 3: down' in desc
assert 'last seen linked there 2026-11-10 10:00:00 UTC' in desc
assert 'Down for at least' in desc and '10 min' in desc
def test_cutsheet_runs_and_history(self, mon, db):
db.cutsheet_load.return_value = {'items': [], 'port_labels': [], 'cables': [
{'id': 4, 'label': 'C04', 'a_device': 'agg', 'a_port': '7', 'a_side': '',
'b_device': 'large1', 'b_port': 'enp1s0f0', 'b_side': '', 'status': 'installed'}]}
db.event_stats.return_value = {'last_7d': 3, 'last_30d': 5,
'last_first_seen': monitor._utcnow(), 'last_resolved_at': monitor._utcnow()}
self._mon(mon)._ticket_interface(1, 'large1', 'enp1s0f0', 2)
desc = self._desc(mon)
assert '[**C04**](https://gandalf.test/cutsheet#cable-4): agg port 7 → large1 enp1s0f0' in desc
assert 'Happened 3 times in the last 7 days and 5 in the last 30' in desc
assert 'This keeps coming back' in desc
def test_other_open_alerts_are_listed(self, mon, db):
db.get_events_for_target.return_value = [
{'id': 1, 'event_type': 'interface_down', 'target_name': 'large1', 'target_detail': 'eno1'},
{'id': 2, 'event_type': 'interface_down', 'target_name': 'large1', 'target_detail': 'eno2',
'first_seen': 'T', 'ticket_id': '55'},
{'id': 3, 'event_type': 'host_unreachable', 'target_name': 'large1', 'resolved_at': 'R'},
]
self._mon(mon)._ticket_interface(1, 'large1', 'eno1', 2)
desc = self._desc(mon)
assert '- interface down on large1 eno2 since T (ticket #55)' in desc
assert 'eno1 since' not in desc and 'host unreachable' not in desc
def test_a_failing_section_still_opens_the_ticket(self, mon, db):
db.event_stats.side_effect = RuntimeError('db gone')
self._mon(mon)._ticket_interface(1, 'large1', 'enp1s0f0', 2)
desc = self._desc(mon)
assert '### History' not in desc and '### Where it is connected' in desc
db.set_ticket_id.assert_called_with(1, '123')
def test_cable_mismatch_shows_the_switch_port(self, mon, db):
self._mon(mon)._ticket_cable_mismatch(9, {
'switch': 'agg', 'port': '7', 'cable_id': 4, 'label': 'C04', 'key': 'cable 4',
'detail': 'A different device is connected', 'path': 'agg port 7 → tv'}, 24)
desc = self._desc(mon)
assert '[C04](https://gandalf.test/cutsheet#cable-4)' in desc
assert '[agg port 7 "Large1": up at 10 Gbps' in desc
def test_titles_keep_the_tag_format(self, mon, db):
mon.unifi_fail_thresh = 3
self._mon(mon)._ticket_unifi(1, {'name': 'ap1', 'type': 'uap', 'model': 'U6', 'ip': '1.2.3.4'})
assert mon.tickets.create.call_args.args[0] == \
'[ap1][auto][production][issue][network][single-node] UniFi device offline'
def test_remember_uplinks_saves_only_on_change(self, mon, db):
self._mon(mon)
mon._known_uplinks = None
mon._remember_uplinks({'unifi_switches': self.SWITCHES})
saved = db.set_state.call_args.args
assert saved[0] == 'known_uplinks'
assert saved[1]['large1']['uplinks'] == [
{'switch': 'agg', 'port': 'Large1', 'port_idx': 7, 'host_interface': 'enp1s0f0'}]
db.set_state.reset_mock()
mon._remember_uplinks({'unifi_switches': self.SWITCHES})
assert not db.set_state.called
class TestRecoveryNotes:
def _event(self, **kw):
now = monitor._utcnow()
e = {'id': 8, 'event_type': 'host_metrics_missing', 'target_name': 'micro1', 'target_detail': 'x:9100',
'ticket_id': '993', 'first_seen': now - timedelta(minutes=12), 'resolved_at': now,
'consecutive_failures': 6}
e.update(kw)
return e
def test_cleared_alert_comments_and_marks(self, mon, db):
mon.public_url = 'https://gandalf.test'
mon.tickets.comments_denied = False
mon.tickets.comment.return_value = True
db.get_pending_recovery_notes.return_value = [self._event()]
mon._post_recovery_notes()
ticket_id, text = mon.tickets.comment.call_args.args
assert ticket_id == '993'
assert 'Cleared' in text and '12 min' in text and 'Failed checks:** 6' in text
assert 'https://gandalf.test/device/micro1' in text
db.mark_recovery_notified.assert_called_once_with(8)
assert db.get_pending_recovery_notes.call_args.kwargs['exclude_types'] == monitor.ONE_SHOT_EVENTS
def test_failed_comment_is_left_for_next_cycle(self, mon, db):
mon.tickets.comments_denied = False
mon.tickets.comment.return_value = False
db.get_pending_recovery_notes.return_value = [self._event(), self._event(id=9)]
mon._post_recovery_notes()
assert not db.mark_recovery_notified.called
assert mon.tickets.comment.call_count == 2
def test_denied_key_stops_trying(self, mon, db):
mon.tickets.comments_denied = True
mon._post_recovery_notes()
assert not db.get_pending_recovery_notes.called
def test_cable_mismatch_links_the_cutsheet(self, mon, db):
mon.public_url = 'https://gandalf.test'
url = mon._event_url({'event_type': 'cable_mismatch', 'target_name': 'sw', 'target_detail': 'cable 7'})
assert url == 'https://gandalf.test/cutsheet#cable-7'
class TestTicketClientComment:
def _client(self):
return monitor.TicketClient({'url': 'http://t.test/create_ticket_api.php', 'api_key': 'k'})
def test_comment_endpoint_sits_beside_create(self):
assert self._client().comment_url == 'http://t.test/api/ticket_comment_api.php'
sub = monitor.TicketClient({'url': 'http://t.test/tinker/create_ticket_api.php', 'api_key': 'k'})
assert sub.comment_url == 'http://t.test/tinker/api/ticket_comment_api.php'
def test_payload(self):
c = self._client()
resp = MagicMock(status_code=200)
resp.json.return_value = {'success': True, 'comment_id': 1}
with patch.object(monitor.requests, 'post', return_value=resp) as post:
assert c.comment('42', 'hi') is True
assert post.call_args.kwargs['json'] == {'ticket_id': '42', 'comment_text': 'hi', 'markdown_enabled': True}
assert post.call_args.kwargs['headers'] == {'Authorization': 'Bearer k'}
def test_forbidden_disables_comments(self):
c = self._client()
with patch.object(monitor.requests, 'post', return_value=MagicMock(status_code=403)) as post:
assert c.comment('42', 'hi') is False
assert c.comment('43', 'hi') is False
assert post.call_count == 1 and c.comments_denied
def test_missing_ticket_counts_as_done(self):
with patch.object(monitor.requests, 'post', return_value=MagicMock(status_code=404)):
assert self._client().comment('42', 'hi') is True
def test_network_error_is_retryable(self):
c = self._client()
with patch.object(monitor.requests, 'post', side_effect=OSError('down')):
assert c.comment('42', 'hi') is False
assert not c.comments_denied
+64
View File
@@ -0,0 +1,64 @@
"""Tests for ticket_text.py: the Markdown Gandalf puts in tickets."""
import os
import sys
from datetime import datetime, timedelta
sys.path.insert(0, os.path.dirname(os.path.dirname(__file__)))
import ticket_text as tt # noqa: E402
NOW = datetime(2026, 11, 10, 18, 0)
def test_duration():
assert tt.duration(42) == '42 s'
assert tt.duration(600) == '10 min'
assert tt.duration(3600) == '1 h'
assert tt.duration(3 * 3600 + 120) == '3 h 2 min'
assert tt.duration(3 * 86400 + 7200) == '3 d 2 h'
def test_speed():
assert tt.speed(100) == '100 Mbps'
assert tt.speed(2500) == '2.5 Gbps'
assert tt.speed(10000) == '10 Gbps'
assert tt.speed(0) == ''
def test_device_url_escapes_names():
assert tt.device_url('https://g', 'USW Flex Mini', 3) == 'https://g/device/USW%20Flex%20Mini#port-3'
assert tt.device_url('', 'x') == ''
def test_port_text():
assert tt.port_text('sw', 'Port 4', {'port_idx': 4, 'up': True, 'speed_mbps': 1000, 'poe_power': 4.21}) == \
'sw port 4: up at 1 Gbps, PoE 4.2 W'
assert tt.port_text('sw', 'Cam', {'port_idx': 9, 'up': False,
'sfp': {'found': True, 'kind': 'optic', 'part': 'SR', 'rx_fault': True}}) == \
'sw port 9 "Cam": down, module: optic SR, **module reports a fault**'
def test_description_skips_empty_facts_and_sections():
text = tt.description('Alert', [('Host', 'a'), ('Blank', '')], 'Summary.', ['One', 'Two'],
[('Empty', []), ('Links', ['- x'])])
assert text.startswith('## Alert\n\n- **Host:** a\n\nSummary.')
assert 'Blank' not in text and '### Empty' not in text
assert '1. One\n2. Two' in text and '### Links\n- x' in text
def test_history_lines():
assert tt.history_lines({'last_7d': 0, 'last_30d': 0}, NOW) == ['- First time in the last 30 days.']
lines = tt.history_lines({'last_7d': 1, 'last_30d': 2, 'last_first_seen': NOW - timedelta(hours=26),
'last_resolved_at': NOW - timedelta(hours=25, minutes=55)}, NOW)
assert lines[0].startswith('- Happened 1 time in the last 7 days and 2')
assert lines[1] == '- Last cleared 2026-11-09 16:05 UTC (25 h 55 min ago), after 5 min.'
def test_recovery_comment():
text = tt.recovery_comment({'event_type': 'interface_down', 'target_name': 'large1', 'target_detail': 'eno1',
'first_seen': NOW - timedelta(minutes=7), 'resolved_at': NOW,
'consecutive_failures': 3}, NOW, 'https://g/device/large1')
assert text.startswith('**✅ Cleared**: interface down on large1 eno1.')
assert '- **Lasted:** 7 min (first seen 2026-11-10 17:53 UTC)' in text
assert '- **Gandalf:** https://g/device/large1' in text
assert 'leaves this ticket open' in text