Dashboard: flag stale alerts and tickets left open
Lint / JS (eslint) (push) Successful in 19s
Lint / Python (flake8) (push) Successful in 1m7s
Lint / Notify on failure (push) Skipped
Lint / Deploy (push) Skipped
Security / Python Security (bandit) (push) Successful in 52s
Test / Python Tests (pytest) (push) Successful in 44s
Lint / JS (eslint) (pull_request) Successful in 7s
Lint / Python (flake8) (pull_request) Successful in 50s
Lint / Notify on failure (pull_request) Skipped
Lint / Deploy (pull_request) Skipped
Security / Python Security (bandit) (pull_request) Successful in 52s
Test / Python Tests (pytest) (pull_request) Successful in 1m35s

Gandalf only comments on tickets, so a ticket can stay open long after its
alert cleared (pbs's ticket sat open for a month), and an alert can stay
active for days once nobody is looking.

- The monitor reads Tinker's open ticket list (tickets_api.php, read scope)
  every 15 minutes and saves it. A failed read keeps the previous list.
- Active alerts get a stale badge after monitor.stale_alert_days (3), and a
  "ticket closed" badge when their ticket was closed while still active.
- A Needs Follow-up panel lists open tickets whose every alert cleared at
  least monitor.stale_ticket_hours (24) ago.
- A "TO FOLLOW UP" status chip counts both. If the open ticket list is over
  2 hours old, the ticket flags are left off rather than guessed.
- Replaced an — in the alert count notice.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-11-14 15:57:58 -05:00
co-authored by Claude Opus 5.5
parent 7b80633088
commit 0921e82c54
10 changed files with 462 additions and 4 deletions
+55
View File
@@ -1,5 +1,7 @@
"""Tests for app.py routes: database and LDAP are mocked."""
import json
import time
from datetime import datetime, timedelta, timezone
import os
import sys
from unittest.mock import patch
@@ -648,3 +650,56 @@ class TestMaintenance:
def test_a_db_error_means_no_banner(self, client):
gandalf.db.get_active_suppressions.side_effect = RuntimeError('db down')
assert gandalf.inject_maintenance() == {'maintenance': None}
class TestFollowUp:
def _state(self, tickets, age_s=60):
state = dict(STATE, open_tickets=json.dumps({'checked_at': time.time() - age_s, 'tickets': tickets}))
gandalf.db.get_state.side_effect = lambda key, default=None: state.get(key, default)
def _event(self, days_old, ticket_id=None):
first = (datetime.now(timezone.utc) - timedelta(days=days_old)).strftime('%Y-%m-%dT%H:%M:%SZ')
return {'id': 1, 'event_type': 'interface_down', 'severity': 'critical', 'target_name': 'micro1',
'target_detail': 'enp3s0', 'description': 'down', 'first_seen': first, 'last_seen': first,
'consecutive_failures': 3, 'ticket_id': ticket_id}
def test_old_alert_is_stale_and_closed_ticket_is_flagged(self, client):
self._state({'9': 'Open'})
gandalf.db.get_active_events.return_value = [self._event(5, '8'), self._event(1, '9')]
gandalf.db.get_cleared_tickets.return_value = []
events = client.get('/api/status', headers=AUTH).get_json()['events']
assert events[0]['stale_days'] == 5 and events[0]['ticket_closed'] is True
assert events[1]['stale_days'] is None and events[1]['ticket_closed'] is False
def test_threshold_comes_from_config(self, client):
self._state({})
gandalf.db.get_active_events.return_value = [self._event(1.5)]
gandalf.db.get_cleared_tickets.return_value = []
cfg = dict(CONFIG, monitor={'stale_alert_days': 1})
with patch.object(gandalf, '_cfg', cfg):
events = client.get('/api/status', headers=AUTH).get_json()['events']
assert events[0]['stale_days'] == 1
def test_tickets_left_open_are_listed(self, client):
self._state({'281331508': 'Open'})
left = [{'ticket_id': '281331508', 'event_type': 'host_unreachable', 'target_name': 'pbs',
'target_detail': '', 'severity': 'critical', 'first_seen': '2026-10-14T12:02:05Z',
'cleared_at': '2026-11-04T22:12:36Z', 'events': 3}]
gandalf.db.get_cleared_tickets.return_value = left
assert client.get('/api/status', headers=AUTH).get_json()['stale_tickets'] == left
gandalf.db.get_cleared_tickets.assert_called_with(['281331508'], older_than_hours=24)
html = client.get('/', headers=AUTH).get_data(as_text=True)
assert '#281331508' in html and 'TO FOLLOW UP' in html
assert '<section class="g-section" id="followup">' in html
def test_out_of_date_list_flags_nothing(self, client):
self._state({}, age_s=3 * 3600)
gandalf.db.get_active_events.return_value = [self._event(1, '8')]
body = client.get('/api/status', headers=AUTH).get_json()
assert body['stale_tickets'] == [] and body['events'][0]['ticket_closed'] is False
gandalf.db.get_cleared_tickets.assert_not_called()
def test_no_list_yet_hides_the_panel(self, client):
html = client.get('/', headers=AUTH).get_data(as_text=True)
assert '<section class="g-section" id="followup" hidden>' in html
assert 'TO FOLLOW UP' not in html
+64
View File
@@ -0,0 +1,64 @@
"""Tests for db.py query helpers that do work beyond the SQL: the connection is faked."""
import os
import sys
from contextlib import contextmanager
from datetime import datetime, timedelta
from unittest.mock import MagicMock, patch
sys.path.insert(0, os.path.dirname(os.path.dirname(__file__)))
import db # noqa: E402
NOW = datetime(2026, 11, 14, 12, 0, 0)
def _fake_conn(rows):
cur = MagicMock()
cur.fetchall.return_value = rows
cur.fetchone.return_value = {'now': NOW}
conn = MagicMock()
conn.cursor.return_value.__enter__.return_value = cur
@contextmanager
def get_conn():
yield conn
return patch.object(db, 'get_conn', get_conn), cur
def _event(tid, hours_ago_first, hours_ago_resolved, etype='host_unreachable', target='pbs'):
return {
'ticket_id': tid, 'event_type': etype, 'target_name': target, 'target_detail': '',
'severity': 'critical', 'first_seen': NOW - timedelta(hours=hours_ago_first),
'resolved_at': None if hours_ago_resolved is None else NOW - timedelta(hours=hours_ago_resolved),
}
class TestGetClearedTickets:
def test_no_ids_skips_the_query(self):
patcher, cur = _fake_conn([])
with patcher:
assert db.get_cleared_tickets([]) == []
cur.execute.assert_not_called()
def test_only_tickets_whose_every_alert_cleared_long_enough_ago(self):
rows = [
_event('1', 300, 250), _event('1', 100, 48), # two outages, both long cleared
_event('2', 50, 40), _event('2', 5, None), # still active
_event('3', 10, 2), # cleared too recently
]
patcher, cur = _fake_conn(rows)
with patcher:
out = db.get_cleared_tickets(['3', '1', '2', '1'], older_than_hours=24)
assert [r['ticket_id'] for r in out] == ['1']
assert out[0]['events'] == 2
assert out[0]['cleared_at'] == (NOW - timedelta(hours=48)).isoformat() + 'Z'
assert out[0]['first_seen'] == (NOW - timedelta(hours=300)).isoformat() + 'Z'
assert cur.execute.call_args_list[0].args[1] == ['1', '2', '3']
def test_newest_clearance_first(self):
rows = [_event('1', 400, 300), _event('2', 100, 30, target='micro1')]
patcher, _ = _fake_conn(rows)
with patcher:
out = db.get_cleared_tickets(['1', '2'])
assert [r['ticket_id'] for r in out] == ['2', '1']
assert out[0]['target_name'] == 'micro1'
+57
View File
@@ -913,3 +913,60 @@ class TestTicketClientCreate:
with self._post({'success': True, 'ticket_id': '8', 'action': 'none'}):
assert c.create('t', 'd') == '8'
assert 't' not in c._closed_matches
class TestTicketClientOpenTickets:
def _client(self):
return monitor.TicketClient({'url': 'http://t.test/create_ticket_api.php', 'api_key': 'k'})
def _page(self, tickets, pages):
resp = MagicMock(status_code=200)
resp.json.return_value = {'success': True, 'tickets': tickets, 'page': 1, 'pages': pages}
return resp
def test_read_endpoint_sits_beside_create(self):
assert self._client().read_url == 'http://t.test/api/tickets_api.php'
def test_reads_every_page_of_open_statuses(self):
pages = [self._page([{'ticket_id': '1', 'status': 'Open'}], 2),
self._page([{'ticket_id': 2, 'status': 'Pending'}], 2)]
with patch.object(monitor.requests, 'get', side_effect=pages) as get:
assert self._client().open_tickets() == {'1': 'Open', '2': 'Pending'}
assert get.call_count == 2
params = get.call_args.kwargs['params']
assert params['status'] == 'Open,Pending,In Progress' and params['page'] == 2
assert get.call_args.kwargs['headers'] == {'Authorization': 'Bearer k'}
def test_failure_means_unknown_not_empty(self):
with patch.object(monitor.requests, 'get', side_effect=OSError('down')):
assert self._client().open_tickets() is None
bad = MagicMock(status_code=200)
bad.json.return_value = {'success': False, 'error': 'nope'}
with patch.object(monitor.requests, 'get', return_value=bad):
assert self._client().open_tickets() is None
def test_unconfigured(self):
assert monitor.TicketClient({}).open_tickets() is None
class TestSyncOpenTickets:
def test_saves_the_list_then_waits(self, mon, db):
mon.tickets.open_tickets.return_value = {'7': 'Open'}
mon._sync_open_tickets()
mon._sync_open_tickets()
assert mon.tickets.open_tickets.call_count == 1
key, value = db.set_state.call_args.args
assert key == monitor.OPEN_TICKETS_STATE and value['tickets'] == {'7': 'Open'}
assert value['checked_at'] > 0
def test_failed_read_keeps_the_previous_list(self, mon, db):
mon.tickets.open_tickets.return_value = None
mon._sync_open_tickets()
db.set_state.assert_not_called()
def test_refreshes_after_the_interval(self, mon, db):
mon.tickets.open_tickets.return_value = {}
mon._sync_open_tickets()
mon._open_tickets_synced -= monitor.OPEN_TICKETS_SYNC_S + 1
mon._sync_open_tickets()
assert mon.tickets.open_tickets.call_count == 2