Dashboard: flag stale alerts and tickets left open
Lint / JS (eslint) (push) Successful in 19s
Lint / Python (flake8) (push) Successful in 1m7s
Lint / Notify on failure (push) Skipped
Lint / Deploy (push) Skipped
Security / Python Security (bandit) (push) Successful in 52s
Test / Python Tests (pytest) (push) Successful in 44s
Lint / JS (eslint) (pull_request) Successful in 7s
Lint / Python (flake8) (pull_request) Successful in 50s
Lint / Notify on failure (pull_request) Skipped
Lint / Deploy (pull_request) Skipped
Security / Python Security (bandit) (pull_request) Successful in 52s
Test / Python Tests (pytest) (pull_request) Successful in 1m35s
Lint / JS (eslint) (push) Successful in 19s
Lint / Python (flake8) (push) Successful in 1m7s
Lint / Notify on failure (push) Skipped
Lint / Deploy (push) Skipped
Security / Python Security (bandit) (push) Successful in 52s
Test / Python Tests (pytest) (push) Successful in 44s
Lint / JS (eslint) (pull_request) Successful in 7s
Lint / Python (flake8) (pull_request) Successful in 50s
Lint / Notify on failure (pull_request) Skipped
Lint / Deploy (pull_request) Skipped
Security / Python Security (bandit) (pull_request) Successful in 52s
Test / Python Tests (pytest) (pull_request) Successful in 1m35s
Gandalf only comments on tickets, so a ticket can stay open long after its alert cleared (pbs's ticket sat open for a month), and an alert can stay active for days once nobody is looking. - The monitor reads Tinker's open ticket list (tickets_api.php, read scope) every 15 minutes and saves it. A failed read keeps the previous list. - Active alerts get a stale badge after monitor.stale_alert_days (3), and a "ticket closed" badge when their ticket was closed while still active. - A Needs Follow-up panel lists open tickets whose every alert cleared at least monitor.stale_ticket_hours (24) ago. - A "TO FOLLOW UP" status chip counts both. If the open ticket list is over 2 hours old, the ticket flags are left off rather than guessed. - Replaced an — in the alert count notice. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -1,5 +1,7 @@
|
||||
"""Tests for app.py routes: database and LDAP are mocked."""
|
||||
import json
|
||||
import time
|
||||
from datetime import datetime, timedelta, timezone
|
||||
import os
|
||||
import sys
|
||||
from unittest.mock import patch
|
||||
@@ -648,3 +650,56 @@ class TestMaintenance:
|
||||
def test_a_db_error_means_no_banner(self, client):
|
||||
gandalf.db.get_active_suppressions.side_effect = RuntimeError('db down')
|
||||
assert gandalf.inject_maintenance() == {'maintenance': None}
|
||||
|
||||
|
||||
class TestFollowUp:
|
||||
def _state(self, tickets, age_s=60):
|
||||
state = dict(STATE, open_tickets=json.dumps({'checked_at': time.time() - age_s, 'tickets': tickets}))
|
||||
gandalf.db.get_state.side_effect = lambda key, default=None: state.get(key, default)
|
||||
|
||||
def _event(self, days_old, ticket_id=None):
|
||||
first = (datetime.now(timezone.utc) - timedelta(days=days_old)).strftime('%Y-%m-%dT%H:%M:%SZ')
|
||||
return {'id': 1, 'event_type': 'interface_down', 'severity': 'critical', 'target_name': 'micro1',
|
||||
'target_detail': 'enp3s0', 'description': 'down', 'first_seen': first, 'last_seen': first,
|
||||
'consecutive_failures': 3, 'ticket_id': ticket_id}
|
||||
|
||||
def test_old_alert_is_stale_and_closed_ticket_is_flagged(self, client):
|
||||
self._state({'9': 'Open'})
|
||||
gandalf.db.get_active_events.return_value = [self._event(5, '8'), self._event(1, '9')]
|
||||
gandalf.db.get_cleared_tickets.return_value = []
|
||||
events = client.get('/api/status', headers=AUTH).get_json()['events']
|
||||
assert events[0]['stale_days'] == 5 and events[0]['ticket_closed'] is True
|
||||
assert events[1]['stale_days'] is None and events[1]['ticket_closed'] is False
|
||||
|
||||
def test_threshold_comes_from_config(self, client):
|
||||
self._state({})
|
||||
gandalf.db.get_active_events.return_value = [self._event(1.5)]
|
||||
gandalf.db.get_cleared_tickets.return_value = []
|
||||
cfg = dict(CONFIG, monitor={'stale_alert_days': 1})
|
||||
with patch.object(gandalf, '_cfg', cfg):
|
||||
events = client.get('/api/status', headers=AUTH).get_json()['events']
|
||||
assert events[0]['stale_days'] == 1
|
||||
|
||||
def test_tickets_left_open_are_listed(self, client):
|
||||
self._state({'281331508': 'Open'})
|
||||
left = [{'ticket_id': '281331508', 'event_type': 'host_unreachable', 'target_name': 'pbs',
|
||||
'target_detail': '', 'severity': 'critical', 'first_seen': '2026-10-14T12:02:05Z',
|
||||
'cleared_at': '2026-11-04T22:12:36Z', 'events': 3}]
|
||||
gandalf.db.get_cleared_tickets.return_value = left
|
||||
assert client.get('/api/status', headers=AUTH).get_json()['stale_tickets'] == left
|
||||
gandalf.db.get_cleared_tickets.assert_called_with(['281331508'], older_than_hours=24)
|
||||
html = client.get('/', headers=AUTH).get_data(as_text=True)
|
||||
assert '#281331508' in html and 'TO FOLLOW UP' in html
|
||||
assert '<section class="g-section" id="followup">' in html
|
||||
|
||||
def test_out_of_date_list_flags_nothing(self, client):
|
||||
self._state({}, age_s=3 * 3600)
|
||||
gandalf.db.get_active_events.return_value = [self._event(1, '8')]
|
||||
body = client.get('/api/status', headers=AUTH).get_json()
|
||||
assert body['stale_tickets'] == [] and body['events'][0]['ticket_closed'] is False
|
||||
gandalf.db.get_cleared_tickets.assert_not_called()
|
||||
|
||||
def test_no_list_yet_hides_the_panel(self, client):
|
||||
html = client.get('/', headers=AUTH).get_data(as_text=True)
|
||||
assert '<section class="g-section" id="followup" hidden>' in html
|
||||
assert 'TO FOLLOW UP' not in html
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
"""Tests for db.py query helpers that do work beyond the SQL: the connection is faked."""
|
||||
import os
|
||||
import sys
|
||||
from contextlib import contextmanager
|
||||
from datetime import datetime, timedelta
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(__file__)))
|
||||
|
||||
import db # noqa: E402
|
||||
|
||||
NOW = datetime(2026, 11, 14, 12, 0, 0)
|
||||
|
||||
|
||||
def _fake_conn(rows):
|
||||
cur = MagicMock()
|
||||
cur.fetchall.return_value = rows
|
||||
cur.fetchone.return_value = {'now': NOW}
|
||||
conn = MagicMock()
|
||||
conn.cursor.return_value.__enter__.return_value = cur
|
||||
|
||||
@contextmanager
|
||||
def get_conn():
|
||||
yield conn
|
||||
return patch.object(db, 'get_conn', get_conn), cur
|
||||
|
||||
|
||||
def _event(tid, hours_ago_first, hours_ago_resolved, etype='host_unreachable', target='pbs'):
|
||||
return {
|
||||
'ticket_id': tid, 'event_type': etype, 'target_name': target, 'target_detail': '',
|
||||
'severity': 'critical', 'first_seen': NOW - timedelta(hours=hours_ago_first),
|
||||
'resolved_at': None if hours_ago_resolved is None else NOW - timedelta(hours=hours_ago_resolved),
|
||||
}
|
||||
|
||||
|
||||
class TestGetClearedTickets:
|
||||
def test_no_ids_skips_the_query(self):
|
||||
patcher, cur = _fake_conn([])
|
||||
with patcher:
|
||||
assert db.get_cleared_tickets([]) == []
|
||||
cur.execute.assert_not_called()
|
||||
|
||||
def test_only_tickets_whose_every_alert_cleared_long_enough_ago(self):
|
||||
rows = [
|
||||
_event('1', 300, 250), _event('1', 100, 48), # two outages, both long cleared
|
||||
_event('2', 50, 40), _event('2', 5, None), # still active
|
||||
_event('3', 10, 2), # cleared too recently
|
||||
]
|
||||
patcher, cur = _fake_conn(rows)
|
||||
with patcher:
|
||||
out = db.get_cleared_tickets(['3', '1', '2', '1'], older_than_hours=24)
|
||||
assert [r['ticket_id'] for r in out] == ['1']
|
||||
assert out[0]['events'] == 2
|
||||
assert out[0]['cleared_at'] == (NOW - timedelta(hours=48)).isoformat() + 'Z'
|
||||
assert out[0]['first_seen'] == (NOW - timedelta(hours=300)).isoformat() + 'Z'
|
||||
assert cur.execute.call_args_list[0].args[1] == ['1', '2', '3']
|
||||
|
||||
def test_newest_clearance_first(self):
|
||||
rows = [_event('1', 400, 300), _event('2', 100, 30, target='micro1')]
|
||||
patcher, _ = _fake_conn(rows)
|
||||
with patcher:
|
||||
out = db.get_cleared_tickets(['1', '2'])
|
||||
assert [r['ticket_id'] for r in out] == ['2', '1']
|
||||
assert out[0]['target_name'] == 'micro1'
|
||||
@@ -913,3 +913,60 @@ class TestTicketClientCreate:
|
||||
with self._post({'success': True, 'ticket_id': '8', 'action': 'none'}):
|
||||
assert c.create('t', 'd') == '8'
|
||||
assert 't' not in c._closed_matches
|
||||
|
||||
|
||||
class TestTicketClientOpenTickets:
|
||||
def _client(self):
|
||||
return monitor.TicketClient({'url': 'http://t.test/create_ticket_api.php', 'api_key': 'k'})
|
||||
|
||||
def _page(self, tickets, pages):
|
||||
resp = MagicMock(status_code=200)
|
||||
resp.json.return_value = {'success': True, 'tickets': tickets, 'page': 1, 'pages': pages}
|
||||
return resp
|
||||
|
||||
def test_read_endpoint_sits_beside_create(self):
|
||||
assert self._client().read_url == 'http://t.test/api/tickets_api.php'
|
||||
|
||||
def test_reads_every_page_of_open_statuses(self):
|
||||
pages = [self._page([{'ticket_id': '1', 'status': 'Open'}], 2),
|
||||
self._page([{'ticket_id': 2, 'status': 'Pending'}], 2)]
|
||||
with patch.object(monitor.requests, 'get', side_effect=pages) as get:
|
||||
assert self._client().open_tickets() == {'1': 'Open', '2': 'Pending'}
|
||||
assert get.call_count == 2
|
||||
params = get.call_args.kwargs['params']
|
||||
assert params['status'] == 'Open,Pending,In Progress' and params['page'] == 2
|
||||
assert get.call_args.kwargs['headers'] == {'Authorization': 'Bearer k'}
|
||||
|
||||
def test_failure_means_unknown_not_empty(self):
|
||||
with patch.object(monitor.requests, 'get', side_effect=OSError('down')):
|
||||
assert self._client().open_tickets() is None
|
||||
bad = MagicMock(status_code=200)
|
||||
bad.json.return_value = {'success': False, 'error': 'nope'}
|
||||
with patch.object(monitor.requests, 'get', return_value=bad):
|
||||
assert self._client().open_tickets() is None
|
||||
|
||||
def test_unconfigured(self):
|
||||
assert monitor.TicketClient({}).open_tickets() is None
|
||||
|
||||
|
||||
class TestSyncOpenTickets:
|
||||
def test_saves_the_list_then_waits(self, mon, db):
|
||||
mon.tickets.open_tickets.return_value = {'7': 'Open'}
|
||||
mon._sync_open_tickets()
|
||||
mon._sync_open_tickets()
|
||||
assert mon.tickets.open_tickets.call_count == 1
|
||||
key, value = db.set_state.call_args.args
|
||||
assert key == monitor.OPEN_TICKETS_STATE and value['tickets'] == {'7': 'Open'}
|
||||
assert value['checked_at'] > 0
|
||||
|
||||
def test_failed_read_keeps_the_previous_list(self, mon, db):
|
||||
mon.tickets.open_tickets.return_value = None
|
||||
mon._sync_open_tickets()
|
||||
db.set_state.assert_not_called()
|
||||
|
||||
def test_refreshes_after_the_interval(self, mon, db):
|
||||
mon.tickets.open_tickets.return_value = {}
|
||||
mon._sync_open_tickets()
|
||||
mon._open_tickets_synced -= monitor.OPEN_TICKETS_SYNC_S + 1
|
||||
mon._sync_open_tickets()
|
||||
assert mon.tickets.open_tickets.call_count == 2
|
||||
|
||||
Reference in New Issue
Block a user