Compare commits

...
Author SHA1 Message Date
jaredandClaude Opus 5.5 d234e36e3f Tickets: don't adopt a Closed ticket Tinker couldn't reopen
Lint / JS (eslint) (push) Successful in 14s
Lint / Python (flake8) (push) Successful in 57s
Lint / Notify on failure (push) Skipped
Lint / Deploy (push) Skipped
Security / Python Security (bandit) (push) Successful in 55s
Lint / Python (flake8) (pull_request) Successful in 1m31s
Test / Python Tests (pytest) (push) Successful in 1m51s
Lint / JS (eslint) (pull_request) Successful in 7s
Lint / Notify on failure (pull_request) Skipped
Lint / Deploy (pull_request) Skipped
Security / Python Security (bandit) (pull_request) Successful in 1m22s
Test / Python Tests (pytest) (pull_request) Successful in 1m46s
Tinker's create response says what it did (action). For recurrence_noted it
matched a Closed ticket it could not reopen unattended: that ticket isn't
tracking the alert (this is what hid micro1 enp3s0 behind an old reboot
ticket), so Gandalf now logs it and keeps no ticket for the alert. Each such
create adds a "recurred" comment to the closed ticket, so the same title is
retried once a day rather than every poll. Other actions are logged.

See tinker_tickets#116 and #115.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-11 01:52:53 -05:00
jared fde1d01f27 Merge pull request 'Revert #31: revived glitch alerts opened tickets' (#33) from revert-31 into main
Lint / Python (flake8) (push) Successful in 57s
Lint / JS (eslint) (push) Successful in 7s
Lint / Notify on failure (push) Skipped
Security / Python Security (bandit) (push) Successful in 1m50s
Test / Python Tests (pytest) (push) Successful in 1m59s
Lint / Deploy (push) Successful in 7s
2026-11-11 01:17:47 -05:00
jaredandClaude Opus 5.5 8edaa78ad7 Revert #31: open link-down alerts from a monitor glitch were revived and ticketed
Lint / JS (eslint) (push) Successful in 12s
Lint / Python (flake8) (push) Successful in 1m47s
Lint / Notify on failure (push) Skipped
Lint / Deploy (push) Skipped
Security / Python Security (bandit) (push) Successful in 1m15s
Lint / JS (eslint) (pull_request) Successful in 7s
Lint / Python (flake8) (pull_request) Successful in 48s
Lint / Notify on failure (pull_request) Skipped
Lint / Deploy (pull_request) Skipped
Test / Python Tests (pytest) (push) Successful in 1m41s
Security / Python Security (bandit) (pull_request) Successful in 1m32s
Test / Python Tests (pytest) (pull_request) Successful in 1m6s
#31 treated any open interface_down alert as previously up. Production had
six such alerts all created in the same second on 2026-03-02 (a monitor
glitch, on ports that were never in use: large1 enp36s0 and a USB adapter,
compute-storage-01 nic1-3, micro1 enp3s0). Reviving them opened per-host
tickets and a P1 cluster-wide ticket. Back to the previous behaviour while a
safer fix is worked out.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-11 01:11:00 -05:00
jared a0e062493d Merge pull request 'Monitor: keep checking interfaces whose link-down alert is still open' (#31) from stale-interface-alerts into main
Lint / JS (eslint) (push) Successful in 21s
Lint / Python (flake8) (push) Successful in 1m44s
Lint / Notify on failure (push) Skipped
Security / Python Security (bandit) (push) Successful in 1m25s
Lint / Deploy (push) Successful in 5s
Test / Python Tests (pytest) (push) Successful in 1m2s
2026-11-11 01:06:37 -05:00
2 changed files with 60 additions and 26 deletions

No files matched your search

+25 -7
View File
@@ -393,6 +393,8 @@ class TicketClient:
self.comment_url = cfg.get('comment_url') or (
urljoin(self.url, 'api/ticket_comment_api.php') if self.url else '')
self.comments_denied = False
# Titles whose create matched a Closed ticket Tinker couldn't reopen: {title: (ticket_id, when)}
self._closed_matches: Dict[str, tuple] = {}
def comment(self, ticket_id: str, text: str) -> bool:
"""Add a Markdown comment to a ticket. Never raises; False means try again later."""
@@ -425,10 +427,23 @@ class TicketClient:
logger.error(f'Ticket comment failed for #{ticket_id}: {e}')
return False
# How long to wait before asking again after a create matched an unreopenable Closed ticket.
# Each ask adds a "recurred" comment to that ticket, so once a poll would pile them up.
CLOSED_MATCH_RETRY_S = 24 * 3600
def create(self, title: str, description: str, priority: str = '2') -> Optional[str]:
"""Open a ticket. Returns its ID, or None if there is no ticket to track.
Tinker deduplicates: the response's `action` says what happened (new, deduplicated,
updated, reopened, recurrence_noted). recurrence_noted means it matched a Closed ticket
it could not reopen unattended; that ticket isn't tracking this alert, so it isn't used.
"""
if not self.api_key or not self.url:
logger.warning('Ticket API not configured: skipping ticket creation')
return None
seen = self._closed_matches.get(title)
if seen and time.time() - seen[1] < self.CLOSED_MATCH_RETRY_S:
return None
try:
resp = requests.post(
self.url,
@@ -450,7 +465,16 @@ class TicketClient:
if not tid:
logger.warning(f'Ticket API success but no ticket_id in response: {data}')
return None
logger.info(f'Created ticket #{tid}: {title}')
action = data.get('action') or 'created'
if action == 'recurrence_noted':
self._closed_matches[title] = (tid, time.time())
logger.warning(
f'Ticket API matched Closed ticket #{tid}, which it could not reopen; not using it '
f'for this alert. Reopen #{tid} in Tinker to track it there. Retrying in 24 h: {title}'
)
return None
self._closed_matches.pop(title, None)
logger.info(f'Ticket #{tid} ({action}): {title}')
return tid
if data.get('existing_ticket_id'):
logger.info(f'Duplicate suppressed by API: existing #{data["existing_ticket_id"]}')
@@ -1195,10 +1219,6 @@ class NetworkMonitor:
def _process_interfaces(self, states: Dict[str, Dict[str, bool]], suppressions: list) -> None:
baseline = db.get_baseline()
new_baseline = {k: dict(v) for k, v in baseline.items()}
# An interface with an open link-down alert was up before it went down, whatever the
# baseline says now: after a baseline reset it reads as "first seen down", which would
# leave the alert open forever without updating, ticketing or resolving it.
open_down = set(db.get_open_event_targets('interface_down'))
# Only count hosts with genuine regressions (UP→DOWN) toward cluster threshold
hosts_with_regression: List[str] = []
@@ -1211,8 +1231,6 @@ class NetworkMonitor:
for iface, is_up in ifaces.items():
prev = baseline.get(host, {}).get(iface) # 'up', 'initial_down', or None
if (host, iface) in open_down:
prev = 'up'
if is_up:
new_baseline[host][iface] = 'up'
+35 -19
View File
@@ -31,7 +31,6 @@ def db():
mock_db.cutsheet_load.return_value = {'items': [], 'cables': [], 'port_labels': []}
mock_db.get_pending_recovery_notes.return_value = []
mock_db.get_device_link.return_value = ''
mock_db.get_open_event_targets.return_value = []
yield mock_db
@@ -118,24 +117,6 @@ class TestDeadHostDetection:
mon._process_interfaces({HOST_A: {'eth0': True}}, [])
assert ('cluster_network_issue', 'test-cluster', '') in upserted(db)
def test_open_alert_survives_a_baseline_reset(self, mon, db):
# The baseline lost eth1 (reset), but its link-down alert is still open: keep alerting
db.get_open_event_targets.return_value = [('alpha', 'eth1')]
db.upsert_event.return_value = (3, False, 2, None)
mon._process_interfaces({HOST_A: {'eth0': True, 'eth1': False}, HOST_B: {'eth0': True}}, [])
assert ('interface_down', 'alpha', 'eth1') in upserted(db)
assert 'Interface eth1 link-down' in mon.tickets.create.call_args.args[0]
def test_open_alert_resolves_when_the_link_returns(self, mon, db):
db.get_open_event_targets.return_value = [('alpha', 'eth1')]
mon._process_interfaces({HOST_A: {'eth0': True, 'eth1': True}, HOST_B: {'eth0': True}}, [])
db.resolve_event.assert_any_call('interface_down', 'alpha', 'eth1')
def test_first_seen_down_without_an_alert_still_stays_quiet(self, mon, db):
db.get_open_event_targets.return_value = []
mon._process_interfaces({HOST_A: {'eth0': True, 'eth1': False}, HOST_B: {'eth0': True}}, [])
assert ('interface_down', 'alpha', 'eth1') not in upserted(db)
def test_dead_host_shows_as_down_on_dashboard(self, mon):
snap = mon._collect_snapshot({HOST_A: {'eth0': True}})
assert snap['hosts']['alpha']['status'] == 'up'
@@ -767,3 +748,38 @@ class TestTicketClientComment:
with patch.object(monitor.requests, 'post', side_effect=OSError('down')):
assert c.comment('42', 'hi') is False
assert not c.comments_denied
class TestTicketClientCreate:
def _client(self):
return monitor.TicketClient({'url': 'http://t.test/create_ticket_api.php', 'api_key': 'k'})
def _post(self, body):
resp = MagicMock(status_code=200)
resp.json.return_value = body
return patch.object(monitor.requests, 'post', return_value=resp)
def test_new_and_deduplicated_tickets_are_used(self):
c = self._client()
with self._post({'success': True, 'ticket_id': '5', 'action': 'deduplicated'}):
assert c.create('t', 'd') == '5'
with self._post({'success': True, 'ticket_id': '6'}):
assert c.create('u', 'd') == '6'
def test_recurrence_on_a_closed_ticket_is_not_used_and_not_retried_each_poll(self):
c = self._client()
with self._post({'success': True, 'ticket_id': '993', 'action': 'recurrence_noted'}) as post:
assert c.create('t', 'd') is None
assert c.create('t', 'd') is None
assert post.call_count == 1 # the second poll doesn't add another comment to the closed ticket
with self._post({'success': True, 'ticket_id': '7', 'action': 'none'}) as post:
assert c.create('other', 'd') == '7' # other alerts are unaffected
def test_retry_after_a_day(self):
c = self._client()
with self._post({'success': True, 'ticket_id': '993', 'action': 'recurrence_noted'}):
c.create('t', 'd')
c._closed_matches['t'] = ('993', time.time() - c.CLOSED_MATCH_RETRY_S - 1)
with self._post({'success': True, 'ticket_id': '8', 'action': 'none'}):
assert c.create('t', 'd') == '8'
assert 't' not in c._closed_matches