Compare commits
4
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
d234e36e3f | ||
|
|
fde1d01f27 | ||
|
|
8edaa78ad7 | ||
|
|
a0e062493d |
No files matched your search
+25
-7
@@ -393,6 +393,8 @@ class TicketClient:
|
||||
self.comment_url = cfg.get('comment_url') or (
|
||||
urljoin(self.url, 'api/ticket_comment_api.php') if self.url else '')
|
||||
self.comments_denied = False
|
||||
# Titles whose create matched a Closed ticket Tinker couldn't reopen: {title: (ticket_id, when)}
|
||||
self._closed_matches: Dict[str, tuple] = {}
|
||||
|
||||
def comment(self, ticket_id: str, text: str) -> bool:
|
||||
"""Add a Markdown comment to a ticket. Never raises; False means try again later."""
|
||||
@@ -425,10 +427,23 @@ class TicketClient:
|
||||
logger.error(f'Ticket comment failed for #{ticket_id}: {e}')
|
||||
return False
|
||||
|
||||
# How long to wait before asking again after a create matched an unreopenable Closed ticket.
|
||||
# Each ask adds a "recurred" comment to that ticket, so once a poll would pile them up.
|
||||
CLOSED_MATCH_RETRY_S = 24 * 3600
|
||||
|
||||
def create(self, title: str, description: str, priority: str = '2') -> Optional[str]:
|
||||
"""Open a ticket. Returns its ID, or None if there is no ticket to track.
|
||||
|
||||
Tinker deduplicates: the response's `action` says what happened (new, deduplicated,
|
||||
updated, reopened, recurrence_noted). recurrence_noted means it matched a Closed ticket
|
||||
it could not reopen unattended; that ticket isn't tracking this alert, so it isn't used.
|
||||
"""
|
||||
if not self.api_key or not self.url:
|
||||
logger.warning('Ticket API not configured: skipping ticket creation')
|
||||
return None
|
||||
seen = self._closed_matches.get(title)
|
||||
if seen and time.time() - seen[1] < self.CLOSED_MATCH_RETRY_S:
|
||||
return None
|
||||
try:
|
||||
resp = requests.post(
|
||||
self.url,
|
||||
@@ -450,7 +465,16 @@ class TicketClient:
|
||||
if not tid:
|
||||
logger.warning(f'Ticket API success but no ticket_id in response: {data}')
|
||||
return None
|
||||
logger.info(f'Created ticket #{tid}: {title}')
|
||||
action = data.get('action') or 'created'
|
||||
if action == 'recurrence_noted':
|
||||
self._closed_matches[title] = (tid, time.time())
|
||||
logger.warning(
|
||||
f'Ticket API matched Closed ticket #{tid}, which it could not reopen; not using it '
|
||||
f'for this alert. Reopen #{tid} in Tinker to track it there. Retrying in 24 h: {title}'
|
||||
)
|
||||
return None
|
||||
self._closed_matches.pop(title, None)
|
||||
logger.info(f'Ticket #{tid} ({action}): {title}')
|
||||
return tid
|
||||
if data.get('existing_ticket_id'):
|
||||
logger.info(f'Duplicate suppressed by API: existing #{data["existing_ticket_id"]}')
|
||||
@@ -1195,10 +1219,6 @@ class NetworkMonitor:
|
||||
def _process_interfaces(self, states: Dict[str, Dict[str, bool]], suppressions: list) -> None:
|
||||
baseline = db.get_baseline()
|
||||
new_baseline = {k: dict(v) for k, v in baseline.items()}
|
||||
# An interface with an open link-down alert was up before it went down, whatever the
|
||||
# baseline says now: after a baseline reset it reads as "first seen down", which would
|
||||
# leave the alert open forever without updating, ticketing or resolving it.
|
||||
open_down = set(db.get_open_event_targets('interface_down'))
|
||||
# Only count hosts with genuine regressions (UP→DOWN) toward cluster threshold
|
||||
hosts_with_regression: List[str] = []
|
||||
|
||||
@@ -1211,8 +1231,6 @@ class NetworkMonitor:
|
||||
|
||||
for iface, is_up in ifaces.items():
|
||||
prev = baseline.get(host, {}).get(iface) # 'up', 'initial_down', or None
|
||||
if (host, iface) in open_down:
|
||||
prev = 'up'
|
||||
|
||||
if is_up:
|
||||
new_baseline[host][iface] = 'up'
|
||||
|
||||
+35
-19
@@ -31,7 +31,6 @@ def db():
|
||||
mock_db.cutsheet_load.return_value = {'items': [], 'cables': [], 'port_labels': []}
|
||||
mock_db.get_pending_recovery_notes.return_value = []
|
||||
mock_db.get_device_link.return_value = ''
|
||||
mock_db.get_open_event_targets.return_value = []
|
||||
yield mock_db
|
||||
|
||||
|
||||
@@ -118,24 +117,6 @@ class TestDeadHostDetection:
|
||||
mon._process_interfaces({HOST_A: {'eth0': True}}, [])
|
||||
assert ('cluster_network_issue', 'test-cluster', '') in upserted(db)
|
||||
|
||||
def test_open_alert_survives_a_baseline_reset(self, mon, db):
|
||||
# The baseline lost eth1 (reset), but its link-down alert is still open: keep alerting
|
||||
db.get_open_event_targets.return_value = [('alpha', 'eth1')]
|
||||
db.upsert_event.return_value = (3, False, 2, None)
|
||||
mon._process_interfaces({HOST_A: {'eth0': True, 'eth1': False}, HOST_B: {'eth0': True}}, [])
|
||||
assert ('interface_down', 'alpha', 'eth1') in upserted(db)
|
||||
assert 'Interface eth1 link-down' in mon.tickets.create.call_args.args[0]
|
||||
|
||||
def test_open_alert_resolves_when_the_link_returns(self, mon, db):
|
||||
db.get_open_event_targets.return_value = [('alpha', 'eth1')]
|
||||
mon._process_interfaces({HOST_A: {'eth0': True, 'eth1': True}, HOST_B: {'eth0': True}}, [])
|
||||
db.resolve_event.assert_any_call('interface_down', 'alpha', 'eth1')
|
||||
|
||||
def test_first_seen_down_without_an_alert_still_stays_quiet(self, mon, db):
|
||||
db.get_open_event_targets.return_value = []
|
||||
mon._process_interfaces({HOST_A: {'eth0': True, 'eth1': False}, HOST_B: {'eth0': True}}, [])
|
||||
assert ('interface_down', 'alpha', 'eth1') not in upserted(db)
|
||||
|
||||
def test_dead_host_shows_as_down_on_dashboard(self, mon):
|
||||
snap = mon._collect_snapshot({HOST_A: {'eth0': True}})
|
||||
assert snap['hosts']['alpha']['status'] == 'up'
|
||||
@@ -767,3 +748,38 @@ class TestTicketClientComment:
|
||||
with patch.object(monitor.requests, 'post', side_effect=OSError('down')):
|
||||
assert c.comment('42', 'hi') is False
|
||||
assert not c.comments_denied
|
||||
|
||||
|
||||
class TestTicketClientCreate:
|
||||
def _client(self):
|
||||
return monitor.TicketClient({'url': 'http://t.test/create_ticket_api.php', 'api_key': 'k'})
|
||||
|
||||
def _post(self, body):
|
||||
resp = MagicMock(status_code=200)
|
||||
resp.json.return_value = body
|
||||
return patch.object(monitor.requests, 'post', return_value=resp)
|
||||
|
||||
def test_new_and_deduplicated_tickets_are_used(self):
|
||||
c = self._client()
|
||||
with self._post({'success': True, 'ticket_id': '5', 'action': 'deduplicated'}):
|
||||
assert c.create('t', 'd') == '5'
|
||||
with self._post({'success': True, 'ticket_id': '6'}):
|
||||
assert c.create('u', 'd') == '6'
|
||||
|
||||
def test_recurrence_on_a_closed_ticket_is_not_used_and_not_retried_each_poll(self):
|
||||
c = self._client()
|
||||
with self._post({'success': True, 'ticket_id': '993', 'action': 'recurrence_noted'}) as post:
|
||||
assert c.create('t', 'd') is None
|
||||
assert c.create('t', 'd') is None
|
||||
assert post.call_count == 1 # the second poll doesn't add another comment to the closed ticket
|
||||
with self._post({'success': True, 'ticket_id': '7', 'action': 'none'}) as post:
|
||||
assert c.create('other', 'd') == '7' # other alerts are unaffected
|
||||
|
||||
def test_retry_after_a_day(self):
|
||||
c = self._client()
|
||||
with self._post({'success': True, 'ticket_id': '993', 'action': 'recurrence_noted'}):
|
||||
c.create('t', 'd')
|
||||
c._closed_matches['t'] = ('993', time.time() - c.CLOSED_MATCH_RETRY_S - 1)
|
||||
with self._post({'success': True, 'ticket_id': '8', 'action': 'none'}):
|
||||
assert c.create('t', 'd') == '8'
|
||||
assert 't' not in c._closed_matches
|
||||
Reference in new issue
Block a user