Add recovery notes and fuller context to tickets
Lint / JS (eslint) (push) Successful in 7s
Lint / Python (flake8) (push) Successful in 1m26s
Lint / Notify on failure (push) Skipped
Lint / Deploy (push) Skipped
Security / Python Security (bandit) (push) Successful in 1m30s
Lint / Python (flake8) (pull_request) Successful in 52s
Test / Python Tests (pytest) (push) Successful in 1m4s
Lint / JS (eslint) (pull_request) Successful in 7s
Lint / Notify on failure (pull_request) Skipped
Lint / Deploy (pull_request) Skipped
Security / Python Security (bandit) (pull_request) Successful in 1m28s
Test / Python Tests (pytest) (pull_request) Successful in 2m0s

When an alert that opened a ticket clears, the monitor now comments on the
ticket (when it cleared, how long it lasted, failed checks, a link back).
It never closes or changes a ticket's status. Failed notes retry each
cycle for 24 hours; a 401/403 from Tinker stops them until restart.

Tickets are now Markdown with a "What to check" list and, where known:
the switch port the device plugs into (kept from the last time it was
seen, so a dead host still shows it), cutsheet cables and their path,
how often the alert fired in the last 7/30 days, other open alerts on the
device, and links into Gandalf (gandalf.public_url). Titles are unchanged
so Tinker's duplicate detection still works.

network_events gains recovery_notified, added at monitor startup; rows
that cleared before the upgrade count as already notified.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-11-10 19:44:50 -05:00
co-authored by Claude Opus 5.5
parent ff0094c8ff
commit 437f7a415f
8 changed files with 886 additions and 133 deletions
+349 -128
View File
@@ -15,7 +15,7 @@ import statistics
import time
from datetime import datetime, timezone
from typing import Dict, List, Optional
from urllib.parse import urlparse
from urllib.parse import urljoin, urlparse
import requests
from urllib3.exceptions import InsecureRequestWarning
@@ -24,6 +24,7 @@ import db
import cutsheet
import devices
import modem as modem_api
import ticket_text
logging.basicConfig(
@@ -388,6 +389,41 @@ class TicketClient:
def __init__(self, cfg: dict):
self.url = cfg.get('url', '')
self.api_key = cfg.get('api_key', '')
# Tinker serves its other API scripts under api/ beside create_ticket_api.php
self.comment_url = cfg.get('comment_url') or (
urljoin(self.url, 'api/ticket_comment_api.php') if self.url else '')
self.comments_denied = False
def comment(self, ticket_id: str, text: str) -> bool:
"""Add a Markdown comment to a ticket. Never raises; False means try again later."""
if not self.api_key or not self.comment_url or self.comments_denied:
return False
try:
resp = requests.post(
self.comment_url,
json={'ticket_id': str(ticket_id), 'comment_text': text, 'markdown_enabled': True},
headers={'Authorization': f'Bearer {self.api_key}'},
timeout=15,
)
if resp.status_code in (401, 403):
# Retrying every cycle can't fix a key without write access, so stop until restart
self.comments_denied = True
logger.error(
f'Ticket API refused a comment on #{ticket_id} (HTTP {resp.status_code}). '
f'The API key needs read_write scope; recovery notes are off until the monitor restarts.'
)
return False
if resp.status_code == 404:
logger.warning(f'Ticket #{ticket_id} no longer exists; skipping its recovery note')
return True
resp.raise_for_status()
if resp.json().get('success'):
logger.info(f'Commented on ticket #{ticket_id}')
return True
logger.warning(f'Unexpected comment API response for #{ticket_id}: {resp.text[:200]}')
except Exception as e:
logger.error(f'Ticket comment failed for #{ticket_id}: {e}')
return False
def create(self, title: str, description: str, priority: str = '2') -> Optional[str]:
if not self.api_key or not self.url:
@@ -893,10 +929,21 @@ def _now_utc() -> str:
return datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M:%S UTC')
def _utcnow() -> datetime:
"""Naive UTC, to compare with database times."""
return datetime.now(timezone.utc).replace(tzinfo=None)
# --------------------------------------------------------------------------
# Monitor
# --------------------------------------------------------------------------
CLUSTER_NAME = 'proxmox-cluster'
DEFAULT_PUBLIC_URL = 'https://gandalf.lotusguild.org'
# Reports of something already over: the ticket is the whole story, no recovery note
ONE_SHOT_EVENTS = ('modem_power_cycled', 'wan_unstable')
# Ports a host or device was last seen on, kept so a ticket for a dead host
# (whose link, and so its LLDP entry, is gone) can still say where it plugs in
KNOWN_UPLINKS_STATE = 'known_uplinks'
class NetworkMonitor:
@@ -950,6 +997,16 @@ class NetworkMonitor:
db.ensure_cutsheet_schema()
except Exception as e:
logger.error(f'Could not create cutsheet tables: {e}')
try:
db.ensure_events_schema()
except Exception as e:
logger.error(f'Could not update the events table: {e}')
# Absolute links in tickets point here
self.public_url = ((self.cfg.get('gandalf') or {}).get('public_url')
or DEFAULT_PUBLIC_URL).rstrip('/')
self._link_data: Optional[dict] = None
self._known_uplinks: Optional[dict] = None
# Last good Prometheus result, shown (as unknown) during a Prometheus outage
self._last_iface_states: Dict[str, Dict[str, bool]] = {}
@@ -964,6 +1021,170 @@ class NetworkMonitor:
def _hostname(self, instance: str) -> str:
return self._instance_map.get(instance, instance.split(':')[0])
# ------------------------------------------------------------------
# Tickets: context, and notes when an alert clears
# ------------------------------------------------------------------
def _base_url(self) -> str:
return getattr(self, 'public_url', DEFAULT_PUBLIC_URL)
def _open_ticket(
self, event_id: int, event_type: str, target: str, detail: str, *,
title: str, heading: str, facts: list, summary: str, checks: list, priority: str,
host: Optional[str] = None, port: Optional[tuple] = None,
) -> None:
"""Open a ticket with the alert's facts plus everything Gandalf knows around it.
host: a device whose switch ports to list. port: (switch, port_idx) the alert is about.
"""
sections = self._ticket_sections(event_id, event_type, target, detail, host=host, port=port)
desc = ticket_text.description(heading, facts, summary, checks, sections)
tid = self.tickets.create(title, desc, priority=priority)
if tid:
db.set_ticket_id(event_id, tid)
def _ticket_sections(self, event_id: int, event_type: str, target: str, detail: str,
host: Optional[str] = None, port: Optional[tuple] = None) -> list:
"""Each section is optional: a failure loses that section, never the ticket."""
sections = []
for title, build in (
('Where it is connected', lambda: self._where_lines(event_type, detail, host, port)),
('History', lambda: ticket_text.history_lines(db.event_stats(event_type, target, detail), _utcnow())),
('Other active alerts here', lambda: self._other_alert_lines(event_id, {target, host})),
('Links', lambda: self._link_lines(host or (port[0] if port else None))),
):
try:
sections.append((title, build()))
except Exception as e:
logger.error(f'Ticket section "{title}" for {target} failed: {e}', exc_info=True)
return sections
def _switches(self) -> dict:
return (getattr(self, '_link_data', None) or {}).get('unifi_switches') or {}
def _remember_uplinks(self, link_data: dict) -> None:
"""Record where every LLDP-visible device is plugged in, for when it disappears."""
known = self._load_known_uplinks()
switches = link_data.get('unifi_switches') or {}
names = {(p.get('lldp') or {}).get('system_name')
for sw in switches.values() for p in (sw.get('ports') or {}).values()}
now = _now_utc()
changed = False
for name in sorted(n for n in names if n):
ups = devices.uplinks_for_host(name, switches)
entry = {'seen': now, 'uplinks': [
{k: u[k] for k in ('switch', 'port', 'port_idx', 'host_interface')} for u in ups]}
if (known.get(name) or {}).get('uplinks') != entry['uplinks']:
changed = True
known[name] = entry
self._known_uplinks = known
# Write when wiring changes, and hourly so "last seen" stays roughly right
last_saved = getattr(self, '_known_uplinks_saved', 0.0)
if changed or time.time() - last_saved > 3600:
db.set_state(KNOWN_UPLINKS_STATE, known)
self._known_uplinks_saved = time.time()
def _load_known_uplinks(self) -> dict:
if getattr(self, '_known_uplinks', None) is None:
self._known_uplinks = {}
raw = db.get_state(KNOWN_UPLINKS_STATE)
if raw:
try:
self._known_uplinks = json.loads(raw)
except ValueError:
logger.error('Failed to parse known_uplinks JSON; starting over')
return self._known_uplinks
def _uplinks_for(self, name: str, switches: dict) -> tuple:
"""(uplinks, None) if the device is linked now, else (last known uplinks, when seen)."""
current = devices.uplinks_for_host(name, switches)
if current:
return current, None
entry = self._load_known_uplinks().get(name) or {}
return entry.get('uplinks') or [], entry.get('seen')
def _where_lines(self, event_type: str, detail: str, host: Optional[str], port: Optional[tuple]) -> list:
base, switches = self._base_url(), self._switches()
lines = []
if host:
ups, seen = self._uplinks_for(host, switches)
if event_type == 'interface_down':
ups = [u for u in ups if u.get('host_interface') == detail] or ups
for u in ups:
live = (switches.get(u['switch'], {}).get('ports') or {}).get(u['port'])
text = ticket_text.port_text(u['switch'], u['port'], live or dict(u, up=False))
iface = f'`{u["host_interface"]}` → ' if u.get('host_interface') else ''
lines.append(f'- {iface}' + ticket_text.link(
text, ticket_text.device_url(base, u['switch'], u.get('port_idx'))))
if seen:
lines[-1] += f' _(last seen linked there {seen})_'
if not ups:
lines.append('- Gandalf has not seen this device on a switch port (no LLDP or client entry).')
if port:
switch, idx = port
live = next((dict(p, name=n) for n, p in (switches.get(switch, {}).get('ports') or {}).items()
if str(p.get('port_idx')) == str(idx)), None)
if live:
lines.append('- ' + ticket_text.link(ticket_text.port_text(switch, live['name'], live),
ticket_text.device_url(base, switch, idx)))
device = host or (port[0] if port else None)
if device:
data = db.cutsheet_load()
items = {i['name']: i for i in data['items']}
runs = cutsheet.runs_for(device, data['cables'], items, switches,
port=str(port[1]) if port else None)
for r in runs:
label = ticket_text.link(f'**{r["label"]}**', ticket_text.cutsheet_url(base, r['cable_id']))
lines.append(f'- Cable {label}: {r["path"]}. Cutsheet check: '
f'{cutsheet.VERDICTS.get(r["verdict"], r["verdict"]).lower()}.')
if not runs:
lines.append('- ' + ticket_text.link('Not on the cutsheet yet', f'{base}/cutsheet' if base else ''))
return lines
def _other_alert_lines(self, event_id: int, names: set) -> list:
lines = []
for name in sorted(n for n in names if n):
for e in db.get_events_for_target(name, limit=25):
if e.get('resolved_at') or e.get('id') == event_id:
continue
what = e['event_type'].replace('_', ' ')
where = f'{e["target_name"]} {e.get("target_detail") or ""}'.strip()
ticket = f' (ticket #{e["ticket_id"]})' if e.get('ticket_id') else ''
lines.append(f'- {what} on {where} since {e.get("first_seen", "?")}{ticket}')
return lines[:10]
def _link_lines(self, device: Optional[str]) -> list:
base = self._base_url()
if not base:
return []
lines = []
if device:
lines.append(f'- {ticket_text.link(f"{device} in Gandalf", ticket_text.device_url(base, device))}')
lines.append(f'- {ticket_text.link("Gandalf dashboard", base + "/")}')
return lines
def _event_url(self, event: dict) -> str:
base = self._base_url()
if not base:
return ''
if event['event_type'] == 'cable_mismatch':
cable_id = str(event.get('target_detail') or '').replace('cable ', '')
return ticket_text.cutsheet_url(base, cable_id)
if event['event_type'] in ('cluster_network_issue', 'monitor_source_unavailable'):
return base + '/'
return ticket_text.device_url(base, event['target_name'])
def _post_recovery_notes(self) -> None:
"""Comment on the tickets of alerts that have cleared. Failures retry next cycle (for 24 h)."""
if self.tickets.comments_denied:
return
now = _utcnow()
for event in db.get_pending_recovery_notes(hours=24, exclude_types=ONE_SHOT_EVENTS):
text = ticket_text.recovery_comment(event, now, self._event_url(event))
if self.tickets.comment(event['ticket_id'], text):
db.mark_recovery_notified(event['id'])
elif self.tickets.comments_denied:
break
# ------------------------------------------------------------------
# Interface monitoring (Prometheus)
# ------------------------------------------------------------------
@@ -1041,47 +1262,54 @@ class NetworkMonitor:
f'{", ".join(hosts_with_regression)}',
)
if not sup and (is_new or not ticket_id):
title = (
f'[{self.cluster_name}][auto][production][issue][network][cluster-wide] '
f'Multiple hosts reporting interface failures'
)
desc = (
f'Cluster Network Alert\n{"=" * 40}\n\n'
f'Affected hosts: {", ".join(hosts_with_regression)}\n'
f'Detected: {_now_utc()}\n\n'
f'{len(hosts_with_regression)} Proxmox hosts simultaneously reported '
f'interface regressions (link-down on interfaces previously known UP) '
f'or stopped reporting metrics.\n'
f'This likely indicates a switch or upstream network failure.\n\n'
f'Please check the core and management switches immediately.'
)
tid = self.tickets.create(title, desc, priority='1')
if tid:
db.set_ticket_id(event_id, tid)
self._ticket_cluster(event_id, hosts_with_regression)
else:
db.resolve_event('cluster_network_issue', self.cluster_name, '')
def _ticket_cluster(self, event_id: int, hosts: List[str]) -> None:
switches = self._switches()
# The switches these hosts hang off are the likeliest shared cause
shared = sorted({u['switch'] for h in hosts for u in self._uplinks_for(h, switches)[0]})
self._open_ticket(
event_id, 'cluster_network_issue', self.cluster_name, '',
title=(f'[{self.cluster_name}][auto][production][issue][network][cluster-wide] '
f'Multiple hosts reporting interface failures'),
heading='Cluster Network Alert',
facts=[('Affected hosts', ', '.join(hosts)),
('Switches they connect through', ', '.join(shared)),
('Detected', _now_utc())],
summary=(f'{len(hosts)} Proxmox hosts reported interface regressions (link-down on '
f'interfaces previously known up) or stopped reporting metrics at the same time. '
f'This usually means a switch or upstream network failure rather than the hosts.'),
checks=['Check the core and management switches (power, uplinks, recent firmware updates).',
'Check whether the hosts share a PDU or circuit that lost power.',
'Look at the per-host tickets and the Gandalf dashboard for which links went down first.'],
priority='1',
)
def _ticket_metrics_missing(
self, event_id: int, host: str, instance: str, consec: int
) -> None:
title = (
f'[{host}][auto][production][issue][network][single-node] '
f'No metrics from host'
self._open_ticket(
event_id, 'host_metrics_missing', host, instance,
title=f'[{host}][auto][production][issue][network][single-node] No metrics from host',
heading='Host Metrics Alert',
facts=[('Host', host), ('Prometheus instance', instance), ('Detected', _now_utc()),
('Consecutive check failures', str(consec)),
('Down for at least', ticket_text.duration(consec * self._poll_seconds()))],
summary=(f'Prometheus is working but has no node_network_up series for {host}. '
f'Either the host is down or node_exporter has stopped on it. '
f'Gandalf waits {self.host_fail_thresh} checks before opening this, so a normal '
f'reboot should not cause it.'),
checks=[f'Is {host} powered on and reachable (Proxmox UI, ping, console)?',
'If it is up, check `systemctl status prometheus-node-exporter` on the host.',
'If it is down, check its switch port below: a dark port with no PoE or link '
'points to power or cabling.'],
priority='2', host=host,
)
desc = (
f'Host Metrics Alert\n{"=" * 40}\n\n'
f'Host: {host}\n'
f'Prometheus instance: {instance}\n'
f'Detected: {_now_utc()}\n'
f'Consecutive check failures: {consec}\n\n'
f'Prometheus is working but has no node_network_up series for {host}.\n'
f'Either the host is down or node_exporter has stopped on it.\n\n'
f'Please check the host power and network connectivity, then '
f'prometheus-node-exporter on the host.'
)
tid = self.tickets.create(title, desc, priority='2')
if tid:
db.set_ticket_id(event_id, tid)
def _poll_seconds(self) -> int:
return int(getattr(self, 'poll_interval', 120) or 120)
def _process_prometheus_outage(self, suppressions: list) -> None:
"""Interface and host alerts can't fire without Prometheus, so say so."""
@@ -1092,44 +1320,38 @@ class NetworkMonitor:
f'Prometheus returned no interface data; host and interface alerting is paused ({_now_utc()})',
)
if not sup and consec >= self.fail_thresh and not ticket_id:
title = (
'[prometheus][auto][production][issue][network][cluster-wide] '
'Gandalf has no Prometheus data'
self._open_ticket(
event_id, 'monitor_source_unavailable', 'prometheus', '',
title='[prometheus][auto][production][issue][network][cluster-wide] Gandalf has no Prometheus data',
heading='Monitoring Source Alert',
facts=[('Prometheus URL', self.prom.url), ('Detected', _now_utc()),
('Consecutive check failures', str(consec))],
summary=('Gandalf could not get node_network_up data from Prometheus. While this lasts, '
'host and interface failures will **not** be detected. UniFi, ping and modem '
'checks keep running.'),
checks=['Is the Prometheus service running on its host?',
'Open Prometheus → Status → Targets and check the node exporters are being scraped.',
'Check disk space on the Prometheus host (a full disk stops ingestion).'],
priority='2',
)
desc = (
f'Monitoring Source Alert\n{"=" * 40}\n\n'
f'Prometheus URL: {self.prom.url}\n'
f'Detected: {_now_utc()}\n'
f'Consecutive check failures: {consec}\n\n'
f'Gandalf could not get node_network_up data from Prometheus.\n'
f'While this lasts, host and interface failures will NOT be detected.\n\n'
f'Please check the Prometheus service and its scrape targets.'
)
tid = self.tickets.create(title, desc, priority='2')
if tid:
db.set_ticket_id(event_id, tid)
def _ticket_interface(
self, event_id: int, host: str, iface: str, consec: int
) -> None:
title = (
f'[{host}][auto][production][issue][network][single-node] '
f'Interface {iface} link-down'
self._open_ticket(
event_id, 'interface_down', host, iface,
title=f'[{host}][auto][production][issue][network][single-node] Interface {iface} link-down',
heading='Network Interface Alert',
facts=[('Host', host), ('Interface', f'`{iface}`'), ('Detected', _now_utc()),
('Consecutive check failures', str(consec))],
summary=(f'Interface {iface} on {host} was up and is now reporting link-down via '
f'Prometheus node_exporter. {host} itself is still reporting, so it may be '
f'reachable over its other interfaces.'),
checks=['Check the switch port below: is it down on the switch side too?',
'Reseat or swap the cable or DAC; for an optic, check the module for a fault.',
f'On {host}, check `ip link show {iface}` and `dmesg` for NIC or driver errors.'],
priority='2', host=host,
)
desc = (
f'Network Interface Alert\n{"=" * 40}\n\n'
f'Host: {host}\n'
f'Interface: {iface}\n'
f'Detected: {_now_utc()}\n'
f'Consecutive check failures: {consec}\n\n'
f'Interface {iface} on {host} is reporting link-down state via '
f'Prometheus node_exporter.\n\n'
f'Note: {host} may still be reachable via its other network interface.\n'
f'Please inspect the cable/SFP/switch port for {host}/{iface}.'
)
tid = self.tickets.create(title, desc, priority='2')
if tid:
db.set_ticket_id(event_id, tid)
# ------------------------------------------------------------------
# UniFi device monitoring
@@ -1155,23 +1377,20 @@ class NetworkMonitor:
def _ticket_unifi(self, event_id: int, device: dict) -> None:
name = device['name']
title = (
f'[{name}][auto][production][issue][network][single-node] '
f'UniFi device offline'
self._open_ticket(
event_id, 'unifi_device_offline', name, device.get('type', ''),
title=f'[{name}][auto][production][issue][network][single-node] UniFi device offline',
heading='UniFi Device Alert',
facts=[('Device', name), ('Type', device.get('type', 'unknown')), ('Model', device.get('model', '')),
('Last known IP', device.get('ip', 'unknown')), ('Detected', _now_utc())],
summary=(f'The UniFi controller reports {name} as offline. Gandalf waits '
f'{self.unifi_fail_thresh} checks first, so a firmware-update reboot should not cause this.'),
checks=['Check the device for power (status LED) and, if it is PoE powered, the PoE '
'budget on the port below.',
'Check the uplink cable and the switch port it plugs into.',
'In the UniFi controller, check whether the device is stuck adopting or upgrading.'],
priority='2', host=name,
)
desc = (
f'UniFi Device Alert\n{"=" * 40}\n\n'
f'Device: {name}\n'
f'Type: {device.get("type","unknown")}\n'
f'Model: {device.get("model","")}\n'
f'Last Known IP: {device.get("ip","unknown")}\n'
f'Detected: {_now_utc()}\n\n'
f'The UniFi device {name} is offline per the UniFi controller.\n'
f'Please check power and cable connectivity.'
)
tid = self.tickets.create(title, desc, priority='2')
if tid:
db.set_ticket_id(event_id, tid)
# ------------------------------------------------------------------
# Cutsheet: documented cabling vs. what discovery sees
@@ -1206,24 +1425,20 @@ class NetworkMonitor:
db.resolve_event('cable_mismatch', target_name, target_detail)
def _ticket_cable_mismatch(self, event_id: int, p: dict, hours: float) -> None:
title = (
f'[{p["switch"]}][auto][production][issue][network][single-node] '
f'Cable {p["label"]} does not match the cutsheet'
url = ticket_text.cutsheet_url(self._base_url(), p['cable_id'])
self._open_ticket(
event_id, 'cable_mismatch', p['switch'], p['key'],
title=(f'[{p["switch"]}][auto][production][issue][network][single-node] '
f'Cable {p["label"]} does not match the cutsheet'),
heading='Cutsheet Mismatch',
facts=[('Cable', ticket_text.link(p['label'], url)), ('Switch port', f'{p["switch"]} port {p["port"]}'),
('Problem', p['detail']), ('Documented path', p['path']), ('Detected', _now_utc())],
summary=f'This has not matched the cutsheet for at least {hours:g} hours.',
checks=['If the cable was moved on purpose, update its record in the cutsheet.',
'Otherwise, follow the label and check it is plugged into the documented port '
'(Trace mode on the cutsheet page can find which port a cord is in).'],
priority='4', port=(p['switch'], p['port']),
)
desc = (
f'Cutsheet Mismatch\n{"=" * 40}\n\n'
f'Cable: {p["label"]}\n'
f'Switch port: {p["switch"]} port {p["port"]}\n'
f'Problem: {p["detail"]}\n'
f'Documented path: {p["path"]}\n'
f'Detected: {_now_utc()}\n\n'
f'This has not matched for at least {hours:g} hours.\n\n'
f'Either the cable was moved (update the cutsheet in Gandalf: /cutsheet#cable-{p["cable_id"]}) '
f'or it is plugged into the wrong place.'
)
tid = self.tickets.create(title, desc, priority='4')
if tid:
db.set_ticket_id(event_id, tid)
# ------------------------------------------------------------------
# Ping-only hosts (no node_exporter)
@@ -1253,24 +1468,20 @@ class NetworkMonitor:
def _ticket_unreachable(
self, event_id: int, name: str, ip: str, consec: int
) -> None:
title = (
f'[{name}][auto][production][issue][network][single-node] '
f'Host unreachable'
self._open_ticket(
event_id, 'host_unreachable', name, ip,
title=f'[{name}][auto][production][issue][network][single-node] Host unreachable',
heading='Host Reachability Alert',
facts=[('Host', name), ('IP', ip), ('Detected', _now_utc()),
('Consecutive check failures', str(consec)),
('Down for at least', ticket_text.duration(consec * self._poll_seconds()))],
summary=(f'{name} ({ip}) is not answering ping from the Pulse worker Gandalf uses. '
f'It has no node_exporter, so ping is the only check Gandalf has for it.'),
checks=[f'Is {name} powered on? Check its console or management interface.',
'Check its switch port below for link and PoE.',
'If the port is up, check the IP and firewall on the device itself.'],
priority='2', host=name,
)
desc = (
f'Host Reachability Alert\n{"=" * 40}\n\n'
f'Host: {name}\n'
f'IP: {ip}\n'
f'Detected: {_now_utc()}\n'
f'Consecutive check failures: {consec}\n\n'
f'Host {name} ({ip}) is not responding to ping from the Gandalf monitor.\n'
f'This host does not have a Prometheus node_exporter, so interface-level '
f'detail is unavailable.\n\n'
f'Please check the host power, management interface, and network connectivity.'
)
tid = self.tickets.create(title, desc, priority='2')
if tid:
db.set_ticket_id(event_id, tid)
# ------------------------------------------------------------------
# WAN modem monitoring
@@ -1291,17 +1502,14 @@ class NetworkMonitor:
event_type, severity, 'modem', name, detail, f'{description} ({_now_utc()})',
)
if not sup and consec >= self.modem_fail_thresh and not ticket_id:
title = f'[{name}][auto][production][issue][network][cluster-wide] {heading}'
desc = (
f'WAN Modem Alert\n{"=" * 40}\n\n'
f'Modem: {name} ({self.modem.base_url})\n'
f'Detected: {_now_utc()}\n'
f'Consecutive check failures: {consec}\n\n'
f'{description}\n\n{advice}'
self._open_ticket(
event_id, event_type, name, detail,
title=f'[{name}][auto][production][issue][network][cluster-wide] {heading}',
heading='WAN Modem Alert',
facts=[('Modem', f'{name} ({self.modem.base_url})'), ('Detected', _now_utc()),
('Consecutive check failures', str(consec))],
summary=description, checks=[advice], priority=priority,
)
tid = self.tickets.create(title, desc, priority=priority)
if tid:
db.set_ticket_id(event_id, tid)
def _process_modem_failure(self, suppressions: list, error: Exception) -> None:
"""Handle a failed poll. Link and optical events stay as they are: we can't see them."""
@@ -1443,7 +1651,8 @@ class NetworkMonitor:
if not db.check_suppressed(suppressions, 'host', name) and not ticket_id:
tid = self.tickets.create(
f'[{name}][auto][production][issue][network][cluster-wide] {heading}',
f'WAN Event\n{"=" * 40}\n\nModem: {name}\nReported: {_now_utc()}\n\n{description}\n\n{advice}',
ticket_text.description('WAN Event', [('Modem', name), ('Reported', _now_utc())],
description, [advice], recovers=False),
priority=priority,
)
if tid:
@@ -1645,6 +1854,12 @@ class NetworkMonitor:
db.set_state('link_stats', link_data)
except Exception as e:
logger.error(f'Link stats collection failed: {e}', exc_info=True)
if link_data is not None:
self._link_data = link_data
try:
self._remember_uplinks(link_data)
except Exception as e:
logger.error(f'Recording known uplinks failed: {e}', exc_info=True)
# 5. Process alerts using already-fetched data
suppressions = db.get_active_suppressions()
@@ -1673,6 +1888,12 @@ class NetworkMonitor:
except Exception as e:
logger.error(f'Modem collection failed: {e}', exc_info=True)
# Note on the tickets of anything that cleared this cycle (or failed to post before)
try:
self._post_recovery_notes()
except Exception as e:
logger.error(f'Posting recovery notes failed: {e}', exc_info=True)
# Housekeeping: deactivate expired suppressions and purge old resolved events
db.cleanup_expired_suppressions()
db.purge_old_resolved_events(days=90)