Add recovery notes and fuller context to tickets
Lint / JS (eslint) (push) Successful in 7s
Lint / Python (flake8) (push) Successful in 1m26s
Lint / Notify on failure (push) Skipped
Lint / Deploy (push) Skipped
Security / Python Security (bandit) (push) Successful in 1m30s
Lint / Python (flake8) (pull_request) Successful in 52s
Test / Python Tests (pytest) (push) Successful in 1m4s
Lint / JS (eslint) (pull_request) Successful in 7s
Lint / Notify on failure (pull_request) Skipped
Lint / Deploy (pull_request) Skipped
Security / Python Security (bandit) (pull_request) Successful in 1m28s
Test / Python Tests (pytest) (pull_request) Successful in 2m0s
Lint / JS (eslint) (push) Successful in 7s
Lint / Python (flake8) (push) Successful in 1m26s
Lint / Notify on failure (push) Skipped
Lint / Deploy (push) Skipped
Security / Python Security (bandit) (push) Successful in 1m30s
Lint / Python (flake8) (pull_request) Successful in 52s
Test / Python Tests (pytest) (push) Successful in 1m4s
Lint / JS (eslint) (pull_request) Successful in 7s
Lint / Notify on failure (pull_request) Skipped
Lint / Deploy (pull_request) Skipped
Security / Python Security (bandit) (pull_request) Successful in 1m28s
Test / Python Tests (pytest) (pull_request) Successful in 2m0s
When an alert that opened a ticket clears, the monitor now comments on the ticket (when it cleared, how long it lasted, failed checks, a link back). It never closes or changes a ticket's status. Failed notes retry each cycle for 24 hours; a 401/403 from Tinker stops them until restart. Tickets are now Markdown with a "What to check" list and, where known: the switch port the device plugs into (kept from the last time it was seen, so a dead host still shows it), cutsheet cables and their path, how often the alert fired in the last 7/30 days, other open alerts on the device, and links into Gandalf (gandalf.public_url). Titles are unchanged so Tinker's duplicate detection still works. network_events gains recovery_notified, added at monitor startup; rows that cleared before the upgrade count as already notified. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
+349
-128
@@ -15,7 +15,7 @@ import statistics
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
from typing import Dict, List, Optional
|
||||
from urllib.parse import urlparse
|
||||
from urllib.parse import urljoin, urlparse
|
||||
|
||||
import requests
|
||||
from urllib3.exceptions import InsecureRequestWarning
|
||||
@@ -24,6 +24,7 @@ import db
|
||||
import cutsheet
|
||||
import devices
|
||||
import modem as modem_api
|
||||
import ticket_text
|
||||
|
||||
|
||||
logging.basicConfig(
|
||||
@@ -388,6 +389,41 @@ class TicketClient:
|
||||
def __init__(self, cfg: dict):
|
||||
self.url = cfg.get('url', '')
|
||||
self.api_key = cfg.get('api_key', '')
|
||||
# Tinker serves its other API scripts under api/ beside create_ticket_api.php
|
||||
self.comment_url = cfg.get('comment_url') or (
|
||||
urljoin(self.url, 'api/ticket_comment_api.php') if self.url else '')
|
||||
self.comments_denied = False
|
||||
|
||||
def comment(self, ticket_id: str, text: str) -> bool:
|
||||
"""Add a Markdown comment to a ticket. Never raises; False means try again later."""
|
||||
if not self.api_key or not self.comment_url or self.comments_denied:
|
||||
return False
|
||||
try:
|
||||
resp = requests.post(
|
||||
self.comment_url,
|
||||
json={'ticket_id': str(ticket_id), 'comment_text': text, 'markdown_enabled': True},
|
||||
headers={'Authorization': f'Bearer {self.api_key}'},
|
||||
timeout=15,
|
||||
)
|
||||
if resp.status_code in (401, 403):
|
||||
# Retrying every cycle can't fix a key without write access, so stop until restart
|
||||
self.comments_denied = True
|
||||
logger.error(
|
||||
f'Ticket API refused a comment on #{ticket_id} (HTTP {resp.status_code}). '
|
||||
f'The API key needs read_write scope; recovery notes are off until the monitor restarts.'
|
||||
)
|
||||
return False
|
||||
if resp.status_code == 404:
|
||||
logger.warning(f'Ticket #{ticket_id} no longer exists; skipping its recovery note')
|
||||
return True
|
||||
resp.raise_for_status()
|
||||
if resp.json().get('success'):
|
||||
logger.info(f'Commented on ticket #{ticket_id}')
|
||||
return True
|
||||
logger.warning(f'Unexpected comment API response for #{ticket_id}: {resp.text[:200]}')
|
||||
except Exception as e:
|
||||
logger.error(f'Ticket comment failed for #{ticket_id}: {e}')
|
||||
return False
|
||||
|
||||
def create(self, title: str, description: str, priority: str = '2') -> Optional[str]:
|
||||
if not self.api_key or not self.url:
|
||||
@@ -893,10 +929,21 @@ def _now_utc() -> str:
|
||||
return datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M:%S UTC')
|
||||
|
||||
|
||||
def _utcnow() -> datetime:
|
||||
"""Naive UTC, to compare with database times."""
|
||||
return datetime.now(timezone.utc).replace(tzinfo=None)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Monitor
|
||||
# --------------------------------------------------------------------------
|
||||
CLUSTER_NAME = 'proxmox-cluster'
|
||||
DEFAULT_PUBLIC_URL = 'https://gandalf.lotusguild.org'
|
||||
# Reports of something already over: the ticket is the whole story, no recovery note
|
||||
ONE_SHOT_EVENTS = ('modem_power_cycled', 'wan_unstable')
|
||||
# Ports a host or device was last seen on, kept so a ticket for a dead host
|
||||
# (whose link, and so its LLDP entry, is gone) can still say where it plugs in
|
||||
KNOWN_UPLINKS_STATE = 'known_uplinks'
|
||||
|
||||
|
||||
class NetworkMonitor:
|
||||
@@ -950,6 +997,16 @@ class NetworkMonitor:
|
||||
db.ensure_cutsheet_schema()
|
||||
except Exception as e:
|
||||
logger.error(f'Could not create cutsheet tables: {e}')
|
||||
try:
|
||||
db.ensure_events_schema()
|
||||
except Exception as e:
|
||||
logger.error(f'Could not update the events table: {e}')
|
||||
|
||||
# Absolute links in tickets point here
|
||||
self.public_url = ((self.cfg.get('gandalf') or {}).get('public_url')
|
||||
or DEFAULT_PUBLIC_URL).rstrip('/')
|
||||
self._link_data: Optional[dict] = None
|
||||
self._known_uplinks: Optional[dict] = None
|
||||
|
||||
# Last good Prometheus result, shown (as unknown) during a Prometheus outage
|
||||
self._last_iface_states: Dict[str, Dict[str, bool]] = {}
|
||||
@@ -964,6 +1021,170 @@ class NetworkMonitor:
|
||||
def _hostname(self, instance: str) -> str:
|
||||
return self._instance_map.get(instance, instance.split(':')[0])
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Tickets: context, and notes when an alert clears
|
||||
# ------------------------------------------------------------------
|
||||
def _base_url(self) -> str:
|
||||
return getattr(self, 'public_url', DEFAULT_PUBLIC_URL)
|
||||
|
||||
def _open_ticket(
|
||||
self, event_id: int, event_type: str, target: str, detail: str, *,
|
||||
title: str, heading: str, facts: list, summary: str, checks: list, priority: str,
|
||||
host: Optional[str] = None, port: Optional[tuple] = None,
|
||||
) -> None:
|
||||
"""Open a ticket with the alert's facts plus everything Gandalf knows around it.
|
||||
|
||||
host: a device whose switch ports to list. port: (switch, port_idx) the alert is about.
|
||||
"""
|
||||
sections = self._ticket_sections(event_id, event_type, target, detail, host=host, port=port)
|
||||
desc = ticket_text.description(heading, facts, summary, checks, sections)
|
||||
tid = self.tickets.create(title, desc, priority=priority)
|
||||
if tid:
|
||||
db.set_ticket_id(event_id, tid)
|
||||
|
||||
def _ticket_sections(self, event_id: int, event_type: str, target: str, detail: str,
|
||||
host: Optional[str] = None, port: Optional[tuple] = None) -> list:
|
||||
"""Each section is optional: a failure loses that section, never the ticket."""
|
||||
sections = []
|
||||
for title, build in (
|
||||
('Where it is connected', lambda: self._where_lines(event_type, detail, host, port)),
|
||||
('History', lambda: ticket_text.history_lines(db.event_stats(event_type, target, detail), _utcnow())),
|
||||
('Other active alerts here', lambda: self._other_alert_lines(event_id, {target, host})),
|
||||
('Links', lambda: self._link_lines(host or (port[0] if port else None))),
|
||||
):
|
||||
try:
|
||||
sections.append((title, build()))
|
||||
except Exception as e:
|
||||
logger.error(f'Ticket section "{title}" for {target} failed: {e}', exc_info=True)
|
||||
return sections
|
||||
|
||||
def _switches(self) -> dict:
|
||||
return (getattr(self, '_link_data', None) or {}).get('unifi_switches') or {}
|
||||
|
||||
def _remember_uplinks(self, link_data: dict) -> None:
|
||||
"""Record where every LLDP-visible device is plugged in, for when it disappears."""
|
||||
known = self._load_known_uplinks()
|
||||
switches = link_data.get('unifi_switches') or {}
|
||||
names = {(p.get('lldp') or {}).get('system_name')
|
||||
for sw in switches.values() for p in (sw.get('ports') or {}).values()}
|
||||
now = _now_utc()
|
||||
changed = False
|
||||
for name in sorted(n for n in names if n):
|
||||
ups = devices.uplinks_for_host(name, switches)
|
||||
entry = {'seen': now, 'uplinks': [
|
||||
{k: u[k] for k in ('switch', 'port', 'port_idx', 'host_interface')} for u in ups]}
|
||||
if (known.get(name) or {}).get('uplinks') != entry['uplinks']:
|
||||
changed = True
|
||||
known[name] = entry
|
||||
self._known_uplinks = known
|
||||
# Write when wiring changes, and hourly so "last seen" stays roughly right
|
||||
last_saved = getattr(self, '_known_uplinks_saved', 0.0)
|
||||
if changed or time.time() - last_saved > 3600:
|
||||
db.set_state(KNOWN_UPLINKS_STATE, known)
|
||||
self._known_uplinks_saved = time.time()
|
||||
|
||||
def _load_known_uplinks(self) -> dict:
|
||||
if getattr(self, '_known_uplinks', None) is None:
|
||||
self._known_uplinks = {}
|
||||
raw = db.get_state(KNOWN_UPLINKS_STATE)
|
||||
if raw:
|
||||
try:
|
||||
self._known_uplinks = json.loads(raw)
|
||||
except ValueError:
|
||||
logger.error('Failed to parse known_uplinks JSON; starting over')
|
||||
return self._known_uplinks
|
||||
|
||||
def _uplinks_for(self, name: str, switches: dict) -> tuple:
|
||||
"""(uplinks, None) if the device is linked now, else (last known uplinks, when seen)."""
|
||||
current = devices.uplinks_for_host(name, switches)
|
||||
if current:
|
||||
return current, None
|
||||
entry = self._load_known_uplinks().get(name) or {}
|
||||
return entry.get('uplinks') or [], entry.get('seen')
|
||||
|
||||
def _where_lines(self, event_type: str, detail: str, host: Optional[str], port: Optional[tuple]) -> list:
|
||||
base, switches = self._base_url(), self._switches()
|
||||
lines = []
|
||||
if host:
|
||||
ups, seen = self._uplinks_for(host, switches)
|
||||
if event_type == 'interface_down':
|
||||
ups = [u for u in ups if u.get('host_interface') == detail] or ups
|
||||
for u in ups:
|
||||
live = (switches.get(u['switch'], {}).get('ports') or {}).get(u['port'])
|
||||
text = ticket_text.port_text(u['switch'], u['port'], live or dict(u, up=False))
|
||||
iface = f'`{u["host_interface"]}` → ' if u.get('host_interface') else ''
|
||||
lines.append(f'- {iface}' + ticket_text.link(
|
||||
text, ticket_text.device_url(base, u['switch'], u.get('port_idx'))))
|
||||
if seen:
|
||||
lines[-1] += f' _(last seen linked there {seen})_'
|
||||
if not ups:
|
||||
lines.append('- Gandalf has not seen this device on a switch port (no LLDP or client entry).')
|
||||
if port:
|
||||
switch, idx = port
|
||||
live = next((dict(p, name=n) for n, p in (switches.get(switch, {}).get('ports') or {}).items()
|
||||
if str(p.get('port_idx')) == str(idx)), None)
|
||||
if live:
|
||||
lines.append('- ' + ticket_text.link(ticket_text.port_text(switch, live['name'], live),
|
||||
ticket_text.device_url(base, switch, idx)))
|
||||
device = host or (port[0] if port else None)
|
||||
if device:
|
||||
data = db.cutsheet_load()
|
||||
items = {i['name']: i for i in data['items']}
|
||||
runs = cutsheet.runs_for(device, data['cables'], items, switches,
|
||||
port=str(port[1]) if port else None)
|
||||
for r in runs:
|
||||
label = ticket_text.link(f'**{r["label"]}**', ticket_text.cutsheet_url(base, r['cable_id']))
|
||||
lines.append(f'- Cable {label}: {r["path"]}. Cutsheet check: '
|
||||
f'{cutsheet.VERDICTS.get(r["verdict"], r["verdict"]).lower()}.')
|
||||
if not runs:
|
||||
lines.append('- ' + ticket_text.link('Not on the cutsheet yet', f'{base}/cutsheet' if base else ''))
|
||||
return lines
|
||||
|
||||
def _other_alert_lines(self, event_id: int, names: set) -> list:
|
||||
lines = []
|
||||
for name in sorted(n for n in names if n):
|
||||
for e in db.get_events_for_target(name, limit=25):
|
||||
if e.get('resolved_at') or e.get('id') == event_id:
|
||||
continue
|
||||
what = e['event_type'].replace('_', ' ')
|
||||
where = f'{e["target_name"]} {e.get("target_detail") or ""}'.strip()
|
||||
ticket = f' (ticket #{e["ticket_id"]})' if e.get('ticket_id') else ''
|
||||
lines.append(f'- {what} on {where} since {e.get("first_seen", "?")}{ticket}')
|
||||
return lines[:10]
|
||||
|
||||
def _link_lines(self, device: Optional[str]) -> list:
|
||||
base = self._base_url()
|
||||
if not base:
|
||||
return []
|
||||
lines = []
|
||||
if device:
|
||||
lines.append(f'- {ticket_text.link(f"{device} in Gandalf", ticket_text.device_url(base, device))}')
|
||||
lines.append(f'- {ticket_text.link("Gandalf dashboard", base + "/")}')
|
||||
return lines
|
||||
|
||||
def _event_url(self, event: dict) -> str:
|
||||
base = self._base_url()
|
||||
if not base:
|
||||
return ''
|
||||
if event['event_type'] == 'cable_mismatch':
|
||||
cable_id = str(event.get('target_detail') or '').replace('cable ', '')
|
||||
return ticket_text.cutsheet_url(base, cable_id)
|
||||
if event['event_type'] in ('cluster_network_issue', 'monitor_source_unavailable'):
|
||||
return base + '/'
|
||||
return ticket_text.device_url(base, event['target_name'])
|
||||
|
||||
def _post_recovery_notes(self) -> None:
|
||||
"""Comment on the tickets of alerts that have cleared. Failures retry next cycle (for 24 h)."""
|
||||
if self.tickets.comments_denied:
|
||||
return
|
||||
now = _utcnow()
|
||||
for event in db.get_pending_recovery_notes(hours=24, exclude_types=ONE_SHOT_EVENTS):
|
||||
text = ticket_text.recovery_comment(event, now, self._event_url(event))
|
||||
if self.tickets.comment(event['ticket_id'], text):
|
||||
db.mark_recovery_notified(event['id'])
|
||||
elif self.tickets.comments_denied:
|
||||
break
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Interface monitoring (Prometheus)
|
||||
# ------------------------------------------------------------------
|
||||
@@ -1041,47 +1262,54 @@ class NetworkMonitor:
|
||||
f'{", ".join(hosts_with_regression)}',
|
||||
)
|
||||
if not sup and (is_new or not ticket_id):
|
||||
title = (
|
||||
f'[{self.cluster_name}][auto][production][issue][network][cluster-wide] '
|
||||
f'Multiple hosts reporting interface failures'
|
||||
)
|
||||
desc = (
|
||||
f'Cluster Network Alert\n{"=" * 40}\n\n'
|
||||
f'Affected hosts: {", ".join(hosts_with_regression)}\n'
|
||||
f'Detected: {_now_utc()}\n\n'
|
||||
f'{len(hosts_with_regression)} Proxmox hosts simultaneously reported '
|
||||
f'interface regressions (link-down on interfaces previously known UP) '
|
||||
f'or stopped reporting metrics.\n'
|
||||
f'This likely indicates a switch or upstream network failure.\n\n'
|
||||
f'Please check the core and management switches immediately.'
|
||||
)
|
||||
tid = self.tickets.create(title, desc, priority='1')
|
||||
if tid:
|
||||
db.set_ticket_id(event_id, tid)
|
||||
self._ticket_cluster(event_id, hosts_with_regression)
|
||||
else:
|
||||
db.resolve_event('cluster_network_issue', self.cluster_name, '')
|
||||
|
||||
def _ticket_cluster(self, event_id: int, hosts: List[str]) -> None:
|
||||
switches = self._switches()
|
||||
# The switches these hosts hang off are the likeliest shared cause
|
||||
shared = sorted({u['switch'] for h in hosts for u in self._uplinks_for(h, switches)[0]})
|
||||
self._open_ticket(
|
||||
event_id, 'cluster_network_issue', self.cluster_name, '',
|
||||
title=(f'[{self.cluster_name}][auto][production][issue][network][cluster-wide] '
|
||||
f'Multiple hosts reporting interface failures'),
|
||||
heading='Cluster Network Alert',
|
||||
facts=[('Affected hosts', ', '.join(hosts)),
|
||||
('Switches they connect through', ', '.join(shared)),
|
||||
('Detected', _now_utc())],
|
||||
summary=(f'{len(hosts)} Proxmox hosts reported interface regressions (link-down on '
|
||||
f'interfaces previously known up) or stopped reporting metrics at the same time. '
|
||||
f'This usually means a switch or upstream network failure rather than the hosts.'),
|
||||
checks=['Check the core and management switches (power, uplinks, recent firmware updates).',
|
||||
'Check whether the hosts share a PDU or circuit that lost power.',
|
||||
'Look at the per-host tickets and the Gandalf dashboard for which links went down first.'],
|
||||
priority='1',
|
||||
)
|
||||
|
||||
def _ticket_metrics_missing(
|
||||
self, event_id: int, host: str, instance: str, consec: int
|
||||
) -> None:
|
||||
title = (
|
||||
f'[{host}][auto][production][issue][network][single-node] '
|
||||
f'No metrics from host'
|
||||
self._open_ticket(
|
||||
event_id, 'host_metrics_missing', host, instance,
|
||||
title=f'[{host}][auto][production][issue][network][single-node] No metrics from host',
|
||||
heading='Host Metrics Alert',
|
||||
facts=[('Host', host), ('Prometheus instance', instance), ('Detected', _now_utc()),
|
||||
('Consecutive check failures', str(consec)),
|
||||
('Down for at least', ticket_text.duration(consec * self._poll_seconds()))],
|
||||
summary=(f'Prometheus is working but has no node_network_up series for {host}. '
|
||||
f'Either the host is down or node_exporter has stopped on it. '
|
||||
f'Gandalf waits {self.host_fail_thresh} checks before opening this, so a normal '
|
||||
f'reboot should not cause it.'),
|
||||
checks=[f'Is {host} powered on and reachable (Proxmox UI, ping, console)?',
|
||||
'If it is up, check `systemctl status prometheus-node-exporter` on the host.',
|
||||
'If it is down, check its switch port below: a dark port with no PoE or link '
|
||||
'points to power or cabling.'],
|
||||
priority='2', host=host,
|
||||
)
|
||||
desc = (
|
||||
f'Host Metrics Alert\n{"=" * 40}\n\n'
|
||||
f'Host: {host}\n'
|
||||
f'Prometheus instance: {instance}\n'
|
||||
f'Detected: {_now_utc()}\n'
|
||||
f'Consecutive check failures: {consec}\n\n'
|
||||
f'Prometheus is working but has no node_network_up series for {host}.\n'
|
||||
f'Either the host is down or node_exporter has stopped on it.\n\n'
|
||||
f'Please check the host power and network connectivity, then '
|
||||
f'prometheus-node-exporter on the host.'
|
||||
)
|
||||
tid = self.tickets.create(title, desc, priority='2')
|
||||
if tid:
|
||||
db.set_ticket_id(event_id, tid)
|
||||
|
||||
def _poll_seconds(self) -> int:
|
||||
return int(getattr(self, 'poll_interval', 120) or 120)
|
||||
|
||||
def _process_prometheus_outage(self, suppressions: list) -> None:
|
||||
"""Interface and host alerts can't fire without Prometheus, so say so."""
|
||||
@@ -1092,44 +1320,38 @@ class NetworkMonitor:
|
||||
f'Prometheus returned no interface data; host and interface alerting is paused ({_now_utc()})',
|
||||
)
|
||||
if not sup and consec >= self.fail_thresh and not ticket_id:
|
||||
title = (
|
||||
'[prometheus][auto][production][issue][network][cluster-wide] '
|
||||
'Gandalf has no Prometheus data'
|
||||
self._open_ticket(
|
||||
event_id, 'monitor_source_unavailable', 'prometheus', '',
|
||||
title='[prometheus][auto][production][issue][network][cluster-wide] Gandalf has no Prometheus data',
|
||||
heading='Monitoring Source Alert',
|
||||
facts=[('Prometheus URL', self.prom.url), ('Detected', _now_utc()),
|
||||
('Consecutive check failures', str(consec))],
|
||||
summary=('Gandalf could not get node_network_up data from Prometheus. While this lasts, '
|
||||
'host and interface failures will **not** be detected. UniFi, ping and modem '
|
||||
'checks keep running.'),
|
||||
checks=['Is the Prometheus service running on its host?',
|
||||
'Open Prometheus → Status → Targets and check the node exporters are being scraped.',
|
||||
'Check disk space on the Prometheus host (a full disk stops ingestion).'],
|
||||
priority='2',
|
||||
)
|
||||
desc = (
|
||||
f'Monitoring Source Alert\n{"=" * 40}\n\n'
|
||||
f'Prometheus URL: {self.prom.url}\n'
|
||||
f'Detected: {_now_utc()}\n'
|
||||
f'Consecutive check failures: {consec}\n\n'
|
||||
f'Gandalf could not get node_network_up data from Prometheus.\n'
|
||||
f'While this lasts, host and interface failures will NOT be detected.\n\n'
|
||||
f'Please check the Prometheus service and its scrape targets.'
|
||||
)
|
||||
tid = self.tickets.create(title, desc, priority='2')
|
||||
if tid:
|
||||
db.set_ticket_id(event_id, tid)
|
||||
|
||||
def _ticket_interface(
|
||||
self, event_id: int, host: str, iface: str, consec: int
|
||||
) -> None:
|
||||
title = (
|
||||
f'[{host}][auto][production][issue][network][single-node] '
|
||||
f'Interface {iface} link-down'
|
||||
self._open_ticket(
|
||||
event_id, 'interface_down', host, iface,
|
||||
title=f'[{host}][auto][production][issue][network][single-node] Interface {iface} link-down',
|
||||
heading='Network Interface Alert',
|
||||
facts=[('Host', host), ('Interface', f'`{iface}`'), ('Detected', _now_utc()),
|
||||
('Consecutive check failures', str(consec))],
|
||||
summary=(f'Interface {iface} on {host} was up and is now reporting link-down via '
|
||||
f'Prometheus node_exporter. {host} itself is still reporting, so it may be '
|
||||
f'reachable over its other interfaces.'),
|
||||
checks=['Check the switch port below: is it down on the switch side too?',
|
||||
'Reseat or swap the cable or DAC; for an optic, check the module for a fault.',
|
||||
f'On {host}, check `ip link show {iface}` and `dmesg` for NIC or driver errors.'],
|
||||
priority='2', host=host,
|
||||
)
|
||||
desc = (
|
||||
f'Network Interface Alert\n{"=" * 40}\n\n'
|
||||
f'Host: {host}\n'
|
||||
f'Interface: {iface}\n'
|
||||
f'Detected: {_now_utc()}\n'
|
||||
f'Consecutive check failures: {consec}\n\n'
|
||||
f'Interface {iface} on {host} is reporting link-down state via '
|
||||
f'Prometheus node_exporter.\n\n'
|
||||
f'Note: {host} may still be reachable via its other network interface.\n'
|
||||
f'Please inspect the cable/SFP/switch port for {host}/{iface}.'
|
||||
)
|
||||
tid = self.tickets.create(title, desc, priority='2')
|
||||
if tid:
|
||||
db.set_ticket_id(event_id, tid)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# UniFi device monitoring
|
||||
@@ -1155,23 +1377,20 @@ class NetworkMonitor:
|
||||
|
||||
def _ticket_unifi(self, event_id: int, device: dict) -> None:
|
||||
name = device['name']
|
||||
title = (
|
||||
f'[{name}][auto][production][issue][network][single-node] '
|
||||
f'UniFi device offline'
|
||||
self._open_ticket(
|
||||
event_id, 'unifi_device_offline', name, device.get('type', ''),
|
||||
title=f'[{name}][auto][production][issue][network][single-node] UniFi device offline',
|
||||
heading='UniFi Device Alert',
|
||||
facts=[('Device', name), ('Type', device.get('type', 'unknown')), ('Model', device.get('model', '')),
|
||||
('Last known IP', device.get('ip', 'unknown')), ('Detected', _now_utc())],
|
||||
summary=(f'The UniFi controller reports {name} as offline. Gandalf waits '
|
||||
f'{self.unifi_fail_thresh} checks first, so a firmware-update reboot should not cause this.'),
|
||||
checks=['Check the device for power (status LED) and, if it is PoE powered, the PoE '
|
||||
'budget on the port below.',
|
||||
'Check the uplink cable and the switch port it plugs into.',
|
||||
'In the UniFi controller, check whether the device is stuck adopting or upgrading.'],
|
||||
priority='2', host=name,
|
||||
)
|
||||
desc = (
|
||||
f'UniFi Device Alert\n{"=" * 40}\n\n'
|
||||
f'Device: {name}\n'
|
||||
f'Type: {device.get("type","unknown")}\n'
|
||||
f'Model: {device.get("model","")}\n'
|
||||
f'Last Known IP: {device.get("ip","unknown")}\n'
|
||||
f'Detected: {_now_utc()}\n\n'
|
||||
f'The UniFi device {name} is offline per the UniFi controller.\n'
|
||||
f'Please check power and cable connectivity.'
|
||||
)
|
||||
tid = self.tickets.create(title, desc, priority='2')
|
||||
if tid:
|
||||
db.set_ticket_id(event_id, tid)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Cutsheet: documented cabling vs. what discovery sees
|
||||
@@ -1206,24 +1425,20 @@ class NetworkMonitor:
|
||||
db.resolve_event('cable_mismatch', target_name, target_detail)
|
||||
|
||||
def _ticket_cable_mismatch(self, event_id: int, p: dict, hours: float) -> None:
|
||||
title = (
|
||||
f'[{p["switch"]}][auto][production][issue][network][single-node] '
|
||||
f'Cable {p["label"]} does not match the cutsheet'
|
||||
url = ticket_text.cutsheet_url(self._base_url(), p['cable_id'])
|
||||
self._open_ticket(
|
||||
event_id, 'cable_mismatch', p['switch'], p['key'],
|
||||
title=(f'[{p["switch"]}][auto][production][issue][network][single-node] '
|
||||
f'Cable {p["label"]} does not match the cutsheet'),
|
||||
heading='Cutsheet Mismatch',
|
||||
facts=[('Cable', ticket_text.link(p['label'], url)), ('Switch port', f'{p["switch"]} port {p["port"]}'),
|
||||
('Problem', p['detail']), ('Documented path', p['path']), ('Detected', _now_utc())],
|
||||
summary=f'This has not matched the cutsheet for at least {hours:g} hours.',
|
||||
checks=['If the cable was moved on purpose, update its record in the cutsheet.',
|
||||
'Otherwise, follow the label and check it is plugged into the documented port '
|
||||
'(Trace mode on the cutsheet page can find which port a cord is in).'],
|
||||
priority='4', port=(p['switch'], p['port']),
|
||||
)
|
||||
desc = (
|
||||
f'Cutsheet Mismatch\n{"=" * 40}\n\n'
|
||||
f'Cable: {p["label"]}\n'
|
||||
f'Switch port: {p["switch"]} port {p["port"]}\n'
|
||||
f'Problem: {p["detail"]}\n'
|
||||
f'Documented path: {p["path"]}\n'
|
||||
f'Detected: {_now_utc()}\n\n'
|
||||
f'This has not matched for at least {hours:g} hours.\n\n'
|
||||
f'Either the cable was moved (update the cutsheet in Gandalf: /cutsheet#cable-{p["cable_id"]}) '
|
||||
f'or it is plugged into the wrong place.'
|
||||
)
|
||||
tid = self.tickets.create(title, desc, priority='4')
|
||||
if tid:
|
||||
db.set_ticket_id(event_id, tid)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Ping-only hosts (no node_exporter)
|
||||
@@ -1253,24 +1468,20 @@ class NetworkMonitor:
|
||||
def _ticket_unreachable(
|
||||
self, event_id: int, name: str, ip: str, consec: int
|
||||
) -> None:
|
||||
title = (
|
||||
f'[{name}][auto][production][issue][network][single-node] '
|
||||
f'Host unreachable'
|
||||
self._open_ticket(
|
||||
event_id, 'host_unreachable', name, ip,
|
||||
title=f'[{name}][auto][production][issue][network][single-node] Host unreachable',
|
||||
heading='Host Reachability Alert',
|
||||
facts=[('Host', name), ('IP', ip), ('Detected', _now_utc()),
|
||||
('Consecutive check failures', str(consec)),
|
||||
('Down for at least', ticket_text.duration(consec * self._poll_seconds()))],
|
||||
summary=(f'{name} ({ip}) is not answering ping from the Pulse worker Gandalf uses. '
|
||||
f'It has no node_exporter, so ping is the only check Gandalf has for it.'),
|
||||
checks=[f'Is {name} powered on? Check its console or management interface.',
|
||||
'Check its switch port below for link and PoE.',
|
||||
'If the port is up, check the IP and firewall on the device itself.'],
|
||||
priority='2', host=name,
|
||||
)
|
||||
desc = (
|
||||
f'Host Reachability Alert\n{"=" * 40}\n\n'
|
||||
f'Host: {name}\n'
|
||||
f'IP: {ip}\n'
|
||||
f'Detected: {_now_utc()}\n'
|
||||
f'Consecutive check failures: {consec}\n\n'
|
||||
f'Host {name} ({ip}) is not responding to ping from the Gandalf monitor.\n'
|
||||
f'This host does not have a Prometheus node_exporter, so interface-level '
|
||||
f'detail is unavailable.\n\n'
|
||||
f'Please check the host power, management interface, and network connectivity.'
|
||||
)
|
||||
tid = self.tickets.create(title, desc, priority='2')
|
||||
if tid:
|
||||
db.set_ticket_id(event_id, tid)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# WAN modem monitoring
|
||||
@@ -1291,17 +1502,14 @@ class NetworkMonitor:
|
||||
event_type, severity, 'modem', name, detail, f'{description} ({_now_utc()})',
|
||||
)
|
||||
if not sup and consec >= self.modem_fail_thresh and not ticket_id:
|
||||
title = f'[{name}][auto][production][issue][network][cluster-wide] {heading}'
|
||||
desc = (
|
||||
f'WAN Modem Alert\n{"=" * 40}\n\n'
|
||||
f'Modem: {name} ({self.modem.base_url})\n'
|
||||
f'Detected: {_now_utc()}\n'
|
||||
f'Consecutive check failures: {consec}\n\n'
|
||||
f'{description}\n\n{advice}'
|
||||
self._open_ticket(
|
||||
event_id, event_type, name, detail,
|
||||
title=f'[{name}][auto][production][issue][network][cluster-wide] {heading}',
|
||||
heading='WAN Modem Alert',
|
||||
facts=[('Modem', f'{name} ({self.modem.base_url})'), ('Detected', _now_utc()),
|
||||
('Consecutive check failures', str(consec))],
|
||||
summary=description, checks=[advice], priority=priority,
|
||||
)
|
||||
tid = self.tickets.create(title, desc, priority=priority)
|
||||
if tid:
|
||||
db.set_ticket_id(event_id, tid)
|
||||
|
||||
def _process_modem_failure(self, suppressions: list, error: Exception) -> None:
|
||||
"""Handle a failed poll. Link and optical events stay as they are: we can't see them."""
|
||||
@@ -1443,7 +1651,8 @@ class NetworkMonitor:
|
||||
if not db.check_suppressed(suppressions, 'host', name) and not ticket_id:
|
||||
tid = self.tickets.create(
|
||||
f'[{name}][auto][production][issue][network][cluster-wide] {heading}',
|
||||
f'WAN Event\n{"=" * 40}\n\nModem: {name}\nReported: {_now_utc()}\n\n{description}\n\n{advice}',
|
||||
ticket_text.description('WAN Event', [('Modem', name), ('Reported', _now_utc())],
|
||||
description, [advice], recovers=False),
|
||||
priority=priority,
|
||||
)
|
||||
if tid:
|
||||
@@ -1645,6 +1854,12 @@ class NetworkMonitor:
|
||||
db.set_state('link_stats', link_data)
|
||||
except Exception as e:
|
||||
logger.error(f'Link stats collection failed: {e}', exc_info=True)
|
||||
if link_data is not None:
|
||||
self._link_data = link_data
|
||||
try:
|
||||
self._remember_uplinks(link_data)
|
||||
except Exception as e:
|
||||
logger.error(f'Recording known uplinks failed: {e}', exc_info=True)
|
||||
|
||||
# 5. Process alerts using already-fetched data
|
||||
suppressions = db.get_active_suppressions()
|
||||
@@ -1673,6 +1888,12 @@ class NetworkMonitor:
|
||||
except Exception as e:
|
||||
logger.error(f'Modem collection failed: {e}', exc_info=True)
|
||||
|
||||
# Note on the tickets of anything that cleared this cycle (or failed to post before)
|
||||
try:
|
||||
self._post_recovery_notes()
|
||||
except Exception as e:
|
||||
logger.error(f'Posting recovery notes failed: {e}', exc_info=True)
|
||||
|
||||
# Housekeeping: deactivate expired suppressions and purge old resolved events
|
||||
db.cleanup_expired_suppressions()
|
||||
db.purge_old_resolved_events(days=90)
|
||||
|
||||
Reference in New Issue
Block a user