From 437f7a415f0e1fc15fe43c43d4ec8fe1288450f3 Mon Sep 17 00:00:00 2001 From: Jared Vititoe Date: Tue, 10 Nov 2026 19:44:50 -0500 Subject: [PATCH] Add recovery notes and fuller context to tickets When an alert that opened a ticket clears, the monitor now comments on the ticket (when it cleared, how long it lasted, failed checks, a link back). It never closes or changes a ticket's status. Failed notes retry each cycle for 24 hours; a 401/403 from Tinker stops them until restart. Tickets are now Markdown with a "What to check" list and, where known: the switch port the device plugs into (kept from the last time it was seen, so a dead host still shows it), cutsheet cables and their path, how often the alert fired in the last 7/30 days, other open alerts on the device, and links into Gandalf (gandalf.public_url). Titles are unchanged so Tinker's duplicate detection still works. network_events gains recovery_notified, added at monitor startup; rows that cleared before the upgrade count as already notified. Co-Authored-By: Claude Opus 5.5 --- README.md | 42 +++- cutsheet.py | 33 ++- db.py | 75 ++++++ monitor.py | 477 ++++++++++++++++++++++++++++---------- schema.sql | 3 +- tests/test_monitor.py | 180 ++++++++++++++ tests/test_ticket_text.py | 64 +++++ ticket_text.py | 145 ++++++++++++ 8 files changed, 886 insertions(+), 133 deletions(-) create mode 100644 tests/test_ticket_text.py create mode 100644 ticket_text.py diff --git a/README.md b/README.md index 386a96d..4308436 100644 --- a/README.md +++ b/README.md @@ -255,6 +255,34 @@ can view it; only the `admin` group can edit, and every change is logged with wh A failed Pulse ping check counts as "unknown": it neither raises nor resolves `host_unreachable`. +### What a ticket contains + +Tickets are Markdown. Besides the alert's own facts and a numbered "What to check" list, each one adds +whatever Gandalf knows around the problem: + +- **Where it is connected**: the switch port(s) the host or device plugs into, with link state, speed, + PoE and SFP module, linked to that port on the switch's page. A host that has dropped off the network + has lost its LLDP entry, so Gandalf keeps the last port it saw each device on (`known_uplinks` in + `monitor_state`) and shows that, marked "last seen linked there". +- The **cutsheet** cables ending on the device or port, with their full path through patch panels +- **History**: how often the same alert fired in the last 7 and 30 days, and when it last cleared; + 3 or more in a week is called out as recurring +- **Other active alerts** on the same device, with their ticket numbers +- **Links** to the device in Gandalf (from `gandalf.public_url`) + +Titles keep the `[target][auto][production]...` tag format, which Tinker uses to spot duplicates. + +### Recovery notes + +When an alert that opened a ticket clears, the monitor adds a comment to that ticket: when it cleared, how +long it lasted, how many checks failed, and a link back to Gandalf. **Gandalf never closes or changes the +status of a ticket**; that is left to a person. The one-shot WAN reports (modem power-cycled, WAN unstable) +get no note, since they describe something already over. + +A note that fails to post is retried every cycle for 24 hours. Posting needs a Tinker API key with +`read_write` scope; if Tinker answers 401 or 403, the monitor logs one error and stops trying until it is +restarted. + ### WAN modem notes The alarm and warning limits are read from the gateway on every poll, so they are never hard-coded here. @@ -310,9 +338,12 @@ Shared by both processes. Located in the working directory (`/var/www/html/prod/ "site_id": "default" }, "ticket_api": { - "url": "https://t.lotusguild.org/api/tickets", + "url": "https://t.lotusguild.org/create_ticket_api.php", "api_key": "..." }, + "gandalf": { + "public_url": "https://gandalf.lotusguild.org" + }, "pulse": { "url": "http://:", "api_key": "...", @@ -360,7 +391,10 @@ Shared by both processes. Located in the working directory (`/var/www/html/prod/ | `unifi.controller` | UniFi controller base URL (HTTPS, self-signed cert ignored) | | `unifi.api_key` | UniFi API key from controller Settings → API | | `unifi.site_id` | UniFi site ID (default: `default`) | -| `ticket_api.api_key` | Tinker Tickets bearer token | +| `ticket_api.url` | Tinker Tickets `create_ticket_api.php` URL | +| `ticket_api.api_key` | Tinker Tickets bearer token. Needs `read_write` scope to add recovery notes | +| `ticket_api.comment_url` | Tinker comment endpoint (default: `api/ticket_comment_api.php` beside `ticket_api.url`) | +| `gandalf.public_url` | Gandalf's own address, for links in tickets (default: `https://gandalf.lotusguild.org`) | | `pulse.url` | Pulse worker API base URL (for SSH relay) | | `pulse.worker_id` | Which Pulse worker runs ethtool collection | | `pulse.timeout` | Max seconds to wait for SSH collection per host | @@ -567,6 +601,7 @@ gandalf/ ├── modem.py # Read-only AT&T gateway status page client and parser ├── devices.py # Device inventory for search and device pages ├── cutsheet.py # Cutsheet validation, path tracing, plan-vs-actual checks, CSV +├── ticket_text.py # Markdown for tickets and recovery notes ├── db.py # Database operations (MariaDB via pymysql, thread-local conn reuse) ├── schema.sql # Database schema (network_events, suppression_rules, monitor_state) ├── config.json # Runtime configuration (not committed with secrets) @@ -617,7 +652,8 @@ Take LED colors from the model's quick start guide. See the comment at the top o ### Database Schema Notes -- `network_events`: one row per active event; `resolved_at` is set when recovered +- `network_events`: one row per active event; `resolved_at` is set when recovered, and + `recovery_notified` once the recovery note is on its ticket (column added automatically at monitor startup) - `suppression_rules`: `active=FALSE` when removed; `expires_at` checked at query time - `monitor_state`: key/value store; `interface_baseline`, `link_stats`, and `modem_stats` are JSON blobs - `monitor_state` also holds `wan_history` (WAN outages, modem restarts) and `unifi_log_cursor` (how far the UniFi log has been read) diff --git a/cutsheet.py b/cutsheet.py index 138eb27..a8e08d3 100644 --- a/cutsheet.py +++ b/cutsheet.py @@ -473,6 +473,37 @@ def parse_csv(text: str, items: Dict[str, dict], max_rows: int = 2000) -> Tuple[ # ── Problems, for the monitor ─────────────────────────────────────────────── +def path_text(hops: list) -> str: + return ' → '.join(describe_node(h) for h in hops if not isinstance(h, int)) + + +def runs_for(device: str, cables: List[dict], items: Dict[str, dict], switches: dict, + port: Optional[str] = None) -> List[dict]: + """The live cables ending on a device (or one of its ports), each with its full run. + + Returns [{'cable_id', 'label', 'port', 'status', 'verdict', 'detail', 'path'}]. + """ + results = None + out = [] + for c in sorted(cables, key=lambda c: _natural(c.get('label') or '')): + if not _live(c): + continue + for end in ('a', 'b'): + if not _same(c.get(f'{end}_device') or '', device): + continue + if port is not None and str(c.get(f'{end}_port') or '') != str(port): + continue + if results is None: + results = evaluate(cables, items, switches or {}) + r = results[c['id']] + out.append({ + 'cable_id': c['id'], 'label': c.get('label') or f'#{c["id"]}', + 'port': c.get(f'{end}_port') or '', 'status': c['status'], + 'verdict': r['verdict'], 'detail': r['detail'], 'path': path_text(r['path']['hops']), + }) + return out + + def problems(cables: List[dict], items: Dict[str, dict], switches: dict) -> List[dict]: """One entry per installed run whose check found a problem. @@ -508,6 +539,6 @@ def problems(cables: List[dict], items: Dict[str, dict], switches: dict) -> List 'port': switch_end[1] if switch_end else lead['a_port'], 'verdict': r['verdict'], 'detail': r['detail'], - 'path': ' → '.join(describe_node(h) for h in hops if not isinstance(h, int)), + 'path': path_text(hops), }) return found diff --git a/db.py b/db.py index 8726162..ce28214 100644 --- a/db.py +++ b/db.py @@ -348,6 +348,81 @@ def cleanup_expired_suppressions() -> int: return n +def ensure_events_schema() -> None: + """Add columns newer than the original events table, so deploys need no manual migration.""" + with get_conn() as conn: + with conn.cursor() as cur: + cur.execute("SHOW COLUMNS FROM network_events LIKE 'recovery_notified'") + if cur.fetchone(): + return + cur.execute( + 'ALTER TABLE network_events ADD COLUMN recovery_notified TINYINT(1) NOT NULL DEFAULT 0' + ) + # Events that cleared before this existed are history, not owed a comment + cur.execute( + 'UPDATE network_events SET recovery_notified=1, last_seen=last_seen WHERE resolved_at IS NOT NULL' + ) + + +def get_pending_recovery_notes(hours: int = 24, exclude_types: tuple = ()) -> list: + """Resolved events with a ticket whose recovery comment hasn't been posted yet.""" + sql = """SELECT id, event_type, target_name, target_detail, ticket_id, + first_seen, resolved_at, consecutive_failures + FROM network_events + WHERE resolved_at IS NOT NULL AND ticket_id IS NOT NULL AND recovery_notified=0 + AND resolved_at > DATE_SUB(NOW(), INTERVAL %s HOUR)""" + params: list = [hours] + if exclude_types: + sql += f' AND event_type NOT IN ({", ".join(["%s"] * len(exclude_types))})' # nosec B608 - placeholders only + params.extend(exclude_types) + with get_conn() as conn: + with conn.cursor() as cur: + cur.execute(sql + ' ORDER BY resolved_at', params) + return list(cur.fetchall()) + + +def mark_recovery_notified(event_id: int) -> None: + with get_conn() as conn: + with conn.cursor() as cur: + # last_seen updates itself on any write; keep it as the last failing check + cur.execute( + 'UPDATE network_events SET recovery_notified=1, last_seen=last_seen WHERE id=%s', + (event_id,), + ) + + +def event_stats(event_type: str, target_name: str, target_detail: str = '') -> dict: + """How often one alert has fired before, for a ticket's history section. + + Returns {'last_7d', 'last_30d', 'last_first_seen', 'last_resolved_at'} over resolved events. + """ + key = (event_type, target_name, target_detail or '') + with get_conn() as conn: + with conn.cursor() as cur: + cur.execute( + """SELECT COALESCE(SUM(first_seen > DATE_SUB(NOW(), INTERVAL 7 DAY)), 0) AS last_7d, + COUNT(*) AS last_30d + FROM network_events + WHERE event_type=%s AND target_name=%s AND target_detail=%s + AND resolved_at IS NOT NULL AND first_seen > DATE_SUB(NOW(), INTERVAL 30 DAY)""", + key, + ) + counts = cur.fetchone() or {} + cur.execute( + """SELECT first_seen, resolved_at FROM network_events + WHERE event_type=%s AND target_name=%s AND target_detail=%s + AND resolved_at IS NOT NULL ORDER BY resolved_at DESC LIMIT 1""", + key, + ) + last = cur.fetchone() or {} + return { + 'last_7d': int(counts.get('last_7d') or 0), + 'last_30d': int(counts.get('last_30d') or 0), + 'last_first_seen': last.get('first_seen'), + 'last_resolved_at': last.get('resolved_at'), + } + + def purge_old_resolved_events(days: int = 90) -> int: """Delete resolved events older than `days` days. Returns count deleted.""" with get_conn() as conn: diff --git a/monitor.py b/monitor.py index bb19297..863d4f3 100644 --- a/monitor.py +++ b/monitor.py @@ -15,7 +15,7 @@ import statistics import time from datetime import datetime, timezone from typing import Dict, List, Optional -from urllib.parse import urlparse +from urllib.parse import urljoin, urlparse import requests from urllib3.exceptions import InsecureRequestWarning @@ -24,6 +24,7 @@ import db import cutsheet import devices import modem as modem_api +import ticket_text logging.basicConfig( @@ -388,6 +389,41 @@ class TicketClient: def __init__(self, cfg: dict): self.url = cfg.get('url', '') self.api_key = cfg.get('api_key', '') + # Tinker serves its other API scripts under api/ beside create_ticket_api.php + self.comment_url = cfg.get('comment_url') or ( + urljoin(self.url, 'api/ticket_comment_api.php') if self.url else '') + self.comments_denied = False + + def comment(self, ticket_id: str, text: str) -> bool: + """Add a Markdown comment to a ticket. Never raises; False means try again later.""" + if not self.api_key or not self.comment_url or self.comments_denied: + return False + try: + resp = requests.post( + self.comment_url, + json={'ticket_id': str(ticket_id), 'comment_text': text, 'markdown_enabled': True}, + headers={'Authorization': f'Bearer {self.api_key}'}, + timeout=15, + ) + if resp.status_code in (401, 403): + # Retrying every cycle can't fix a key without write access, so stop until restart + self.comments_denied = True + logger.error( + f'Ticket API refused a comment on #{ticket_id} (HTTP {resp.status_code}). ' + f'The API key needs read_write scope; recovery notes are off until the monitor restarts.' + ) + return False + if resp.status_code == 404: + logger.warning(f'Ticket #{ticket_id} no longer exists; skipping its recovery note') + return True + resp.raise_for_status() + if resp.json().get('success'): + logger.info(f'Commented on ticket #{ticket_id}') + return True + logger.warning(f'Unexpected comment API response for #{ticket_id}: {resp.text[:200]}') + except Exception as e: + logger.error(f'Ticket comment failed for #{ticket_id}: {e}') + return False def create(self, title: str, description: str, priority: str = '2') -> Optional[str]: if not self.api_key or not self.url: @@ -893,10 +929,21 @@ def _now_utc() -> str: return datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M:%S UTC') +def _utcnow() -> datetime: + """Naive UTC, to compare with database times.""" + return datetime.now(timezone.utc).replace(tzinfo=None) + + # -------------------------------------------------------------------------- # Monitor # -------------------------------------------------------------------------- CLUSTER_NAME = 'proxmox-cluster' +DEFAULT_PUBLIC_URL = 'https://gandalf.lotusguild.org' +# Reports of something already over: the ticket is the whole story, no recovery note +ONE_SHOT_EVENTS = ('modem_power_cycled', 'wan_unstable') +# Ports a host or device was last seen on, kept so a ticket for a dead host +# (whose link, and so its LLDP entry, is gone) can still say where it plugs in +KNOWN_UPLINKS_STATE = 'known_uplinks' class NetworkMonitor: @@ -950,6 +997,16 @@ class NetworkMonitor: db.ensure_cutsheet_schema() except Exception as e: logger.error(f'Could not create cutsheet tables: {e}') + try: + db.ensure_events_schema() + except Exception as e: + logger.error(f'Could not update the events table: {e}') + + # Absolute links in tickets point here + self.public_url = ((self.cfg.get('gandalf') or {}).get('public_url') + or DEFAULT_PUBLIC_URL).rstrip('/') + self._link_data: Optional[dict] = None + self._known_uplinks: Optional[dict] = None # Last good Prometheus result, shown (as unknown) during a Prometheus outage self._last_iface_states: Dict[str, Dict[str, bool]] = {} @@ -964,6 +1021,170 @@ class NetworkMonitor: def _hostname(self, instance: str) -> str: return self._instance_map.get(instance, instance.split(':')[0]) + # ------------------------------------------------------------------ + # Tickets: context, and notes when an alert clears + # ------------------------------------------------------------------ + def _base_url(self) -> str: + return getattr(self, 'public_url', DEFAULT_PUBLIC_URL) + + def _open_ticket( + self, event_id: int, event_type: str, target: str, detail: str, *, + title: str, heading: str, facts: list, summary: str, checks: list, priority: str, + host: Optional[str] = None, port: Optional[tuple] = None, + ) -> None: + """Open a ticket with the alert's facts plus everything Gandalf knows around it. + + host: a device whose switch ports to list. port: (switch, port_idx) the alert is about. + """ + sections = self._ticket_sections(event_id, event_type, target, detail, host=host, port=port) + desc = ticket_text.description(heading, facts, summary, checks, sections) + tid = self.tickets.create(title, desc, priority=priority) + if tid: + db.set_ticket_id(event_id, tid) + + def _ticket_sections(self, event_id: int, event_type: str, target: str, detail: str, + host: Optional[str] = None, port: Optional[tuple] = None) -> list: + """Each section is optional: a failure loses that section, never the ticket.""" + sections = [] + for title, build in ( + ('Where it is connected', lambda: self._where_lines(event_type, detail, host, port)), + ('History', lambda: ticket_text.history_lines(db.event_stats(event_type, target, detail), _utcnow())), + ('Other active alerts here', lambda: self._other_alert_lines(event_id, {target, host})), + ('Links', lambda: self._link_lines(host or (port[0] if port else None))), + ): + try: + sections.append((title, build())) + except Exception as e: + logger.error(f'Ticket section "{title}" for {target} failed: {e}', exc_info=True) + return sections + + def _switches(self) -> dict: + return (getattr(self, '_link_data', None) or {}).get('unifi_switches') or {} + + def _remember_uplinks(self, link_data: dict) -> None: + """Record where every LLDP-visible device is plugged in, for when it disappears.""" + known = self._load_known_uplinks() + switches = link_data.get('unifi_switches') or {} + names = {(p.get('lldp') or {}).get('system_name') + for sw in switches.values() for p in (sw.get('ports') or {}).values()} + now = _now_utc() + changed = False + for name in sorted(n for n in names if n): + ups = devices.uplinks_for_host(name, switches) + entry = {'seen': now, 'uplinks': [ + {k: u[k] for k in ('switch', 'port', 'port_idx', 'host_interface')} for u in ups]} + if (known.get(name) or {}).get('uplinks') != entry['uplinks']: + changed = True + known[name] = entry + self._known_uplinks = known + # Write when wiring changes, and hourly so "last seen" stays roughly right + last_saved = getattr(self, '_known_uplinks_saved', 0.0) + if changed or time.time() - last_saved > 3600: + db.set_state(KNOWN_UPLINKS_STATE, known) + self._known_uplinks_saved = time.time() + + def _load_known_uplinks(self) -> dict: + if getattr(self, '_known_uplinks', None) is None: + self._known_uplinks = {} + raw = db.get_state(KNOWN_UPLINKS_STATE) + if raw: + try: + self._known_uplinks = json.loads(raw) + except ValueError: + logger.error('Failed to parse known_uplinks JSON; starting over') + return self._known_uplinks + + def _uplinks_for(self, name: str, switches: dict) -> tuple: + """(uplinks, None) if the device is linked now, else (last known uplinks, when seen).""" + current = devices.uplinks_for_host(name, switches) + if current: + return current, None + entry = self._load_known_uplinks().get(name) or {} + return entry.get('uplinks') or [], entry.get('seen') + + def _where_lines(self, event_type: str, detail: str, host: Optional[str], port: Optional[tuple]) -> list: + base, switches = self._base_url(), self._switches() + lines = [] + if host: + ups, seen = self._uplinks_for(host, switches) + if event_type == 'interface_down': + ups = [u for u in ups if u.get('host_interface') == detail] or ups + for u in ups: + live = (switches.get(u['switch'], {}).get('ports') or {}).get(u['port']) + text = ticket_text.port_text(u['switch'], u['port'], live or dict(u, up=False)) + iface = f'`{u["host_interface"]}` → ' if u.get('host_interface') else '' + lines.append(f'- {iface}' + ticket_text.link( + text, ticket_text.device_url(base, u['switch'], u.get('port_idx')))) + if seen: + lines[-1] += f' _(last seen linked there {seen})_' + if not ups: + lines.append('- Gandalf has not seen this device on a switch port (no LLDP or client entry).') + if port: + switch, idx = port + live = next((dict(p, name=n) for n, p in (switches.get(switch, {}).get('ports') or {}).items() + if str(p.get('port_idx')) == str(idx)), None) + if live: + lines.append('- ' + ticket_text.link(ticket_text.port_text(switch, live['name'], live), + ticket_text.device_url(base, switch, idx))) + device = host or (port[0] if port else None) + if device: + data = db.cutsheet_load() + items = {i['name']: i for i in data['items']} + runs = cutsheet.runs_for(device, data['cables'], items, switches, + port=str(port[1]) if port else None) + for r in runs: + label = ticket_text.link(f'**{r["label"]}**', ticket_text.cutsheet_url(base, r['cable_id'])) + lines.append(f'- Cable {label}: {r["path"]}. Cutsheet check: ' + f'{cutsheet.VERDICTS.get(r["verdict"], r["verdict"]).lower()}.') + if not runs: + lines.append('- ' + ticket_text.link('Not on the cutsheet yet', f'{base}/cutsheet' if base else '')) + return lines + + def _other_alert_lines(self, event_id: int, names: set) -> list: + lines = [] + for name in sorted(n for n in names if n): + for e in db.get_events_for_target(name, limit=25): + if e.get('resolved_at') or e.get('id') == event_id: + continue + what = e['event_type'].replace('_', ' ') + where = f'{e["target_name"]} {e.get("target_detail") or ""}'.strip() + ticket = f' (ticket #{e["ticket_id"]})' if e.get('ticket_id') else '' + lines.append(f'- {what} on {where} since {e.get("first_seen", "?")}{ticket}') + return lines[:10] + + def _link_lines(self, device: Optional[str]) -> list: + base = self._base_url() + if not base: + return [] + lines = [] + if device: + lines.append(f'- {ticket_text.link(f"{device} in Gandalf", ticket_text.device_url(base, device))}') + lines.append(f'- {ticket_text.link("Gandalf dashboard", base + "/")}') + return lines + + def _event_url(self, event: dict) -> str: + base = self._base_url() + if not base: + return '' + if event['event_type'] == 'cable_mismatch': + cable_id = str(event.get('target_detail') or '').replace('cable ', '') + return ticket_text.cutsheet_url(base, cable_id) + if event['event_type'] in ('cluster_network_issue', 'monitor_source_unavailable'): + return base + '/' + return ticket_text.device_url(base, event['target_name']) + + def _post_recovery_notes(self) -> None: + """Comment on the tickets of alerts that have cleared. Failures retry next cycle (for 24 h).""" + if self.tickets.comments_denied: + return + now = _utcnow() + for event in db.get_pending_recovery_notes(hours=24, exclude_types=ONE_SHOT_EVENTS): + text = ticket_text.recovery_comment(event, now, self._event_url(event)) + if self.tickets.comment(event['ticket_id'], text): + db.mark_recovery_notified(event['id']) + elif self.tickets.comments_denied: + break + # ------------------------------------------------------------------ # Interface monitoring (Prometheus) # ------------------------------------------------------------------ @@ -1041,47 +1262,54 @@ class NetworkMonitor: f'{", ".join(hosts_with_regression)}', ) if not sup and (is_new or not ticket_id): - title = ( - f'[{self.cluster_name}][auto][production][issue][network][cluster-wide] ' - f'Multiple hosts reporting interface failures' - ) - desc = ( - f'Cluster Network Alert\n{"=" * 40}\n\n' - f'Affected hosts: {", ".join(hosts_with_regression)}\n' - f'Detected: {_now_utc()}\n\n' - f'{len(hosts_with_regression)} Proxmox hosts simultaneously reported ' - f'interface regressions (link-down on interfaces previously known UP) ' - f'or stopped reporting metrics.\n' - f'This likely indicates a switch or upstream network failure.\n\n' - f'Please check the core and management switches immediately.' - ) - tid = self.tickets.create(title, desc, priority='1') - if tid: - db.set_ticket_id(event_id, tid) + self._ticket_cluster(event_id, hosts_with_regression) else: db.resolve_event('cluster_network_issue', self.cluster_name, '') + def _ticket_cluster(self, event_id: int, hosts: List[str]) -> None: + switches = self._switches() + # The switches these hosts hang off are the likeliest shared cause + shared = sorted({u['switch'] for h in hosts for u in self._uplinks_for(h, switches)[0]}) + self._open_ticket( + event_id, 'cluster_network_issue', self.cluster_name, '', + title=(f'[{self.cluster_name}][auto][production][issue][network][cluster-wide] ' + f'Multiple hosts reporting interface failures'), + heading='Cluster Network Alert', + facts=[('Affected hosts', ', '.join(hosts)), + ('Switches they connect through', ', '.join(shared)), + ('Detected', _now_utc())], + summary=(f'{len(hosts)} Proxmox hosts reported interface regressions (link-down on ' + f'interfaces previously known up) or stopped reporting metrics at the same time. ' + f'This usually means a switch or upstream network failure rather than the hosts.'), + checks=['Check the core and management switches (power, uplinks, recent firmware updates).', + 'Check whether the hosts share a PDU or circuit that lost power.', + 'Look at the per-host tickets and the Gandalf dashboard for which links went down first.'], + priority='1', + ) + def _ticket_metrics_missing( self, event_id: int, host: str, instance: str, consec: int ) -> None: - title = ( - f'[{host}][auto][production][issue][network][single-node] ' - f'No metrics from host' + self._open_ticket( + event_id, 'host_metrics_missing', host, instance, + title=f'[{host}][auto][production][issue][network][single-node] No metrics from host', + heading='Host Metrics Alert', + facts=[('Host', host), ('Prometheus instance', instance), ('Detected', _now_utc()), + ('Consecutive check failures', str(consec)), + ('Down for at least', ticket_text.duration(consec * self._poll_seconds()))], + summary=(f'Prometheus is working but has no node_network_up series for {host}. ' + f'Either the host is down or node_exporter has stopped on it. ' + f'Gandalf waits {self.host_fail_thresh} checks before opening this, so a normal ' + f'reboot should not cause it.'), + checks=[f'Is {host} powered on and reachable (Proxmox UI, ping, console)?', + 'If it is up, check `systemctl status prometheus-node-exporter` on the host.', + 'If it is down, check its switch port below: a dark port with no PoE or link ' + 'points to power or cabling.'], + priority='2', host=host, ) - desc = ( - f'Host Metrics Alert\n{"=" * 40}\n\n' - f'Host: {host}\n' - f'Prometheus instance: {instance}\n' - f'Detected: {_now_utc()}\n' - f'Consecutive check failures: {consec}\n\n' - f'Prometheus is working but has no node_network_up series for {host}.\n' - f'Either the host is down or node_exporter has stopped on it.\n\n' - f'Please check the host power and network connectivity, then ' - f'prometheus-node-exporter on the host.' - ) - tid = self.tickets.create(title, desc, priority='2') - if tid: - db.set_ticket_id(event_id, tid) + + def _poll_seconds(self) -> int: + return int(getattr(self, 'poll_interval', 120) or 120) def _process_prometheus_outage(self, suppressions: list) -> None: """Interface and host alerts can't fire without Prometheus, so say so.""" @@ -1092,44 +1320,38 @@ class NetworkMonitor: f'Prometheus returned no interface data; host and interface alerting is paused ({_now_utc()})', ) if not sup and consec >= self.fail_thresh and not ticket_id: - title = ( - '[prometheus][auto][production][issue][network][cluster-wide] ' - 'Gandalf has no Prometheus data' + self._open_ticket( + event_id, 'monitor_source_unavailable', 'prometheus', '', + title='[prometheus][auto][production][issue][network][cluster-wide] Gandalf has no Prometheus data', + heading='Monitoring Source Alert', + facts=[('Prometheus URL', self.prom.url), ('Detected', _now_utc()), + ('Consecutive check failures', str(consec))], + summary=('Gandalf could not get node_network_up data from Prometheus. While this lasts, ' + 'host and interface failures will **not** be detected. UniFi, ping and modem ' + 'checks keep running.'), + checks=['Is the Prometheus service running on its host?', + 'Open Prometheus → Status → Targets and check the node exporters are being scraped.', + 'Check disk space on the Prometheus host (a full disk stops ingestion).'], + priority='2', ) - desc = ( - f'Monitoring Source Alert\n{"=" * 40}\n\n' - f'Prometheus URL: {self.prom.url}\n' - f'Detected: {_now_utc()}\n' - f'Consecutive check failures: {consec}\n\n' - f'Gandalf could not get node_network_up data from Prometheus.\n' - f'While this lasts, host and interface failures will NOT be detected.\n\n' - f'Please check the Prometheus service and its scrape targets.' - ) - tid = self.tickets.create(title, desc, priority='2') - if tid: - db.set_ticket_id(event_id, tid) def _ticket_interface( self, event_id: int, host: str, iface: str, consec: int ) -> None: - title = ( - f'[{host}][auto][production][issue][network][single-node] ' - f'Interface {iface} link-down' + self._open_ticket( + event_id, 'interface_down', host, iface, + title=f'[{host}][auto][production][issue][network][single-node] Interface {iface} link-down', + heading='Network Interface Alert', + facts=[('Host', host), ('Interface', f'`{iface}`'), ('Detected', _now_utc()), + ('Consecutive check failures', str(consec))], + summary=(f'Interface {iface} on {host} was up and is now reporting link-down via ' + f'Prometheus node_exporter. {host} itself is still reporting, so it may be ' + f'reachable over its other interfaces.'), + checks=['Check the switch port below: is it down on the switch side too?', + 'Reseat or swap the cable or DAC; for an optic, check the module for a fault.', + f'On {host}, check `ip link show {iface}` and `dmesg` for NIC or driver errors.'], + priority='2', host=host, ) - desc = ( - f'Network Interface Alert\n{"=" * 40}\n\n' - f'Host: {host}\n' - f'Interface: {iface}\n' - f'Detected: {_now_utc()}\n' - f'Consecutive check failures: {consec}\n\n' - f'Interface {iface} on {host} is reporting link-down state via ' - f'Prometheus node_exporter.\n\n' - f'Note: {host} may still be reachable via its other network interface.\n' - f'Please inspect the cable/SFP/switch port for {host}/{iface}.' - ) - tid = self.tickets.create(title, desc, priority='2') - if tid: - db.set_ticket_id(event_id, tid) # ------------------------------------------------------------------ # UniFi device monitoring @@ -1155,23 +1377,20 @@ class NetworkMonitor: def _ticket_unifi(self, event_id: int, device: dict) -> None: name = device['name'] - title = ( - f'[{name}][auto][production][issue][network][single-node] ' - f'UniFi device offline' + self._open_ticket( + event_id, 'unifi_device_offline', name, device.get('type', ''), + title=f'[{name}][auto][production][issue][network][single-node] UniFi device offline', + heading='UniFi Device Alert', + facts=[('Device', name), ('Type', device.get('type', 'unknown')), ('Model', device.get('model', '')), + ('Last known IP', device.get('ip', 'unknown')), ('Detected', _now_utc())], + summary=(f'The UniFi controller reports {name} as offline. Gandalf waits ' + f'{self.unifi_fail_thresh} checks first, so a firmware-update reboot should not cause this.'), + checks=['Check the device for power (status LED) and, if it is PoE powered, the PoE ' + 'budget on the port below.', + 'Check the uplink cable and the switch port it plugs into.', + 'In the UniFi controller, check whether the device is stuck adopting or upgrading.'], + priority='2', host=name, ) - desc = ( - f'UniFi Device Alert\n{"=" * 40}\n\n' - f'Device: {name}\n' - f'Type: {device.get("type","unknown")}\n' - f'Model: {device.get("model","")}\n' - f'Last Known IP: {device.get("ip","unknown")}\n' - f'Detected: {_now_utc()}\n\n' - f'The UniFi device {name} is offline per the UniFi controller.\n' - f'Please check power and cable connectivity.' - ) - tid = self.tickets.create(title, desc, priority='2') - if tid: - db.set_ticket_id(event_id, tid) # ------------------------------------------------------------------ # Cutsheet: documented cabling vs. what discovery sees @@ -1206,24 +1425,20 @@ class NetworkMonitor: db.resolve_event('cable_mismatch', target_name, target_detail) def _ticket_cable_mismatch(self, event_id: int, p: dict, hours: float) -> None: - title = ( - f'[{p["switch"]}][auto][production][issue][network][single-node] ' - f'Cable {p["label"]} does not match the cutsheet' + url = ticket_text.cutsheet_url(self._base_url(), p['cable_id']) + self._open_ticket( + event_id, 'cable_mismatch', p['switch'], p['key'], + title=(f'[{p["switch"]}][auto][production][issue][network][single-node] ' + f'Cable {p["label"]} does not match the cutsheet'), + heading='Cutsheet Mismatch', + facts=[('Cable', ticket_text.link(p['label'], url)), ('Switch port', f'{p["switch"]} port {p["port"]}'), + ('Problem', p['detail']), ('Documented path', p['path']), ('Detected', _now_utc())], + summary=f'This has not matched the cutsheet for at least {hours:g} hours.', + checks=['If the cable was moved on purpose, update its record in the cutsheet.', + 'Otherwise, follow the label and check it is plugged into the documented port ' + '(Trace mode on the cutsheet page can find which port a cord is in).'], + priority='4', port=(p['switch'], p['port']), ) - desc = ( - f'Cutsheet Mismatch\n{"=" * 40}\n\n' - f'Cable: {p["label"]}\n' - f'Switch port: {p["switch"]} port {p["port"]}\n' - f'Problem: {p["detail"]}\n' - f'Documented path: {p["path"]}\n' - f'Detected: {_now_utc()}\n\n' - f'This has not matched for at least {hours:g} hours.\n\n' - f'Either the cable was moved (update the cutsheet in Gandalf: /cutsheet#cable-{p["cable_id"]}) ' - f'or it is plugged into the wrong place.' - ) - tid = self.tickets.create(title, desc, priority='4') - if tid: - db.set_ticket_id(event_id, tid) # ------------------------------------------------------------------ # Ping-only hosts (no node_exporter) @@ -1253,24 +1468,20 @@ class NetworkMonitor: def _ticket_unreachable( self, event_id: int, name: str, ip: str, consec: int ) -> None: - title = ( - f'[{name}][auto][production][issue][network][single-node] ' - f'Host unreachable' + self._open_ticket( + event_id, 'host_unreachable', name, ip, + title=f'[{name}][auto][production][issue][network][single-node] Host unreachable', + heading='Host Reachability Alert', + facts=[('Host', name), ('IP', ip), ('Detected', _now_utc()), + ('Consecutive check failures', str(consec)), + ('Down for at least', ticket_text.duration(consec * self._poll_seconds()))], + summary=(f'{name} ({ip}) is not answering ping from the Pulse worker Gandalf uses. ' + f'It has no node_exporter, so ping is the only check Gandalf has for it.'), + checks=[f'Is {name} powered on? Check its console or management interface.', + 'Check its switch port below for link and PoE.', + 'If the port is up, check the IP and firewall on the device itself.'], + priority='2', host=name, ) - desc = ( - f'Host Reachability Alert\n{"=" * 40}\n\n' - f'Host: {name}\n' - f'IP: {ip}\n' - f'Detected: {_now_utc()}\n' - f'Consecutive check failures: {consec}\n\n' - f'Host {name} ({ip}) is not responding to ping from the Gandalf monitor.\n' - f'This host does not have a Prometheus node_exporter, so interface-level ' - f'detail is unavailable.\n\n' - f'Please check the host power, management interface, and network connectivity.' - ) - tid = self.tickets.create(title, desc, priority='2') - if tid: - db.set_ticket_id(event_id, tid) # ------------------------------------------------------------------ # WAN modem monitoring @@ -1291,17 +1502,14 @@ class NetworkMonitor: event_type, severity, 'modem', name, detail, f'{description} ({_now_utc()})', ) if not sup and consec >= self.modem_fail_thresh and not ticket_id: - title = f'[{name}][auto][production][issue][network][cluster-wide] {heading}' - desc = ( - f'WAN Modem Alert\n{"=" * 40}\n\n' - f'Modem: {name} ({self.modem.base_url})\n' - f'Detected: {_now_utc()}\n' - f'Consecutive check failures: {consec}\n\n' - f'{description}\n\n{advice}' + self._open_ticket( + event_id, event_type, name, detail, + title=f'[{name}][auto][production][issue][network][cluster-wide] {heading}', + heading='WAN Modem Alert', + facts=[('Modem', f'{name} ({self.modem.base_url})'), ('Detected', _now_utc()), + ('Consecutive check failures', str(consec))], + summary=description, checks=[advice], priority=priority, ) - tid = self.tickets.create(title, desc, priority=priority) - if tid: - db.set_ticket_id(event_id, tid) def _process_modem_failure(self, suppressions: list, error: Exception) -> None: """Handle a failed poll. Link and optical events stay as they are: we can't see them.""" @@ -1443,7 +1651,8 @@ class NetworkMonitor: if not db.check_suppressed(suppressions, 'host', name) and not ticket_id: tid = self.tickets.create( f'[{name}][auto][production][issue][network][cluster-wide] {heading}', - f'WAN Event\n{"=" * 40}\n\nModem: {name}\nReported: {_now_utc()}\n\n{description}\n\n{advice}', + ticket_text.description('WAN Event', [('Modem', name), ('Reported', _now_utc())], + description, [advice], recovers=False), priority=priority, ) if tid: @@ -1645,6 +1854,12 @@ class NetworkMonitor: db.set_state('link_stats', link_data) except Exception as e: logger.error(f'Link stats collection failed: {e}', exc_info=True) + if link_data is not None: + self._link_data = link_data + try: + self._remember_uplinks(link_data) + except Exception as e: + logger.error(f'Recording known uplinks failed: {e}', exc_info=True) # 5. Process alerts using already-fetched data suppressions = db.get_active_suppressions() @@ -1673,6 +1888,12 @@ class NetworkMonitor: except Exception as e: logger.error(f'Modem collection failed: {e}', exc_info=True) + # Note on the tickets of anything that cleared this cycle (or failed to post before) + try: + self._post_recovery_notes() + except Exception as e: + logger.error(f'Posting recovery notes failed: {e}', exc_info=True) + # Housekeeping: deactivate expired suppressions and purge old resolved events db.cleanup_expired_suppressions() db.purge_old_resolved_events(days=90) diff --git a/schema.sql b/schema.sql index af8b1ad..fbfc6cf 100644 --- a/schema.sql +++ b/schema.sql @@ -21,8 +21,9 @@ CREATE TABLE IF NOT EXISTS network_events ( resolved_at TIMESTAMP NULL, consecutive_failures INT NOT NULL DEFAULT 1, ticket_id VARCHAR(20) NULL, + recovery_notified TINYINT(1) NOT NULL DEFAULT 0, -- recovery comment posted on ticket_id - INDEX idx_active (resolved_at), + INDEX idx_active (resolved_at), INDEX idx_target (target_name, target_detail), INDEX idx_type (event_type), -- Compound index for hot upsert_event lookup: event_type+target+resolved_at diff --git a/tests/test_monitor.py b/tests/test_monitor.py index 973d424..ca462f6 100644 --- a/tests/test_monitor.py +++ b/tests/test_monitor.py @@ -3,6 +3,7 @@ import json import os import sys import time +from datetime import timedelta from unittest.mock import MagicMock, patch import pytest @@ -24,6 +25,11 @@ def db(): mock_db.upsert_event.return_value = (1, False, 2, None) mock_db.get_modem_rx_values.return_value = [] mock_db.get_state.return_value = None + mock_db.event_stats.return_value = {'last_7d': 0, 'last_30d': 0, + 'last_first_seen': None, 'last_resolved_at': None} + mock_db.get_events_for_target.return_value = [] + mock_db.cutsheet_load.return_value = {'items': [], 'cables': [], 'port_labels': []} + mock_db.get_pending_recovery_notes.return_value = [] yield mock_db @@ -560,3 +566,177 @@ class TestCutsheetMismatch: assert not db.upsert_event.called resolved = {c.args for c in db.resolve_event.call_args_list} assert ('cable_mismatch', 'sw', 'cable 7') in resolved and ('cable_mismatch', 'other', 'cable 9') in resolved + + +class TestTicketContext: + SWITCHES = {'agg': {'ports': { + 'Large1': {'port_idx': 7, 'up': True, 'speed_mbps': 10000, + 'lldp': {'system_name': 'large1', 'port_id': 'enp1s0f0'}, + 'sfp': {'found': True, 'kind': 'dac', 'part': 'DAC-10G'}}, + 'Port 3': {'port_idx': 3, 'up': False, 'speed_mbps': 0}, + }}} + + def _mon(self, mon, switches=None): + mon.public_url = 'https://gandalf.test' + mon.poll_interval = 120 + mon._link_data = {'unifi_switches': switches if switches is not None else self.SWITCHES} + return mon + + def _desc(self, mon): + return mon.tickets.create.call_args.args[1] + + def test_interface_ticket_names_the_switch_port_and_links(self, mon, db): + self._mon(mon)._ticket_interface(1, 'large1', 'enp1s0f0', 2) + desc = self._desc(mon) + assert '### Where it is connected' in desc + assert '`enp1s0f0` → [agg port 7 "Large1": up at 10 Gbps, module: DAC DAC-10G]' in desc + assert '(https://gandalf.test/device/agg#port-7)' in desc + assert 'https://gandalf.test/device/large1' in desc + assert 'Opened automatically by Gandalf' in desc + + def test_dead_host_uses_the_last_known_port(self, mon, db): + mon._known_uplinks = {'large1': {'seen': '2026-11-10 10:00:00 UTC', 'uplinks': [ + {'switch': 'agg', 'port': 'Port 3', 'port_idx': 3, 'host_interface': 'eno1'}]}} + self._mon(mon)._ticket_metrics_missing(1, 'micro1', 'x:9100', 5) + assert 'not seen this device on a switch port' in self._desc(mon) + mon.tickets.create.reset_mock() + unplugged = {'agg': {'ports': {'Port 3': {'port_idx': 3, 'up': False}}}} + self._mon(mon, unplugged)._ticket_unreachable(1, 'large1', '10.0.0.9', 5) + desc = self._desc(mon) + assert 'agg port 3: down' in desc + assert 'last seen linked there 2026-11-10 10:00:00 UTC' in desc + assert 'Down for at least' in desc and '10 min' in desc + + def test_cutsheet_runs_and_history(self, mon, db): + db.cutsheet_load.return_value = {'items': [], 'port_labels': [], 'cables': [ + {'id': 4, 'label': 'C04', 'a_device': 'agg', 'a_port': '7', 'a_side': '', + 'b_device': 'large1', 'b_port': 'enp1s0f0', 'b_side': '', 'status': 'installed'}]} + db.event_stats.return_value = {'last_7d': 3, 'last_30d': 5, + 'last_first_seen': monitor._utcnow(), 'last_resolved_at': monitor._utcnow()} + self._mon(mon)._ticket_interface(1, 'large1', 'enp1s0f0', 2) + desc = self._desc(mon) + assert '[**C04**](https://gandalf.test/cutsheet#cable-4): agg port 7 → large1 enp1s0f0' in desc + assert 'Happened 3 times in the last 7 days and 5 in the last 30' in desc + assert 'This keeps coming back' in desc + + def test_other_open_alerts_are_listed(self, mon, db): + db.get_events_for_target.return_value = [ + {'id': 1, 'event_type': 'interface_down', 'target_name': 'large1', 'target_detail': 'eno1'}, + {'id': 2, 'event_type': 'interface_down', 'target_name': 'large1', 'target_detail': 'eno2', + 'first_seen': 'T', 'ticket_id': '55'}, + {'id': 3, 'event_type': 'host_unreachable', 'target_name': 'large1', 'resolved_at': 'R'}, + ] + self._mon(mon)._ticket_interface(1, 'large1', 'eno1', 2) + desc = self._desc(mon) + assert '- interface down on large1 eno2 since T (ticket #55)' in desc + assert 'eno1 since' not in desc and 'host unreachable' not in desc + + def test_a_failing_section_still_opens_the_ticket(self, mon, db): + db.event_stats.side_effect = RuntimeError('db gone') + self._mon(mon)._ticket_interface(1, 'large1', 'enp1s0f0', 2) + desc = self._desc(mon) + assert '### History' not in desc and '### Where it is connected' in desc + db.set_ticket_id.assert_called_with(1, '123') + + def test_cable_mismatch_shows_the_switch_port(self, mon, db): + self._mon(mon)._ticket_cable_mismatch(9, { + 'switch': 'agg', 'port': '7', 'cable_id': 4, 'label': 'C04', 'key': 'cable 4', + 'detail': 'A different device is connected', 'path': 'agg port 7 → tv'}, 24) + desc = self._desc(mon) + assert '[C04](https://gandalf.test/cutsheet#cable-4)' in desc + assert '[agg port 7 "Large1": up at 10 Gbps' in desc + + def test_titles_keep_the_tag_format(self, mon, db): + mon.unifi_fail_thresh = 3 + self._mon(mon)._ticket_unifi(1, {'name': 'ap1', 'type': 'uap', 'model': 'U6', 'ip': '1.2.3.4'}) + assert mon.tickets.create.call_args.args[0] == \ + '[ap1][auto][production][issue][network][single-node] UniFi device offline' + + def test_remember_uplinks_saves_only_on_change(self, mon, db): + self._mon(mon) + mon._known_uplinks = None + mon._remember_uplinks({'unifi_switches': self.SWITCHES}) + saved = db.set_state.call_args.args + assert saved[0] == 'known_uplinks' + assert saved[1]['large1']['uplinks'] == [ + {'switch': 'agg', 'port': 'Large1', 'port_idx': 7, 'host_interface': 'enp1s0f0'}] + db.set_state.reset_mock() + mon._remember_uplinks({'unifi_switches': self.SWITCHES}) + assert not db.set_state.called + + +class TestRecoveryNotes: + def _event(self, **kw): + now = monitor._utcnow() + e = {'id': 8, 'event_type': 'host_metrics_missing', 'target_name': 'micro1', 'target_detail': 'x:9100', + 'ticket_id': '993', 'first_seen': now - timedelta(minutes=12), 'resolved_at': now, + 'consecutive_failures': 6} + e.update(kw) + return e + + def test_cleared_alert_comments_and_marks(self, mon, db): + mon.public_url = 'https://gandalf.test' + mon.tickets.comments_denied = False + mon.tickets.comment.return_value = True + db.get_pending_recovery_notes.return_value = [self._event()] + mon._post_recovery_notes() + ticket_id, text = mon.tickets.comment.call_args.args + assert ticket_id == '993' + assert 'Cleared' in text and '12 min' in text and 'Failed checks:** 6' in text + assert 'https://gandalf.test/device/micro1' in text + db.mark_recovery_notified.assert_called_once_with(8) + assert db.get_pending_recovery_notes.call_args.kwargs['exclude_types'] == monitor.ONE_SHOT_EVENTS + + def test_failed_comment_is_left_for_next_cycle(self, mon, db): + mon.tickets.comments_denied = False + mon.tickets.comment.return_value = False + db.get_pending_recovery_notes.return_value = [self._event(), self._event(id=9)] + mon._post_recovery_notes() + assert not db.mark_recovery_notified.called + assert mon.tickets.comment.call_count == 2 + + def test_denied_key_stops_trying(self, mon, db): + mon.tickets.comments_denied = True + mon._post_recovery_notes() + assert not db.get_pending_recovery_notes.called + + def test_cable_mismatch_links_the_cutsheet(self, mon, db): + mon.public_url = 'https://gandalf.test' + url = mon._event_url({'event_type': 'cable_mismatch', 'target_name': 'sw', 'target_detail': 'cable 7'}) + assert url == 'https://gandalf.test/cutsheet#cable-7' + + +class TestTicketClientComment: + def _client(self): + return monitor.TicketClient({'url': 'http://t.test/create_ticket_api.php', 'api_key': 'k'}) + + def test_comment_endpoint_sits_beside_create(self): + assert self._client().comment_url == 'http://t.test/api/ticket_comment_api.php' + sub = monitor.TicketClient({'url': 'http://t.test/tinker/create_ticket_api.php', 'api_key': 'k'}) + assert sub.comment_url == 'http://t.test/tinker/api/ticket_comment_api.php' + + def test_payload(self): + c = self._client() + resp = MagicMock(status_code=200) + resp.json.return_value = {'success': True, 'comment_id': 1} + with patch.object(monitor.requests, 'post', return_value=resp) as post: + assert c.comment('42', 'hi') is True + assert post.call_args.kwargs['json'] == {'ticket_id': '42', 'comment_text': 'hi', 'markdown_enabled': True} + assert post.call_args.kwargs['headers'] == {'Authorization': 'Bearer k'} + + def test_forbidden_disables_comments(self): + c = self._client() + with patch.object(monitor.requests, 'post', return_value=MagicMock(status_code=403)) as post: + assert c.comment('42', 'hi') is False + assert c.comment('43', 'hi') is False + assert post.call_count == 1 and c.comments_denied + + def test_missing_ticket_counts_as_done(self): + with patch.object(monitor.requests, 'post', return_value=MagicMock(status_code=404)): + assert self._client().comment('42', 'hi') is True + + def test_network_error_is_retryable(self): + c = self._client() + with patch.object(monitor.requests, 'post', side_effect=OSError('down')): + assert c.comment('42', 'hi') is False + assert not c.comments_denied diff --git a/tests/test_ticket_text.py b/tests/test_ticket_text.py new file mode 100644 index 0000000..ab27e5e --- /dev/null +++ b/tests/test_ticket_text.py @@ -0,0 +1,64 @@ +"""Tests for ticket_text.py: the Markdown Gandalf puts in tickets.""" +import os +import sys +from datetime import datetime, timedelta + +sys.path.insert(0, os.path.dirname(os.path.dirname(__file__))) + +import ticket_text as tt # noqa: E402 + +NOW = datetime(2026, 11, 10, 18, 0) + + +def test_duration(): + assert tt.duration(42) == '42 s' + assert tt.duration(600) == '10 min' + assert tt.duration(3600) == '1 h' + assert tt.duration(3 * 3600 + 120) == '3 h 2 min' + assert tt.duration(3 * 86400 + 7200) == '3 d 2 h' + + +def test_speed(): + assert tt.speed(100) == '100 Mbps' + assert tt.speed(2500) == '2.5 Gbps' + assert tt.speed(10000) == '10 Gbps' + assert tt.speed(0) == '' + + +def test_device_url_escapes_names(): + assert tt.device_url('https://g', 'USW Flex Mini', 3) == 'https://g/device/USW%20Flex%20Mini#port-3' + assert tt.device_url('', 'x') == '' + + +def test_port_text(): + assert tt.port_text('sw', 'Port 4', {'port_idx': 4, 'up': True, 'speed_mbps': 1000, 'poe_power': 4.21}) == \ + 'sw port 4: up at 1 Gbps, PoE 4.2 W' + assert tt.port_text('sw', 'Cam', {'port_idx': 9, 'up': False, + 'sfp': {'found': True, 'kind': 'optic', 'part': 'SR', 'rx_fault': True}}) == \ + 'sw port 9 "Cam": down, module: optic SR, **module reports a fault**' + + +def test_description_skips_empty_facts_and_sections(): + text = tt.description('Alert', [('Host', 'a'), ('Blank', '')], 'Summary.', ['One', 'Two'], + [('Empty', []), ('Links', ['- x'])]) + assert text.startswith('## Alert\n\n- **Host:** a\n\nSummary.') + assert 'Blank' not in text and '### Empty' not in text + assert '1. One\n2. Two' in text and '### Links\n- x' in text + + +def test_history_lines(): + assert tt.history_lines({'last_7d': 0, 'last_30d': 0}, NOW) == ['- First time in the last 30 days.'] + lines = tt.history_lines({'last_7d': 1, 'last_30d': 2, 'last_first_seen': NOW - timedelta(hours=26), + 'last_resolved_at': NOW - timedelta(hours=25, minutes=55)}, NOW) + assert lines[0].startswith('- Happened 1 time in the last 7 days and 2') + assert lines[1] == '- Last cleared 2026-11-09 16:05 UTC (25 h 55 min ago), after 5 min.' + + +def test_recovery_comment(): + text = tt.recovery_comment({'event_type': 'interface_down', 'target_name': 'large1', 'target_detail': 'eno1', + 'first_seen': NOW - timedelta(minutes=7), 'resolved_at': NOW, + 'consecutive_failures': 3}, NOW, 'https://g/device/large1') + assert text.startswith('**✅ Cleared**: interface down on large1 eno1.') + assert '- **Lasted:** 7 min (first seen 2026-11-10 17:53 UTC)' in text + assert '- **Gandalf:** https://g/device/large1' in text + assert 'leaves this ticket open' in text diff --git a/ticket_text.py b/ticket_text.py new file mode 100644 index 0000000..b98fd3a --- /dev/null +++ b/ticket_text.py @@ -0,0 +1,145 @@ +"""Markdown for the tickets Gandalf opens in Tinker Tickets, and the notes it adds to them. + +Pure formatting only: monitor.py gathers the facts and passes them in. +""" +from datetime import datetime, timezone +from typing import List, Optional, Sequence, Tuple +from urllib.parse import quote + +Section = Tuple[str, List[str]] + + +def duration(seconds) -> str: + seconds = max(0, int(seconds)) + if seconds < 60: + return f'{seconds} s' + minutes = seconds // 60 + if minutes < 60: + return f'{minutes} min' + hours, minutes = divmod(minutes, 60) + if hours < 48: + return f'{hours} h {minutes} min' if minutes else f'{hours} h' + days, hours = divmod(hours, 24) + return f'{days} d {hours} h' if hours else f'{days} d' + + +def utc(dt: Optional[datetime]) -> str: + """Database times are naive UTC.""" + if dt is None: + return 'unknown' + if dt.tzinfo is not None: + dt = dt.astimezone(timezone.utc).replace(tzinfo=None) + return dt.strftime('%Y-%m-%d %H:%M UTC') + + +def speed(mbps) -> str: + if not mbps: + return '' + mbps = int(mbps) + if mbps >= 1000: + gbps = mbps / 1000 + return f'{gbps:g} Gbps' + return f'{mbps} Mbps' + + +def device_url(base: str, name: str, port_idx=None) -> str: + if not base or not name: + return '' + url = f'{base}/device/{quote(name, safe="")}' + return f'{url}#port-{port_idx}' if port_idx not in (None, '') else url + + +def cutsheet_url(base: str, cable_id) -> str: + return f'{base}/cutsheet#cable-{cable_id}' if base else '' + + +def link(text: str, url: str) -> str: + return f'[{text}]({url})' if url else text + + +def port_text(switch: str, port_name: str, port: dict) -> str: + """'8SFP-AGG port 7 "Large1": up at 10 Gbps, DAC'""" + idx = port.get('port_idx') + text = f'{switch} port {idx}' if idx is not None else f'{switch} {port_name}' + if port_name and idx is not None and port_name != f'Port {idx}': + text += f' "{port_name}"' + state = [] + if port.get('up'): + rate = speed(port.get('speed_mbps')) + state.append(f'up at {rate}' if rate else 'up') + else: + state.append('down') + if port.get('poe_power'): + state.append(f'PoE {float(port["poe_power"]):.1f} W') + sfp = port.get('sfp') or {} + if sfp.get('found'): + kind = 'DAC' if sfp.get('kind') == 'dac' else sfp.get('kind', '') + module = ' '.join(p for p in (kind, sfp.get('part', '')) if p) + if module: + state.append(f'module: {module}') + if sfp.get('rx_fault') or sfp.get('tx_fault'): + state.append('**module reports a fault**') + return f'{text}: {", ".join(state)}' + + +def description(heading: str, facts: Sequence[Tuple[str, str]], summary: str, + checks: Sequence[str], sections: Sequence[Section] = (), recovers: bool = True) -> str: + lines = [f'## {heading}', ''] + lines += [f'- **{key}:** {value}' for key, value in facts if value not in (None, '')] + lines += ['', summary.strip()] + if checks: + lines += ['', '### What to check'] + lines += [f'{n}. {check}' for n, check in enumerate(checks, 1)] + for title, body in sections: + if body: + lines += ['', f'### {title}'] + list(body) + if recovers: + lines += ['', '_Opened automatically by Gandalf. It adds a note here when the problem clears, ' + 'but leaves closing the ticket to you._'] + else: + lines += ['', '_Reported automatically by Gandalf. This already happened, so there will be no ' + 'recovery note._'] + return '\n'.join(lines) + + +def history_lines(stats: dict, now: datetime) -> List[str]: + week, month = stats.get('last_7d', 0), stats.get('last_30d', 0) + if not month: + lines = ['- First time in the last 30 days.'] + else: + lines = [f'- Happened {week} time{"s" if week != 1 else ""} in the last 7 days ' + f'and {month} in the last 30 (not counting this one).'] + if week >= 3: + lines.append('- **This keeps coming back**: look for a cause that comes and goes ' + '(a loose cable, a failing optic, power, or a scheduled job).') + first, resolved = stats.get('last_first_seen'), stats.get('last_resolved_at') + if resolved: + took = duration((resolved - first).total_seconds()) if first else 'unknown' + ago = duration((now - resolved).total_seconds()) + lines.append(f'- Last cleared {utc(resolved)} ({ago} ago), after {took}.') + return lines + + +ISSUE_FIXED = { + 'cable_mismatch': 'The cable now matches the cutsheet (or the cutsheet was updated).', + 'monitor_source_unavailable': 'Prometheus is returning data again; host and interface alerting has resumed.', +} + + +def recovery_comment(event: dict, now: datetime, url: str = '') -> str: + """The note Gandalf adds to a ticket when its alert clears.""" + first, resolved = event.get('first_seen'), event.get('resolved_at') or now + target = event['target_name'] + (f' {event["target_detail"]}' if event.get('target_detail') else '') + lines = [f'**✅ Cleared**: {event["event_type"].replace("_", " ")} on {target}.', ''] + lines.append(f'- **Cleared:** {utc(resolved)}') + if first: + lines.append(f'- **Lasted:** {duration((resolved - first).total_seconds())} ' + f'(first seen {utc(first)})') + if event.get('consecutive_failures'): + lines.append(f'- **Failed checks:** {event["consecutive_failures"]}') + if url: + lines.append(f'- **Gandalf:** {url}') + lines += ['', ISSUE_FIXED.get(event['event_type'], 'Gandalf no longer sees the problem.')] + lines += ['', '_Gandalf leaves this ticket open. Close it once you know why it happened, ' + 'or leave it open if it needs follow-up._'] + return '\n'.join(lines)