Merge pull request 'Recovery notes and fuller context on tickets' (#20) from ticket-recovery-and-context into main

README.md

@@ -255,6 +255,34 @@ can view it; only the `admin` group can edit, and every change is logged with wh
 
 A failed Pulse ping check counts as "unknown": it neither raises nor resolves `host_unreachable`.
 
+### What a ticket contains
+
+Tickets are Markdown. Besides the alert's own facts and a numbered "What to check" list, each one adds
+whatever Gandalf knows around the problem:
+
+- **Where it is connected**: the switch port(s) the host or device plugs into, with link state, speed,
+  PoE and SFP module, linked to that port on the switch's page. A host that has dropped off the network
+  has lost its LLDP entry, so Gandalf keeps the last port it saw each device on (`known_uplinks` in
+  `monitor_state`) and shows that, marked "last seen linked there".
+- The **cutsheet** cables ending on the device or port, with their full path through patch panels
+- **History**: how often the same alert fired in the last 7 and 30 days, and when it last cleared;
+  3 or more in a week is called out as recurring
+- **Other active alerts** on the same device, with their ticket numbers
+- **Links** to the device in Gandalf (from `gandalf.public_url`)
+
+Titles keep the `[target][auto][production]...` tag format, which Tinker uses to spot duplicates.
+
+### Recovery notes
+
+When an alert that opened a ticket clears, the monitor adds a comment to that ticket: when it cleared, how
+long it lasted, how many checks failed, and a link back to Gandalf. **Gandalf never closes or changes the
+status of a ticket**; that is left to a person. The one-shot WAN reports (modem power-cycled, WAN unstable)
+get no note, since they describe something already over.
+
+A note that fails to post is retried every cycle for 24 hours. Posting needs a Tinker API key with
+`read_write` scope; if Tinker answers 401 or 403, the monitor logs one error and stops trying until it is
+restarted.
+
 ### WAN modem notes
 
 The alarm and warning limits are read from the gateway on every poll, so they are never hard-coded here.
@@ -310,9 +338,12 @@ Shared by both processes. Located in the working directory (`/var/www/html/prod/
     "site_id": "default"
   },
   "ticket_api": {
-    "url": "https://t.lotusguild.org/api/tickets",
+    "url": "https://t.lotusguild.org/create_ticket_api.php",
     "api_key": "..."
   },
+  "gandalf": {
+    "public_url": "https://gandalf.lotusguild.org"
+  },
   "pulse": {
     "url": "http://<pulse-host>:<port>",
     "api_key": "...",
@@ -360,7 +391,10 @@ Shared by both processes. Located in the working directory (`/var/www/html/prod/
 | `unifi.controller` | UniFi controller base URL (HTTPS, self-signed cert ignored) |
 | `unifi.api_key` | UniFi API key from controller Settings → API |
 | `unifi.site_id` | UniFi site ID (default: `default`) |
-| `ticket_api.api_key` | Tinker Tickets bearer token |
+| `ticket_api.url` | Tinker Tickets `create_ticket_api.php` URL |
+| `ticket_api.api_key` | Tinker Tickets bearer token. Needs `read_write` scope to add recovery notes |
+| `ticket_api.comment_url` | Tinker comment endpoint (default: `api/ticket_comment_api.php` beside `ticket_api.url`) |
+| `gandalf.public_url` | Gandalf's own address, for links in tickets (default: `https://gandalf.lotusguild.org`) |
 | `pulse.url` | Pulse worker API base URL (for SSH relay) |
 | `pulse.worker_id` | Which Pulse worker runs ethtool collection |
 | `pulse.timeout` | Max seconds to wait for SSH collection per host |
@@ -567,6 +601,7 @@ gandalf/
 ├── modem.py            # Read-only AT&T gateway status page client and parser
 ├── devices.py          # Device inventory for search and device pages
 ├── cutsheet.py         # Cutsheet validation, path tracing, plan-vs-actual checks, CSV
+├── ticket_text.py      # Markdown for tickets and recovery notes
 ├── db.py               # Database operations (MariaDB via pymysql, thread-local conn reuse)
 ├── schema.sql          # Database schema (network_events, suppression_rules, monitor_state)
 ├── config.json         # Runtime configuration (not committed with secrets)
@@ -617,7 +652,8 @@ Take LED colors from the model's quick start guide. See the comment at the top o
 
 ### Database Schema Notes
 
-- `network_events`: one row per active event; `resolved_at` is set when recovered
+- `network_events`: one row per active event; `resolved_at` is set when recovered, and
+  `recovery_notified` once the recovery note is on its ticket (column added automatically at monitor startup)
 - `suppression_rules`: `active=FALSE` when removed; `expires_at` checked at query time
 - `monitor_state`: key/value store; `interface_baseline`, `link_stats`, and `modem_stats` are JSON blobs
 - `monitor_state` also holds `wan_history` (WAN outages, modem restarts) and `unifi_log_cursor` (how far the UniFi log has been read)

cutsheet.py

@@ -473,6 +473,37 @@ def parse_csv(text: str, items: Dict[str, dict], max_rows: int = 2000) -> Tuple[
 
 # ── Problems, for the monitor ───────────────────────────────────────────────
 
+def path_text(hops: list) -> str:
+    return ' → '.join(describe_node(h) for h in hops if not isinstance(h, int))
+
+
+def runs_for(device: str, cables: List[dict], items: Dict[str, dict], switches: dict,
+             port: Optional[str] = None) -> List[dict]:
+    """The live cables ending on a device (or one of its ports), each with its full run.
+
+    Returns [{'cable_id', 'label', 'port', 'status', 'verdict', 'detail', 'path'}].
+    """
+    results = None
+    out = []
+    for c in sorted(cables, key=lambda c: _natural(c.get('label') or '')):
+        if not _live(c):
+            continue
+        for end in ('a', 'b'):
+            if not _same(c.get(f'{end}_device') or '', device):
+                continue
+            if port is not None and str(c.get(f'{end}_port') or '') != str(port):
+                continue
+            if results is None:
+                results = evaluate(cables, items, switches or {})
+            r = results[c['id']]
+            out.append({
+                'cable_id': c['id'], 'label': c.get('label') or f'#{c["id"]}',
+                'port': c.get(f'{end}_port') or '', 'status': c['status'],
+                'verdict': r['verdict'], 'detail': r['detail'], 'path': path_text(r['path']['hops']),
+            })
+    return out
+
+
 def problems(cables: List[dict], items: Dict[str, dict], switches: dict) -> List[dict]:
     """One entry per installed run whose check found a problem.
 
@@ -508,6 +539,6 @@ def problems(cables: List[dict], items: Dict[str, dict], switches: dict) -> List
             'port': switch_end[1] if switch_end else lead['a_port'],
             'verdict': r['verdict'],
             'detail': r['detail'],
-            'path': ' → '.join(describe_node(h) for h in hops if not isinstance(h, int)),
+            'path': path_text(hops),
         })
     return found

db.py

@@ -348,6 +348,81 @@ def cleanup_expired_suppressions() -> int:
     return n
 
 
+def ensure_events_schema() -> None:
+    """Add columns newer than the original events table, so deploys need no manual migration."""
+    with get_conn() as conn:
+        with conn.cursor() as cur:
+            cur.execute("SHOW COLUMNS FROM network_events LIKE 'recovery_notified'")
+            if cur.fetchone():
+                return
+            cur.execute(
+                'ALTER TABLE network_events ADD COLUMN recovery_notified TINYINT(1) NOT NULL DEFAULT 0'
+            )
+            # Events that cleared before this existed are history, not owed a comment
+            cur.execute(
+                'UPDATE network_events SET recovery_notified=1, last_seen=last_seen WHERE resolved_at IS NOT NULL'
+            )
+
+
+def get_pending_recovery_notes(hours: int = 24, exclude_types: tuple = ()) -> list:
+    """Resolved events with a ticket whose recovery comment hasn't been posted yet."""
+    sql = """SELECT id, event_type, target_name, target_detail, ticket_id,
+                    first_seen, resolved_at, consecutive_failures
+             FROM network_events
+             WHERE resolved_at IS NOT NULL AND ticket_id IS NOT NULL AND recovery_notified=0
+             AND resolved_at > DATE_SUB(NOW(), INTERVAL %s HOUR)"""
+    params: list = [hours]
+    if exclude_types:
+        sql += f' AND event_type NOT IN ({", ".join(["%s"] * len(exclude_types))})'  # nosec B608 - placeholders only
+        params.extend(exclude_types)
+    with get_conn() as conn:
+        with conn.cursor() as cur:
+            cur.execute(sql + ' ORDER BY resolved_at', params)
+            return list(cur.fetchall())
+
+
+def mark_recovery_notified(event_id: int) -> None:
+    with get_conn() as conn:
+        with conn.cursor() as cur:
+            # last_seen updates itself on any write; keep it as the last failing check
+            cur.execute(
+                'UPDATE network_events SET recovery_notified=1, last_seen=last_seen WHERE id=%s',
+                (event_id,),
+            )
+
+
+def event_stats(event_type: str, target_name: str, target_detail: str = '') -> dict:
+    """How often one alert has fired before, for a ticket's history section.
+
+    Returns {'last_7d', 'last_30d', 'last_first_seen', 'last_resolved_at'} over resolved events.
+    """
+    key = (event_type, target_name, target_detail or '')
+    with get_conn() as conn:
+        with conn.cursor() as cur:
+            cur.execute(
+                """SELECT COALESCE(SUM(first_seen > DATE_SUB(NOW(), INTERVAL 7 DAY)), 0) AS last_7d,
+                          COUNT(*) AS last_30d
+                   FROM network_events
+                   WHERE event_type=%s AND target_name=%s AND target_detail=%s
+                   AND resolved_at IS NOT NULL AND first_seen > DATE_SUB(NOW(), INTERVAL 30 DAY)""",
+                key,
+            )
+            counts = cur.fetchone() or {}
+            cur.execute(
+                """SELECT first_seen, resolved_at FROM network_events
+                   WHERE event_type=%s AND target_name=%s AND target_detail=%s
+                   AND resolved_at IS NOT NULL ORDER BY resolved_at DESC LIMIT 1""",
+                key,
+            )
+            last = cur.fetchone() or {}
+    return {
+        'last_7d': int(counts.get('last_7d') or 0),
+        'last_30d': int(counts.get('last_30d') or 0),
+        'last_first_seen': last.get('first_seen'),
+        'last_resolved_at': last.get('resolved_at'),
+    }
+
+
 def purge_old_resolved_events(days: int = 90) -> int:
     """Delete resolved events older than `days` days. Returns count deleted."""
     with get_conn() as conn:

monitor.py

@@ -15,7 +15,7 @@ import statistics
 import time
 from datetime import datetime, timezone
 from typing import Dict, List, Optional
-from urllib.parse import urlparse
+from urllib.parse import urljoin, urlparse
 
 import requests
 from urllib3.exceptions import InsecureRequestWarning
@@ -24,6 +24,7 @@ import db
 import cutsheet
 import devices
 import modem as modem_api
+import ticket_text
 
 
 logging.basicConfig(
@@ -388,6 +389,41 @@ class TicketClient:
     def __init__(self, cfg: dict):
         self.url = cfg.get('url', '')
         self.api_key = cfg.get('api_key', '')
+        # Tinker serves its other API scripts under api/ beside create_ticket_api.php
+        self.comment_url = cfg.get('comment_url') or (
+            urljoin(self.url, 'api/ticket_comment_api.php') if self.url else '')
+        self.comments_denied = False
+
+    def comment(self, ticket_id: str, text: str) -> bool:
+        """Add a Markdown comment to a ticket. Never raises; False means try again later."""
+        if not self.api_key or not self.comment_url or self.comments_denied:
+            return False
+        try:
+            resp = requests.post(
+                self.comment_url,
+                json={'ticket_id': str(ticket_id), 'comment_text': text, 'markdown_enabled': True},
+                headers={'Authorization': f'Bearer {self.api_key}'},
+                timeout=15,
+            )
+            if resp.status_code in (401, 403):
+                # Retrying every cycle can't fix a key without write access, so stop until restart
+                self.comments_denied = True
+                logger.error(
+                    f'Ticket API refused a comment on #{ticket_id} (HTTP {resp.status_code}). '
+                    f'The API key needs read_write scope; recovery notes are off until the monitor restarts.'
+                )
+                return False
+            if resp.status_code == 404:
+                logger.warning(f'Ticket #{ticket_id} no longer exists; skipping its recovery note')
+                return True
+            resp.raise_for_status()
+            if resp.json().get('success'):
+                logger.info(f'Commented on ticket #{ticket_id}')
+                return True
+            logger.warning(f'Unexpected comment API response for #{ticket_id}: {resp.text[:200]}')
+        except Exception as e:
+            logger.error(f'Ticket comment failed for #{ticket_id}: {e}')
+        return False
 
     def create(self, title: str, description: str, priority: str = '2') -> Optional[str]:
         if not self.api_key or not self.url:
@@ -893,10 +929,21 @@ def _now_utc() -> str:
     return datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M:%S UTC')
 
 
+def _utcnow() -> datetime:
+    """Naive UTC, to compare with database times."""
+    return datetime.now(timezone.utc).replace(tzinfo=None)
+
+
 # --------------------------------------------------------------------------
 # Monitor
 # --------------------------------------------------------------------------
 CLUSTER_NAME = 'proxmox-cluster'
+DEFAULT_PUBLIC_URL = 'https://gandalf.lotusguild.org'
+# Reports of something already over: the ticket is the whole story, no recovery note
+ONE_SHOT_EVENTS = ('modem_power_cycled', 'wan_unstable')
+# Ports a host or device was last seen on, kept so a ticket for a dead host
+# (whose link, and so its LLDP entry, is gone) can still say where it plugs in
+KNOWN_UPLINKS_STATE = 'known_uplinks'
 
 
 class NetworkMonitor:
@@ -950,6 +997,16 @@ class NetworkMonitor:
             db.ensure_cutsheet_schema()
         except Exception as e:
             logger.error(f'Could not create cutsheet tables: {e}')
+        try:
+            db.ensure_events_schema()
+        except Exception as e:
+            logger.error(f'Could not update the events table: {e}')
+
+        # Absolute links in tickets point here
+        self.public_url = ((self.cfg.get('gandalf') or {}).get('public_url')
+                           or DEFAULT_PUBLIC_URL).rstrip('/')
+        self._link_data: Optional[dict] = None
+        self._known_uplinks: Optional[dict] = None
 
         # Last good Prometheus result, shown (as unknown) during a Prometheus outage
         self._last_iface_states: Dict[str, Dict[str, bool]] = {}
@@ -964,6 +1021,170 @@ class NetworkMonitor:
     def _hostname(self, instance: str) -> str:
         return self._instance_map.get(instance, instance.split(':')[0])
 
+    # ------------------------------------------------------------------
+    # Tickets: context, and notes when an alert clears
+    # ------------------------------------------------------------------
+    def _base_url(self) -> str:
+        return getattr(self, 'public_url', DEFAULT_PUBLIC_URL)
+
+    def _open_ticket(
+        self, event_id: int, event_type: str, target: str, detail: str, *,
+        title: str, heading: str, facts: list, summary: str, checks: list, priority: str,
+        host: Optional[str] = None, port: Optional[tuple] = None,
+    ) -> None:
+        """Open a ticket with the alert's facts plus everything Gandalf knows around it.
+
+        host: a device whose switch ports to list. port: (switch, port_idx) the alert is about.
+        """
+        sections = self._ticket_sections(event_id, event_type, target, detail, host=host, port=port)
+        desc = ticket_text.description(heading, facts, summary, checks, sections)
+        tid = self.tickets.create(title, desc, priority=priority)
+        if tid:
+            db.set_ticket_id(event_id, tid)
+
+    def _ticket_sections(self, event_id: int, event_type: str, target: str, detail: str,
+                         host: Optional[str] = None, port: Optional[tuple] = None) -> list:
+        """Each section is optional: a failure loses that section, never the ticket."""
+        sections = []
+        for title, build in (
+            ('Where it is connected', lambda: self._where_lines(event_type, detail, host, port)),
+            ('History', lambda: ticket_text.history_lines(db.event_stats(event_type, target, detail), _utcnow())),
+            ('Other active alerts here', lambda: self._other_alert_lines(event_id, {target, host})),
+            ('Links', lambda: self._link_lines(host or (port[0] if port else None))),
+        ):
+            try:
+                sections.append((title, build()))
+            except Exception as e:
+                logger.error(f'Ticket section "{title}" for {target} failed: {e}', exc_info=True)
+        return sections
+
+    def _switches(self) -> dict:
+        return (getattr(self, '_link_data', None) or {}).get('unifi_switches') or {}
+
+    def _remember_uplinks(self, link_data: dict) -> None:
+        """Record where every LLDP-visible device is plugged in, for when it disappears."""
+        known = self._load_known_uplinks()
+        switches = link_data.get('unifi_switches') or {}
+        names = {(p.get('lldp') or {}).get('system_name')
+                 for sw in switches.values() for p in (sw.get('ports') or {}).values()}
+        now = _now_utc()
+        changed = False
+        for name in sorted(n for n in names if n):
+            ups = devices.uplinks_for_host(name, switches)
+            entry = {'seen': now, 'uplinks': [
+                {k: u[k] for k in ('switch', 'port', 'port_idx', 'host_interface')} for u in ups]}
+            if (known.get(name) or {}).get('uplinks') != entry['uplinks']:
+                changed = True
+            known[name] = entry
+        self._known_uplinks = known
+        # Write when wiring changes, and hourly so "last seen" stays roughly right
+        last_saved = getattr(self, '_known_uplinks_saved', 0.0)
+        if changed or time.time() - last_saved > 3600:
+            db.set_state(KNOWN_UPLINKS_STATE, known)
+            self._known_uplinks_saved = time.time()
+
+    def _load_known_uplinks(self) -> dict:
+        if getattr(self, '_known_uplinks', None) is None:
+            self._known_uplinks = {}
+            raw = db.get_state(KNOWN_UPLINKS_STATE)
+            if raw:
+                try:
+                    self._known_uplinks = json.loads(raw)
+                except ValueError:
+                    logger.error('Failed to parse known_uplinks JSON; starting over')
+        return self._known_uplinks
+
+    def _uplinks_for(self, name: str, switches: dict) -> tuple:
+        """(uplinks, None) if the device is linked now, else (last known uplinks, when seen)."""
+        current = devices.uplinks_for_host(name, switches)
+        if current:
+            return current, None
+        entry = self._load_known_uplinks().get(name) or {}
+        return entry.get('uplinks') or [], entry.get('seen')
+
+    def _where_lines(self, event_type: str, detail: str, host: Optional[str], port: Optional[tuple]) -> list:
+        base, switches = self._base_url(), self._switches()
+        lines = []
+        if host:
+            ups, seen = self._uplinks_for(host, switches)
+            if event_type == 'interface_down':
+                ups = [u for u in ups if u.get('host_interface') == detail] or ups
+            for u in ups:
+                live = (switches.get(u['switch'], {}).get('ports') or {}).get(u['port'])
+                text = ticket_text.port_text(u['switch'], u['port'], live or dict(u, up=False))
+                iface = f'`{u["host_interface"]}` → ' if u.get('host_interface') else ''
+                lines.append(f'- {iface}' + ticket_text.link(
+                    text, ticket_text.device_url(base, u['switch'], u.get('port_idx'))))
+                if seen:
+                    lines[-1] += f' _(last seen linked there {seen})_'
+            if not ups:
+                lines.append('- Gandalf has not seen this device on a switch port (no LLDP or client entry).')
+        if port:
+            switch, idx = port
+            live = next((dict(p, name=n) for n, p in (switches.get(switch, {}).get('ports') or {}).items()
+                        if str(p.get('port_idx')) == str(idx)), None)
+            if live:
+                lines.append('- ' + ticket_text.link(ticket_text.port_text(switch, live['name'], live),
+                                                     ticket_text.device_url(base, switch, idx)))
+        device = host or (port[0] if port else None)
+        if device:
+            data = db.cutsheet_load()
+            items = {i['name']: i for i in data['items']}
+            runs = cutsheet.runs_for(device, data['cables'], items, switches,
+                                     port=str(port[1]) if port else None)
+            for r in runs:
+                label = ticket_text.link(f'**{r["label"]}**', ticket_text.cutsheet_url(base, r['cable_id']))
+                lines.append(f'- Cable {label}: {r["path"]}. Cutsheet check: '
+                             f'{cutsheet.VERDICTS.get(r["verdict"], r["verdict"]).lower()}.')
+            if not runs:
+                lines.append('- ' + ticket_text.link('Not on the cutsheet yet', f'{base}/cutsheet' if base else ''))
+        return lines
+
+    def _other_alert_lines(self, event_id: int, names: set) -> list:
+        lines = []
+        for name in sorted(n for n in names if n):
+            for e in db.get_events_for_target(name, limit=25):
+                if e.get('resolved_at') or e.get('id') == event_id:
+                    continue
+                what = e['event_type'].replace('_', ' ')
+                where = f'{e["target_name"]} {e.get("target_detail") or ""}'.strip()
+                ticket = f' (ticket #{e["ticket_id"]})' if e.get('ticket_id') else ''
+                lines.append(f'- {what} on {where} since {e.get("first_seen", "?")}{ticket}')
+        return lines[:10]
+
+    def _link_lines(self, device: Optional[str]) -> list:
+        base = self._base_url()
+        if not base:
+            return []
+        lines = []
+        if device:
+            lines.append(f'- {ticket_text.link(f"{device} in Gandalf", ticket_text.device_url(base, device))}')
+        lines.append(f'- {ticket_text.link("Gandalf dashboard", base + "/")}')
+        return lines
+
+    def _event_url(self, event: dict) -> str:
+        base = self._base_url()
+        if not base:
+            return ''
+        if event['event_type'] == 'cable_mismatch':
+            cable_id = str(event.get('target_detail') or '').replace('cable ', '')
+            return ticket_text.cutsheet_url(base, cable_id)
+        if event['event_type'] in ('cluster_network_issue', 'monitor_source_unavailable'):
+            return base + '/'
+        return ticket_text.device_url(base, event['target_name'])
+
+    def _post_recovery_notes(self) -> None:
+        """Comment on the tickets of alerts that have cleared. Failures retry next cycle (for 24 h)."""
+        if self.tickets.comments_denied:
+            return
+        now = _utcnow()
+        for event in db.get_pending_recovery_notes(hours=24, exclude_types=ONE_SHOT_EVENTS):
+            text = ticket_text.recovery_comment(event, now, self._event_url(event))
+            if self.tickets.comment(event['ticket_id'], text):
+                db.mark_recovery_notified(event['id'])
+            elif self.tickets.comments_denied:
+                break
+
     # ------------------------------------------------------------------
     # Interface monitoring (Prometheus)
     # ------------------------------------------------------------------
@@ -1041,47 +1262,54 @@ class NetworkMonitor:
                 f'{", ".join(hosts_with_regression)}',
             )
             if not sup and (is_new or not ticket_id):
-                title = (
-                    f'[{self.cluster_name}][auto][production][issue][network][cluster-wide] '
-                    f'Multiple hosts reporting interface failures'
-                )
-                desc = (
-                    f'Cluster Network Alert\n{"=" * 40}\n\n'
-                    f'Affected hosts: {", ".join(hosts_with_regression)}\n'
-                    f'Detected: {_now_utc()}\n\n'
-                    f'{len(hosts_with_regression)} Proxmox hosts simultaneously reported '
-                    f'interface regressions (link-down on interfaces previously known UP) '
-                    f'or stopped reporting metrics.\n'
-                    f'This likely indicates a switch or upstream network failure.\n\n'
-                    f'Please check the core and management switches immediately.'
-                )
-                tid = self.tickets.create(title, desc, priority='1')
-                if tid:
-                    db.set_ticket_id(event_id, tid)
+                self._ticket_cluster(event_id, hosts_with_regression)
         else:
             db.resolve_event('cluster_network_issue', self.cluster_name, '')
 
+    def _ticket_cluster(self, event_id: int, hosts: List[str]) -> None:
+        switches = self._switches()
+        # The switches these hosts hang off are the likeliest shared cause
+        shared = sorted({u['switch'] for h in hosts for u in self._uplinks_for(h, switches)[0]})
+        self._open_ticket(
+            event_id, 'cluster_network_issue', self.cluster_name, '',
+            title=(f'[{self.cluster_name}][auto][production][issue][network][cluster-wide] '
+                   f'Multiple hosts reporting interface failures'),
+            heading='Cluster Network Alert',
+            facts=[('Affected hosts', ', '.join(hosts)),
+                   ('Switches they connect through', ', '.join(shared)),
+                   ('Detected', _now_utc())],
+            summary=(f'{len(hosts)} Proxmox hosts reported interface regressions (link-down on '
+                     f'interfaces previously known up) or stopped reporting metrics at the same time. '
+                     f'This usually means a switch or upstream network failure rather than the hosts.'),
+            checks=['Check the core and management switches (power, uplinks, recent firmware updates).',
+                    'Check whether the hosts share a PDU or circuit that lost power.',
+                    'Look at the per-host tickets and the Gandalf dashboard for which links went down first.'],
+            priority='1',
+        )
+
     def _ticket_metrics_missing(
         self, event_id: int, host: str, instance: str, consec: int
     ) -> None:
-        title = (
-            f'[{host}][auto][production][issue][network][single-node] '
-            f'No metrics from host'
+        self._open_ticket(
+            event_id, 'host_metrics_missing', host, instance,
+            title=f'[{host}][auto][production][issue][network][single-node] No metrics from host',
+            heading='Host Metrics Alert',
+            facts=[('Host', host), ('Prometheus instance', instance), ('Detected', _now_utc()),
+                   ('Consecutive check failures', str(consec)),
+                   ('Down for at least', ticket_text.duration(consec * self._poll_seconds()))],
+            summary=(f'Prometheus is working but has no node_network_up series for {host}. '
+                     f'Either the host is down or node_exporter has stopped on it. '
+                     f'Gandalf waits {self.host_fail_thresh} checks before opening this, so a normal '
+                     f'reboot should not cause it.'),
+            checks=[f'Is {host} powered on and reachable (Proxmox UI, ping, console)?',
+                    'If it is up, check `systemctl status prometheus-node-exporter` on the host.',
+                    'If it is down, check its switch port below: a dark port with no PoE or link '
+                    'points to power or cabling.'],
+            priority='2', host=host,
         )
-        desc = (
-            f'Host Metrics Alert\n{"=" * 40}\n\n'
-            f'Host: {host}\n'
-            f'Prometheus instance: {instance}\n'
-            f'Detected: {_now_utc()}\n'
-            f'Consecutive check failures: {consec}\n\n'
-            f'Prometheus is working but has no node_network_up series for {host}.\n'
-            f'Either the host is down or node_exporter has stopped on it.\n\n'
-            f'Please check the host power and network connectivity, then '
-            f'prometheus-node-exporter on the host.'
-        )
-        tid = self.tickets.create(title, desc, priority='2')
-        if tid:
-            db.set_ticket_id(event_id, tid)
+
+    def _poll_seconds(self) -> int:
+        return int(getattr(self, 'poll_interval', 120) or 120)
 
     def _process_prometheus_outage(self, suppressions: list) -> None:
         """Interface and host alerts can't fire without Prometheus, so say so."""
@@ -1092,44 +1320,38 @@ class NetworkMonitor:
             f'Prometheus returned no interface data; host and interface alerting is paused ({_now_utc()})',
         )
         if not sup and consec >= self.fail_thresh and not ticket_id:
-            title = (
-                '[prometheus][auto][production][issue][network][cluster-wide] '
-                'Gandalf has no Prometheus data'
+            self._open_ticket(
+                event_id, 'monitor_source_unavailable', 'prometheus', '',
+                title='[prometheus][auto][production][issue][network][cluster-wide] Gandalf has no Prometheus data',
+                heading='Monitoring Source Alert',
+                facts=[('Prometheus URL', self.prom.url), ('Detected', _now_utc()),
+                       ('Consecutive check failures', str(consec))],
+                summary=('Gandalf could not get node_network_up data from Prometheus. While this lasts, '
+                         'host and interface failures will **not** be detected. UniFi, ping and modem '
+                         'checks keep running.'),
+                checks=['Is the Prometheus service running on its host?',
+                        'Open Prometheus → Status → Targets and check the node exporters are being scraped.',
+                        'Check disk space on the Prometheus host (a full disk stops ingestion).'],
+                priority='2',
             )
-            desc = (
-                f'Monitoring Source Alert\n{"=" * 40}\n\n'
-                f'Prometheus URL: {self.prom.url}\n'
-                f'Detected: {_now_utc()}\n'
-                f'Consecutive check failures: {consec}\n\n'
-                f'Gandalf could not get node_network_up data from Prometheus.\n'
-                f'While this lasts, host and interface failures will NOT be detected.\n\n'
-                f'Please check the Prometheus service and its scrape targets.'
-            )
-            tid = self.tickets.create(title, desc, priority='2')
-            if tid:
-                db.set_ticket_id(event_id, tid)
 
     def _ticket_interface(
         self, event_id: int, host: str, iface: str, consec: int
     ) -> None:
-        title = (
-            f'[{host}][auto][production][issue][network][single-node] '
-            f'Interface {iface} link-down'
+        self._open_ticket(
+            event_id, 'interface_down', host, iface,
+            title=f'[{host}][auto][production][issue][network][single-node] Interface {iface} link-down',
+            heading='Network Interface Alert',
+            facts=[('Host', host), ('Interface', f'`{iface}`'), ('Detected', _now_utc()),
+                   ('Consecutive check failures', str(consec))],
+            summary=(f'Interface {iface} on {host} was up and is now reporting link-down via '
+                     f'Prometheus node_exporter. {host} itself is still reporting, so it may be '
+                     f'reachable over its other interfaces.'),
+            checks=['Check the switch port below: is it down on the switch side too?',
+                    'Reseat or swap the cable or DAC; for an optic, check the module for a fault.',
+                    f'On {host}, check `ip link show {iface}` and `dmesg` for NIC or driver errors.'],
+            priority='2', host=host,
         )
-        desc = (
-            f'Network Interface Alert\n{"=" * 40}\n\n'
-            f'Host: {host}\n'
-            f'Interface: {iface}\n'
-            f'Detected: {_now_utc()}\n'
-            f'Consecutive check failures: {consec}\n\n'
-            f'Interface {iface} on {host} is reporting link-down state via '
-            f'Prometheus node_exporter.\n\n'
-            f'Note: {host} may still be reachable via its other network interface.\n'
-            f'Please inspect the cable/SFP/switch port for {host}/{iface}.'
-        )
-        tid = self.tickets.create(title, desc, priority='2')
-        if tid:
-            db.set_ticket_id(event_id, tid)
 
     # ------------------------------------------------------------------
     # UniFi device monitoring
@@ -1155,23 +1377,20 @@ class NetworkMonitor:
 
     def _ticket_unifi(self, event_id: int, device: dict) -> None:
         name = device['name']
-        title = (
-            f'[{name}][auto][production][issue][network][single-node] '
-            f'UniFi device offline'
+        self._open_ticket(
+            event_id, 'unifi_device_offline', name, device.get('type', ''),
+            title=f'[{name}][auto][production][issue][network][single-node] UniFi device offline',
+            heading='UniFi Device Alert',
+            facts=[('Device', name), ('Type', device.get('type', 'unknown')), ('Model', device.get('model', '')),
+                   ('Last known IP', device.get('ip', 'unknown')), ('Detected', _now_utc())],
+            summary=(f'The UniFi controller reports {name} as offline. Gandalf waits '
+                     f'{self.unifi_fail_thresh} checks first, so a firmware-update reboot should not cause this.'),
+            checks=['Check the device for power (status LED) and, if it is PoE powered, the PoE '
+                    'budget on the port below.',
+                    'Check the uplink cable and the switch port it plugs into.',
+                    'In the UniFi controller, check whether the device is stuck adopting or upgrading.'],
+            priority='2', host=name,
         )
-        desc = (
-            f'UniFi Device Alert\n{"=" * 40}\n\n'
-            f'Device: {name}\n'
-            f'Type: {device.get("type","unknown")}\n'
-            f'Model: {device.get("model","")}\n'
-            f'Last Known IP: {device.get("ip","unknown")}\n'
-            f'Detected: {_now_utc()}\n\n'
-            f'The UniFi device {name} is offline per the UniFi controller.\n'
-            f'Please check power and cable connectivity.'
-        )
-        tid = self.tickets.create(title, desc, priority='2')
-        if tid:
-            db.set_ticket_id(event_id, tid)
 
     # ------------------------------------------------------------------
     # Cutsheet: documented cabling vs. what discovery sees
@@ -1206,24 +1425,20 @@ class NetworkMonitor:
                 db.resolve_event('cable_mismatch', target_name, target_detail)
 
     def _ticket_cable_mismatch(self, event_id: int, p: dict, hours: float) -> None:
-        title = (
-            f'[{p["switch"]}][auto][production][issue][network][single-node] '
-            f'Cable {p["label"]} does not match the cutsheet'
+        url = ticket_text.cutsheet_url(self._base_url(), p['cable_id'])
+        self._open_ticket(
+            event_id, 'cable_mismatch', p['switch'], p['key'],
+            title=(f'[{p["switch"]}][auto][production][issue][network][single-node] '
+                   f'Cable {p["label"]} does not match the cutsheet'),
+            heading='Cutsheet Mismatch',
+            facts=[('Cable', ticket_text.link(p['label'], url)), ('Switch port', f'{p["switch"]} port {p["port"]}'),
+                   ('Problem', p['detail']), ('Documented path', p['path']), ('Detected', _now_utc())],
+            summary=f'This has not matched the cutsheet for at least {hours:g} hours.',
+            checks=['If the cable was moved on purpose, update its record in the cutsheet.',
+                    'Otherwise, follow the label and check it is plugged into the documented port '
+                    '(Trace mode on the cutsheet page can find which port a cord is in).'],
+            priority='4', port=(p['switch'], p['port']),
         )
-        desc = (
-            f'Cutsheet Mismatch\n{"=" * 40}\n\n'
-            f'Cable: {p["label"]}\n'
-            f'Switch port: {p["switch"]} port {p["port"]}\n'
-            f'Problem: {p["detail"]}\n'
-            f'Documented path: {p["path"]}\n'
-            f'Detected: {_now_utc()}\n\n'
-            f'This has not matched for at least {hours:g} hours.\n\n'
-            f'Either the cable was moved (update the cutsheet in Gandalf: /cutsheet#cable-{p["cable_id"]}) '
-            f'or it is plugged into the wrong place.'
-        )
-        tid = self.tickets.create(title, desc, priority='4')
-        if tid:
-            db.set_ticket_id(event_id, tid)
 
     # ------------------------------------------------------------------
     # Ping-only hosts (no node_exporter)
@@ -1253,24 +1468,20 @@ class NetworkMonitor:
     def _ticket_unreachable(
         self, event_id: int, name: str, ip: str, consec: int
     ) -> None:
-        title = (
-            f'[{name}][auto][production][issue][network][single-node] '
-            f'Host unreachable'
+        self._open_ticket(
+            event_id, 'host_unreachable', name, ip,
+            title=f'[{name}][auto][production][issue][network][single-node] Host unreachable',
+            heading='Host Reachability Alert',
+            facts=[('Host', name), ('IP', ip), ('Detected', _now_utc()),
+                   ('Consecutive check failures', str(consec)),
+                   ('Down for at least', ticket_text.duration(consec * self._poll_seconds()))],
+            summary=(f'{name} ({ip}) is not answering ping from the Pulse worker Gandalf uses. '
+                     f'It has no node_exporter, so ping is the only check Gandalf has for it.'),
+            checks=[f'Is {name} powered on? Check its console or management interface.',
+                    'Check its switch port below for link and PoE.',
+                    'If the port is up, check the IP and firewall on the device itself.'],
+            priority='2', host=name,
         )
-        desc = (
-            f'Host Reachability Alert\n{"=" * 40}\n\n'
-            f'Host: {name}\n'
-            f'IP: {ip}\n'
-            f'Detected: {_now_utc()}\n'
-            f'Consecutive check failures: {consec}\n\n'
-            f'Host {name} ({ip}) is not responding to ping from the Gandalf monitor.\n'
-            f'This host does not have a Prometheus node_exporter, so interface-level '
-            f'detail is unavailable.\n\n'
-            f'Please check the host power, management interface, and network connectivity.'
-        )
-        tid = self.tickets.create(title, desc, priority='2')
-        if tid:
-            db.set_ticket_id(event_id, tid)
 
     # ------------------------------------------------------------------
     # WAN modem monitoring
@@ -1291,17 +1502,14 @@ class NetworkMonitor:
             event_type, severity, 'modem', name, detail, f'{description} ({_now_utc()})',
         )
         if not sup and consec >= self.modem_fail_thresh and not ticket_id:
-            title = f'[{name}][auto][production][issue][network][cluster-wide] {heading}'
-            desc = (
-                f'WAN Modem Alert\n{"=" * 40}\n\n'
-                f'Modem: {name} ({self.modem.base_url})\n'
-                f'Detected: {_now_utc()}\n'
-                f'Consecutive check failures: {consec}\n\n'
-                f'{description}\n\n{advice}'
+            self._open_ticket(
+                event_id, event_type, name, detail,
+                title=f'[{name}][auto][production][issue][network][cluster-wide] {heading}',
+                heading='WAN Modem Alert',
+                facts=[('Modem', f'{name} ({self.modem.base_url})'), ('Detected', _now_utc()),
+                       ('Consecutive check failures', str(consec))],
+                summary=description, checks=[advice], priority=priority,
             )
-            tid = self.tickets.create(title, desc, priority=priority)
-            if tid:
-                db.set_ticket_id(event_id, tid)
 
     def _process_modem_failure(self, suppressions: list, error: Exception) -> None:
         """Handle a failed poll. Link and optical events stay as they are: we can't see them."""
@@ -1443,7 +1651,8 @@ class NetworkMonitor:
         if not db.check_suppressed(suppressions, 'host', name) and not ticket_id:
             tid = self.tickets.create(
                 f'[{name}][auto][production][issue][network][cluster-wide] {heading}',
-                f'WAN Event\n{"=" * 40}\n\nModem: {name}\nReported: {_now_utc()}\n\n{description}\n\n{advice}',
+                ticket_text.description('WAN Event', [('Modem', name), ('Reported', _now_utc())],
+                                        description, [advice], recovers=False),
                 priority=priority,
             )
             if tid:
@@ -1645,6 +1854,12 @@ class NetworkMonitor:
                     db.set_state('link_stats', link_data)
                 except Exception as e:
                     logger.error(f'Link stats collection failed: {e}', exc_info=True)
+                if link_data is not None:
+                    self._link_data = link_data
+                    try:
+                        self._remember_uplinks(link_data)
+                    except Exception as e:
+                        logger.error(f'Recording known uplinks failed: {e}', exc_info=True)
 
                 # 5. Process alerts using already-fetched data
                 suppressions = db.get_active_suppressions()
@@ -1673,6 +1888,12 @@ class NetworkMonitor:
                     except Exception as e:
                         logger.error(f'Modem collection failed: {e}', exc_info=True)
 
+                # Note on the tickets of anything that cleared this cycle (or failed to post before)
+                try:
+                    self._post_recovery_notes()
+                except Exception as e:
+                    logger.error(f'Posting recovery notes failed: {e}', exc_info=True)
+
                 # Housekeeping: deactivate expired suppressions and purge old resolved events
                 db.cleanup_expired_suppressions()
                 db.purge_old_resolved_events(days=90)

schema.sql

@@ -21,8 +21,9 @@ CREATE TABLE IF NOT EXISTS network_events (
   resolved_at         TIMESTAMP    NULL,
   consecutive_failures INT          NOT NULL DEFAULT 1,
   ticket_id           VARCHAR(20)  NULL,
+  recovery_notified   TINYINT(1)   NOT NULL DEFAULT 0, -- recovery comment posted on ticket_id
 
-  INDEX idx_active       (resolved_at),
+  INDEX idx_active      (resolved_at),
   INDEX idx_target       (target_name, target_detail),
   INDEX idx_type         (event_type),
   -- Compound index for hot upsert_event lookup: event_type+target+resolved_at

tests/test_monitor.py

@@ -3,6 +3,7 @@ import json
 import os
 import sys
 import time
+from datetime import timedelta
 from unittest.mock import MagicMock, patch
 
 import pytest
@@ -24,6 +25,11 @@ def db():
         mock_db.upsert_event.return_value = (1, False, 2, None)
         mock_db.get_modem_rx_values.return_value = []
         mock_db.get_state.return_value = None
+        mock_db.event_stats.return_value = {'last_7d': 0, 'last_30d': 0,
+                                            'last_first_seen': None, 'last_resolved_at': None}
+        mock_db.get_events_for_target.return_value = []
+        mock_db.cutsheet_load.return_value = {'items': [], 'cables': [], 'port_labels': []}
+        mock_db.get_pending_recovery_notes.return_value = []
         yield mock_db
 
 
@@ -560,3 +566,177 @@ class TestCutsheetMismatch:
         assert not db.upsert_event.called
         resolved = {c.args for c in db.resolve_event.call_args_list}
         assert ('cable_mismatch', 'sw', 'cable 7') in resolved and ('cable_mismatch', 'other', 'cable 9') in resolved
+
+
+class TestTicketContext:
+    SWITCHES = {'agg': {'ports': {
+        'Large1': {'port_idx': 7, 'up': True, 'speed_mbps': 10000,
+                   'lldp': {'system_name': 'large1', 'port_id': 'enp1s0f0'},
+                   'sfp': {'found': True, 'kind': 'dac', 'part': 'DAC-10G'}},
+        'Port 3': {'port_idx': 3, 'up': False, 'speed_mbps': 0},
+    }}}
+
+    def _mon(self, mon, switches=None):
+        mon.public_url = 'https://gandalf.test'
+        mon.poll_interval = 120
+        mon._link_data = {'unifi_switches': switches if switches is not None else self.SWITCHES}
+        return mon
+
+    def _desc(self, mon):
+        return mon.tickets.create.call_args.args[1]
+
+    def test_interface_ticket_names_the_switch_port_and_links(self, mon, db):
+        self._mon(mon)._ticket_interface(1, 'large1', 'enp1s0f0', 2)
+        desc = self._desc(mon)
+        assert '### Where it is connected' in desc
+        assert '`enp1s0f0` → [agg port 7 "Large1": up at 10 Gbps, module: DAC DAC-10G]' in desc
+        assert '(https://gandalf.test/device/agg#port-7)' in desc
+        assert 'https://gandalf.test/device/large1' in desc
+        assert 'Opened automatically by Gandalf' in desc
+
+    def test_dead_host_uses_the_last_known_port(self, mon, db):
+        mon._known_uplinks = {'large1': {'seen': '2026-11-10 10:00:00 UTC', 'uplinks': [
+            {'switch': 'agg', 'port': 'Port 3', 'port_idx': 3, 'host_interface': 'eno1'}]}}
+        self._mon(mon)._ticket_metrics_missing(1, 'micro1', 'x:9100', 5)
+        assert 'not seen this device on a switch port' in self._desc(mon)
+        mon.tickets.create.reset_mock()
+        unplugged = {'agg': {'ports': {'Port 3': {'port_idx': 3, 'up': False}}}}
+        self._mon(mon, unplugged)._ticket_unreachable(1, 'large1', '10.0.0.9', 5)
+        desc = self._desc(mon)
+        assert 'agg port 3: down' in desc
+        assert 'last seen linked there 2026-11-10 10:00:00 UTC' in desc
+        assert 'Down for at least' in desc and '10 min' in desc
+
+    def test_cutsheet_runs_and_history(self, mon, db):
+        db.cutsheet_load.return_value = {'items': [], 'port_labels': [], 'cables': [
+            {'id': 4, 'label': 'C04', 'a_device': 'agg', 'a_port': '7', 'a_side': '',
+             'b_device': 'large1', 'b_port': 'enp1s0f0', 'b_side': '', 'status': 'installed'}]}
+        db.event_stats.return_value = {'last_7d': 3, 'last_30d': 5,
+                                       'last_first_seen': monitor._utcnow(), 'last_resolved_at': monitor._utcnow()}
+        self._mon(mon)._ticket_interface(1, 'large1', 'enp1s0f0', 2)
+        desc = self._desc(mon)
+        assert '[**C04**](https://gandalf.test/cutsheet#cable-4): agg port 7 → large1 enp1s0f0' in desc
+        assert 'Happened 3 times in the last 7 days and 5 in the last 30' in desc
+        assert 'This keeps coming back' in desc
+
+    def test_other_open_alerts_are_listed(self, mon, db):
+        db.get_events_for_target.return_value = [
+            {'id': 1, 'event_type': 'interface_down', 'target_name': 'large1', 'target_detail': 'eno1'},
+            {'id': 2, 'event_type': 'interface_down', 'target_name': 'large1', 'target_detail': 'eno2',
+             'first_seen': 'T', 'ticket_id': '55'},
+            {'id': 3, 'event_type': 'host_unreachable', 'target_name': 'large1', 'resolved_at': 'R'},
+        ]
+        self._mon(mon)._ticket_interface(1, 'large1', 'eno1', 2)
+        desc = self._desc(mon)
+        assert '- interface down on large1 eno2 since T (ticket #55)' in desc
+        assert 'eno1 since' not in desc and 'host unreachable' not in desc
+
+    def test_a_failing_section_still_opens_the_ticket(self, mon, db):
+        db.event_stats.side_effect = RuntimeError('db gone')
+        self._mon(mon)._ticket_interface(1, 'large1', 'enp1s0f0', 2)
+        desc = self._desc(mon)
+        assert '### History' not in desc and '### Where it is connected' in desc
+        db.set_ticket_id.assert_called_with(1, '123')
+
+    def test_cable_mismatch_shows_the_switch_port(self, mon, db):
+        self._mon(mon)._ticket_cable_mismatch(9, {
+            'switch': 'agg', 'port': '7', 'cable_id': 4, 'label': 'C04', 'key': 'cable 4',
+            'detail': 'A different device is connected', 'path': 'agg port 7 → tv'}, 24)
+        desc = self._desc(mon)
+        assert '[C04](https://gandalf.test/cutsheet#cable-4)' in desc
+        assert '[agg port 7 "Large1": up at 10 Gbps' in desc
+
+    def test_titles_keep_the_tag_format(self, mon, db):
+        mon.unifi_fail_thresh = 3
+        self._mon(mon)._ticket_unifi(1, {'name': 'ap1', 'type': 'uap', 'model': 'U6', 'ip': '1.2.3.4'})
+        assert mon.tickets.create.call_args.args[0] == \
+            '[ap1][auto][production][issue][network][single-node] UniFi device offline'
+
+    def test_remember_uplinks_saves_only_on_change(self, mon, db):
+        self._mon(mon)
+        mon._known_uplinks = None
+        mon._remember_uplinks({'unifi_switches': self.SWITCHES})
+        saved = db.set_state.call_args.args
+        assert saved[0] == 'known_uplinks'
+        assert saved[1]['large1']['uplinks'] == [
+            {'switch': 'agg', 'port': 'Large1', 'port_idx': 7, 'host_interface': 'enp1s0f0'}]
+        db.set_state.reset_mock()
+        mon._remember_uplinks({'unifi_switches': self.SWITCHES})
+        assert not db.set_state.called
+
+
+class TestRecoveryNotes:
+    def _event(self, **kw):
+        now = monitor._utcnow()
+        e = {'id': 8, 'event_type': 'host_metrics_missing', 'target_name': 'micro1', 'target_detail': 'x:9100',
+             'ticket_id': '993', 'first_seen': now - timedelta(minutes=12), 'resolved_at': now,
+             'consecutive_failures': 6}
+        e.update(kw)
+        return e
+
+    def test_cleared_alert_comments_and_marks(self, mon, db):
+        mon.public_url = 'https://gandalf.test'
+        mon.tickets.comments_denied = False
+        mon.tickets.comment.return_value = True
+        db.get_pending_recovery_notes.return_value = [self._event()]
+        mon._post_recovery_notes()
+        ticket_id, text = mon.tickets.comment.call_args.args
+        assert ticket_id == '993'
+        assert 'Cleared' in text and '12 min' in text and 'Failed checks:** 6' in text
+        assert 'https://gandalf.test/device/micro1' in text
+        db.mark_recovery_notified.assert_called_once_with(8)
+        assert db.get_pending_recovery_notes.call_args.kwargs['exclude_types'] == monitor.ONE_SHOT_EVENTS
+
+    def test_failed_comment_is_left_for_next_cycle(self, mon, db):
+        mon.tickets.comments_denied = False
+        mon.tickets.comment.return_value = False
+        db.get_pending_recovery_notes.return_value = [self._event(), self._event(id=9)]
+        mon._post_recovery_notes()
+        assert not db.mark_recovery_notified.called
+        assert mon.tickets.comment.call_count == 2
+
+    def test_denied_key_stops_trying(self, mon, db):
+        mon.tickets.comments_denied = True
+        mon._post_recovery_notes()
+        assert not db.get_pending_recovery_notes.called
+
+    def test_cable_mismatch_links_the_cutsheet(self, mon, db):
+        mon.public_url = 'https://gandalf.test'
+        url = mon._event_url({'event_type': 'cable_mismatch', 'target_name': 'sw', 'target_detail': 'cable 7'})
+        assert url == 'https://gandalf.test/cutsheet#cable-7'
+
+
+class TestTicketClientComment:
+    def _client(self):
+        return monitor.TicketClient({'url': 'http://t.test/create_ticket_api.php', 'api_key': 'k'})
+
+    def test_comment_endpoint_sits_beside_create(self):
+        assert self._client().comment_url == 'http://t.test/api/ticket_comment_api.php'
+        sub = monitor.TicketClient({'url': 'http://t.test/tinker/create_ticket_api.php', 'api_key': 'k'})
+        assert sub.comment_url == 'http://t.test/tinker/api/ticket_comment_api.php'
+
+    def test_payload(self):
+        c = self._client()
+        resp = MagicMock(status_code=200)
+        resp.json.return_value = {'success': True, 'comment_id': 1}
+        with patch.object(monitor.requests, 'post', return_value=resp) as post:
+            assert c.comment('42', 'hi') is True
+        assert post.call_args.kwargs['json'] == {'ticket_id': '42', 'comment_text': 'hi', 'markdown_enabled': True}
+        assert post.call_args.kwargs['headers'] == {'Authorization': 'Bearer k'}
+
+    def test_forbidden_disables_comments(self):
+        c = self._client()
+        with patch.object(monitor.requests, 'post', return_value=MagicMock(status_code=403)) as post:
+            assert c.comment('42', 'hi') is False
+            assert c.comment('43', 'hi') is False
+        assert post.call_count == 1 and c.comments_denied
+
+    def test_missing_ticket_counts_as_done(self):
+        with patch.object(monitor.requests, 'post', return_value=MagicMock(status_code=404)):
+            assert self._client().comment('42', 'hi') is True
+
+    def test_network_error_is_retryable(self):
+        c = self._client()
+        with patch.object(monitor.requests, 'post', side_effect=OSError('down')):
+            assert c.comment('42', 'hi') is False
+        assert not c.comments_denied

tests/test_ticket_text.py

@@ -0,0 +1,64 @@
+"""Tests for ticket_text.py: the Markdown Gandalf puts in tickets."""
+import os
+import sys
+from datetime import datetime, timedelta
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(__file__)))
+
+import ticket_text as tt  # noqa: E402
+
+NOW = datetime(2026, 11, 10, 18, 0)
+
+
+def test_duration():
+    assert tt.duration(42) == '42 s'
+    assert tt.duration(600) == '10 min'
+    assert tt.duration(3600) == '1 h'
+    assert tt.duration(3 * 3600 + 120) == '3 h 2 min'
+    assert tt.duration(3 * 86400 + 7200) == '3 d 2 h'
+
+
+def test_speed():
+    assert tt.speed(100) == '100 Mbps'
+    assert tt.speed(2500) == '2.5 Gbps'
+    assert tt.speed(10000) == '10 Gbps'
+    assert tt.speed(0) == ''
+
+
+def test_device_url_escapes_names():
+    assert tt.device_url('https://g', 'USW Flex Mini', 3) == 'https://g/device/USW%20Flex%20Mini#port-3'
+    assert tt.device_url('', 'x') == ''
+
+
+def test_port_text():
+    assert tt.port_text('sw', 'Port 4', {'port_idx': 4, 'up': True, 'speed_mbps': 1000, 'poe_power': 4.21}) == \
+        'sw port 4: up at 1 Gbps, PoE 4.2 W'
+    assert tt.port_text('sw', 'Cam', {'port_idx': 9, 'up': False,
+                                      'sfp': {'found': True, 'kind': 'optic', 'part': 'SR', 'rx_fault': True}}) == \
+        'sw port 9 "Cam": down, module: optic SR, **module reports a fault**'
+
+
+def test_description_skips_empty_facts_and_sections():
+    text = tt.description('Alert', [('Host', 'a'), ('Blank', '')], 'Summary.', ['One', 'Two'],
+                          [('Empty', []), ('Links', ['- x'])])
+    assert text.startswith('## Alert\n\n- **Host:** a\n\nSummary.')
+    assert 'Blank' not in text and '### Empty' not in text
+    assert '1. One\n2. Two' in text and '### Links\n- x' in text
+
+
+def test_history_lines():
+    assert tt.history_lines({'last_7d': 0, 'last_30d': 0}, NOW) == ['- First time in the last 30 days.']
+    lines = tt.history_lines({'last_7d': 1, 'last_30d': 2, 'last_first_seen': NOW - timedelta(hours=26),
+                              'last_resolved_at': NOW - timedelta(hours=25, minutes=55)}, NOW)
+    assert lines[0].startswith('- Happened 1 time in the last 7 days and 2')
+    assert lines[1] == '- Last cleared 2026-11-09 16:05 UTC (25 h 55 min ago), after 5 min.'
+
+
+def test_recovery_comment():
+    text = tt.recovery_comment({'event_type': 'interface_down', 'target_name': 'large1', 'target_detail': 'eno1',
+                                'first_seen': NOW - timedelta(minutes=7), 'resolved_at': NOW,
+                                'consecutive_failures': 3}, NOW, 'https://g/device/large1')
+    assert text.startswith('**✅ Cleared**: interface down on large1 eno1.')
+    assert '- **Lasted:** 7 min (first seen 2026-11-10 17:53 UTC)' in text
+    assert '- **Gandalf:** https://g/device/large1' in text
+    assert 'leaves this ticket open' in text

ticket_text.py

@@ -0,0 +1,145 @@
+"""Markdown for the tickets Gandalf opens in Tinker Tickets, and the notes it adds to them.
+
+Pure formatting only: monitor.py gathers the facts and passes them in.
+"""
+from datetime import datetime, timezone
+from typing import List, Optional, Sequence, Tuple
+from urllib.parse import quote
+
+Section = Tuple[str, List[str]]
+
+
+def duration(seconds) -> str:
+    seconds = max(0, int(seconds))
+    if seconds < 60:
+        return f'{seconds} s'
+    minutes = seconds // 60
+    if minutes < 60:
+        return f'{minutes} min'
+    hours, minutes = divmod(minutes, 60)
+    if hours < 48:
+        return f'{hours} h {minutes} min' if minutes else f'{hours} h'
+    days, hours = divmod(hours, 24)
+    return f'{days} d {hours} h' if hours else f'{days} d'
+
+
+def utc(dt: Optional[datetime]) -> str:
+    """Database times are naive UTC."""
+    if dt is None:
+        return 'unknown'
+    if dt.tzinfo is not None:
+        dt = dt.astimezone(timezone.utc).replace(tzinfo=None)
+    return dt.strftime('%Y-%m-%d %H:%M UTC')
+
+
+def speed(mbps) -> str:
+    if not mbps:
+        return ''
+    mbps = int(mbps)
+    if mbps >= 1000:
+        gbps = mbps / 1000
+        return f'{gbps:g} Gbps'
+    return f'{mbps} Mbps'
+
+
+def device_url(base: str, name: str, port_idx=None) -> str:
+    if not base or not name:
+        return ''
+    url = f'{base}/device/{quote(name, safe="")}'
+    return f'{url}#port-{port_idx}' if port_idx not in (None, '') else url
+
+
+def cutsheet_url(base: str, cable_id) -> str:
+    return f'{base}/cutsheet#cable-{cable_id}' if base else ''
+
+
+def link(text: str, url: str) -> str:
+    return f'[{text}]({url})' if url else text
+
+
+def port_text(switch: str, port_name: str, port: dict) -> str:
+    """'8SFP-AGG port 7 "Large1": up at 10 Gbps, DAC'"""
+    idx = port.get('port_idx')
+    text = f'{switch} port {idx}' if idx is not None else f'{switch} {port_name}'
+    if port_name and idx is not None and port_name != f'Port {idx}':
+        text += f' "{port_name}"'
+    state = []
+    if port.get('up'):
+        rate = speed(port.get('speed_mbps'))
+        state.append(f'up at {rate}' if rate else 'up')
+    else:
+        state.append('down')
+    if port.get('poe_power'):
+        state.append(f'PoE {float(port["poe_power"]):.1f} W')
+    sfp = port.get('sfp') or {}
+    if sfp.get('found'):
+        kind = 'DAC' if sfp.get('kind') == 'dac' else sfp.get('kind', '')
+        module = ' '.join(p for p in (kind, sfp.get('part', '')) if p)
+        if module:
+            state.append(f'module: {module}')
+        if sfp.get('rx_fault') or sfp.get('tx_fault'):
+            state.append('**module reports a fault**')
+    return f'{text}: {", ".join(state)}'
+
+
+def description(heading: str, facts: Sequence[Tuple[str, str]], summary: str,
+                checks: Sequence[str], sections: Sequence[Section] = (), recovers: bool = True) -> str:
+    lines = [f'## {heading}', '']
+    lines += [f'- **{key}:** {value}' for key, value in facts if value not in (None, '')]
+    lines += ['', summary.strip()]
+    if checks:
+        lines += ['', '### What to check']
+        lines += [f'{n}. {check}' for n, check in enumerate(checks, 1)]
+    for title, body in sections:
+        if body:
+            lines += ['', f'### {title}'] + list(body)
+    if recovers:
+        lines += ['', '_Opened automatically by Gandalf. It adds a note here when the problem clears, '
+                      'but leaves closing the ticket to you._']
+    else:
+        lines += ['', '_Reported automatically by Gandalf. This already happened, so there will be no '
+                      'recovery note._']
+    return '\n'.join(lines)
+
+
+def history_lines(stats: dict, now: datetime) -> List[str]:
+    week, month = stats.get('last_7d', 0), stats.get('last_30d', 0)
+    if not month:
+        lines = ['- First time in the last 30 days.']
+    else:
+        lines = [f'- Happened {week} time{"s" if week != 1 else ""} in the last 7 days '
+                 f'and {month} in the last 30 (not counting this one).']
+        if week >= 3:
+            lines.append('- **This keeps coming back**: look for a cause that comes and goes '
+                         '(a loose cable, a failing optic, power, or a scheduled job).')
+    first, resolved = stats.get('last_first_seen'), stats.get('last_resolved_at')
+    if resolved:
+        took = duration((resolved - first).total_seconds()) if first else 'unknown'
+        ago = duration((now - resolved).total_seconds())
+        lines.append(f'- Last cleared {utc(resolved)} ({ago} ago), after {took}.')
+    return lines
+
+
+ISSUE_FIXED = {
+    'cable_mismatch': 'The cable now matches the cutsheet (or the cutsheet was updated).',
+    'monitor_source_unavailable': 'Prometheus is returning data again; host and interface alerting has resumed.',
+}
+
+
+def recovery_comment(event: dict, now: datetime, url: str = '') -> str:
+    """The note Gandalf adds to a ticket when its alert clears."""
+    first, resolved = event.get('first_seen'), event.get('resolved_at') or now
+    target = event['target_name'] + (f' {event["target_detail"]}' if event.get('target_detail') else '')
+    lines = [f'**✅ Cleared**: {event["event_type"].replace("_", " ")} on {target}.', '']
+    lines.append(f'- **Cleared:** {utc(resolved)}')
+    if first:
+        lines.append(f'- **Lasted:** {duration((resolved - first).total_seconds())} '
+                     f'(first seen {utc(first)})')
+    if event.get('consecutive_failures'):
+        lines.append(f'- **Failed checks:** {event["consecutive_failures"]}')
+    if url:
+        lines.append(f'- **Gandalf:** {url}')
+    lines += ['', ISSUE_FIXED.get(event['event_type'], 'Gandalf no longer sees the problem.')]
+    lines += ['', '_Gandalf leaves this ticket open. Close it once you know why it happened, '
+                  'or leave it open if it needs follow-up._']
+    return '\n'.join(lines)