diff --git a/README.md b/README.md
index a4c3329..386a96d 100644
--- a/README.md
+++ b/README.md
@@ -213,6 +213,14 @@ can view it; only the `admin` group can edit, and every change is logged with wh
- **CSV**: export, and import that adds or updates cables by ID. A file with any invalid row imports
nothing.
- Cables, panels and jacks are included in the dashboard search.
+- **Trace a cord** (admins): pick a panel port and press Start, then unplug that cord at the panel.
+ Gandalf reads every switch port's state from UniFi every 3 seconds and shows the one that drops,
+ with a button to record the cable, then confirms when it's plugged back in. The device on that
+ port loses its link for those seconds. A cord whose far end has nothing linked can't be traced
+ this way.
+- **Mismatch alerts**: the monitor runs the same check every poll. A mismatched run opens a
+ `cable_mismatch` event on the switch straight away and resolves it once fixed; it becomes a P4
+ ticket only after `cutsheet.mismatch_ticket_hours`.
### Suppressions (`/suppressions`)
- Create timed (30 min / 1 hr / 4 hr / 8 hr) or manual suppressions
@@ -243,6 +251,7 @@ can view it; only the `admin` group can edit, and every change is logged with wh
| Modem pages unreadable, e.g. after a firmware update (≥`modem.failure_threshold`) | P3 |
| UniFi logs that Smart Power auto power-cycled the modem | P3 |
| UniFi logs that the WAN failed multiple times in 24 hours (at most one ticket a day) | P2 |
+| An installed cable doesn't match the cutsheet (wrong neighbor, no link, missing port, port conflict) for `cutsheet.mismatch_ticket_hours` | P4 |
A failed Pulse ping check counts as "unknown": it neither raises nor resolves `host_unreachable`.
@@ -332,6 +341,9 @@ Shared by both processes. Located in the working directory (`/var/www/html/prod/
{ "name": "pbs", "ip": "10.10.10.3" }
]
},
+ "cutsheet": {
+ "mismatch_ticket_hours": 24
+ },
"modem": {
"url": "http://192.168.1.254",
"name": "att-modem"
@@ -361,6 +373,7 @@ Shared by both processes. Located in the working directory (`/var/www/html/prod/
| `monitor.unifi_failure_threshold` | Consecutive failures before ticketing a UniFi device as offline (default: same as `failure_threshold`). Set higher than `failure_threshold` — UniFi switches/APs can take several minutes to rejoin after a firmware-update reboot, so a low threshold cuts tickets for momentary maintenance reboots instead of real failures. |
| `monitor.cluster_threshold` | Hosts with failures to trigger cluster-wide P1 (default: 3) |
| `monitor.ping_hosts` | Hosts checked only by ping (no node_exporter) |
+| `cutsheet.mismatch_ticket_hours` | Hours a cutsheet mismatch must last before it opens a P4 ticket (default: 24; `0` never tickets). The mismatch shows on the dashboard and the switch's page straight away, so recabling never pages anyone. Suppressing the switch or its port also stops the ticket. |
| `modem.url` | AT&T gateway base URL. Omit the `modem` block to disable modem monitoring. Must be reachable from the monitor host. |
| `modem.name` | Name used for the modem's alerts, tickets, and suppressions (default: `att-modem`) |
| `modem.timeout` | Seconds to wait per status page (default: 10) |
diff --git a/app.py b/app.py
index 26a22e1..5d2f390 100644
--- a/app.py
+++ b/app.py
@@ -24,7 +24,7 @@ import cutsheet
import db
import devices
import diagnose
-from monitor import PulseClient
+from monitor import PulseClient, UnifiClient
logging.basicConfig(
level=logging.INFO,
@@ -1014,6 +1014,37 @@ def api_cutsheet_port_label(item_id: int, port: str):
return jsonify({'success': True})
+_live_ports_cache: dict = {'at': 0.0, 'data': None, 'client': None}
+_live_ports_lock = threading.Lock()
+_LIVE_PORTS_TTL = 2.0
+
+
+@app.route('/api/cutsheet/live-ports')
+@require_auth
+@require_admin
+def api_cutsheet_live_ports():
+ """Current up/down state of every switch port, read from UniFi now.
+
+ Trace mode polls this while someone unplugs a cord, to see which port dropped.
+ The monitor's cached state is minutes old, so it can't be used for this.
+ """
+ with _live_ports_lock:
+ now = time.time()
+ if _live_ports_cache['data'] is None or now - _live_ports_cache['at'] > _LIVE_PORTS_TTL:
+ if _live_ports_cache['client'] is None:
+ _live_ports_cache['client'] = UnifiClient(_config()['unifi'])
+ raw = _live_ports_cache['client'].get_switch_ports()
+ if raw is None:
+ return jsonify({'error': 'The UniFi controller did not answer'}), 502
+ _live_ports_cache['data'] = {
+ sw: {str(p['port_idx']): {'up': bool(p.get('up')), 'name': name}
+ for name, p in (d.get('ports') or {}).items() if p.get('port_idx') is not None}
+ for sw, d in raw.items()
+ }
+ _live_ports_cache['at'] = now
+ return jsonify({'switches': _live_ports_cache['data'], 'at': _live_ports_cache['at']})
+
+
@app.route('/api/cutsheet/cables.csv')
@require_auth
def api_cutsheet_export():
diff --git a/cutsheet.py b/cutsheet.py
index 1c1c3c1..138eb27 100644
--- a/cutsheet.py
+++ b/cutsheet.py
@@ -469,3 +469,45 @@ def parse_csv(text: str, items: Dict[str, dict], max_rows: int = 2000) -> Tuple[
labels.add(cable['label'].lower())
rows.append(cable)
return rows, errors
+
+
+# ── Problems, for the monitor ───────────────────────────────────────────────
+
+def problems(cables: List[dict], items: Dict[str, dict], switches: dict) -> List[dict]:
+ """One entry per installed run whose check found a problem.
+
+ A run through a panel is several cables with the same verdict; it is reported
+ once, under the cable on the switch port (the one people look for).
+ Returns [{'key', 'cable_id', 'label', 'switch', 'port', 'verdict', 'detail', 'path'}].
+ """
+ switches = switches or {}
+ results = evaluate(cables, items, switches)
+ by_id = {c['id']: c for c in cables}
+ seen_runs = set()
+ found = []
+ for cid, r in sorted(results.items()):
+ if r['verdict'] not in PROBLEM_VERDICTS or by_id[cid]['status'] != 'installed':
+ continue
+ run = tuple(sorted(r['path']['cable_ids']))
+ if run in seen_runs and r['verdict'] != 'conflict':
+ continue
+ seen_runs.add(run)
+ hops = r['path']['hops']
+ ends = [n for n in (hops[0], hops[-1]) if n and n[0] in switches]
+ switch_end = ends[0] if ends else _node(by_id[cid], 'a')
+ # Name the run after the cable plugged into the switch, when there is one
+ lead = next((by_id[i] for i in r['path']['cable_ids']
+ if switch_end in (_node(by_id[i], 'a'), _node(by_id[i], 'b'))), by_id[cid])
+ if r['verdict'] == 'conflict':
+ lead = by_id[cid]
+ found.append({
+ 'key': f'cable {lead["id"]}',
+ 'cable_id': lead['id'],
+ 'label': lead.get('label') or f'#{lead["id"]}',
+ 'switch': switch_end[0] if switch_end else lead['a_device'],
+ 'port': switch_end[1] if switch_end else lead['a_port'],
+ 'verdict': r['verdict'],
+ 'detail': r['detail'],
+ 'path': ' → '.join(describe_node(h) for h in hops if not isinstance(h, int)),
+ })
+ return found
diff --git a/db.py b/db.py
index ffa18aa..8726162 100644
--- a/db.py
+++ b/db.py
@@ -178,6 +178,17 @@ def resolve_event(event_type: str, target_name: str, target_detail: str = '') ->
)
+def get_open_event_targets(event_type: str) -> list:
+ """(target_name, target_detail) of each unresolved event of one type."""
+ with get_conn() as conn:
+ with conn.cursor() as cur:
+ cur.execute(
+ 'SELECT target_name, target_detail FROM network_events WHERE event_type=%s AND resolved_at IS NULL',
+ (event_type,),
+ )
+ return [(r['target_name'], r['target_detail']) for r in cur.fetchall()]
+
+
def set_ticket_id(event_id: int, ticket_id: str) -> None:
with get_conn() as conn:
with conn.cursor() as cur:
diff --git a/monitor.py b/monitor.py
index 1f7527b..bb19297 100644
--- a/monitor.py
+++ b/monitor.py
@@ -21,6 +21,7 @@ import requests
from urllib3.exceptions import InsecureRequestWarning
import db
+import cutsheet
import devices
import modem as modem_api
@@ -943,6 +944,13 @@ class NetworkMonitor:
except Exception as e:
logger.error(f'Could not create modem history table: {e}')
+ # Cutsheet mismatches open a P4 ticket once they've lasted this long (0 = never ticket)
+ self.cutsheet_ticket_hours = (self.cfg.get('cutsheet') or {}).get('mismatch_ticket_hours', 24)
+ try:
+ db.ensure_cutsheet_schema()
+ except Exception as e:
+ logger.error(f'Could not create cutsheet tables: {e}')
+
# Last good Prometheus result, shown (as unknown) during a Prometheus outage
self._last_iface_states: Dict[str, Dict[str, bool]] = {}
@@ -1165,6 +1173,58 @@ class NetworkMonitor:
if tid:
db.set_ticket_id(event_id, tid)
+ # ------------------------------------------------------------------
+ # Cutsheet: documented cabling vs. what discovery sees
+ # ------------------------------------------------------------------
+ def _process_cutsheet(self, link_data: dict, suppressions: list) -> None:
+ """Keep one event per mismatched cable run, and ticket the ones that last.
+
+ Events appear on the dashboard and the switch's page straight away; a
+ ticket waits for cutsheet.mismatch_ticket_hours, so recabling never pages.
+ """
+ data = db.cutsheet_load()
+ items = {i['name']: i for i in data['items']}
+ switches = link_data.get('unifi_switches') or {}
+ found = cutsheet.problems(data['cables'], items, switches)
+ hours = self.cutsheet_ticket_hours
+ threshold = max(1, -(-int(hours * 3600) // max(1, self.poll_interval))) if hours else 0
+
+ current = set()
+ for p in found:
+ current.add((p['switch'], p['key']))
+ description = f'{p["label"]}: {p["detail"]} (path: {p["path"]})'
+ event_id, _, consec, ticket_id = db.upsert_event(
+ 'cable_mismatch', 'warning', 'cutsheet', p['switch'], p['key'], description,
+ )
+ sup = (db.check_suppressed(suppressions, 'unifi_device', p['switch']) or
+ db.check_suppressed(suppressions, 'interface', p['switch'], p['port']))
+ if threshold and not sup and consec >= threshold and not ticket_id:
+ self._ticket_cable_mismatch(event_id, p, hours)
+
+ for target_name, target_detail in db.get_open_event_targets('cable_mismatch'):
+ if (target_name, target_detail) not in current:
+ db.resolve_event('cable_mismatch', target_name, target_detail)
+
+ def _ticket_cable_mismatch(self, event_id: int, p: dict, hours: float) -> None:
+ title = (
+ f'[{p["switch"]}][auto][production][issue][network][single-node] '
+ f'Cable {p["label"]} does not match the cutsheet'
+ )
+ desc = (
+ f'Cutsheet Mismatch\n{"=" * 40}\n\n'
+ f'Cable: {p["label"]}\n'
+ f'Switch port: {p["switch"]} port {p["port"]}\n'
+ f'Problem: {p["detail"]}\n'
+ f'Documented path: {p["path"]}\n'
+ f'Detected: {_now_utc()}\n\n'
+ f'This has not matched for at least {hours:g} hours.\n\n'
+ f'Either the cable was moved (update the cutsheet in Gandalf: /cutsheet#cable-{p["cable_id"]}) '
+ f'or it is plugged into the wrong place.'
+ )
+ tid = self.tickets.create(title, desc, priority='4')
+ if tid:
+ db.set_ticket_id(event_id, tid)
+
# ------------------------------------------------------------------
# Ping-only hosts (no node_exporter)
# ------------------------------------------------------------------
@@ -1579,6 +1639,7 @@ class NetworkMonitor:
db.set_state('last_check', _now_utc())
# 4. Collect link stats (ethtool + traffic metrics)
+ link_data = None
try:
link_data = self.link_stats.collect(self._instance_map)
db.set_state('link_stats', link_data)
@@ -1596,6 +1657,13 @@ class NetworkMonitor:
self._process_ping_hosts(suppressions, ping_states)
+ # Cutsheet: compare documented cabling with what was just discovered
+ if link_data is not None:
+ try:
+ self._process_cutsheet(link_data, suppressions)
+ except Exception as e:
+ logger.error(f'Cutsheet check failed: {e}', exc_info=True)
+
# WAN modem: isolated so a failure here can't stop the rest of the cycle
if self.modem:
try:
diff --git a/static/cutsheet.js b/static/cutsheet.js
index ca386ce..a16daf8 100644
--- a/static/cutsheet.js
+++ b/static/cutsheet.js
@@ -329,6 +329,7 @@ function render() {
renderItems();
renderHistory();
renderDatalists();
+ renderTracePanels();
}
async function load() {
@@ -500,6 +501,136 @@ async function importCsv(e, dryRun) {
await load();
}
+// ── Trace mode ──────────────────────────────────────────────────────────
+// Snapshot which switch ports are up, then watch for the one that drops when a
+// cord is unplugged at the panel. UniFi takes a few seconds to notice.
+const TRACE_POLL_MS = 3000;
+const TRACE_TIMEOUT_MS = 180000;
+let _trace = null;
+
+function renderTracePanels() {
+ const select = document.getElementById('cs-trace-panel');
+ if (!select) return;
+ const panels = _cs.items.filter(i => i.kind === 'patch_panel').sort((a, b) => natural(a.name, b.name));
+ const keep = select.value;
+ select.innerHTML = panels.map(p => ``).join('');
+ if (panels.some(p => p.name === keep)) select.value = keep;
+ document.querySelector('[data-action="trace-start"]').disabled = !panels.length || !!_trace;
+ if (!panels.length) traceStatus('Add a patch panel first.');
+}
+
+function traceStatus(html) {
+ const el = document.getElementById('cs-trace-status');
+ if (el) el.innerHTML = html;
+}
+
+function portKey(sw, idx) { return JSON.stringify([sw, String(idx)]); }
+
+function upPorts(switches) {
+ const up = new Set();
+ for (const [sw, ports] of Object.entries(switches || {})) {
+ for (const [idx, p] of Object.entries(ports)) if (p.up) up.add(portKey(sw, idx));
+ }
+ return up;
+}
+
+function traceButtons(running) {
+ document.querySelector('[data-action="trace-start"]').hidden = running;
+ document.querySelector('[data-action="trace-stop"]').hidden = !running;
+}
+
+function stopTrace() {
+ if (_trace && _trace.timer) clearTimeout(_trace.timer);
+ _trace = null;
+ traceButtons(false);
+ renderTracePanels();
+}
+
+async function startTrace() {
+ const panel = document.getElementById('cs-trace-panel').value;
+ const port = String(parseInt(document.getElementById('cs-trace-port').value, 10) || '');
+ const item = itemByName(panel);
+ if (!item || !port || Number(port) < 1 || Number(port) > (item.port_count || 0)) {
+ traceStatus(`Choose a port between 1 and ${esc(item ? item.port_count : '?')}.`);
+ return;
+ }
+ traceButtons(true);
+ traceStatus('Reading switch ports…');
+ let live;
+ try {
+ live = (await lt.api.get('/api/cutsheet/live-ports')).switches;
+ } catch (err) {
+ traceStatus(`${esc(err.message)}`);
+ traceButtons(false);
+ return;
+ }
+ _trace = { panel, port, baseline: upPorts(live), started: Date.now(), found: null, timer: null };
+ traceStatus(`Ready. Unplug the cord from ${esc(panel)} port ${esc(port)} now.
+ Watching ${_trace.baseline.size} linked switch ports… UniFi can take up to about 30 seconds to notice.`);
+ _trace.timer = setTimeout(pollTrace, TRACE_POLL_MS);
+}
+
+async function pollTrace() {
+ const t = _trace;
+ if (!t) return;
+ let live;
+ try {
+ live = (await lt.api.get('/api/cutsheet/live-ports')).switches;
+ } catch (err) {
+ if (_trace === t) t.timer = setTimeout(pollTrace, TRACE_POLL_MS);
+ return;
+ }
+ if (_trace !== t) return;
+ const up = upPorts(live);
+ if (t.found) {
+ if (up.has(t.found.key)) {
+ showTraceResult(t, true);
+ stopTrace();
+ } else {
+ t.timer = setTimeout(pollTrace, TRACE_POLL_MS);
+ }
+ return;
+ }
+ const dropped = [...t.baseline].filter(k => !up.has(k)).map(k => JSON.parse(k));
+ if (dropped.length === 1) {
+ const [sw, idx] = dropped[0];
+ t.found = { key: portKey(sw, idx), sw, idx, name: ((live[sw] || {})[idx] || {}).name || '' };
+ showTraceResult(t, false);
+ t.timer = setTimeout(pollTrace, TRACE_POLL_MS);
+ } else if (dropped.length > 1) {
+ traceStatus(`Several ports dropped at once, so the cord can't be told apart:
+ ${dropped.map(([sw, idx]) => `${esc(sw)} port ${esc(idx)}`).join(', ')}. Plug everything back in and try again.`);
+ stopTrace();
+ } else if (Date.now() - t.started > TRACE_TIMEOUT_MS) {
+ traceStatus(`No switch port dropped in 3 minutes. If nothing is plugged in at the far end of
+ ${esc(t.panel)} port ${esc(t.port)}, its switch port was never linked; trace that one by hand.`);
+ stopTrace();
+ } else {
+ t.timer = setTimeout(pollTrace, TRACE_POLL_MS);
+ }
+}
+
+function showTraceResult(t, replugged) {
+ const f = t.found;
+ const existing = cableAt(f.sw, f.idx, '');
+ const panelCable = cableAt(t.panel, t.port, 'front');
+ let action;
+ if (existing && panelCable && existing.id === panelCable.id) {
+ action = `Matches ${esc(existing.label || 'the cutsheet')}.`;
+ } else if (existing) {
+ action = `That switch port is documented as ${esc(existing.label || 'an unlabeled cable')}
+ to ${esc(existing.a_device === f.sw ? existing.b_text : existing.a_text)}.
+ `;
+ } else {
+ action = ``;
+ }
+ traceStatus(`
+
${esc(t.panel)} port ${esc(t.port)} → ${esc(f.sw)} port ${esc(f.idx)}${f.name ? ` (${esc(f.name)})` : ''}
+
${action}
+
${replugged ? 'Plugged back in, link is up again.' : 'Plug the cord back in now.'}
Find which switch port a panel port's cord goes to: start, then unplug that cord at the panel.
+ Gandalf watches every switch port and shows the one that drops. Plug it back in afterwards. The device on it
+ loses its link for those few seconds.