From 3834546f384f6eef82191c6e167a3994dd9eb36b Mon Sep 17 00:00:00 2001 From: Jared Vititoe Date: Wed, 11 Nov 2026 17:19:45 -0500 Subject: [PATCH] Mark host interfaces as unused Ports that are never meant to be up (compute-storage-01 nic1-3, large1's spare NIC and USB adapter) kept their hosts "degraded" and were the source of tonight's false alerts. An admin can now mark an interface unused from the device page, with an optional reason. Unused interfaces raise no link-down alerts, don't count toward the cluster-wide alert or the host's status, and show greyed as "unused" on the dashboard, device page and rack. Marking one closes any open link-down alert on it without a recovery note. New table unused_interfaces, created with the cutsheet tables. Co-Authored-By: Claude Opus 5.5 --- README.md | 13 +++++++++++ app.py | 40 ++++++++++++++++++++++++++++++-- db.py | 53 +++++++++++++++++++++++++++++++++++++++++++ monitor.py | 23 ++++++++++++++++--- static/device.js | 52 ++++++++++++++++++++++++++++++++++++------ static/fronts.js | 2 +- static/style.css | 8 +++++++ tests/test_app.py | 29 ++++++++++++++++++++++- tests/test_monitor.py | 13 +++++++++++ 9 files changed, 219 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index b3dcceb..31a518b 100644 --- a/README.md +++ b/README.md @@ -352,6 +352,18 @@ whatever Gandalf knows around the problem: Titles keep the `[target][auto][production]...` tag format, which Tinker uses to spot duplicates. +### Unused interfaces + +An admin can mark a host interface as unused on its device page (**Mark unused** on a down interface, +with an optional reason; **In use again** to undo). An unused interface: + +- never raises a link-down alert, and can't count toward the cluster-wide alert; +- doesn't count toward the host's status, so a host whose only down ports are unused shows as up; +- shows as "unused" (greyed, dashed dot) on the dashboard, the device page and the rack drawings; +- has any open link-down alert closed quietly when it is marked (no recovery note: nothing recovered). + +Marks are stored in `unused_interfaces` (host, interface, reason, who, when). + ### Recovery notes When an alert that opened a ticket clears, the monitor adds a comment to that ticket: when it cleared, how @@ -753,6 +765,7 @@ Take LED colors from the model's quick start guide. See the comment at the top o - `cutsheet_items` gained `rack`, `shelf`, `device`, `mount`, `chassis`, `facing` and `slot` columns; they are added automatically on first use - `cutsheet_racks`: racks saved on the rack page (name, height, notes) +- `unused_interfaces`: host interfaces marked unused (no alerts, not counted in host status) - `device_links`: each device's inventory URL, by device name; created with the cutsheet tables - `network_events`: one row per active event; `resolved_at` is set when recovered, and `recovery_notified` once the recovery note is on its ticket (column added automatically at monitor startup) diff --git a/app.py b/app.py index 4b49211..683a760 100644 --- a/app.py +++ b/app.py @@ -528,6 +528,12 @@ def api_device(name: str): logger.error(f'Cutsheet unavailable for {name}: {e}') detail['cutsheet'] = [] detail['inventory'] = _inventory_for(name, detail.get('model') or '') + try: + _ensure_cutsheet_schema() + detail['unused_interfaces'] = db.get_unused_interfaces().get(name, {}) + except Exception as e: + logger.error(f'Unused interfaces unavailable for {name}: {e}') + detail['unused_interfaces'] = {} return jsonify(detail) @@ -1114,8 +1120,8 @@ def _physical_interfaces(name: str, snapshot: dict) -> list: Drawn fronts light one RJ45 per NIC in this order (first NIC on the leftmost jack). """ ifaces = ((snapshot.get('hosts') or {}).get(name) or {}).get('interfaces') or {} - return [{'name': n, 'up': str(state).lower() == 'up'} for n, state in sorted(ifaces.items()) - if not n.startswith(_VIRTUAL_IFACE)] + return [{'name': n, 'up': str(state).lower() == 'up', 'unused': str(state).lower() == 'unused'} + for n, state in sorted(ifaces.items()) if not n.startswith(_VIRTUAL_IFACE)] def _device_status(name: str, snapshot: dict) -> str: @@ -1257,6 +1263,36 @@ def api_cutsheet_delete_rack(rack_id: int): return jsonify({'success': True}) +# --------------------------------------------------------------------------- +# Unused interfaces +# --------------------------------------------------------------------------- + +_IFACE_RE = re.compile(r'^[A-Za-z0-9_.:@-]{1,64}$') + + +@app.route('/api/device//interfaces//unused', methods=['PUT', 'DELETE']) +@require_auth +@require_admin +@_cutsheet_write +def api_interface_unused(name: str, iface: str): + """Mark a host interface unused (PUT {reason}) or in use again (DELETE). + + Unused interfaces raise no link-down alerts and don't count toward the host's status; + marking one quietly closes any link-down alert it has open. + """ + if len(name) > 255 or not _IFACE_RE.match(iface): + raise cutsheet.ValidationError('Not a valid host or interface name') + user = _get_user()['username'] + if request.method == 'DELETE': + found = db.clear_unused_interface(name, iface) + logger.info(f'{user} marked {name} {iface} in use again') + return jsonify({'success': True, 'changed': found}) + reason = str(_json_body().get('reason') or '').strip()[:255] + db.set_unused_interface(name, iface, reason, user) + logger.info(f'{user} marked {name} {iface} unused: {reason or "(no reason)"}') + return jsonify({'success': True}) + + @app.route('/api/cutsheet/cables', methods=['POST']) @require_auth @require_admin diff --git a/db.py b/db.py index e55da2d..8f80984 100644 --- a/db.py +++ b/db.py @@ -738,6 +738,15 @@ _CUTSHEET_SCHEMA = ( updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, UNIQUE KEY uq_rack_name (name) ) ENGINE=InnoDB""", + # Host interfaces an admin has marked as not in use: no alerts, not counted in host status + """CREATE TABLE IF NOT EXISTS unused_interfaces ( + host VARCHAR(255) NOT NULL, + iface VARCHAR(64) NOT NULL, + reason VARCHAR(255) NOT NULL DEFAULT '', + set_by VARCHAR(255) NOT NULL, + set_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (host, iface) + ) ENGINE=InnoDB""", # Where a device lives in the asset inventory (HomeBox), set by hand on its Gandalf page """CREATE TABLE IF NOT EXISTS device_links ( name VARCHAR(255) NOT NULL PRIMARY KEY, @@ -1107,3 +1116,47 @@ def cutsheet_delete_rack(rack_id: int, user: str) -> bool: cur.execute('DELETE FROM cutsheet_racks WHERE id=%s', (rack_id,)) _cs_log(cur, user, 'rack', rack_id, 'delete', f'Removed rack {before["name"]}', _cs_jsonable(before), None) return True + + +# --------------------------------------------------------------------------- +# Unused interfaces: ports an admin says aren't meant to be up +# --------------------------------------------------------------------------- + +def get_unused_interfaces() -> dict: + """{host: {iface: {'reason', 'set_by', 'set_at'}}}""" + with get_conn() as conn: + with conn.cursor() as cur: + cur.execute('SELECT host, iface, reason, set_by, set_at FROM unused_interfaces') + out: dict = {} + for r in cur.fetchall(): + at = r['set_at'].isoformat() + 'Z' if hasattr(r['set_at'], 'isoformat') else r['set_at'] + out.setdefault(r['host'], {})[r['iface']] = {'reason': r['reason'], 'set_by': r['set_by'], + 'set_at': at} + return out + + +def set_unused_interface(host: str, iface: str, reason: str, user: str) -> None: + """Mark an interface unused and quietly close any link-down alert it has open. + + The alert is resolved without a recovery note: nothing recovered, the port just isn't used. + """ + with get_conn() as conn: + with conn.cursor() as cur: + cur.execute( + """INSERT INTO unused_interfaces (host, iface, reason, set_by) VALUES (%s, %s, %s, %s) + ON DUPLICATE KEY UPDATE reason=VALUES(reason), set_by=VALUES(set_by)""", + (host, iface, reason, user), + ) + cur.execute( + """UPDATE network_events SET resolved_at=NOW(), recovery_notified=1, last_seen=last_seen + WHERE event_type='interface_down' AND target_name=%s AND target_detail=%s + AND resolved_at IS NULL""", + (host, iface), + ) + + +def clear_unused_interface(host: str, iface: str) -> bool: + with get_conn() as conn: + with conn.cursor() as cur: + cur.execute('DELETE FROM unused_interfaces WHERE host=%s AND iface=%s', (host, iface)) + return cur.rowcount > 0 diff --git a/monitor.py b/monitor.py index 4d75dec..c19d7b5 100644 --- a/monitor.py +++ b/monitor.py @@ -1045,6 +1045,16 @@ class NetworkMonitor: def _hostname(self, instance: str) -> str: return self._instance_map.get(instance, instance.split(':')[0]) + def _unused_for(self, host: str) -> dict: + return (getattr(self, '_unused', None) or {}).get(host) or {} + + def _load_unused(self) -> None: + """Reload the interfaces marked unused (once a cycle; keeps the last list on error).""" + try: + self._unused = db.get_unused_interfaces() + except Exception as e: + logger.error(f'Could not load unused interfaces: {e}') + # ------------------------------------------------------------------ # Tickets: context, and notes when an alert clears # ------------------------------------------------------------------ @@ -1229,7 +1239,10 @@ class NetworkMonitor: new_baseline.setdefault(host, {}) host_has_regression = False + unused = self._unused_for(host) for iface, is_up in ifaces.items(): + if iface in unused: + continue # marked unused: no alerts, and it can't count as a regression prev = baseline.get(host, {}).get(iface) # 'up', 'initial_down', or None if is_up: @@ -1799,8 +1812,11 @@ class NetworkMonitor: continue # skip Prometheus instances not in config (e.g. LXC app servers) host = self._hostname(instance) phys = {k: v for k, v in ifaces.items()} - up_count = sum(1 for v in phys.values() if v) - total = len(phys) + # Interfaces marked unused don't count toward the host's status + unused = self._unused_for(host) + counted = {k: v for k, v in phys.items() if k not in unused} + up_count = sum(1 for v in counted.values() if v) + total = len(counted) if not prom_ok: status = 'unknown' elif total == 0 or up_count == total: @@ -1812,7 +1828,7 @@ class NetworkMonitor: hosts[host] = { 'ip': instance.split(':')[0], - 'interfaces': {k: ('up' if v else 'down') for k, v in phys.items()}, + 'interfaces': {k: ('unused' if k in unused else ('up' if v else 'down')) for k, v in phys.items()}, 'status': status, 'source': 'prometheus', } @@ -1855,6 +1871,7 @@ class NetworkMonitor: while True: try: logger.info('Starting network check cycle') + self._load_unused() # 1. Fetch interface states once: shared by snapshot and alert processing iface_states = self.prom.get_interface_states() diff --git a/static/device.js b/static/device.js index 78dd75f..c9afcb1 100644 --- a/static/device.js +++ b/static/device.js @@ -122,9 +122,11 @@ function hostRows(interfaces) { return Object.entries(interfaces) .sort(([a], [b]) => a.localeCompare(b, undefined, { numeric: true })) .map(([name, d]) => { - const up = d.state ? d.state === 'up' : d.link_detected !== false; + const mark = (_device.unused_interfaces || {})[name]; + const unused = !!mark || d.state === 'unused'; + const up = !unused && (d.state ? d.state === 'up' : d.link_detected !== false); return { - id: name, label: name, up, data: d, sub: d.port_type || '', + id: name, label: name, up, unused, unusedReason: mark ? mark.reason : '', data: d, sub: d.port_type || '', detail: `
${stat('Duplex', esc(d.duplex || '-'), 'Full = both directions at once. Half can cause collisions.')} @@ -154,7 +156,21 @@ function portRows(ports) { } function rowHasProblem(r) { - return !!badges(r.data); + return !r.unused && !!badges(r.data); +} + +// The Health cell: an unused interface says so (with an admin toggle) instead of reporting down +function healthCell(r) { + if (r.unused) { + const tip = r.unusedReason ? ` title="${esc(r.unusedReason)}"` : ''; + const btn = CAN_EDIT ? ` ` : ''; + return `unused${btn}`; + } + const state = badges(r.data) || (r.up ? 'OK' : 'down'); + const btn = CAN_EDIT && !r.up && r.num === undefined + ? ` ` : ''; + return state + btn; } function renderTable(rows) { @@ -173,15 +189,15 @@ function renderTable(rows) { ? (r.neighbor.name ? `${esc(r.neighbor.name)}` : esc(r.neighbor.chassis || r.neighbor.port || '')) : '-'; return ` - + - + ${r.num !== undefined ? `#${esc(r.num)} ` : ''}${esc(r.label)}
${esc(r.sub)}
${r.up ? fmtSpeed(d.speed_mbps) : '-'} ${fmtRate(d.rx_bytes_rate)} ${fmtRate(d.tx_bytes_rate)} ${isPorts ? `${neighbor}` : ''} - ${badges(d) || (r.up ? 'OK' : 'down')} + ${healthCell(r)} ${open ? `${r.detail}` : ''}`; }).join(''); @@ -326,8 +342,10 @@ function updateInterfaceSection(rows, title, hasChassis) { document.getElementById('device-ports-title').textContent = 'Front panel'; return; } + const inUse = rows.filter(r => !r.unused); + const unusedCount = rows.length - inUse.length; document.getElementById('device-ports-title').textContent = - `${title} · ${rows.filter(r => r.up).length} of ${rows.length} up`; + `${title} · ${inUse.filter(r => r.up).length} of ${inUse.length} up${unusedCount ? ` · ${unusedCount} unused` : ''}`; document.getElementById('device-table-search').setAttribute('aria-label', `Filter ${title.toLowerCase()}`); document.getElementById('device-error-count').textContent = rows.filter(rowHasProblem).length; } @@ -369,7 +387,27 @@ async function saveInventory(e) { document.addEventListener('click', e => { if (e.target.closest('[data-action="edit-inventory"]')) openInventoryEditor(); + const mark = e.target.closest('[data-unused]'); + if (mark) toggleUnused(mark.dataset.iface, mark.dataset.unused === '1'); }); + +async function toggleUnused(iface, unused) { + const url = `/api/device/${encodeURIComponent(DEVICE_NAME)}/interfaces/${encodeURIComponent(iface)}/unused`; + try { + if (unused) { + const reason = window.prompt(`Why isn't ${iface} in use? (optional, e.g. "spare, never cabled")`, ''); + if (reason === null) return; + await lt.api.put(url, { reason }); + lt.toast.success(`${iface} marked unused: it won't alert or count against ${DEVICE_NAME}`); + } else { + await lt.api.delete(url); + lt.toast.success(`${iface} is in use again`); + } + await loadDevice(); + } catch (err) { + lt.toast.error(err.message || 'Save failed'); + } +} const _inventoryForm = document.getElementById('inventory-form'); if (_inventoryForm) _inventoryForm.addEventListener('submit', saveInventory); diff --git a/static/fronts.js b/static/fronts.js index feab522..545a285 100644 --- a/static/fronts.js +++ b/static/fronts.js @@ -391,7 +391,7 @@ function shelfSvg(item, esc, faceplate) { ${esc(c.name)}`; } const top = floor - h; - const nics = (c.interfaces || []).map(n => `${n.name} ${n.up ? 'up' : 'down'}`).join(', '); + const nics = (c.interfaces || []).map(n => `${n.name} ${n.unused ? 'unused' : (n.up ? 'up' : 'down')}`).join(', '); const label = gear ? gear.label : (desk ? desk.label : c.kind); const tip = [c.name, label, c.status, nics, gear && gear.specs].filter(Boolean).join(' · '); // A live faceplate's ports are links of their own, so it isn't wrapped in one diff --git a/static/style.css b/static/style.css index 23dcd59..7b78ab9 100644 --- a/static/style.css +++ b/static/style.css @@ -1648,3 +1648,11 @@ body.fp-marks-off .fp-svg .fp-mark { display: none; } .ub-opening { fill: #0d0e10; } .ub-seam { fill: #050506; opacity: .8; } .ub-strand { fill: #000; } + +/* Interfaces marked unused: greyed, never "down" */ +.dot-unused { display: inline-block; width: 8px; height: 8px; border: 1px dashed var(--text-muted); + background: transparent; flex-shrink: 0; vertical-align: middle; } +.state-unused { color: var(--text-muted); } +.device-row--unused td { opacity: .6; } +.device-row--unused td:last-child { opacity: 1; } +.lt-btn-xs { padding: 1px 6px; font-size: .68rem; margin-left: 6px; } diff --git a/tests/test_app.py b/tests/test_app.py index 8d83b6a..37b1235 100644 --- a/tests/test_app.py +++ b/tests/test_app.py @@ -37,6 +37,7 @@ def client(): db.check_suppressed.return_value = False db.get_device_link.return_value = '' db.cutsheet_racks.return_value = [] + db.get_unused_interfaces.return_value = {} with patch.object(gandalf, '_cfg', CONFIG): yield gandalf.app.test_client() @@ -551,7 +552,7 @@ class TestRackFacesAndSettings: things = data['items'][0]['contents'] assert [(c['name'], c['slot'], c['chassis']) for c in things] == [('b-box', 1, 'zimaboard-832'), ('large1', 2, '')] - assert things[1]['interfaces'] == [{'name': 'eth0', 'up': True}] + assert things[1]['interfaces'] == [{'name': 'eth0', 'up': True, 'unused': False}] assert things[0]['device'] == 'b-box' and things[0]['ports'] == {} def test_saved_racks_replace_config(self, client): @@ -583,3 +584,29 @@ class TestRackFacesAndSettings: assert client.put('/api/cutsheet/racks/1', json={'name': 'x', 'units': 2}, headers=user).status_code == 403 gandalf.db.cutsheet_delete_rack.return_value = False assert client.delete('/api/cutsheet/racks/9', headers=AUTH).status_code == 404 + + +class TestUnusedInterfaces: + def _url(self, iface='nic1'): + gandalf._cutsheet_schema_ready = True + return f'/api/device/compute-storage-01/interfaces/{iface}/unused' + + def test_mark_and_clear(self, client): + assert client.put(self._url(), json={'reason': ' never cabled '}, headers=AUTH).status_code == 200 + gandalf.db.set_unused_interface.assert_called_with('compute-storage-01', 'nic1', 'never cabled', 'jared') + gandalf.db.clear_unused_interface.return_value = True + resp = client.delete(self._url(), headers=AUTH) + assert resp.get_json() == {'success': True, 'changed': True} + + def test_needs_admin_json_and_a_sane_name(self, client): + user = dict(AUTH, **{'Remote-Groups': 'staff'}) + with patch.object(gandalf, '_cfg', dict(CONFIG, auth=dict(CONFIG['auth'], allowed_groups=['admin', 'staff']))): + assert client.put(self._url(), json={}, headers=user).status_code == 403 + assert client.put(self._url(), data='x', headers=AUTH).status_code == 400 + assert client.put(self._url('bad name!'), json={}, headers=AUTH).status_code in (400, 404) + + def test_device_api_lists_them(self, client): + gandalf._cutsheet_schema_ready = True + gandalf.db.get_unused_interfaces.return_value = {'large1': {'enp36s0': {'reason': 'spare'}}} + data = client.get('/api/device/large1', headers=AUTH).get_json() + assert data['unused_interfaces'] == {'enp36s0': {'reason': 'spare'}} diff --git a/tests/test_monitor.py b/tests/test_monitor.py index 6c694f2..24c0369 100644 --- a/tests/test_monitor.py +++ b/tests/test_monitor.py @@ -117,6 +117,19 @@ class TestDeadHostDetection: mon._process_interfaces({HOST_A: {'eth0': True}}, []) assert ('cluster_network_issue', 'test-cluster', '') in upserted(db) + def test_unused_interface_never_alerts(self, mon, db): + mon._unused = {'alpha': {'eth1': {'reason': 'spare'}}} + db.get_baseline.return_value = {'alpha': {'eth0': 'up', 'eth1': 'up'}, 'beta': {'eth0': 'up'}} + mon._process_interfaces({HOST_A: {'eth0': True, 'eth1': False}, HOST_B: {'eth0': True}}, []) + assert ('interface_down', 'alpha', 'eth1') not in upserted(db) + assert not mon.tickets.create.called + + def test_unused_interface_does_not_degrade_the_host(self, mon): + mon._unused = {'alpha': {'eth1': {}}} + snap = mon._collect_snapshot({HOST_A: {'eth0': True, 'eth1': False}, HOST_B: {'eth0': True}}) + assert snap['hosts']['alpha']['status'] == 'up' + assert snap['hosts']['alpha']['interfaces'] == {'eth0': 'up', 'eth1': 'unused'} + def test_dead_host_shows_as_down_on_dashboard(self, mon): snap = mon._collect_snapshot({HOST_A: {'eth0': True}}) assert snap['hosts']['alpha']['status'] == 'up' -- 2.47.3