Mark host interfaces as unused #35

Merged
jared merged 1 commits from unused-interfaces into main 2026-11-11 17:50:14 -05:00
9 changed files with 219 additions and 14 deletions
+13
View File
@@ -352,6 +352,18 @@ whatever Gandalf knows around the problem:
Titles keep the `[target][auto][production]...` tag format, which Tinker uses to spot duplicates.
### Unused interfaces
An admin can mark a host interface as unused on its device page (**Mark unused** on a down interface,
with an optional reason; **In use again** to undo). An unused interface:
- never raises a link-down alert, and can't count toward the cluster-wide alert;
- doesn't count toward the host's status, so a host whose only down ports are unused shows as up;
- shows as "unused" (greyed, dashed dot) on the dashboard, the device page and the rack drawings;
- has any open link-down alert closed quietly when it is marked (no recovery note: nothing recovered).
Marks are stored in `unused_interfaces` (host, interface, reason, who, when).
### Recovery notes
When an alert that opened a ticket clears, the monitor adds a comment to that ticket: when it cleared, how
@@ -753,6 +765,7 @@ Take LED colors from the model's quick start guide. See the comment at the top o
- `cutsheet_items` gained `rack`, `shelf`, `device`, `mount`, `chassis`, `facing` and `slot` columns; they are added automatically on first use
- `cutsheet_racks`: racks saved on the rack page (name, height, notes)
- `unused_interfaces`: host interfaces marked unused (no alerts, not counted in host status)
- `device_links`: each device's inventory URL, by device name; created with the cutsheet tables
- `network_events`: one row per active event; `resolved_at` is set when recovered, and
`recovery_notified` once the recovery note is on its ticket (column added automatically at monitor startup)
+38 -2
View File
@@ -528,6 +528,12 @@ def api_device(name: str):
logger.error(f'Cutsheet unavailable for {name}: {e}')
detail['cutsheet'] = []
detail['inventory'] = _inventory_for(name, detail.get('model') or '')
try:
_ensure_cutsheet_schema()
detail['unused_interfaces'] = db.get_unused_interfaces().get(name, {})
except Exception as e:
logger.error(f'Unused interfaces unavailable for {name}: {e}')
detail['unused_interfaces'] = {}
return jsonify(detail)
@@ -1114,8 +1120,8 @@ def _physical_interfaces(name: str, snapshot: dict) -> list:
Drawn fronts light one RJ45 per NIC in this order (first NIC on the leftmost jack).
"""
ifaces = ((snapshot.get('hosts') or {}).get(name) or {}).get('interfaces') or {}
return [{'name': n, 'up': str(state).lower() == 'up'} for n, state in sorted(ifaces.items())
if not n.startswith(_VIRTUAL_IFACE)]
return [{'name': n, 'up': str(state).lower() == 'up', 'unused': str(state).lower() == 'unused'}
for n, state in sorted(ifaces.items()) if not n.startswith(_VIRTUAL_IFACE)]
def _device_status(name: str, snapshot: dict) -> str:
@@ -1257,6 +1263,36 @@ def api_cutsheet_delete_rack(rack_id: int):
return jsonify({'success': True})
# ---------------------------------------------------------------------------
# Unused interfaces
# ---------------------------------------------------------------------------
_IFACE_RE = re.compile(r'^[A-Za-z0-9_.:@-]{1,64}$')
@app.route('/api/device/<path:name>/interfaces/<iface>/unused', methods=['PUT', 'DELETE'])
@require_auth
@require_admin
@_cutsheet_write
def api_interface_unused(name: str, iface: str):
"""Mark a host interface unused (PUT {reason}) or in use again (DELETE).
Unused interfaces raise no link-down alerts and don't count toward the host's status;
marking one quietly closes any link-down alert it has open.
"""
if len(name) > 255 or not _IFACE_RE.match(iface):
raise cutsheet.ValidationError('Not a valid host or interface name')
user = _get_user()['username']
if request.method == 'DELETE':
found = db.clear_unused_interface(name, iface)
logger.info(f'{user} marked {name} {iface} in use again')
return jsonify({'success': True, 'changed': found})
reason = str(_json_body().get('reason') or '').strip()[:255]
db.set_unused_interface(name, iface, reason, user)
logger.info(f'{user} marked {name} {iface} unused: {reason or "(no reason)"}')
return jsonify({'success': True})
@app.route('/api/cutsheet/cables', methods=['POST'])
@require_auth
@require_admin
+53
View File
@@ -738,6 +738,15 @@ _CUTSHEET_SCHEMA = (
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
UNIQUE KEY uq_rack_name (name)
) ENGINE=InnoDB""",
# Host interfaces an admin has marked as not in use: no alerts, not counted in host status
"""CREATE TABLE IF NOT EXISTS unused_interfaces (
host VARCHAR(255) NOT NULL,
iface VARCHAR(64) NOT NULL,
reason VARCHAR(255) NOT NULL DEFAULT '',
set_by VARCHAR(255) NOT NULL,
set_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (host, iface)
) ENGINE=InnoDB""",
# Where a device lives in the asset inventory (HomeBox), set by hand on its Gandalf page
"""CREATE TABLE IF NOT EXISTS device_links (
name VARCHAR(255) NOT NULL PRIMARY KEY,
@@ -1107,3 +1116,47 @@ def cutsheet_delete_rack(rack_id: int, user: str) -> bool:
cur.execute('DELETE FROM cutsheet_racks WHERE id=%s', (rack_id,))
_cs_log(cur, user, 'rack', rack_id, 'delete', f'Removed rack {before["name"]}', _cs_jsonable(before), None)
return True
# ---------------------------------------------------------------------------
# Unused interfaces: ports an admin says aren't meant to be up
# ---------------------------------------------------------------------------
def get_unused_interfaces() -> dict:
"""{host: {iface: {'reason', 'set_by', 'set_at'}}}"""
with get_conn() as conn:
with conn.cursor() as cur:
cur.execute('SELECT host, iface, reason, set_by, set_at FROM unused_interfaces')
out: dict = {}
for r in cur.fetchall():
at = r['set_at'].isoformat() + 'Z' if hasattr(r['set_at'], 'isoformat') else r['set_at']
out.setdefault(r['host'], {})[r['iface']] = {'reason': r['reason'], 'set_by': r['set_by'],
'set_at': at}
return out
def set_unused_interface(host: str, iface: str, reason: str, user: str) -> None:
"""Mark an interface unused and quietly close any link-down alert it has open.
The alert is resolved without a recovery note: nothing recovered, the port just isn't used.
"""
with get_conn() as conn:
with conn.cursor() as cur:
cur.execute(
"""INSERT INTO unused_interfaces (host, iface, reason, set_by) VALUES (%s, %s, %s, %s)
ON DUPLICATE KEY UPDATE reason=VALUES(reason), set_by=VALUES(set_by)""",
(host, iface, reason, user),
)
cur.execute(
"""UPDATE network_events SET resolved_at=NOW(), recovery_notified=1, last_seen=last_seen
WHERE event_type='interface_down' AND target_name=%s AND target_detail=%s
AND resolved_at IS NULL""",
(host, iface),
)
def clear_unused_interface(host: str, iface: str) -> bool:
with get_conn() as conn:
with conn.cursor() as cur:
cur.execute('DELETE FROM unused_interfaces WHERE host=%s AND iface=%s', (host, iface))
return cur.rowcount > 0
+20 -3
View File
@@ -1045,6 +1045,16 @@ class NetworkMonitor:
def _hostname(self, instance: str) -> str:
return self._instance_map.get(instance, instance.split(':')[0])
def _unused_for(self, host: str) -> dict:
return (getattr(self, '_unused', None) or {}).get(host) or {}
def _load_unused(self) -> None:
"""Reload the interfaces marked unused (once a cycle; keeps the last list on error)."""
try:
self._unused = db.get_unused_interfaces()
except Exception as e:
logger.error(f'Could not load unused interfaces: {e}')
# ------------------------------------------------------------------
# Tickets: context, and notes when an alert clears
# ------------------------------------------------------------------
@@ -1229,7 +1239,10 @@ class NetworkMonitor:
new_baseline.setdefault(host, {})
host_has_regression = False
unused = self._unused_for(host)
for iface, is_up in ifaces.items():
if iface in unused:
continue # marked unused: no alerts, and it can't count as a regression
prev = baseline.get(host, {}).get(iface) # 'up', 'initial_down', or None
if is_up:
@@ -1799,8 +1812,11 @@ class NetworkMonitor:
continue # skip Prometheus instances not in config (e.g. LXC app servers)
host = self._hostname(instance)
phys = {k: v for k, v in ifaces.items()}
up_count = sum(1 for v in phys.values() if v)
total = len(phys)
# Interfaces marked unused don't count toward the host's status
unused = self._unused_for(host)
counted = {k: v for k, v in phys.items() if k not in unused}
up_count = sum(1 for v in counted.values() if v)
total = len(counted)
if not prom_ok:
status = 'unknown'
elif total == 0 or up_count == total:
@@ -1812,7 +1828,7 @@ class NetworkMonitor:
hosts[host] = {
'ip': instance.split(':')[0],
'interfaces': {k: ('up' if v else 'down') for k, v in phys.items()},
'interfaces': {k: ('unused' if k in unused else ('up' if v else 'down')) for k, v in phys.items()},
'status': status,
'source': 'prometheus',
}
@@ -1855,6 +1871,7 @@ class NetworkMonitor:
while True:
try:
logger.info('Starting network check cycle')
self._load_unused()
# 1. Fetch interface states once: shared by snapshot and alert processing
iface_states = self.prom.get_interface_states()
+45 -7
View File
@@ -122,9 +122,11 @@ function hostRows(interfaces) {
return Object.entries(interfaces)
.sort(([a], [b]) => a.localeCompare(b, undefined, { numeric: true }))
.map(([name, d]) => {
const up = d.state ? d.state === 'up' : d.link_detected !== false;
const mark = (_device.unused_interfaces || {})[name];
const unused = !!mark || d.state === 'unused';
const up = !unused && (d.state ? d.state === 'up' : d.link_detected !== false);
return {
id: name, label: name, up, data: d, sub: d.port_type || '',
id: name, label: name, up, unused, unusedReason: mark ? mark.reason : '', data: d, sub: d.port_type || '',
detail: `
<div class="link-stats-grid">
${stat('Duplex', esc(d.duplex || '-'), 'Full = both directions at once. Half can cause collisions.')}
@@ -154,7 +156,21 @@ function portRows(ports) {
}
function rowHasProblem(r) {
return !!badges(r.data);
return !r.unused && !!badges(r.data);
}
// The Health cell: an unused interface says so (with an admin toggle) instead of reporting down
function healthCell(r) {
if (r.unused) {
const tip = r.unusedReason ? ` title="${esc(r.unusedReason)}"` : '';
const btn = CAN_EDIT ? ` <button type="button" class="lt-btn lt-btn-ghost lt-btn-xs" data-unused="0" data-iface="${esc(r.id)}">In use again</button>` : '';
return `<span class="lt-badge badge-neutral"${tip}>unused</span>${btn}`;
}
const state = badges(r.data) || (r.up ? '<span class="val-good">OK</span>' : '<span class="device-row-sub">down</span>');
const btn = CAN_EDIT && !r.up && r.num === undefined
? ` <button type="button" class="lt-btn lt-btn-ghost lt-btn-xs" data-unused="1" data-iface="${esc(r.id)}"
title="Not meant to be up: stop alerting and stop counting it against the host">Mark unused</button>` : '';
return state + btn;
}
function renderTable(rows) {
@@ -173,15 +189,15 @@ function renderTable(rows) {
? (r.neighbor.name ? `<a class="device-link" href="${esc(deviceUrl(r.neighbor.name))}">${esc(r.neighbor.name)}</a>` : esc(r.neighbor.chassis || r.neighbor.port || ''))
: '-';
return `
<tr class="device-row${r.up ? '' : ' device-row--down'}"${isPorts ? ` data-port-idx="${esc(r.num)}"` : ''}>
<tr class="device-row${r.unused ? ' device-row--unused' : (r.up ? '' : ' device-row--down')}"${isPorts ? ` data-port-idx="${esc(r.num)}"` : ''}>
<td><button type="button" class="device-row-toggle" data-row="${esc(r.id)}"${isPorts ? '' : ` aria-expanded="${open}"`} aria-label="Details for ${esc(r.label)}">${open ? '▾' : '▸'}</button></td>
<td><span class="dot-${r.up ? 'up' : 'down'}"></span></td>
<td><span class="dot-${r.unused ? 'unused' : (r.up ? 'up' : 'down')}"></span></td>
<td class="device-row-name">${r.num !== undefined ? `<span class="port-badge port-badge-num">#${esc(r.num)}</span> ` : ''}${esc(r.label)}<div class="device-row-sub">${esc(r.sub)}</div></td>
<td class="device-num">${r.up ? fmtSpeed(d.speed_mbps) : '-'}</td>
<td class="device-num device-col-wide">${fmtRate(d.rx_bytes_rate)}</td>
<td class="device-num device-col-wide">${fmtRate(d.tx_bytes_rate)}</td>
${isPorts ? `<td class="device-col-wide">${neighbor}</td>` : ''}
<td>${badges(d) || (r.up ? '<span class="val-good">OK</span>' : '<span class="device-row-sub">down</span>')}</td>
<td>${healthCell(r)}</td>
</tr>
${open ? `<tr class="device-row-detail"><td colspan="7">${r.detail}</td></tr>` : ''}`;
}).join('');
@@ -326,8 +342,10 @@ function updateInterfaceSection(rows, title, hasChassis) {
document.getElementById('device-ports-title').textContent = 'Front panel';
return;
}
const inUse = rows.filter(r => !r.unused);
const unusedCount = rows.length - inUse.length;
document.getElementById('device-ports-title').textContent =
`${title} · ${rows.filter(r => r.up).length} of ${rows.length} up`;
`${title} · ${inUse.filter(r => r.up).length} of ${inUse.length} up${unusedCount ? ` · ${unusedCount} unused` : ''}`;
document.getElementById('device-table-search').setAttribute('aria-label', `Filter ${title.toLowerCase()}`);
document.getElementById('device-error-count').textContent = rows.filter(rowHasProblem).length;
}
@@ -369,7 +387,27 @@ async function saveInventory(e) {
document.addEventListener('click', e => {
if (e.target.closest('[data-action="edit-inventory"]')) openInventoryEditor();
const mark = e.target.closest('[data-unused]');
if (mark) toggleUnused(mark.dataset.iface, mark.dataset.unused === '1');
});
async function toggleUnused(iface, unused) {
const url = `/api/device/${encodeURIComponent(DEVICE_NAME)}/interfaces/${encodeURIComponent(iface)}/unused`;
try {
if (unused) {
const reason = window.prompt(`Why isn't ${iface} in use? (optional, e.g. "spare, never cabled")`, '');
if (reason === null) return;
await lt.api.put(url, { reason });
lt.toast.success(`${iface} marked unused: it won't alert or count against ${DEVICE_NAME}`);
} else {
await lt.api.delete(url);
lt.toast.success(`${iface} is in use again`);
}
await loadDevice();
} catch (err) {
lt.toast.error(err.message || 'Save failed');
}
}
const _inventoryForm = document.getElementById('inventory-form');
if (_inventoryForm) _inventoryForm.addEventListener('submit', saveInventory);
+1 -1
View File
@@ -391,7 +391,7 @@ function shelfSvg(item, esc, faceplate) {
<text class="sh-box-name" x="${x + w / 2}" y="${floor - h / 2 + 5}" text-anchor="middle">${esc(c.name)}</text>`;
}
const top = floor - h;
const nics = (c.interfaces || []).map(n => `${n.name} ${n.up ? 'up' : 'down'}`).join(', ');
const nics = (c.interfaces || []).map(n => `${n.name} ${n.unused ? 'unused' : (n.up ? 'up' : 'down')}`).join(', ');
const label = gear ? gear.label : (desk ? desk.label : c.kind);
const tip = [c.name, label, c.status, nics, gear && gear.specs].filter(Boolean).join(' · ');
// A live faceplate's ports are links of their own, so it isn't wrapped in one
+8
View File
@@ -1648,3 +1648,11 @@ body.fp-marks-off .fp-svg .fp-mark { display: none; }
.ub-opening { fill: #0d0e10; }
.ub-seam { fill: #050506; opacity: .8; }
.ub-strand { fill: #000; }
/* Interfaces marked unused: greyed, never "down" */
.dot-unused { display: inline-block; width: 8px; height: 8px; border: 1px dashed var(--text-muted);
background: transparent; flex-shrink: 0; vertical-align: middle; }
.state-unused { color: var(--text-muted); }
.device-row--unused td { opacity: .6; }
.device-row--unused td:last-child { opacity: 1; }
.lt-btn-xs { padding: 1px 6px; font-size: .68rem; margin-left: 6px; }
+28 -1
View File
@@ -37,6 +37,7 @@ def client():
db.check_suppressed.return_value = False
db.get_device_link.return_value = ''
db.cutsheet_racks.return_value = []
db.get_unused_interfaces.return_value = {}
with patch.object(gandalf, '_cfg', CONFIG):
yield gandalf.app.test_client()
@@ -551,7 +552,7 @@ class TestRackFacesAndSettings:
things = data['items'][0]['contents']
assert [(c['name'], c['slot'], c['chassis']) for c in things] == [('b-box', 1, 'zimaboard-832'),
('large1', 2, '')]
assert things[1]['interfaces'] == [{'name': 'eth0', 'up': True}]
assert things[1]['interfaces'] == [{'name': 'eth0', 'up': True, 'unused': False}]
assert things[0]['device'] == 'b-box' and things[0]['ports'] == {}
def test_saved_racks_replace_config(self, client):
@@ -583,3 +584,29 @@ class TestRackFacesAndSettings:
assert client.put('/api/cutsheet/racks/1', json={'name': 'x', 'units': 2}, headers=user).status_code == 403
gandalf.db.cutsheet_delete_rack.return_value = False
assert client.delete('/api/cutsheet/racks/9', headers=AUTH).status_code == 404
class TestUnusedInterfaces:
def _url(self, iface='nic1'):
gandalf._cutsheet_schema_ready = True
return f'/api/device/compute-storage-01/interfaces/{iface}/unused'
def test_mark_and_clear(self, client):
assert client.put(self._url(), json={'reason': ' never cabled '}, headers=AUTH).status_code == 200
gandalf.db.set_unused_interface.assert_called_with('compute-storage-01', 'nic1', 'never cabled', 'jared')
gandalf.db.clear_unused_interface.return_value = True
resp = client.delete(self._url(), headers=AUTH)
assert resp.get_json() == {'success': True, 'changed': True}
def test_needs_admin_json_and_a_sane_name(self, client):
user = dict(AUTH, **{'Remote-Groups': 'staff'})
with patch.object(gandalf, '_cfg', dict(CONFIG, auth=dict(CONFIG['auth'], allowed_groups=['admin', 'staff']))):
assert client.put(self._url(), json={}, headers=user).status_code == 403
assert client.put(self._url(), data='x', headers=AUTH).status_code == 400
assert client.put(self._url('bad name!'), json={}, headers=AUTH).status_code in (400, 404)
def test_device_api_lists_them(self, client):
gandalf._cutsheet_schema_ready = True
gandalf.db.get_unused_interfaces.return_value = {'large1': {'enp36s0': {'reason': 'spare'}}}
data = client.get('/api/device/large1', headers=AUTH).get_json()
assert data['unused_interfaces'] == {'enp36s0': {'reason': 'spare'}}
+13
View File
@@ -117,6 +117,19 @@ class TestDeadHostDetection:
mon._process_interfaces({HOST_A: {'eth0': True}}, [])
assert ('cluster_network_issue', 'test-cluster', '') in upserted(db)
def test_unused_interface_never_alerts(self, mon, db):
mon._unused = {'alpha': {'eth1': {'reason': 'spare'}}}
db.get_baseline.return_value = {'alpha': {'eth0': 'up', 'eth1': 'up'}, 'beta': {'eth0': 'up'}}
mon._process_interfaces({HOST_A: {'eth0': True, 'eth1': False}, HOST_B: {'eth0': True}}, [])
assert ('interface_down', 'alpha', 'eth1') not in upserted(db)
assert not mon.tickets.create.called
def test_unused_interface_does_not_degrade_the_host(self, mon):
mon._unused = {'alpha': {'eth1': {}}}
snap = mon._collect_snapshot({HOST_A: {'eth0': True, 'eth1': False}, HOST_B: {'eth0': True}})
assert snap['hosts']['alpha']['status'] == 'up'
assert snap['hosts']['alpha']['interfaces'] == {'eth0': 'up', 'eth1': 'unused'}
def test_dead_host_shows_as_down_on_dashboard(self, mon):
snap = mon._collect_snapshot({HOST_A: {'eth0': True}})
assert snap['hosts']['alpha']['status'] == 'up'