Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
e1716d5228 | ||
|
|
4a1998046b | ||
|
|
fed279a0bd |
@@ -11,6 +11,52 @@ Deployed on **LXC 157** (monitor-02 / 10.10.10.9), reachable at `gandalf.lotusgu
|
||||
|
||||
**Design System**: [web_template](https://code.lotusguild.org/LotusGuild/web_template) — shared CSS, JS, and layout patterns for all LotusGuild apps
|
||||
|
||||

|
||||
|
||||
## Why this exists
|
||||
|
||||
A Proxmox/Ceph cluster lives or dies by its network, and network problems are sneaky: a link that flaps, a fibre with weak receive power, a switch port that quietly negotiated 1 Gbps. GANDALF watches the cluster's links from three angles at once (host NIC state from Prometheus, switch ports and LLDP from the UniFi controller, and reachability pings), alerts only on **regressions** (a link that was up and went down; ports that were never connected are learned as baseline and ignored), and files one ticket per real problem. When something looks wrong, a click on the switch port runs a full on-host diagnostic through [PULSE](https://code.lotusguild.org/LotusGuild/pulse) (`ethtool`, SFP optics, kernel log, LLDP) and explains what it found.
|
||||
|
||||
## Gallery
|
||||
|
||||
**Dashboard**: alert queue with linked tickets, a live topology, per-host interface status and the UniFi device inventory.
|
||||
|
||||

|
||||
|
||||

|
||||
|
||||

|
||||
|
||||
**Link debug**: per-interface speed, duplex, carrier flaps, error and drop rates, and **SFP optical readings** (temperature, TX/RX power) with colour-coded thresholds. Here a fibre with weak receive power is flagged while its neighbour is healthy.
|
||||
|
||||

|
||||
|
||||
| Weak receive power on a fibre link | Healthy fibre link |
|
||||
|---|---|
|
||||
|  |  |
|
||||
|
||||
Switch ports are shown too, with PoE draw and the LLDP neighbour on each port:
|
||||
|
||||

|
||||
|
||||
**Network inspector**: chassis diagrams for each UniFi switch. Click a port for its stats, LLDP neighbour and a side-by-side switch/server path check.
|
||||
|
||||

|
||||
|
||||
**One-click diagnostics**: GANDALF asks a PULSE worker to SSH to the host on the other end of the link and collect everything in one pass, then analyzes it: carrier, duplex and speed mismatches, SFP power levels, CRC errors, link flaps, recent kernel events and LLDP validation.
|
||||
|
||||

|
||||
|
||||
**Suppressions**: timed or manual maintenance windows per host, interface or UniFi device, with a full history.
|
||||
|
||||

|
||||
|
||||
**Light theme:**
|
||||
|
||||

|
||||
|
||||
<sub>Screenshots come from a throwaway local instance seeded with fictional data (documentation IP ranges), with PULSE replaced by a small mock. See `scripts/demo/`.</sub>
|
||||
|
||||
## Styling & Layout
|
||||
|
||||
GANDALF uses the **LotusGuild Terminal Design System**. For all styling, component, and layout documentation see:
|
||||
|
||||
@@ -436,6 +436,7 @@ def api_diagnose_start():
|
||||
if pd.get('port_idx') == port_idx:
|
||||
port_data = dict(pd)
|
||||
port_data['name'] = pname
|
||||
port_data['switch_name'] = switch_name
|
||||
break
|
||||
if not port_data:
|
||||
return jsonify({'error': f'Port {port_idx} not found on switch "{switch_name}"'}), 404
|
||||
|
||||
@@ -530,15 +530,17 @@ class DiagnosticsRunner:
|
||||
add(warnings, 'KERNEL_EVENTS',
|
||||
f'{len(err_events)} recent kernel error event(s) for this interface in dmesg')
|
||||
|
||||
# LLDP validation
|
||||
# LLDP validation: the neighbour the *server's* lldpd sees should be the switch
|
||||
# that this port belongs to. (The switch port's own LLDP entry describes the server,
|
||||
# so it must not be compared with the server's view of the switch.)
|
||||
if lldp.get('available'):
|
||||
sw_lldp = switch_port_data.get('lldp') or {}
|
||||
sw_system = (sw_lldp.get('system_name') or '').lower()
|
||||
expected_switch = (switch_port_data.get('switch_name') or '').lower()
|
||||
srv_neighbor = (lldp.get('neighbor_system') or '').lower()
|
||||
if sw_system and srv_neighbor and sw_system not in srv_neighbor and srv_neighbor not in sw_system:
|
||||
names_differ = expected_switch not in srv_neighbor and srv_neighbor not in expected_switch
|
||||
if expected_switch and srv_neighbor and names_differ:
|
||||
add(warnings, 'LLDP_MISMATCH',
|
||||
f'LLDP mismatch: switch sees "{sw_lldp.get("system_name")}" but '
|
||||
f'server lldpctl sees "{lldp.get("neighbor_system")}" — cross-cabled port?')
|
||||
f'LLDP mismatch: this port is on "{switch_port_data.get("switch_name")}" but the server\'s '
|
||||
f'lldpctl sees "{lldp.get("neighbor_system")}" — cross-cabled port?')
|
||||
else:
|
||||
add(info, 'LLDP_MISSING',
|
||||
'lldpd not running on server — install lldpd for full path validation')
|
||||
|
||||
|
After Width: | Height: | Size: 88 KiB |
|
After Width: | Height: | Size: 164 KiB |
|
After Width: | Height: | Size: 674 KiB |
|
After Width: | Height: | Size: 67 KiB |
|
After Width: | Height: | Size: 145 KiB |
|
After Width: | Height: | Size: 162 KiB |
|
After Width: | Height: | Size: 108 KiB |
|
After Width: | Height: | Size: 49 KiB |
|
After Width: | Height: | Size: 71 KiB |
|
After Width: | Height: | Size: 206 KiB |
|
After Width: | Height: | Size: 145 KiB |
|
After Width: | Height: | Size: 214 KiB |
|
After Width: | Height: | Size: 90 KiB |
@@ -0,0 +1,19 @@
|
||||
# Demo environment (for README screenshots)
|
||||
|
||||
Fictional data only (RFC 5737 documentation addresses). A throwaway MariaDB on loopback, the Flask app, and a tiny mock of the PULSE internal API so the **Run Diagnostics** button works.
|
||||
|
||||
```bash
|
||||
D=$(mktemp -d); rsync -a --exclude .git . $D/gandalf && cd $D
|
||||
mariadb-install-db --no-defaults --datadir=$D/db --auth-root-authentication-method=normal --skip-test-db
|
||||
mariadbd --no-defaults --datadir=$D/db --socket=$D/sock --port=3398 --bind-address=127.0.0.1 --user=root &
|
||||
mariadb --no-defaults -S $D/sock -e "CREATE USER 'gandalf'@'127.0.0.1' IDENTIFIED BY 'demo-only'" < /dev/null
|
||||
mariadb --no-defaults -S $D/sock < gandalf/schema.sql
|
||||
mariadb --no-defaults -S $D/sock -e "GRANT ALL ON gandalf.* TO 'gandalf'@'127.0.0.1'"
|
||||
# gandalf/config.json: database -> 127.0.0.1:3398 gandalf/demo-only; pulse.url -> http://127.0.0.1:8650 (api_key/worker_id any value)
|
||||
python3 gandalf/scripts/demo/seed.py # events, suppressions, snapshot and link stats
|
||||
python3 gandalf/scripts/demo/mock_pulse.py & # canned diagnostics output
|
||||
(cd gandalf && gunicorn -b 127.0.0.1:8500 app:app) &
|
||||
python3 gandalf/scripts/demo/capture.py # screenshots + GIF -> docs/img/
|
||||
```
|
||||
|
||||
The app trusts Authelia `Remote-User` / `Remote-Groups` headers, so `capture.py` sends them directly. Requires `playwright` and `pillow`.
|
||||
@@ -0,0 +1,164 @@
|
||||
"""Regenerates docs/img/* from a LOCAL demo instance (fictional data). See README.md in this folder."""
|
||||
|
||||
# flake8: noqa: E501 (demo data and canned output contain long literal lines)
|
||||
|
||||
import asyncio
|
||||
import io
|
||||
import pathlib
|
||||
import sys
|
||||
|
||||
from PIL import Image
|
||||
from playwright.async_api import async_playwright
|
||||
|
||||
OUT = str(pathlib.Path(__file__).resolve().parents[2] / "docs" / "img")
|
||||
B = "http://127.0.0.1:8500"
|
||||
H = {
|
||||
"Remote-User": "alex.rivera",
|
||||
"Remote-Name": "Alex Rivera",
|
||||
"Remote-Email": "alex@example.com",
|
||||
"Remote-Groups": "admin",
|
||||
}
|
||||
TOP = (
|
||||
"document.activeElement&&document.activeElement.blur();"
|
||||
"document.documentElement.style.scrollBehavior='auto';window.scrollTo(0,0)"
|
||||
)
|
||||
AV = (
|
||||
'<svg xmlns="http://www.w3.org/2000/svg" width="64" height="64">'
|
||||
'<rect width="64" height="64" fill="#050a10"/>'
|
||||
'<text x="32" y="40" font-family="monospace" font-size="24" font-weight="700" fill="#00d4ff" text-anchor="middle">AR</text></svg>'
|
||||
)
|
||||
|
||||
|
||||
async def mk(p, theme="dark", w=1440, h=900):
|
||||
b = await p.chromium.launch()
|
||||
c = await b.new_context(viewport={"width": w, "height": h}, extra_http_headers=H)
|
||||
await c.add_init_script(f"try{{localStorage.setItem('lt_theme','{theme}')}}catch(e){{}}")
|
||||
|
||||
async def av(route):
|
||||
await route.fulfill(status=200, content_type="image/svg+xml", body=AV)
|
||||
|
||||
await c.route("**/api/avatar*", av)
|
||||
return b, c
|
||||
|
||||
|
||||
async def go(pg, path, wait=5500):
|
||||
await pg.goto(B + path)
|
||||
await pg.wait_for_timeout(wait)
|
||||
await pg.evaluate(TOP)
|
||||
await pg.wait_for_timeout(300)
|
||||
|
||||
|
||||
async def el(pg, sel, name, idx=0):
|
||||
loc = pg.locator(sel).nth(idx)
|
||||
await loc.scroll_into_view_if_needed()
|
||||
await pg.wait_for_timeout(300)
|
||||
await loc.screenshot(path=f"{OUT}/{name}.png")
|
||||
print(name, "ok")
|
||||
|
||||
|
||||
async def select_port(pg, sw, idx):
|
||||
await pg.locator(f'[data-action="select-port"][data-switch="{sw}"][data-port-idx="{idx}"]').first.click()
|
||||
await pg.wait_for_timeout(1200)
|
||||
|
||||
|
||||
async def shots():
|
||||
async with async_playwright() as p:
|
||||
b, c = await mk(p)
|
||||
pg = await c.new_page()
|
||||
errs = []
|
||||
pg.on("pageerror", lambda e: errs.append(str(e)[:100]))
|
||||
await go(pg, "/")
|
||||
await pg.screenshot(path=f"{OUT}/dashboard.png")
|
||||
print("dashboard ok")
|
||||
await el(pg, "#topology-diagram", "topology")
|
||||
await el(pg, ".lt-section-header >> text=ALERT QUEUE", "alerts-tmp") if False else None
|
||||
# host cards + unifi table via element crops
|
||||
hb = (
|
||||
await pg.locator("#hosts-grid, .host-grid, .hosts-grid").first.bounding_box()
|
||||
if await pg.locator("#hosts-grid, .host-grid, .hosts-grid").count()
|
||||
else None
|
||||
)
|
||||
if hb:
|
||||
await pg.locator("#hosts-grid, .host-grid, .hosts-grid").first.screenshot(path=f"{OUT}/host-cards.png")
|
||||
print("host-cards ok")
|
||||
await go(pg, "/links")
|
||||
await pg.screenshot(path=f"{OUT}/link-debug.png")
|
||||
names = await pg.evaluate("[...document.querySelectorAll('.link-host-name')].map(e=>e.textContent.trim())")
|
||||
print("hosts", names)
|
||||
for nm, fn in (
|
||||
("large1", "link-debug-sfp-warning"),
|
||||
("compute-storage-gpu-01", "link-debug-sfp"),
|
||||
("Pro 24 PoE", "link-debug-switch"),
|
||||
):
|
||||
i = next((k for k, n in enumerate(names) if nm in n), None)
|
||||
if i is not None:
|
||||
await el(pg, ".link-host-panel", fn, i)
|
||||
await go(pg, "/inspector")
|
||||
await pg.screenshot(path=f"{OUT}/inspector.png")
|
||||
await select_port(pg, "USW-Aggregation", 4)
|
||||
await pg.screenshot(path=f"{OUT}/inspector-port.png", full_page=False)
|
||||
await pg.locator('[data-action="run-diagnostic"]').first.click()
|
||||
await pg.wait_for_timeout(9000)
|
||||
await pg.evaluate("document.getElementById('diag-results')?.scrollIntoView({block:'start'})")
|
||||
await pg.wait_for_timeout(600)
|
||||
await pg.screenshot(path=f"{OUT}/inspector-diagnostics.png")
|
||||
print("diagnostics ok")
|
||||
await go(pg, "/suppressions")
|
||||
await pg.screenshot(path=f"{OUT}/suppressions.png", full_page=True)
|
||||
print("errors", errs[:3])
|
||||
await b.close()
|
||||
b, c = await mk(p, "light")
|
||||
pg = await c.new_page()
|
||||
await go(pg, "/")
|
||||
await pg.screenshot(path=f"{OUT}/dashboard-light.png")
|
||||
await b.close()
|
||||
|
||||
|
||||
async def gif():
|
||||
frames = []
|
||||
async with async_playwright() as p:
|
||||
b, c = await mk(p, "dark", 1280, 720)
|
||||
pg = await c.new_page()
|
||||
|
||||
async def snap(n=1, ms=0):
|
||||
if ms:
|
||||
await pg.wait_for_timeout(ms)
|
||||
im = Image.open(io.BytesIO(await pg.screenshot())).convert("RGB").resize((960, 540), Image.LANCZOS)
|
||||
for _ in range(n):
|
||||
frames.append(im)
|
||||
|
||||
await go(pg, "/", 5000)
|
||||
await snap(3)
|
||||
await pg.mouse.wheel(0, 620)
|
||||
await snap(3, 600)
|
||||
await pg.mouse.wheel(0, 700)
|
||||
await snap(2, 600)
|
||||
await go(pg, "/links", 4500)
|
||||
await snap(3)
|
||||
await pg.mouse.wheel(0, 500)
|
||||
await snap(2, 500)
|
||||
await go(pg, "/inspector", 4000)
|
||||
await snap(2)
|
||||
await select_port(pg, "USW-Aggregation", 4)
|
||||
await snap(3)
|
||||
await pg.locator('[data-action="run-diagnostic"]').first.click()
|
||||
await snap(2, 1500)
|
||||
await snap(2, 7500)
|
||||
await pg.evaluate("document.getElementById('diag-results')?.scrollIntoView({block:'start'})")
|
||||
await snap(4, 600)
|
||||
await go(pg, "/suppressions", 4000)
|
||||
await snap(3)
|
||||
await b.close()
|
||||
pal = [f.quantize(colors=96, method=Image.MEDIANCUT, dither=Image.NONE) for f in frames]
|
||||
pal[0].save(f"{OUT}/demo.gif", save_all=True, append_images=pal[1:], duration=1100, loop=0, optimize=True)
|
||||
print("gif", len(frames))
|
||||
|
||||
|
||||
async def main():
|
||||
if "gif" not in sys.argv:
|
||||
await shots()
|
||||
if "shots" not in sys.argv:
|
||||
await gif()
|
||||
|
||||
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,133 @@
|
||||
"""Minimal stand-in for the PULSE internal API, returning canned diagnostics output (fictional data)."""
|
||||
|
||||
# flake8: noqa: E501 (demo data and canned output contain long literal lines)
|
||||
|
||||
import json
|
||||
from http.server import BaseHTTPRequestHandler, HTTPServer
|
||||
|
||||
CANNED = """=== carrier ===
|
||||
1
|
||||
=== operstate ===
|
||||
up
|
||||
=== sysfs_stats ===
|
||||
rx_bytes:9183029113
|
||||
tx_bytes:7732918231
|
||||
rx_errors:14
|
||||
tx_errors:0
|
||||
rx_dropped:0
|
||||
tx_dropped:0
|
||||
rx_crc_errors:14
|
||||
rx_frame_errors:0
|
||||
rx_fifo_errors:0
|
||||
tx_carrier_errors:0
|
||||
collisions:0
|
||||
rx_missed_errors:0
|
||||
=== carrier_changes ===
|
||||
6
|
||||
=== ethtool ===
|
||||
Settings for enp5s0:
|
||||
Supported ports: [ FIBRE ]
|
||||
Supported link modes: 10000baseT/Full
|
||||
Speed: 10000Mb/s
|
||||
Duplex: Full
|
||||
Port: FIBRE
|
||||
PHYAD: 0
|
||||
Auto-negotiation: off
|
||||
Link detected: yes
|
||||
=== ethtool_driver ===
|
||||
driver: ixgbe
|
||||
version: 6.8.12-4-pve
|
||||
firmware-version: 0x800003e1, 1.3429.0
|
||||
bus-info: 0000:05:00.0
|
||||
=== ethtool_pause ===
|
||||
Pause parameters for enp5s0:
|
||||
Autonegotiate: off
|
||||
RX: off
|
||||
TX: off
|
||||
=== ethtool_ring ===
|
||||
Ring parameters for enp5s0:
|
||||
Pre-set maximums:
|
||||
RX: 8192
|
||||
RX Mini: n/a
|
||||
RX Jumbo: n/a
|
||||
TX: 8192
|
||||
Current hardware settings:
|
||||
RX: 512
|
||||
RX Mini: n/a
|
||||
RX Jumbo: n/a
|
||||
TX: 512
|
||||
=== ethtool_stats ===
|
||||
NIC statistics:
|
||||
rx_packets: 183029113
|
||||
tx_packets: 120918231
|
||||
rx_errors: 14
|
||||
tx_errors: 0
|
||||
rx_crc_errors: 14
|
||||
rx_missed_errors: 0
|
||||
tx_timeout_count: 0
|
||||
fdir_match: 41022
|
||||
tx_flow_control_xon: 0
|
||||
rx_flow_control_xon: 0
|
||||
=== ethtool_dom ===
|
||||
Identifier : 0x03 (SFP)
|
||||
Connector : 0x07 (LC)
|
||||
Vendor name : FS
|
||||
Vendor PN : SFP-10GSR-85
|
||||
Laser wavelength : 850nm
|
||||
Laser bias current : 6.100 mA
|
||||
Laser output power : 0.5530 mW / -2.57 dBm
|
||||
Receiver signal average optical power : 0.0120 mW / -19.21 dBm
|
||||
Module temperature : 47.50 degrees C / 117.50 degrees F
|
||||
Module voltage : 3.3120 V
|
||||
=== ip_link ===
|
||||
5: enp5s0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 9000 qdisc mq master vmbr1 state UP mode DEFAULT group default qlen 1000
|
||||
link/ether 02:00:5e:10:00:05 brd ff:ff:ff:ff:ff:ff
|
||||
RX: bytes packets errors dropped missed mcast
|
||||
9183029113 183029113 14 0 0 10234
|
||||
TX: bytes packets errors dropped carrier collsns
|
||||
7732918231 120918231 0 0 0 0
|
||||
=== ip_addr ===
|
||||
5: enp5s0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 9000 qdisc mq master vmbr1 state UP group default qlen 1000
|
||||
link/ether 02:00:5e:10:00:05 brd ff:ff:ff:ff:ff:ff
|
||||
=== ip_route ===
|
||||
=== dmesg ===
|
||||
[ 412.118233] ixgbe 0000:05:00.0 enp5s0: NIC Link is Up 10 Gbps, Flow Control: None
|
||||
[1188221.402917] ixgbe 0000:05:00.0 enp5s0: NIC Link is Down
|
||||
[1188224.190331] ixgbe 0000:05:00.0 enp5s0: NIC Link is Up 10 Gbps, Flow Control: None
|
||||
[2230418.550112] ixgbe 0000:05:00.0 enp5s0: Reset adapter
|
||||
[2230421.118904] ixgbe 0000:05:00.0 enp5s0: NIC Link is Up 10 Gbps, Flow Control: None
|
||||
=== lldpctl ===
|
||||
-------------------------------------------------------------------------------
|
||||
LLDP neighbors:
|
||||
-------------------------------------------------------------------------------
|
||||
Interface: enp5s0, via: LLDP, RID: 1, Time: 12 days, 03:11:42
|
||||
Chassis:
|
||||
ChassisID: mac aa:bb:cc:00:00:03
|
||||
SysName: USW-Aggregation
|
||||
Port:
|
||||
PortID: ifname Port 4
|
||||
=== end ===
|
||||
"""
|
||||
|
||||
|
||||
class H(BaseHTTPRequestHandler):
|
||||
def _send(self, obj, code=200):
|
||||
b = json.dumps(obj).encode()
|
||||
self.send_response(code)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.send_header("Content-Length", str(len(b)))
|
||||
self.end_headers()
|
||||
self.wfile.write(b)
|
||||
|
||||
def do_POST(self):
|
||||
self.rfile.read(int(self.headers.get("Content-Length", 0) or 0))
|
||||
self._send({"execution_id": "demo-exec-0001"})
|
||||
|
||||
def do_GET(self):
|
||||
self._send({"status": "completed", "logs": [{"action": "command_result", "stdout": CANNED}]})
|
||||
|
||||
def log_message(self, *a):
|
||||
pass
|
||||
|
||||
|
||||
HTTPServer(("127.0.0.1", 8650), H).serve_forever()
|
||||
@@ -0,0 +1,425 @@
|
||||
"""Seed a throwaway GANDALF database with fictional data (documentation IP ranges only)."""
|
||||
|
||||
# flake8: noqa: E501 (demo data and canned output contain long literal lines)
|
||||
|
||||
import json
|
||||
import random
|
||||
from datetime import datetime, timedelta, timezone
|
||||
import pymysql
|
||||
|
||||
random.seed(11)
|
||||
conn = pymysql.connect(
|
||||
host="127.0.0.1",
|
||||
port=3398,
|
||||
user="gandalf",
|
||||
password="demo-only",
|
||||
database="gandalf",
|
||||
autocommit=True,
|
||||
charset="utf8mb4",
|
||||
)
|
||||
cur = conn.cursor()
|
||||
now = datetime.now(timezone.utc)
|
||||
|
||||
|
||||
def ts(h):
|
||||
return (now - timedelta(hours=h)).strftime("%Y-%m-%d %H:%M:%S")
|
||||
|
||||
|
||||
def state(k, v):
|
||||
cur.execute(
|
||||
"REPLACE INTO monitor_state (key_name,value) VALUES (%s,%s)", (k, v if isinstance(v, str) else json.dumps(v))
|
||||
)
|
||||
|
||||
|
||||
# ---- hosts / interfaces: (name, ip, [(iface, speed_mbps, up, kind)]) kind: rj45 | fiber | dac
|
||||
HOSTS = [
|
||||
(
|
||||
"compute-storage-01",
|
||||
"192.0.2.4",
|
||||
[("eno1", 1000, True, "rj45"), ("enp1s0f0", 10000, True, "dac"), ("enp1s0f1", 10000, True, "dac")],
|
||||
),
|
||||
("compute-storage-gpu-01", "192.0.2.10", [("enp2s0", 1000, True, "rj45"), ("enp4s0", 10000, True, "fiber")]),
|
||||
("large1", "192.0.2.2", [("enp3s0", 1000, True, "rj45"), ("enp5s0", 10000, True, "fiber")]),
|
||||
(
|
||||
"storage-01",
|
||||
"192.0.2.11",
|
||||
[("enp5s0", 1000, True, "rj45"), ("enp1s0", 10000, False, "fiber")],
|
||||
), # regression: link down
|
||||
(
|
||||
"micro1",
|
||||
"192.0.2.8",
|
||||
[("enp2s0", 1000, True, "rj45"), ("enp3s0", 1000, False, "rj45")],
|
||||
), # unused bond leg (baseline, not alerted)
|
||||
("monitor-02", "192.0.2.9", [("enp2s0", 1000, True, "rj45"), ("enp3s0", 1000, True, "rj45")]),
|
||||
]
|
||||
hosts = {}
|
||||
for name, ip, ifs in HOSTS:
|
||||
st = [u for _, _, u, _ in ifs]
|
||||
status = "up" if all(st) else ("down" if not any(st) else "degraded")
|
||||
hosts[name] = {
|
||||
"ip": ip,
|
||||
"interfaces": {i: ("up" if u else "down") for i, _, u, _ in ifs},
|
||||
"status": status,
|
||||
"source": "prometheus",
|
||||
}
|
||||
hosts["backup-01"] = {"ip": "198.51.100.20", "interfaces": {}, "status": "up", "source": "ping"}
|
||||
unifi = [
|
||||
{
|
||||
"name": "Dream Machine Pro",
|
||||
"mac": "aa:bb:cc:00:00:01",
|
||||
"ip": "192.0.2.1",
|
||||
"type": "udm",
|
||||
"model": "UDMPRO",
|
||||
"state": 1,
|
||||
"connected": True,
|
||||
},
|
||||
{
|
||||
"name": "Pro 24 PoE",
|
||||
"mac": "aa:bb:cc:00:00:02",
|
||||
"ip": "192.0.2.30",
|
||||
"type": "usw",
|
||||
"model": "US24PRO",
|
||||
"state": 1,
|
||||
"connected": True,
|
||||
},
|
||||
{
|
||||
"name": "USW-Aggregation",
|
||||
"mac": "aa:bb:cc:00:00:03",
|
||||
"ip": "192.0.2.31",
|
||||
"type": "usw",
|
||||
"model": "USL8A",
|
||||
"state": 1,
|
||||
"connected": True,
|
||||
},
|
||||
{
|
||||
"name": "UNVR Pro",
|
||||
"mac": "aa:bb:cc:00:00:04",
|
||||
"ip": "192.0.2.40",
|
||||
"type": "unvr",
|
||||
"model": "UNVRPRO",
|
||||
"state": 1,
|
||||
"connected": True,
|
||||
},
|
||||
{
|
||||
"name": "Garage AP",
|
||||
"mac": "aa:bb:cc:00:00:05",
|
||||
"ip": "192.0.2.41",
|
||||
"type": "uap",
|
||||
"model": "U6LR",
|
||||
"state": 0,
|
||||
"connected": False,
|
||||
},
|
||||
{
|
||||
"name": "Office AP",
|
||||
"mac": "aa:bb:cc:00:00:06",
|
||||
"ip": "192.0.2.42",
|
||||
"type": "uap",
|
||||
"model": "U6PRO",
|
||||
"state": 1,
|
||||
"connected": True,
|
||||
},
|
||||
]
|
||||
state("network_snapshot", {"hosts": hosts, "unifi": unifi, "updated": now.isoformat().replace("+00:00", "Z")})
|
||||
state("last_check", now.strftime("%Y-%m-%d %H:%M:%S UTC"))
|
||||
|
||||
|
||||
# ---- per-interface link stats
|
||||
def rate(speed, frac):
|
||||
return speed * 1e6 / 8 * frac
|
||||
|
||||
|
||||
link_hosts = {}
|
||||
for name, ip, ifs in HOSTS:
|
||||
d = {}
|
||||
for iface, speed, up, kind in ifs:
|
||||
e = {"host_ip": ip, "link_detected": up, "carrier_changes": random.choice([1, 2, 2, 3]) if up else 9}
|
||||
if up:
|
||||
f = random.uniform(0.01, 0.18) if speed == 1000 else random.uniform(0.04, 0.38)
|
||||
e.update(
|
||||
{
|
||||
"speed_mbps": speed,
|
||||
"duplex": "Full",
|
||||
"auto_neg": True,
|
||||
"tx_bytes_rate": rate(speed, f),
|
||||
"rx_bytes_rate": rate(speed, f * random.uniform(0.5, 1.4)),
|
||||
"tx_errs_rate": 0.0,
|
||||
"rx_errs_rate": 0.0,
|
||||
"tx_drops_rate": 0.0,
|
||||
"rx_drops_rate": 0.0,
|
||||
}
|
||||
)
|
||||
else:
|
||||
e.update(
|
||||
{
|
||||
"speed_mbps": None,
|
||||
"duplex": None,
|
||||
"auto_neg": True,
|
||||
"tx_bytes_rate": 0,
|
||||
"rx_bytes_rate": 0,
|
||||
"tx_errs_rate": 0,
|
||||
"rx_errs_rate": 0,
|
||||
"tx_drops_rate": 0,
|
||||
"rx_drops_rate": 0,
|
||||
}
|
||||
)
|
||||
if up and kind == "fiber":
|
||||
e["sfp"] = {
|
||||
"sfp_type": "SFP+ 10GBASE-SR",
|
||||
"vendor": "FS",
|
||||
"part_no": "SFP-10GSR-85",
|
||||
"temp_c": round(random.uniform(38, 47), 1),
|
||||
"tx_power_dbm": round(random.uniform(-2.8, -1.6), 2),
|
||||
"rx_power_dbm": round(random.uniform(-4.5, -3.0), 2),
|
||||
}
|
||||
if up and kind == "dac":
|
||||
e["sfp"] = {"sfp_type": "SFP+ passive DAC", "vendor": "Ubiquiti", "part_no": "UACC-DAC-SFP10-1M"}
|
||||
d[iface] = e
|
||||
link_hosts[name] = d
|
||||
# one marginal fibre (low RX power) for the warning colour
|
||||
link_hosts["large1"]["enp5s0"]["sfp"].update({"rx_power_dbm": -10.9, "temp_c": 58.4})
|
||||
link_hosts["large1"]["enp5s0"]["carrier_changes"] = 6
|
||||
|
||||
|
||||
# ---- UniFi switch ports (with LLDP neighbours = the hosts above)
|
||||
def port(idx, up, speed, sw_ip, uplink=False, media="GE", lldp=None, poe=None, frac=0.05):
|
||||
|
||||
def r():
|
||||
return rate(speed, frac * random.uniform(0.5, 1.5)) if up else 0
|
||||
|
||||
p = {
|
||||
"port_idx": idx,
|
||||
"switch_ip": sw_ip,
|
||||
"up": up,
|
||||
"speed_mbps": speed if up else 0,
|
||||
"full_duplex": up,
|
||||
"autoneg": True,
|
||||
"is_uplink": uplink,
|
||||
"media": media,
|
||||
"poe_power": poe[0] if poe else None,
|
||||
"poe_class": poe[1] if poe else None,
|
||||
"poe_max_power": poe[2] if poe else None,
|
||||
"poe_mode": "auto" if poe else "",
|
||||
"lldp": lldp,
|
||||
"tx_bytes": int(random.uniform(1e11, 9e12)) if up else 0,
|
||||
"rx_bytes": int(random.uniform(1e11, 9e12)) if up else 0,
|
||||
"tx_errors": 0,
|
||||
"rx_errors": 0,
|
||||
"tx_dropped": 0,
|
||||
"rx_dropped": 0,
|
||||
"tx_bytes_rate": r(),
|
||||
"rx_bytes_rate": r(),
|
||||
"tx_errs_rate": 0.0,
|
||||
"rx_errs_rate": 0.0,
|
||||
"tx_drops_rate": 0.0,
|
||||
"rx_drops_rate": 0.0,
|
||||
}
|
||||
return p
|
||||
|
||||
|
||||
def L(host, iface):
|
||||
return {
|
||||
"chassis_id": "aa:bb:cc:11:22:33",
|
||||
"system_name": host,
|
||||
"port_id": iface,
|
||||
"port_desc": iface,
|
||||
"mgmt_ips": [dict(HOSTS_IP)[host]] if False else [],
|
||||
}
|
||||
|
||||
|
||||
HOSTS_IP = {n: ip for n, ip, _ in HOSTS}
|
||||
pro = {}
|
||||
mgmt = [
|
||||
(1, "compute-storage-01", "eno1"),
|
||||
(2, "compute-storage-gpu-01", "enp2s0"),
|
||||
(3, "large1", "enp3s0"),
|
||||
(4, "storage-01", "enp5s0"),
|
||||
(5, "micro1", "enp2s0"),
|
||||
(6, "monitor-02", "enp2s0"),
|
||||
(7, "monitor-02", "enp3s0"),
|
||||
]
|
||||
for i in range(1, 25):
|
||||
m = [x for x in mgmt if x[0] == i]
|
||||
if m:
|
||||
pro[f"Port {i}"] = port(i, True, 1000, "192.0.2.30", lldp=L(m[0][1], m[0][2]), frac=0.06)
|
||||
elif i == 8:
|
||||
pro[f"Port {i}"] = port(i, False, 0, "192.0.2.30") # micro1 unused bond leg
|
||||
elif i in (10, 11, 12):
|
||||
pro[f"Port {i}"] = port(
|
||||
i,
|
||||
True,
|
||||
100 if i == 12 else 1000,
|
||||
"192.0.2.30",
|
||||
lldp=None,
|
||||
poe=(round(random.uniform(3, 9), 1), 4, 30.0),
|
||||
frac=0.02,
|
||||
) # APs / cameras on PoE
|
||||
elif i == 9:
|
||||
pro[f"Port {i}"] = port(
|
||||
i,
|
||||
True,
|
||||
1000,
|
||||
"192.0.2.30",
|
||||
lldp={
|
||||
"chassis_id": "aa:bb:cc:00:00:04",
|
||||
"system_name": "UNVR Pro",
|
||||
"port_id": "eth0",
|
||||
"port_desc": "eth0",
|
||||
"mgmt_ips": [],
|
||||
},
|
||||
poe=(0.0, 0, 30.0),
|
||||
frac=0.2,
|
||||
)
|
||||
else:
|
||||
pro[f"Port {i}"] = port(i, False, 0, "192.0.2.30")
|
||||
pro["Port 25"] = port(
|
||||
25,
|
||||
True,
|
||||
10000,
|
||||
"192.0.2.30",
|
||||
uplink=True,
|
||||
media="SFP+",
|
||||
lldp={
|
||||
"chassis_id": "aa:bb:cc:00:00:03",
|
||||
"system_name": "USW-Aggregation",
|
||||
"port_id": "Port 7",
|
||||
"port_desc": "Port 7",
|
||||
"mgmt_ips": [],
|
||||
},
|
||||
frac=0.08,
|
||||
)
|
||||
pro["Port 26"] = port(26, False, 0, "192.0.2.30", media="SFP+")
|
||||
agg = {}
|
||||
aggmap = [
|
||||
(1, "compute-storage-01", "enp1s0f0"),
|
||||
(2, "compute-storage-01", "enp1s0f1"),
|
||||
(3, "compute-storage-gpu-01", "enp4s0"),
|
||||
(4, "large1", "enp5s0"),
|
||||
(5, "storage-01", "enp1s0"),
|
||||
]
|
||||
for i in range(1, 9):
|
||||
m = [x for x in aggmap if x[0] == i]
|
||||
if i == 5:
|
||||
agg[f"Port {i}"] = port(i, False, 0, "192.0.2.31", media="SFP+", lldp=None) # storage-01 10G link is down
|
||||
elif m:
|
||||
agg[f"Port {i}"] = port(i, True, 10000, "192.0.2.31", media="SFP+", lldp=L(m[0][1], m[0][2]), frac=0.15)
|
||||
elif i == 7:
|
||||
agg[f"Port {i}"] = port(
|
||||
i,
|
||||
True,
|
||||
10000,
|
||||
"192.0.2.31",
|
||||
uplink=True,
|
||||
media="SFP+",
|
||||
lldp={
|
||||
"chassis_id": "aa:bb:cc:00:00:02",
|
||||
"system_name": "Pro 24 PoE",
|
||||
"port_id": "Port 25",
|
||||
"port_desc": "Port 25",
|
||||
"mgmt_ips": [],
|
||||
},
|
||||
frac=0.08,
|
||||
)
|
||||
else:
|
||||
agg[f"Port {i}"] = port(i, False, 0, "192.0.2.31", media="SFP+")
|
||||
state(
|
||||
"link_stats",
|
||||
{
|
||||
"hosts": link_hosts,
|
||||
"unifi_switches": {
|
||||
"Pro 24 PoE": {"ip": "192.0.2.30", "model": "US24PRO", "ports": pro},
|
||||
"USW-Aggregation": {"ip": "192.0.2.31", "model": "USL8A", "ports": agg},
|
||||
},
|
||||
"updated": now.strftime("%Y-%m-%d %H:%M:%S UTC"),
|
||||
},
|
||||
)
|
||||
|
||||
# ---- events
|
||||
EV = [ # type, sev, source, target, detail, desc, first_seen_h, last_seen_h, resolved_h|None, fails, ticket
|
||||
(
|
||||
"interface_down",
|
||||
"warning",
|
||||
"prometheus",
|
||||
"storage-01",
|
||||
"enp1s0",
|
||||
"Interface enp1s0 on storage-01 went down (UP -> DOWN regression).",
|
||||
3.2,
|
||||
0.0,
|
||||
None,
|
||||
5,
|
||||
"731905284",
|
||||
),
|
||||
(
|
||||
"unifi_device_offline",
|
||||
"warning",
|
||||
"unifi",
|
||||
"Garage AP",
|
||||
"uap / 192.0.2.41",
|
||||
'UniFi device "Garage AP" has been offline for 3 consecutive checks.',
|
||||
1.1,
|
||||
0.0,
|
||||
None,
|
||||
3,
|
||||
"731905301",
|
||||
),
|
||||
(
|
||||
"interface_down",
|
||||
"warning",
|
||||
"prometheus",
|
||||
"compute-storage-01",
|
||||
"enp1s0f1",
|
||||
"Interface enp1s0f1 on compute-storage-01 went down.",
|
||||
9.0,
|
||||
7.5,
|
||||
7.4,
|
||||
2,
|
||||
None,
|
||||
),
|
||||
(
|
||||
"host_unreachable",
|
||||
"warning",
|
||||
"ping",
|
||||
"backup-01",
|
||||
"198.51.100.20",
|
||||
"Host backup-01 stopped answering ping for 2 consecutive checks.",
|
||||
30.0,
|
||||
28.0,
|
||||
27.9,
|
||||
2,
|
||||
"731881772",
|
||||
),
|
||||
(
|
||||
"unifi_device_offline",
|
||||
"warning",
|
||||
"unifi",
|
||||
"Office AP",
|
||||
"uap / 192.0.2.42",
|
||||
'UniFi device "Office AP" was offline for 3 consecutive checks.',
|
||||
19.0,
|
||||
18.0,
|
||||
17.8,
|
||||
3,
|
||||
"731890014",
|
||||
),
|
||||
]
|
||||
for t, sev, src, tgt, det, desc, f, l, r, fails, tid in EV:
|
||||
cur.execute(
|
||||
"INSERT INTO network_events (event_type,severity,source_type,target_name,target_detail,description,first_seen,last_seen,resolved_at,consecutive_failures,ticket_id) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)",
|
||||
(t, sev, src, tgt, det, desc, ts(f), ts(l), ts(r) if r is not None else None, fails, tid),
|
||||
)
|
||||
# ---- suppressions (one active timed, one manual, one expired in history)
|
||||
SUP = [
|
||||
("interface", "micro1", "enp3s0", "Unused bond leg, cable not connected", "alex.rivera", 48, None, 1),
|
||||
("host", "monitor-02", "", "Planned firmware update window", "alex.rivera", 0.4, 0.6, 1),
|
||||
("unifi_device", "Office AP", "", "Replacing the access point", "sam.chen", 30, 29, 0),
|
||||
]
|
||||
for tt, tn, td, reason, by, created_h, exp_in, active in SUP:
|
||||
exp = (
|
||||
(now + timedelta(hours=exp_in)).strftime("%Y-%m-%d %H:%M:%S")
|
||||
if exp_in and exp_in > 0
|
||||
else (ts(-exp_in) if exp_in else None)
|
||||
)
|
||||
cur.execute(
|
||||
"INSERT INTO suppression_rules (target_type,target_name,target_detail,reason,suppressed_by,created_at,expires_at,active) VALUES (%s,%s,%s,%s,%s,%s,%s,%s)",
|
||||
(tt, tn, td, reason, by, ts(created_h), exp, active),
|
||||
)
|
||||
print("seeded: hosts", len(hosts), "unifi", len(unifi), "events", len(EV), "suppressions", len(SUP))
|
||||
@@ -473,12 +473,27 @@ class TestAnalyze:
|
||||
assert 'LLDP_MISSING' in codes
|
||||
|
||||
def test_lldp_mismatch_is_warning(self):
|
||||
# The server's lldpd sees a different switch than the one this port belongs to.
|
||||
sections = self._sections(lldpctl={'available': True, 'neighbor_system': 'wrong-switch'})
|
||||
switch_data = {'speed_mbps': 1000, 'lldp': {'system_name': 'core-sw-01'}}
|
||||
switch_data = {'speed_mbps': 1000, 'switch_name': 'core-sw-01', 'lldp': {'system_name': 'server-a'}}
|
||||
result = DiagnosticsRunner.analyze(sections, switch_data)
|
||||
codes = [w['code'] for w in result['warnings']]
|
||||
assert 'LLDP_MISMATCH' in codes
|
||||
|
||||
def test_lldp_match_is_not_a_warning(self):
|
||||
# Correct cabling: the switch sees the server, the server sees the switch.
|
||||
sections = self._sections(lldpctl={'available': True, 'neighbor_system': 'core-sw-01'})
|
||||
switch_data = {'speed_mbps': 1000, 'switch_name': 'core-sw-01', 'lldp': {'system_name': 'server-a'}}
|
||||
result = DiagnosticsRunner.analyze(sections, switch_data)
|
||||
codes = [w['code'] for w in result['warnings']]
|
||||
assert 'LLDP_MISMATCH' not in codes
|
||||
|
||||
def test_lldp_without_switch_name_skips_check(self):
|
||||
sections = self._sections(lldpctl={'available': True, 'neighbor_system': 'anything'})
|
||||
result = DiagnosticsRunner.analyze(sections, {'speed_mbps': 1000, 'lldp': {'system_name': 'server-a'}})
|
||||
codes = [w['code'] for w in result['warnings']]
|
||||
assert 'LLDP_MISMATCH' not in codes
|
||||
|
||||
def test_healthy_link_no_issues(self):
|
||||
result = DiagnosticsRunner.analyze(self._sections(), {})
|
||||
assert result['issues'] == []
|
||||
|
||||