Compare commits
119
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
e6649a4c39 | ||
|
|
f0759ed24c | ||
|
|
9d15d4b802 | ||
|
|
6f9787b74a | ||
|
|
032d0db81e | ||
|
|
5fdb77d664 | ||
|
|
d05deb6ccd | ||
|
|
3a9cc472e1 | ||
|
|
54369bf6c3 | ||
|
|
a5fcd619f8 | ||
|
|
1767c12f1b | ||
|
|
6461afe12f | ||
|
|
6473d932a7 | ||
|
|
b31b5dd122 | ||
|
|
1e9ce08dca | ||
|
|
c0f8975dee | ||
|
|
51c18fee5c | ||
|
|
b61501d81d | ||
|
|
3e36dc9505 | ||
|
|
0d4f792a46 | ||
|
|
9dc10422e3 | ||
|
|
903dc960c3 | ||
|
|
6880abdff3 | ||
|
|
a4fd74754c | ||
|
|
5c3db5e934 | ||
|
|
3aff52f972 | ||
|
|
de5677fe4d | ||
|
|
eb9fb3f88f | ||
|
|
44d4a7d1eb | ||
|
|
ee0ea8fd47 | ||
|
|
95ec211037 | ||
|
|
253bc568d6 | ||
|
|
892d1cfe2d | ||
|
|
834acfd393 | ||
|
|
148fc57cc2 | ||
|
|
35d785db64 | ||
|
|
f3753092e0 | ||
|
|
55989f4c50 | ||
|
|
bc1d3fb33b | ||
|
|
3840d2ccb7 | ||
|
|
df7b7e36e5 | ||
|
|
fb6848b7e0 | ||
|
|
34758768a4 | ||
|
|
a7c3f97252 | ||
|
|
4413bb8b4c | ||
|
|
df2a2cc48c | ||
|
|
c51c24bbb5 | ||
|
|
4ea5ae61e4 | ||
|
|
300b1cc68c | ||
|
|
18112541c8 | ||
|
|
468dbda7a1 | ||
|
|
c8e183fc4f | ||
|
|
124a8c4ea2 | ||
|
|
565bd4ee24 | ||
|
|
150300ff91 | ||
|
|
997c626c3c | ||
|
|
c201604dca | ||
|
|
04a5182259 | ||
|
|
4a66c41d18 | ||
|
|
517843886b | ||
|
|
ec1e8055de | ||
|
|
773802815a | ||
|
|
3688cf6d62 | ||
|
|
c8fb605dbb | ||
|
|
2498388715 | ||
|
|
640a3b9993 | ||
|
|
a7b164f340 | ||
|
|
6f79dc905d | ||
|
|
705be36c3f | ||
|
|
9188401374 | ||
|
|
59546cd080 | ||
|
|
6b0ef3fcb6 | ||
|
|
2b1af750f4 | ||
|
|
361e85fd69 | ||
|
|
b3b118395a | ||
|
|
bc2590dc34 | ||
|
|
df460afe68 | ||
|
|
0438bd32ce | ||
|
|
05671360cd | ||
|
|
f3b9d26ec2 | ||
|
|
27a1707b81 | ||
|
|
dfdafc07c4 | ||
|
|
8c72a1494a | ||
|
|
3115b714aa | ||
|
|
4edffca6af | ||
|
|
231de5714b | ||
|
|
bb2f6b96cf | ||
|
|
d37e58af15 | ||
|
|
4f126f1ffb | ||
|
|
071f5e954d | ||
|
|
646a1c3687 | ||
|
|
0894f35cde | ||
|
|
09823f8619 | ||
|
|
62a4ec16a2 | ||
|
|
bd5707160a | ||
|
|
a21c21fbfc | ||
|
|
786c4be2ff | ||
|
|
1a4ffa107d | ||
|
|
637b9876a3 | ||
|
|
d7b7b1d17a | ||
|
|
b2a183b0e2 | ||
|
|
fa7ec4cfe2 | ||
|
|
0b068c824b | ||
|
|
d4a28b37fb | ||
|
|
21bd5d8dbf | ||
|
|
a35f0527e3 | ||
|
|
d671916100 | ||
|
|
c2cfa6370e | ||
|
|
eefa162863 | ||
|
|
8c81c19785 | ||
|
|
f3eab76c63 | ||
|
|
9e091cd8fe | ||
|
|
5173302eb4 | ||
|
|
d4cdad7006 | ||
|
|
8f73514297 | ||
|
|
21dfad35bf | ||
|
|
81ba85845b | ||
|
|
109dff1cd0 | ||
|
|
a746140e8b |
No files matched your search
@@ -0,0 +1,7 @@
|
||||
[run]
|
||||
omit =
|
||||
tests/*
|
||||
*/site-packages/*
|
||||
|
||||
[report]
|
||||
show_missing = True
|
||||
@@ -0,0 +1,7 @@
|
||||
[flake8]
|
||||
max-line-length = 120
|
||||
# E221: multiple spaces before operator — intentional column alignment
|
||||
# E305: two blank lines after function — relaxed for module-level code
|
||||
extend-ignore = E221, E305
|
||||
exclude = __pycache__, .git, node_modules
|
||||
extend-exclude = node_modules
|
||||
@@ -0,0 +1,86 @@
|
||||
name: Lint
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: ["**"]
|
||||
pull_request:
|
||||
branches: ["**"]
|
||||
|
||||
jobs:
|
||||
python-lint:
|
||||
name: Python (flake8)
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
|
||||
- name: Install Python and flake8
|
||||
run: |
|
||||
apt-get update -qq
|
||||
apt-get install -y -qq python3 python3-pip
|
||||
pip3 install flake8
|
||||
|
||||
- name: Run flake8
|
||||
run: flake8 . --exclude=__pycache__,.git
|
||||
|
||||
js-lint:
|
||||
name: JS (eslint)
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
|
||||
- name: Install ESLint
|
||||
run: npm install --save-dev eslint@8
|
||||
|
||||
- name: Run ESLint
|
||||
run: npx eslint --ext .js static/
|
||||
|
||||
notify-failure:
|
||||
name: Notify on failure
|
||||
runs-on: ubuntu-latest
|
||||
needs: [python-lint, js-lint]
|
||||
if: failure() && github.event_name == 'push'
|
||||
steps:
|
||||
- name: Send Matrix alert
|
||||
env:
|
||||
MATRIX_WEBHOOK_URL: ${{ secrets.MATRIX_WEBHOOK_URL }}
|
||||
REPO: ${{ github.repository }}
|
||||
BRANCH: ${{ github.ref_name }}
|
||||
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
|
||||
run: |
|
||||
if [ -z "$MATRIX_WEBHOOK_URL" ] || [ "$MATRIX_WEBHOOK_URL" = "CONFIGURE_ME" ]; then exit 0; fi
|
||||
curl -sf -X POST "$MATRIX_WEBHOOK_URL" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{\"text\":\"CI FAILED: ${REPO} @ ${BRANCH} — ${RUN_URL}\"}"
|
||||
|
||||
deploy:
|
||||
name: Deploy
|
||||
runs-on: ubuntu-latest
|
||||
needs: [python-lint, js-lint]
|
||||
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
|
||||
permissions:
|
||||
contents: write
|
||||
steps:
|
||||
- name: Trigger webhook
|
||||
env:
|
||||
WEBHOOK_SECRET: ${{ secrets.WEBHOOK_SECRET }}
|
||||
GIT_REF: ${{ github.ref }}
|
||||
run: |
|
||||
PAYLOAD="{\"ref\":\"${GIT_REF}\"}"
|
||||
SIG=$(echo -n "$PAYLOAD" | openssl dgst -sha256 -hmac "$WEBHOOK_SECRET" | awk '{print $2}')
|
||||
curl -sf --connect-timeout 10 \
|
||||
-X POST \
|
||||
-H "Content-Type: application/json" \
|
||||
-H "X-Gitea-Signature: ${SIG}" \
|
||||
-d "$PAYLOAD" \
|
||||
"http://10.10.10.61:9000/hooks/gandalf-deploy"
|
||||
|
||||
- name: Tag deployed commit
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
TAG="deploy-$(date -u +%Y.%m.%d)-${{ github.run_number }}"
|
||||
curl -sf -X POST \
|
||||
-H "Authorization: token $GITHUB_TOKEN" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{\"tag_name\":\"${TAG}\",\"target\":\"${{ github.sha }}\",\"message\":\"Deployed to production\"}" \
|
||||
"https://code.lotusguild.org/api/v1/repos/${{ github.repository }}/tags"
|
||||
@@ -0,0 +1,25 @@
|
||||
name: Security
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: ["**"]
|
||||
pull_request:
|
||||
branches: ["**"]
|
||||
schedule:
|
||||
- cron: '0 6 * * 1'
|
||||
|
||||
jobs:
|
||||
bandit:
|
||||
name: Python Security (bandit)
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
|
||||
- name: Install bandit
|
||||
run: |
|
||||
apt-get update -qq
|
||||
apt-get install -y -qq python3 python3-pip
|
||||
pip3 install bandit
|
||||
|
||||
- name: Run bandit
|
||||
run: bandit -r . --exclude .git,__pycache__,node_modules -ll
|
||||
@@ -0,0 +1,24 @@
|
||||
name: Test
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: ["**"]
|
||||
pull_request:
|
||||
branches: ["**"]
|
||||
|
||||
jobs:
|
||||
pytest:
|
||||
name: Python Tests (pytest)
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
|
||||
- name: Install Python and dependencies
|
||||
run: |
|
||||
apt-get update -qq
|
||||
apt-get install -y -qq python3 python3-pip
|
||||
pip3 install pytest pytest-cov
|
||||
pip3 install -r requirements.txt --quiet
|
||||
|
||||
- name: Run pytest with coverage
|
||||
run: python3 -m pytest tests/ -v --cov=. --cov-report=term-missing --cov-config=.coveragerc
|
||||
@@ -0,0 +1,5 @@
|
||||
log.txt
|
||||
config.json
|
||||
__pycache__/
|
||||
*.pyc
|
||||
node_modules/
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 239 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 156 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 106 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 217 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 153 KiB |
@@ -0,0 +1,509 @@
|
||||
# GANDALF (Global Advanced Network Detection And Link Facilitator)
|
||||
|
||||
[](https://code.lotusguild.org/LotusGuild/gandalf/actions?workflow=lint.yml)
|
||||
[](https://code.lotusguild.org/LotusGuild/gandalf/actions?workflow=test.yml)
|
||||
[](https://code.lotusguild.org/LotusGuild/gandalf/actions?workflow=security.yml)
|
||||
|
||||
> Because it shall not let problems pass.
|
||||
|
||||
Network monitoring dashboard for the LotusGuild Proxmox cluster.
|
||||
Deployed on **LXC 157** (monitor-02 / 10.10.10.9), reachable at `gandalf.lotusguild.org`.
|
||||
|
||||
**Design System**: [web_template](https://code.lotusguild.org/LotusGuild/web_template) — shared CSS, JS, and layout patterns for all LotusGuild apps
|
||||
|
||||
## Styling & Layout
|
||||
|
||||
GANDALF uses the **LotusGuild Terminal Design System**. For all styling, component, and layout documentation see:
|
||||
|
||||
- [`web_template/README.md`](https://code.lotusguild.org/LotusGuild/web_template/src/branch/main/README.md) — full component reference, CSS variables, JS API
|
||||
- [`web_template/base.css`](https://code.lotusguild.org/LotusGuild/web_template/src/branch/main/base.css) — unified CSS (`.lt-*` classes)
|
||||
- [`web_template/base.js`](https://code.lotusguild.org/LotusGuild/web_template/src/branch/main/base.js) — `window.lt` utilities (toast, modal, auto-refresh, fetch helpers)
|
||||
- [`web_template/python/base.html`](https://code.lotusguild.org/LotusGuild/web_template/src/branch/main/python/base.html) — Jinja2 base template
|
||||
- [`web_template/python/auth.py`](https://code.lotusguild.org/LotusGuild/web_template/src/branch/main/python/auth.py) — `@require_auth` decorator pattern
|
||||
|
||||
---
|
||||
|
||||
## Architecture
|
||||
|
||||
Two processes share a MariaDB database:
|
||||
|
||||
| Process | Service | Role |
|
||||
|---|---|---|
|
||||
| `app.py` | `gandalf.service` | Flask web dashboard (gunicorn, port 8000) |
|
||||
| `monitor.py` | `gandalf-monitor.service` | Background polling daemon |
|
||||
|
||||
```
|
||||
[Prometheus :9090] ──▶
|
||||
[UniFi Controller] ──▶ monitor.py ──▶ MariaDB ◀── app.py ──▶ nginx ──▶ Authelia ──▶ Browser
|
||||
[Pulse Worker] ──▶
|
||||
[SSH / ethtool] ──▶
|
||||
```
|
||||
|
||||
### Data Sources
|
||||
|
||||
| Source | What it provides |
|
||||
|---|---|
|
||||
| **Prometheus** (`10.10.10.48:9090`) | Physical NIC link state + traffic/error rates via `node_exporter` |
|
||||
| **UniFi API** (`https://10.10.10.1`) | Switch port stats, device status, LLDP neighbor table, PoE data |
|
||||
| **Pulse Worker** | SSH relay — runs `ethtool` + SFP DOM queries on each Proxmox host |
|
||||
| **Ping** | Reachability for hosts without `node_exporter` (e.g. PBS) |
|
||||
|
||||
### Monitored Hosts (Prometheus / node_exporter)
|
||||
|
||||
| Host | Prometheus Instance |
|
||||
|---|---|
|
||||
| large1 | 10.10.10.2:9100 |
|
||||
| compute-storage-01 | 10.10.10.4:9100 |
|
||||
| micro1 | 10.10.10.8:9100 |
|
||||
| monitor-02 | 10.10.10.9:9100 |
|
||||
| compute-storage-gpu-01 | 10.10.10.10:9100 |
|
||||
| storage-01 | 10.10.10.11:9100 |
|
||||
|
||||
Ping-only (no node_exporter): **pbs** (10.10.10.3)
|
||||
|
||||
---
|
||||
|
||||
## Pages
|
||||
|
||||
### Dashboard (`/`)
|
||||
- Real-time host status grid with per-NIC link state (UP / DOWN / degraded)
|
||||
- Network topology diagram (Internet → Gateway → Switches → Hosts)
|
||||
- UniFi device table (switches, APs, gateway)
|
||||
- Active alerts table with severity, target, consecutive failures, ticket link
|
||||
- Quick-suppress modal: apply timed or manual suppression from any alert row
|
||||
- Auto-refreshes every 30 seconds via `/api/status` + `/api/network`
|
||||
|
||||
### Link Debug (`/links`)
|
||||
Per-interface statistics collected every poll cycle. All panels are collapsible
|
||||
(click header or use Collapse All / Expand All). Collapse state persists across
|
||||
page refreshes via `sessionStorage`.
|
||||
|
||||
**Server NICs** (via Prometheus + SSH/ethtool):
|
||||
- Speed, duplex, auto-negotiation, link detected
|
||||
- TX/RX rate bars (bandwidth utilisation % of link capacity)
|
||||
- TX/RX error and drop rates per second
|
||||
- Carrier changes (cumulative since boot — watch for flapping)
|
||||
- **SFP / Optical panel** (when SFP module present): vendor/PN, temp, voltage,
|
||||
bias current, TX power (dBm), RX power (dBm), RX−TX delta, per-stat bars
|
||||
|
||||
**UniFi Switch Ports** (via UniFi API):
|
||||
- Port number badge (`#N`), UPLINK badge, PoE draw badge
|
||||
- LLDP neighbor line: `→ system_name (port_id)` when neighbor is detected
|
||||
- PoE class and max wattage line
|
||||
- Speed, duplex, auto-neg, TX/RX rates, errors, drops
|
||||
|
||||
### Inspector (`/inspector`)
|
||||
Visual switch chassis diagrams. Each switch is rendered model-accurately using
|
||||
layout config in the template (`SWITCH_LAYOUTS`).
|
||||
|
||||
**Port block colours:**
|
||||
| Colour | State |
|
||||
|---|---|
|
||||
| Green | Up, no active PoE |
|
||||
| Amber | Up with active PoE draw |
|
||||
| Cyan | Uplink port (up) |
|
||||
| Grey | Down |
|
||||
| White outline | Currently selected |
|
||||
|
||||
**Clicking a port** opens the right-side detail panel showing:
|
||||
- Link stats (status, speed, duplex, auto-neg, media type)
|
||||
- PoE (class, max wattage, current draw, mode)
|
||||
- Traffic (TX/RX rates)
|
||||
- Errors/drops per second
|
||||
- **LLDP Neighbor** section (system name, port ID, chassis ID, management IPs)
|
||||
- **Path Debug** (auto-appears when LLDP `system_name` matches a known server):
|
||||
two-column comparison of the switch port stats vs. the server NIC stats,
|
||||
including SFP DOM data if the server side has an SFP module
|
||||
|
||||
**LLDP path debug requirements:**
|
||||
1. Server must run `lldpd`: `apt install lldpd && systemctl enable --now lldpd`
|
||||
2. `lldpd` hostname must match the key in `data.hosts` (set via `config.json → hosts`)
|
||||
3. Switch has LLDP enabled (UniFi default: on)
|
||||
|
||||
**Supported switch models** (set `SWITCH_LAYOUTS` keys to your UniFi model codes):
|
||||
|
||||
| Key | Model | Layout |
|
||||
|---|---|---|
|
||||
| `USF5P` | UniFi Switch Flex 5 PoE | 4×RJ45 + 1×SFP uplink |
|
||||
| `USL8A` | UniFi Switch Lite 8 PoE | 8×SFP (2 rows of 4) |
|
||||
| `US24PRO` | UniFi Switch Pro 24 | 24×RJ45 staggered + 2×SFP |
|
||||
| `USPPDUP` | Custom/other | Single-port fallback |
|
||||
| `USMINI` | UniFi Switch Mini | 5-port row |
|
||||
|
||||
Add new layouts by adding a key to `SWITCH_LAYOUTS` matching the `model` field
|
||||
returned by the UniFi API for that device.
|
||||
|
||||
### Suppressions (`/suppressions`)
|
||||
- Create timed (30 min / 1 hr / 4 hr / 8 hr) or manual suppressions
|
||||
- Target types: host, interface, UniFi device, or global
|
||||
- Active suppressions table with one-click removal
|
||||
- Suppression history (last 50)
|
||||
- Available targets reference grid (all known hosts + interfaces)
|
||||
|
||||
---
|
||||
|
||||
## Alert Logic
|
||||
|
||||
### Ticket Triggers
|
||||
|
||||
| Condition | Priority |
|
||||
|---|---|
|
||||
| UniFi device offline (≥2 consecutive checks) | P2 High |
|
||||
| Proxmox host NIC link-down regression (≥2 consecutive checks) | P2 High |
|
||||
| Host unreachable via ping (≥2 consecutive checks) | P2 High |
|
||||
| ≥3 hosts simultaneously reporting interface failures | P1 Critical |
|
||||
|
||||
### Baseline Tracking
|
||||
|
||||
Interfaces that are **down on first observation** (unused ports, unplugged cables)
|
||||
are recorded as `initial_down` and never alerted. Only **UP→DOWN regressions**
|
||||
generate tickets. Baseline is stored in MariaDB and survives daemon restarts.
|
||||
|
||||
### Suppression Targets
|
||||
|
||||
| Type | Suppresses |
|
||||
|---|---|
|
||||
| `host` | All interface alerts for a named host |
|
||||
| `interface` | A specific NIC on a specific host |
|
||||
| `unifi_device` | A specific UniFi device |
|
||||
| `all` | Everything (global maintenance mode) |
|
||||
|
||||
Suppressions can be manual (persist until removed) or timed (auto-expire).
|
||||
Expired suppressions are checked at evaluation time — no background cleanup needed.
|
||||
|
||||
---
|
||||
|
||||
## Configuration (`config.json`)
|
||||
|
||||
Shared by both processes. Located in the working directory (`/var/www/html/prod/`).
|
||||
|
||||
```json
|
||||
{
|
||||
"database": {
|
||||
"host": "10.10.10.50",
|
||||
"port": 3306,
|
||||
"user": "gandalf",
|
||||
"password": "...",
|
||||
"name": "gandalf"
|
||||
},
|
||||
"prometheus": {
|
||||
"url": "http://10.10.10.48:9090"
|
||||
},
|
||||
"unifi": {
|
||||
"controller": "https://10.10.10.1",
|
||||
"api_key": "...",
|
||||
"site_id": "default"
|
||||
},
|
||||
"ticket_api": {
|
||||
"url": "https://t.lotusguild.org/api/tickets",
|
||||
"api_key": "..."
|
||||
},
|
||||
"pulse": {
|
||||
"url": "http://<pulse-host>:<port>",
|
||||
"api_key": "...",
|
||||
"worker_id": "...",
|
||||
"timeout": 45
|
||||
},
|
||||
"auth": {
|
||||
"allowed_groups": ["admin"]
|
||||
},
|
||||
"hosts": [
|
||||
{ "name": "large1", "prometheus_instance": "10.10.10.2:9100" },
|
||||
{ "name": "compute-storage-01", "prometheus_instance": "10.10.10.4:9100" },
|
||||
{ "name": "micro1", "prometheus_instance": "10.10.10.8:9100" },
|
||||
{ "name": "monitor-02", "prometheus_instance": "10.10.10.9:9100" },
|
||||
{ "name": "compute-storage-gpu-01", "prometheus_instance": "10.10.10.10:9100" },
|
||||
{ "name": "storage-01", "prometheus_instance": "10.10.10.11:9100" }
|
||||
],
|
||||
"monitor": {
|
||||
"poll_interval": 120,
|
||||
"failure_threshold": 2,
|
||||
"cluster_threshold": 3,
|
||||
"ping_hosts": [
|
||||
{ "name": "pbs", "ip": "10.10.10.3" }
|
||||
]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Key Config Fields
|
||||
|
||||
| Key | Description |
|
||||
|---|---|
|
||||
| `database.*` | MariaDB credentials (LXC 149 at 10.10.10.50) |
|
||||
| `prometheus.url` | Prometheus base URL |
|
||||
| `unifi.controller` | UniFi controller base URL (HTTPS, self-signed cert ignored) |
|
||||
| `unifi.api_key` | UniFi API key from controller Settings → API |
|
||||
| `unifi.site_id` | UniFi site ID (default: `default`) |
|
||||
| `ticket_api.api_key` | Tinker Tickets bearer token |
|
||||
| `pulse.url` | Pulse worker API base URL (for SSH relay) |
|
||||
| `pulse.worker_id` | Which Pulse worker runs ethtool collection |
|
||||
| `pulse.timeout` | Max seconds to wait for SSH collection per host |
|
||||
| `auth.allowed_groups` | Authelia groups that may access Gandalf |
|
||||
| `hosts` | Maps Prometheus instance labels → display hostnames |
|
||||
| `monitor.poll_interval` | Seconds between full check cycles (default: 120) |
|
||||
| `monitor.failure_threshold` | Consecutive failures before creating ticket (default: 2) |
|
||||
| `monitor.cluster_threshold` | Hosts with failures to trigger cluster-wide P1 (default: 3) |
|
||||
| `monitor.ping_hosts` | Hosts checked only by ping (no node_exporter) |
|
||||
|
||||
---
|
||||
|
||||
## Deployment (LXC 157)
|
||||
|
||||
### 1. Database — MariaDB LXC 149 (`10.10.10.50`)
|
||||
|
||||
```sql
|
||||
CREATE DATABASE gandalf CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
|
||||
CREATE USER 'gandalf'@'10.10.10.61' IDENTIFIED BY 'your_password';
|
||||
GRANT ALL PRIVILEGES ON gandalf.* TO 'gandalf'@'10.10.10.61';
|
||||
FLUSH PRIVILEGES;
|
||||
```
|
||||
|
||||
Import schema:
|
||||
```bash
|
||||
mysql -h 10.10.10.50 -u gandalf -p gandalf < schema.sql
|
||||
```
|
||||
|
||||
### 2. LXC 157 — Install dependencies
|
||||
|
||||
```bash
|
||||
pip3 install -r requirements.txt
|
||||
# Ensure sshpass is available (used by deploy scripts)
|
||||
apt install sshpass
|
||||
```
|
||||
|
||||
### 3. Deploy files
|
||||
|
||||
```bash
|
||||
# From dev machine / root/code/gandalf:
|
||||
for f in app.py db.py monitor.py config.json schema.sql \
|
||||
static/style.css static/app.js \
|
||||
templates/*.html; do
|
||||
sshpass -p 'yourpass' scp -o StrictHostKeyChecking=no \
|
||||
"$f" "root@10.10.10.61:/var/www/html/prod/$f"
|
||||
done
|
||||
systemctl restart gandalf gandalf-monitor
|
||||
```
|
||||
|
||||
### 4. systemd services
|
||||
|
||||
**`gandalf.service`** (Flask/gunicorn web app):
|
||||
```ini
|
||||
[Unit]
|
||||
Description=Gandalf Web Dashboard
|
||||
After=network.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
WorkingDirectory=/var/www/html/prod
|
||||
ExecStart=/usr/bin/python3 -m gunicorn --workers 1 --bind 127.0.0.1:8000 app:app
|
||||
Restart=always
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
```
|
||||
|
||||
**`gandalf-monitor.service`** (background polling daemon):
|
||||
```ini
|
||||
[Unit]
|
||||
Description=Gandalf Network Monitor Daemon
|
||||
After=network.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
WorkingDirectory=/var/www/html/prod
|
||||
ExecStart=/usr/bin/python3 monitor.py
|
||||
Restart=always
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
```
|
||||
|
||||
### 5. Authelia rule (LXC 167)
|
||||
|
||||
```yaml
|
||||
access_control:
|
||||
rules:
|
||||
- domain: gandalf.lotusguild.org
|
||||
policy: one_factor
|
||||
subject:
|
||||
- group:admin
|
||||
```
|
||||
|
||||
```bash
|
||||
systemctl restart authelia
|
||||
```
|
||||
|
||||
### 6. NPM reverse proxy
|
||||
|
||||
- **Domain:** `gandalf.lotusguild.org`
|
||||
- **Forward to:** `http://10.10.10.61:8000` (gunicorn direct, no nginx needed on LXC)
|
||||
- **Forward Auth:** Authelia at `http://10.10.10.167:9091`
|
||||
- **WebSockets:** Not required
|
||||
|
||||
---
|
||||
|
||||
## Service Management
|
||||
|
||||
```bash
|
||||
# Status
|
||||
systemctl status gandalf gandalf-monitor
|
||||
|
||||
# Logs (live)
|
||||
journalctl -u gandalf -f
|
||||
journalctl -u gandalf-monitor -f
|
||||
|
||||
# Restart after code or config changes
|
||||
systemctl restart gandalf gandalf-monitor
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Monitor not creating tickets
|
||||
- Verify `config.json → ticket_api.api_key` is set and valid
|
||||
- Check `journalctl -u gandalf-monitor` for `Ticket creation failed` lines
|
||||
- Confirm the Tinker Tickets API is reachable from LXC 157
|
||||
|
||||
### Link Debug shows no data / "Loading…" forever
|
||||
- Check `gandalf-monitor.service` is running and has completed at least one cycle
|
||||
- Check `journalctl -u gandalf-monitor` for Prometheus or UniFi errors
|
||||
- Verify Prometheus is reachable: `curl http://10.10.10.48:9090/api/v1/query?query=up`
|
||||
|
||||
### Link Debug: SFP DOM panel missing
|
||||
- SFP data requires Pulse worker + SSH access to hosts
|
||||
- Verify `config.json → pulse.*` is configured and the Pulse worker is running
|
||||
- Confirm `sshpass` + SSH access from the Pulse worker to each Proxmox host
|
||||
- Only interfaces with physical SFP modules return DOM data (`ethtool -m`)
|
||||
|
||||
### Inspector: path debug section not appearing
|
||||
- Requires LLDP: run `apt install lldpd && systemctl enable --now lldpd` on each server
|
||||
- The LLDP `system_name` broadcast by `lldpd` must match the hostname in `config.json → hosts[].name`
|
||||
- Override: `echo 'configure system hostname large1' > /etc/lldpd.d/hostname.conf && systemctl restart lldpd`
|
||||
- Allow up to 2 poll cycles (240s) after installing lldpd for LLDP table to populate
|
||||
|
||||
### Inspector: switch chassis shows as flat list (no layout)
|
||||
- The switch's `model` field from UniFi doesn't match any key in `SWITCH_LAYOUTS` in `inspector.html`
|
||||
- Check the UniFi API: the model appears in the `link_stats` API response under `unifi_switches.<name>.model`
|
||||
- Add the model key to `SWITCH_LAYOUTS` in `inspector.html` with the correct row/SFP layout
|
||||
|
||||
### Baseline re-initializing on every restart
|
||||
- `interface_baseline` is stored in the `monitor_state` DB table; survives restarts
|
||||
- If it appears to reset: check DB connectivity from the monitor daemon
|
||||
|
||||
### Interface stuck at "initial_down" forever
|
||||
- This means the interface was down when the monitor first saw it
|
||||
- It will begin tracking once it comes up; or manually clear it:
|
||||
```sql
|
||||
-- In MariaDB on 10.10.10.50:
|
||||
UPDATE monitor_state SET value='{}' WHERE key_name='interface_baseline';
|
||||
```
|
||||
Then restart the monitor: `systemctl restart gandalf-monitor`
|
||||
|
||||
### Prometheus data missing for a host
|
||||
```bash
|
||||
# On the affected host:
|
||||
systemctl status prometheus-node-exporter
|
||||
# Verify it's scraped:
|
||||
curl http://10.10.10.48:9090/api/v1/query?query=up | jq '.data.result[] | select(.metric.job=="node")'
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Development Notes
|
||||
|
||||
### File Layout
|
||||
|
||||
```
|
||||
gandalf/
|
||||
├── app.py # Flask web app (routes, auth, API endpoints)
|
||||
├── monitor.py # Background daemon (Prometheus, UniFi, Pulse, alert logic)
|
||||
├── db.py # Database operations (MariaDB via pymysql, thread-local conn reuse)
|
||||
├── schema.sql # Database schema (network_events, suppression_rules, monitor_state)
|
||||
├── config.json # Runtime configuration (not committed with secrets)
|
||||
├── requirements.txt # Python dependencies
|
||||
├── static/
|
||||
│ ├── style.css # Terminal aesthetic CSS (CRT scanlines, green-on-black)
|
||||
│ └── app.js # Dashboard JS (auto-refresh, host grid, events, suppress modal)
|
||||
└── templates/
|
||||
├── base.html # Shared layout (header, nav, footer)
|
||||
├── index.html # Dashboard page
|
||||
├── links.html # Link Debug page (server NICs + UniFi switch ports)
|
||||
├── inspector.html # Visual switch inspector + LLDP path debug
|
||||
└── suppressions.html # Suppression management page
|
||||
```
|
||||
|
||||
### Adding a New Monitored Host
|
||||
|
||||
1. Install `prometheus-node-exporter` on the host
|
||||
2. Add a scrape target to Prometheus config
|
||||
3. Add an entry to `config.json → hosts`:
|
||||
```json
|
||||
{ "name": "newhost", "prometheus_instance": "10.10.10.X:9100" }
|
||||
```
|
||||
4. Restart monitor: `systemctl restart gandalf-monitor`
|
||||
5. For SFP DOM / ethtool: ensure the host is SSH-accessible from the Pulse worker
|
||||
|
||||
### Adding a New Switch Layout (Inspector)
|
||||
|
||||
Find the UniFi model code for the switch (it appears in the `/api/links` JSON response
|
||||
under `unifi_switches.<switch_name>.model`), then add to `SWITCH_LAYOUTS` in
|
||||
`templates/inspector.html`:
|
||||
|
||||
```javascript
|
||||
'MYNEWMODEL': {
|
||||
rows: [[1,2,3,4,5,6,7,8], [9,10,11,12,13,14,15,16]], // port_idx by row
|
||||
sfp_section: [17, 18], // separate SFP cage ports (rendered below rows)
|
||||
sfp_ports: [], // port_idx values that are SFP-type within rows
|
||||
},
|
||||
```
|
||||
|
||||
### Database Schema Notes
|
||||
|
||||
- `network_events`: one row per active event; `resolved_at` is set when recovered
|
||||
- `suppression_rules`: `active=FALSE` when removed; `expires_at` checked at query time
|
||||
- `monitor_state`: key/value store; `interface_baseline` and `link_stats` are JSON blobs
|
||||
|
||||
### Security Notes
|
||||
|
||||
- **XSS prevention**: all user-controlled data in dynamically generated HTML uses
|
||||
`escHtml()` (JS) or Jinja2 auto-escaping (Python). Suppress buttons use `data-*`
|
||||
attributes + a single delegated click listener rather than inline `onclick` with
|
||||
interpolated strings.
|
||||
- **Interface name validation**: `monitor.py` validates SSH interface names against
|
||||
`^[a-zA-Z0-9_.@-]+$` before use, and additionally wraps them with `shlex.quote()`
|
||||
for defense-in-depth.
|
||||
- **DB parameters**: all SQL uses parameterised queries via pymysql — no string
|
||||
concatenation into SQL.
|
||||
- **Auth**: Authelia enforces admin-only access at the nginx/LXC 167 layer; the Flask
|
||||
app additionally checks the `Remote-User` header via `@require_auth`.
|
||||
|
||||
### Known Limitations
|
||||
|
||||
- Single gunicorn worker (`--workers 1`) — required because `db.py` uses thread-local
|
||||
connection reuse (one connection per thread). Multiple workers would each have their
|
||||
own connection, which is fine, but the thread-local optimisation only helps within
|
||||
one worker.
|
||||
- No CSRF tokens on API endpoints — mitigated by Authelia session cookies being
|
||||
`SameSite=Strict` and the site being admin-only.
|
||||
- SSH collection via Pulse is synchronous — if Pulse is slow, the entire monitor cycle
|
||||
is delayed. The `pulse.timeout` config controls the max wait.
|
||||
- UniFi LLDP data is only as fresh as the last monitor poll (120s default).
|
||||
|
||||
---
|
||||
|
||||
## CI / CD
|
||||
|
||||
| Workflow | Purpose | Triggers |
|
||||
|---|---|---|
|
||||
| `lint.yml` (python-lint) | flake8 on all `.py` files | Every push and PR |
|
||||
| `lint.yml` (js-lint) | ESLint on `static/` | Every push and PR |
|
||||
| `test.yml` | pytest — 33 tests for `diagnose.py` static methods | Every push and PR |
|
||||
| `security.yml` | bandit `-ll` (medium+ severity) | Every push, PR, and weekly Monday 6am |
|
||||
| `deploy` job in `lint.yml` | Calls the `gandalf-deploy` webhook on CT157 (10.10.10.61) | Push to `main` only, after both lint jobs pass |
|
||||
|
||||
Branch protection is enabled on `main` — both lint jobs must pass before any PR can merge.
|
||||
|
||||
Tests live in `tests/test_diagnose.py` and cover `DiagnosticsRunner` static methods:
|
||||
`build_ssh_command`, `parse_output`, `parse_sysfs_stats`, `parse_ethtool`, and variants.
|
||||
@@ -0,0 +1,559 @@
|
||||
"""Gandalf – Global Advanced Network Detection And Link Facilitator.
|
||||
|
||||
Flask web application serving the monitoring dashboard and suppression
|
||||
management UI. Authentication via Authelia forward-auth headers.
|
||||
All monitoring and alerting is handled by the separate monitor.py daemon.
|
||||
"""
|
||||
import hashlib
|
||||
import ipaddress
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
from functools import wraps
|
||||
|
||||
from flask import Flask, jsonify, make_response, render_template, request, send_file
|
||||
|
||||
import db
|
||||
import diagnose
|
||||
from monitor import PulseClient
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format='%(asctime)s %(levelname)s %(name)s %(message)s',
|
||||
)
|
||||
logger = logging.getLogger('gandalf.web')
|
||||
|
||||
app = Flask(__name__)
|
||||
|
||||
_AVATAR_COLORS = ['lt-avatar--orange', 'lt-avatar--green', 'lt-avatar--purple', '']
|
||||
|
||||
|
||||
@app.template_filter('avatar_color')
|
||||
def avatar_color_filter(name: str) -> str:
|
||||
return _AVATAR_COLORS[int(hashlib.md5(name.encode()).hexdigest(), 16) % len(_AVATAR_COLORS)] # nosec B324
|
||||
|
||||
_cfg = None
|
||||
_cfg_lock = threading.Lock()
|
||||
|
||||
|
||||
@app.context_processor
|
||||
def inject_config():
|
||||
"""Inject safe config values into all templates."""
|
||||
cfg = _config()
|
||||
return {
|
||||
'config': {
|
||||
'ticket_api': {
|
||||
'web_url': cfg.get('ticket_api', {}).get('web_url', 'http://t.lotusguild.org/ticket/'),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
# In-memory diagnostic job store { job_id: { status, result, created_at } }
|
||||
_diag_jobs: dict = {}
|
||||
_diag_lock = threading.Lock()
|
||||
|
||||
|
||||
def _purge_old_jobs_loop():
|
||||
"""Background thread: remove stale diag jobs and run daily event purge."""
|
||||
while True:
|
||||
time.sleep(120)
|
||||
cutoff = time.time() - 600
|
||||
stuck_cutoff = time.time() - 300 # 5 min: job still 'running' → thread must have crashed
|
||||
with _diag_lock:
|
||||
stale = [jid for jid, j in _diag_jobs.items() if j.get('created_at', 0) < cutoff]
|
||||
for jid in stale:
|
||||
del _diag_jobs[jid]
|
||||
for jid, j in list(_diag_jobs.items()):
|
||||
if j['status'] == 'running' and j.get('created_at', 0) < stuck_cutoff:
|
||||
j['status'] = 'done'
|
||||
j['result'] = {'status': 'error', 'error': 'Diagnostic timed out (thread crash)'}
|
||||
logger.error(f'Diagnostic job {jid} appeared stuck; marked as errored')
|
||||
|
||||
|
||||
_purge_thread = threading.Thread(target=_purge_old_jobs_loop, daemon=True)
|
||||
_purge_thread.start()
|
||||
|
||||
|
||||
def _config() -> dict:
|
||||
global _cfg
|
||||
if _cfg is None:
|
||||
with _cfg_lock:
|
||||
if _cfg is None:
|
||||
with open('config.json') as f:
|
||||
_cfg = json.load(f)
|
||||
return _cfg
|
||||
|
||||
|
||||
def _daemon_ok(last_check: str) -> bool:
|
||||
"""Return True if monitor last checked within 20 minutes."""
|
||||
if not last_check or last_check == 'Never':
|
||||
return False
|
||||
try:
|
||||
ts = datetime.strptime(last_check, '%Y-%m-%d %H:%M:%S UTC').replace(tzinfo=timezone.utc)
|
||||
return (datetime.now(timezone.utc) - ts).total_seconds() < 1200
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Auth helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _get_user() -> dict:
|
||||
return {
|
||||
'username': request.headers.get('Remote-User', ''),
|
||||
'name': request.headers.get('Remote-Name', ''),
|
||||
'email': request.headers.get('Remote-Email', ''),
|
||||
'groups': [
|
||||
g.strip()
|
||||
for g in request.headers.get('Remote-Groups', '').split(',')
|
||||
if g.strip()
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def require_auth(f):
|
||||
@wraps(f)
|
||||
def wrapper(*args, **kwargs):
|
||||
user = _get_user()
|
||||
if not user['username']:
|
||||
return (
|
||||
'<h1>401 – Not authenticated</h1>'
|
||||
'<p>Please access Gandalf through '
|
||||
'<a href="https://auth.lotusguild.org">auth.lotusguild.org</a>.</p>',
|
||||
401,
|
||||
)
|
||||
allowed = _config().get('auth', {}).get('allowed_groups', ['admin'])
|
||||
if not any(g in allowed for g in user['groups']):
|
||||
return (
|
||||
f'<h1>403 – Access denied</h1>'
|
||||
f'<p>Your account ({user["username"]}) is not in an allowed group '
|
||||
f'({", ".join(allowed)}).</p>',
|
||||
403,
|
||||
)
|
||||
return f(*args, **kwargs)
|
||||
return wrapper
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Page routes
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_PAGE_LIMIT = 200 # max events returned per request
|
||||
|
||||
|
||||
@app.route('/')
|
||||
@require_auth
|
||||
def index():
|
||||
user = _get_user()
|
||||
events = db.get_active_events(limit=_PAGE_LIMIT)
|
||||
total_active = db.count_active_events()
|
||||
summary = db.get_status_summary()
|
||||
snapshot_raw = db.get_state('network_snapshot')
|
||||
last_check = db.get_state('last_check', 'Never')
|
||||
snapshot = json.loads(snapshot_raw) if snapshot_raw else {}
|
||||
suppressions = db.get_active_suppressions()
|
||||
recent_resolved = db.get_recent_resolved(hours=24, limit=10)
|
||||
return render_template(
|
||||
'index.html',
|
||||
user=user,
|
||||
events=events,
|
||||
total_active=total_active,
|
||||
summary=summary,
|
||||
snapshot=snapshot,
|
||||
last_check=last_check,
|
||||
suppressions=suppressions,
|
||||
recent_resolved=recent_resolved,
|
||||
daemon_ok=_daemon_ok(last_check),
|
||||
)
|
||||
|
||||
|
||||
@app.route('/links')
|
||||
@require_auth
|
||||
def links_page():
|
||||
user = _get_user()
|
||||
return render_template('links.html', user=user)
|
||||
|
||||
|
||||
@app.route('/inspector')
|
||||
@require_auth
|
||||
def inspector():
|
||||
user = _get_user()
|
||||
return render_template('inspector.html', user=user)
|
||||
|
||||
|
||||
@app.route('/suppressions')
|
||||
@require_auth
|
||||
def suppressions_page():
|
||||
user = _get_user()
|
||||
active = db.get_active_suppressions()
|
||||
history = db.get_suppression_history(limit=50)
|
||||
snapshot_raw = db.get_state('network_snapshot')
|
||||
snapshot = json.loads(snapshot_raw) if snapshot_raw else {}
|
||||
return render_template(
|
||||
'suppressions.html',
|
||||
user=user,
|
||||
active=active,
|
||||
history=history,
|
||||
snapshot=snapshot,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# API routes
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@app.route('/api/status')
|
||||
@require_auth
|
||||
def api_status():
|
||||
active = db.get_active_events(limit=_PAGE_LIMIT)
|
||||
last_check = db.get_state('last_check', 'Never')
|
||||
return jsonify({
|
||||
'summary': db.get_status_summary(),
|
||||
'last_check': last_check,
|
||||
'events': active,
|
||||
'total_active': db.count_active_events(),
|
||||
'daemon_ok': _daemon_ok(last_check),
|
||||
})
|
||||
|
||||
|
||||
@app.route('/api/network')
|
||||
@require_auth
|
||||
def api_network():
|
||||
raw = db.get_state('network_snapshot')
|
||||
if raw:
|
||||
try:
|
||||
return jsonify(json.loads(raw))
|
||||
except Exception:
|
||||
logger.error('Failed to parse network_snapshot JSON')
|
||||
return jsonify({'hosts': {}, 'unifi': [], 'updated': None})
|
||||
|
||||
|
||||
@app.route('/api/links')
|
||||
@require_auth
|
||||
def api_links():
|
||||
raw = db.get_state('link_stats')
|
||||
if raw:
|
||||
try:
|
||||
return jsonify(json.loads(raw))
|
||||
except Exception:
|
||||
logger.error('Failed to parse link_stats JSON')
|
||||
return jsonify({'hosts': {}, 'updated': None})
|
||||
|
||||
|
||||
@app.route('/api/events')
|
||||
@require_auth
|
||||
def api_events():
|
||||
try:
|
||||
limit = min(int(request.args.get('limit', _PAGE_LIMIT)), 1000)
|
||||
offset = max(int(request.args.get('offset', 0)), 0)
|
||||
except ValueError:
|
||||
return jsonify({'error': 'limit and offset must be integers'}), 400
|
||||
status_filter = request.args.get('status', 'active')
|
||||
if status_filter not in ('active', 'resolved', 'all'):
|
||||
return jsonify({'error': 'status must be active, resolved, or all'}), 400
|
||||
|
||||
result: dict = {}
|
||||
if status_filter in ('active', 'all'):
|
||||
result['active'] = db.get_active_events(limit=limit, offset=offset)
|
||||
result['total_active'] = db.count_active_events()
|
||||
if status_filter in ('resolved', 'all'):
|
||||
result['resolved'] = db.get_recent_resolved(hours=24, limit=30)
|
||||
return jsonify(result)
|
||||
|
||||
|
||||
@app.route('/api/suppressions', methods=['GET'])
|
||||
@require_auth
|
||||
def api_get_suppressions():
|
||||
return jsonify(db.get_active_suppressions())
|
||||
|
||||
|
||||
@app.route('/api/suppressions', methods=['POST'])
|
||||
@require_auth
|
||||
def api_create_suppression():
|
||||
user = _get_user()
|
||||
data = request.get_json(silent=True) or {}
|
||||
|
||||
target_type = data.get('target_type', 'host')
|
||||
target_name = (data.get('target_name') or '').strip()
|
||||
target_detail = (data.get('target_detail') or '').strip()
|
||||
reason = (data.get('reason') or '').strip()
|
||||
expires_minutes = data.get('expires_minutes') # None = manual/permanent
|
||||
|
||||
if target_type not in ('host', 'interface', 'unifi_device', 'all'):
|
||||
return jsonify({'error': 'Invalid target_type'}), 400
|
||||
if target_type != 'all' and not target_name:
|
||||
return jsonify({'error': 'target_name required'}), 400
|
||||
if not reason:
|
||||
return jsonify({'error': 'reason required'}), 400
|
||||
if len(reason) > 500:
|
||||
return jsonify({'error': 'reason must be 500 characters or fewer'}), 400
|
||||
if len(target_name) > 255:
|
||||
return jsonify({'error': 'target_name must be 255 characters or fewer'}), 400
|
||||
if len(target_detail) > 255:
|
||||
return jsonify({'error': 'target_detail must be 255 characters or fewer'}), 400
|
||||
|
||||
sup_id = db.create_suppression(
|
||||
target_type=target_type,
|
||||
target_name=target_name,
|
||||
target_detail=target_detail,
|
||||
reason=reason,
|
||||
suppressed_by=user['username'],
|
||||
expires_minutes=int(expires_minutes) if expires_minutes else None,
|
||||
)
|
||||
logger.info(
|
||||
f'Suppression #{sup_id} created by {user["username"]}: '
|
||||
f'{target_type}/{target_name}/{target_detail} – {reason}'
|
||||
)
|
||||
return jsonify({'success': True, 'id': sup_id})
|
||||
|
||||
|
||||
@app.route('/api/suppressions/<int:sup_id>', methods=['DELETE'])
|
||||
@require_auth
|
||||
def api_delete_suppression(sup_id: int):
|
||||
user = _get_user()
|
||||
db.deactivate_suppression(sup_id)
|
||||
logger.info(f'Suppression #{sup_id} removed by {user["username"]}')
|
||||
return jsonify({'success': True})
|
||||
|
||||
|
||||
@app.route('/api/diagnose', methods=['POST'])
|
||||
@require_auth
|
||||
def api_diagnose_start():
|
||||
"""Start a link diagnostic job. Returns {job_id}."""
|
||||
data = request.get_json(silent=True) or {}
|
||||
switch_name = (data.get('switch_name') or '').strip()
|
||||
try:
|
||||
port_idx = int(data.get('port_idx'))
|
||||
except (TypeError, ValueError):
|
||||
return jsonify({'error': 'port_idx must be an integer'}), 400
|
||||
|
||||
if not switch_name:
|
||||
return jsonify({'error': 'switch_name and port_idx required'}), 400
|
||||
|
||||
# Look up switch + port in cached link_stats
|
||||
raw = db.get_state('link_stats')
|
||||
if not raw:
|
||||
return jsonify({'error': 'No link_stats data available'}), 503
|
||||
try:
|
||||
link_data = json.loads(raw)
|
||||
except Exception:
|
||||
logger.error('Failed to parse link_stats JSON in /api/diagnose')
|
||||
return jsonify({'error': 'Internal data error'}), 500
|
||||
|
||||
switches = link_data.get('unifi_switches', {})
|
||||
sw = switches.get(switch_name)
|
||||
if not sw:
|
||||
return jsonify({'error': f'Switch "{switch_name}" not found'}), 404
|
||||
|
||||
# Find port by port_idx
|
||||
port_data = None
|
||||
for pname, pd in sw.get('ports', {}).items():
|
||||
if pd.get('port_idx') == port_idx:
|
||||
port_data = dict(pd)
|
||||
port_data['name'] = pname
|
||||
break
|
||||
if not port_data:
|
||||
return jsonify({'error': f'Port {port_idx} not found on switch "{switch_name}"'}), 404
|
||||
|
||||
# LLDP neighbor required to know which host+iface to SSH into
|
||||
lldp = port_data.get('lldp')
|
||||
if not lldp or not lldp.get('system_name'):
|
||||
return jsonify({'error': 'No LLDP neighbor data for this port'}), 400
|
||||
|
||||
server_name = lldp['system_name']
|
||||
lldp_port_id = lldp.get('port_id', '')
|
||||
|
||||
# Find matching host + interface in link_stats hosts
|
||||
hosts = link_data.get('hosts', {})
|
||||
server_ifaces = hosts.get(server_name)
|
||||
if not server_ifaces:
|
||||
return jsonify({'error': f'Host "{server_name}" not in link stats'}), 404
|
||||
|
||||
# Match interface by LLDP port_id (exact then fuzzy)
|
||||
matched_iface = None
|
||||
if lldp_port_id and lldp_port_id in server_ifaces:
|
||||
matched_iface = lldp_port_id
|
||||
if not matched_iface and lldp_port_id:
|
||||
matched_iface = next(
|
||||
(k for k in server_ifaces if lldp_port_id in k or k in lldp_port_id),
|
||||
None
|
||||
)
|
||||
if not matched_iface:
|
||||
matched_iface = next(iter(server_ifaces), None)
|
||||
if not matched_iface:
|
||||
return jsonify({'error': 'Cannot determine server interface'}), 400
|
||||
|
||||
# Resolve host IP from link_stats host data
|
||||
host_ip = (server_ifaces.get(matched_iface) or {}).get('host_ip')
|
||||
if not host_ip:
|
||||
# Fallback: use LLDP mgmt IPs
|
||||
mgmt_ips = lldp.get('mgmt_ips') or []
|
||||
host_ip = mgmt_ips[0] if mgmt_ips else None
|
||||
if not host_ip:
|
||||
return jsonify({'error': 'Cannot determine host IP for SSH'}), 400
|
||||
|
||||
# Validate resolved values before passing to SSH command builder
|
||||
try:
|
||||
ipaddress.ip_address(host_ip)
|
||||
except ValueError:
|
||||
logger.error(f'Refusing diagnostic: invalid host_ip "{host_ip}" for {server_name}')
|
||||
return jsonify({'error': 'Resolved host IP is not a valid IP address'}), 400
|
||||
if not re.fullmatch(r'[a-zA-Z0-9._-]+', matched_iface):
|
||||
logger.error(f'Refusing diagnostic: invalid iface "{matched_iface}" for {server_name}')
|
||||
return jsonify({'error': 'Resolved interface name contains invalid characters'}), 400
|
||||
|
||||
job_id = str(uuid.uuid4())
|
||||
with _diag_lock:
|
||||
_diag_jobs[job_id] = {'status': 'running', 'result': None, 'created_at': time.time()}
|
||||
|
||||
def _run():
|
||||
try:
|
||||
cfg = _config()
|
||||
pulse = PulseClient(cfg)
|
||||
runner = diagnose.DiagnosticsRunner(pulse)
|
||||
result = runner.run(host_ip, server_name, matched_iface, port_data)
|
||||
except Exception as e:
|
||||
logger.error(f'Diagnostic job {job_id} failed: {e}', exc_info=True)
|
||||
result = {'status': 'error', 'error': str(e)}
|
||||
with _diag_lock:
|
||||
if job_id in _diag_jobs:
|
||||
_diag_jobs[job_id]['status'] = 'done'
|
||||
_diag_jobs[job_id]['result'] = result
|
||||
|
||||
t = threading.Thread(target=_run, daemon=True)
|
||||
t.start()
|
||||
|
||||
return jsonify({'job_id': job_id})
|
||||
|
||||
|
||||
@app.route('/api/diagnose/<job_id>', methods=['GET'])
|
||||
@require_auth
|
||||
def api_diagnose_poll(job_id: str):
|
||||
"""Poll a diagnostic job. Returns {status, result}."""
|
||||
with _diag_lock:
|
||||
job = _diag_jobs.get(job_id)
|
||||
if not job:
|
||||
return jsonify({'error': 'Job not found'}), 404
|
||||
return jsonify({'status': job['status'], 'result': job.get('result')})
|
||||
|
||||
|
||||
@app.route('/api/avatar')
|
||||
@require_auth
|
||||
def api_avatar():
|
||||
"""Serve the current user's LDAP avatar photo (JPEG), cached to disk."""
|
||||
username = request.headers.get('Remote-User', '').strip()
|
||||
if not username:
|
||||
return '', 404
|
||||
|
||||
ldap_cfg = _config().get('ldap', {})
|
||||
if not ldap_cfg.get('host') or not ldap_cfg.get('bind_dn'):
|
||||
return '', 404
|
||||
|
||||
# Build a safe cache filename from the username (alphanumeric + - _ .)
|
||||
safe_name = re.sub(r'[^a-zA-Z0-9._-]', '_', username)
|
||||
cache_dir = ldap_cfg.get('cache_dir', '/tmp/gandalf_avatars')
|
||||
os.makedirs(cache_dir, exist_ok=True)
|
||||
cache_file = os.path.join(cache_dir, f'user_{safe_name}.jpg')
|
||||
sentinel = os.path.join(cache_dir, f'user_{safe_name}.none')
|
||||
cache_ttl = int(ldap_cfg.get('cache_ttl', 3600))
|
||||
|
||||
now = time.time()
|
||||
|
||||
# Serve cached image if fresh
|
||||
if os.path.exists(cache_file) and now - os.path.getmtime(cache_file) < cache_ttl:
|
||||
return send_file(cache_file, mimetype='image/jpeg',
|
||||
max_age=cache_ttl, conditional=True)
|
||||
|
||||
# Skip LDAP if we already know this user has no avatar
|
||||
if os.path.exists(sentinel) and now - os.path.getmtime(sentinel) < cache_ttl:
|
||||
return '', 404
|
||||
|
||||
# Query lldap
|
||||
avatar_data = None
|
||||
try:
|
||||
import ldap3
|
||||
server = ldap3.Server(ldap_cfg['host'], port=int(ldap_cfg.get('port', 3890)))
|
||||
conn = ldap3.Connection(server,
|
||||
user=ldap_cfg['bind_dn'],
|
||||
password=ldap_cfg.get('bind_pw', ''),
|
||||
auto_bind=True, receive_timeout=5)
|
||||
safe_uid = ldap3.utils.conv.escape_filter_chars(username)
|
||||
conn.search(ldap_cfg.get('user_base', 'ou=people,dc=example,dc=com'),
|
||||
f'(uid={safe_uid})', attributes=['avatar'])
|
||||
if conn.entries and conn.entries[0]['avatar'].value:
|
||||
avatar_data = conn.entries[0]['avatar'].value
|
||||
conn.unbind()
|
||||
except ImportError:
|
||||
logger.error('ldap3 not installed — run: pip install ldap3')
|
||||
return '', 404
|
||||
except Exception as e:
|
||||
logger.error(f'LDAP avatar lookup failed for {username}: {e}')
|
||||
return '', 404
|
||||
|
||||
if not avatar_data or len(avatar_data) < 100:
|
||||
open(sentinel, 'w').close()
|
||||
return '', 404
|
||||
|
||||
# Validate JPEG magic bytes (FF D8 FF)
|
||||
if isinstance(avatar_data, str):
|
||||
avatar_data = avatar_data.encode('latin-1')
|
||||
if avatar_data[:3] != b'\xFF\xD8\xFF':
|
||||
logger.warning(f'Non-JPEG avatar data for {username}')
|
||||
open(sentinel, 'w').close()
|
||||
return '', 404
|
||||
|
||||
with open(cache_file, 'wb') as f:
|
||||
f.write(avatar_data)
|
||||
if os.path.exists(sentinel):
|
||||
os.unlink(sentinel)
|
||||
|
||||
resp = make_response(avatar_data)
|
||||
resp.headers['Content-Type'] = 'image/jpeg'
|
||||
resp.headers['Cache-Control'] = f'private, max-age={cache_ttl}'
|
||||
return resp
|
||||
|
||||
|
||||
@app.route('/health')
|
||||
def health():
|
||||
"""Health check endpoint (no auth). Checks DB and monitor freshness."""
|
||||
checks = {}
|
||||
overall = 'ok'
|
||||
|
||||
# DB connectivity
|
||||
try:
|
||||
db.get_state('last_check')
|
||||
checks['db'] = 'ok'
|
||||
except Exception as e:
|
||||
checks['db'] = f'error: {e}'
|
||||
overall = 'degraded'
|
||||
|
||||
# Monitor freshness: fail if last_check is older than 20 minutes
|
||||
try:
|
||||
last_check = db.get_state('last_check', '')
|
||||
if last_check:
|
||||
ts = datetime.strptime(last_check, '%Y-%m-%d %H:%M:%S UTC').replace(tzinfo=timezone.utc)
|
||||
age_s = (datetime.now(timezone.utc) - ts).total_seconds()
|
||||
if age_s > 1200:
|
||||
checks['monitor'] = f'stale ({int(age_s)}s since last check)'
|
||||
overall = 'degraded'
|
||||
else:
|
||||
checks['monitor'] = f'ok ({int(age_s)}s ago)'
|
||||
else:
|
||||
checks['monitor'] = 'no data yet'
|
||||
except Exception as e:
|
||||
checks['monitor'] = f'error: {e}'
|
||||
overall = 'degraded'
|
||||
|
||||
status_code = 200 if overall == 'ok' else 503
|
||||
return jsonify({'status': overall, 'service': 'gandalf', 'checks': checks}), status_code
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
app.run(debug=True, host='0.0.0.0', port=5000) # nosec B201 B104 — dev runner only; production uses gunicorn
|
||||
@@ -0,0 +1,378 @@
|
||||
"""Database operations for Gandalf network monitor."""
|
||||
import json
|
||||
import logging
|
||||
import threading
|
||||
from contextlib import contextmanager
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Optional
|
||||
|
||||
import pymysql
|
||||
import pymysql.cursors
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_config_cache = None
|
||||
_local = threading.local()
|
||||
|
||||
|
||||
def _config() -> dict:
|
||||
global _config_cache
|
||||
if _config_cache is None:
|
||||
with open('config.json') as f:
|
||||
_config_cache = json.load(f)['database']
|
||||
return _config_cache
|
||||
|
||||
|
||||
def _new_conn(cfg: dict):
|
||||
return pymysql.connect(
|
||||
host=cfg['host'],
|
||||
port=cfg.get('port', 3306),
|
||||
user=cfg['user'],
|
||||
password=cfg['password'],
|
||||
database=cfg['name'],
|
||||
autocommit=True,
|
||||
cursorclass=pymysql.cursors.DictCursor,
|
||||
connect_timeout=10,
|
||||
charset='utf8mb4',
|
||||
)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def get_conn():
|
||||
"""Yield a per-thread cached database connection, reconnecting as needed."""
|
||||
cfg = _config()
|
||||
conn = getattr(_local, 'conn', None)
|
||||
if conn is None:
|
||||
conn = _new_conn(cfg)
|
||||
_local.conn = conn
|
||||
else:
|
||||
try:
|
||||
conn.ping(reconnect=True)
|
||||
except Exception:
|
||||
try:
|
||||
conn.close()
|
||||
except Exception:
|
||||
pass
|
||||
_local.conn = None
|
||||
conn = _new_conn(cfg)
|
||||
_local.conn = conn
|
||||
yield conn
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Monitor state (key/value store)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def set_state(key: str, value) -> None:
|
||||
if not isinstance(value, str):
|
||||
value = json.dumps(value, default=str)
|
||||
with get_conn() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"""INSERT INTO monitor_state (key_name, value)
|
||||
VALUES (%s, %s)
|
||||
ON DUPLICATE KEY UPDATE value=VALUES(value), updated_at=NOW()""",
|
||||
(key, value),
|
||||
)
|
||||
|
||||
|
||||
def get_state(key: str, default=None):
|
||||
with get_conn() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute('SELECT value FROM monitor_state WHERE key_name=%s', (key,))
|
||||
row = cur.fetchone()
|
||||
return row['value'] if row else default
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Interface baseline tracking
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def get_baseline() -> dict:
|
||||
raw = get_state('interface_baseline')
|
||||
if raw:
|
||||
try:
|
||||
return json.loads(raw)
|
||||
except Exception:
|
||||
logger.error('Failed to parse interface_baseline JSON; resetting baseline')
|
||||
return {}
|
||||
|
||||
|
||||
def set_baseline(baseline: dict) -> None:
|
||||
set_state('interface_baseline', json.dumps(baseline))
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Network events
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def upsert_event(
|
||||
event_type: str,
|
||||
severity: str,
|
||||
source_type: str,
|
||||
target_name: str,
|
||||
target_detail: str,
|
||||
description: str,
|
||||
) -> tuple:
|
||||
"""Insert or update a network event. Returns (id, is_new, consecutive_failures)."""
|
||||
detail = target_detail or ''
|
||||
with get_conn() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"""SELECT id, consecutive_failures FROM network_events
|
||||
WHERE event_type=%s AND target_name=%s AND target_detail=%s
|
||||
AND resolved_at IS NULL LIMIT 1""",
|
||||
(event_type, target_name, detail),
|
||||
)
|
||||
existing = cur.fetchone()
|
||||
|
||||
if existing:
|
||||
new_count = existing['consecutive_failures'] + 1
|
||||
cur.execute(
|
||||
"""UPDATE network_events
|
||||
SET last_seen=NOW(), consecutive_failures=%s, description=%s
|
||||
WHERE id=%s""",
|
||||
(new_count, description, existing['id']),
|
||||
)
|
||||
return existing['id'], False, new_count
|
||||
else:
|
||||
cur.execute(
|
||||
"""INSERT INTO network_events
|
||||
(event_type, severity, source_type, target_name, target_detail, description)
|
||||
VALUES (%s, %s, %s, %s, %s, %s)""",
|
||||
(event_type, severity, source_type, target_name, detail, description),
|
||||
)
|
||||
return cur.lastrowid, True, 1
|
||||
|
||||
|
||||
def resolve_event(event_type: str, target_name: str, target_detail: str = '') -> None:
|
||||
detail = target_detail or ''
|
||||
with get_conn() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"""UPDATE network_events SET resolved_at=NOW()
|
||||
WHERE event_type=%s AND target_name=%s AND target_detail=%s
|
||||
AND resolved_at IS NULL""",
|
||||
(event_type, target_name, detail),
|
||||
)
|
||||
|
||||
|
||||
def set_ticket_id(event_id: int, ticket_id: str) -> None:
|
||||
with get_conn() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
'UPDATE network_events SET ticket_id=%s WHERE id=%s',
|
||||
(ticket_id, event_id),
|
||||
)
|
||||
|
||||
|
||||
def get_active_events(limit: int = 200, offset: int = 0) -> list:
|
||||
with get_conn() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"""SELECT * FROM network_events
|
||||
WHERE resolved_at IS NULL
|
||||
ORDER BY
|
||||
FIELD(severity,'critical','warning','info'),
|
||||
first_seen DESC
|
||||
LIMIT %s OFFSET %s""",
|
||||
(limit, offset),
|
||||
)
|
||||
rows = cur.fetchall()
|
||||
for r in rows:
|
||||
for k in ('first_seen', 'last_seen'):
|
||||
if r.get(k) and hasattr(r[k], 'isoformat'):
|
||||
r[k] = r[k].isoformat()
|
||||
return rows
|
||||
|
||||
|
||||
def count_active_events() -> int:
|
||||
"""Return count of all unresolved events (for pagination)."""
|
||||
with get_conn() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"SELECT COUNT(*) AS n FROM network_events WHERE resolved_at IS NULL"
|
||||
)
|
||||
return cur.fetchone()['n']
|
||||
|
||||
|
||||
def get_recent_resolved(hours: int = 24, limit: int = 50) -> list:
|
||||
with get_conn() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"""SELECT * FROM network_events
|
||||
WHERE resolved_at IS NOT NULL
|
||||
AND resolved_at > DATE_SUB(NOW(), INTERVAL %s HOUR)
|
||||
ORDER BY resolved_at DESC LIMIT %s""",
|
||||
(hours, limit),
|
||||
)
|
||||
rows = cur.fetchall()
|
||||
for r in rows:
|
||||
for k in ('first_seen', 'last_seen', 'resolved_at'):
|
||||
if r.get(k) and hasattr(r[k], 'isoformat'):
|
||||
r[k] = r[k].isoformat()
|
||||
return rows
|
||||
|
||||
|
||||
def get_status_summary() -> dict:
|
||||
with get_conn() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"""SELECT severity, COUNT(*) as cnt FROM network_events
|
||||
WHERE resolved_at IS NULL GROUP BY severity"""
|
||||
)
|
||||
counts = {r['severity']: r['cnt'] for r in cur.fetchall()}
|
||||
return {
|
||||
'critical': counts.get('critical', 0),
|
||||
'warning': counts.get('warning', 0),
|
||||
'info': counts.get('info', 0),
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Suppression rules
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def get_active_suppressions() -> list:
|
||||
with get_conn() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"""SELECT * FROM suppression_rules
|
||||
WHERE active=TRUE AND (expires_at IS NULL OR expires_at > NOW())
|
||||
ORDER BY created_at DESC"""
|
||||
)
|
||||
rows = cur.fetchall()
|
||||
for r in rows:
|
||||
for k in ('created_at', 'expires_at'):
|
||||
if r.get(k) and hasattr(r[k], 'isoformat'):
|
||||
r[k] = r[k].isoformat()
|
||||
return rows
|
||||
|
||||
|
||||
def get_suppression_history(limit: int = 50) -> list:
|
||||
with get_conn() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
'SELECT * FROM suppression_rules ORDER BY created_at DESC LIMIT %s',
|
||||
(limit,),
|
||||
)
|
||||
rows = cur.fetchall()
|
||||
for r in rows:
|
||||
for k in ('created_at', 'expires_at'):
|
||||
if r.get(k) and hasattr(r[k], 'isoformat'):
|
||||
r[k] = r[k].isoformat()
|
||||
return rows
|
||||
|
||||
|
||||
def create_suppression(
|
||||
target_type: str,
|
||||
target_name: str,
|
||||
target_detail: str,
|
||||
reason: str,
|
||||
suppressed_by: str,
|
||||
expires_minutes: Optional[int] = None,
|
||||
) -> int:
|
||||
expires_at = None
|
||||
if expires_minutes:
|
||||
expires_at = datetime.utcnow() + timedelta(minutes=int(expires_minutes))
|
||||
with get_conn() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"""INSERT INTO suppression_rules
|
||||
(target_type, target_name, target_detail, reason, suppressed_by, expires_at, active)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, TRUE)""",
|
||||
(target_type, target_name or '', target_detail or '', reason, suppressed_by, expires_at),
|
||||
)
|
||||
return cur.lastrowid
|
||||
|
||||
|
||||
def deactivate_suppression(sup_id: int) -> None:
|
||||
with get_conn() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
'UPDATE suppression_rules SET active=FALSE WHERE id=%s', (sup_id,)
|
||||
)
|
||||
|
||||
|
||||
def cleanup_expired_suppressions() -> int:
|
||||
"""Mark expired time-limited suppressions as inactive. Returns count deactivated."""
|
||||
with get_conn() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"""UPDATE suppression_rules
|
||||
SET active=FALSE
|
||||
WHERE active=TRUE AND expires_at IS NOT NULL AND expires_at <= NOW()"""
|
||||
)
|
||||
n = cur.rowcount
|
||||
if n:
|
||||
logger.info(f'Deactivated {n} expired suppression(s)')
|
||||
return n
|
||||
|
||||
|
||||
def purge_old_resolved_events(days: int = 90) -> int:
|
||||
"""Delete resolved events older than `days` days. Returns count deleted."""
|
||||
with get_conn() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"""DELETE FROM network_events
|
||||
WHERE resolved_at IS NOT NULL
|
||||
AND resolved_at < DATE_SUB(NOW(), INTERVAL %s DAY)""",
|
||||
(days,),
|
||||
)
|
||||
n = cur.rowcount
|
||||
if n:
|
||||
logger.info(f'Purged {n} old resolved event(s) (>{days}d)')
|
||||
return n
|
||||
|
||||
|
||||
def check_suppressed(suppressions: list, target_type: str, target_name: str, target_detail: str = '') -> bool:
|
||||
"""Check suppression against a pre-loaded list (avoids per-call DB queries)."""
|
||||
for s in suppressions:
|
||||
if s['target_type'] == 'all':
|
||||
return True
|
||||
if s['target_type'] == target_type and s['target_name'] == target_name:
|
||||
if not (s.get('target_detail') or ''):
|
||||
return True
|
||||
if target_detail and s.get('target_detail') == target_detail:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def is_suppressed(target_type: str, target_name: str, target_detail: str = '') -> bool:
|
||||
with get_conn() as conn:
|
||||
with conn.cursor() as cur:
|
||||
# Global suppression (all)
|
||||
cur.execute(
|
||||
"""SELECT id FROM suppression_rules
|
||||
WHERE active=TRUE AND (expires_at IS NULL OR expires_at > NOW())
|
||||
AND target_type='all' LIMIT 1"""
|
||||
)
|
||||
if cur.fetchone():
|
||||
return True
|
||||
|
||||
if not target_name:
|
||||
return False
|
||||
|
||||
# Host-level suppression (covers all interfaces on that host)
|
||||
cur.execute(
|
||||
"""SELECT id FROM suppression_rules
|
||||
WHERE active=TRUE AND (expires_at IS NULL OR expires_at > NOW())
|
||||
AND target_type=%s AND target_name=%s
|
||||
AND (target_detail IS NULL OR target_detail='') LIMIT 1""",
|
||||
(target_type, target_name),
|
||||
)
|
||||
if cur.fetchone():
|
||||
return True
|
||||
|
||||
# Interface/device-specific suppression
|
||||
if target_detail:
|
||||
cur.execute(
|
||||
"""SELECT id FROM suppression_rules
|
||||
WHERE active=TRUE AND (expires_at IS NULL OR expires_at > NOW())
|
||||
AND target_type=%s AND target_name=%s AND target_detail=%s LIMIT 1""",
|
||||
(target_type, target_name, target_detail),
|
||||
)
|
||||
if cur.fetchone():
|
||||
return True
|
||||
|
||||
return False
|
||||
+548
@@ -0,0 +1,548 @@
|
||||
"""Gandalf – Link Diagnostics module.
|
||||
|
||||
Runs a comprehensive SSH-based diagnostic against a server NIC and
|
||||
analyses the result against switch port data to surface root causes.
|
||||
Executed in a background thread; results stored in _diag_jobs (app.py).
|
||||
"""
|
||||
import re
|
||||
import shlex
|
||||
import logging
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
logger = logging.getLogger('gandalf.diagnose')
|
||||
|
||||
# sysfs counters collected per interface
|
||||
_SYSFS_STATS = [
|
||||
'rx_bytes', 'tx_bytes', 'rx_errors', 'tx_errors',
|
||||
'rx_dropped', 'tx_dropped', 'rx_crc_errors',
|
||||
'rx_frame_errors', 'rx_fifo_errors', 'tx_carrier_errors',
|
||||
'collisions', 'rx_missed_errors',
|
||||
]
|
||||
|
||||
|
||||
class DiagnosticsRunner:
|
||||
"""Build and run a link diagnostic against a server NIC via PulseClient."""
|
||||
|
||||
def __init__(self, pulse_client):
|
||||
self.pulse = pulse_client
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# SSH command builder
|
||||
# ------------------------------------------------------------------
|
||||
@staticmethod
|
||||
def build_ssh_command(host_ip: str, iface: str) -> str:
|
||||
"""Return a single-line SSH command that collects all diagnostic data."""
|
||||
q = shlex.quote(iface)
|
||||
ip_q = shlex.quote(host_ip)
|
||||
|
||||
sysfs_loop = '; '.join(
|
||||
f'echo "{s}:$(cat /sys/class/net/{q}/statistics/{s} 2>/dev/null || echo 0)"'
|
||||
for s in _SYSFS_STATS
|
||||
)
|
||||
|
||||
remote_cmd = (
|
||||
f'echo "=== carrier ===";'
|
||||
f' cat /sys/class/net/{q}/carrier 2>/dev/null || echo "?";'
|
||||
f' echo "=== operstate ===";'
|
||||
f' cat /sys/class/net/{q}/operstate 2>/dev/null || echo "?";'
|
||||
f' echo "=== sysfs_stats ===";'
|
||||
f' {sysfs_loop};'
|
||||
f' echo "=== carrier_changes ===";'
|
||||
f' cat /sys/class/net/{q}/carrier_changes 2>/dev/null || echo "0";'
|
||||
f' echo "=== ethtool ===";'
|
||||
f' ethtool {q} 2>/dev/null;'
|
||||
f' echo "=== ethtool_driver ===";'
|
||||
f' ethtool -i {q} 2>/dev/null;'
|
||||
f' echo "=== ethtool_pause ===";'
|
||||
f' ethtool -a {q} 2>/dev/null;'
|
||||
f' echo "=== ethtool_ring ===";'
|
||||
f' ethtool -g {q} 2>/dev/null;'
|
||||
f' echo "=== ethtool_stats ===";'
|
||||
f' ethtool -S {q} 2>/dev/null;'
|
||||
f' echo "=== ethtool_dom ===";'
|
||||
f' ethtool -m {q} 2>/dev/null;'
|
||||
f' echo "=== ip_link ===";'
|
||||
f' ip -s link show {q} 2>/dev/null;'
|
||||
f' echo "=== ip_addr ===";'
|
||||
f' ip addr show {q} 2>/dev/null;'
|
||||
f' echo "=== ip_route ===";'
|
||||
f' ip route show dev {q} 2>/dev/null;'
|
||||
f' echo "=== dmesg ===";'
|
||||
f' dmesg 2>/dev/null | grep {q} | tail -50;'
|
||||
f' echo "=== lldpctl ===";'
|
||||
f' lldpctl 2>/dev/null || echo "lldpd not running";'
|
||||
f' echo "=== end ==="'
|
||||
)
|
||||
|
||||
return (
|
||||
f'ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 '
|
||||
f'-o BatchMode=yes -o LogLevel=ERROR '
|
||||
f'-o ServerAliveInterval=10 -o ServerAliveCountMax=2 '
|
||||
f'root@{ip_q} \'{remote_cmd}\''
|
||||
)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Main entry point
|
||||
# ------------------------------------------------------------------
|
||||
def run(self, host_ip: str, host_name: str, iface: str,
|
||||
switch_port_data: dict) -> dict:
|
||||
"""Execute diagnostics and return structured result dict."""
|
||||
cmd = self.build_ssh_command(host_ip, iface)
|
||||
logger.info(f'Running link diagnostic: {host_name}/{iface} via Pulse')
|
||||
|
||||
# Reset execution_id before call
|
||||
self.pulse.last_execution_id = None
|
||||
output = self.pulse.run_command(cmd)
|
||||
execution_id = getattr(self.pulse, 'last_execution_id', None)
|
||||
|
||||
if output is None:
|
||||
return {
|
||||
'status': 'error',
|
||||
'error': 'Pulse command failed or timed out',
|
||||
'host': host_name,
|
||||
'iface': iface,
|
||||
'pulse_execution_id': execution_id,
|
||||
}
|
||||
|
||||
sections = self.parse_output(output)
|
||||
health = self.analyze(sections, switch_port_data)
|
||||
|
||||
pulse_url = None
|
||||
if execution_id:
|
||||
base = getattr(self.pulse, 'url', '').rstrip('/')
|
||||
pulse_url = f'{base}/executions/{execution_id}' if base else None
|
||||
|
||||
return {
|
||||
'status': 'done',
|
||||
'host': host_name,
|
||||
'iface': iface,
|
||||
'sections': sections,
|
||||
'health': health,
|
||||
'pulse_execution_id': execution_id,
|
||||
'pulse_url': pulse_url,
|
||||
'switch_port': switch_port_data,
|
||||
}
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Output parser (splits on === SECTION_NAME === sentinels)
|
||||
# ------------------------------------------------------------------
|
||||
@staticmethod
|
||||
def parse_output(raw: str) -> dict:
|
||||
sections: Dict[str, str] = {}
|
||||
current: Optional[str] = None
|
||||
buf: List[str] = []
|
||||
|
||||
for line in raw.splitlines():
|
||||
m = re.match(r'^=== (.+?) ===$', line.strip())
|
||||
if m:
|
||||
if current and current != 'end':
|
||||
sections[current] = '\n'.join(buf).strip()
|
||||
name = m.group(1)
|
||||
if name == 'end':
|
||||
current = None
|
||||
else:
|
||||
current = name
|
||||
buf = []
|
||||
elif current:
|
||||
buf.append(line)
|
||||
|
||||
if current and current != 'end':
|
||||
sections[current] = '\n'.join(buf).strip()
|
||||
|
||||
parsed: dict = {}
|
||||
|
||||
# Simple string sections
|
||||
parsed['carrier'] = sections.get('carrier', '?').strip()
|
||||
parsed['operstate'] = sections.get('operstate', '?').strip()
|
||||
|
||||
# carrier_changes
|
||||
cc_raw = sections.get('carrier_changes', '0').strip()
|
||||
try:
|
||||
parsed['carrier_changes'] = int(cc_raw)
|
||||
except ValueError:
|
||||
parsed['carrier_changes'] = None
|
||||
|
||||
# Structured sections
|
||||
parsed['sysfs_stats'] = DiagnosticsRunner.parse_sysfs_stats(sections.get('sysfs_stats', ''))
|
||||
parsed['ethtool'] = DiagnosticsRunner.parse_ethtool(sections.get('ethtool', ''))
|
||||
parsed['ethtool_driver'] = DiagnosticsRunner.parse_ethtool_driver(sections.get('ethtool_driver', ''))
|
||||
parsed['ethtool_pause'] = DiagnosticsRunner.parse_ethtool_pause(sections.get('ethtool_pause', ''))
|
||||
parsed['ethtool_ring'] = DiagnosticsRunner.parse_ethtool_ring(sections.get('ethtool_ring', ''))
|
||||
parsed['ethtool_stats'] = DiagnosticsRunner.parse_nic_stats(sections.get('ethtool_stats', ''))
|
||||
parsed['ethtool_dom'] = DiagnosticsRunner.parse_ethtool_dom(sections.get('ethtool_dom', ''))
|
||||
parsed['ip_link'] = DiagnosticsRunner.parse_ip_link(sections.get('ip_link', ''))
|
||||
parsed['ip_addr'] = sections.get('ip_addr', '').strip()
|
||||
parsed['ip_route'] = sections.get('ip_route', '').strip()
|
||||
parsed['dmesg'] = DiagnosticsRunner.parse_dmesg(sections.get('dmesg', ''))
|
||||
parsed['lldpctl'] = DiagnosticsRunner.parse_lldpctl(sections.get('lldpctl', ''))
|
||||
|
||||
return parsed
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Individual parsers
|
||||
# ------------------------------------------------------------------
|
||||
@staticmethod
|
||||
def parse_sysfs_stats(text: str) -> dict:
|
||||
result: dict = {}
|
||||
for line in text.splitlines():
|
||||
if ':' not in line:
|
||||
continue
|
||||
key, _, val = line.partition(':')
|
||||
key = key.strip()
|
||||
val = val.strip()
|
||||
if key in _SYSFS_STATS:
|
||||
try:
|
||||
result[key] = int(val)
|
||||
except ValueError:
|
||||
result[key] = 0
|
||||
return result
|
||||
|
||||
@staticmethod
|
||||
def parse_ethtool(text: str) -> dict:
|
||||
"""Parse ethtool <iface> output."""
|
||||
data: dict = {}
|
||||
for line in text.splitlines():
|
||||
if ':' not in line:
|
||||
continue
|
||||
key, _, val = line.partition(':')
|
||||
key = key.strip()
|
||||
val = val.strip()
|
||||
if key == 'Speed':
|
||||
m = re.match(r'(\d+)\s*Mb/s', val)
|
||||
if m:
|
||||
data['speed_mbps'] = int(m.group(1))
|
||||
elif 'Unknown' in val or 'unknown' in val:
|
||||
data['speed_mbps'] = None
|
||||
elif key == 'Duplex':
|
||||
data['duplex'] = val.lower()
|
||||
elif key == 'Port':
|
||||
data['port_type'] = val
|
||||
elif key == 'Auto-negotiation':
|
||||
data['auto_neg'] = (val.lower() == 'on')
|
||||
elif key == 'Link detected':
|
||||
data['link_detected'] = (val.lower() == 'yes')
|
||||
elif 'Supported link modes' in key:
|
||||
data.setdefault('supported_modes', []).append(val)
|
||||
return data
|
||||
|
||||
@staticmethod
|
||||
def parse_ethtool_driver(text: str) -> dict:
|
||||
data: dict = {}
|
||||
for line in text.splitlines():
|
||||
if ':' not in line:
|
||||
continue
|
||||
key, _, val = line.partition(':')
|
||||
key = key.strip()
|
||||
val = val.strip()
|
||||
if key == 'driver':
|
||||
data['driver'] = val
|
||||
elif key == 'version':
|
||||
data['version'] = val
|
||||
elif key == 'firmware-version':
|
||||
data['firmware_version'] = val
|
||||
elif key == 'bus-info':
|
||||
data['bus_info'] = val
|
||||
return data
|
||||
|
||||
@staticmethod
|
||||
def parse_ethtool_pause(text: str) -> dict:
|
||||
data = {'rx_pause': False, 'tx_pause': False}
|
||||
for line in text.splitlines():
|
||||
if ':' not in line:
|
||||
continue
|
||||
key, _, val = line.partition(':')
|
||||
key = key.strip()
|
||||
val = val.strip().lower()
|
||||
if key == 'RX':
|
||||
data['rx_pause'] = (val == 'on')
|
||||
elif key == 'TX':
|
||||
data['tx_pause'] = (val == 'on')
|
||||
return data
|
||||
|
||||
@staticmethod
|
||||
def parse_ethtool_ring(text: str) -> dict:
|
||||
data: dict = {}
|
||||
in_current = False
|
||||
for line in text.splitlines():
|
||||
if 'Current hardware settings' in line:
|
||||
in_current = True
|
||||
continue
|
||||
if 'Pre-set maximums' in line:
|
||||
in_current = False
|
||||
continue
|
||||
if ':' not in line:
|
||||
continue
|
||||
key, _, val = line.partition(':')
|
||||
key = key.strip()
|
||||
val = val.strip()
|
||||
try:
|
||||
v = int(val)
|
||||
except ValueError:
|
||||
continue
|
||||
if in_current:
|
||||
if 'RX' in key and 'rx_current' not in data:
|
||||
data['rx_current'] = v
|
||||
elif 'TX' in key and 'tx_current' not in data:
|
||||
data['tx_current'] = v
|
||||
else:
|
||||
if 'RX' in key and 'rx_max' not in data:
|
||||
data['rx_max'] = v
|
||||
elif 'TX' in key and 'tx_max' not in data:
|
||||
data['tx_max'] = v
|
||||
return data
|
||||
|
||||
@staticmethod
|
||||
def parse_nic_stats(text: str) -> dict:
|
||||
"""Parse ethtool -S output into {key: int} dict."""
|
||||
data: dict = {}
|
||||
for line in text.splitlines():
|
||||
if ':' not in line:
|
||||
continue
|
||||
key, _, val = line.partition(':')
|
||||
key = key.strip()
|
||||
val = val.strip()
|
||||
try:
|
||||
data[key] = int(val)
|
||||
except ValueError:
|
||||
pass
|
||||
return data
|
||||
|
||||
@staticmethod
|
||||
def parse_ethtool_dom(text: str) -> dict:
|
||||
"""Parse ethtool -m (SFP DOM) output."""
|
||||
if not text:
|
||||
return {}
|
||||
lower = text.lower()
|
||||
if any(s in lower for s in ('cannot get', 'not supported', 'no sfp', 'operation not supported')):
|
||||
return {}
|
||||
|
||||
data: dict = {}
|
||||
for line in text.splitlines():
|
||||
if ':' not in line:
|
||||
continue
|
||||
key, _, val = line.partition(':')
|
||||
key = key.strip()
|
||||
val = val.strip()
|
||||
|
||||
if key == 'Vendor name':
|
||||
data['vendor'] = val
|
||||
elif key == 'Vendor PN':
|
||||
data['part_no'] = val
|
||||
elif key == 'Identifier':
|
||||
m = re.search(r'\((.+?)\)', val)
|
||||
if m:
|
||||
data['sfp_type'] = m.group(1)
|
||||
elif key == 'Connector':
|
||||
m = re.search(r'\((.+?)\)', val)
|
||||
if m:
|
||||
data['connector'] = m.group(1)
|
||||
elif key == 'Laser wavelength':
|
||||
m = re.match(r'(\d+)', val)
|
||||
if m:
|
||||
data['wavelength_nm'] = int(m.group(1))
|
||||
elif key == 'Laser bias current':
|
||||
m = re.match(r'([\d.]+)\s+mA', val)
|
||||
if m:
|
||||
data['bias_ma'] = float(m.group(1))
|
||||
elif key == 'Laser output power':
|
||||
m = re.search(r'/\s*([-\d.]+)\s*dBm', val)
|
||||
if m:
|
||||
try:
|
||||
data['tx_power_dbm'] = float(m.group(1))
|
||||
except ValueError:
|
||||
pass
|
||||
elif 'receiver' in key.lower() and ('power' in key.lower() or 'optical' in key.lower()):
|
||||
m = re.search(r'/\s*([-\d.]+)\s*dBm', val)
|
||||
if m:
|
||||
try:
|
||||
data['rx_power_dbm'] = float(m.group(1))
|
||||
except ValueError:
|
||||
pass
|
||||
elif key == 'Module temperature':
|
||||
m = re.match(r'([\d.]+)\s+degrees', val)
|
||||
if m:
|
||||
data['temp_c'] = float(m.group(1))
|
||||
elif key == 'Module voltage':
|
||||
m = re.match(r'([\d.]+)\s+V', val)
|
||||
if m:
|
||||
data['voltage_v'] = float(m.group(1))
|
||||
|
||||
return data
|
||||
|
||||
@staticmethod
|
||||
def parse_ip_link(text: str) -> dict:
|
||||
"""Parse ip -s link show output for basic link state and counters."""
|
||||
data: dict = {}
|
||||
lines = text.splitlines()
|
||||
for i, line in enumerate(lines):
|
||||
# MTU and state: "2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 ..."
|
||||
m = re.search(r'mtu\s+(\d+)', line)
|
||||
if m:
|
||||
data['mtu'] = int(m.group(1))
|
||||
m = re.search(r'state\s+(\S+)', line)
|
||||
if m:
|
||||
data['state'] = m.group(1).lower()
|
||||
# RX line follows "RX:" label
|
||||
if line.strip().startswith('RX:') and i + 1 < len(lines):
|
||||
vals = lines[i + 1].split()
|
||||
if len(vals) >= 5:
|
||||
try:
|
||||
data['ip_rx_bytes'] = int(vals[0])
|
||||
data['ip_rx_packets'] = int(vals[1])
|
||||
data['ip_rx_errors'] = int(vals[2])
|
||||
data['ip_rx_dropped'] = int(vals[3])
|
||||
except (ValueError, IndexError):
|
||||
pass
|
||||
if line.strip().startswith('TX:') and i + 1 < len(lines):
|
||||
vals = lines[i + 1].split()
|
||||
if len(vals) >= 5:
|
||||
try:
|
||||
data['ip_tx_bytes'] = int(vals[0])
|
||||
data['ip_tx_packets'] = int(vals[1])
|
||||
data['ip_tx_errors'] = int(vals[2])
|
||||
data['ip_tx_dropped'] = int(vals[3])
|
||||
except (ValueError, IndexError):
|
||||
pass
|
||||
return data
|
||||
|
||||
@staticmethod
|
||||
def parse_dmesg(text: str) -> List[dict]:
|
||||
"""Parse dmesg lines into [{timestamp, msg, severity}]."""
|
||||
events = []
|
||||
for line in text.splitlines():
|
||||
if not line.strip():
|
||||
continue
|
||||
# Extract timestamp from [ 123.456789]
|
||||
m = re.match(r'^\[\s*([\d.]+)\]\s*(.*)', line)
|
||||
if m:
|
||||
ts = m.group(1)
|
||||
msg = m.group(2)
|
||||
else:
|
||||
ts = ''
|
||||
msg = line
|
||||
|
||||
lower = msg.lower()
|
||||
if any(w in lower for w in ('error', 'fail', 'reset', 'panic', 'oops', 'hung', 'timeout')):
|
||||
severity = 'error'
|
||||
elif any(w in lower for w in ('warn', 'drop', 'lost', 'miss')):
|
||||
severity = 'warn'
|
||||
else:
|
||||
severity = 'info'
|
||||
|
||||
events.append({'timestamp': ts, 'msg': msg, 'severity': severity})
|
||||
return events
|
||||
|
||||
@staticmethod
|
||||
def parse_lldpctl(text: str) -> dict:
|
||||
"""Extract neighbor info from lldpctl output."""
|
||||
if not text or 'lldpd not running' in text or 'not found' in text.lower():
|
||||
return {'available': False}
|
||||
|
||||
data: dict = {'available': True}
|
||||
for line in text.splitlines():
|
||||
if ':' not in line:
|
||||
continue
|
||||
key, _, val = line.partition(':')
|
||||
key = key.strip()
|
||||
val = val.strip()
|
||||
if 'SysName' in key:
|
||||
data['neighbor_system'] = val
|
||||
elif 'PortID' in key and 'neighbor_port' not in data:
|
||||
data['neighbor_port'] = val
|
||||
elif 'ChassisID' in key and 'neighbor_chassis_id' not in data:
|
||||
data['neighbor_chassis_id'] = val
|
||||
return data
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Health analysis
|
||||
# ------------------------------------------------------------------
|
||||
@staticmethod
|
||||
def analyze(sections: dict, switch_port_data: dict) -> dict:
|
||||
"""Return {issues: [...], warnings: [...], info: [...]} health analysis."""
|
||||
issues: List[dict] = []
|
||||
warnings: List[dict] = []
|
||||
info: List[dict] = []
|
||||
|
||||
def add(collection, code, message):
|
||||
collection.append({'code': code, 'message': message})
|
||||
|
||||
carrier = sections.get('carrier', '?')
|
||||
eth = sections.get('ethtool', {})
|
||||
sysfs = sections.get('sysfs_stats', {})
|
||||
dom = sections.get('ethtool_dom', {})
|
||||
dmesg = sections.get('dmesg', [])
|
||||
lldp = sections.get('lldpctl', {})
|
||||
cc = sections.get('carrier_changes')
|
||||
|
||||
# Physical carrier
|
||||
if carrier == '0':
|
||||
add(issues, 'NO_CARRIER',
|
||||
'No physical carrier — cable/SFP disconnected or switch port disabled')
|
||||
elif eth.get('link_detected') is False and carrier != '0':
|
||||
add(issues, 'LINK_NOT_DETECTED',
|
||||
'NIC does not detect link signal despite carrier sysfs reading non-zero')
|
||||
|
||||
# Duplex
|
||||
if eth.get('duplex') == 'half':
|
||||
add(issues, 'HALF_DUPLEX',
|
||||
'Half-duplex detected — likely duplex mismatch; force full-duplex on both ends')
|
||||
|
||||
# Speed mismatch (switch vs server NIC)
|
||||
sw_speed = switch_port_data.get('speed_mbps', 0) or 0
|
||||
srv_speed = eth.get('speed_mbps', 0) or 0
|
||||
if sw_speed > 0 and srv_speed > 0 and sw_speed != srv_speed:
|
||||
add(warnings, 'SPEED_MISMATCH',
|
||||
f'Speed mismatch: switch reports {sw_speed} Mbps, NIC reports {srv_speed} Mbps')
|
||||
|
||||
# SFP DOM power levels
|
||||
rx_dbm = dom.get('rx_power_dbm')
|
||||
tx_dbm = dom.get('tx_power_dbm')
|
||||
if rx_dbm is not None:
|
||||
if rx_dbm < -25:
|
||||
add(issues, 'SFP_RX_CRITICAL',
|
||||
f'RX power critically low ({rx_dbm:.2f} dBm) — fiber not connected or SFP failed')
|
||||
elif rx_dbm < -18:
|
||||
add(warnings, 'SFP_RX_LOW',
|
||||
f'RX power low ({rx_dbm:.2f} dBm) — check fiber cleanliness and SFP seating')
|
||||
if tx_dbm is not None and tx_dbm < -10:
|
||||
add(warnings, 'SFP_TX_LOW',
|
||||
f'TX power low ({tx_dbm:.2f} dBm) — SFP may be failing or requires cleaning')
|
||||
|
||||
# Carrier changes (flapping)
|
||||
if cc is not None:
|
||||
if cc > 100:
|
||||
add(issues, 'CARRIER_FLAPPING',
|
||||
f'Link has flapped {cc} times — severe physical instability')
|
||||
elif cc > 20:
|
||||
add(warnings, 'CARRIER_FLAPS',
|
||||
f'Link has flapped {cc} times — intermittent physical issue')
|
||||
|
||||
# CRC errors
|
||||
crc = sysfs.get('rx_crc_errors', 0) or 0
|
||||
if crc > 100:
|
||||
add(issues, 'CRC_ERRORS_HIGH',
|
||||
f'High CRC error count ({crc}) — dirty fiber/connector or cable damage')
|
||||
elif crc > 10:
|
||||
add(warnings, 'CRC_ERRORS_LOW',
|
||||
f'CRC errors present ({crc}) — cable or SFP quality issue')
|
||||
|
||||
# Kernel events
|
||||
err_events = [e for e in dmesg if e['severity'] == 'error']
|
||||
if err_events:
|
||||
add(warnings, 'KERNEL_EVENTS',
|
||||
f'{len(err_events)} recent kernel error event(s) for this interface in dmesg')
|
||||
|
||||
# LLDP validation
|
||||
if lldp.get('available'):
|
||||
sw_lldp = switch_port_data.get('lldp') or {}
|
||||
sw_system = (sw_lldp.get('system_name') or '').lower()
|
||||
srv_neighbor = (lldp.get('neighbor_system') or '').lower()
|
||||
if sw_system and srv_neighbor and sw_system not in srv_neighbor and srv_neighbor not in sw_system:
|
||||
add(warnings, 'LLDP_MISMATCH',
|
||||
f'LLDP mismatch: switch sees "{sw_lldp.get("system_name")}" but '
|
||||
f'server lldpctl sees "{lldp.get("neighbor_system")}" — cross-cabled port?')
|
||||
else:
|
||||
add(info, 'LLDP_MISSING',
|
||||
'lldpd not running on server — install lldpd for full path validation')
|
||||
|
||||
return {'issues': issues, 'warnings': warnings, 'info': info}
|
||||
@@ -0,0 +1,22 @@
|
||||
[Unit]
|
||||
Description=Gandalf Network Monitor Daemon
|
||||
Documentation=https://gitea.lotusguild.org/LotusGuild/gandalf
|
||||
After=network.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=www-data
|
||||
WorkingDirectory=/var/www/html/prod
|
||||
ExecStart=/usr/bin/python3 /var/www/html/prod/monitor.py
|
||||
Restart=on-failure
|
||||
RestartSec=30
|
||||
TimeoutStopSec=10
|
||||
|
||||
# Logging
|
||||
StandardOutput=journal
|
||||
StandardError=journal
|
||||
SyslogIdentifier=gandalf-monitor
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
+976
@@ -0,0 +1,976 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Gandalf network monitor daemon.
|
||||
|
||||
Polls Prometheus (node_exporter) and the UniFi controller for network
|
||||
interface and device state. Creates tickets in Tinker Tickets when issues
|
||||
are detected, with deduplication and suppression support.
|
||||
|
||||
Run as a separate systemd service alongside the Flask web app.
|
||||
"""
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import shlex
|
||||
import subprocess
|
||||
import time
|
||||
from datetime import datetime
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
import requests
|
||||
from urllib3.exceptions import InsecureRequestWarning
|
||||
|
||||
import db
|
||||
|
||||
requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format='%(asctime)s %(levelname)s %(name)s %(message)s',
|
||||
)
|
||||
logger = logging.getLogger('gandalf.monitor')
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Interface filtering
|
||||
# --------------------------------------------------------------------------
|
||||
_SKIP_PREFIXES = (
|
||||
'lo', 'veth', 'tap', 'fwbr', 'fwln', 'fwpr',
|
||||
'docker', 'dummy', 'br-', 'virbr', 'vmbr',
|
||||
)
|
||||
_VLAN_SUFFIX = re.compile(r'\.\d+$')
|
||||
|
||||
|
||||
def is_physical_interface(name: str) -> bool:
|
||||
"""Return True for physical/bond interfaces worth monitoring."""
|
||||
if any(name.startswith(p) for p in _SKIP_PREFIXES):
|
||||
return False
|
||||
if _VLAN_SUFFIX.search(name):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Prometheus client
|
||||
# --------------------------------------------------------------------------
|
||||
class PrometheusClient:
|
||||
def __init__(self, url: str):
|
||||
self.url = url.rstrip('/')
|
||||
|
||||
def query(self, promql: str) -> list:
|
||||
try:
|
||||
resp = requests.get(
|
||||
f'{self.url}/api/v1/query',
|
||||
params={'query': promql},
|
||||
timeout=15,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
if data.get('status') == 'success':
|
||||
return data['data']['result']
|
||||
except Exception as e:
|
||||
logger.error(f'Prometheus query failed ({promql!r}): {e}')
|
||||
return []
|
||||
|
||||
def get_interface_states(self) -> Dict[str, Dict[str, bool]]:
|
||||
"""Return {instance: {device: is_up}} for physical interfaces."""
|
||||
results = self.query('node_network_up')
|
||||
hosts: Dict[str, Dict[str, bool]] = {}
|
||||
for r in results:
|
||||
instance = r['metric'].get('instance', '')
|
||||
device = r['metric'].get('device', '')
|
||||
if not is_physical_interface(device):
|
||||
continue
|
||||
hosts.setdefault(instance, {})[device] = (r['value'][1] == '1')
|
||||
return hosts
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# UniFi client
|
||||
# --------------------------------------------------------------------------
|
||||
class UnifiClient:
|
||||
def __init__(self, cfg: dict):
|
||||
self.base_url = cfg['controller']
|
||||
self.site_id = cfg.get('site_id', 'default')
|
||||
self.session = requests.Session()
|
||||
self.session.verify = False
|
||||
self.headers = {
|
||||
'X-API-KEY': cfg['api_key'],
|
||||
'Accept': 'application/json',
|
||||
}
|
||||
|
||||
def get_devices(self) -> Optional[List[dict]]:
|
||||
"""Return list of UniFi devices, or None if the controller is unreachable."""
|
||||
try:
|
||||
url = f'{self.base_url}/proxy/network/v2/api/site/{self.site_id}/device'
|
||||
resp = self.session.get(url, headers=self.headers, timeout=15)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
devices = []
|
||||
for d in data.get('network_devices', []):
|
||||
state = d.get('state', 1)
|
||||
devices.append({
|
||||
'name': d.get('name') or d.get('mac', 'unknown'),
|
||||
'mac': d.get('mac', ''),
|
||||
'ip': d.get('ip', ''),
|
||||
'type': d.get('type', 'unknown'),
|
||||
'model': d.get('model', ''),
|
||||
'state': state,
|
||||
'connected': state == 1,
|
||||
})
|
||||
return devices
|
||||
except Exception as e:
|
||||
logger.error(f'UniFi API error: {e}')
|
||||
return None
|
||||
|
||||
def get_switch_ports(self) -> Optional[Dict[str, dict]]:
|
||||
"""Return per-port stats for all UniFi switches, keyed by switch name.
|
||||
|
||||
Uses the v1 stat API which includes full port_table data.
|
||||
Returns {switch_name: {'ip': str, 'model': str, 'ports': {port_name: {...}}}}.
|
||||
"""
|
||||
try:
|
||||
url = f'{self.base_url}/proxy/network/api/s/{self.site_id}/stat/device'
|
||||
resp = self.session.get(url, headers=self.headers, timeout=15)
|
||||
resp.raise_for_status()
|
||||
devices = resp.json().get('data', [])
|
||||
result: Dict[str, dict] = {}
|
||||
for dev in devices:
|
||||
if dev.get('type', '').lower() != 'usw':
|
||||
continue
|
||||
sw_name = dev.get('name') or dev.get('mac', 'unknown')
|
||||
sw_ip = dev.get('ip', '')
|
||||
sw_model = dev.get('model', '')
|
||||
ports: Dict[str, dict] = {}
|
||||
# Build LLDP neighbor map (keyed by port_idx)
|
||||
lldp_map: Dict[int, dict] = {}
|
||||
for entry in dev.get('lldp_table', []):
|
||||
pidx = entry.get('lldp_port_idx')
|
||||
if pidx is not None:
|
||||
lldp_map[int(pidx)] = {
|
||||
'chassis_id': entry.get('chassis_id', ''),
|
||||
'system_name': entry.get('system_name', ''),
|
||||
'port_id': entry.get('port_id', ''),
|
||||
'port_desc': entry.get('port_desc', ''),
|
||||
'mgmt_ips': entry.get('management_ips', []),
|
||||
}
|
||||
for port in dev.get('port_table', []):
|
||||
idx = port.get('port_idx', 0)
|
||||
pname = port.get('name') or f'Port {idx}'
|
||||
raw_poe = port.get('poe_power')
|
||||
raw_poe_max = port.get('poe_max_power')
|
||||
ports[pname] = {
|
||||
'port_idx': idx,
|
||||
'switch_ip': sw_ip,
|
||||
'up': port.get('up', False),
|
||||
'speed_mbps': port.get('speed', 0),
|
||||
'full_duplex': port.get('full_duplex', False),
|
||||
'autoneg': port.get('autoneg', False),
|
||||
'is_uplink': port.get('is_uplink', False),
|
||||
'media': port.get('media', ''),
|
||||
'poe_power': float(raw_poe) if raw_poe is not None else None,
|
||||
'poe_class': port.get('poe_class'),
|
||||
'poe_max_power': float(raw_poe_max) if raw_poe_max is not None else None,
|
||||
'poe_mode': port.get('poe_mode', ''),
|
||||
'lldp': lldp_map.get(idx),
|
||||
'tx_bytes': port.get('tx_bytes', 0),
|
||||
'rx_bytes': port.get('rx_bytes', 0),
|
||||
'tx_errors': port.get('tx_errors', 0),
|
||||
'rx_errors': port.get('rx_errors', 0),
|
||||
'tx_dropped': port.get('tx_dropped', 0),
|
||||
'rx_dropped': port.get('rx_dropped', 0),
|
||||
}
|
||||
if ports:
|
||||
result[sw_name] = {'ip': sw_ip, 'model': sw_model, 'ports': ports}
|
||||
return result
|
||||
except Exception as e:
|
||||
logger.error(f'UniFi switch port stats error: {e}')
|
||||
return None
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Ticket client
|
||||
# --------------------------------------------------------------------------
|
||||
class TicketClient:
|
||||
def __init__(self, cfg: dict):
|
||||
self.url = cfg.get('url', '')
|
||||
self.api_key = cfg.get('api_key', '')
|
||||
|
||||
def create(self, title: str, description: str, priority: str = '2') -> Optional[str]:
|
||||
if not self.api_key or not self.url:
|
||||
logger.warning('Ticket API not configured – skipping ticket creation')
|
||||
return None
|
||||
try:
|
||||
resp = requests.post(
|
||||
self.url,
|
||||
json={
|
||||
'title': title,
|
||||
'description': description,
|
||||
'status': 'Open',
|
||||
'priority': priority,
|
||||
'category': 'Network',
|
||||
'type': 'Issue',
|
||||
},
|
||||
headers={'Authorization': f'Bearer {self.api_key}'},
|
||||
timeout=15,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
if data.get('success'):
|
||||
tid = data['ticket_id']
|
||||
logger.info(f'Created ticket #{tid}: {title}')
|
||||
return tid
|
||||
if data.get('existing_ticket_id'):
|
||||
logger.info(f'Duplicate suppressed by API – existing #{data["existing_ticket_id"]}')
|
||||
return data['existing_ticket_id']
|
||||
logger.warning(f'Unexpected ticket API response: {data}')
|
||||
except Exception as e:
|
||||
logger.error(f'Ticket creation failed: {e}')
|
||||
return None
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Pulse HTTP client (delegates SSH commands to Pulse worker)
|
||||
# --------------------------------------------------------------------------
|
||||
class PulseClient:
|
||||
"""Submit a command to a Pulse worker via the internal M2M API and poll for result."""
|
||||
|
||||
def __init__(self, cfg: dict):
|
||||
p = cfg.get('pulse', {})
|
||||
self.url = p.get('url', '').rstrip('/')
|
||||
self.api_key = p.get('api_key', '')
|
||||
self.worker_id = p.get('worker_id', '')
|
||||
self.timeout = p.get('timeout', 45)
|
||||
self.last_execution_id: Optional[str] = None
|
||||
self.session = requests.Session()
|
||||
self.session.headers.update({
|
||||
'X-Gandalf-API-Key': self.api_key,
|
||||
'Content-Type': 'application/json',
|
||||
})
|
||||
|
||||
def run_command(self, command: str, _retry: bool = True) -> Optional[str]:
|
||||
"""Submit *command* to Pulse, poll until done, return stdout or None.
|
||||
|
||||
Retries once automatically on transient submit failures or timeouts.
|
||||
"""
|
||||
self.last_execution_id = None
|
||||
if not self.url or not self.api_key or not self.worker_id:
|
||||
return None
|
||||
|
||||
# Submit
|
||||
try:
|
||||
resp = self.session.post(
|
||||
f'{self.url}/api/internal/command',
|
||||
json={'worker_id': self.worker_id, 'command': command},
|
||||
timeout=10,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
execution_id = resp.json()['execution_id']
|
||||
self.last_execution_id = execution_id
|
||||
except Exception as e:
|
||||
logger.error(f'Pulse command submit failed: {e}')
|
||||
if _retry:
|
||||
logger.info('Retrying Pulse command submit in 5s...')
|
||||
time.sleep(5)
|
||||
return self.run_command(command, _retry=False)
|
||||
return None
|
||||
|
||||
# Poll
|
||||
deadline = time.time() + self.timeout
|
||||
while time.time() < deadline:
|
||||
time.sleep(2)
|
||||
try:
|
||||
r = self.session.get(
|
||||
f'{self.url}/api/internal/executions/{execution_id}',
|
||||
timeout=10,
|
||||
)
|
||||
r.raise_for_status()
|
||||
data = r.json()
|
||||
status = data.get('status')
|
||||
if status == 'completed':
|
||||
logs = data.get('logs', [])
|
||||
for entry in logs:
|
||||
if entry.get('action') == 'command_result':
|
||||
return entry.get('stdout', '')
|
||||
return ''
|
||||
if status in ('failed', 'timed_out', 'cancelled'):
|
||||
logger.error(
|
||||
f'Pulse execution {execution_id} ended with status={status!r}; '
|
||||
f'view at {self.url}/executions/{execution_id}'
|
||||
)
|
||||
if _retry and status != 'cancelled':
|
||||
logger.info('Retrying failed Pulse command in 5s...')
|
||||
time.sleep(5)
|
||||
return self.run_command(command, _retry=False)
|
||||
return None
|
||||
except Exception as e:
|
||||
logger.error(f'Pulse poll failed for {execution_id}: {e}')
|
||||
|
||||
logger.warning(
|
||||
f'Pulse command timed out after {self.timeout}s '
|
||||
f'(execution_id={execution_id}); '
|
||||
f'view at {self.url}/executions/{execution_id}'
|
||||
)
|
||||
if _retry:
|
||||
logger.info('Retrying timed-out Pulse command in 5s...')
|
||||
time.sleep(5)
|
||||
return self.run_command(command, _retry=False)
|
||||
return None
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Link stats collector (ethtool + Prometheus traffic metrics)
|
||||
# --------------------------------------------------------------------------
|
||||
class LinkStatsCollector:
|
||||
"""Collects detailed per-interface statistics via SSH (ethtool) and Prometheus,
|
||||
plus per-port stats from UniFi switches."""
|
||||
|
||||
def __init__(self, cfg: dict, prom: 'PrometheusClient',
|
||||
unifi: Optional['UnifiClient'] = None):
|
||||
self.cfg = cfg
|
||||
self.prom = prom
|
||||
self.pulse = PulseClient(cfg)
|
||||
self.unifi = unifi
|
||||
# State for UniFi rate calculation (previous snapshot + timestamp)
|
||||
self._prev_unifi: Dict[str, dict] = {}
|
||||
self._prev_unifi_time: float = 0.0
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# SSH collection (via Pulse worker)
|
||||
# ------------------------------------------------------------------
|
||||
def _ssh_batch(self, ip: str, ifaces: List[str]) -> Dict[str, dict]:
|
||||
"""
|
||||
Delegate one SSH session to the Pulse worker to collect ethtool + SFP DOM
|
||||
data for all *ifaces*. Returns {iface: {speed_mbps, duplex, ..., sfp: {...}}}.
|
||||
"""
|
||||
if not ifaces or not self.pulse.url:
|
||||
return {}
|
||||
|
||||
# Validate interface names (kernel names only contain [a-zA-Z0-9_.-])
|
||||
safe_ifaces = [i for i in ifaces if re.match(r'^[a-zA-Z0-9_.@-]+$', i)]
|
||||
if not safe_ifaces:
|
||||
return {}
|
||||
|
||||
# Build a single shell command: for each iface output ethtool + -m with sentinels
|
||||
parts = []
|
||||
for iface in safe_ifaces:
|
||||
q = shlex.quote(iface)
|
||||
parts.append(
|
||||
f'echo "___IFACE:{iface}___";'
|
||||
f' ethtool {q} 2>/dev/null;'
|
||||
f' echo "___DOM:{iface}___";'
|
||||
f' ethtool -m {q} 2>/dev/null;'
|
||||
f' echo "___END___"'
|
||||
)
|
||||
shell_cmd = ' '.join(parts)
|
||||
|
||||
ssh_cmd = (
|
||||
f'ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 '
|
||||
f'-o BatchMode=yes -o LogLevel=ERROR '
|
||||
f'-o ServerAliveInterval=10 -o ServerAliveCountMax=2 '
|
||||
f'root@{ip} "{shell_cmd}"'
|
||||
)
|
||||
output = self.pulse.run_command(ssh_cmd)
|
||||
if output is None:
|
||||
logger.error(f'Pulse ethtool collection returned None for {ip}')
|
||||
return {}
|
||||
|
||||
return self._parse_ssh_output(output)
|
||||
|
||||
@staticmethod
|
||||
def _parse_ssh_output(output: str) -> Dict[str, dict]:
|
||||
result: Dict[str, dict] = {}
|
||||
current_iface: Optional[str] = None
|
||||
current_section: Optional[str] = None
|
||||
buf: List[str] = []
|
||||
|
||||
def flush(iface, section, lines):
|
||||
if not iface or not lines:
|
||||
return
|
||||
text = '\n'.join(lines)
|
||||
if section == 'ethtool':
|
||||
result.setdefault(iface, {}).update(
|
||||
LinkStatsCollector._parse_ethtool(text)
|
||||
)
|
||||
elif section == 'dom':
|
||||
sfp = LinkStatsCollector._parse_ethtool_m(text)
|
||||
if sfp:
|
||||
result.setdefault(iface, {})['sfp'] = sfp
|
||||
|
||||
for line in output.splitlines():
|
||||
if line.startswith('___IFACE:') and line.endswith('___'):
|
||||
flush(current_iface, current_section, buf)
|
||||
current_iface = line[9:-3]
|
||||
current_section = 'ethtool'
|
||||
buf = []
|
||||
elif line.startswith('___DOM:') and line.endswith('___'):
|
||||
flush(current_iface, current_section, buf)
|
||||
current_iface = line[7:-3]
|
||||
current_section = 'dom'
|
||||
buf = []
|
||||
elif line == '___END___':
|
||||
flush(current_iface, current_section, buf)
|
||||
current_iface = None
|
||||
current_section = None
|
||||
buf = []
|
||||
else:
|
||||
buf.append(line)
|
||||
|
||||
flush(current_iface, current_section, buf)
|
||||
return result
|
||||
|
||||
@staticmethod
|
||||
def _parse_ethtool(output: str) -> dict:
|
||||
data: dict = {}
|
||||
for line in output.splitlines():
|
||||
if ':' not in line:
|
||||
continue
|
||||
key, _, val = line.partition(':')
|
||||
key = key.strip()
|
||||
val = val.strip()
|
||||
if key == 'Speed':
|
||||
m = re.match(r'(\d+)Mb/s', val)
|
||||
if m:
|
||||
data['speed_mbps'] = int(m.group(1))
|
||||
elif key == 'Duplex':
|
||||
data['duplex'] = val.lower()
|
||||
elif key == 'Port':
|
||||
data['port_type'] = val
|
||||
elif key == 'Auto-negotiation':
|
||||
data['auto_neg'] = (val.lower() == 'on')
|
||||
elif key == 'Link detected':
|
||||
data['link_detected'] = (val.lower() == 'yes')
|
||||
return data
|
||||
|
||||
@staticmethod
|
||||
def _parse_ethtool_m(output: str) -> dict:
|
||||
"""Parse ethtool -m (SFP DOM / digital optical monitoring) output."""
|
||||
if not output:
|
||||
return {}
|
||||
# Skip if module diagnostics unsupported
|
||||
lower = output.lower()
|
||||
if 'cannot get' in lower or 'not supported' in lower or 'no sfp' in lower:
|
||||
return {}
|
||||
|
||||
data: dict = {}
|
||||
for line in output.splitlines():
|
||||
if ':' not in line:
|
||||
continue
|
||||
key, _, val = line.partition(':')
|
||||
key = key.strip()
|
||||
val = val.strip()
|
||||
|
||||
if key == 'Vendor name':
|
||||
data['vendor'] = val
|
||||
elif key == 'Vendor PN':
|
||||
data['part_no'] = val
|
||||
elif key == 'Identifier':
|
||||
m = re.search(r'\((.+?)\)', val)
|
||||
if m:
|
||||
data['sfp_type'] = m.group(1)
|
||||
elif key == 'Connector':
|
||||
m = re.search(r'\((.+?)\)', val)
|
||||
if m:
|
||||
data['connector'] = m.group(1)
|
||||
elif key == 'Laser wavelength':
|
||||
m = re.match(r'(\d+)', val)
|
||||
if m:
|
||||
data['wavelength_nm'] = int(m.group(1))
|
||||
elif key == 'Laser bias current':
|
||||
# e.g. "4.340 mA"
|
||||
m = re.match(r'([\d.]+)\s+mA', val)
|
||||
if m:
|
||||
data['bias_ma'] = float(m.group(1))
|
||||
elif key == 'Laser output power':
|
||||
# e.g. "0.1234 mW / -9.09 dBm"
|
||||
m = re.search(r'/\s*([-\d.]+)\s*dBm', val)
|
||||
if m:
|
||||
try:
|
||||
data['tx_power_dbm'] = float(m.group(1))
|
||||
except ValueError:
|
||||
pass
|
||||
elif 'receiver' in key.lower() and ('power' in key.lower() or 'optical' in key.lower()):
|
||||
m = re.search(r'/\s*([-\d.]+)\s*dBm', val)
|
||||
if m:
|
||||
try:
|
||||
data['rx_power_dbm'] = float(m.group(1))
|
||||
except ValueError:
|
||||
pass
|
||||
elif key == 'Module temperature':
|
||||
# e.g. "36.00 degrees C / 96.80 degrees F"
|
||||
m = re.match(r'([\d.]+)\s+degrees', val)
|
||||
if m:
|
||||
data['temp_c'] = float(m.group(1))
|
||||
elif key == 'Module voltage':
|
||||
# e.g. "3.3180 V"
|
||||
m = re.match(r'([\d.]+)\s+V', val)
|
||||
if m:
|
||||
data['voltage_v'] = float(m.group(1))
|
||||
|
||||
return data
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Prometheus traffic / error metrics
|
||||
# ------------------------------------------------------------------
|
||||
def _collect_prom_metrics(self) -> Dict[str, Dict[str, dict]]:
|
||||
"""Return {instance: {device: {tx_bytes_rate, rx_bytes_rate, ...}}}."""
|
||||
metrics: Dict[str, Dict[str, dict]] = {}
|
||||
|
||||
queries = [
|
||||
('tx_bytes_rate', 'rate(node_network_transmit_bytes_total[5m])'),
|
||||
('rx_bytes_rate', 'rate(node_network_receive_bytes_total[5m])'),
|
||||
('tx_errs_rate', 'rate(node_network_transmit_errs_total[5m])'),
|
||||
('rx_errs_rate', 'rate(node_network_receive_errs_total[5m])'),
|
||||
('tx_drops_rate', 'rate(node_network_transmit_drop_total[5m])'),
|
||||
('rx_drops_rate', 'rate(node_network_receive_drop_total[5m])'),
|
||||
('carrier_changes', 'node_network_carrier_changes_total'),
|
||||
]
|
||||
|
||||
for field, promql in queries:
|
||||
for r in self.prom.query(promql):
|
||||
instance = r['metric'].get('instance', '')
|
||||
device = r['metric'].get('device', '')
|
||||
if not is_physical_interface(device):
|
||||
continue
|
||||
raw = r['value'][1]
|
||||
try:
|
||||
val: Optional[float] = float(raw)
|
||||
if val != val: # NaN
|
||||
val = None
|
||||
except (ValueError, TypeError):
|
||||
val = None
|
||||
metrics.setdefault(instance, {}).setdefault(device, {})[field] = val
|
||||
|
||||
return metrics
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Main collection entry point
|
||||
# ------------------------------------------------------------------
|
||||
def collect(self, instance_map: Dict[str, str]) -> dict:
|
||||
"""
|
||||
Collect full link stats for all Prometheus-monitored hosts.
|
||||
|
||||
*instance_map*: ``{'10.10.10.2:9100': 'large1', ...}``
|
||||
|
||||
Returns a dict suitable for ``db.set_state('link_stats', ...)``.
|
||||
"""
|
||||
prom_metrics = self._collect_prom_metrics()
|
||||
result_hosts: Dict[str, Dict[str, dict]] = {}
|
||||
exclude_ips = set(self.cfg.get('monitor', {}).get('links_exclude_ips', []))
|
||||
|
||||
for instance, iface_metrics in prom_metrics.items():
|
||||
host_ip = instance.split(':')[0]
|
||||
if host_ip in exclude_ips:
|
||||
continue
|
||||
host = instance_map.get(instance, host_ip)
|
||||
ifaces = list(iface_metrics.keys())
|
||||
|
||||
# SSH ethtool collection via Pulse worker — only for explicitly configured
|
||||
# hosts (instance_map keys). Hosts like postgresql/matrix may report
|
||||
# node_exporter metrics to Prometheus but don't need link diagnostics.
|
||||
ethtool_data: Dict[str, dict] = {}
|
||||
if self.pulse.url and ifaces and instance in instance_map:
|
||||
try:
|
||||
ethtool_data = self._ssh_batch(host_ip, ifaces)
|
||||
except Exception as e:
|
||||
logger.warning(f'ethtool collection failed for {host} ({host_ip}): {e}')
|
||||
|
||||
# Merge Prometheus metrics + ethtool data per interface
|
||||
merged: Dict[str, dict] = {}
|
||||
for iface in ifaces:
|
||||
d: dict = {'host_ip': host_ip}
|
||||
d.update(iface_metrics.get(iface, {}))
|
||||
eth = ethtool_data.get(iface, {})
|
||||
for k, v in eth.items():
|
||||
if k != 'sfp':
|
||||
d[k] = v
|
||||
if 'sfp' in eth:
|
||||
d['sfp'] = eth['sfp']
|
||||
merged[iface] = d
|
||||
|
||||
result_hosts[host] = merged
|
||||
|
||||
# Collect UniFi switch port stats
|
||||
unifi_switches: dict = {}
|
||||
if self.unifi:
|
||||
try:
|
||||
raw = self.unifi.get_switch_ports()
|
||||
if raw is not None:
|
||||
now = time.time()
|
||||
unifi_switches = self._compute_unifi_rates(raw, now)
|
||||
self._prev_unifi = raw
|
||||
self._prev_unifi_time = now
|
||||
except Exception as e:
|
||||
logger.warning(f'UniFi switch port collection failed: {e}')
|
||||
|
||||
return {
|
||||
'hosts': result_hosts,
|
||||
'unifi_switches': unifi_switches,
|
||||
'updated': datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S UTC'),
|
||||
}
|
||||
|
||||
def _compute_unifi_rates(self, raw: Dict[str, dict], now: float) -> Dict[str, dict]:
|
||||
"""Compute per-port byte/error rates from delta against previous snapshot."""
|
||||
dt = now - self._prev_unifi_time if self._prev_unifi_time > 0 else 0
|
||||
|
||||
def rate(new_val: int, old_val: int) -> Optional[float]:
|
||||
if dt <= 0:
|
||||
return None
|
||||
return max(0.0, (new_val - old_val) / dt)
|
||||
|
||||
result: Dict[str, dict] = {}
|
||||
for sw_name, sw_data in raw.items():
|
||||
prev_ports = self._prev_unifi.get(sw_name, {}).get('ports', {})
|
||||
merged_ports: Dict[str, dict] = {}
|
||||
for pname, d in sw_data['ports'].items():
|
||||
entry = dict(d)
|
||||
prev = prev_ports.get(pname, {})
|
||||
entry['tx_bytes_rate'] = rate(d['tx_bytes'], prev.get('tx_bytes', 0))
|
||||
entry['rx_bytes_rate'] = rate(d['rx_bytes'], prev.get('rx_bytes', 0))
|
||||
entry['tx_errs_rate'] = rate(d['tx_errors'], prev.get('tx_errors', 0))
|
||||
entry['rx_errs_rate'] = rate(d['rx_errors'], prev.get('rx_errors', 0))
|
||||
entry['tx_drops_rate'] = rate(d['tx_dropped'], prev.get('tx_dropped', 0))
|
||||
entry['rx_drops_rate'] = rate(d['rx_dropped'], prev.get('rx_dropped', 0))
|
||||
merged_ports[pname] = entry
|
||||
result[sw_name] = {'ip': sw_data['ip'], 'model': sw_data['model'],
|
||||
'ports': merged_ports}
|
||||
return result
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# --------------------------------------------------------------------------
|
||||
def ping(ip: str, count: int = 3, timeout: int = 2) -> bool:
|
||||
try:
|
||||
r = subprocess.run(
|
||||
['ping', '-c', str(count), '-W', str(timeout), ip],
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL,
|
||||
timeout=30,
|
||||
)
|
||||
return r.returncode == 0
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def _now_utc() -> str:
|
||||
return datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S UTC')
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Monitor
|
||||
# --------------------------------------------------------------------------
|
||||
CLUSTER_NAME = 'proxmox-cluster'
|
||||
|
||||
|
||||
class NetworkMonitor:
|
||||
def __init__(self):
|
||||
with open('config.json') as f:
|
||||
self.cfg = json.load(f)
|
||||
|
||||
prom_url = self.cfg['prometheus']['url']
|
||||
self.prom = PrometheusClient(prom_url)
|
||||
self.unifi = UnifiClient(self.cfg['unifi'])
|
||||
self.tickets = TicketClient(self.cfg.get('ticket_api', {}))
|
||||
self.link_stats = LinkStatsCollector(self.cfg, self.prom, self.unifi)
|
||||
|
||||
mon = self.cfg.get('monitor', {})
|
||||
self.poll_interval = mon.get('poll_interval', 120)
|
||||
self.fail_thresh = mon.get('failure_threshold', 2)
|
||||
self.cluster_thresh = mon.get('cluster_threshold', 3)
|
||||
self.cluster_name = mon.get('cluster_name', CLUSTER_NAME)
|
||||
|
||||
# Build Prometheus instance → hostname lookup
|
||||
self._instance_map: Dict[str, str] = {
|
||||
h['prometheus_instance']: h['name']
|
||||
for h in self.cfg.get('hosts', [])
|
||||
if 'prometheus_instance' in h
|
||||
}
|
||||
|
||||
def _hostname(self, instance: str) -> str:
|
||||
return self._instance_map.get(instance, instance.split(':')[0])
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Interface monitoring (Prometheus)
|
||||
# ------------------------------------------------------------------
|
||||
def _process_interfaces(self, states: Dict[str, Dict[str, bool]], suppressions: list) -> None:
|
||||
baseline = db.get_baseline()
|
||||
new_baseline = {k: dict(v) for k, v in baseline.items()}
|
||||
# Only count hosts with genuine regressions (UP→DOWN) toward cluster threshold
|
||||
hosts_with_regression: List[str] = []
|
||||
|
||||
for instance, ifaces in states.items():
|
||||
if instance not in self._instance_map:
|
||||
continue # skip unconfigured Prometheus instances
|
||||
host = self._hostname(instance)
|
||||
new_baseline.setdefault(host, {})
|
||||
host_has_regression = False
|
||||
|
||||
for iface, is_up in ifaces.items():
|
||||
prev = baseline.get(host, {}).get(iface) # 'up', 'initial_down', or None
|
||||
|
||||
if is_up:
|
||||
new_baseline[host][iface] = 'up'
|
||||
db.resolve_event('interface_down', host, iface)
|
||||
else:
|
||||
if prev is None:
|
||||
# First observation is down – could be unused port, don't alert
|
||||
new_baseline[host][iface] = 'initial_down'
|
||||
|
||||
elif prev == 'initial_down':
|
||||
# Persistently down since first observation – no alert
|
||||
pass
|
||||
|
||||
else: # prev == 'up'
|
||||
# Regression: was UP, now DOWN
|
||||
host_has_regression = True
|
||||
sup = (
|
||||
db.check_suppressed(suppressions, 'interface', host, iface) or
|
||||
db.check_suppressed(suppressions, 'host', host)
|
||||
)
|
||||
event_id, is_new, consec = db.upsert_event(
|
||||
'interface_down', 'critical', 'prometheus',
|
||||
host, iface,
|
||||
f'Interface {iface} on {host} went link-down ({_now_utc()})',
|
||||
)
|
||||
if not sup and consec >= self.fail_thresh:
|
||||
self._ticket_interface(event_id, is_new, host, iface, consec)
|
||||
|
||||
if host_has_regression:
|
||||
hosts_with_regression.append(host)
|
||||
|
||||
db.set_baseline(new_baseline)
|
||||
|
||||
# Cluster-wide check – only genuine regressions count
|
||||
if len(hosts_with_regression) >= self.cluster_thresh:
|
||||
sup = db.check_suppressed(suppressions, 'all', '')
|
||||
event_id, is_new, consec = db.upsert_event(
|
||||
'cluster_network_issue', 'critical', 'prometheus',
|
||||
self.cluster_name, '',
|
||||
f'{len(hosts_with_regression)} hosts reporting simultaneous interface failures: '
|
||||
f'{", ".join(hosts_with_regression)}',
|
||||
)
|
||||
if not sup and is_new:
|
||||
title = (
|
||||
f'[{self.cluster_name}][auto][production][issue][network][cluster-wide] '
|
||||
f'Multiple hosts reporting interface failures'
|
||||
)
|
||||
desc = (
|
||||
f'Cluster Network Alert\n{"=" * 40}\n\n'
|
||||
f'Affected hosts: {", ".join(hosts_with_regression)}\n'
|
||||
f'Detected: {_now_utc()}\n\n'
|
||||
f'{len(hosts_with_regression)} Proxmox hosts simultaneously reported '
|
||||
f'interface regressions (link-down on interfaces previously known UP).\n'
|
||||
f'This likely indicates a switch or upstream network failure.\n\n'
|
||||
f'Please check the core and management switches immediately.'
|
||||
)
|
||||
tid = self.tickets.create(title, desc, priority='1')
|
||||
if tid:
|
||||
db.set_ticket_id(event_id, tid)
|
||||
else:
|
||||
db.resolve_event('cluster_network_issue', self.cluster_name, '')
|
||||
|
||||
def _ticket_interface(
|
||||
self, event_id: int, is_new: bool, host: str, iface: str, consec: int
|
||||
) -> None:
|
||||
title = (
|
||||
f'[{host}][auto][production][issue][network][single-node] '
|
||||
f'Interface {iface} link-down'
|
||||
)
|
||||
desc = (
|
||||
f'Network Interface Alert\n{"=" * 40}\n\n'
|
||||
f'Host: {host}\n'
|
||||
f'Interface: {iface}\n'
|
||||
f'Detected: {_now_utc()}\n'
|
||||
f'Consecutive check failures: {consec}\n\n'
|
||||
f'Interface {iface} on {host} is reporting link-down state via '
|
||||
f'Prometheus node_exporter.\n\n'
|
||||
f'Note: {host} may still be reachable via its other network interface.\n'
|
||||
f'Please inspect the cable/SFP/switch port for {host}/{iface}.'
|
||||
)
|
||||
tid = self.tickets.create(title, desc, priority='2')
|
||||
if tid and is_new:
|
||||
db.set_ticket_id(event_id, tid)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# UniFi device monitoring
|
||||
# ------------------------------------------------------------------
|
||||
def _process_unifi(self, devices: Optional[List[dict]], suppressions: list) -> None:
|
||||
if devices is None:
|
||||
logger.warning('UniFi API unreachable this cycle')
|
||||
return
|
||||
|
||||
for d in devices:
|
||||
name = d['name']
|
||||
if not d['connected']:
|
||||
sup = db.check_suppressed(suppressions, 'unifi_device', name)
|
||||
event_id, is_new, consec = db.upsert_event(
|
||||
'unifi_device_offline', 'critical', 'unifi',
|
||||
name, d.get('type', ''),
|
||||
f'UniFi {name} ({d.get("ip","")}) offline ({_now_utc()})',
|
||||
)
|
||||
if not sup and consec >= self.fail_thresh:
|
||||
self._ticket_unifi(event_id, is_new, d)
|
||||
else:
|
||||
db.resolve_event('unifi_device_offline', name, d.get('type', ''))
|
||||
|
||||
def _ticket_unifi(self, event_id: int, is_new: bool, device: dict) -> None:
|
||||
name = device['name']
|
||||
title = (
|
||||
f'[{name}][auto][production][issue][network][single-node] '
|
||||
f'UniFi device offline'
|
||||
)
|
||||
desc = (
|
||||
f'UniFi Device Alert\n{"=" * 40}\n\n'
|
||||
f'Device: {name}\n'
|
||||
f'Type: {device.get("type","unknown")}\n'
|
||||
f'Model: {device.get("model","")}\n'
|
||||
f'Last Known IP: {device.get("ip","unknown")}\n'
|
||||
f'Detected: {_now_utc()}\n\n'
|
||||
f'The UniFi device {name} is offline per the UniFi controller.\n'
|
||||
f'Please check power and cable connectivity.'
|
||||
)
|
||||
tid = self.tickets.create(title, desc, priority='2')
|
||||
if tid and is_new:
|
||||
db.set_ticket_id(event_id, tid)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Ping-only hosts (no node_exporter)
|
||||
# ------------------------------------------------------------------
|
||||
def _process_ping_hosts(self, suppressions: list) -> None:
|
||||
for h in self.cfg.get('monitor', {}).get('ping_hosts', []):
|
||||
name, ip = h['name'], h['ip']
|
||||
reachable = ping(ip)
|
||||
|
||||
if not reachable:
|
||||
sup = db.check_suppressed(suppressions, 'host', name)
|
||||
event_id, is_new, consec = db.upsert_event(
|
||||
'host_unreachable', 'critical', 'ping',
|
||||
name, ip,
|
||||
f'Host {name} ({ip}) unreachable via ping ({_now_utc()})',
|
||||
)
|
||||
if not sup and consec >= self.fail_thresh:
|
||||
self._ticket_unreachable(event_id, is_new, name, ip, consec)
|
||||
else:
|
||||
db.resolve_event('host_unreachable', name, ip)
|
||||
|
||||
def _ticket_unreachable(
|
||||
self, event_id: int, is_new: bool, name: str, ip: str, consec: int
|
||||
) -> None:
|
||||
title = (
|
||||
f'[{name}][auto][production][issue][network][single-node] '
|
||||
f'Host unreachable'
|
||||
)
|
||||
desc = (
|
||||
f'Host Reachability Alert\n{"=" * 40}\n\n'
|
||||
f'Host: {name}\n'
|
||||
f'IP: {ip}\n'
|
||||
f'Detected: {_now_utc()}\n'
|
||||
f'Consecutive check failures: {consec}\n\n'
|
||||
f'Host {name} ({ip}) is not responding to ping from the Gandalf monitor.\n'
|
||||
f'This host does not have a Prometheus node_exporter, so interface-level '
|
||||
f'detail is unavailable.\n\n'
|
||||
f'Please check the host power, management interface, and network connectivity.'
|
||||
)
|
||||
tid = self.tickets.create(title, desc, priority='2')
|
||||
if tid and is_new:
|
||||
db.set_ticket_id(event_id, tid)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Snapshot collection (for dashboard)
|
||||
# ------------------------------------------------------------------
|
||||
def _collect_snapshot(
|
||||
self, iface_states: Dict[str, Dict[str, bool]],
|
||||
unifi_devices: Optional[List[dict]] = None,
|
||||
) -> dict:
|
||||
# Accept pre-fetched devices; fall back to empty list if unavailable
|
||||
display_unifi = unifi_devices if unifi_devices is not None else []
|
||||
|
||||
hosts = {}
|
||||
for instance, ifaces in iface_states.items():
|
||||
if instance not in self._instance_map:
|
||||
continue # skip Prometheus instances not in config (e.g. LXC app servers)
|
||||
host = self._hostname(instance)
|
||||
phys = {k: v for k, v in ifaces.items()}
|
||||
up_count = sum(1 for v in phys.values() if v)
|
||||
total = len(phys)
|
||||
if total == 0 or up_count == total:
|
||||
status = 'up'
|
||||
elif up_count == 0:
|
||||
status = 'down'
|
||||
else:
|
||||
status = 'degraded'
|
||||
|
||||
hosts[host] = {
|
||||
'ip': instance.split(':')[0],
|
||||
'interfaces': {k: ('up' if v else 'down') for k, v in phys.items()},
|
||||
'status': status,
|
||||
'source': 'prometheus',
|
||||
}
|
||||
|
||||
for h in self.cfg.get('monitor', {}).get('ping_hosts', []):
|
||||
name, ip = h['name'], h['ip']
|
||||
reachable = ping(ip, count=1, timeout=2)
|
||||
hosts[name] = {
|
||||
'ip': ip,
|
||||
'interfaces': {},
|
||||
'status': 'up' if reachable else 'down',
|
||||
'source': 'ping',
|
||||
}
|
||||
|
||||
return {
|
||||
'hosts': hosts,
|
||||
'unifi': display_unifi,
|
||||
'updated': datetime.utcnow().isoformat(),
|
||||
}
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Main loop
|
||||
# ------------------------------------------------------------------
|
||||
def run(self) -> None:
|
||||
logger.info(
|
||||
f'Gandalf monitor started – poll_interval={self.poll_interval}s '
|
||||
f'fail_thresh={self.fail_thresh}'
|
||||
)
|
||||
while True:
|
||||
try:
|
||||
logger.info('Starting network check cycle')
|
||||
|
||||
# 1. Fetch interface states once — shared by snapshot and alert processing
|
||||
iface_states = self.prom.get_interface_states()
|
||||
|
||||
# 2. Fetch UniFi devices once — used by both snapshot and alert processing
|
||||
unifi_devices = self.unifi.get_devices()
|
||||
|
||||
# 3. Collect and store snapshot for dashboard
|
||||
snapshot = self._collect_snapshot(iface_states, unifi_devices)
|
||||
db.set_state('network_snapshot', snapshot)
|
||||
db.set_state('last_check', _now_utc())
|
||||
|
||||
# 4. Collect link stats (ethtool + traffic metrics)
|
||||
try:
|
||||
link_data = self.link_stats.collect(self._instance_map)
|
||||
db.set_state('link_stats', link_data)
|
||||
except Exception as e:
|
||||
logger.error(f'Link stats collection failed: {e}', exc_info=True)
|
||||
|
||||
# 5. Process alerts using already-fetched data
|
||||
suppressions = db.get_active_suppressions()
|
||||
self._process_interfaces(iface_states, suppressions)
|
||||
self._process_unifi(unifi_devices, suppressions)
|
||||
|
||||
self._process_ping_hosts(suppressions)
|
||||
|
||||
# Housekeeping: deactivate expired suppressions and purge old resolved events
|
||||
db.cleanup_expired_suppressions()
|
||||
db.purge_old_resolved_events(days=90)
|
||||
|
||||
logger.info('Network check cycle complete')
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f'Monitor loop error: {e}', exc_info=True)
|
||||
|
||||
time.sleep(self.poll_interval)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
monitor = NetworkMonitor()
|
||||
monitor.run()
|
||||
Generated
+1140
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,5 @@
|
||||
{
|
||||
"devDependencies": {
|
||||
"eslint": "^8.57.1"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
flask>=2.2.0
|
||||
gunicorn>=20.1.0
|
||||
ldap3>=2.9
|
||||
pymysql>=1.1.0
|
||||
requests>=2.31.0
|
||||
urllib3>=2.0.0
|
||||
+54
@@ -0,0 +1,54 @@
|
||||
-- Gandalf Network Monitor – Database Schema
|
||||
-- Run on MariaDB LXC 149 (10.10.10.50)
|
||||
|
||||
CREATE DATABASE IF NOT EXISTS gandalf
|
||||
CHARACTER SET utf8mb4
|
||||
COLLATE utf8mb4_unicode_ci;
|
||||
|
||||
USE gandalf;
|
||||
|
||||
-- ── Network events (open and resolved alerts) ─────────────────────────
|
||||
CREATE TABLE IF NOT EXISTS network_events (
|
||||
id INT AUTO_INCREMENT PRIMARY KEY,
|
||||
event_type VARCHAR(60) NOT NULL,
|
||||
severity ENUM('critical','warning','info') NOT NULL DEFAULT 'warning',
|
||||
source_type VARCHAR(20) NOT NULL, -- 'prometheus', 'unifi', 'ping'
|
||||
target_name VARCHAR(255) NOT NULL, -- hostname or device name
|
||||
target_detail VARCHAR(255) NOT NULL DEFAULT '', -- interface name, device type, IP
|
||||
description TEXT,
|
||||
first_seen TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
last_seen TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
||||
resolved_at TIMESTAMP NULL,
|
||||
consecutive_failures INT NOT NULL DEFAULT 1,
|
||||
ticket_id VARCHAR(20) NULL,
|
||||
|
||||
INDEX idx_active (resolved_at),
|
||||
INDEX idx_target (target_name, target_detail),
|
||||
INDEX idx_type (event_type),
|
||||
-- Compound index for hot upsert_event lookup: event_type+target+resolved_at
|
||||
INDEX idx_event_lookup (event_type, target_name, target_detail, resolved_at)
|
||||
) ENGINE=InnoDB;
|
||||
|
||||
-- ── Suppression rules ─────────────────────────────────────────────────
|
||||
CREATE TABLE IF NOT EXISTS suppression_rules (
|
||||
id INT AUTO_INCREMENT PRIMARY KEY,
|
||||
target_type VARCHAR(50) NOT NULL, -- 'host', 'interface', 'unifi_device', 'all'
|
||||
target_name VARCHAR(255) NOT NULL DEFAULT '',
|
||||
target_detail VARCHAR(255) NOT NULL DEFAULT '',
|
||||
reason TEXT NOT NULL,
|
||||
suppressed_by VARCHAR(255) NOT NULL,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
expires_at TIMESTAMP NULL, -- NULL = manual (never auto-expires)
|
||||
active BOOLEAN NOT NULL DEFAULT TRUE,
|
||||
|
||||
INDEX idx_active_exp (active, expires_at),
|
||||
-- For suppression lookup by type+name
|
||||
INDEX idx_sup_lookup (active, target_type, target_name, target_detail)
|
||||
) ENGINE=InnoDB;
|
||||
|
||||
-- ── Monitor state (key/value store for snapshot + baseline) ───────────
|
||||
CREATE TABLE IF NOT EXISTS monitor_state (
|
||||
key_name VARCHAR(100) PRIMARY KEY,
|
||||
value MEDIUMTEXT NOT NULL,
|
||||
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP
|
||||
) ENGINE=InnoDB;
|
||||
@@ -0,0 +1,13 @@
|
||||
{
|
||||
"env": { "browser": true, "es2021": true },
|
||||
"extends": "eslint:recommended",
|
||||
"parserOptions": { "ecmaVersion": 2021, "sourceType": "script" },
|
||||
"rules": {
|
||||
"no-unused-vars": "warn",
|
||||
"no-undef": "off",
|
||||
"no-empty": "warn",
|
||||
"no-inner-declarations": "warn",
|
||||
"semi": ["error", "always"],
|
||||
"eqeqeq": "warn"
|
||||
}
|
||||
}
|
||||
+360
@@ -0,0 +1,360 @@
|
||||
'use strict';
|
||||
|
||||
// ── Auto-redirect on auth timeout ─────────────────────────────────────
|
||||
// Wraps fetch so a 401 (Authelia session expired) forces a full reload.
|
||||
// lt.api uses fetch internally, so this covers all API calls too.
|
||||
(function () {
|
||||
const _fetch = window.fetch;
|
||||
window.fetch = async function (...args) {
|
||||
const resp = await _fetch(...args);
|
||||
if (resp.status === 401) {
|
||||
window.location.reload();
|
||||
throw new Error('Session expired — reloading');
|
||||
}
|
||||
return resp;
|
||||
};
|
||||
})();
|
||||
|
||||
// ── Toast notifications — thin wrapper over lt.toast ──────────────────
|
||||
function showToast(msg, type = 'success') {
|
||||
if (type === 'error') return lt.toast.error(msg);
|
||||
if (type === 'warning') return lt.toast.warning(msg);
|
||||
if (type === 'info') return lt.toast.info(msg);
|
||||
return lt.toast.success(msg);
|
||||
}
|
||||
|
||||
// ── Normalise UTC timestamp string for Date() parsing ─────────────────
|
||||
// Server returns "2026-03-14 14:14:21 UTC"; Date() needs ISO 8601.
|
||||
function _toIso(s) {
|
||||
if (!s) return s;
|
||||
return s.replace(' UTC', 'Z').replace(' ', 'T');
|
||||
}
|
||||
|
||||
// ── Dashboard auto-refresh ────────────────────────────────────────────
|
||||
async function refreshAll() {
|
||||
const refreshBtn = document.querySelector('[data-action="refresh"]');
|
||||
const spinner = document.getElementById('refresh-spinner');
|
||||
if (refreshBtn) refreshBtn.classList.add('is-loading');
|
||||
if (spinner) spinner.style.display = '';
|
||||
try {
|
||||
const [netResult, statusResult] = await Promise.allSettled([
|
||||
lt.api.get('/api/network'),
|
||||
lt.api.get('/api/status'),
|
||||
]);
|
||||
if (netResult.status === 'fulfilled') {
|
||||
const net = netResult.value;
|
||||
updateHostGrid(net.hosts || {});
|
||||
updateUnifiTable(net.unifi || []);
|
||||
updateTopology(net.hosts || {});
|
||||
} else {
|
||||
showToast('Network data unavailable', 'warning');
|
||||
}
|
||||
if (statusResult.status === 'fulfilled') {
|
||||
const status = statusResult.value;
|
||||
updateEventsTable(status.events || [], status.total_active);
|
||||
updateStatusBar(status.summary || {}, status.last_check || '', status.daemon_ok);
|
||||
if (typeof window.gandalfNotifUpdate === 'function') window.gandalfNotifUpdate(status.events || []);
|
||||
} else {
|
||||
showToast('Status data unavailable', 'warning');
|
||||
}
|
||||
} finally {
|
||||
if (refreshBtn) refreshBtn.classList.remove('is-loading');
|
||||
if (spinner) spinner.style.display = 'none';
|
||||
}
|
||||
}
|
||||
|
||||
function updateStatusBar(summary, lastCheck, daemonOk) {
|
||||
const bar = document.querySelector('.status-chips');
|
||||
if (!bar) return;
|
||||
const chips = [];
|
||||
if (daemonOk === false) chips.push('<span class="chip chip-critical">⚠ MONITOR OFFLINE</span>');
|
||||
if (summary.critical) chips.push(`<span class="chip chip-critical">● ${summary.critical} CRITICAL</span>`);
|
||||
if (summary.warning) chips.push(`<span class="chip chip-warning">● ${summary.warning} WARNING</span>`);
|
||||
if (!summary.critical && !summary.warning && daemonOk !== false) chips.push('<span class="chip chip-ok">✔ ALL SYSTEMS NOMINAL</span>');
|
||||
bar.innerHTML = chips.join('');
|
||||
|
||||
const lc = document.getElementById('last-check');
|
||||
if (lc && lastCheck) lc.textContent = lastCheck;
|
||||
|
||||
const critCount = summary.critical || 0;
|
||||
const warnCount = summary.warning || 0;
|
||||
if (critCount) document.title = `(${critCount} CRIT) GANDALF`;
|
||||
else if (warnCount) document.title = `(${warnCount} WARN) GANDALF`;
|
||||
else document.title = 'GANDALF';
|
||||
|
||||
const alertBadge = document.getElementById('alert-count-badge');
|
||||
if (alertBadge) {
|
||||
const total = critCount + warnCount;
|
||||
alertBadge.textContent = total;
|
||||
alertBadge.style.display = total ? '' : 'none';
|
||||
}
|
||||
|
||||
// Update stat cards
|
||||
const scCrit = document.getElementById('stat-critical-val');
|
||||
const scWarn = document.getElementById('stat-warning-val');
|
||||
if (scCrit) scCrit.textContent = critCount;
|
||||
if (scWarn) scWarn.textContent = warnCount;
|
||||
const statCritCard = document.getElementById('stat-critical');
|
||||
if (statCritCard) statCritCard.classList.toggle('lt-stat-card--alert', critCount > 0);
|
||||
|
||||
// Stale data banner: warn if last_check is older than 15 minutes
|
||||
let staleBanner = document.getElementById('stale-banner');
|
||||
if (lastCheck) {
|
||||
const checkAge = (Date.now() - new Date(_toIso(lastCheck))) / 1000;
|
||||
if (checkAge > 900) {
|
||||
if (!staleBanner) {
|
||||
staleBanner = document.createElement('div');
|
||||
staleBanner.id = 'stale-banner';
|
||||
staleBanner.className = 'lt-alert lt-alert--warning';
|
||||
staleBanner.innerHTML = '<span class="lt-alert-icon">⚠</span><div class="lt-alert-body"><div class="lt-alert-msg"></div></div>';
|
||||
document.querySelector('.lt-main').prepend(staleBanner);
|
||||
}
|
||||
const mins = Math.floor(checkAge / 60);
|
||||
staleBanner.querySelector('.lt-alert-msg').textContent =
|
||||
`Monitoring data is stale — last check was ${mins} minute${mins !== 1 ? 's' : ''} ago. The monitor daemon may be down.`;
|
||||
staleBanner.style.display = '';
|
||||
} else if (staleBanner) {
|
||||
staleBanner.style.display = 'none';
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function updateHostGrid(hosts) {
|
||||
const scHosts = document.getElementById('stat-hosts-val');
|
||||
if (scHosts) scHosts.textContent = Object.keys(hosts).length;
|
||||
|
||||
for (const [name, host] of Object.entries(hosts)) {
|
||||
const card = document.querySelector(`.host-card[data-host="${CSS.escape(name)}"]`);
|
||||
if (!card) continue;
|
||||
|
||||
card.className = card.className.replace(/host-card-(up|down|degraded|unknown)/g, '');
|
||||
card.classList.add(`host-card-${host.status}`);
|
||||
|
||||
const dot = card.querySelector('.host-status-dot');
|
||||
if (dot) dot.className = `host-status-dot dot-${host.status}`;
|
||||
|
||||
const ifaceList = card.querySelector('.iface-list');
|
||||
if (ifaceList && host.interfaces && Object.keys(host.interfaces).length > 0) {
|
||||
ifaceList.innerHTML = Object.entries(host.interfaces)
|
||||
.sort(([a], [b]) => a.localeCompare(b))
|
||||
.map(([iface, state]) => `
|
||||
<div class="iface-row">
|
||||
<span class="iface-dot dot-${state}"></span>
|
||||
<span class="iface-name">${lt.escHtml(iface)}</span>
|
||||
<span class="iface-state state-${state}">${state}</span>
|
||||
</div>
|
||||
`).join('');
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function updateTopology(hosts) {
|
||||
document.querySelectorAll('.topo-host').forEach(node => {
|
||||
const name = node.dataset.host;
|
||||
const host = hosts[name];
|
||||
if (!host) return;
|
||||
node.className = node.className.replace(/topo-v2-status-(up|down|degraded|unknown)/g, '');
|
||||
node.classList.add(`topo-v2-status-${host.status}`);
|
||||
const badge = node.querySelector('.topo-badge');
|
||||
if (badge) {
|
||||
badge.className = `topo-badge topo-badge-${host.status}`;
|
||||
badge.textContent = host.status;
|
||||
}
|
||||
|
||||
// Animate the 10G drop-wire red+dashed when host is down
|
||||
document.querySelectorAll(`.topo-v2-wire-10g[data-host="${CSS.escape(name)}"]`).forEach(wire => {
|
||||
wire.classList.toggle('wire-down', host.status === 'down');
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function updateUnifiTable(devices) {
|
||||
const tbody = document.querySelector('#unifi-table tbody');
|
||||
if (!tbody || !devices.length) return;
|
||||
|
||||
tbody.innerHTML = devices.map(d => {
|
||||
const statusClass = d.connected ? '' : 'row-critical';
|
||||
const dotClass = d.connected ? 'dot-up' : 'dot-down';
|
||||
const statusText = d.connected ? 'ONLINE' : 'OFFLINE';
|
||||
const suppressBtn = !d.connected
|
||||
? `<button class="lt-btn lt-btn-ghost lt-btn-sm btn-suppress"
|
||||
data-sup-type="unifi_device"
|
||||
data-sup-name="${lt.escHtml(d.name)}"
|
||||
data-sup-detail=""
|
||||
aria-label="Suppress alerts for ${lt.escHtml(d.name)}">🔕 Suppress</button>`
|
||||
: '';
|
||||
return `
|
||||
<tr class="${statusClass}">
|
||||
<td><span class="${dotClass}"></span> ${statusText}</td>
|
||||
<td><strong>${lt.escHtml(d.name)}</strong></td>
|
||||
<td>${lt.escHtml(d.type)}</td>
|
||||
<td>${lt.escHtml(d.model)}</td>
|
||||
<td>${lt.escHtml(d.ip)}</td>
|
||||
<td>${suppressBtn}</td>
|
||||
</tr>`;
|
||||
}).join('');
|
||||
}
|
||||
|
||||
function updateEventsTable(events, totalActive) {
|
||||
const wrap = document.getElementById('events-table-wrap');
|
||||
if (!wrap) return;
|
||||
|
||||
const active = (events || []).filter(e => e.severity !== 'info');
|
||||
if (!active.length) {
|
||||
wrap.innerHTML = '<div class="lt-empty-state lt-empty-state--sm"><div class="lt-empty-state-icon">✔</div><div class="lt-empty-state-title">No active alerts</div></div>';
|
||||
return;
|
||||
}
|
||||
|
||||
const truncated = totalActive != null && totalActive > active.length;
|
||||
const countNotice = truncated
|
||||
? `<div class="pagination-notice">Showing ${active.length} of ${totalActive} active alerts — <a href="/api/events?limit=1000">view all via API</a></div>`
|
||||
: '';
|
||||
|
||||
const rows = active.map(e => {
|
||||
const supType = e.event_type === 'unifi_device_offline' ? 'unifi_device'
|
||||
: e.event_type === 'interface_down' ? 'interface'
|
||||
: 'host';
|
||||
const ticketBase = (typeof GANDALF_CONFIG !== 'undefined' && GANDALF_CONFIG.ticket_web_url)
|
||||
? GANDALF_CONFIG.ticket_web_url : 'http://t.lotusguild.org/ticket/';
|
||||
const ticket = e.ticket_id
|
||||
? `<a href="${lt.escHtml(ticketBase)}${lt.escHtml(String(e.ticket_id))}" target="_blank"
|
||||
class="ticket-link">#${e.ticket_id}</a>`
|
||||
: '–';
|
||||
return `
|
||||
<tr class="row-${e.severity}">
|
||||
<td><span class="lt-badge badge-${e.severity}">${e.severity}</span></td>
|
||||
<td>${lt.escHtml(e.event_type.replace(/_/g,' '))}</td>
|
||||
<td><strong>${lt.escHtml(e.target_name)}</strong></td>
|
||||
<td>${lt.escHtml(e.target_detail || '–')}</td>
|
||||
<td class="desc-cell" title="${lt.escHtml(e.description || '')}">${lt.escHtml((e.description||'').substring(0,60))}${(e.description||'').length>60?'…':''}</td>
|
||||
<td class="ts-cell" title="${lt.escHtml(e.first_seen||'')}">${lt.time.ago(_toIso(e.first_seen))}</td>
|
||||
<td class="ts-cell" title="${lt.escHtml(e.last_seen||'')}">${lt.time.ago(_toIso(e.last_seen))}</td>
|
||||
<td>${e.consecutive_failures}</td>
|
||||
<td>${ticket}</td>
|
||||
<td>
|
||||
<button class="lt-btn lt-btn-ghost lt-btn-sm btn-suppress"
|
||||
data-sup-type="${lt.escHtml(supType)}"
|
||||
data-sup-name="${lt.escHtml(e.target_name)}"
|
||||
data-sup-detail="${lt.escHtml(e.target_detail||'')}"
|
||||
aria-label="Suppress alert for ${lt.escHtml(e.target_name)}">🔕</button>
|
||||
</td>
|
||||
</tr>`;
|
||||
}).join('');
|
||||
|
||||
wrap.innerHTML = `
|
||||
${countNotice}
|
||||
<div class="lt-table-wrap">
|
||||
<table class="lt-table" id="events-table">
|
||||
<caption class="lt-sr-only">Active network alerts</caption>
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Sev</th><th>Type</th><th>Target</th><th>Detail</th>
|
||||
<th>Description</th><th>First Seen</th><th>Last Seen</th><th>Failures</th><th>Ticket</th><th>Actions</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>${rows}</tbody>
|
||||
</table>
|
||||
</div>`;
|
||||
}
|
||||
|
||||
// ── Suppression modal ─────────────────────────────────────────────────
|
||||
function openSuppressModal(type, name, detail) {
|
||||
const modal = document.getElementById('suppress-modal');
|
||||
if (!modal) return;
|
||||
|
||||
document.getElementById('sup-type').value = type;
|
||||
document.getElementById('sup-name').value = name;
|
||||
document.getElementById('sup-detail').value = detail;
|
||||
document.getElementById('sup-reason').value = '';
|
||||
document.getElementById('sup-expires').value = '';
|
||||
|
||||
updateSuppressForm();
|
||||
lt.modal.open('suppress-modal');
|
||||
|
||||
document.querySelectorAll('#suppress-modal .pill').forEach(p => p.classList.remove('active'));
|
||||
const manualPill = document.querySelector('#suppress-modal .pill-manual');
|
||||
if (manualPill) manualPill.classList.add('active');
|
||||
const hint = document.getElementById('duration-hint');
|
||||
if (hint) hint.textContent = 'Suppression will persist until manually removed.';
|
||||
}
|
||||
|
||||
function closeSuppressModal() {
|
||||
lt.modal.close('suppress-modal');
|
||||
}
|
||||
|
||||
function updateSuppressForm() {
|
||||
const type = document.getElementById('sup-type').value;
|
||||
const nameGrp = document.getElementById('sup-name-group');
|
||||
const detailGrp = document.getElementById('sup-detail-group');
|
||||
if (nameGrp) nameGrp.style.display = (type === 'all') ? 'none' : '';
|
||||
if (detailGrp) detailGrp.style.display = (type === 'interface') ? '' : 'none';
|
||||
}
|
||||
|
||||
function setDuration(mins, el) {
|
||||
document.getElementById('sup-expires').value = mins || '';
|
||||
document.querySelectorAll('#suppress-modal .pill').forEach(p => p.classList.remove('active'));
|
||||
if (el) el.classList.add('active');
|
||||
const hint = document.getElementById('duration-hint');
|
||||
if (hint) {
|
||||
if (mins) {
|
||||
const h = Math.floor(mins / 60), m = mins % 60;
|
||||
hint.textContent = `Expires in ${h ? h + 'h ' : ''}${m ? m + 'm' : ''}.`;
|
||||
} else {
|
||||
hint.textContent = 'Suppression will persist until manually removed.';
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function submitSuppress(e) {
|
||||
e.preventDefault();
|
||||
const type = document.getElementById('sup-type').value;
|
||||
const name = document.getElementById('sup-name').value;
|
||||
const detail = document.getElementById('sup-detail').value;
|
||||
const reason = document.getElementById('sup-reason').value;
|
||||
const expires = document.getElementById('sup-expires').value;
|
||||
|
||||
if (!reason.trim()) { showToast('Reason is required', 'error'); return; }
|
||||
if (type !== 'all' && !name.trim()) { showToast('Target name is required', 'error'); return; }
|
||||
|
||||
try {
|
||||
await lt.api.post('/api/suppressions', {
|
||||
target_type: type,
|
||||
target_name: name,
|
||||
target_detail: detail,
|
||||
reason,
|
||||
expires_minutes: expires ? parseInt(expires) : null,
|
||||
});
|
||||
closeSuppressModal();
|
||||
showToast('Suppression applied ✔', 'success');
|
||||
setTimeout(refreshAll, 500);
|
||||
} catch (err) {
|
||||
showToast(err.message || 'Failed to apply suppression', 'error');
|
||||
}
|
||||
}
|
||||
|
||||
// ── Global click delegation ───────────────────────────────────────────
|
||||
document.addEventListener('click', e => {
|
||||
// Refresh button
|
||||
if (e.target.closest('[data-action="refresh"]')) {
|
||||
lt.autoRefresh.now();
|
||||
return;
|
||||
}
|
||||
|
||||
// Duration pills (data-duration="" = manual/forever)
|
||||
const pill = e.target.closest('.pill[data-duration]');
|
||||
if (pill) {
|
||||
const val = pill.dataset.duration;
|
||||
setDuration(val ? parseInt(val) : null, pill);
|
||||
return;
|
||||
}
|
||||
|
||||
// Suppress buttons
|
||||
const btn = e.target.closest('.btn-suppress[data-sup-type]');
|
||||
if (btn) {
|
||||
openSuppressModal(
|
||||
btn.dataset.supType || '',
|
||||
btn.dataset.supName || '',
|
||||
btn.dataset.supDetail || '',
|
||||
);
|
||||
}
|
||||
});
|
||||
+5614
File diff suppressed because it is too large.
Load diff
+2947
File diff suppressed because it is too large.
Load diff
+1020
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,493 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en" data-theme="dark">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0, viewport-fit=cover">
|
||||
<meta name="theme-color" content="#030508">
|
||||
<meta name="robots" content="noindex, nofollow">
|
||||
<title>{% block title %}GANDALF{% endblock %} — GANDALF</title>
|
||||
|
||||
<link rel="preconnect" href="https://fonts.googleapis.com">
|
||||
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
|
||||
<link href="https://fonts.googleapis.com/css2?family=JetBrains+Mono:ital,wght@0,400;0,600;0,700;1,400&family=VT323&display=swap" rel="stylesheet">
|
||||
|
||||
<link rel="stylesheet" href="{{ url_for('static', filename='base.css') }}">
|
||||
<link rel="stylesheet" href="{{ url_for('static', filename='style.css') }}">
|
||||
|
||||
<!-- base.js loaded in head so lt.* is available for inline scripts -->
|
||||
<script src="{{ url_for('static', filename='base.js') }}"></script>
|
||||
</head>
|
||||
<body>
|
||||
|
||||
<a class="lt-skip-link" href="#main-content">Skip to main content</a>
|
||||
|
||||
<!-- BOOT OVERLAY -->
|
||||
<div id="lt-boot" class="lt-boot-overlay" data-app-name="GANDALF" style="display:none" aria-hidden="true">
|
||||
<pre id="lt-boot-text" class="lt-boot-text"></pre>
|
||||
</div>
|
||||
|
||||
<!-- MOBILE NAV DRAWER -->
|
||||
<div id="lt-nav-drawer" class="lt-nav-drawer" aria-hidden="true" role="dialog" aria-modal="true" aria-label="Navigation menu">
|
||||
<div class="lt-nav-drawer-header">
|
||||
<span class="lt-brand-title">GANDALF</span>
|
||||
<button type="button" class="lt-nav-drawer-close" id="lt-nav-drawer-close" aria-label="Close navigation">✕</button>
|
||||
</div>
|
||||
<nav class="lt-nav-drawer-links" aria-label="Mobile navigation">
|
||||
<a href="{{ url_for('index') }}"
|
||||
class="lt-nav-drawer-link{% if request.endpoint == 'index' %} active{% endif %}"
|
||||
{% if request.endpoint == 'index' %}aria-current="page"{% endif %}>Dashboard</a>
|
||||
<a href="{{ url_for('links_page') }}"
|
||||
class="lt-nav-drawer-link{% if request.endpoint == 'links_page' %} active{% endif %}"
|
||||
{% if request.endpoint == 'links_page' %}aria-current="page"{% endif %}>Link Debug</a>
|
||||
<a href="{{ url_for('inspector') }}"
|
||||
class="lt-nav-drawer-link{% if request.endpoint == 'inspector' %} active{% endif %}"
|
||||
{% if request.endpoint == 'inspector' %}aria-current="page"{% endif %}>Inspector</a>
|
||||
{% if user.groups and 'admin' in user.groups %}
|
||||
<a href="{{ url_for('suppressions_page') }}"
|
||||
class="lt-nav-drawer-link{% if request.endpoint == 'suppressions_page' %} active{% endif %}"
|
||||
{% if request.endpoint == 'suppressions_page' %}aria-current="page"{% endif %}>Suppressions</a>
|
||||
{% endif %}
|
||||
</nav>
|
||||
</div>
|
||||
<div id="lt-nav-overlay" class="lt-nav-drawer-overlay"></div>
|
||||
|
||||
<!-- PRIMARY HEADER -->
|
||||
<header class="lt-header" role="banner">
|
||||
<div class="lt-header-left">
|
||||
|
||||
<!-- Hamburger (mobile) -->
|
||||
<button type="button"
|
||||
class="lt-menu-btn"
|
||||
id="lt-menu-btn"
|
||||
data-action="open-nav-drawer"
|
||||
aria-label="Open navigation menu"
|
||||
aria-expanded="false"
|
||||
aria-controls="lt-nav-drawer">
|
||||
<span class="lt-menu-btn-bar"></span>
|
||||
<span class="lt-menu-btn-bar"></span>
|
||||
<span class="lt-menu-btn-bar"></span>
|
||||
</button>
|
||||
|
||||
<!-- Brand -->
|
||||
<div class="lt-brand">
|
||||
<a href="{{ url_for('index') }}"
|
||||
class="lt-brand-title lt-glitch"
|
||||
data-text="GANDALF"
|
||||
style="text-decoration:none"
|
||||
aria-label="GANDALF home">GANDALF</a>
|
||||
<span class="lt-brand-subtitle">Network Monitor // LotusGuild</span>
|
||||
</div>
|
||||
|
||||
<!-- Desktop nav -->
|
||||
<nav class="lt-nav" aria-label="Main navigation">
|
||||
<a href="{{ url_for('index') }}"
|
||||
class="lt-nav-link{% if request.endpoint == 'index' %} active{% endif %}"
|
||||
{% if request.endpoint == 'index' %}aria-current="page"{% endif %}>
|
||||
Dashboard
|
||||
</a>
|
||||
<a href="{{ url_for('links_page') }}"
|
||||
class="lt-nav-link{% if request.endpoint == 'links_page' %} active{% endif %}"
|
||||
{% if request.endpoint == 'links_page' %}aria-current="page"{% endif %}>
|
||||
Link Debug
|
||||
</a>
|
||||
<a href="{{ url_for('inspector') }}"
|
||||
class="lt-nav-link{% if request.endpoint == 'inspector' %} active{% endif %}"
|
||||
{% if request.endpoint == 'inspector' %}aria-current="page"{% endif %}>
|
||||
Inspector
|
||||
</a>
|
||||
{% if user.groups and 'admin' in user.groups %}
|
||||
<div class="lt-nav-dropdown" data-action="toggle-nav-dropdown">
|
||||
<a href="#"
|
||||
class="lt-nav-link{% if request.endpoint == 'suppressions_page' %} active{% endif %}"
|
||||
role="button"
|
||||
aria-haspopup="true"
|
||||
aria-expanded="false"
|
||||
aria-controls="lt-admin-dropdown-menu">
|
||||
Admin ▾
|
||||
</a>
|
||||
<ul class="lt-nav-dropdown-menu"
|
||||
id="lt-admin-dropdown-menu"
|
||||
role="menu"
|
||||
aria-label="Admin menu">
|
||||
<li role="none">
|
||||
<a href="{{ url_for('suppressions_page') }}" role="menuitem"
|
||||
class="{% if request.endpoint == 'suppressions_page' %}active{% endif %}">
|
||||
Suppressions
|
||||
</a>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
{% else %}
|
||||
<a href="{{ url_for('suppressions_page') }}"
|
||||
class="lt-nav-link{% if request.endpoint == 'suppressions_page' %} active{% endif %}"
|
||||
{% if request.endpoint == 'suppressions_page' %}aria-current="page"{% endif %}>
|
||||
Suppressions
|
||||
</a>
|
||||
{% endif %}
|
||||
</nav>
|
||||
</div>
|
||||
|
||||
<div class="lt-header-right">
|
||||
{% set _uname = user.name or user.username %}
|
||||
{% set _words = _uname.split() %}
|
||||
{% set _initials = (_words[0][0] ~ (_words[1][0] if _words|length > 1 else ''))|upper %}
|
||||
<div class="lt-avatar lt-avatar--sm {{ _uname | avatar_color }}"
|
||||
aria-hidden="true" title="{{ _uname }}">
|
||||
<img src="{{ url_for('api_avatar') }}" alt="" class="lt-avatar-img">
|
||||
<span class="lt-avatar-initials">{{ _initials }}</span>
|
||||
</div>
|
||||
<span class="lt-header-user">{{ _uname }}</span>
|
||||
{% if user.groups and 'admin' in user.groups %}
|
||||
<span class="lt-badge lt-badge-admin" aria-label="Administrator">ADMIN</span>
|
||||
{% endif %}
|
||||
|
||||
<!-- Notification bell — shows active monitoring alerts -->
|
||||
<div class="lt-notif-dropdown-wrap" id="lt-notif-wrap">
|
||||
<button type="button"
|
||||
class="lt-btn lt-btn-ghost lt-btn-sm lt-notif-bell-btn"
|
||||
id="lt-notif-bell"
|
||||
aria-label="Active alerts"
|
||||
aria-expanded="false"
|
||||
aria-controls="lt-notif-panel"
|
||||
title="Active alerts">🔔</button>
|
||||
<div class="lt-notif-panel" id="lt-notif-panel" aria-hidden="true" role="dialog" aria-label="Active alerts">
|
||||
<div class="lt-notif-panel-header">
|
||||
<span>Active Alerts</span>
|
||||
<button type="button" class="lt-notif-panel-clear" id="lt-notif-clear-btn">Mark all read</button>
|
||||
</div>
|
||||
<div class="lt-notif-panel-list" id="lt-notif-list">
|
||||
<div style="padding:0.75rem;font-size:0.75rem;color:var(--text-muted);text-align:center">Loading…</div>
|
||||
</div>
|
||||
<div class="lt-notif-panel-footer">
|
||||
<a href="{{ url_for('index') }}" class="lt-btn lt-btn-ghost lt-btn-sm" style="width:100%;text-align:center;display:block;font-size:0.72rem">View dashboard</a>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- ⌘K affordance -->
|
||||
<button type="button"
|
||||
class="lt-btn lt-btn-ghost lt-btn-sm"
|
||||
title="Command palette (Ctrl+K)"
|
||||
aria-label="Open command palette"
|
||||
onclick="if(window.lt&<.cmdPalette)lt.cmdPalette.open()"
|
||||
style="font-size:0.65rem;opacity:0.55;letter-spacing:0.03em;padding:0.2rem 0.45rem">⌕ K</button>
|
||||
|
||||
<button type="button" class="lt-theme-btn" id="lt-theme-btn"
|
||||
aria-label="Toggle theme" title="Toggle light/dark mode">☀</button>
|
||||
</div>
|
||||
</header>
|
||||
|
||||
<!-- COMMAND PALETTE -->
|
||||
<div id="lt-cmd-overlay" class="lt-cmd-overlay" role="dialog" aria-modal="true" aria-label="Command palette" aria-hidden="true">
|
||||
<div class="lt-cmd-palette" id="lt-cmd-palette">
|
||||
<div class="lt-cmd-input-wrap">
|
||||
<span class="lt-cmd-prompt">></span>
|
||||
<input id="lt-cmd-input" class="lt-cmd-input" type="text"
|
||||
placeholder="Search commands…" autocomplete="off"
|
||||
spellcheck="false" aria-label="Search commands">
|
||||
</div>
|
||||
<div class="lt-cmd-results" id="lt-cmd-results">
|
||||
<div class="lt-cmd-empty">Start typing to search…</div>
|
||||
</div>
|
||||
<div class="lt-cmd-footer">
|
||||
<span><kbd>↑</kbd><kbd>↓</kbd> Navigate</span>
|
||||
<span><kbd>Enter</kbd> Select</span>
|
||||
<span><kbd>Esc</kbd> Close</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- MAIN CONTENT -->
|
||||
<main class="lt-main lt-container" id="main-content">
|
||||
{% block content %}{% endblock %}
|
||||
</main>
|
||||
|
||||
<!-- FOOTER — context-sensitive per page -->
|
||||
<footer class="lt-footer" role="contentinfo">
|
||||
<nav class="lt-footer-hints" aria-label="Keyboard shortcuts">
|
||||
{% if request.endpoint == 'index' %}
|
||||
<span class="lt-footer-hint"><span class="lt-footer-key">[ R ]</span> REFRESH</span>
|
||||
<span class="lt-footer-sep">|</span>
|
||||
<span class="lt-footer-hint"><span class="lt-footer-key">[ S ]</span> SUPPRESS</span>
|
||||
<span class="lt-footer-sep">|</span>
|
||||
{% elif request.endpoint == 'links_page' %}
|
||||
<span class="lt-footer-hint"><span class="lt-footer-key">[ R ]</span> REFRESH</span>
|
||||
<span class="lt-footer-sep">|</span>
|
||||
{% endif %}
|
||||
<button type="button" class="lt-footer-hint" data-action="open-settings"><span class="lt-footer-key">[ * ]</span> CFG</button>
|
||||
<span class="lt-footer-sep">|</span>
|
||||
<button type="button" class="lt-footer-hint" data-action="show-keyboard-help"><span class="lt-footer-key">[ ? ]</span> HELP</button>
|
||||
</nav>
|
||||
<span>GANDALF — TDS v1.2</span>
|
||||
</footer>
|
||||
|
||||
<!-- KEYBOARD SHORTCUTS MODAL -->
|
||||
<div id="lt-keys-help" class="lt-modal-overlay" aria-hidden="true">
|
||||
<div class="lt-modal" role="dialog" aria-modal="true" aria-labelledby="keys-help-title">
|
||||
<div class="lt-modal-header">
|
||||
<span class="lt-modal-title" id="keys-help-title">Keyboard Shortcuts</span>
|
||||
<button type="button" class="lt-modal-close" data-modal-close aria-label="Close">✕</button>
|
||||
</div>
|
||||
<div class="lt-modal-body">
|
||||
<table class="lt-table" style="width:100%">
|
||||
<thead><tr><th>Shortcut</th><th>Action</th></tr></thead>
|
||||
<tbody>
|
||||
<tr><td>Ctrl / ⌘ + K</td><td>Command palette</td></tr>
|
||||
<tr><td>R</td><td>Refresh data (Dashboard / Link Debug)</td></tr>
|
||||
<tr><td>S</td><td>Quick-suppress alert (Dashboard)</td></tr>
|
||||
<tr><td>*</td><td>Open settings</td></tr>
|
||||
<tr><td>?</td><td>Show this help</td></tr>
|
||||
<tr><td>ESC</td><td>Close modal / panel</td></tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
<div class="lt-modal-footer">
|
||||
<button type="button" class="lt-btn" data-modal-close>Close</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- SETTINGS MODAL -->
|
||||
<div id="lt-settings-modal" class="lt-modal-overlay" aria-hidden="true">
|
||||
<div class="lt-modal" role="dialog" aria-modal="true" aria-labelledby="settings-modal-title">
|
||||
<div class="lt-modal-header">
|
||||
<span class="lt-modal-title" id="settings-modal-title">Settings</span>
|
||||
<button type="button" class="lt-modal-close" data-modal-close aria-label="Close">✕</button>
|
||||
</div>
|
||||
<div class="lt-modal-body">
|
||||
<div class="lt-form-group">
|
||||
<label class="lt-label">Auto-refresh interval</label>
|
||||
<div class="duration-pills" id="settings-refresh-pills">
|
||||
<button type="button" class="pill" data-refresh-interval="15">15 s</button>
|
||||
<button type="button" class="pill" data-refresh-interval="30">30 s</button>
|
||||
<button type="button" class="pill" data-refresh-interval="60">1 min</button>
|
||||
<button type="button" class="pill" data-refresh-interval="300">5 min</button>
|
||||
<button type="button" class="pill" data-refresh-interval="0">Off</button>
|
||||
</div>
|
||||
<div class="lt-field-hint" id="settings-refresh-hint"></div>
|
||||
</div>
|
||||
<div class="lt-divider" style="margin:1rem 0 0.75rem"></div>
|
||||
<div class="lt-kv-grid">
|
||||
<div class="lt-kv-row">
|
||||
<span class="lt-kv-label">User</span>
|
||||
<span class="lt-kv-value lt-text-cyan">{{ user.name or user.username }}</span>
|
||||
</div>
|
||||
{% if user.groups %}
|
||||
<div class="lt-kv-row">
|
||||
<span class="lt-kv-label">Groups</span>
|
||||
<span class="lt-kv-value">{{ user.groups | join(', ') }}</span>
|
||||
</div>
|
||||
{% endif %}
|
||||
</div>
|
||||
</div>
|
||||
<div class="lt-modal-footer">
|
||||
<button type="button" class="lt-btn" data-modal-close>Close</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
const GANDALF_CONFIG = {
|
||||
ticket_web_url: "{{ config.get('ticket_api', {}).get('web_url', 'http://t.lotusguild.org/ticket/') }}"
|
||||
};
|
||||
</script>
|
||||
<script src="{{ url_for('static', filename='app.js') }}"></script>
|
||||
{% block scripts %}{% endblock %}
|
||||
|
||||
<script>
|
||||
if (window.lt) {
|
||||
lt.init({ bootName: 'GANDALF' });
|
||||
|
||||
// Theme toggle
|
||||
var themeBtn = document.getElementById('lt-theme-btn');
|
||||
if (themeBtn) themeBtn.addEventListener('click', function() { lt.theme.toggle(); });
|
||||
|
||||
// Command palette
|
||||
lt.cmdPalette.init([
|
||||
{ id: 'nav-dashboard', group: 'Navigate', icon: '~', label: 'Dashboard', action: function() { window.location.href = '/'; } },
|
||||
{ id: 'nav-links', group: 'Navigate', icon: '↗', label: 'Link Debug', action: function() { window.location.href = '/links'; } },
|
||||
{ id: 'nav-inspector', group: 'Navigate', icon: '⬡', label: 'Inspector', action: function() { window.location.href = '/inspector'; } },
|
||||
{ id: 'nav-suppressions', group: 'Navigate', icon: '🔕', label: 'Suppressions', action: function() { window.location.href = '/suppressions'; } },
|
||||
{ id: 'action-refresh', group: 'Actions', icon: '↻', label: 'Refresh Data', kbd: 'R', action: function() { lt.autoRefresh.now(); } },
|
||||
{ id: 'action-suppress', group: 'Actions', icon: '🔕', label: 'New Suppression', kbd: 'S', action: function() { if (typeof openSuppressModal === 'function') openSuppressModal('host','',''); else window.location.href='/suppressions'; } },
|
||||
{ id: 'help-shortcuts', group: 'Help', icon: '?', label: 'Keyboard Shortcuts', kbd: '?', action: function() { lt.modal.open('lt-keys-help'); } },
|
||||
{ id: 'help-settings', group: 'Help', icon: '*', label: 'Settings', kbd: '*', action: function() { lt.modal.open('lt-settings-modal'); } },
|
||||
{ id: 'help-theme', group: 'Help', icon: '☀', label: 'Toggle Theme', action: function() { lt.theme.toggle(); } },
|
||||
]);
|
||||
}
|
||||
|
||||
// ── Global footer + key actions ───────────────────────────────────────
|
||||
document.addEventListener('click', function(e) {
|
||||
var btn = e.target.closest('[data-action]');
|
||||
if (!btn) return;
|
||||
var action = btn.getAttribute('data-action');
|
||||
if (action === 'show-keyboard-help' && window.lt) lt.modal.open('lt-keys-help');
|
||||
if (action === 'open-settings' && window.lt) lt.modal.open('lt-settings-modal');
|
||||
});
|
||||
|
||||
lt.keys.on('r', function() { lt.autoRefresh.now(); });
|
||||
lt.keys.on('?', function() { if (window.lt) lt.modal.open('lt-keys-help'); });
|
||||
lt.keys.on('*', function() { if (window.lt) lt.modal.open('lt-settings-modal'); });
|
||||
lt.keys.on('s', function() {
|
||||
if (typeof openSuppressModal === 'function') openSuppressModal('host', '', '');
|
||||
});
|
||||
|
||||
// ── Avatar image error fallback ───────────────────────────────────────
|
||||
document.addEventListener('error', function(e) {
|
||||
if (e.target.tagName === 'IMG' && e.target.closest('.lt-avatar')) {
|
||||
e.target.classList.add('lt-avatar-img-err');
|
||||
}
|
||||
}, true);
|
||||
|
||||
// ── Settings modal ────────────────────────────────────────────────────
|
||||
(function() {
|
||||
var LS_KEY = 'gandalf_settings';
|
||||
var DEFAULT = { refreshInterval: 30 };
|
||||
|
||||
function loadSettings() {
|
||||
try { return Object.assign({}, DEFAULT, JSON.parse(localStorage.getItem(LS_KEY) || '{}')); }
|
||||
catch(_) { return Object.assign({}, DEFAULT); }
|
||||
}
|
||||
|
||||
function saveSettings(s) {
|
||||
try { localStorage.setItem(LS_KEY, JSON.stringify(s)); } catch(_) {}
|
||||
if (typeof window.onGandalfSettingsChanged === 'function') window.onGandalfSettingsChanged(s);
|
||||
}
|
||||
|
||||
function applyRefreshPillUI(interval) {
|
||||
document.querySelectorAll('#settings-refresh-pills .pill').forEach(function(p) {
|
||||
p.classList.toggle('active', parseInt(p.dataset.refreshInterval) === interval);
|
||||
});
|
||||
var hint = document.getElementById('settings-refresh-hint');
|
||||
if (hint) {
|
||||
if (interval === 0) hint.textContent = 'Auto-refresh disabled.';
|
||||
else if (interval < 60) hint.textContent = 'Refreshes every ' + interval + ' seconds.';
|
||||
else hint.textContent = 'Refreshes every ' + Math.floor(interval/60) + ' minute' + (interval > 60 ? 's' : '') + '.';
|
||||
}
|
||||
}
|
||||
|
||||
// Init pill UI from saved settings
|
||||
var _settings = loadSettings();
|
||||
applyRefreshPillUI(_settings.refreshInterval);
|
||||
|
||||
// Expose for pages that need to read it (e.g. index.html for autoRefresh)
|
||||
window.gandalfSettings = _settings;
|
||||
|
||||
document.addEventListener('click', function(e) {
|
||||
var pill = e.target.closest('#settings-refresh-pills .pill[data-refresh-interval]');
|
||||
if (!pill) return;
|
||||
var interval = parseInt(pill.dataset.refreshInterval);
|
||||
_settings.refreshInterval = interval;
|
||||
saveSettings(_settings);
|
||||
applyRefreshPillUI(interval);
|
||||
});
|
||||
})();
|
||||
|
||||
// ── Notification Bell — shows active monitoring alerts ────────────────
|
||||
(function() {
|
||||
var bell = document.getElementById('lt-notif-bell');
|
||||
var panel = document.getElementById('lt-notif-panel');
|
||||
var list = document.getElementById('lt-notif-list');
|
||||
var clearBtn = document.getElementById('lt-notif-clear-btn');
|
||||
var wrapEl = document.getElementById('lt-notif-wrap');
|
||||
if (!bell || !panel) return;
|
||||
|
||||
var _open = false;
|
||||
var _lastEvents = [];
|
||||
var LS_READ_KEY = 'gandalf_notif_read_before';
|
||||
|
||||
function getReadBefore() {
|
||||
try { return parseInt(localStorage.getItem(LS_READ_KEY) || '0'); } catch(_) { return 0; }
|
||||
}
|
||||
function setReadBefore(ts) {
|
||||
try { localStorage.setItem(LS_READ_KEY, String(ts)); } catch(_) {}
|
||||
}
|
||||
|
||||
function esc(s) {
|
||||
return (window.lt && lt.escHtml) ? lt.escHtml(String(s)) : String(s).replace(/&/g,'&').replace(/</g,'<').replace(/>/g,'>');
|
||||
}
|
||||
|
||||
function toMs(dateStr) {
|
||||
if (!dateStr) return 0;
|
||||
return new Date(dateStr.replace(' UTC','Z').replace(' ','T')).getTime();
|
||||
}
|
||||
|
||||
function fmtAgo(dateStr) {
|
||||
var diff = Math.floor((Date.now() - toMs(dateStr)) / 1000);
|
||||
if (diff < 60) return diff + 's ago';
|
||||
if (diff < 3600) return Math.floor(diff/60) + 'm ago';
|
||||
if (diff < 86400) return Math.floor(diff/3600) + 'h ago';
|
||||
return Math.floor(diff/86400) + 'd ago';
|
||||
}
|
||||
|
||||
var SEV_DOT = { critical: 'var(--red)', warning: 'var(--amber)' };
|
||||
|
||||
function renderAlerts(events) {
|
||||
_lastEvents = events || [];
|
||||
var readBefore = getReadBefore();
|
||||
var active = _lastEvents.filter(function(e) { return e.severity !== 'info'; });
|
||||
var unreadCount = active.filter(function(e) { return toMs(e.last_seen) > readBefore; }).length;
|
||||
lt.notif.set(bell, unreadCount);
|
||||
|
||||
if (!active.length) {
|
||||
list.innerHTML = '<div style="padding:1rem;font-size:0.75rem;color:var(--text-muted);text-align:center">✔ No active alerts</div>';
|
||||
return;
|
||||
}
|
||||
list.innerHTML = active.slice(0, 25).map(function(e) {
|
||||
var isUnread = toMs(e.last_seen) > readBefore;
|
||||
var dotColor = SEV_DOT[e.severity] || 'var(--text-muted)';
|
||||
return '<div class="lt-notif-item' + (isUnread ? ' lt-notif-item--unread' : '') + '">' +
|
||||
'<div class="lt-notif-dot' + (isUnread ? '' : ' lt-notif-dot--read') + '" style="background:' + dotColor + ';border-radius:50%;margin-top:4px"></div>' +
|
||||
'<div class="lt-notif-item-body">' +
|
||||
'<div class="lt-notif-item-title">' + esc(e.target_name) + (e.target_detail ? ' · ' + esc(e.target_detail) : '') + '</div>' +
|
||||
'<div class="lt-notif-item-time">' + esc(e.event_type.replace(/_/g,' ')) + ' · ' + fmtAgo(e.last_seen) + '</div>' +
|
||||
'</div></div>';
|
||||
}).join('');
|
||||
}
|
||||
|
||||
function fetchAlerts(andRender) {
|
||||
fetch('/api/status', { credentials: 'same-origin' })
|
||||
.then(function(r) { return r.json(); })
|
||||
.then(function(data) {
|
||||
var events = data.events || [];
|
||||
if (andRender) {
|
||||
renderAlerts(events);
|
||||
} else {
|
||||
_lastEvents = events;
|
||||
var readBefore = getReadBefore();
|
||||
var active = events.filter(function(e) { return e.severity !== 'info'; });
|
||||
var unread = active.filter(function(e) { return toMs(e.last_seen) > readBefore; }).length;
|
||||
lt.notif.set(bell, unread);
|
||||
}
|
||||
})
|
||||
.catch(function() {
|
||||
if (andRender) list.innerHTML = '<div style="padding:0.75rem;font-size:0.75rem;color:var(--text-muted);text-align:center">Could not load</div>';
|
||||
});
|
||||
}
|
||||
|
||||
function openPanel() { _open = true; panel.removeAttribute('aria-hidden'); bell.setAttribute('aria-expanded','true'); fetchAlerts(true); }
|
||||
function closePanel() { _open = false; panel.setAttribute('aria-hidden','true'); bell.setAttribute('aria-expanded','false'); }
|
||||
|
||||
bell.addEventListener('click', function(e) { e.stopPropagation(); _open ? closePanel() : openPanel(); });
|
||||
|
||||
if (clearBtn) {
|
||||
clearBtn.addEventListener('click', function() {
|
||||
setReadBefore(Date.now());
|
||||
renderAlerts(_lastEvents);
|
||||
});
|
||||
}
|
||||
|
||||
document.addEventListener('click', function(e) { if (_open && wrapEl && !wrapEl.contains(e.target)) closePanel(); });
|
||||
document.addEventListener('keydown', function(e) { if (e.key === 'Escape' && _open) closePanel(); });
|
||||
|
||||
// Initial badge load + poll every 60 s
|
||||
fetchAlerts(false);
|
||||
setInterval(function() { fetchAlerts(_open); }, 60000);
|
||||
|
||||
// Allow refreshAll() to also push fresh events to the bell
|
||||
window.gandalfNotifUpdate = function(events) { renderAlerts(events); };
|
||||
})();
|
||||
</script>
|
||||
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,580 @@
|
||||
{% extends "base.html" %}
|
||||
{% block title %}Dashboard – GANDALF{% endblock %}
|
||||
|
||||
{% block content %}
|
||||
|
||||
<!-- ── Status bar ──────────────────────────────────────────────────── -->
|
||||
<div class="status-bar">
|
||||
<div class="status-chips">
|
||||
{% if not daemon_ok %}
|
||||
<span class="chip chip-critical">⚠ MONITOR OFFLINE</span>
|
||||
{% endif %}
|
||||
{% if summary.critical %}
|
||||
<span class="chip chip-critical">● {{ summary.critical }} CRITICAL</span>
|
||||
{% endif %}
|
||||
{% if summary.warning %}
|
||||
<span class="chip chip-warning">● {{ summary.warning }} WARNING</span>
|
||||
{% endif %}
|
||||
{% if daemon_ok and not summary.critical and not summary.warning %}
|
||||
<span class="chip chip-ok">✔ ALL SYSTEMS NOMINAL</span>
|
||||
{% endif %}
|
||||
</div>
|
||||
<div class="status-meta">
|
||||
<span class="last-check" id="last-check" data-tooltip="Last monitor poll timestamp" data-tooltip-pos="bottom">{{ last_check }}</span>
|
||||
<span class="lt-spinner lt-spinner--sm lt-spinner--cyan" id="refresh-spinner" style="display:none"></span>
|
||||
<button class="lt-btn lt-btn-ghost lt-btn-sm" data-action="refresh" data-tooltip="Refresh all data now" data-tooltip-pos="bottom">↻ REFRESH</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- ── Stats summary cards ──────────────────────────────────────────── -->
|
||||
<div class="lt-stats-grid">
|
||||
<div class="lt-stat-card{% if summary.critical %} lt-stat-card--alert{% endif %}"
|
||||
id="stat-critical" role="button" tabindex="0"
|
||||
data-stat-filter="critical" aria-label="{{ summary.critical or 0 }} critical alerts">
|
||||
<span class="lt-stat-icon" aria-hidden="true" style="color:var(--red);text-shadow:var(--glow-red)">●</span>
|
||||
<div class="lt-stat-info">
|
||||
<span class="lt-stat-value" id="stat-critical-val" style="color:var(--red)">{{ summary.critical or 0 }}</span>
|
||||
<span class="lt-stat-label">Critical</span>
|
||||
</div>
|
||||
</div>
|
||||
<div class="lt-stat-card"
|
||||
id="stat-warning" role="button" tabindex="0"
|
||||
data-stat-filter="warning" aria-label="{{ summary.warning or 0 }} warning alerts">
|
||||
<span class="lt-stat-icon" aria-hidden="true" style="color:var(--amber)">●</span>
|
||||
<div class="lt-stat-info">
|
||||
<span class="lt-stat-value" id="stat-warning-val" style="color:var(--amber)">{{ summary.warning or 0 }}</span>
|
||||
<span class="lt-stat-label">Warning</span>
|
||||
</div>
|
||||
</div>
|
||||
<div class="lt-stat-card" id="stat-hosts" aria-label="Monitored hosts">
|
||||
<span class="lt-stat-icon" aria-hidden="true" style="color:var(--cyan)">⬡</span>
|
||||
<div class="lt-stat-info">
|
||||
<span class="lt-stat-value" id="stat-hosts-val" style="color:var(--cyan)">{{ snapshot.hosts | length }}</span>
|
||||
<span class="lt-stat-label">Hosts</span>
|
||||
</div>
|
||||
</div>
|
||||
<div class="lt-stat-card" id="stat-resolved" aria-label="{{ recent_resolved | length }} alerts resolved in last 24 hours">
|
||||
<span class="lt-stat-icon" aria-hidden="true" style="color:var(--green);text-shadow:var(--glow)">✔</span>
|
||||
<div class="lt-stat-info">
|
||||
<span class="lt-stat-value" id="stat-resolved-val" style="color:var(--green)">{{ recent_resolved | length }}</span>
|
||||
<span class="lt-stat-label">Resolved 24h</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- ── Network topology + host grid ───────────────────────────────── -->
|
||||
<section class="g-section">
|
||||
<div class="g-section-header">
|
||||
<h2 class="g-section-title">Network Hosts</h2>
|
||||
</div>
|
||||
|
||||
<div class="topology" id="topology-diagram">
|
||||
<div class="topo-v2">
|
||||
|
||||
{%- set topo_h = snapshot.hosts if snapshot.hosts else {} -%}
|
||||
|
||||
<!-- ══════════════════════════════════════════════════════════════
|
||||
TIER 1: Internet (WAN edge)
|
||||
══════════════════════════════════════════════════════════ -->
|
||||
<div class="topo-tier">
|
||||
<div class="topo-v2-node topo-v2-internet">
|
||||
<span class="topo-v2-icon">◈</span>
|
||||
<span class="topo-v2-label">INTERNET</span>
|
||||
<span class="topo-v2-sub">WAN uplink</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- WAN wire: cyan → green gradient, labeled -->
|
||||
<div class="topo-vc">
|
||||
<div class="topo-vc-wire" style="background:linear-gradient(to bottom,var(--cyan),rgba(0,212,255,.3)); opacity:.7;"></div>
|
||||
<span class="topo-vc-label">WAN · 10G SFP+</span>
|
||||
</div>
|
||||
|
||||
<!-- ══════════════════════════════════════════════════════════════
|
||||
TIER 2: Router – UDM-Pro
|
||||
══════════════════════════════════════════════════════════ -->
|
||||
<div class="topo-tier">
|
||||
<div class="topo-v2-node topo-v2-router">
|
||||
<span class="topo-v2-icon">⬡</span>
|
||||
<span class="topo-v2-label">UDM-Pro</span>
|
||||
<span class="topo-v2-sub">Dream Machine Pro</span>
|
||||
<span class="topo-v2-sub">RU24</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- UDM-Pro → USW-Agg (10G SFP+) -->
|
||||
<div class="topo-vc">
|
||||
<div class="topo-vc-wire" style="background:var(--amber);opacity:.6;"></div>
|
||||
<span class="topo-vc-label">10G SFP+</span>
|
||||
</div>
|
||||
|
||||
<!-- ══════════════════════════════════════════════════════════════
|
||||
TIER 3: USW-Aggregation
|
||||
══════════════════════════════════════════════════════════ -->
|
||||
<div class="topo-tier">
|
||||
<div class="topo-v2-node topo-v2-switch" id="topo-switch-agg">
|
||||
<span class="topo-v2-icon">⬡</span>
|
||||
<span class="topo-v2-label">USW-Agg</span>
|
||||
<span class="topo-v2-sub">Aggregation · RU22</span>
|
||||
<span class="topo-v2-sub">8 × 10G SFP+</span>
|
||||
<span class="topo-v2-vlan">VLAN90 · 10.10.90.x/24</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- USW-Agg → Pro 24 PoE (10G trunk) -->
|
||||
<div class="topo-vc">
|
||||
<div class="topo-vc-wire" style="background:var(--amber);opacity:.6;"></div>
|
||||
<span class="topo-vc-label">10G trunk</span>
|
||||
</div>
|
||||
|
||||
<!-- ══════════════════════════════════════════════════════════════
|
||||
TIER 4: Pro 24 PoE
|
||||
══════════════════════════════════════════════════════════ -->
|
||||
<div class="topo-tier">
|
||||
<div class="topo-v2-node topo-v2-switch" id="topo-switch-poe">
|
||||
<span class="topo-v2-icon">⬡</span>
|
||||
<span class="topo-v2-label">Pro 24 PoE</span>
|
||||
<span class="topo-v2-sub">24-Port · RU23</span>
|
||||
<span class="topo-v2-sub">24 × 1G PoE</span>
|
||||
<span class="topo-v2-vlan">DHCP · mgmt</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Pro 24 PoE → host bus section -->
|
||||
<div class="topo-vc">
|
||||
<div class="topo-vc-wire" style="background:var(--border-color);opacity:.5;"></div>
|
||||
</div>
|
||||
|
||||
<!-- ══════════════════════════════════════════════════════════════
|
||||
TIER 4 connecting bus – two rails (10G green + 1G amber dashed)
|
||||
showing dual-homing for all 6 servers
|
||||
══════════════════════════════════════════════════════════ -->
|
||||
<div class="topo-bus-section" style="max-width:860px;">
|
||||
|
||||
<!-- 10G storage bus (Agg → VLAN90) -->
|
||||
<div class="topo-bus-10g">
|
||||
<span class="topo-bus-10g-label">← USW-Agg · 10G SFP+ · VLAN90 →</span>
|
||||
<div class="topo-bus-10g-line"></div>
|
||||
</div>
|
||||
|
||||
<!-- 1G management bus (PoE → DHCP) -->
|
||||
<div class="topo-bus-1g">
|
||||
<span class="topo-bus-1g-label">← Pro 24 PoE · 1G · DHCP mgmt →</span>
|
||||
<div class="topo-bus-1g-line"></div>
|
||||
</div>
|
||||
|
||||
<!-- ── Host nodes with drop wires ── -->
|
||||
<div class="topo-v2-hosts">
|
||||
{%- set all_defs = [
|
||||
('compute-storage-gpu-01', 'csg-01', 'RU4–12', 'Ceph · VLAN90', False),
|
||||
('compute-storage-01', 'cs-01', 'RU14–17', 'Ceph · VLAN90', False),
|
||||
('storage-01', 'sto-01', 'rack', 'Ceph · VLAN90', False),
|
||||
('monitor-01', 'mon-01', 'ZimaBoard', 'mgmt', False),
|
||||
('monitor-02', 'mon-02', 'ZimaBoard', 'mgmt', False),
|
||||
('large1', 'large1', 'off-rack', 'table', True),
|
||||
] -%}
|
||||
{%- for hname, hlabel, hsub, hvlan, off_rack in all_defs -%}
|
||||
{%- set st = topo_h[hname].status if hname in topo_h else 'unknown' -%}
|
||||
<div class="topo-v2-host-wrap">
|
||||
<!-- dual-homing wires: 10G solid green + 1G dashed amber -->
|
||||
<div class="topo-v2-host-wires">
|
||||
<div class="topo-v2-wire-10g" data-host="{{ hname }}" title="10G SFP+ → USW-Agg"></div>
|
||||
<div class="topo-v2-wire-1g" data-host="{{ hname }}" title="1G → Pro 24 PoE"></div>
|
||||
</div>
|
||||
<!-- host box -->
|
||||
<div class="topo-v2-node topo-v2-host topo-host topo-v2-status-{{ st }}{{ ' topo-v2-offrack' if off_rack else '' }}"
|
||||
data-host="{{ hname }}" style="min-width:80px; max-width:96px;">
|
||||
<span class="topo-v2-icon">▣</span>
|
||||
<span class="topo-v2-label">{{ hlabel }}</span>
|
||||
<span class="topo-v2-sub">{{ hsub }}</span>
|
||||
<span class="topo-v2-vlan">{{ hvlan }}</span>
|
||||
<span class="topo-badge topo-badge-{{ st }}">{{ st if st != 'unknown' else '–' }}</span>
|
||||
</div>
|
||||
</div>
|
||||
{%- endfor -%}
|
||||
</div>
|
||||
|
||||
</div><!-- /topo-bus-section -->
|
||||
|
||||
<!-- ── Legend ── -->
|
||||
<div class="topo-legend">
|
||||
<div class="topo-legend-item"><span class="topo-legend-line-wan"></span> WAN / uplink</div>
|
||||
<div class="topo-legend-item"><span class="topo-legend-line-10g"></span> 10G SFP+ (Ceph / VLAN90)</div>
|
||||
<div class="topo-legend-item"><span class="topo-legend-line-1g"></span> 1G DHCP (mgmt)</div>
|
||||
<div class="topo-legend-item" style="border:1px dashed var(--border-color); padding:1px 5px; font-size:.56em; color:var(--text-muted);">dashed border = off-rack</div>
|
||||
</div>
|
||||
|
||||
</div><!-- /topo-v2 -->
|
||||
</div>
|
||||
|
||||
<!-- Host cards -->
|
||||
<div class="host-grid" id="host-grid">
|
||||
{% for name, host in snapshot.hosts.items() %}
|
||||
{% set suppressed = suppressions | selectattr('target_name', 'equalto', name) | list %}
|
||||
<div class="host-card host-card-{{ host.status }}" data-host="{{ name }}">
|
||||
<div class="host-card-header">
|
||||
<div class="host-name-row">
|
||||
<span class="host-status-dot dot-{{ host.status }}"></span>
|
||||
<span class="host-name">{{ name }}</span>
|
||||
{% if suppressed %}
|
||||
<span class="badge-suppressed" title="Suppressed">🔕</span>
|
||||
{% endif %}
|
||||
</div>
|
||||
<div class="host-meta">
|
||||
<span class="host-ip">{{ host.ip }}</span>
|
||||
<span class="host-source source-{{ host.source }}">{{ host.source }}</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{% if host.interfaces %}
|
||||
<div class="iface-list">
|
||||
{% for iface, state in host.interfaces.items() | sort %}
|
||||
<div class="iface-row">
|
||||
<span class="iface-dot dot-{{ state }}"></span>
|
||||
<span class="iface-name">{{ iface }}</span>
|
||||
<span class="iface-state state-{{ state }}">{{ state }}</span>
|
||||
</div>
|
||||
{% endfor %}
|
||||
</div>
|
||||
{% else %}
|
||||
<div class="host-ping-note">ping-only / no node_exporter</div>
|
||||
{% endif %}
|
||||
|
||||
<div class="host-actions">
|
||||
<button class="lt-btn lt-btn-ghost lt-btn-sm btn-suppress"
|
||||
data-sup-type="host"
|
||||
data-sup-name="{{ name }}"
|
||||
data-sup-detail=""
|
||||
title="Suppress alerts for this host">
|
||||
🔕 Suppress
|
||||
</button>
|
||||
<a href="{{ url_for('links_page') }}#{{ name }}"
|
||||
class="lt-btn lt-btn-secondary lt-btn-sm" style="text-decoration:none">
|
||||
↗ Links
|
||||
</a>
|
||||
</div>
|
||||
</div>
|
||||
{% else %}
|
||||
<div class="lt-empty-state lt-empty-state--sm">
|
||||
<div class="lt-empty-state-icon">⌛</div>
|
||||
<div class="lt-empty-state-title">No host data yet</div>
|
||||
<div class="lt-empty-state-body">The monitor daemon may still be starting up.</div>
|
||||
</div>
|
||||
{% endfor %}
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<!-- ── UniFi devices ────────────────────────────────────────────────── -->
|
||||
{% if snapshot.unifi %}
|
||||
<section class="g-section">
|
||||
<div class="g-section-header">
|
||||
<h2 class="g-section-title">UniFi Devices</h2>
|
||||
</div>
|
||||
<div class="lt-table-wrap">
|
||||
<table class="lt-table" id="unifi-table">
|
||||
<caption class="lt-sr-only">UniFi network devices</caption>
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Status</th>
|
||||
<th>Name</th>
|
||||
<th>Type</th>
|
||||
<th>Model</th>
|
||||
<th>IP</th>
|
||||
<th>Actions</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{% for d in snapshot.unifi %}
|
||||
<tr class="{% if not d.connected %}row-critical{% endif %}">
|
||||
<td>
|
||||
<span class="dot-{{ 'up' if d.connected else 'down' }}"></span>
|
||||
{{ 'ONLINE' if d.connected else 'OFFLINE' }}
|
||||
</td>
|
||||
<td><strong>{{ d.name }}</strong></td>
|
||||
<td>{{ d.type }}</td>
|
||||
<td>{{ d.model }}</td>
|
||||
<td>{{ d.ip }}</td>
|
||||
<td>
|
||||
{% if not d.connected %}
|
||||
<button class="lt-btn lt-btn-ghost lt-btn-sm btn-suppress"
|
||||
data-sup-type="unifi_device"
|
||||
data-sup-name="{{ d.name }}"
|
||||
data-sup-detail="">
|
||||
🔕 Suppress
|
||||
</button>
|
||||
{% endif %}
|
||||
</td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</section>
|
||||
{% endif %}
|
||||
|
||||
<!-- ── Active alerts ───────────────────────────────────────────────── -->
|
||||
<section class="g-section">
|
||||
<div class="g-section-header">
|
||||
<h2 class="g-section-title">Active Alerts</h2>
|
||||
<span class="g-section-badge" id="alert-count-badge"
|
||||
{% if not summary.critical and not summary.warning %}style="display:none"{% endif %}>{{ (summary.critical or 0) + (summary.warning or 0) }}</span>
|
||||
</div>
|
||||
<div class="lt-toolbar">
|
||||
<div class="lt-toolbar-left">
|
||||
<div class="lt-search">
|
||||
<input type="search" class="lt-input lt-search-input" id="events-search"
|
||||
placeholder="Filter by target, type, description…" autocomplete="off">
|
||||
</div>
|
||||
<div class="sev-pills">
|
||||
<button type="button" class="pill active" data-sev="">All</button>
|
||||
<button type="button" class="pill" data-sev="critical">Critical</button>
|
||||
<button type="button" class="pill" data-sev="warning">Warning</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="lt-frame">
|
||||
<span class="lt-frame-bl">╚</span>
|
||||
<span class="lt-frame-br">╝</span>
|
||||
<div class="lt-section-header">Alert Queue</div>
|
||||
<div id="events-table-wrap">
|
||||
{% if events %}
|
||||
{% if total_active is defined and total_active > events|length %}
|
||||
<div class="pagination-notice">Showing {{ events|length }} of {{ total_active }} active alerts — <a href="/api/events?limit=1000">view all via API</a></div>
|
||||
{% endif %}
|
||||
<div class="lt-table-wrap">
|
||||
<table class="lt-table" id="events-table">
|
||||
<caption class="lt-sr-only">Active network alerts</caption>
|
||||
<thead>
|
||||
<tr>
|
||||
<th data-tooltip="Alert severity level" data-tooltip-pos="bottom">Sev</th>
|
||||
<th>Type</th>
|
||||
<th>Target</th>
|
||||
<th>Detail</th>
|
||||
<th>Description</th>
|
||||
<th data-tooltip="When this alert was first raised" data-tooltip-pos="bottom">First Seen</th>
|
||||
<th data-tooltip="Most recent check failure" data-tooltip-pos="bottom">Last Seen</th>
|
||||
<th data-tooltip="Consecutive check failures since first seen" data-tooltip-pos="bottom">Failures</th>
|
||||
<th>Ticket</th>
|
||||
<th>Actions</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{% for e in events %}
|
||||
{% if e.severity != 'info' %}
|
||||
<tr class="row-{{ e.severity }}">
|
||||
<td><span class="lt-badge badge-{{ e.severity }}">{{ e.severity }}</span></td>
|
||||
<td>{{ e.event_type | replace('_', ' ') }}</td>
|
||||
<td><strong>{{ e.target_name }}</strong></td>
|
||||
<td>{{ e.target_detail or '–' }}</td>
|
||||
<td class="desc-cell" title="{{ e.description | e }}">{{ e.description | truncate(60) }}</td>
|
||||
<td class="ts-cell" title="{{ e.first_seen }}">
|
||||
<span class="event-age" data-ts="{{ e.first_seen }}">{{ e.first_seen }}</span>
|
||||
</td>
|
||||
<td class="ts-cell" title="{{ e.last_seen }}">
|
||||
<span class="event-age" data-ts="{{ e.last_seen }}">{{ e.last_seen }}</span>
|
||||
</td>
|
||||
<td>{{ e.consecutive_failures }}</td>
|
||||
<td>
|
||||
{% if e.ticket_id %}
|
||||
<a href="{{ config.ticket_api.web_url }}{{ e.ticket_id }}" target="_blank"
|
||||
class="ticket-link">#{{ e.ticket_id }}</a>
|
||||
{% else %}–{% endif %}
|
||||
</td>
|
||||
<td>
|
||||
<button class="lt-btn lt-btn-ghost lt-btn-sm btn-suppress"
|
||||
data-sup-type="{{ 'unifi_device' if e.event_type == 'unifi_device_offline' else 'interface' if e.event_type == 'interface_down' else 'host' }}"
|
||||
data-sup-name="{{ e.target_name }}"
|
||||
data-sup-detail="{{ e.target_detail or '' }}"
|
||||
title="Suppress" aria-label="Suppress alert for {{ e.target_name }}">🔕</button>
|
||||
</td>
|
||||
</tr>
|
||||
{% endif %}
|
||||
{% else %}
|
||||
<tr><td colspan="10">
|
||||
<div class="lt-empty-state lt-empty-state--sm">
|
||||
<div class="lt-empty-state-icon">✔</div>
|
||||
<div class="lt-empty-state-title">No active alerts</div>
|
||||
</div>
|
||||
</td></tr>
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
{% else %}
|
||||
<div class="lt-empty-state lt-empty-state--sm">
|
||||
<div class="lt-empty-state-icon">✔</div>
|
||||
<div class="lt-empty-state-title">No active alerts</div>
|
||||
</div>
|
||||
{% endif %}
|
||||
</div><!-- /#events-table-wrap -->
|
||||
</div><!-- /.lt-frame -->
|
||||
</section>
|
||||
|
||||
<!-- ── Recently Resolved (last 24h) ───────────────────────────────── -->
|
||||
{% if recent_resolved %}
|
||||
<section class="g-section">
|
||||
<div class="g-section-header">
|
||||
<h2 class="g-section-title">Recently Resolved</h2>
|
||||
<span class="g-section-badge g-section-badge-resolved">{{ recent_resolved | length }} in last 24h</span>
|
||||
</div>
|
||||
<div class="lt-timeline">
|
||||
{% for e in recent_resolved %}
|
||||
{%- set dot_cls = 'lt-timeline-item--green' if e.severity == 'info' else 'lt-timeline-item--dim' -%}
|
||||
<div class="lt-timeline-item {{ dot_cls }}">
|
||||
<div class="lt-timeline-meta">
|
||||
<strong class="lt-timeline-actor">{{ e.target_name }}</strong>
|
||||
{% if e.target_detail %}<span>· {{ e.target_detail }}</span>{% endif %}
|
||||
<span class="lt-timeline-time event-age" data-ts="{{ e.resolved_at }}">{{ e.resolved_at }}</span>
|
||||
</div>
|
||||
<div class="lt-timeline-body">
|
||||
{{ e.event_type | replace('_', ' ') }}
|
||||
·
|
||||
<span class="lt-badge badge-resolved">{{ e.severity }}</span>
|
||||
· duration
|
||||
<span class="event-duration" data-first="{{ e.first_seen }}" data-resolved="{{ e.resolved_at }}">–</span>
|
||||
</div>
|
||||
</div>
|
||||
{% endfor %}
|
||||
</div>
|
||||
</section>
|
||||
{% endif %}
|
||||
|
||||
<!-- ── Quick-suppress modal ─────────────────────────────────────────── -->
|
||||
<div id="suppress-modal" class="lt-modal-overlay"
|
||||
role="dialog" aria-modal="true" aria-labelledby="suppress-modal-title" aria-hidden="true">
|
||||
<div class="lt-modal">
|
||||
<div class="lt-modal-header">
|
||||
<h3 class="lt-modal-title" id="suppress-modal-title">Suppress Alert</h3>
|
||||
<button type="button" class="lt-modal-close" data-modal-close aria-label="Close">✕</button>
|
||||
</div>
|
||||
<form id="suppress-form">
|
||||
<div class="lt-modal-body">
|
||||
<div class="lt-form-group" style="margin-bottom:12px">
|
||||
<label class="lt-label" for="sup-type">Target Type</label>
|
||||
<select class="lt-select" id="sup-type" name="target_type">
|
||||
<option value="host">Host (all interfaces)</option>
|
||||
<option value="interface">Specific Interface</option>
|
||||
<option value="unifi_device">UniFi Device</option>
|
||||
<option value="all">Global Maintenance</option>
|
||||
</select>
|
||||
</div>
|
||||
<div class="lt-form-group" id="sup-name-group" style="margin-bottom:12px">
|
||||
<label class="lt-label" for="sup-name">Target Name</label>
|
||||
<input type="text" class="lt-input" id="sup-name" name="target_name" placeholder="e.g. large1">
|
||||
</div>
|
||||
<div class="lt-form-group" id="sup-detail-group" style="margin-bottom:12px;display:none">
|
||||
<label class="lt-label" for="sup-detail">Interface <span class="lt-field-hint">(interface type only)</span></label>
|
||||
<input type="text" class="lt-input" id="sup-detail" name="target_detail" placeholder="e.g. enp35s0">
|
||||
</div>
|
||||
<div class="lt-form-group" style="margin-bottom:12px">
|
||||
<label class="lt-label" for="sup-reason">Reason <span class="required">*</span></label>
|
||||
<input type="text" class="lt-input" id="sup-reason" name="reason"
|
||||
placeholder="e.g. Planned switch reboot" required>
|
||||
</div>
|
||||
<div class="lt-form-group" style="margin-bottom:0">
|
||||
<label class="lt-label">Duration</label>
|
||||
<div class="duration-pills">
|
||||
<button type="button" class="pill" data-duration="30">30 min</button>
|
||||
<button type="button" class="pill" data-duration="60">1 hr</button>
|
||||
<button type="button" class="pill" data-duration="240">4 hr</button>
|
||||
<button type="button" class="pill" data-duration="480">8 hr</button>
|
||||
<button type="button" class="pill pill-manual active" data-duration="">Manual ∞</button>
|
||||
</div>
|
||||
<input type="hidden" id="sup-expires" name="expires_minutes" value="">
|
||||
<div class="lt-field-hint" id="duration-hint">Persists until manually removed.</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="lt-modal-footer">
|
||||
<button type="button" class="lt-btn lt-btn-secondary" data-modal-close>Cancel</button>
|
||||
<button type="submit" class="lt-btn lt-btn-primary">Apply</button>
|
||||
</div>
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{% endblock %}
|
||||
|
||||
{% block scripts %}
|
||||
<script>
|
||||
// Start auto-refresh using saved settings interval (default 30 s)
|
||||
var _savedInterval = (window.gandalfSettings && window.gandalfSettings.refreshInterval) || 30;
|
||||
if (_savedInterval > 0) lt.autoRefresh.start(refreshAll, _savedInterval * 1000);
|
||||
|
||||
// When settings change, restart auto-refresh with new interval
|
||||
window.onGandalfSettingsChanged = function(s) {
|
||||
lt.autoRefresh.stop();
|
||||
if (s.refreshInterval > 0) lt.autoRefresh.start(refreshAll, s.refreshInterval * 1000);
|
||||
};
|
||||
|
||||
document.getElementById('suppress-form')?.addEventListener('submit', submitSuppress);
|
||||
document.getElementById('sup-type')?.addEventListener('change', updateSuppressForm);
|
||||
|
||||
function updateEventAges() {
|
||||
document.querySelectorAll('.event-age[data-ts]').forEach(el => {
|
||||
el.textContent = lt.time.ago(_toIso(el.dataset.ts));
|
||||
});
|
||||
}
|
||||
|
||||
updateEventAges();
|
||||
setInterval(updateEventAges, 60000);
|
||||
|
||||
// ── Event duration (resolved_at - first_seen) ──────────────────
|
||||
function fmtDuration(firstTs, resolvedTs) {
|
||||
if (!firstTs || !resolvedTs) return '–';
|
||||
const secs = Math.floor((new Date(_toIso(resolvedTs)) - new Date(_toIso(firstTs))) / 1000);
|
||||
if (secs < 0) return '–';
|
||||
if (secs < 60) return `${secs}s`;
|
||||
if (secs < 3600) return `${Math.floor(secs/60)}m`;
|
||||
if (secs < 86400) return `${Math.floor(secs/3600)}h ${Math.floor((secs%3600)/60)}m`;
|
||||
return `${Math.floor(secs/86400)}d`;
|
||||
}
|
||||
|
||||
document.querySelectorAll('.event-duration[data-first][data-resolved]').forEach(el => {
|
||||
el.textContent = fmtDuration(el.dataset.first, el.dataset.resolved);
|
||||
});
|
||||
|
||||
// ── Events table filter ────────────────────────────────────────
|
||||
let _filterSev = '';
|
||||
|
||||
function applyEventsFilter() {
|
||||
const q = (document.getElementById('events-search')?.value || '').toLowerCase();
|
||||
const tbody = document.querySelector('#events-table tbody');
|
||||
if (!tbody) return;
|
||||
tbody.querySelectorAll('tr').forEach(row => {
|
||||
if (row.children.length < 3) { row.style.display = ''; return; }
|
||||
const sevMatch = !_filterSev || row.classList.contains(`row-${_filterSev}`);
|
||||
const textMatch = !q || row.textContent.toLowerCase().includes(q);
|
||||
row.style.display = (sevMatch && textMatch) ? '' : 'none';
|
||||
});
|
||||
}
|
||||
|
||||
document.getElementById('events-search')?.addEventListener('input', applyEventsFilter);
|
||||
|
||||
document.querySelector('.sev-pills')?.addEventListener('click', e => {
|
||||
const pill = e.target.closest('.pill[data-sev]');
|
||||
if (!pill) return;
|
||||
document.querySelectorAll('.sev-pills .pill').forEach(p => p.classList.remove('active'));
|
||||
pill.classList.add('active');
|
||||
_filterSev = pill.dataset.sev;
|
||||
applyEventsFilter();
|
||||
});
|
||||
|
||||
// Re-apply filter after dynamic table updates
|
||||
new MutationObserver(applyEventsFilter)
|
||||
.observe(document.getElementById('events-table-wrap'), { childList: true, subtree: true });
|
||||
|
||||
// Stat card clicks — filter events table by severity
|
||||
document.querySelectorAll('.lt-stat-card[data-stat-filter]').forEach(card => {
|
||||
card.addEventListener('click', () => {
|
||||
const sev = card.dataset.statFilter;
|
||||
document.querySelectorAll('.sev-pills .pill').forEach(p => p.classList.remove('active'));
|
||||
const matchPill = document.querySelector(`.sev-pills .pill[data-sev="${sev}"]`);
|
||||
if (matchPill) matchPill.classList.add('active');
|
||||
_filterSev = sev;
|
||||
applyEventsFilter();
|
||||
document.getElementById('events-table-wrap')?.scrollIntoView({ behavior: 'smooth', block: 'start' });
|
||||
});
|
||||
card.addEventListener('keydown', e => { if (e.key === 'Enter' || e.key === ' ') { e.preventDefault(); card.click(); } });
|
||||
});
|
||||
</script>
|
||||
{% endblock %}
|
||||
@@ -0,0 +1,796 @@
|
||||
{% extends "base.html" %}
|
||||
{% block title %}Inspector – GANDALF{% endblock %}
|
||||
|
||||
{% block content %}
|
||||
|
||||
<div class="lt-page-header">
|
||||
<div>
|
||||
<h1 class="lt-page-title">Network Inspector</h1>
|
||||
<p class="g-page-sub" style="margin-top:4px">
|
||||
Visual switch chassis diagrams. Click a port to see detailed stats and LLDP path debug.
|
||||
<span id="inspector-updated" style="margin-left:8px"></span>
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="inspector-layout">
|
||||
<div class="inspector-main" id="inspector-main">
|
||||
<div class="link-loading">Loading inspector data</div>
|
||||
</div>
|
||||
<div class="inspector-panel" id="inspector-panel">
|
||||
<div class="inspector-panel-inner" id="inspector-panel-inner"></div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{% endblock %}
|
||||
|
||||
{% block scripts %}
|
||||
<script>
|
||||
const escHtml = s => lt.escHtml(s);
|
||||
|
||||
// ── Switch layout config ─────────────────────────────────────────────────
|
||||
// keys match the model field returned by the UniFi API
|
||||
// rows: array of rows, each row is an array of port_idx values
|
||||
// sfp_ports: port_idx values that are SFP cages (in rows, rendered differently)
|
||||
// sfp_section: port_idx values rendered as a separate SFP bank below the rows
|
||||
const SWITCH_LAYOUTS = {
|
||||
'USF5P': { rows: [[1,2,3,4]], sfp_section: [5] },
|
||||
'USL8A': { rows: [[1,2,3,4],[5,6,7,8]], sfp_ports: [1,2,3,4,5,6,7,8] },
|
||||
'US24PRO': {
|
||||
rows: [
|
||||
[1,3,5,7,9,11,13,15,17,19,21,23],
|
||||
[2,4,6,8,10,12,14,16,18,20,22,24],
|
||||
],
|
||||
sfp_section: [25,26],
|
||||
},
|
||||
'USPPDUP': { rows: [[1]] },
|
||||
'USMINI': { rows: [[1,2,3,4,5]] },
|
||||
};
|
||||
|
||||
// ── Formatting helpers ───────────────────────────────────────────────────
|
||||
function fmtSpeed(mbps) {
|
||||
if (!mbps) return '–';
|
||||
if (mbps >= 1000) return (mbps / 1000).toFixed(0) + 'G';
|
||||
return mbps + 'M';
|
||||
}
|
||||
|
||||
function fmtRate(bytesPerSec) {
|
||||
if (bytesPerSec === null || bytesPerSec === undefined) return '–';
|
||||
const bps = bytesPerSec * 8;
|
||||
if (bps < 1e3) return bps.toFixed(0) + ' bps';
|
||||
if (bps < 1e6) return (bps / 1e3).toFixed(1) + ' Kbps';
|
||||
if (bps < 1e9) return (bps / 1e6).toFixed(2) + ' Mbps';
|
||||
return (bps / 1e9).toFixed(3) + ' Gbps';
|
||||
}
|
||||
|
||||
function fmtErrors(rate) {
|
||||
if (rate === null || rate === undefined) return '–';
|
||||
if (rate < 0.001) return '<span class="val-good">0 /s</span>';
|
||||
return `<span class="val-crit">${rate.toFixed(3)} /s</span>`;
|
||||
}
|
||||
|
||||
// ── Build port_idx → port data map ──────────────────────────────────────
|
||||
function buildPortIdxMap(ports) {
|
||||
const map = {};
|
||||
for (const [pname, d] of Object.entries(ports)) {
|
||||
if (d.port_idx != null) {
|
||||
map[d.port_idx] = Object.assign({ name: pname }, d);
|
||||
}
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
// ── Determine port block CSS state class ────────────────────────────────
|
||||
function portBlockState(d) {
|
||||
if (!d || !d.up) return 'down';
|
||||
if (d.is_uplink) return 'uplink';
|
||||
if (d.poe_power != null && d.poe_power > 0) return 'poe-active';
|
||||
return 'up';
|
||||
}
|
||||
|
||||
// ── Speed label helper ───────────────────────────────────────────────────
|
||||
function portSpeedLabel(port) {
|
||||
if (!port || !port.up) return '';
|
||||
const spd = port.speed; // speed in Mbps from UniFi API
|
||||
if (!spd) return '';
|
||||
if (spd >= 10000) return '10G';
|
||||
if (spd >= 1000) return '1G';
|
||||
if (spd >= 100) return '100M';
|
||||
return spd + 'M';
|
||||
}
|
||||
|
||||
// ── Render a single port block element ──────────────────────────────────
|
||||
function portBlockHtml(idx, port, swName, sfpBlock) {
|
||||
const state = portBlockState(port);
|
||||
const numLabel = sfpBlock ? 'SFP' : idx;
|
||||
const title = port ? escHtml(port.name) : `Port ${idx}`;
|
||||
const sfpCls = sfpBlock ? ' sfp-block' : '';
|
||||
const speedTxt = portSpeedLabel(port);
|
||||
// LLDP neighbor: first 6 chars of hostname
|
||||
const lldpName = (port && port.lldp_table && port.lldp_table.length)
|
||||
? escHtml((port.lldp_table[0].chassis_id_subtype === 'local'
|
||||
? port.lldp_table[0].chassis_id
|
||||
: port.lldp_table[0].system_name || port.lldp_table[0].chassis_id || '').slice(0, 6))
|
||||
: '';
|
||||
const lldpHtml = lldpName ? `<span class="port-lldp">${lldpName}</span>` : '';
|
||||
const speedHtml = speedTxt ? `<span class="port-speed">${speedTxt}</span>` : '';
|
||||
return `<div class="switch-port-block ${state}${sfpCls}"
|
||||
data-switch="${escHtml(swName)}" data-port-idx="${idx}"
|
||||
title="${title}" aria-label="${title}"
|
||||
data-action="select-port"><span class="port-num">${numLabel}</span>${speedHtml}${lldpHtml}</div>`;
|
||||
}
|
||||
|
||||
// ── Chassis legend HTML ──────────────────────────────────────────────────
|
||||
function chassisLegendHtml() {
|
||||
return `<div class="chassis-legend">
|
||||
<div class="chassis-legend-item"><span class="chassis-legend-swatch cls-down"></span>down</div>
|
||||
<div class="chassis-legend-item"><span class="chassis-legend-swatch cls-up"></span>up</div>
|
||||
<div class="chassis-legend-item"><span class="chassis-legend-swatch cls-poe"></span>poe active</div>
|
||||
<div class="chassis-legend-item"><span class="chassis-legend-swatch cls-uplink"></span>uplink</div>
|
||||
</div>`;
|
||||
}
|
||||
|
||||
// ── Render one switch chassis ────────────────────────────────────────────
|
||||
function renderChassis(swName, sw) {
|
||||
const model = sw.model || '';
|
||||
const layout = SWITCH_LAYOUTS[model] || null;
|
||||
const portMap = buildPortIdxMap(sw.ports || {});
|
||||
|
||||
const upCount = Object.values(sw.ports || {}).filter(p => p.up).length;
|
||||
const totCount = Object.keys(sw.ports || {}).length;
|
||||
const downCount = totCount - upCount;
|
||||
const meta = [model, `${upCount}/${totCount} up`, downCount ? `${downCount} down` : ''].filter(Boolean).join(' · ');
|
||||
|
||||
// Is this a US24PRO? Used to add group-separator class
|
||||
const isUs24Pro = (model === 'US24PRO');
|
||||
|
||||
let chassisHtml = '';
|
||||
|
||||
if (layout) {
|
||||
const sfpPortSet = new Set(layout.sfp_ports || []);
|
||||
const sfpSectionSet = new Set(layout.sfp_section || []);
|
||||
|
||||
// Main port rows
|
||||
chassisHtml += '<div class="chassis-rows">';
|
||||
for (const row of layout.rows) {
|
||||
const rowCls = isUs24Pro ? ' us24pro-row' : '';
|
||||
chassisHtml += `<div class="chassis-row${rowCls}">`;
|
||||
for (const idx of row) {
|
||||
const port = portMap[idx];
|
||||
const isSfp = sfpPortSet.has(idx);
|
||||
const sfpCls = isSfp ? ' sfp-port' : '';
|
||||
const state = portBlockState(port);
|
||||
const title = port ? escHtml(port.name) : `Port ${idx}`;
|
||||
const speedTxt = portSpeedLabel(port);
|
||||
const lldpName = (port && port.lldp_table && port.lldp_table.length)
|
||||
? escHtml((port.lldp_table[0].chassis_id_subtype === 'local'
|
||||
? port.lldp_table[0].chassis_id
|
||||
: port.lldp_table[0].system_name || port.lldp_table[0].chassis_id || '').slice(0, 6))
|
||||
: '';
|
||||
const speedHtml = speedTxt ? `<span class="port-speed">${speedTxt}</span>` : '';
|
||||
const lldpHtml = lldpName ? `<span class="port-lldp">${lldpName}</span>` : '';
|
||||
chassisHtml += `<div class="switch-port-block ${state}${sfpCls}"
|
||||
data-switch="${escHtml(swName)}" data-port-idx="${idx}"
|
||||
title="${title}" aria-label="${title}"
|
||||
data-action="select-port"><span class="port-num">${idx}</span>${speedHtml}${lldpHtml}</div>`;
|
||||
}
|
||||
chassisHtml += '</div>';
|
||||
}
|
||||
chassisHtml += '</div>';
|
||||
|
||||
// Separate SFP section (if present)
|
||||
if (sfpSectionSet.size) {
|
||||
chassisHtml += '<div class="chassis-sfp-section">';
|
||||
for (const idx of layout.sfp_section) {
|
||||
chassisHtml += portBlockHtml(idx, portMap[idx], swName, true);
|
||||
}
|
||||
chassisHtml += '</div>';
|
||||
}
|
||||
} else {
|
||||
// Fallback: render all ports sorted by idx
|
||||
const allPorts = Object.entries(sw.ports || {})
|
||||
.sort(([, a], [, b]) => (a.port_idx || 0) - (b.port_idx || 0));
|
||||
chassisHtml += '<div class="chassis-rows"><div class="chassis-row">';
|
||||
for (const [pname, d] of allPorts) {
|
||||
chassisHtml += portBlockHtml(d.port_idx || 0, Object.assign({ name: pname }, d), swName, false);
|
||||
}
|
||||
chassisHtml += '</div></div>';
|
||||
}
|
||||
|
||||
return `
|
||||
<div class="inspector-chassis" id="chassis-${escHtml(swName)}">
|
||||
<div class="chassis-header">
|
||||
<span class="chassis-name">${escHtml(swName)}</span>
|
||||
${sw.ip ? `<span class="chassis-ip">${escHtml(sw.ip)}</span>` : ''}
|
||||
<span class="chassis-meta">${escHtml(meta)}</span>
|
||||
</div>
|
||||
<div class="chassis-body">
|
||||
<div class="chassis-ear-l"></div>
|
||||
<div class="chassis-ear-r"></div>
|
||||
${chassisHtml}
|
||||
</div>
|
||||
${chassisLegendHtml()}
|
||||
</div>`;
|
||||
}
|
||||
|
||||
// ── State ────────────────────────────────────────────────────────────────
|
||||
let _selectedSwitch = null;
|
||||
let _selectedIdx = null;
|
||||
let _apiData = null;
|
||||
|
||||
// ── Port selection ───────────────────────────────────────────────────────
|
||||
function selectPort(el) {
|
||||
const swName = el.dataset.switch;
|
||||
const idx = parseInt(el.dataset.portIdx, 10);
|
||||
document.querySelectorAll('.switch-port-block.selected')
|
||||
.forEach(e => e.classList.remove('selected'));
|
||||
el.classList.add('selected');
|
||||
_selectedSwitch = swName;
|
||||
_selectedIdx = idx;
|
||||
renderPanel(swName, idx);
|
||||
}
|
||||
|
||||
function closePanel() {
|
||||
document.getElementById('inspector-panel').classList.remove('open');
|
||||
document.querySelectorAll('.switch-port-block.selected')
|
||||
.forEach(el => el.classList.remove('selected'));
|
||||
_selectedSwitch = null;
|
||||
_selectedIdx = null;
|
||||
}
|
||||
|
||||
// ── Render detail panel ──────────────────────────────────────────────────
|
||||
function renderPanel(swName, idx) {
|
||||
if (!_apiData) return;
|
||||
const sw = _apiData.unifi_switches && _apiData.unifi_switches[swName];
|
||||
if (!sw) return;
|
||||
const portMap = buildPortIdxMap(sw.ports || {});
|
||||
const d = portMap[idx];
|
||||
if (!d) return;
|
||||
|
||||
const upStr = d.up ? '<span class="val-good">UP</span>' : '<span class="val-crit">DOWN</span>';
|
||||
const speedStr = d.speed_mbps ? `<span class="val-cyan">${fmtSpeed(d.speed_mbps)}bps</span>` : '–';
|
||||
const duplexCls = d.full_duplex ? 'val-good' : (d.up ? 'val-warn' : 'val-neutral');
|
||||
const duplexStr = d.up ? `<span class="${duplexCls}">${d.full_duplex ? 'Full' : 'Half'}</span>` : '–';
|
||||
const autoneg = d.autoneg ? 'On' : 'Off';
|
||||
const mediaStr = d.media || '–';
|
||||
const isUplinkBadge = d.is_uplink ? ' <span class="port-badge port-badge-uplink">UPLINK</span>' : '';
|
||||
|
||||
// PoE section
|
||||
let poeHtml = '';
|
||||
if (d.poe_class != null) {
|
||||
const poeMaxStr = d.poe_max_power != null ? ` / max ${d.poe_max_power.toFixed(1)}W` : '';
|
||||
const poeCurStr = (d.poe_power != null && d.poe_power > 0) ? ` / draw <span class="val-amber">${d.poe_power.toFixed(1)}W</span>` : '';
|
||||
poeHtml = `
|
||||
<div class="lt-divider"><span class="lt-divider-label">PoE</span></div>
|
||||
<div class="panel-row"><span class="panel-label">Class</span><span class="panel-val">class ${d.poe_class}${poeMaxStr}</span></div>
|
||||
${d.poe_power != null ? `<div class="panel-row"><span class="panel-label">Draw</span><span class="panel-val">${d.poe_power > 0 ? `<span class="val-amber">${d.poe_power.toFixed(1)}W</span>` : '0W'}</span></div>` : ''}
|
||||
${d.poe_mode ? `<div class="panel-row"><span class="panel-label">Mode</span><span class="panel-val">${escHtml(d.poe_mode)}</span></div>` : ''}`;
|
||||
}
|
||||
|
||||
// Traffic section
|
||||
let trafficHtml = '';
|
||||
if (d.tx_bytes_rate != null || d.rx_bytes_rate != null) {
|
||||
trafficHtml = `
|
||||
<div class="lt-divider"><span class="lt-divider-label">Traffic</span></div>
|
||||
<div class="panel-row"><span class="panel-label" data-tooltip="Transmit — outgoing from this port">TX</span><span class="panel-val">${fmtRate(d.tx_bytes_rate)}</span></div>
|
||||
<div class="panel-row"><span class="panel-label" data-tooltip="Receive — incoming to this port">RX</span><span class="panel-val">${fmtRate(d.rx_bytes_rate)}</span></div>`;
|
||||
}
|
||||
|
||||
// Errors / drops section
|
||||
let errHtml = '';
|
||||
if (d.tx_errs_rate != null || d.rx_errs_rate != null) {
|
||||
errHtml = `
|
||||
<div class="lt-divider"><span class="lt-divider-label">Errors / Drops</span></div>
|
||||
<div class="panel-row"><span class="panel-label">TX Err</span><span class="panel-val">${fmtErrors(d.tx_errs_rate)}</span></div>
|
||||
<div class="panel-row"><span class="panel-label">RX Err</span><span class="panel-val">${fmtErrors(d.rx_errs_rate)}</span></div>
|
||||
<div class="panel-row"><span class="panel-label">TX Drop</span><span class="panel-val">${fmtErrors(d.tx_drops_rate)}</span></div>
|
||||
<div class="panel-row"><span class="panel-label">RX Drop</span><span class="panel-val">${fmtErrors(d.rx_drops_rate)}</span></div>`;
|
||||
}
|
||||
|
||||
// LLDP + path debug
|
||||
let lldpHtml = '';
|
||||
let pathHtml = '';
|
||||
if (d.lldp && d.lldp.system_name) {
|
||||
const l = d.lldp;
|
||||
lldpHtml = `
|
||||
<div class="lt-divider"><span class="lt-divider-label">LLDP Neighbor</span></div>
|
||||
<div class="panel-row"><span class="panel-label">System</span><span class="panel-val val-cyan">${escHtml(l.system_name)}</span></div>
|
||||
${l.port_id ? `<div class="panel-row"><span class="panel-label">Port</span><span class="panel-val">${escHtml(l.port_id)}</span></div>` : ''}
|
||||
${l.port_desc ? `<div class="panel-row"><span class="panel-label">Port Desc</span><span class="panel-val">${escHtml(l.port_desc)}</span></div>` : ''}
|
||||
${l.chassis_id ? `<div class="panel-row"><span class="panel-label">Chassis</span><span class="panel-val">${escHtml(l.chassis_id)}</span></div>` : ''}
|
||||
${l.mgmt_ips && l.mgmt_ips.length ? `<div class="panel-row"><span class="panel-label">Mgmt IP</span><span class="panel-val">${escHtml(l.mgmt_ips.join(', '))}</span></div>` : ''}`;
|
||||
|
||||
// Path debug: look for matching server interface
|
||||
const hosts = _apiData.hosts || {};
|
||||
const serverIfaces = hosts[l.system_name];
|
||||
if (serverIfaces) {
|
||||
let matchedIface = l.port_id && serverIfaces[l.port_id] ? l.port_id : null;
|
||||
if (!matchedIface && l.port_id) {
|
||||
// fuzzy match
|
||||
matchedIface = Object.keys(serverIfaces).find(k => l.port_id.includes(k) || k.includes(l.port_id)) || null;
|
||||
}
|
||||
if (matchedIface) {
|
||||
pathHtml = buildPathDebug(swName, d, l.system_name, matchedIface, serverIfaces[matchedIface]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Diagnose button (only when LLDP has an identified neighbor we can map)
|
||||
const hasDiagTarget = !!(d.lldp && d.lldp.system_name &&
|
||||
_apiData.hosts && _apiData.hosts[d.lldp.system_name]);
|
||||
const diagHtml = hasDiagTarget ? `
|
||||
<div class="diag-bar">
|
||||
<button class="btn-diag" data-action="run-diagnostic" data-sw="${escHtml(swName)}" data-idx="${idx}">Run Link Diagnostics</button>
|
||||
<span class="diag-status" id="diag-status"></span>
|
||||
</div>
|
||||
<div class="diag-results" id="diag-results"></div>` : '';
|
||||
|
||||
const inner = document.getElementById('inspector-panel-inner');
|
||||
inner.innerHTML = `
|
||||
<div class="panel-header">
|
||||
<div>
|
||||
<span class="panel-port-name">${escHtml(d.name)}</span>${isUplinkBadge}
|
||||
<div class="panel-meta">${escHtml(swName)} · port #${idx}</div>
|
||||
</div>
|
||||
<button class="panel-close" data-action="close-panel" aria-label="Close panel">✕</button>
|
||||
</div>
|
||||
|
||||
<div class="lt-divider"><span class="lt-divider-label">Link</span></div>
|
||||
<div class="panel-row"><span class="panel-label">Status</span><span class="panel-val">${upStr}</span></div>
|
||||
<div class="panel-row"><span class="panel-label">Speed</span><span class="panel-val">${speedStr}</span></div>
|
||||
<div class="panel-row"><span class="panel-label" data-tooltip="Full = simultaneous send/receive. Half = one direction at a time.">Duplex</span><span class="panel-val">${duplexStr}</span></div>
|
||||
<div class="panel-row"><span class="panel-label" data-tooltip="Autonegotiation: NIC and switch automatically agree speed and duplex.">Auto-neg</span><span class="panel-val val-neutral">${autoneg}</span></div>
|
||||
<div class="panel-row"><span class="panel-label">Media</span><span class="panel-val">${escHtml(mediaStr)}</span></div>
|
||||
|
||||
${poeHtml}
|
||||
${trafficHtml}
|
||||
${errHtml}
|
||||
${lldpHtml}
|
||||
${pathHtml}
|
||||
${diagHtml}
|
||||
`;
|
||||
|
||||
document.getElementById('inspector-panel').classList.add('open');
|
||||
}
|
||||
|
||||
// ── Build path debug two-column section ─────────────────────────────────
|
||||
function buildPathDebug(swName, swPort, serverName, ifaceName, svrData) {
|
||||
const isFiber = (swPort.media || '').toLowerCase().includes('sfp') ||
|
||||
(svrData.port_type || '').toLowerCase().includes('fibre') ||
|
||||
(svrData.port_type || '').toLowerCase().includes('fiber') ||
|
||||
!!svrData.sfp;
|
||||
const connType = isFiber ? 'SFP / Fiber' : 'Copper';
|
||||
|
||||
let sfpDomHtml = '';
|
||||
if (svrData.sfp && Object.keys(svrData.sfp).length) {
|
||||
const sfp = svrData.sfp;
|
||||
sfpDomHtml = '<div class="path-dom">';
|
||||
if (sfp.vendor) sfpDomHtml += `<div class="path-dom-row"><span>Vendor</span><span>${escHtml(sfp.vendor)}${sfp.part_no ? ' / ' + escHtml(sfp.part_no) : ''}</span></div>`;
|
||||
if (sfp.temp_c != null) sfpDomHtml += `<div class="path-dom-row"><span>Temp</span><span>${sfp.temp_c.toFixed(1)}°C</span></div>`;
|
||||
if (sfp.tx_power_dbm != null) sfpDomHtml += `<div class="path-dom-row"><span>TX</span><span>${sfp.tx_power_dbm.toFixed(2)} dBm</span></div>`;
|
||||
if (sfp.rx_power_dbm != null) sfpDomHtml += `<div class="path-dom-row"><span>RX</span><span>${sfp.rx_power_dbm.toFixed(2)} dBm</span></div>`;
|
||||
sfpDomHtml += '</div>';
|
||||
}
|
||||
|
||||
const svrErrTx = (svrData.tx_errs_rate > 0.001) ? 'val-crit' : 'val-good';
|
||||
const svrErrRx = (svrData.rx_errs_rate > 0.001) ? 'val-crit' : 'val-good';
|
||||
const swErrTx = (swPort.tx_errs_rate > 0.001) ? 'val-crit' : 'val-good';
|
||||
const swErrRx = (swPort.rx_errs_rate > 0.001) ? 'val-crit' : 'val-good';
|
||||
|
||||
// Detect duplex mismatch (switch full_duplex vs server duplex string)
|
||||
const swFull = swPort.full_duplex;
|
||||
const svrFull = (svrData.duplex || '').toLowerCase().includes('full');
|
||||
const duplexMismatch = swPort.up && svrData.duplex &&
|
||||
((swFull && !svrFull) || (!swFull && svrFull));
|
||||
const duplexWarnHtml = duplexMismatch
|
||||
? `<div class="path-mismatch-alert">⚠ DUPLEX MISMATCH — Switch: ${swFull ? 'Full' : 'Half'} · Server: ${escHtml(svrData.duplex)}</div>`
|
||||
: '';
|
||||
|
||||
// Detect speed mismatch
|
||||
const swSpd = swPort.speed_mbps, svrSpd = svrData.speed_mbps;
|
||||
const speedMismatch = swSpd && svrSpd && swSpd > 0 && svrSpd > 0 && swSpd !== svrSpd;
|
||||
const speedWarnHtml = speedMismatch
|
||||
? `<div class="path-mismatch-alert">⚠ SPEED MISMATCH — Switch: ${fmtSpeed(swSpd)} · Server: ${fmtSpeed(svrSpd)}</div>`
|
||||
: '';
|
||||
|
||||
return `
|
||||
<div class="lt-divider"><span class="lt-divider-label">Path Debug · ${escHtml(connType)}</span></div>
|
||||
${duplexWarnHtml}${speedWarnHtml}
|
||||
<div class="path-debug-cols">
|
||||
<div class="path-col">
|
||||
<div class="path-col-header">Switch</div>
|
||||
<div class="path-row"><span>Port</span><span>${escHtml(swPort.name)}</span></div>
|
||||
<div class="path-row"><span>Speed</span><span>${fmtSpeed(swPort.speed_mbps)}bps</span></div>
|
||||
<div class="path-row"><span>TX</span><span>${fmtRate(swPort.tx_bytes_rate)}</span></div>
|
||||
<div class="path-row"><span>RX</span><span>${fmtRate(swPort.rx_bytes_rate)}</span></div>
|
||||
<div class="path-row"><span>TX Err</span><span class="${swErrTx}">${fmtErrors(swPort.tx_errs_rate)}</span></div>
|
||||
<div class="path-row"><span>RX Err</span><span class="${swErrRx}">${fmtErrors(swPort.rx_errs_rate)}</span></div>
|
||||
${(swPort.poe_power != null && swPort.poe_power > 0) ? `<div class="path-row"><span>PoE</span><span class="val-amber">${swPort.poe_power.toFixed(1)}W</span></div>` : ''}
|
||||
</div>
|
||||
<div class="path-col">
|
||||
<div class="path-col-header">Server: ${escHtml(serverName)}</div>
|
||||
<div class="path-row"><span>Iface</span><span>${escHtml(ifaceName)}</span></div>
|
||||
<div class="path-row"><span>Speed</span><span>${svrData.speed_mbps ? fmtSpeed(svrData.speed_mbps) + 'bps' : '–'}</span></div>
|
||||
<div class="path-row"><span>TX</span><span>${fmtRate(svrData.tx_bytes_rate)}</span></div>
|
||||
<div class="path-row"><span>RX</span><span>${fmtRate(svrData.rx_bytes_rate)}</span></div>
|
||||
<div class="path-row"><span>TX Err</span><span class="${svrErrTx}">${fmtErrors(svrData.tx_errs_rate)}</span></div>
|
||||
<div class="path-row"><span>RX Err</span><span class="${svrErrRx}">${fmtErrors(svrData.rx_errs_rate)}</span></div>
|
||||
${svrData.carrier_changes != null ? `<div class="path-row"><span>Carrier Chg</span><span class="${(svrData.carrier_changes||0)>10?'val-crit':(svrData.carrier_changes||0)>2?'val-warn':'val-good'}">${svrData.carrier_changes}</span></div>` : ''}
|
||||
${sfpDomHtml}
|
||||
</div>
|
||||
</div>`;
|
||||
}
|
||||
|
||||
// ── Render all switches ──────────────────────────────────────────────────
|
||||
function renderInspector(data) {
|
||||
_apiData = data;
|
||||
const main = document.getElementById('inspector-main');
|
||||
const switches = data.unifi_switches || {};
|
||||
|
||||
const updEl = document.getElementById('inspector-updated');
|
||||
if (updEl && data.updated) {
|
||||
updEl.textContent = 'Updated: ' + new Date(data.updated + (data.updated.includes('Z') ? '' : 'Z')).toLocaleTimeString();
|
||||
}
|
||||
|
||||
if (!Object.keys(switches).length) {
|
||||
main.innerHTML = '<p class="empty-state">No switch data available. Monitor may still be initialising.</p>';
|
||||
return;
|
||||
}
|
||||
|
||||
main.innerHTML = Object.entries(switches)
|
||||
.map(([swName, sw]) => renderChassis(swName, sw))
|
||||
.join('');
|
||||
|
||||
// Re-apply selection highlight after re-render (dataset compare — no CSS escaping)
|
||||
if (_selectedSwitch && _selectedIdx !== null) {
|
||||
const block = Array.from(document.querySelectorAll('.switch-port-block')).find(
|
||||
el => el.dataset.switch === _selectedSwitch && parseInt(el.dataset.portIdx, 10) === _selectedIdx
|
||||
);
|
||||
if (block) {
|
||||
block.classList.add('selected');
|
||||
renderPanel(_selectedSwitch, _selectedIdx);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ── Fetch and render ─────────────────────────────────────────────────────
|
||||
async function loadInspector() {
|
||||
try {
|
||||
const data = await lt.api.get('/api/links');
|
||||
renderInspector(data);
|
||||
} catch (e) {
|
||||
document.getElementById('inspector-main').innerHTML =
|
||||
'<p class="empty-state">Failed to load inspector data.</p>';
|
||||
lt.toast.error('Failed to load inspector data');
|
||||
}
|
||||
}
|
||||
|
||||
loadInspector();
|
||||
var _inspInterval = (window.gandalfSettings && window.gandalfSettings.refreshInterval) || 60;
|
||||
if (_inspInterval > 0) lt.autoRefresh.start(loadInspector, Math.max(_inspInterval, 15) * 1000);
|
||||
|
||||
window.onGandalfSettingsChanged = function(s) {
|
||||
lt.autoRefresh.stop();
|
||||
if (s.refreshInterval > 0) lt.autoRefresh.start(loadInspector, Math.max(s.refreshInterval, 15) * 1000);
|
||||
};
|
||||
lt.keys.on('Escape', () => {
|
||||
if (document.getElementById('inspector-panel').classList.contains('open')) closePanel();
|
||||
});
|
||||
|
||||
document.addEventListener('click', e => {
|
||||
const portBlock = e.target.closest('[data-action="select-port"]');
|
||||
if (portBlock) { selectPort(portBlock); return; }
|
||||
|
||||
const closeBtn = e.target.closest('[data-action="close-panel"]');
|
||||
if (closeBtn) { closePanel(); return; }
|
||||
|
||||
const diagBtn = e.target.closest('[data-action="run-diagnostic"]');
|
||||
if (diagBtn) { runDiagnostic(diagBtn.dataset.sw, parseInt(diagBtn.dataset.idx, 10)); return; }
|
||||
|
||||
const toggleDiag = e.target.closest('[data-action="toggle-diag"]');
|
||||
if (toggleDiag) { toggleDiag.parentElement.classList.toggle('diag-open'); return; }
|
||||
});
|
||||
|
||||
// ── Link Diagnostics ─────────────────────────────────────────────────
|
||||
let _diagPollTimer = null;
|
||||
|
||||
function runDiagnostic(swName, portIdx) {
|
||||
const statusEl = document.getElementById('diag-status');
|
||||
const resultsEl = document.getElementById('diag-results');
|
||||
if (!statusEl || !resultsEl) return;
|
||||
|
||||
// Clear any previous poll
|
||||
if (_diagPollTimer) { clearInterval(_diagPollTimer); _diagPollTimer = null; }
|
||||
|
||||
statusEl.textContent = 'Submitting to Pulse...';
|
||||
resultsEl.innerHTML = '';
|
||||
|
||||
lt.api.post('/api/diagnose', {switch_name: swName, port_idx: portIdx})
|
||||
.then(resp => {
|
||||
statusEl.textContent = 'Collecting diagnostics via Pulse...';
|
||||
pollDiagnostic(resp.job_id, statusEl, resultsEl);
|
||||
})
|
||||
.catch(e => {
|
||||
statusEl.textContent = 'Error: ' + (e.message || 'Request failed');
|
||||
});
|
||||
}
|
||||
|
||||
function pollDiagnostic(jobId, statusEl, resultsEl) {
|
||||
let attempts = 0;
|
||||
_diagPollTimer = setInterval(() => {
|
||||
attempts++;
|
||||
if (attempts > 120) { // 2min timeout
|
||||
clearInterval(_diagPollTimer);
|
||||
statusEl.textContent = 'Timed out waiting for results.';
|
||||
return;
|
||||
}
|
||||
lt.api.get(`/api/diagnose/${jobId}`)
|
||||
.then(resp => {
|
||||
if (resp.status === 'done') {
|
||||
clearInterval(_diagPollTimer);
|
||||
_diagPollTimer = null;
|
||||
statusEl.textContent = '';
|
||||
renderDiagnosticResults(resp.result, resultsEl);
|
||||
}
|
||||
})
|
||||
.catch(() => {
|
||||
clearInterval(_diagPollTimer);
|
||||
_diagPollTimer = null;
|
||||
statusEl.textContent = 'Error: lost connection while collecting diagnostics.';
|
||||
});
|
||||
}, 2000);
|
||||
}
|
||||
|
||||
function renderDiagnosticResults(d, container) {
|
||||
if (!d || d.status === 'error') {
|
||||
container.innerHTML = `<div class="diag-error">Diagnostic error: ${escHtml((d && d.error) || 'unknown')}</div>`;
|
||||
return;
|
||||
}
|
||||
|
||||
const health = d.health || {};
|
||||
const issues = health.issues || [];
|
||||
const warns = health.warnings || [];
|
||||
const infoArr = health.info || [];
|
||||
const secs = d.sections || {};
|
||||
const eth = secs.ethtool || {};
|
||||
const drv = secs.ethtool_driver || {};
|
||||
const pause = secs.ethtool_pause || {};
|
||||
const ring = secs.ethtool_ring || {};
|
||||
const dom = secs.ethtool_dom || {};
|
||||
const sysfs = secs.sysfs_stats || {};
|
||||
const dmesg = secs.dmesg || [];
|
||||
const lldpctl = secs.lldpctl || {};
|
||||
const nicStats = secs.ethtool_stats || {};
|
||||
const swPort = d.switch_port || {};
|
||||
|
||||
// ── Health banner ──
|
||||
let bannerHtml = '';
|
||||
if (issues.length === 0 && warns.length === 0) {
|
||||
bannerHtml = '<div class="diag-health-banner"><span class="diag-health-ok">ALL OK</span></div>';
|
||||
} else {
|
||||
const parts = [];
|
||||
if (issues.length) parts.push(`<span class="diag-health-critical">${issues.length} CRITICAL</span>`);
|
||||
if (warns.length) parts.push(`<span class="diag-health-warning">${warns.length} WARNING</span>`);
|
||||
bannerHtml = `<div class="diag-health-banner">${parts.join(' ')}</div>`;
|
||||
}
|
||||
|
||||
const issueRows = [...issues, ...warns, ...infoArr].map(item => {
|
||||
const cls = issues.includes(item) ? 'diag-val-bad' : warns.includes(item) ? 'diag-val-warn' : 'diag-val-good';
|
||||
const label = issues.includes(item) ? 'CRIT' : warns.includes(item) ? 'WARN' : 'INFO';
|
||||
return `<div class="diag-issue-row"><span class="${cls}">[${label}]</span> <span class="diag-code">${escHtml(item.code)}</span> — ${escHtml(item.message)}</div>`;
|
||||
}).join('');
|
||||
|
||||
// ── Physical layer ──
|
||||
const carrierVal = secs.carrier === '1' ? '<span class="diag-val-good">YES</span>' :
|
||||
secs.carrier === '0' ? '<span class="diag-val-bad">NO</span>' : '–';
|
||||
const operstateVal = (secs.operstate || '?').toUpperCase();
|
||||
const opstateCls = secs.operstate === 'up' ? 'diag-val-good' : secs.operstate === 'down' ? 'diag-val-bad' : 'diag-val-warn';
|
||||
const speedVal = eth.speed_mbps ? `<span class="diag-val-good">${fmtSpeed(eth.speed_mbps)}bps</span>` : '<span class="diag-val-warn">–</span>';
|
||||
const duplexVal = eth.duplex === 'full' ? '<span class="diag-val-good">Full</span>' :
|
||||
eth.duplex === 'half' ? '<span class="diag-val-bad">Half</span>' : '–';
|
||||
const linkDetVal = eth.link_detected === true ? '<span class="diag-val-good">Yes</span>' :
|
||||
eth.link_detected === false ? '<span class="diag-val-bad">No</span>' : '–';
|
||||
const autonegVal = eth.auto_neg === true ? '<span class="diag-val-good">On</span>' :
|
||||
eth.auto_neg === false ? '<span class="diag-val-warn">Off</span>' : '–';
|
||||
|
||||
const physHtml = `
|
||||
<div class="diag-section">
|
||||
<div class="diag-section-header">Physical Layer</div>
|
||||
<table class="diag-table">
|
||||
<tr><td>Carrier</td><td>${carrierVal}</td></tr>
|
||||
<tr><td>Oper State</td><td><span class="${opstateCls}">${escHtml(operstateVal)}</span></td></tr>
|
||||
<tr><td>Speed</td><td>${speedVal}</td></tr>
|
||||
<tr><td>Duplex</td><td>${duplexVal}</td></tr>
|
||||
<tr><td>Link Detected</td><td>${linkDetVal}</td></tr>
|
||||
<tr><td>Auto-neg</td><td>${autonegVal}</td></tr>
|
||||
${secs.carrier_changes != null ? `<tr><td>Carrier Changes</td><td><span class="${secs.carrier_changes > 20 ? 'diag-val-warn' : 'diag-val-good'}">${secs.carrier_changes}</span></td></tr>` : ''}
|
||||
</table>
|
||||
</div>`;
|
||||
|
||||
// ── SFP / DOM ──
|
||||
let domHtml = '';
|
||||
if (dom && Object.keys(dom).length > 0) {
|
||||
const rxBar = dom.rx_power_dbm != null ? renderPowerBar(dom.rx_power_dbm, -18, -25) : '';
|
||||
const txBar = dom.tx_power_dbm != null ? renderPowerBar(dom.tx_power_dbm, -10, -13) : '';
|
||||
domHtml = `
|
||||
<div class="diag-section">
|
||||
<div class="diag-section-header">SFP / DOM</div>
|
||||
<table class="diag-table">
|
||||
${dom.vendor ? `<tr><td>Vendor</td><td>${escHtml(dom.vendor)}${dom.part_no ? ' / ' + escHtml(dom.part_no) : ''}</td></tr>` : ''}
|
||||
${dom.sfp_type ? `<tr><td>Type</td><td>${escHtml(dom.sfp_type)}</td></tr>` : ''}
|
||||
${dom.connector ? `<tr><td>Connector</td><td>${escHtml(dom.connector)}</td></tr>` : ''}
|
||||
${dom.wavelength_nm != null ? `<tr><td>Wavelength</td><td>${dom.wavelength_nm} nm</td></tr>` : ''}
|
||||
${dom.temp_c != null ? `<tr><td>Temperature</td><td>${dom.temp_c.toFixed(1)} °C</td></tr>` : ''}
|
||||
${dom.voltage_v != null ? `<tr><td>Voltage</td><td>${dom.voltage_v.toFixed(4)} V</td></tr>` : ''}
|
||||
${dom.bias_ma != null ? `<tr><td>Bias Current</td><td>${dom.bias_ma.toFixed(3)} mA</td></tr>` : ''}
|
||||
${dom.tx_power_dbm != null ? `<tr><td>TX Power</td><td>${dom.tx_power_dbm.toFixed(2)} dBm ${txBar}</td></tr>` : ''}
|
||||
${dom.rx_power_dbm != null ? `<tr><td>RX Power</td><td>${dom.rx_power_dbm.toFixed(2)} dBm ${rxBar}</td></tr>` : ''}
|
||||
</table>
|
||||
</div>`;
|
||||
}
|
||||
|
||||
// ── NIC Error Counters ──
|
||||
const errCounters = ['rx_crc_errors','rx_frame_errors','collisions','tx_carrier_errors','rx_missed_errors','rx_fifo_errors'];
|
||||
const nonZeroCounters = errCounters.filter(k => sysfs[k] > 0);
|
||||
let errCounterHtml = '';
|
||||
if (nonZeroCounters.length > 0 || secs.carrier_changes > 0) {
|
||||
const rows = nonZeroCounters.map(k => {
|
||||
const v = sysfs[k];
|
||||
const cls = v > 100 ? 'diag-val-bad' : 'diag-val-warn';
|
||||
return `<tr><td>${escHtml(k)}</td><td class="${cls}">${v.toLocaleString()}</td></tr>`;
|
||||
}).join('');
|
||||
errCounterHtml = `
|
||||
<div class="diag-section">
|
||||
<div class="diag-section-header">NIC Error Counters</div>
|
||||
<table class="diag-table">
|
||||
${rows || '<tr><td colspan="2" class="diag-val-good">All zero</td></tr>'}
|
||||
</table>
|
||||
</div>`;
|
||||
}
|
||||
|
||||
// ── ethtool -S (collapsible) ──
|
||||
let nicStatHtml = '';
|
||||
if (Object.keys(nicStats).length > 0) {
|
||||
const _ERR_KEYS = /err|drop|miss|crc|frame|fifo|abort|carrier|collision|fault|discard|overflow|reset/i;
|
||||
const rows = Object.entries(nicStats).map(([k, v]) => {
|
||||
const cls = _ERR_KEYS.test(k) && v > 0 ? ' class="diag-stat-nonzero-warn"' : '';
|
||||
return `<tr${cls}><td>${escHtml(k)}</td><td>${v.toLocaleString()}</td></tr>`;
|
||||
}).join('');
|
||||
nicStatHtml = `
|
||||
<div class="diag-section diag-collapsible">
|
||||
<div class="diag-section-header diag-toggle" data-action="toggle-diag">
|
||||
ethtool -S (NIC stats) <span class="diag-toggle-hint">[expand]</span>
|
||||
</div>
|
||||
<div class="diag-section-body">
|
||||
<table class="diag-stat-table">${rows}</table>
|
||||
</div>
|
||||
</div>`;
|
||||
}
|
||||
|
||||
// ── Flow Control + Ring Buffers ──
|
||||
let flowRingHtml = '';
|
||||
const hasPause = Object.keys(pause).length > 0;
|
||||
const hasRing = Object.keys(ring).length > 0;
|
||||
if (hasPause || hasRing) {
|
||||
flowRingHtml = `
|
||||
<div class="diag-section">
|
||||
<div class="diag-section-header">Flow Control & Ring Buffers</div>
|
||||
<table class="diag-table">
|
||||
${hasPause ? `
|
||||
<tr><td>RX Pause</td><td>${pause.rx_pause ? '<span class="diag-val-good">On</span>' : 'Off'}</td></tr>
|
||||
<tr><td>TX Pause</td><td>${pause.tx_pause ? '<span class="diag-val-good">On</span>' : 'Off'}</td></tr>` : ''}
|
||||
${hasRing ? `
|
||||
<tr><td>RX Ring</td><td>${ring.rx_current != null ? ring.rx_current : '–'} / ${ring.rx_max != null ? ring.rx_max : '–'} max</td></tr>
|
||||
<tr><td>TX Ring</td><td>${ring.tx_current != null ? ring.tx_current : '–'} / ${ring.tx_max != null ? ring.tx_max : '–'} max</td></tr>` : ''}
|
||||
</table>
|
||||
</div>`;
|
||||
}
|
||||
|
||||
// ── Driver Info ──
|
||||
let drvHtml = '';
|
||||
if (Object.keys(drv).length > 0) {
|
||||
drvHtml = `
|
||||
<div class="diag-section">
|
||||
<div class="diag-section-header">Driver Info</div>
|
||||
<table class="diag-table">
|
||||
${drv.driver ? `<tr><td>Driver</td><td>${escHtml(drv.driver)}</td></tr>` : ''}
|
||||
${drv.version ? `<tr><td>Version</td><td>${escHtml(drv.version)}</td></tr>` : ''}
|
||||
${drv.firmware_version ? `<tr><td>Firmware</td><td>${escHtml(drv.firmware_version)}</td></tr>` : ''}
|
||||
${drv.bus_info ? `<tr><td>Bus</td><td>${escHtml(drv.bus_info)}</td></tr>` : ''}
|
||||
</table>
|
||||
</div>`;
|
||||
}
|
||||
|
||||
// ── LLDP Validation ──
|
||||
let lldpValHtml = '';
|
||||
const swLldp = swPort.lldp || {};
|
||||
lldpValHtml = `
|
||||
<div class="diag-section">
|
||||
<div class="diag-section-header">LLDP Validation</div>
|
||||
<div class="path-debug-cols">
|
||||
<div class="path-col">
|
||||
<div class="path-col-header">Switch sees</div>
|
||||
<div class="path-row"><span>System</span><span>${escHtml(swLldp.system_name || '–')}</span></div>
|
||||
<div class="path-row"><span>Port</span><span>${escHtml(swLldp.port_id || '–')}</span></div>
|
||||
<div class="path-row"><span>Chassis</span><span>${escHtml(swLldp.chassis_id || '–')}</span></div>
|
||||
</div>
|
||||
<div class="path-col">
|
||||
<div class="path-col-header">Server lldpctl</div>
|
||||
${lldpctl.available
|
||||
? `<div class="path-row"><span>Neighbor</span><span>${escHtml(lldpctl.neighbor_system || '–')}</span></div>
|
||||
<div class="path-row"><span>Port</span><span>${escHtml(lldpctl.neighbor_port || '–')}</span></div>`
|
||||
: '<div class="path-row"><span class="diag-val-warn">lldpd not running</span></div>'}
|
||||
</div>
|
||||
</div>
|
||||
</div>`;
|
||||
|
||||
// ── dmesg ──
|
||||
let dmesgHtml = '';
|
||||
if (dmesg.length > 0) {
|
||||
const dlines = dmesg.map(e => {
|
||||
const cls = e.severity === 'error' ? ' diag-dmesg-err' : e.severity === 'warn' ? ' diag-dmesg-warn' : '';
|
||||
const ts = e.timestamp ? `[${e.timestamp}] ` : '';
|
||||
return `<div class="diag-dmesg-line${cls}">${escHtml(ts + e.msg)}</div>`;
|
||||
}).join('');
|
||||
dmesgHtml = `
|
||||
<div class="diag-section diag-collapsible">
|
||||
<div class="diag-section-header diag-toggle" data-action="toggle-diag">
|
||||
Kernel Events (dmesg) <span class="diag-toggle-hint">[expand]</span>
|
||||
</div>
|
||||
<div class="diag-section-body">
|
||||
<div class="diag-dmesg-wrap">${dlines}</div>
|
||||
</div>
|
||||
</div>`;
|
||||
}
|
||||
|
||||
// ── Switch Port Summary ──
|
||||
const swSummaryHtml = `
|
||||
<div class="diag-section">
|
||||
<div class="diag-section-header">Switch Port Summary</div>
|
||||
<table class="diag-table">
|
||||
<tr><td>Status</td><td>${swPort.up ? '<span class="diag-val-good">UP</span>' : '<span class="diag-val-bad">DOWN</span>'}</td></tr>
|
||||
<tr><td>Speed</td><td>${swPort.speed_mbps ? fmtSpeed(swPort.speed_mbps) + 'bps' : '–'}</td></tr>
|
||||
<tr><td>Duplex</td><td>${swPort.full_duplex ? 'Full' : (swPort.up ? '<span class="diag-val-bad">Half</span>' : '–')}</td></tr>
|
||||
<tr><td>TX Err</td><td>${fmtErrors(swPort.tx_errs_rate)}</td></tr>
|
||||
<tr><td>RX Err</td><td>${fmtErrors(swPort.rx_errs_rate)}</td></tr>
|
||||
${swPort.poe_power != null ? `<tr><td>PoE</td><td><span class="val-amber">${swPort.poe_power.toFixed(1)}W</span></td></tr>` : ''}
|
||||
</table>
|
||||
</div>`;
|
||||
|
||||
// ── Pulse link ──
|
||||
const pulseLink = d.pulse_url
|
||||
? `<div class="diag-pulse-link"><a href="${escHtml(d.pulse_url)}" target="_blank" rel="noopener">View raw output in Pulse ↗</a></div>`
|
||||
: '';
|
||||
|
||||
container.innerHTML = `
|
||||
<div class="diag-results-inner">
|
||||
${bannerHtml}
|
||||
<div class="diag-issue-list">${issueRows}</div>
|
||||
${physHtml}
|
||||
${domHtml}
|
||||
${errCounterHtml}
|
||||
${nicStatHtml}
|
||||
${flowRingHtml}
|
||||
${drvHtml}
|
||||
${lldpValHtml}
|
||||
${dmesgHtml}
|
||||
${swSummaryHtml}
|
||||
${pulseLink}
|
||||
</div>`;
|
||||
}
|
||||
|
||||
// SFP power bar: range is 0 dBm (best) to -35 dBm (worst)
|
||||
function renderPowerBar(dbm, warnThreshold, critThreshold) {
|
||||
const minDbm = -35, maxDbm = 0;
|
||||
const pct = Math.max(0, Math.min(100, ((dbm - minDbm) / (maxDbm - minDbm)) * 100));
|
||||
const warnPct = ((warnThreshold - minDbm) / (maxDbm - minDbm)) * 100;
|
||||
const critPct = ((critThreshold - minDbm) / (maxDbm - minDbm)) * 100;
|
||||
const barCls = dbm < critThreshold ? 'diag-val-bad' : dbm < warnThreshold ? 'diag-val-warn' : 'diag-val-good';
|
||||
return `<span class="diag-power-bar-wrap">
|
||||
<span class="diag-power-bar ${barCls}" style="width:${pct.toFixed(1)}%"></span>
|
||||
<span class="diag-power-zone-warn" style="left:${warnPct.toFixed(1)}%"></span>
|
||||
<span class="diag-power-zone-crit" style="left:${critPct.toFixed(1)}%"></span>
|
||||
</span>`;
|
||||
}
|
||||
</script>
|
||||
{% endblock %}
|
||||
@@ -0,0 +1,556 @@
|
||||
{% extends "base.html" %}
|
||||
{% block title %}Link Debug – GANDALF{% endblock %}
|
||||
|
||||
{% block content %}
|
||||
|
||||
<div class="lt-page-header">
|
||||
<div>
|
||||
<h1 class="lt-page-title">Link Debug</h1>
|
||||
<p class="g-page-sub" style="margin-top:4px">
|
||||
Per-interface stats: speed, duplex, SFP optical levels, TX/RX rates, errors, and carrier changes.
|
||||
<span id="links-updated" style="margin-left:8px"></span>
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div id="links-container">
|
||||
<div class="link-loading">Loading link statistics</div>
|
||||
</div>
|
||||
|
||||
{% endblock %}
|
||||
|
||||
{% block scripts %}
|
||||
<script>
|
||||
const escHtml = s => lt.escHtml(s);
|
||||
|
||||
// ── Formatting helpers ────────────────────────────────────────────
|
||||
function fmtRate(bytesPerSec) {
|
||||
if (bytesPerSec === null || bytesPerSec === undefined) return '–';
|
||||
const bps = bytesPerSec * 8;
|
||||
if (bps < 1e3) return bps.toFixed(0) + ' bps';
|
||||
if (bps < 1e6) return (bps/1e3).toFixed(1) + ' Kbps';
|
||||
if (bps < 1e9) return (bps/1e6).toFixed(2) + ' Mbps';
|
||||
return (bps/1e9).toFixed(3) + ' Gbps';
|
||||
}
|
||||
|
||||
function fmtRateBar(bytesPerSec, linkSpeedMbps) {
|
||||
if (!linkSpeedMbps || linkSpeedMbps <= 0) return 0;
|
||||
const mbps = (bytesPerSec * 8) / 1e6;
|
||||
return Math.min(100, (mbps / linkSpeedMbps) * 100);
|
||||
}
|
||||
|
||||
function trafficBarClass(pct, isTx) {
|
||||
if (pct > 85) return 'lt-progress--red';
|
||||
if (pct > 65) return 'lt-progress--amber';
|
||||
return isTx ? '' : 'lt-progress--cyan';
|
||||
}
|
||||
|
||||
function fmtSpeed(mbps) {
|
||||
if (mbps === null || mbps === undefined) return '–';
|
||||
if (mbps >= 1000) return (mbps/1000).toFixed(0) + ' Gbps';
|
||||
return mbps + ' Mbps';
|
||||
}
|
||||
|
||||
function fmtDuplex(d) {
|
||||
if (!d) return '–';
|
||||
return d.charAt(0).toUpperCase() + d.slice(1);
|
||||
}
|
||||
|
||||
function fmtTemp(c) {
|
||||
if (c === null || c === undefined) return '–';
|
||||
return c.toFixed(1) + ' °C';
|
||||
}
|
||||
|
||||
function fmtVoltage(v) {
|
||||
if (v === null || v === undefined) return '–';
|
||||
return v.toFixed(2) + ' V';
|
||||
}
|
||||
|
||||
function fmtPower(dbm) {
|
||||
if (dbm === null || dbm === undefined) return '–';
|
||||
return dbm.toFixed(2) + ' dBm';
|
||||
}
|
||||
|
||||
function fmtBias(ma) {
|
||||
if (ma === null || ma === undefined) return '–';
|
||||
return ma.toFixed(2) + ' mA';
|
||||
}
|
||||
|
||||
function fmtErrors(rate) {
|
||||
if (rate === null || rate === undefined) return '–';
|
||||
if (rate < 0.001) return '<span class="val-good">0 /s</span>';
|
||||
return `<span class="val-crit">${rate.toFixed(3)} /s</span>`;
|
||||
}
|
||||
|
||||
function fmtCarrier(n) {
|
||||
if (n === null || n === undefined) return '–';
|
||||
if (n === 0) return '<span class="counter-zero">0</span>';
|
||||
return `<span class="counter-nonzero">${n}</span>`;
|
||||
}
|
||||
|
||||
// ── SFP/DOM value classification ─────────────────────────────────
|
||||
function rxPowerClass(dbm) {
|
||||
if (dbm === null || dbm === undefined) return 'val-neutral';
|
||||
if (dbm < -15) return 'val-crit';
|
||||
if (dbm < -10) return 'val-warn';
|
||||
return 'val-good';
|
||||
}
|
||||
function txPowerClass(dbm) {
|
||||
if (dbm === null || dbm === undefined) return 'val-neutral';
|
||||
if (dbm < -5) return 'val-crit';
|
||||
return 'val-good';
|
||||
}
|
||||
function tempClass(c) {
|
||||
if (c === null || c === undefined) return 'val-neutral';
|
||||
if (c > 80) return 'val-crit';
|
||||
if (c > 70) return 'val-warn';
|
||||
return 'val-good';
|
||||
}
|
||||
function voltageClass(v) {
|
||||
if (v === null || v === undefined) return 'val-neutral';
|
||||
if (v < 3.0 || v > 3.6) return 'val-crit';
|
||||
if (v < 3.1 || v > 3.5) return 'val-warn';
|
||||
return 'val-good';
|
||||
}
|
||||
|
||||
function errorBadges(d) {
|
||||
const badges = [];
|
||||
if ((d.tx_errs_rate || 0) > 0.001 || (d.rx_errs_rate || 0) > 0.001)
|
||||
badges.push('<span class="link-alert-badge">ERR</span>');
|
||||
if ((d.tx_drops_rate || 0) > 0.001 || (d.rx_drops_rate || 0) > 0.001)
|
||||
badges.push('<span class="link-alert-badge link-alert-amber">DROP</span>');
|
||||
if ((d.carrier_changes || 0) > 3)
|
||||
badges.push('<span class="link-alert-badge link-alert-amber">FLAP</span>');
|
||||
return badges.join('');
|
||||
}
|
||||
|
||||
// ── Render a single server interface card ─────────────────────────
|
||||
function renderIfaceCard(ifaceName, d) {
|
||||
const isDown = d.link_detected === false;
|
||||
const pt = (d.port_type || '').toUpperCase();
|
||||
const mediaTag = pt === 'FIBRE' || pt === 'SFP' || pt.includes('FIBRE') ? 'type-fibre'
|
||||
: pt === 'DA' ? 'type-da'
|
||||
: 'type-copper';
|
||||
const mediaLabel = d.port_type || '–';
|
||||
const speedStr = d.speed_mbps ? fmtSpeed(d.speed_mbps) : '–';
|
||||
const txPct = fmtRateBar(d.tx_bytes_rate, d.speed_mbps);
|
||||
const rxPct = fmtRateBar(d.rx_bytes_rate, d.speed_mbps);
|
||||
|
||||
let sfpHtml = '';
|
||||
if (d.sfp && Object.keys(d.sfp).length > 0) {
|
||||
const s = d.sfp;
|
||||
const rxClass = rxPowerClass(s.rx_power_dbm);
|
||||
const txClass = txPowerClass(s.tx_power_dbm);
|
||||
const tmpClass = tempClass(s.temp_c);
|
||||
const vClass = voltageClass(s.voltage_v);
|
||||
const rxPct2 = s.rx_power_dbm != null ? Math.min(100, Math.max(0, (s.rx_power_dbm + 20) / 15 * 100)) : 0;
|
||||
const txPct2 = s.tx_power_dbm != null ? Math.min(100, Math.max(0, (s.tx_power_dbm + 10) / 8 * 100)) : 0;
|
||||
|
||||
sfpHtml = `
|
||||
<div class="sfp-panel">
|
||||
<div class="sfp-vendor-row">
|
||||
${s.vendor ? `<span>${escHtml(s.vendor)}</span>` : ''}
|
||||
${s.part_no ? ` / <span>${escHtml(s.part_no)}</span>` : ''}
|
||||
</div>
|
||||
<div class="sfp-grid">
|
||||
<div class="sfp-stat">
|
||||
<span class="sfp-stat-label" data-tooltip="SFP module temperature. Normal: below 70°C. Warn: 70–85°C. Critical: above 85°C.">Temp</span>
|
||||
<span class="sfp-stat-value ${tmpClass}">${fmtTemp(s.temp_c)}</span>
|
||||
</div>
|
||||
<div class="sfp-stat">
|
||||
<span class="sfp-stat-label" data-tooltip="SFP supply voltage. Normal: 3.1–3.5V.">Voltage</span>
|
||||
<span class="sfp-stat-value ${vClass}">${fmtVoltage(s.voltage_v)}</span>
|
||||
</div>
|
||||
<div class="sfp-stat">
|
||||
<span class="sfp-stat-label" data-tooltip="Laser bias current in mA. High values may indicate end-of-life laser diode.">Bias</span>
|
||||
<span class="sfp-stat-value">${fmtBias(s.bias_ma)}</span>
|
||||
</div>
|
||||
<div class="sfp-stat">
|
||||
<span class="sfp-stat-label" data-tooltip="Optical transmit power in dBm. Typical good range: -3 to -9 dBm.">TX Power</span>
|
||||
<span class="sfp-stat-value ${txClass}">${fmtPower(s.tx_power_dbm)}</span>
|
||||
<div class="power-row">
|
||||
<div class="power-track"><div class="power-fill ${txClass === 'val-good' ? 'power-ok' : txClass === 'val-warn' ? 'power-warn' : 'power-crit'}" style="width:${txPct2}%"></div></div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="sfp-stat">
|
||||
<span class="sfp-stat-label" data-tooltip="Optical receive power in dBm. Typical good range: -3 to -18 dBm. Below -20 dBm may indicate dirty/damaged fiber.">RX Power</span>
|
||||
<span class="sfp-stat-value ${rxClass}">${fmtPower(s.rx_power_dbm)}</span>
|
||||
<div class="power-row">
|
||||
<div class="power-track"><div class="power-fill ${rxClass === 'val-good' ? 'power-ok' : rxClass === 'val-warn' ? 'power-warn' : 'power-crit'}" style="width:${rxPct2}%"></div></div>
|
||||
</div>
|
||||
</div>
|
||||
${s.rx_power_dbm != null && s.tx_power_dbm != null ? `
|
||||
<div class="sfp-stat">
|
||||
<span class="sfp-stat-label" data-tooltip="Insertion loss: difference between transmit and receive power. Large negative values indicate fiber loss or connector issues.">RX−TX Δ</span>
|
||||
<span class="sfp-stat-value">${(s.rx_power_dbm - s.tx_power_dbm).toFixed(2)} dBm</span>
|
||||
</div>` : ''}
|
||||
</div>
|
||||
</div>`;
|
||||
}
|
||||
|
||||
return `
|
||||
<div class="link-iface-card ${isDown ? 'port-down' : ''}">
|
||||
<div class="link-iface-header">
|
||||
<span class="link-iface-name">${escHtml(ifaceName)}</span>
|
||||
<span class="link-iface-speed">${speedStr}</span>
|
||||
<span class="link-iface-type ${mediaTag}">${escHtml(mediaLabel)}</span>
|
||||
${errorBadges(d)}
|
||||
</div>
|
||||
<div class="link-stats-grid">
|
||||
<div class="link-stat">
|
||||
<span class="link-stat-label">Link</span>
|
||||
<span class="link-stat-value ${isDown ? 'val-crit' : 'val-good'}">${isDown ? 'DOWN' : 'UP'}</span>
|
||||
</div>
|
||||
<div class="link-stat">
|
||||
<span class="link-stat-label" data-tooltip="Full = simultaneous send/receive at full speed. Half = one direction at a time, can cause collisions.">Duplex</span>
|
||||
<span class="link-stat-value">${fmtDuplex(d.duplex)}</span>
|
||||
</div>
|
||||
<div class="link-stat">
|
||||
<span class="link-stat-label" data-tooltip="Autonegotiation: NIC and switch automatically agree on link speed and duplex mode.">Auto-neg</span>
|
||||
<span class="link-stat-value">${d.auto_neg == null ? '–' : d.auto_neg ? 'On' : 'Off'}</span>
|
||||
</div>
|
||||
<div class="link-stat">
|
||||
<span class="link-stat-label" data-tooltip="Carrier changes: number of times the link went up or down. High values indicate a flapping or unstable cable/SFP.">Carrier Δ</span>
|
||||
<span class="link-stat-value">${fmtCarrier(d.carrier_changes)}</span>
|
||||
</div>
|
||||
<div class="link-stat">
|
||||
<span class="link-stat-label" data-tooltip="Transmit errors per second reported by the kernel network driver.">TX Err/s</span>
|
||||
<span class="link-stat-value">${fmtErrors(d.tx_errs_rate)}</span>
|
||||
</div>
|
||||
<div class="link-stat">
|
||||
<span class="link-stat-label" data-tooltip="Receive errors per second reported by the kernel network driver.">RX Err/s</span>
|
||||
<span class="link-stat-value">${fmtErrors(d.rx_errs_rate)}</span>
|
||||
</div>
|
||||
<div class="link-stat">
|
||||
<span class="link-stat-label" data-tooltip="Transmit packets dropped per second (ring buffer full or driver overrun).">TX Drop/s</span>
|
||||
<span class="link-stat-value">${fmtErrors(d.tx_drops_rate)}</span>
|
||||
</div>
|
||||
<div class="link-stat">
|
||||
<span class="link-stat-label" data-tooltip="Receive packets dropped per second (ring buffer full or driver overrun).">RX Drop/s</span>
|
||||
<span class="link-stat-value">${fmtErrors(d.rx_drops_rate)}</span>
|
||||
</div>
|
||||
</div>
|
||||
<div class="traffic-section">
|
||||
<div class="traffic-row">
|
||||
<span class="traffic-label" data-tooltip="Transmit — outgoing traffic from this server">TX</span>
|
||||
<div class="lt-progress ${trafficBarClass(txPct, true)}"><div class="lt-progress-bar" style="width:${txPct}%"></div></div>
|
||||
<span class="traffic-value">${fmtRate(d.tx_bytes_rate)}</span>
|
||||
</div>
|
||||
<div class="traffic-row">
|
||||
<span class="traffic-label" data-tooltip="Receive — incoming traffic to this server">RX</span>
|
||||
<div class="lt-progress ${trafficBarClass(rxPct, false)}"><div class="lt-progress-bar" style="width:${rxPct}%"></div></div>
|
||||
<span class="traffic-value">${fmtRate(d.rx_bytes_rate)}</span>
|
||||
</div>
|
||||
</div>
|
||||
${sfpHtml}
|
||||
</div>`;
|
||||
}
|
||||
|
||||
// ── Render a single UniFi switch port card ────────────────────────
|
||||
function renderPortCard(portName, d) {
|
||||
const isDown = !d.up;
|
||||
const speedStr = d.speed_mbps ? fmtSpeed(d.speed_mbps) : '–';
|
||||
const txPct = fmtRateBar(d.tx_bytes_rate, d.speed_mbps);
|
||||
const rxPct = fmtRateBar(d.rx_bytes_rate, d.speed_mbps);
|
||||
const numBadge = d.port_idx != null ? `<span class="port-badge port-badge-num">#${d.port_idx}</span>` : '';
|
||||
const uplinkBadge = d.is_uplink ? `<span class="port-badge port-badge-uplink">UPLINK</span>` : '';
|
||||
const poeBadge = d.poe_power ? `<span class="port-badge port-badge-poe">PoE ${d.poe_power.toFixed(1)}W</span>` : '';
|
||||
const lldpLine = d.lldp ? `<div class="port-lldp">→ ${escHtml(d.lldp.system_name || '')} (${escHtml(d.lldp.port_id || '')})</div>` : '';
|
||||
const poeLine = d.poe_class ? `<div class="port-poe-info">PoE ${escHtml(d.poe_class)} · max ${d.poe_max_power != null ? d.poe_max_power.toFixed(1)+'W' : '–'}</div>` : '';
|
||||
|
||||
return `
|
||||
<div class="link-iface-card ${isDown ? 'port-down' : ''}">
|
||||
<div class="link-iface-header">
|
||||
<span class="link-iface-name">${numBadge} ${escHtml(portName)}</span>
|
||||
<span class="link-iface-speed">${speedStr}</span>
|
||||
${uplinkBadge}${poeBadge}
|
||||
${errorBadges(d)}
|
||||
</div>
|
||||
${lldpLine}${poeLine}
|
||||
<div class="link-stats-grid">
|
||||
<div class="link-stat">
|
||||
<span class="link-stat-label">Link</span>
|
||||
<span class="link-stat-value ${isDown ? 'val-crit' : 'val-good'}">${isDown ? 'DOWN' : 'UP'}</span>
|
||||
</div>
|
||||
<div class="link-stat">
|
||||
<span class="link-stat-label">Duplex</span>
|
||||
<span class="link-stat-value">${d.full_duplex == null ? '–' : d.full_duplex ? 'Full' : 'Half'}</span>
|
||||
</div>
|
||||
<div class="link-stat">
|
||||
<span class="link-stat-label">Auto-neg</span>
|
||||
<span class="link-stat-value">${d.autoneg == null ? '–' : d.autoneg ? 'On' : 'Off'}</span>
|
||||
</div>
|
||||
<div class="link-stat">
|
||||
<span class="link-stat-label">TX Err/s</span>
|
||||
<span class="link-stat-value">${fmtErrors(d.tx_errs_rate)}</span>
|
||||
</div>
|
||||
<div class="link-stat">
|
||||
<span class="link-stat-label">RX Err/s</span>
|
||||
<span class="link-stat-value">${fmtErrors(d.rx_errs_rate)}</span>
|
||||
</div>
|
||||
<div class="link-stat">
|
||||
<span class="link-stat-label">TX Drop/s</span>
|
||||
<span class="link-stat-value">${fmtErrors(d.tx_drops_rate)}</span>
|
||||
</div>
|
||||
</div>
|
||||
<div class="traffic-section">
|
||||
<div class="traffic-row">
|
||||
<span class="traffic-label">TX</span>
|
||||
<div class="lt-progress ${trafficBarClass(txPct, true)}"><div class="lt-progress-bar" style="width:${txPct}%"></div></div>
|
||||
<span class="traffic-value">${fmtRate(d.tx_bytes_rate)}</span>
|
||||
</div>
|
||||
<div class="traffic-row">
|
||||
<span class="traffic-label">RX</span>
|
||||
<div class="lt-progress ${trafficBarClass(rxPct, false)}"><div class="lt-progress-bar" style="width:${rxPct}%"></div></div>
|
||||
<span class="traffic-value">${fmtRate(d.rx_bytes_rate)}</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>`;
|
||||
}
|
||||
|
||||
// ── Render all UniFi switches ─────────────────────────────────────
|
||||
function renderUnifiSwitches(unifiSwitches, dataUpdated) {
|
||||
if (!unifiSwitches || !Object.keys(unifiSwitches).length) return '';
|
||||
const updStr = dataUpdated
|
||||
? new Date(dataUpdated.replace(' UTC', 'Z').replace(' ', 'T')).toLocaleTimeString()
|
||||
: '';
|
||||
const html = Object.entries(unifiSwitches).map(([swName, sw]) => {
|
||||
const ports = sw.ports || {};
|
||||
const portValues = Object.values(ports);
|
||||
const portCards = Object.entries(ports)
|
||||
.sort(([,a],[,b]) => (a.port_idx||0) - (b.port_idx||0))
|
||||
.map(([pname, d]) => renderPortCard(pname, d)).join('');
|
||||
const poe_total_w = portValues.reduce((s, p) => s + (p.poe_power || 0), 0);
|
||||
const poe_max_w = portValues.reduce((s, p) => s + (p.poe_max_power || 0), 0);
|
||||
const poeLoad = poe_total_w > 0 ? ` · PoE ${poe_total_w.toFixed(1)}W` : '';
|
||||
|
||||
// PoE utilisation bar
|
||||
let poebar = '';
|
||||
if (poe_total_w > 0 && poe_max_w > 0) {
|
||||
const pct = Math.min(100, (poe_total_w / poe_max_w) * 100);
|
||||
const cls = pct > 80 ? 'poe-bar-crit' : pct > 60 ? 'poe-bar-warn' : 'poe-bar-ok';
|
||||
poebar = `<div class="poe-bar-track"><div class="poe-bar-fill ${cls}" style="width:${pct}%"></div></div>`;
|
||||
}
|
||||
|
||||
return `
|
||||
<div class="link-host-panel" id="panel-${CSS.escape(swName)}">
|
||||
<div class="link-host-title" data-action="toggle-panel">
|
||||
<span class="link-host-name">${escHtml(swName)}</span>
|
||||
<span class="link-host-ip">${escHtml(sw.ip || '')}</span>
|
||||
<span class="link-host-upd">${escHtml(sw.model || '')}${updStr ? ' · ' + updStr : ''}${poeLoad}</span>
|
||||
${poebar}
|
||||
<span class="panel-toggle">[–]</span>
|
||||
</div>
|
||||
<div class="link-ifaces-grid">${portCards}</div>
|
||||
</div>`;
|
||||
}).join('');
|
||||
|
||||
return `<div class="unifi-section-header">UNIFI SWITCH PORTS</div>${html}`;
|
||||
}
|
||||
|
||||
// ── Panel collapse / expand ───────────────────────────────────────
|
||||
function togglePanel(panel) {
|
||||
panel.classList.toggle('collapsed');
|
||||
const btn = panel.querySelector('.panel-toggle');
|
||||
if (btn) btn.textContent = panel.classList.contains('collapsed') ? '[+]' : '[–]';
|
||||
const id = panel.id;
|
||||
if (id) {
|
||||
const collapsed = JSON.parse(sessionStorage.getItem('linksCollapsed') || '{}');
|
||||
collapsed[id] = panel.classList.contains('collapsed');
|
||||
sessionStorage.setItem('linksCollapsed', JSON.stringify(collapsed));
|
||||
}
|
||||
}
|
||||
|
||||
function restoreCollapseState() {
|
||||
const collapsed = JSON.parse(sessionStorage.getItem('linksCollapsed') || '{}');
|
||||
for (const [id, isCollapsed] of Object.entries(collapsed)) {
|
||||
const panel = document.getElementById(id);
|
||||
if (!panel) continue;
|
||||
if (isCollapsed) {
|
||||
panel.classList.add('collapsed');
|
||||
const btn = panel.querySelector('.panel-toggle');
|
||||
if (btn) btn.textContent = '[+]';
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ── Build summary stats header ────────────────────────────────────
|
||||
function buildLinkSummary(hosts, unifiSwitches) {
|
||||
let totalIfaces = 0, downIfaces = 0, errIfaces = 0, totalPoe = 0;
|
||||
for (const ifaces of Object.values(hosts || {})) {
|
||||
for (const d of Object.values(ifaces)) {
|
||||
totalIfaces++;
|
||||
if (d.link_detected === false) downIfaces++;
|
||||
if ((d.tx_errs_rate || 0) > 0 || (d.rx_errs_rate || 0) > 0) errIfaces++;
|
||||
}
|
||||
}
|
||||
let swTotal = 0, swDown = 0;
|
||||
for (const sw of Object.values(unifiSwitches || {})) {
|
||||
for (const p of Object.values(sw.ports || {})) {
|
||||
totalPoe += p.poe_power || 0;
|
||||
swTotal++;
|
||||
if (!p.up) swDown++;
|
||||
}
|
||||
}
|
||||
const allTotal = totalIfaces + swTotal;
|
||||
const allDown = downIfaces + swDown;
|
||||
const downColor = allDown > 0 ? 'var(--red)' : 'var(--green)';
|
||||
const errColor = errIfaces > 0 ? 'var(--amber)' : 'var(--green)';
|
||||
const downCardCls = allDown > 0 ? ' lt-stat-card--alert' : '';
|
||||
const poeCard = totalPoe > 0 ? `
|
||||
<div class="lt-stat-card">
|
||||
<span class="lt-stat-icon" aria-hidden="true" style="color:var(--amber)">⚡</span>
|
||||
<div class="lt-stat-info">
|
||||
<span class="lt-stat-value" style="color:var(--amber)">${totalPoe.toFixed(1)}</span>
|
||||
<span class="lt-stat-label">PoE Load (W)</span>
|
||||
</div>
|
||||
</div>` : '';
|
||||
return `
|
||||
<div class="lt-stats-grid" style="margin-bottom:16px">
|
||||
<div class="lt-stat-card">
|
||||
<span class="lt-stat-icon" aria-hidden="true" style="color:var(--cyan)">⬡</span>
|
||||
<div class="lt-stat-info">
|
||||
<span class="lt-stat-value" style="color:var(--cyan)">${allTotal}</span>
|
||||
<span class="lt-stat-label">Interfaces</span>
|
||||
</div>
|
||||
</div>
|
||||
<div class="lt-stat-card${downCardCls}">
|
||||
<span class="lt-stat-icon" aria-hidden="true" style="color:${downColor}">●</span>
|
||||
<div class="lt-stat-info">
|
||||
<span class="lt-stat-value" style="color:${downColor}">${allDown}</span>
|
||||
<span class="lt-stat-label">Ports Down</span>
|
||||
</div>
|
||||
</div>
|
||||
<div class="lt-stat-card">
|
||||
<span class="lt-stat-icon" aria-hidden="true" style="color:${errColor}">▲</span>
|
||||
<div class="lt-stat-info">
|
||||
<span class="lt-stat-value" style="color:${errColor}">${errIfaces}</span>
|
||||
<span class="lt-stat-label">With Errors</span>
|
||||
</div>
|
||||
</div>
|
||||
${poeCard}
|
||||
</div>`;
|
||||
}
|
||||
|
||||
// ── Main render ───────────────────────────────────────────────────
|
||||
function renderLinks(data) {
|
||||
const hosts = data.hosts || {};
|
||||
const unifiSwitches = data.unifi_switches || {};
|
||||
const parts = [];
|
||||
|
||||
parts.push(buildLinkSummary(hosts, unifiSwitches));
|
||||
parts.push(`<div class="link-collapse-bar">
|
||||
<button class="lt-btn lt-btn-ghost lt-btn-sm" data-action="collapse-all">Collapse All</button>
|
||||
<button class="lt-btn lt-btn-ghost lt-btn-sm" data-action="expand-all">Expand All</button>
|
||||
</div>`);
|
||||
parts.push('<div class="link-host-list">');
|
||||
|
||||
for (const [hostname, ifaces] of Object.entries(hosts)) {
|
||||
const ifaceCards = Object.entries(ifaces)
|
||||
.sort(([a],[b]) => a.localeCompare(b))
|
||||
.map(([iname, d]) => renderIfaceCard(iname, d)).join('');
|
||||
const sample = Object.values(ifaces)[0] || {};
|
||||
const ip = sample.host_ip || '';
|
||||
const updStr = data.updated
|
||||
? new Date(data.updated.replace(' UTC', 'Z').replace(' ', 'T')).toLocaleTimeString()
|
||||
: '';
|
||||
|
||||
parts.push(`
|
||||
<div class="link-host-panel" id="panel-${CSS.escape(hostname)}">
|
||||
<div class="link-host-title" data-action="toggle-panel">
|
||||
<span class="link-host-name">${escHtml(hostname)}</span>
|
||||
<span class="link-host-ip">${escHtml(ip)}</span>
|
||||
<span class="link-host-upd">${updStr}</span>
|
||||
<span class="panel-toggle">[–]</span>
|
||||
</div>
|
||||
<div class="link-ifaces-grid">${ifaceCards}</div>
|
||||
</div>`);
|
||||
}
|
||||
|
||||
parts.push(renderUnifiSwitches(unifiSwitches, data.updated));
|
||||
parts.push('</div>');
|
||||
document.getElementById('links-container').innerHTML = parts.join('');
|
||||
restoreCollapseState();
|
||||
}
|
||||
|
||||
function collapseAll() {
|
||||
document.querySelectorAll('.link-host-panel').forEach(p => {
|
||||
p.classList.add('collapsed');
|
||||
const btn = p.querySelector('.panel-toggle');
|
||||
if (btn) btn.textContent = '[+]';
|
||||
});
|
||||
sessionStorage.setItem('linksCollapsed', JSON.stringify(
|
||||
Object.fromEntries([...document.querySelectorAll('.link-host-panel')].map(p => [p.id, true]))
|
||||
));
|
||||
}
|
||||
|
||||
function expandAll() {
|
||||
document.querySelectorAll('.link-host-panel').forEach(p => {
|
||||
p.classList.remove('collapsed');
|
||||
const btn = p.querySelector('.panel-toggle');
|
||||
if (btn) btn.textContent = '[–]';
|
||||
});
|
||||
sessionStorage.setItem('linksCollapsed', '{}');
|
||||
}
|
||||
|
||||
// ── Stale data warning ────────────────────────────────────────────
|
||||
function checkLinksStale(updatedStr) {
|
||||
if (!updatedStr) return;
|
||||
const age = (Date.now() - new Date(updatedStr + (updatedStr.includes('Z') ? '' : 'Z'))) / 1000;
|
||||
let banner = document.getElementById('links-stale-banner');
|
||||
if (age > 120) {
|
||||
if (!banner) {
|
||||
banner = document.createElement('div');
|
||||
banner.id = 'links-stale-banner';
|
||||
banner.className = 'lt-alert lt-alert--warning';
|
||||
banner.innerHTML = '<span class="lt-alert-icon">⚠</span><div class="lt-alert-body"><div class="lt-alert-msg"></div></div>';
|
||||
document.getElementById('links-container').prepend(banner);
|
||||
}
|
||||
banner.querySelector('.lt-alert-msg').textContent =
|
||||
`Link data may be stale — last updated ${Math.floor(age/60)}m ago.`;
|
||||
banner.style.display = '';
|
||||
} else if (banner) {
|
||||
banner.style.display = 'none';
|
||||
}
|
||||
}
|
||||
|
||||
// ── Fetch + render ────────────────────────────────────────────────
|
||||
async function loadLinks() {
|
||||
try {
|
||||
const data = await lt.api.get('/api/links');
|
||||
if (!data.hosts && !data.unifi_switches) {
|
||||
document.getElementById('links-container').innerHTML =
|
||||
'<div class="link-no-data">No link data yet — monitor has not completed a full cycle.</div>';
|
||||
return;
|
||||
}
|
||||
const updEl = document.getElementById('links-updated');
|
||||
if (updEl && data.updated) {
|
||||
updEl.textContent = 'Updated: ' + new Date(data.updated + (data.updated.includes('Z') ? '' : 'Z')).toLocaleTimeString();
|
||||
}
|
||||
renderLinks(data);
|
||||
checkLinksStale(data.updated);
|
||||
} catch (e) {
|
||||
document.getElementById('links-container').innerHTML =
|
||||
'<div class="error-state">Network error loading link statistics.</div>';
|
||||
lt.toast.error('Failed to load link statistics');
|
||||
}
|
||||
}
|
||||
|
||||
loadLinks();
|
||||
var _linksInterval = (window.gandalfSettings && window.gandalfSettings.refreshInterval) || 60;
|
||||
if (_linksInterval > 0) lt.autoRefresh.start(loadLinks, Math.max(_linksInterval, 15) * 1000);
|
||||
|
||||
window.onGandalfSettingsChanged = function(s) {
|
||||
lt.autoRefresh.stop();
|
||||
if (s.refreshInterval > 0) lt.autoRefresh.start(loadLinks, Math.max(s.refreshInterval, 15) * 1000);
|
||||
};
|
||||
|
||||
document.addEventListener('click', e => {
|
||||
const toggleTitle = e.target.closest('[data-action="toggle-panel"]');
|
||||
if (toggleTitle) { togglePanel(toggleTitle.closest('.link-host-panel')); return; }
|
||||
|
||||
if (e.target.closest('[data-action="collapse-all"]')) { collapseAll(); return; }
|
||||
if (e.target.closest('[data-action="expand-all"]')) { expandAll(); return; }
|
||||
});
|
||||
</script>
|
||||
{% endblock %}
|
||||
@@ -0,0 +1,337 @@
|
||||
{% extends "base.html" %}
|
||||
{% block title %}Suppressions – GANDALF{% endblock %}
|
||||
|
||||
{% block content %}
|
||||
|
||||
<div class="lt-page-header">
|
||||
<div>
|
||||
<h1 class="lt-page-title">Alert Suppressions</h1>
|
||||
<p class="g-page-sub" style="margin-top:4px">Manage maintenance windows and per-target alert suppression rules.</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- ── Create suppression ─────────────────────────────────────────── -->
|
||||
<section class="g-section">
|
||||
<div class="g-section-header">
|
||||
<h2 class="g-section-title">Create Suppression</h2>
|
||||
</div>
|
||||
<div class="lt-card">
|
||||
<div class="lt-card-body">
|
||||
<form id="create-suppression-form">
|
||||
<div class="form-row">
|
||||
<div class="lt-form-group">
|
||||
<label class="lt-label" for="s-type">Target Type <span class="required">*</span></label>
|
||||
<select class="lt-select" id="s-type" name="target_type">
|
||||
<option value="host">Host (all interfaces)</option>
|
||||
<option value="interface">Specific Interface</option>
|
||||
<option value="unifi_device">UniFi Device</option>
|
||||
<option value="all">Global (suppress everything)</option>
|
||||
</select>
|
||||
</div>
|
||||
<div class="lt-form-group" id="name-group">
|
||||
<label class="lt-label" for="s-name">Target Name <span class="required">*</span></label>
|
||||
<input type="text" class="lt-input" id="s-name" name="target_name"
|
||||
placeholder="hostname or device name" autocomplete="off"
|
||||
list="target-name-list">
|
||||
<datalist id="target-name-list">
|
||||
{% for name in snapshot.hosts.keys() | sort %}
|
||||
<option value="{{ name }}">
|
||||
{% endfor %}
|
||||
</datalist>
|
||||
</div>
|
||||
<div class="lt-form-group" id="detail-group" style="display:none">
|
||||
<label class="lt-label" for="s-detail">Interface Name</label>
|
||||
<input type="text" class="lt-input" id="s-detail" name="target_detail"
|
||||
placeholder="e.g. enp35s0 or bond0" autocomplete="off">
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="form-row">
|
||||
<div class="lt-form-group form-group-wide">
|
||||
<label class="lt-label" for="s-reason">Reason <span class="required">*</span></label>
|
||||
<input type="text" class="lt-input" id="s-reason" name="reason"
|
||||
placeholder="e.g. Planned switch maintenance, replacing SFP on large1/enp43s0"
|
||||
required>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="form-row form-row-align">
|
||||
<div class="lt-form-group">
|
||||
<label class="lt-label">Duration</label>
|
||||
<div class="duration-pills">
|
||||
<button type="button" class="pill" data-duration="30">30 min</button>
|
||||
<button type="button" class="pill" data-duration="60">1 hr</button>
|
||||
<button type="button" class="pill" data-duration="240">4 hr</button>
|
||||
<button type="button" class="pill" data-duration="480">8 hr</button>
|
||||
<button type="button" class="pill pill-manual active" data-duration="">Manual ∞</button>
|
||||
</div>
|
||||
<input type="hidden" id="s-expires" name="expires_minutes" value="">
|
||||
<div class="lt-field-hint" id="s-dur-hint">Persists until manually removed.</div>
|
||||
</div>
|
||||
<div class="lt-form-group form-group-submit">
|
||||
<button type="submit" class="lt-btn lt-btn-primary lt-btn-lg">🔕 Apply Suppression</button>
|
||||
</div>
|
||||
</div>
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<!-- ── Active suppressions ────────────────────────────────────────── -->
|
||||
<section class="g-section" id="active-sup-section">
|
||||
<div class="g-section-header">
|
||||
<h2 class="g-section-title">Active Suppressions</h2>
|
||||
<span class="g-section-badge" id="active-sup-badge">{{ active | length }}</span>
|
||||
</div>
|
||||
<div id="active-sup-wrap">
|
||||
{% if active %}
|
||||
<div class="lt-frame">
|
||||
<span class="lt-frame-bl">╚</span>
|
||||
<span class="lt-frame-br">╝</span>
|
||||
<div class="lt-section-header">Active Rules</div>
|
||||
<div class="lt-table-wrap">
|
||||
<table class="lt-table" id="active-sup-table">
|
||||
<caption class="lt-sr-only">Active suppression rules</caption>
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Type</th><th>Target</th><th>Detail</th><th>Reason</th>
|
||||
<th>By</th><th>Created</th><th>Expires</th><th>Actions</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{% for s in active %}
|
||||
<tr id="sup-row-{{ s.id }}">
|
||||
{%- set _sup_badge = {'host':'badge-warning','interface':'badge-info','unifi_device':'badge-purple','all':'badge-critical'} -%}
|
||||
<td><span class="lt-badge {{ _sup_badge.get(s.target_type, 'badge-neutral') }}">{{ s.target_type }}</span></td>
|
||||
<td>{{ s.target_name or 'all' }}</td>
|
||||
<td>{{ s.target_detail or '–' }}</td>
|
||||
<td>{{ s.reason }}</td>
|
||||
<td>{{ s.suppressed_by }}</td>
|
||||
<td class="ts-cell">{{ s.created_at }}</td>
|
||||
<td class="ts-cell">{% if s.expires_at %}{{ s.expires_at }}{% else %}<em>manual</em>{% endif %}</td>
|
||||
<td>
|
||||
<button class="lt-btn lt-btn-danger lt-btn-sm" data-action="remove-sup" data-sup-id="{{ s.id }}">Remove</button>
|
||||
</td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
{% else %}
|
||||
<div class="lt-empty-state lt-empty-state--sm" id="no-active-msg">
|
||||
<div class="lt-empty-state-icon">🔕</div>
|
||||
<div class="lt-empty-state-title">No active suppressions</div>
|
||||
<div class="lt-empty-state-body">All alerts are active. Use the form above to silence a host or interface.</div>
|
||||
</div>
|
||||
{% endif %}
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<!-- ── Suppression history ────────────────────────────────────────── -->
|
||||
<section class="g-section">
|
||||
<div class="g-section-header">
|
||||
<h2 class="g-section-title">History</h2>
|
||||
<span class="g-section-badge">{{ history | length }}</span>
|
||||
</div>
|
||||
{% if history %}
|
||||
<div class="lt-frame">
|
||||
<span class="lt-frame-bl">╚</span>
|
||||
<span class="lt-frame-br">╝</span>
|
||||
<div class="lt-section-header">Suppression Log</div>
|
||||
<div class="lt-table-wrap">
|
||||
<table class="lt-table lt-table-sm">
|
||||
<caption class="lt-sr-only">Suppression history</caption>
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Type</th><th>Target</th><th>Detail</th><th>Reason</th>
|
||||
<th>By</th><th>Created</th><th>Expires</th><th>Active</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{% for s in history %}
|
||||
<tr class="{% if not s.active %}row-resolved{% endif %}">
|
||||
<td>{{ s.target_type }}</td>
|
||||
<td>{{ s.target_name or 'all' }}</td>
|
||||
<td>{{ s.target_detail or '–' }}</td>
|
||||
<td>{{ s.reason }}</td>
|
||||
<td>{{ s.suppressed_by }}</td>
|
||||
<td class="ts-cell">{{ s.created_at }}</td>
|
||||
<td class="ts-cell">{% if s.expires_at %}{{ s.expires_at }}{% else %}<em>manual</em>{% endif %}</td>
|
||||
<td>
|
||||
{% if s.active %}
|
||||
<span class="lt-badge badge-ok">Yes</span>
|
||||
{% else %}
|
||||
<span class="lt-badge badge-neutral">No</span>
|
||||
{% endif %}
|
||||
</td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
{% else %}
|
||||
<div class="lt-empty-state lt-empty-state--sm">
|
||||
<div class="lt-empty-state-icon">📋</div>
|
||||
<div class="lt-empty-state-title">No suppression history yet</div>
|
||||
</div>
|
||||
{% endif %}
|
||||
</section>
|
||||
|
||||
<!-- ── Available targets reference ───────────────────────────────── -->
|
||||
<section class="g-section">
|
||||
<div class="g-section-header">
|
||||
<h2 class="g-section-title">Available Targets</h2>
|
||||
</div>
|
||||
<div class="targets-grid">
|
||||
{% for name, host in snapshot.hosts.items() %}
|
||||
<div class="target-card">
|
||||
<div class="target-name">{{ name }}</div>
|
||||
<div class="target-type">{{ 'Proxmox Host (prometheus)' if host.source == 'prometheus' else 'Ping-only host' }}</div>
|
||||
{% if host.interfaces %}
|
||||
<div class="target-ifaces">
|
||||
{% for iface in host.interfaces.keys() | sort %}
|
||||
<code class="iface-chip">{{ iface }}</code>
|
||||
{% endfor %}
|
||||
</div>
|
||||
{% endif %}
|
||||
</div>
|
||||
{% endfor %}
|
||||
</div>
|
||||
</section>
|
||||
|
||||
{% endblock %}
|
||||
|
||||
{% block scripts %}
|
||||
<script>
|
||||
function onTypeChange() {
|
||||
const t = document.getElementById('s-type').value;
|
||||
document.getElementById('name-group').style.display = (t==='all') ? 'none' : '';
|
||||
document.getElementById('detail-group').style.display = (t==='interface') ? '' : 'none';
|
||||
document.getElementById('s-name').required = (t!=='all');
|
||||
}
|
||||
|
||||
function setDur(mins, el) {
|
||||
document.getElementById('s-expires').value = mins || '';
|
||||
document.querySelectorAll('.duration-pills .pill').forEach(p => p.classList.remove('active'));
|
||||
if (el) el.classList.add('active');
|
||||
const hint = document.getElementById('s-dur-hint');
|
||||
if (mins) {
|
||||
const h = Math.floor(mins/60), m = mins%60;
|
||||
hint.textContent = `Expires in ${h?h+'h ':''}${m?m+'m':''}`.trim()+'.';
|
||||
} else {
|
||||
hint.textContent = 'Persists until manually removed.';
|
||||
}
|
||||
}
|
||||
|
||||
function renderActiveRows(rows) {
|
||||
const wrap = document.getElementById('active-sup-wrap');
|
||||
const badge = document.getElementById('active-sup-badge');
|
||||
if (!rows || !rows.length) {
|
||||
wrap.innerHTML = '<div class="lt-empty-state lt-empty-state--sm" id="no-active-msg"><div class="lt-empty-state-icon">🔕</div><div class="lt-empty-state-title">No active suppressions</div><div class="lt-empty-state-body">All alerts are active. Use the form above to silence a host or interface.</div></div>';
|
||||
if (badge) badge.textContent = '0';
|
||||
return;
|
||||
}
|
||||
if (badge) badge.textContent = rows.length;
|
||||
const SUP_BADGE = {host:'badge-warning', interface:'badge-info', unifi_device:'badge-purple', all:'badge-critical'};
|
||||
const tbody = rows.map(s => `
|
||||
<tr id="sup-row-${s.id}">
|
||||
<td><span class="lt-badge ${SUP_BADGE[s.target_type] || 'badge-neutral'}">${lt.escHtml(s.target_type)}</span></td>
|
||||
<td>${lt.escHtml(s.target_name || 'all')}</td>
|
||||
<td>${lt.escHtml(s.target_detail || '–')}</td>
|
||||
<td>${lt.escHtml(s.reason)}</td>
|
||||
<td>${lt.escHtml(s.suppressed_by)}</td>
|
||||
<td class="ts-cell">${lt.escHtml(s.created_at || '')}</td>
|
||||
<td class="ts-cell">${s.expires_at ? lt.escHtml(s.expires_at) : '<em>manual</em>'}</td>
|
||||
<td><button class="lt-btn lt-btn-danger lt-btn-sm" data-action="remove-sup" data-sup-id="${s.id}">Remove</button></td>
|
||||
</tr>`).join('');
|
||||
wrap.innerHTML = `
|
||||
<div class="lt-frame">
|
||||
<span class="lt-frame-bl">╚</span>
|
||||
<span class="lt-frame-br">╝</span>
|
||||
<div class="lt-section-header">Active Rules</div>
|
||||
<div class="lt-table-wrap">
|
||||
<table class="lt-table" id="active-sup-table">
|
||||
<caption class="lt-sr-only">Active suppression rules</caption>
|
||||
<thead><tr>
|
||||
<th>Type</th><th>Target</th><th>Detail</th><th>Reason</th>
|
||||
<th>By</th><th>Created</th><th>Expires</th><th>Actions</th>
|
||||
</tr></thead>
|
||||
<tbody>${tbody}</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>`;
|
||||
}
|
||||
|
||||
async function refreshActive() {
|
||||
try {
|
||||
const rows = await lt.api.get('/api/suppressions');
|
||||
renderActiveRows(rows);
|
||||
} catch (err) {
|
||||
showToast('Failed to refresh suppressions', 'warning');
|
||||
}
|
||||
}
|
||||
|
||||
async function createSuppression(e) {
|
||||
e.preventDefault();
|
||||
const form = e.target;
|
||||
const btn = form.querySelector('[type="submit"]');
|
||||
btn.classList.add('is-loading');
|
||||
const payload = {
|
||||
target_type: form.target_type.value,
|
||||
target_name: form.target_name ? form.target_name.value : '',
|
||||
target_detail: document.getElementById('s-detail').value,
|
||||
reason: form.reason.value,
|
||||
expires_minutes: form.expires_minutes.value ? parseInt(form.expires_minutes.value) : null,
|
||||
};
|
||||
try {
|
||||
await lt.api.post('/api/suppressions', payload);
|
||||
showToast('Suppression applied', 'success');
|
||||
form.reset();
|
||||
onTypeChange();
|
||||
document.querySelectorAll('.duration-pills .pill').forEach(p => p.classList.remove('active'));
|
||||
document.querySelector('.duration-pills .pill-manual')?.classList.add('active');
|
||||
document.getElementById('s-dur-hint').textContent = 'Persists until manually removed.';
|
||||
await refreshActive();
|
||||
} catch (err) {
|
||||
showToast(err.message || 'Error', 'error');
|
||||
} finally {
|
||||
btn.classList.remove('is-loading');
|
||||
}
|
||||
}
|
||||
|
||||
async function removeSuppression(id) {
|
||||
if (!confirm('Remove this suppression?')) return;
|
||||
try {
|
||||
await lt.api.delete(`/api/suppressions/${id}`);
|
||||
document.getElementById(`sup-row-${id}`)?.remove();
|
||||
const badge = document.getElementById('active-sup-badge');
|
||||
if (badge) badge.textContent = Math.max(0, parseInt(badge.textContent || '0') - 1);
|
||||
const tbody = document.querySelector('#active-sup-table tbody');
|
||||
if (tbody && !tbody.children.length) {
|
||||
document.getElementById('active-sup-wrap').innerHTML =
|
||||
'<div class="lt-empty-state lt-empty-state--sm" id="no-active-msg"><div class="lt-empty-state-icon">🔕</div><div class="lt-empty-state-title">No active suppressions</div><div class="lt-empty-state-body">All alerts are active. Use the form above to silence a host or interface.</div></div>';
|
||||
if (badge) badge.textContent = '0';
|
||||
}
|
||||
showToast('Suppression removed', 'success');
|
||||
} catch (err) {
|
||||
showToast(err.message || 'Failed to remove suppression', 'error');
|
||||
}
|
||||
}
|
||||
|
||||
document.getElementById('s-type')?.addEventListener('change', onTypeChange);
|
||||
document.getElementById('create-suppression-form')?.addEventListener('submit', createSuppression);
|
||||
|
||||
document.addEventListener('click', e => {
|
||||
const pill = e.target.closest('#create-suppression-form .pill[data-duration]');
|
||||
if (pill) {
|
||||
const val = pill.dataset.duration;
|
||||
setDur(val ? parseInt(val) : null, pill);
|
||||
return;
|
||||
}
|
||||
const removeBtn = e.target.closest('[data-action="remove-sup"]');
|
||||
if (removeBtn) removeSuppression(parseInt(removeBtn.dataset.supId));
|
||||
});
|
||||
</script>
|
||||
{% endblock %}
|
||||
@@ -0,0 +1,479 @@
|
||||
"""Tests for gandalf.diagnose — all pure static methods, no external deps."""
|
||||
import sys
|
||||
import os
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(__file__)))
|
||||
|
||||
from diagnose import DiagnosticsRunner # noqa: E402
|
||||
|
||||
|
||||
# ── build_ssh_command ────────────────────────────────────────────────────────
|
||||
|
||||
class TestBuildSshCommand:
|
||||
def test_contains_stricthostkeychecking_no(self):
|
||||
cmd = DiagnosticsRunner.build_ssh_command('10.0.0.1', 'eth0')
|
||||
assert 'StrictHostKeyChecking=no' in cmd
|
||||
|
||||
def test_contains_host_ip(self):
|
||||
cmd = DiagnosticsRunner.build_ssh_command('10.0.0.1', 'eth0')
|
||||
assert 'root@10.0.0.1' in cmd
|
||||
|
||||
def test_contains_interface_name(self):
|
||||
cmd = DiagnosticsRunner.build_ssh_command('10.0.0.1', 'eth0')
|
||||
assert 'eth0' in cmd
|
||||
|
||||
def test_shell_quotes_special_chars_in_iface(self):
|
||||
# shlex.quote wraps the whole iface string in single quotes so
|
||||
# shell metacharacters are not interpreted as commands
|
||||
cmd = DiagnosticsRunner.build_ssh_command('10.0.0.1', "eth0; rm -rf /")
|
||||
# The iface must appear inside single quotes
|
||||
assert "'eth0; rm -rf /'" in cmd
|
||||
|
||||
def test_contains_sysfs_stats_section(self):
|
||||
cmd = DiagnosticsRunner.build_ssh_command('10.0.0.1', 'eth0')
|
||||
assert 'sysfs_stats' in cmd
|
||||
|
||||
def test_contains_ethtool_section(self):
|
||||
cmd = DiagnosticsRunner.build_ssh_command('10.0.0.1', 'eth0')
|
||||
assert 'ethtool' in cmd
|
||||
|
||||
|
||||
# ── parse_output ─────────────────────────────────────────────────────────────
|
||||
|
||||
SAMPLE_OUTPUT = """\
|
||||
=== carrier ===
|
||||
1
|
||||
=== operstate ===
|
||||
up
|
||||
=== sysfs_stats ===
|
||||
rx_bytes:12345
|
||||
tx_bytes:67890
|
||||
rx_errors:0
|
||||
tx_errors:0
|
||||
rx_dropped:0
|
||||
tx_dropped:0
|
||||
rx_crc_errors:0
|
||||
rx_frame_errors:0
|
||||
rx_fifo_errors:0
|
||||
tx_carrier_errors:0
|
||||
collisions:0
|
||||
rx_missed_errors:0
|
||||
=== carrier_changes ===
|
||||
3
|
||||
=== ethtool ===
|
||||
Speed: 1000Mb/s
|
||||
Duplex: Full
|
||||
Link detected: yes
|
||||
Auto-negotiation: on
|
||||
=== ethtool_driver ===
|
||||
driver: igb
|
||||
version: 5.6.0-k
|
||||
firmware-version: 1.67, 0x80000d38
|
||||
bus-info: 0000:03:00.0
|
||||
=== ethtool_pause ===
|
||||
RX: on
|
||||
TX: off
|
||||
=== ethtool_ring ===
|
||||
Pre-set maximums:
|
||||
RX: 4096
|
||||
TX: 4096
|
||||
Current hardware settings:
|
||||
RX: 256
|
||||
TX: 256
|
||||
=== ethtool_stats ===
|
||||
rx_packets: 1000
|
||||
tx_packets: 2000
|
||||
=== ethtool_dom ===
|
||||
=== ip_link ===
|
||||
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500
|
||||
=== ip_addr ===
|
||||
inet 10.0.0.1/24
|
||||
=== ip_route ===
|
||||
default via 10.0.0.254
|
||||
=== dmesg ===
|
||||
eth0: renamed from ens3
|
||||
=== lldpctl ===
|
||||
lldpd not running
|
||||
=== end ===
|
||||
"""
|
||||
|
||||
|
||||
class TestParseOutput:
|
||||
def setup_method(self):
|
||||
self.parsed = DiagnosticsRunner.parse_output(SAMPLE_OUTPUT)
|
||||
|
||||
def test_carrier_parsed(self):
|
||||
assert self.parsed['carrier'] == '1'
|
||||
|
||||
def test_operstate_parsed(self):
|
||||
assert self.parsed['operstate'] == 'up'
|
||||
|
||||
def test_carrier_changes_is_int(self):
|
||||
assert self.parsed['carrier_changes'] == 3
|
||||
|
||||
def test_sysfs_stats_rx_bytes(self):
|
||||
assert self.parsed['sysfs_stats']['rx_bytes'] == 12345
|
||||
|
||||
def test_sysfs_stats_tx_bytes(self):
|
||||
assert self.parsed['sysfs_stats']['tx_bytes'] == 67890
|
||||
|
||||
def test_ethtool_speed(self):
|
||||
assert self.parsed['ethtool']['speed_mbps'] == 1000
|
||||
|
||||
def test_ethtool_link_detected(self):
|
||||
assert self.parsed['ethtool']['link_detected'] is True
|
||||
|
||||
def test_ethtool_duplex(self):
|
||||
assert self.parsed['ethtool']['duplex'] == 'full'
|
||||
|
||||
def test_ethtool_auto_neg(self):
|
||||
assert self.parsed['ethtool']['auto_neg'] is True
|
||||
|
||||
def test_ethtool_driver(self):
|
||||
assert self.parsed['ethtool_driver']['driver'] == 'igb'
|
||||
|
||||
def test_ethtool_pause_rx_on(self):
|
||||
assert self.parsed['ethtool_pause']['rx_pause'] is True
|
||||
|
||||
def test_ethtool_pause_tx_off(self):
|
||||
assert self.parsed['ethtool_pause']['tx_pause'] is False
|
||||
|
||||
def test_ethtool_ring_current(self):
|
||||
assert self.parsed['ethtool_ring']['rx_current'] == 256
|
||||
assert self.parsed['ethtool_ring']['tx_current'] == 256
|
||||
|
||||
def test_ethtool_ring_max(self):
|
||||
assert self.parsed['ethtool_ring']['rx_max'] == 4096
|
||||
|
||||
def test_ip_addr_present(self):
|
||||
assert '10.0.0.1/24' in self.parsed['ip_addr']
|
||||
|
||||
def test_ip_route_present(self):
|
||||
assert 'default via 10.0.0.254' in self.parsed['ip_route']
|
||||
|
||||
|
||||
class TestParseOutputEdgeCases:
|
||||
def test_empty_string_returns_dict(self):
|
||||
result = DiagnosticsRunner.parse_output('')
|
||||
assert isinstance(result, dict)
|
||||
assert result['carrier'] == '?'
|
||||
assert result['operstate'] == '?'
|
||||
assert result['carrier_changes'] == 0
|
||||
|
||||
def test_no_end_sentinel_still_parses(self):
|
||||
output = "=== carrier ===\n1\n=== operstate ===\nup\n"
|
||||
result = DiagnosticsRunner.parse_output(output)
|
||||
assert result['carrier'] == '1'
|
||||
|
||||
|
||||
# ── parse_sysfs_stats ────────────────────────────────────────────────────────
|
||||
|
||||
class TestParseSysfsStats:
|
||||
def test_parses_known_keys(self):
|
||||
text = "rx_bytes:100\ntx_bytes:200\nrx_errors:0\n"
|
||||
result = DiagnosticsRunner.parse_sysfs_stats(text)
|
||||
assert result['rx_bytes'] == 100
|
||||
assert result['tx_bytes'] == 200
|
||||
assert result['rx_errors'] == 0
|
||||
|
||||
def test_ignores_unknown_keys(self):
|
||||
text = "unknown_counter:999\nrx_bytes:50\n"
|
||||
result = DiagnosticsRunner.parse_sysfs_stats(text)
|
||||
assert 'unknown_counter' not in result
|
||||
assert result['rx_bytes'] == 50
|
||||
|
||||
def test_non_numeric_value_defaults_to_zero(self):
|
||||
text = "rx_bytes:not_a_number\n"
|
||||
result = DiagnosticsRunner.parse_sysfs_stats(text)
|
||||
assert result['rx_bytes'] == 0
|
||||
|
||||
def test_empty_text_returns_empty_dict(self):
|
||||
assert DiagnosticsRunner.parse_sysfs_stats('') == {}
|
||||
|
||||
|
||||
# ── parse_ethtool ─────────────────────────────────────────────────────────────
|
||||
|
||||
class TestParseEthtool:
|
||||
def test_speed_parsed(self):
|
||||
text = "Speed: 10000Mb/s\n"
|
||||
result = DiagnosticsRunner.parse_ethtool(text)
|
||||
assert result['speed_mbps'] == 10000
|
||||
|
||||
def test_unknown_speed(self):
|
||||
text = "Speed: Unknown! (0)\n"
|
||||
result = DiagnosticsRunner.parse_ethtool(text)
|
||||
assert result['speed_mbps'] is None
|
||||
|
||||
def test_link_detected_no(self):
|
||||
text = "Link detected: no\n"
|
||||
result = DiagnosticsRunner.parse_ethtool(text)
|
||||
assert result['link_detected'] is False
|
||||
|
||||
def test_auto_neg_off(self):
|
||||
text = "Auto-negotiation: off\n"
|
||||
result = DiagnosticsRunner.parse_ethtool(text)
|
||||
assert result['auto_neg'] is False
|
||||
|
||||
def test_empty_returns_empty(self):
|
||||
assert DiagnosticsRunner.parse_ethtool('') == {}
|
||||
|
||||
|
||||
# ── parse_ethtool_driver ──────────────────────────────────────────────────────
|
||||
|
||||
class TestParseEthtoolDriver:
|
||||
def test_driver_name(self):
|
||||
text = "driver: igb\nversion: 5.6.0-k\nfirmware-version: 1.67\nbus-info: 0000:03:00.0\n"
|
||||
result = DiagnosticsRunner.parse_ethtool_driver(text)
|
||||
assert result['driver'] == 'igb'
|
||||
|
||||
def test_version(self):
|
||||
text = "driver: igb\nversion: 5.6.0-k\n"
|
||||
result = DiagnosticsRunner.parse_ethtool_driver(text)
|
||||
assert result['version'] == '5.6.0-k'
|
||||
|
||||
def test_firmware_version(self):
|
||||
text = "firmware-version: 1.67, 0x80000d38\n"
|
||||
result = DiagnosticsRunner.parse_ethtool_driver(text)
|
||||
assert result['firmware_version'] == '1.67, 0x80000d38'
|
||||
|
||||
def test_bus_info(self):
|
||||
text = "bus-info: 0000:03:00.0\n"
|
||||
result = DiagnosticsRunner.parse_ethtool_driver(text)
|
||||
assert result['bus_info'] == '0000:03:00.0'
|
||||
|
||||
def test_empty_returns_empty(self):
|
||||
assert DiagnosticsRunner.parse_ethtool_driver('') == {}
|
||||
|
||||
|
||||
# ── parse_nic_stats ───────────────────────────────────────────────────────────
|
||||
|
||||
class TestParseNicStats:
|
||||
def test_parses_integer_values(self):
|
||||
text = "rx_packets: 1234\ntx_packets: 5678\n"
|
||||
result = DiagnosticsRunner.parse_nic_stats(text)
|
||||
assert result['rx_packets'] == 1234
|
||||
assert result['tx_packets'] == 5678
|
||||
|
||||
def test_non_numeric_skipped(self):
|
||||
text = "rx_packets: 100\nbad_stat: N/A\n"
|
||||
result = DiagnosticsRunner.parse_nic_stats(text)
|
||||
assert 'bad_stat' not in result
|
||||
assert result['rx_packets'] == 100
|
||||
|
||||
def test_empty_returns_empty(self):
|
||||
assert DiagnosticsRunner.parse_nic_stats('') == {}
|
||||
|
||||
|
||||
# ── parse_ethtool_dom ─────────────────────────────────────────────────────────
|
||||
|
||||
class TestParseEthtoolDom:
|
||||
def test_unsupported_returns_empty(self):
|
||||
assert DiagnosticsRunner.parse_ethtool_dom('Cannot get module EEPROM information') == {}
|
||||
|
||||
def test_empty_returns_empty(self):
|
||||
assert DiagnosticsRunner.parse_ethtool_dom('') == {}
|
||||
|
||||
def test_vendor_name(self):
|
||||
text = "Vendor name : CISCO-FINISAR\n"
|
||||
result = DiagnosticsRunner.parse_ethtool_dom(text)
|
||||
assert result['vendor'] == 'CISCO-FINISAR'
|
||||
|
||||
def test_wavelength(self):
|
||||
text = "Laser wavelength : 1310 nm\n"
|
||||
result = DiagnosticsRunner.parse_ethtool_dom(text)
|
||||
assert result['wavelength_nm'] == 1310
|
||||
|
||||
def test_tx_power_dbm(self):
|
||||
text = "Laser output power : 0.3000 mW / -5.23 dBm\n"
|
||||
result = DiagnosticsRunner.parse_ethtool_dom(text)
|
||||
assert abs(result['tx_power_dbm'] - (-5.23)) < 0.01
|
||||
|
||||
def test_rx_power_dbm(self):
|
||||
text = "Receiver signal average optical power : 0.1000 mW / -10.00 dBm\n"
|
||||
result = DiagnosticsRunner.parse_ethtool_dom(text)
|
||||
assert abs(result['rx_power_dbm'] - (-10.00)) < 0.01
|
||||
|
||||
|
||||
# ── parse_ip_link ─────────────────────────────────────────────────────────────
|
||||
|
||||
class TestParseIpLink:
|
||||
IP_LINK_SAMPLE = """\
|
||||
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP mode DEFAULT group default qlen 1000
|
||||
link/ether aa:bb:cc:dd:ee:ff brd ff:ff:ff:ff:ff:ff
|
||||
RX: bytes packets errors dropped missed mcast
|
||||
123456789 1000000 0 5 0 0
|
||||
TX: bytes packets errors dropped carrier collsns
|
||||
98765432 900000 0 0 0 0
|
||||
"""
|
||||
|
||||
def test_mtu_parsed(self):
|
||||
result = DiagnosticsRunner.parse_ip_link(self.IP_LINK_SAMPLE)
|
||||
assert result['mtu'] == 1500
|
||||
|
||||
def test_state_parsed(self):
|
||||
result = DiagnosticsRunner.parse_ip_link(self.IP_LINK_SAMPLE)
|
||||
assert result['state'] == 'up'
|
||||
|
||||
def test_rx_bytes(self):
|
||||
result = DiagnosticsRunner.parse_ip_link(self.IP_LINK_SAMPLE)
|
||||
assert result['ip_rx_bytes'] == 123456789
|
||||
|
||||
def test_tx_bytes(self):
|
||||
result = DiagnosticsRunner.parse_ip_link(self.IP_LINK_SAMPLE)
|
||||
assert result['ip_tx_bytes'] == 98765432
|
||||
|
||||
def test_rx_dropped(self):
|
||||
result = DiagnosticsRunner.parse_ip_link(self.IP_LINK_SAMPLE)
|
||||
assert result['ip_rx_dropped'] == 5
|
||||
|
||||
def test_empty_returns_empty(self):
|
||||
assert DiagnosticsRunner.parse_ip_link('') == {}
|
||||
|
||||
|
||||
# ── parse_dmesg ───────────────────────────────────────────────────────────────
|
||||
|
||||
class TestParseDmesg:
|
||||
def test_error_severity(self):
|
||||
text = "[ 1.234567] eth0: Link failure detected\n"
|
||||
events = DiagnosticsRunner.parse_dmesg(text)
|
||||
assert len(events) == 1
|
||||
assert events[0]['severity'] == 'error'
|
||||
|
||||
def test_warn_severity(self):
|
||||
text = "[ 2.000000] eth0: dropped packet\n"
|
||||
events = DiagnosticsRunner.parse_dmesg(text)
|
||||
assert events[0]['severity'] == 'warn'
|
||||
|
||||
def test_info_severity(self):
|
||||
text = "[ 3.000000] eth0: NIC Link is Up\n"
|
||||
events = DiagnosticsRunner.parse_dmesg(text)
|
||||
assert events[0]['severity'] == 'info'
|
||||
|
||||
def test_timestamp_extracted(self):
|
||||
text = "[ 5.678900] eth0: reset\n"
|
||||
events = DiagnosticsRunner.parse_dmesg(text)
|
||||
assert events[0]['timestamp'] == '5.678900'
|
||||
|
||||
def test_no_timestamp(self):
|
||||
text = "eth0: some timeout event\n"
|
||||
events = DiagnosticsRunner.parse_dmesg(text)
|
||||
assert events[0]['timestamp'] == ''
|
||||
assert events[0]['severity'] == 'error'
|
||||
|
||||
def test_empty_returns_empty_list(self):
|
||||
assert DiagnosticsRunner.parse_dmesg('') == []
|
||||
|
||||
|
||||
# ── parse_lldpctl ─────────────────────────────────────────────────────────────
|
||||
|
||||
class TestParseLldpctl:
|
||||
def test_unavailable_when_not_running(self):
|
||||
result = DiagnosticsRunner.parse_lldpctl('lldpd not running')
|
||||
assert result == {'available': False}
|
||||
|
||||
def test_unavailable_on_empty(self):
|
||||
result = DiagnosticsRunner.parse_lldpctl('')
|
||||
assert result == {'available': False}
|
||||
|
||||
def test_neighbor_system_parsed(self):
|
||||
text = " SysName: core-sw-01\n PortID: Gi1/0/5\n ChassisID: aa:bb:cc:dd:ee:ff\n"
|
||||
result = DiagnosticsRunner.parse_lldpctl(text)
|
||||
assert result['available'] is True
|
||||
assert result['neighbor_system'] == 'core-sw-01'
|
||||
|
||||
def test_neighbor_port_parsed(self):
|
||||
text = " SysName: core-sw-01\n PortID: Gi1/0/5\n"
|
||||
result = DiagnosticsRunner.parse_lldpctl(text)
|
||||
assert result['neighbor_port'] == 'Gi1/0/5'
|
||||
|
||||
def test_chassis_id_parsed(self):
|
||||
text = " ChassisID: aa:bb:cc:dd:ee:ff\n"
|
||||
result = DiagnosticsRunner.parse_lldpctl(text)
|
||||
assert result['neighbor_chassis_id'] == 'aa:bb:cc:dd:ee:ff'
|
||||
|
||||
|
||||
# ── analyze ───────────────────────────────────────────────────────────────────
|
||||
|
||||
class TestAnalyze:
|
||||
def _sections(self, **overrides):
|
||||
base = {
|
||||
'carrier': '1',
|
||||
'operstate': 'up',
|
||||
'ethtool': {'link_detected': True, 'duplex': 'full', 'speed_mbps': 1000},
|
||||
'sysfs_stats': {'rx_crc_errors': 0},
|
||||
'ethtool_dom': {},
|
||||
'dmesg': [],
|
||||
'lldpctl': {'available': False},
|
||||
'carrier_changes': 0,
|
||||
}
|
||||
base.update(overrides)
|
||||
return base
|
||||
|
||||
def test_no_carrier_is_issue(self):
|
||||
result = DiagnosticsRunner.analyze(self._sections(carrier='0'), {})
|
||||
codes = [i['code'] for i in result['issues']]
|
||||
assert 'NO_CARRIER' in codes
|
||||
|
||||
def test_half_duplex_is_issue(self):
|
||||
sections = self._sections(ethtool={'duplex': 'half', 'link_detected': True, 'speed_mbps': 1000})
|
||||
result = DiagnosticsRunner.analyze(sections, {})
|
||||
codes = [i['code'] for i in result['issues']]
|
||||
assert 'HALF_DUPLEX' in codes
|
||||
|
||||
def test_speed_mismatch_is_warning(self):
|
||||
result = DiagnosticsRunner.analyze(
|
||||
self._sections(ethtool={'duplex': 'full', 'link_detected': True, 'speed_mbps': 100}),
|
||||
{'speed_mbps': 1000}
|
||||
)
|
||||
codes = [w['code'] for w in result['warnings']]
|
||||
assert 'SPEED_MISMATCH' in codes
|
||||
|
||||
def test_sfp_rx_critical_low(self):
|
||||
sections = self._sections(ethtool_dom={'rx_power_dbm': -30.0})
|
||||
result = DiagnosticsRunner.analyze(sections, {})
|
||||
codes = [i['code'] for i in result['issues']]
|
||||
assert 'SFP_RX_CRITICAL' in codes
|
||||
|
||||
def test_sfp_rx_low_is_warning(self):
|
||||
sections = self._sections(ethtool_dom={'rx_power_dbm': -20.0})
|
||||
result = DiagnosticsRunner.analyze(sections, {})
|
||||
codes = [w['code'] for w in result['warnings']]
|
||||
assert 'SFP_RX_LOW' in codes
|
||||
|
||||
def test_high_crc_is_issue(self):
|
||||
sections = self._sections(sysfs_stats={'rx_crc_errors': 200})
|
||||
result = DiagnosticsRunner.analyze(sections, {})
|
||||
codes = [i['code'] for i in result['issues']]
|
||||
assert 'CRC_ERRORS_HIGH' in codes
|
||||
|
||||
def test_low_crc_is_warning(self):
|
||||
sections = self._sections(sysfs_stats={'rx_crc_errors': 50})
|
||||
result = DiagnosticsRunner.analyze(sections, {})
|
||||
codes = [w['code'] for w in result['warnings']]
|
||||
assert 'CRC_ERRORS_LOW' in codes
|
||||
|
||||
def test_carrier_flapping_issue(self):
|
||||
result = DiagnosticsRunner.analyze(self._sections(carrier_changes=150), {})
|
||||
codes = [i['code'] for i in result['issues']]
|
||||
assert 'CARRIER_FLAPPING' in codes
|
||||
|
||||
def test_carrier_flaps_warning(self):
|
||||
result = DiagnosticsRunner.analyze(self._sections(carrier_changes=25), {})
|
||||
codes = [w['code'] for w in result['warnings']]
|
||||
assert 'CARRIER_FLAPS' in codes
|
||||
|
||||
def test_lldp_missing_is_info(self):
|
||||
result = DiagnosticsRunner.analyze(self._sections(lldpctl={'available': False}), {})
|
||||
codes = [i['code'] for i in result['info']]
|
||||
assert 'LLDP_MISSING' in codes
|
||||
|
||||
def test_lldp_mismatch_is_warning(self):
|
||||
sections = self._sections(lldpctl={'available': True, 'neighbor_system': 'wrong-switch'})
|
||||
switch_data = {'speed_mbps': 1000, 'lldp': {'system_name': 'core-sw-01'}}
|
||||
result = DiagnosticsRunner.analyze(sections, switch_data)
|
||||
codes = [w['code'] for w in result['warnings']]
|
||||
assert 'LLDP_MISMATCH' in codes
|
||||
|
||||
def test_healthy_link_no_issues(self):
|
||||
result = DiagnosticsRunner.analyze(self._sections(), {})
|
||||
assert result['issues'] == []
|
||||
assert result['warnings'] == []
|
||||
Reference in new issue
Block a user