Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
e1716d5228 | ||
|
|
4a1998046b | ||
|
|
fed279a0bd | ||
|
|
e0ff3e2168 | ||
|
|
c64853f643 | ||
|
|
42cf754796 | ||
|
|
28fd5aad14 | ||
|
|
65536d58c5 | ||
|
|
0e852ebd11 | ||
|
|
8aaaa89f03 | ||
|
|
7467236ffd | ||
|
|
dc16a583a5 | ||
|
|
12ef20901a | ||
|
|
0693ad56f9 | ||
|
|
c455c5d4b9 | ||
|
|
13e6839cdf | ||
|
|
e7c83eff2b | ||
|
|
5c2a92eaee | ||
|
|
f942e96c24 | ||
|
|
8173b9dc7f | ||
|
|
2896692d93 | ||
|
|
c2492bbfe8 | ||
|
|
33b214eac0 | ||
|
|
0331c2bd00 | ||
|
|
2710eaa118 | ||
|
|
9c011cfce1 | ||
|
|
182cc8c451 | ||
|
|
0d4f209c79 | ||
|
|
b3495caac8 |
@@ -0,0 +1,21 @@
|
|||||||
|
{
|
||||||
|
"env": {
|
||||||
|
"browser": true,
|
||||||
|
"es2021": true
|
||||||
|
},
|
||||||
|
"globals": {
|
||||||
|
"lt": "readonly",
|
||||||
|
"GANDALF_CONFIG": "readonly",
|
||||||
|
"CSS": "readonly"
|
||||||
|
},
|
||||||
|
"rules": {
|
||||||
|
"no-undef": "error",
|
||||||
|
"no-unused-vars": ["warn", { "argsIgnorePattern": "^_", "varsIgnorePattern": "^_" }],
|
||||||
|
"no-console": "off",
|
||||||
|
"eqeqeq": ["error", "always", { "null": "ignore" }]
|
||||||
|
},
|
||||||
|
"parserOptions": {
|
||||||
|
"ecmaVersion": 2021,
|
||||||
|
"sourceType": "script"
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -17,7 +17,7 @@ jobs:
|
|||||||
run: |
|
run: |
|
||||||
apt-get update -qq
|
apt-get update -qq
|
||||||
apt-get install -y -qq python3 python3-pip
|
apt-get install -y -qq python3 python3-pip
|
||||||
pip3 install flake8
|
pip3 install --break-system-packages flake8
|
||||||
|
|
||||||
- name: Run flake8
|
- name: Run flake8
|
||||||
run: flake8 . --exclude=__pycache__,.git
|
run: flake8 . --exclude=__pycache__,.git
|
||||||
|
|||||||
@@ -19,7 +19,7 @@ jobs:
|
|||||||
run: |
|
run: |
|
||||||
apt-get update -qq
|
apt-get update -qq
|
||||||
apt-get install -y -qq python3 python3-pip
|
apt-get install -y -qq python3 python3-pip
|
||||||
pip3 install bandit
|
pip3 install --break-system-packages bandit
|
||||||
|
|
||||||
- name: Run bandit
|
- name: Run bandit
|
||||||
run: bandit -r . --exclude .git,__pycache__,node_modules -ll
|
run: bandit -r . --exclude .git,__pycache__,node_modules -ll
|
||||||
|
|||||||
@@ -17,8 +17,8 @@ jobs:
|
|||||||
run: |
|
run: |
|
||||||
apt-get update -qq
|
apt-get update -qq
|
||||||
apt-get install -y -qq python3 python3-pip
|
apt-get install -y -qq python3 python3-pip
|
||||||
pip3 install pytest pytest-cov
|
pip3 install --break-system-packages pytest pytest-cov
|
||||||
pip3 install -r requirements.txt --quiet
|
pip3 install --break-system-packages -r requirements.txt --quiet
|
||||||
|
|
||||||
- name: Run pytest with coverage
|
- name: Run pytest with coverage
|
||||||
run: python3 -m pytest tests/ -v --cov=. --cov-report=term-missing --cov-config=.coveragerc
|
run: python3 -m pytest tests/ -v --cov=. --cov-report=term-missing --cov-config=.coveragerc
|
||||||
|
|||||||
@@ -0,0 +1,21 @@
|
|||||||
|
MIT License
|
||||||
|
|
||||||
|
Copyright (c) 2025-2026 Jared Vititoe
|
||||||
|
|
||||||
|
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||||
|
of this software and associated documentation files (the "Software"), to deal
|
||||||
|
in the Software without restriction, including without limitation the rights
|
||||||
|
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||||
|
copies of the Software, and to permit persons to whom the Software is
|
||||||
|
furnished to do so, subject to the following conditions:
|
||||||
|
|
||||||
|
The above copyright notice and this permission notice shall be included in all
|
||||||
|
copies or substantial portions of the Software.
|
||||||
|
|
||||||
|
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||||
|
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||||
|
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||||
|
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||||
|
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||||
|
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||||
|
SOFTWARE.
|
||||||
@@ -11,6 +11,52 @@ Deployed on **LXC 157** (monitor-02 / 10.10.10.9), reachable at `gandalf.lotusgu
|
|||||||
|
|
||||||
**Design System**: [web_template](https://code.lotusguild.org/LotusGuild/web_template) — shared CSS, JS, and layout patterns for all LotusGuild apps
|
**Design System**: [web_template](https://code.lotusguild.org/LotusGuild/web_template) — shared CSS, JS, and layout patterns for all LotusGuild apps
|
||||||
|
|
||||||
|

|
||||||
|
|
||||||
|
## Why this exists
|
||||||
|
|
||||||
|
A Proxmox/Ceph cluster lives or dies by its network, and network problems are sneaky: a link that flaps, a fibre with weak receive power, a switch port that quietly negotiated 1 Gbps. GANDALF watches the cluster's links from three angles at once (host NIC state from Prometheus, switch ports and LLDP from the UniFi controller, and reachability pings), alerts only on **regressions** (a link that was up and went down; ports that were never connected are learned as baseline and ignored), and files one ticket per real problem. When something looks wrong, a click on the switch port runs a full on-host diagnostic through [PULSE](https://code.lotusguild.org/LotusGuild/pulse) (`ethtool`, SFP optics, kernel log, LLDP) and explains what it found.
|
||||||
|
|
||||||
|
## Gallery
|
||||||
|
|
||||||
|
**Dashboard**: alert queue with linked tickets, a live topology, per-host interface status and the UniFi device inventory.
|
||||||
|
|
||||||
|

|
||||||
|
|
||||||
|

|
||||||
|
|
||||||
|

|
||||||
|
|
||||||
|
**Link debug**: per-interface speed, duplex, carrier flaps, error and drop rates, and **SFP optical readings** (temperature, TX/RX power) with colour-coded thresholds. Here a fibre with weak receive power is flagged while its neighbour is healthy.
|
||||||
|
|
||||||
|

|
||||||
|
|
||||||
|
| Weak receive power on a fibre link | Healthy fibre link |
|
||||||
|
|---|---|
|
||||||
|
|  |  |
|
||||||
|
|
||||||
|
Switch ports are shown too, with PoE draw and the LLDP neighbour on each port:
|
||||||
|
|
||||||
|

|
||||||
|
|
||||||
|
**Network inspector**: chassis diagrams for each UniFi switch. Click a port for its stats, LLDP neighbour and a side-by-side switch/server path check.
|
||||||
|
|
||||||
|

|
||||||
|
|
||||||
|
**One-click diagnostics**: GANDALF asks a PULSE worker to SSH to the host on the other end of the link and collect everything in one pass, then analyzes it: carrier, duplex and speed mismatches, SFP power levels, CRC errors, link flaps, recent kernel events and LLDP validation.
|
||||||
|
|
||||||
|

|
||||||
|
|
||||||
|
**Suppressions**: timed or manual maintenance windows per host, interface or UniFi device, with a full history.
|
||||||
|
|
||||||
|

|
||||||
|
|
||||||
|
**Light theme:**
|
||||||
|
|
||||||
|

|
||||||
|
|
||||||
|
<sub>Screenshots come from a throwaway local instance seeded with fictional data (documentation IP ranges), with PULSE replaced by a small mock. See `scripts/demo/`.</sub>
|
||||||
|
|
||||||
## Styling & Layout
|
## Styling & Layout
|
||||||
|
|
||||||
GANDALF uses the **LotusGuild Terminal Design System**. For all styling, component, and layout documentation see:
|
GANDALF uses the **LotusGuild Terminal Design System**. For all styling, component, and layout documentation see:
|
||||||
@@ -148,7 +194,7 @@ returned by the UniFi API for that device.
|
|||||||
|
|
||||||
| Condition | Priority |
|
| Condition | Priority |
|
||||||
|---|---|
|
|---|---|
|
||||||
| UniFi device offline (≥2 consecutive checks) | P2 High |
|
| UniFi device offline (≥`unifi_failure_threshold` consecutive checks) | P2 High |
|
||||||
| Proxmox host NIC link-down regression (≥2 consecutive checks) | P2 High |
|
| Proxmox host NIC link-down regression (≥2 consecutive checks) | P2 High |
|
||||||
| Host unreachable via ping (≥2 consecutive checks) | P2 High |
|
| Host unreachable via ping (≥2 consecutive checks) | P2 High |
|
||||||
| ≥3 hosts simultaneously reporting interface failures | P1 Critical |
|
| ≥3 hosts simultaneously reporting interface failures | P1 Critical |
|
||||||
@@ -218,6 +264,7 @@ Shared by both processes. Located in the working directory (`/var/www/html/prod/
|
|||||||
"monitor": {
|
"monitor": {
|
||||||
"poll_interval": 120,
|
"poll_interval": 120,
|
||||||
"failure_threshold": 2,
|
"failure_threshold": 2,
|
||||||
|
"unifi_failure_threshold": 5,
|
||||||
"cluster_threshold": 3,
|
"cluster_threshold": 3,
|
||||||
"ping_hosts": [
|
"ping_hosts": [
|
||||||
{ "name": "pbs", "ip": "10.10.10.3" }
|
{ "name": "pbs", "ip": "10.10.10.3" }
|
||||||
@@ -243,6 +290,7 @@ Shared by both processes. Located in the working directory (`/var/www/html/prod/
|
|||||||
| `hosts` | Maps Prometheus instance labels → display hostnames |
|
| `hosts` | Maps Prometheus instance labels → display hostnames |
|
||||||
| `monitor.poll_interval` | Seconds between full check cycles (default: 120) |
|
| `monitor.poll_interval` | Seconds between full check cycles (default: 120) |
|
||||||
| `monitor.failure_threshold` | Consecutive failures before creating ticket (default: 2) |
|
| `monitor.failure_threshold` | Consecutive failures before creating ticket (default: 2) |
|
||||||
|
| `monitor.unifi_failure_threshold` | Consecutive failures before ticketing a UniFi device as offline (default: same as `failure_threshold`). Set higher than `failure_threshold` — UniFi switches/APs can take several minutes to rejoin after a firmware-update reboot, so a low threshold cuts tickets for momentary maintenance reboots instead of real failures. |
|
||||||
| `monitor.cluster_threshold` | Hosts with failures to trigger cluster-wide P1 (default: 3) |
|
| `monitor.cluster_threshold` | Hosts with failures to trigger cluster-wide P1 (default: 3) |
|
||||||
| `monitor.ping_hosts` | Hosts checked only by ping (no node_exporter) |
|
| `monitor.ping_hosts` | Hosts checked only by ping (no node_exporter) |
|
||||||
|
|
||||||
|
|||||||
@@ -64,7 +64,7 @@ _diag_rate: dict = {}
|
|||||||
|
|
||||||
|
|
||||||
def _purge_old_jobs_loop():
|
def _purge_old_jobs_loop():
|
||||||
"""Background thread: remove stale diag jobs and run daily event purge."""
|
"""Background thread: remove stale diagnostic jobs and mark stuck ones done."""
|
||||||
while True:
|
while True:
|
||||||
time.sleep(120)
|
time.sleep(120)
|
||||||
cutoff = time.time() - 600
|
cutoff = time.time() - 600
|
||||||
@@ -155,6 +155,17 @@ def require_auth(f):
|
|||||||
return wrapper
|
return wrapper
|
||||||
|
|
||||||
|
|
||||||
|
def require_admin(f):
|
||||||
|
"""Decorator: require require_auth AND membership in the 'admin' group."""
|
||||||
|
@wraps(f)
|
||||||
|
def wrapper(*args, **kwargs):
|
||||||
|
user = _get_user()
|
||||||
|
if 'admin' not in user.get('groups', []):
|
||||||
|
return jsonify({'error': 'Admin access required'}), 403
|
||||||
|
return f(*args, **kwargs)
|
||||||
|
return wrapper
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# Helpers
|
# Helpers
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
@@ -163,17 +174,26 @@ _PAGE_LIMIT = 200 # max events returned per request
|
|||||||
|
|
||||||
|
|
||||||
def _annotate_suppressions(events: list, suppressions: list) -> None:
|
def _annotate_suppressions(events: list, suppressions: list) -> None:
|
||||||
"""Annotate each event dict in-place with an is_suppressed bool."""
|
"""Annotate each event dict in-place with an is_suppressed bool.
|
||||||
|
|
||||||
|
Mirrors the suppression check order in monitor.py exactly:
|
||||||
|
interface_down → interface OR host
|
||||||
|
unifi_device_* → unifi_device
|
||||||
|
everything else → host
|
||||||
|
"""
|
||||||
for ev in events:
|
for ev in events:
|
||||||
sup_type = (
|
etype = ev.get('event_type', '')
|
||||||
'unifi_device' if ev.get('event_type') == 'unifi_device_offline'
|
name = ev.get('target_name', '')
|
||||||
else 'interface' if ev.get('event_type') == 'interface_down'
|
detail = ev.get('target_detail', '') or ''
|
||||||
else 'host'
|
if etype == 'interface_down':
|
||||||
)
|
ev['is_suppressed'] = (
|
||||||
ev['is_suppressed'] = db.check_suppressed(
|
db.check_suppressed(suppressions, 'interface', name, detail) or
|
||||||
suppressions, sup_type,
|
db.check_suppressed(suppressions, 'host', name)
|
||||||
ev.get('target_name', ''), ev.get('target_detail', '') or '',
|
)
|
||||||
)
|
elif etype == 'unifi_device_offline':
|
||||||
|
ev['is_suppressed'] = db.check_suppressed(suppressions, 'unifi_device', name, detail)
|
||||||
|
else:
|
||||||
|
ev['is_suppressed'] = db.check_suppressed(suppressions, 'host', name, detail)
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
@@ -228,6 +248,7 @@ def inspector():
|
|||||||
|
|
||||||
@app.route('/suppressions')
|
@app.route('/suppressions')
|
||||||
@require_auth
|
@require_auth
|
||||||
|
@require_admin
|
||||||
def suppressions_page():
|
def suppressions_page():
|
||||||
user = _get_user()
|
user = _get_user()
|
||||||
active = db.get_active_suppressions()
|
active = db.get_active_suppressions()
|
||||||
@@ -323,6 +344,7 @@ def api_get_suppressions():
|
|||||||
|
|
||||||
@app.route('/api/suppressions', methods=['POST'])
|
@app.route('/api/suppressions', methods=['POST'])
|
||||||
@require_auth
|
@require_auth
|
||||||
|
@require_admin
|
||||||
def api_create_suppression():
|
def api_create_suppression():
|
||||||
user = _get_user()
|
user = _get_user()
|
||||||
data = request.get_json(silent=True) or {}
|
data = request.get_json(silent=True) or {}
|
||||||
@@ -371,6 +393,7 @@ def api_create_suppression():
|
|||||||
|
|
||||||
@app.route('/api/suppressions/<int:sup_id>', methods=['DELETE'])
|
@app.route('/api/suppressions/<int:sup_id>', methods=['DELETE'])
|
||||||
@require_auth
|
@require_auth
|
||||||
|
@require_admin
|
||||||
def api_delete_suppression(sup_id: int):
|
def api_delete_suppression(sup_id: int):
|
||||||
user = _get_user()
|
user = _get_user()
|
||||||
db.deactivate_suppression(sup_id)
|
db.deactivate_suppression(sup_id)
|
||||||
@@ -413,6 +436,7 @@ def api_diagnose_start():
|
|||||||
if pd.get('port_idx') == port_idx:
|
if pd.get('port_idx') == port_idx:
|
||||||
port_data = dict(pd)
|
port_data = dict(pd)
|
||||||
port_data['name'] = pname
|
port_data['name'] = pname
|
||||||
|
port_data['switch_name'] = switch_name
|
||||||
break
|
break
|
||||||
if not port_data:
|
if not port_data:
|
||||||
return jsonify({'error': f'Port {port_idx} not found on switch "{switch_name}"'}), 404
|
return jsonify({'error': f'Port {port_idx} not found on switch "{switch_name}"'}), 404
|
||||||
@@ -612,7 +636,8 @@ def api_avatar():
|
|||||||
avatar_data = avatar_data.encode('latin-1')
|
avatar_data = avatar_data.encode('latin-1')
|
||||||
if avatar_data[:3] != b'\xFF\xD8\xFF':
|
if avatar_data[:3] != b'\xFF\xD8\xFF':
|
||||||
logger.warning(f'Non-JPEG avatar data for {username}')
|
logger.warning(f'Non-JPEG avatar data for {username}')
|
||||||
open(sentinel, 'w').close()
|
with open(sentinel, 'w'):
|
||||||
|
pass
|
||||||
return '', 404
|
return '', 404
|
||||||
|
|
||||||
with open(cache_file, 'wb') as f:
|
with open(cache_file, 'wb') as f:
|
||||||
|
|||||||
@@ -3,7 +3,7 @@ import json
|
|||||||
import logging
|
import logging
|
||||||
import threading
|
import threading
|
||||||
from contextlib import contextmanager
|
from contextlib import contextmanager
|
||||||
from datetime import datetime, timedelta
|
from datetime import datetime, timedelta, timezone
|
||||||
from typing import Optional
|
from typing import Optional
|
||||||
|
|
||||||
import pymysql
|
import pymysql
|
||||||
@@ -114,12 +114,12 @@ def upsert_event(
|
|||||||
target_detail: str,
|
target_detail: str,
|
||||||
description: str,
|
description: str,
|
||||||
) -> tuple:
|
) -> tuple:
|
||||||
"""Insert or update a network event. Returns (id, is_new, consecutive_failures)."""
|
"""Insert or update a network event. Returns (id, is_new, consecutive_failures, ticket_id)."""
|
||||||
detail = target_detail or ''
|
detail = target_detail or ''
|
||||||
with get_conn() as conn:
|
with get_conn() as conn:
|
||||||
with conn.cursor() as cur:
|
with conn.cursor() as cur:
|
||||||
cur.execute(
|
cur.execute(
|
||||||
"""SELECT id, consecutive_failures FROM network_events
|
"""SELECT id, consecutive_failures, ticket_id FROM network_events
|
||||||
WHERE event_type=%s AND target_name=%s AND target_detail=%s
|
WHERE event_type=%s AND target_name=%s AND target_detail=%s
|
||||||
AND resolved_at IS NULL LIMIT 1""",
|
AND resolved_at IS NULL LIMIT 1""",
|
||||||
(event_type, target_name, detail),
|
(event_type, target_name, detail),
|
||||||
@@ -134,7 +134,7 @@ def upsert_event(
|
|||||||
WHERE id=%s""",
|
WHERE id=%s""",
|
||||||
(new_count, description, existing['id']),
|
(new_count, description, existing['id']),
|
||||||
)
|
)
|
||||||
return existing['id'], False, new_count
|
return existing['id'], False, new_count, existing.get('ticket_id')
|
||||||
else:
|
else:
|
||||||
cur.execute(
|
cur.execute(
|
||||||
"""INSERT INTO network_events
|
"""INSERT INTO network_events
|
||||||
@@ -142,7 +142,7 @@ def upsert_event(
|
|||||||
VALUES (%s, %s, %s, %s, %s, %s)""",
|
VALUES (%s, %s, %s, %s, %s, %s)""",
|
||||||
(event_type, severity, source_type, target_name, detail, description),
|
(event_type, severity, source_type, target_name, detail, description),
|
||||||
)
|
)
|
||||||
return cur.lastrowid, True, 1
|
return cur.lastrowid, True, 1, None
|
||||||
|
|
||||||
|
|
||||||
def resolve_event(event_type: str, target_name: str, target_detail: str = '') -> None:
|
def resolve_event(event_type: str, target_name: str, target_detail: str = '') -> None:
|
||||||
@@ -182,7 +182,7 @@ def get_active_events(limit: int = 200, offset: int = 0) -> list:
|
|||||||
for r in rows:
|
for r in rows:
|
||||||
for k in ('first_seen', 'last_seen'):
|
for k in ('first_seen', 'last_seen'):
|
||||||
if r.get(k) and hasattr(r[k], 'isoformat'):
|
if r.get(k) and hasattr(r[k], 'isoformat'):
|
||||||
r[k] = r[k].isoformat()
|
r[k] = r[k].isoformat() + 'Z'
|
||||||
return rows
|
return rows
|
||||||
|
|
||||||
|
|
||||||
@@ -210,7 +210,7 @@ def get_recent_resolved(hours: int = 24, limit: int = 50) -> list:
|
|||||||
for r in rows:
|
for r in rows:
|
||||||
for k in ('first_seen', 'last_seen', 'resolved_at'):
|
for k in ('first_seen', 'last_seen', 'resolved_at'):
|
||||||
if r.get(k) and hasattr(r[k], 'isoformat'):
|
if r.get(k) and hasattr(r[k], 'isoformat'):
|
||||||
r[k] = r[k].isoformat()
|
r[k] = r[k].isoformat() + 'Z'
|
||||||
return rows
|
return rows
|
||||||
|
|
||||||
|
|
||||||
@@ -252,7 +252,7 @@ def get_active_suppressions() -> list:
|
|||||||
for r in rows:
|
for r in rows:
|
||||||
for k in ('created_at', 'expires_at'):
|
for k in ('created_at', 'expires_at'):
|
||||||
if r.get(k) and hasattr(r[k], 'isoformat'):
|
if r.get(k) and hasattr(r[k], 'isoformat'):
|
||||||
r[k] = r[k].isoformat()
|
r[k] = r[k].isoformat() + 'Z'
|
||||||
return rows
|
return rows
|
||||||
|
|
||||||
|
|
||||||
@@ -267,7 +267,7 @@ def get_suppression_history(limit: int = 50) -> list:
|
|||||||
for r in rows:
|
for r in rows:
|
||||||
for k in ('created_at', 'expires_at'):
|
for k in ('created_at', 'expires_at'):
|
||||||
if r.get(k) and hasattr(r[k], 'isoformat'):
|
if r.get(k) and hasattr(r[k], 'isoformat'):
|
||||||
r[k] = r[k].isoformat()
|
r[k] = r[k].isoformat() + 'Z'
|
||||||
return rows
|
return rows
|
||||||
|
|
||||||
|
|
||||||
@@ -281,7 +281,7 @@ def create_suppression(
|
|||||||
) -> int:
|
) -> int:
|
||||||
expires_at = None
|
expires_at = None
|
||||||
if expires_minutes:
|
if expires_minutes:
|
||||||
expires_at = datetime.utcnow() + timedelta(minutes=int(expires_minutes))
|
expires_at = datetime.now(timezone.utc) + timedelta(minutes=int(expires_minutes))
|
||||||
with get_conn() as conn:
|
with get_conn() as conn:
|
||||||
with conn.cursor() as cur:
|
with conn.cursor() as cur:
|
||||||
cur.execute(
|
cur.execute(
|
||||||
|
|||||||
@@ -68,7 +68,7 @@ class DiagnosticsRunner:
|
|||||||
f' echo "=== ip_route ===";'
|
f' echo "=== ip_route ===";'
|
||||||
f' ip route show dev {q} 2>/dev/null;'
|
f' ip route show dev {q} 2>/dev/null;'
|
||||||
f' echo "=== dmesg ===";'
|
f' echo "=== dmesg ===";'
|
||||||
f' dmesg 2>/dev/null | grep {q} | tail -50;'
|
f' dmesg 2>/dev/null | grep -F -- {q} | tail -50;'
|
||||||
f' echo "=== lldpctl ===";'
|
f' echo "=== lldpctl ===";'
|
||||||
f' lldpctl 2>/dev/null || echo "lldpd not running";'
|
f' lldpctl 2>/dev/null || echo "lldpd not running";'
|
||||||
f' echo "=== end ==="'
|
f' echo "=== end ==="'
|
||||||
@@ -78,7 +78,7 @@ class DiagnosticsRunner:
|
|||||||
f'ssh -o StrictHostKeyChecking=accept-new -o ConnectTimeout=5 '
|
f'ssh -o StrictHostKeyChecking=accept-new -o ConnectTimeout=5 '
|
||||||
f'-o BatchMode=yes -o LogLevel=ERROR '
|
f'-o BatchMode=yes -o LogLevel=ERROR '
|
||||||
f'-o ServerAliveInterval=10 -o ServerAliveCountMax=2 '
|
f'-o ServerAliveInterval=10 -o ServerAliveCountMax=2 '
|
||||||
f'root@{ip_q} \'{remote_cmd}\''
|
f'root@{ip_q} {shlex.quote(remote_cmd)}'
|
||||||
)
|
)
|
||||||
|
|
||||||
# ------------------------------------------------------------------
|
# ------------------------------------------------------------------
|
||||||
@@ -530,15 +530,17 @@ class DiagnosticsRunner:
|
|||||||
add(warnings, 'KERNEL_EVENTS',
|
add(warnings, 'KERNEL_EVENTS',
|
||||||
f'{len(err_events)} recent kernel error event(s) for this interface in dmesg')
|
f'{len(err_events)} recent kernel error event(s) for this interface in dmesg')
|
||||||
|
|
||||||
# LLDP validation
|
# LLDP validation: the neighbour the *server's* lldpd sees should be the switch
|
||||||
|
# that this port belongs to. (The switch port's own LLDP entry describes the server,
|
||||||
|
# so it must not be compared with the server's view of the switch.)
|
||||||
if lldp.get('available'):
|
if lldp.get('available'):
|
||||||
sw_lldp = switch_port_data.get('lldp') or {}
|
expected_switch = (switch_port_data.get('switch_name') or '').lower()
|
||||||
sw_system = (sw_lldp.get('system_name') or '').lower()
|
|
||||||
srv_neighbor = (lldp.get('neighbor_system') or '').lower()
|
srv_neighbor = (lldp.get('neighbor_system') or '').lower()
|
||||||
if sw_system and srv_neighbor and sw_system not in srv_neighbor and srv_neighbor not in sw_system:
|
names_differ = expected_switch not in srv_neighbor and srv_neighbor not in expected_switch
|
||||||
|
if expected_switch and srv_neighbor and names_differ:
|
||||||
add(warnings, 'LLDP_MISMATCH',
|
add(warnings, 'LLDP_MISMATCH',
|
||||||
f'LLDP mismatch: switch sees "{sw_lldp.get("system_name")}" but '
|
f'LLDP mismatch: this port is on "{switch_port_data.get("switch_name")}" but the server\'s '
|
||||||
f'server lldpctl sees "{lldp.get("neighbor_system")}" — cross-cabled port?')
|
f'lldpctl sees "{lldp.get("neighbor_system")}" — cross-cabled port?')
|
||||||
else:
|
else:
|
||||||
add(info, 'LLDP_MISSING',
|
add(info, 'LLDP_MISSING',
|
||||||
'lldpd not running on server — install lldpd for full path validation')
|
'lldpd not running on server — install lldpd for full path validation')
|
||||||
|
|||||||
|
After Width: | Height: | Size: 88 KiB |
|
After Width: | Height: | Size: 164 KiB |
|
After Width: | Height: | Size: 674 KiB |
|
After Width: | Height: | Size: 67 KiB |
|
After Width: | Height: | Size: 145 KiB |
|
After Width: | Height: | Size: 162 KiB |
|
After Width: | Height: | Size: 108 KiB |
|
After Width: | Height: | Size: 49 KiB |
|
After Width: | Height: | Size: 71 KiB |
|
After Width: | Height: | Size: 206 KiB |
|
After Width: | Height: | Size: 145 KiB |
|
After Width: | Height: | Size: 214 KiB |
|
After Width: | Height: | Size: 90 KiB |
@@ -12,7 +12,7 @@ import logging
|
|||||||
import re
|
import re
|
||||||
import shlex
|
import shlex
|
||||||
import time
|
import time
|
||||||
from datetime import datetime
|
from datetime import datetime, timezone
|
||||||
from typing import Dict, List, Optional
|
from typing import Dict, List, Optional
|
||||||
|
|
||||||
import requests
|
import requests
|
||||||
@@ -215,7 +215,10 @@ class TicketClient:
|
|||||||
resp.raise_for_status()
|
resp.raise_for_status()
|
||||||
data = resp.json()
|
data = resp.json()
|
||||||
if data.get('success'):
|
if data.get('success'):
|
||||||
tid = data['ticket_id']
|
tid = data.get('ticket_id')
|
||||||
|
if not tid:
|
||||||
|
logger.warning(f'Ticket API success but no ticket_id in response: {data}')
|
||||||
|
return None
|
||||||
logger.info(f'Created ticket #{tid}: {title}')
|
logger.info(f'Created ticket #{tid}: {title}')
|
||||||
return tid
|
return tid
|
||||||
if data.get('existing_ticket_id'):
|
if data.get('existing_ticket_id'):
|
||||||
@@ -377,7 +380,7 @@ class LinkStatsCollector:
|
|||||||
f'ssh -o StrictHostKeyChecking=accept-new -o ConnectTimeout=5 '
|
f'ssh -o StrictHostKeyChecking=accept-new -o ConnectTimeout=5 '
|
||||||
f'-o BatchMode=yes -o LogLevel=ERROR '
|
f'-o BatchMode=yes -o LogLevel=ERROR '
|
||||||
f'-o ServerAliveInterval=10 -o ServerAliveCountMax=2 '
|
f'-o ServerAliveInterval=10 -o ServerAliveCountMax=2 '
|
||||||
f'root@{ip} "{shell_cmd}"'
|
f'root@{ip} {shlex.quote(shell_cmd)}'
|
||||||
)
|
)
|
||||||
output = self.pulse.run_command(ssh_cmd)
|
output = self.pulse.run_command(ssh_cmd)
|
||||||
if output is None:
|
if output is None:
|
||||||
@@ -615,7 +618,7 @@ class LinkStatsCollector:
|
|||||||
return {
|
return {
|
||||||
'hosts': result_hosts,
|
'hosts': result_hosts,
|
||||||
'unifi_switches': unifi_switches,
|
'unifi_switches': unifi_switches,
|
||||||
'updated': datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S UTC'),
|
'updated': datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M:%S UTC'),
|
||||||
}
|
}
|
||||||
|
|
||||||
def _compute_unifi_rates(self, raw: Dict[str, dict], now: float) -> Dict[str, dict]:
|
def _compute_unifi_rates(self, raw: Dict[str, dict], now: float) -> Dict[str, dict]:
|
||||||
@@ -650,7 +653,7 @@ class LinkStatsCollector:
|
|||||||
# Helpers
|
# Helpers
|
||||||
# --------------------------------------------------------------------------
|
# --------------------------------------------------------------------------
|
||||||
def _now_utc() -> str:
|
def _now_utc() -> str:
|
||||||
return datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S UTC')
|
return datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M:%S UTC')
|
||||||
|
|
||||||
|
|
||||||
# --------------------------------------------------------------------------
|
# --------------------------------------------------------------------------
|
||||||
@@ -674,6 +677,11 @@ class NetworkMonitor:
|
|||||||
mon = self.cfg.get('monitor', {})
|
mon = self.cfg.get('monitor', {})
|
||||||
self.poll_interval = mon.get('poll_interval', 120)
|
self.poll_interval = mon.get('poll_interval', 120)
|
||||||
self.fail_thresh = mon.get('failure_threshold', 2)
|
self.fail_thresh = mon.get('failure_threshold', 2)
|
||||||
|
# UniFi gear (switches/APs) can take several minutes to rejoin after a
|
||||||
|
# firmware-update reboot, much longer than a Proxmox host link flap.
|
||||||
|
# Use a separate, more forgiving threshold so those momentary reboots
|
||||||
|
# don't cut tickets, while still catching genuine device failures.
|
||||||
|
self.unifi_fail_thresh = mon.get('unifi_failure_threshold', self.fail_thresh)
|
||||||
self.cluster_thresh = mon.get('cluster_threshold', 3)
|
self.cluster_thresh = mon.get('cluster_threshold', 3)
|
||||||
self.cluster_name = mon.get('cluster_name', CLUSTER_NAME)
|
self.cluster_name = mon.get('cluster_name', CLUSTER_NAME)
|
||||||
|
|
||||||
@@ -725,13 +733,13 @@ class NetworkMonitor:
|
|||||||
db.check_suppressed(suppressions, 'interface', host, iface) or
|
db.check_suppressed(suppressions, 'interface', host, iface) or
|
||||||
db.check_suppressed(suppressions, 'host', host)
|
db.check_suppressed(suppressions, 'host', host)
|
||||||
)
|
)
|
||||||
event_id, is_new, consec = db.upsert_event(
|
event_id, is_new, consec, ticket_id = db.upsert_event(
|
||||||
'interface_down', 'critical', 'prometheus',
|
'interface_down', 'critical', 'prometheus',
|
||||||
host, iface,
|
host, iface,
|
||||||
f'Interface {iface} on {host} went link-down ({_now_utc()})',
|
f'Interface {iface} on {host} went link-down ({_now_utc()})',
|
||||||
)
|
)
|
||||||
if not sup and consec >= self.fail_thresh:
|
if not sup and consec >= self.fail_thresh and not ticket_id:
|
||||||
self._ticket_interface(event_id, is_new, host, iface, consec)
|
self._ticket_interface(event_id, host, iface, consec)
|
||||||
|
|
||||||
if host_has_regression:
|
if host_has_regression:
|
||||||
hosts_with_regression.append(host)
|
hosts_with_regression.append(host)
|
||||||
@@ -741,13 +749,13 @@ class NetworkMonitor:
|
|||||||
# Cluster-wide check – only genuine regressions count
|
# Cluster-wide check – only genuine regressions count
|
||||||
if len(hosts_with_regression) >= self.cluster_thresh:
|
if len(hosts_with_regression) >= self.cluster_thresh:
|
||||||
sup = db.check_suppressed(suppressions, 'all', '')
|
sup = db.check_suppressed(suppressions, 'all', '')
|
||||||
event_id, is_new, consec = db.upsert_event(
|
event_id, is_new, consec, ticket_id = db.upsert_event(
|
||||||
'cluster_network_issue', 'critical', 'prometheus',
|
'cluster_network_issue', 'critical', 'prometheus',
|
||||||
self.cluster_name, '',
|
self.cluster_name, '',
|
||||||
f'{len(hosts_with_regression)} hosts reporting simultaneous interface failures: '
|
f'{len(hosts_with_regression)} hosts reporting simultaneous interface failures: '
|
||||||
f'{", ".join(hosts_with_regression)}',
|
f'{", ".join(hosts_with_regression)}',
|
||||||
)
|
)
|
||||||
if not sup and is_new:
|
if not sup and (is_new or not ticket_id):
|
||||||
title = (
|
title = (
|
||||||
f'[{self.cluster_name}][auto][production][issue][network][cluster-wide] '
|
f'[{self.cluster_name}][auto][production][issue][network][cluster-wide] '
|
||||||
f'Multiple hosts reporting interface failures'
|
f'Multiple hosts reporting interface failures'
|
||||||
@@ -768,7 +776,7 @@ class NetworkMonitor:
|
|||||||
db.resolve_event('cluster_network_issue', self.cluster_name, '')
|
db.resolve_event('cluster_network_issue', self.cluster_name, '')
|
||||||
|
|
||||||
def _ticket_interface(
|
def _ticket_interface(
|
||||||
self, event_id: int, is_new: bool, host: str, iface: str, consec: int
|
self, event_id: int, host: str, iface: str, consec: int
|
||||||
) -> None:
|
) -> None:
|
||||||
title = (
|
title = (
|
||||||
f'[{host}][auto][production][issue][network][single-node] '
|
f'[{host}][auto][production][issue][network][single-node] '
|
||||||
@@ -786,7 +794,7 @@ class NetworkMonitor:
|
|||||||
f'Please inspect the cable/SFP/switch port for {host}/{iface}.'
|
f'Please inspect the cable/SFP/switch port for {host}/{iface}.'
|
||||||
)
|
)
|
||||||
tid = self.tickets.create(title, desc, priority='2')
|
tid = self.tickets.create(title, desc, priority='2')
|
||||||
if tid and is_new:
|
if tid:
|
||||||
db.set_ticket_id(event_id, tid)
|
db.set_ticket_id(event_id, tid)
|
||||||
|
|
||||||
# ------------------------------------------------------------------
|
# ------------------------------------------------------------------
|
||||||
@@ -801,17 +809,17 @@ class NetworkMonitor:
|
|||||||
name = d['name']
|
name = d['name']
|
||||||
if not d['connected']:
|
if not d['connected']:
|
||||||
sup = db.check_suppressed(suppressions, 'unifi_device', name)
|
sup = db.check_suppressed(suppressions, 'unifi_device', name)
|
||||||
event_id, is_new, consec = db.upsert_event(
|
event_id, is_new, consec, ticket_id = db.upsert_event(
|
||||||
'unifi_device_offline', 'critical', 'unifi',
|
'unifi_device_offline', 'critical', 'unifi',
|
||||||
name, d.get('type', ''),
|
name, d.get('type', ''),
|
||||||
f'UniFi {name} ({d.get("ip","")}) offline ({_now_utc()})',
|
f'UniFi {name} ({d.get("ip","")}) offline ({_now_utc()})',
|
||||||
)
|
)
|
||||||
if not sup and consec >= self.fail_thresh:
|
if not sup and consec >= self.unifi_fail_thresh and not ticket_id:
|
||||||
self._ticket_unifi(event_id, is_new, d)
|
self._ticket_unifi(event_id, d)
|
||||||
else:
|
else:
|
||||||
db.resolve_event('unifi_device_offline', name, d.get('type', ''))
|
db.resolve_event('unifi_device_offline', name, d.get('type', ''))
|
||||||
|
|
||||||
def _ticket_unifi(self, event_id: int, is_new: bool, device: dict) -> None:
|
def _ticket_unifi(self, event_id: int, device: dict) -> None:
|
||||||
name = device['name']
|
name = device['name']
|
||||||
title = (
|
title = (
|
||||||
f'[{name}][auto][production][issue][network][single-node] '
|
f'[{name}][auto][production][issue][network][single-node] '
|
||||||
@@ -828,31 +836,31 @@ class NetworkMonitor:
|
|||||||
f'Please check power and cable connectivity.'
|
f'Please check power and cable connectivity.'
|
||||||
)
|
)
|
||||||
tid = self.tickets.create(title, desc, priority='2')
|
tid = self.tickets.create(title, desc, priority='2')
|
||||||
if tid and is_new:
|
if tid:
|
||||||
db.set_ticket_id(event_id, tid)
|
db.set_ticket_id(event_id, tid)
|
||||||
|
|
||||||
# ------------------------------------------------------------------
|
# ------------------------------------------------------------------
|
||||||
# Ping-only hosts (no node_exporter)
|
# Ping-only hosts (no node_exporter)
|
||||||
# ------------------------------------------------------------------
|
# ------------------------------------------------------------------
|
||||||
def _process_ping_hosts(self, suppressions: list) -> None:
|
def _process_ping_hosts(self, suppressions: list, ping_states: Dict[str, bool]) -> None:
|
||||||
for h in self.cfg.get('monitor', {}).get('ping_hosts', []):
|
for h in self.cfg.get('monitor', {}).get('ping_hosts', []):
|
||||||
name, ip = h['name'], h['ip']
|
name, ip = h['name'], h['ip']
|
||||||
reachable = self.pulse.ping(ip)
|
reachable = ping_states.get(name, False)
|
||||||
|
|
||||||
if not reachable:
|
if not reachable:
|
||||||
sup = db.check_suppressed(suppressions, 'host', name)
|
sup = db.check_suppressed(suppressions, 'host', name)
|
||||||
event_id, is_new, consec = db.upsert_event(
|
event_id, is_new, consec, ticket_id = db.upsert_event(
|
||||||
'host_unreachable', 'critical', 'ping',
|
'host_unreachable', 'critical', 'ping',
|
||||||
name, ip,
|
name, ip,
|
||||||
f'Host {name} ({ip}) unreachable via ping ({_now_utc()})',
|
f'Host {name} ({ip}) unreachable via ping ({_now_utc()})',
|
||||||
)
|
)
|
||||||
if not sup and consec >= self.fail_thresh:
|
if not sup and consec >= self.fail_thresh and not ticket_id:
|
||||||
self._ticket_unreachable(event_id, is_new, name, ip, consec)
|
self._ticket_unreachable(event_id, name, ip, consec)
|
||||||
else:
|
else:
|
||||||
db.resolve_event('host_unreachable', name, ip)
|
db.resolve_event('host_unreachable', name, ip)
|
||||||
|
|
||||||
def _ticket_unreachable(
|
def _ticket_unreachable(
|
||||||
self, event_id: int, is_new: bool, name: str, ip: str, consec: int
|
self, event_id: int, name: str, ip: str, consec: int
|
||||||
) -> None:
|
) -> None:
|
||||||
title = (
|
title = (
|
||||||
f'[{name}][auto][production][issue][network][single-node] '
|
f'[{name}][auto][production][issue][network][single-node] '
|
||||||
@@ -870,7 +878,7 @@ class NetworkMonitor:
|
|||||||
f'Please check the host power, management interface, and network connectivity.'
|
f'Please check the host power, management interface, and network connectivity.'
|
||||||
)
|
)
|
||||||
tid = self.tickets.create(title, desc, priority='2')
|
tid = self.tickets.create(title, desc, priority='2')
|
||||||
if tid and is_new:
|
if tid:
|
||||||
db.set_ticket_id(event_id, tid)
|
db.set_ticket_id(event_id, tid)
|
||||||
|
|
||||||
# ------------------------------------------------------------------
|
# ------------------------------------------------------------------
|
||||||
@@ -879,6 +887,7 @@ class NetworkMonitor:
|
|||||||
def _collect_snapshot(
|
def _collect_snapshot(
|
||||||
self, iface_states: Dict[str, Dict[str, bool]],
|
self, iface_states: Dict[str, Dict[str, bool]],
|
||||||
unifi_devices: Optional[List[dict]] = None,
|
unifi_devices: Optional[List[dict]] = None,
|
||||||
|
ping_states: Optional[Dict[str, bool]] = None,
|
||||||
) -> dict:
|
) -> dict:
|
||||||
# Accept pre-fetched devices; fall back to empty list if unavailable
|
# Accept pre-fetched devices; fall back to empty list if unavailable
|
||||||
display_unifi = unifi_devices if unifi_devices is not None else []
|
display_unifi = unifi_devices if unifi_devices is not None else []
|
||||||
@@ -907,7 +916,7 @@ class NetworkMonitor:
|
|||||||
|
|
||||||
for h in self.cfg.get('monitor', {}).get('ping_hosts', []):
|
for h in self.cfg.get('monitor', {}).get('ping_hosts', []):
|
||||||
name, ip = h['name'], h['ip']
|
name, ip = h['name'], h['ip']
|
||||||
reachable = self.pulse.ping(ip, count=1, timeout=2)
|
reachable = (ping_states or {}).get(name, False)
|
||||||
hosts[name] = {
|
hosts[name] = {
|
||||||
'ip': ip,
|
'ip': ip,
|
||||||
'interfaces': {},
|
'interfaces': {},
|
||||||
@@ -918,7 +927,7 @@ class NetworkMonitor:
|
|||||||
return {
|
return {
|
||||||
'hosts': hosts,
|
'hosts': hosts,
|
||||||
'unifi': display_unifi,
|
'unifi': display_unifi,
|
||||||
'updated': datetime.utcnow().isoformat(),
|
'updated': datetime.now(timezone.utc).isoformat().replace('+00:00', 'Z'),
|
||||||
}
|
}
|
||||||
|
|
||||||
# ------------------------------------------------------------------
|
# ------------------------------------------------------------------
|
||||||
@@ -927,7 +936,7 @@ class NetworkMonitor:
|
|||||||
def run(self) -> None:
|
def run(self) -> None:
|
||||||
logger.info(
|
logger.info(
|
||||||
f'Gandalf monitor started – poll_interval={self.poll_interval}s '
|
f'Gandalf monitor started – poll_interval={self.poll_interval}s '
|
||||||
f'fail_thresh={self.fail_thresh}'
|
f'fail_thresh={self.fail_thresh} unifi_fail_thresh={self.unifi_fail_thresh}'
|
||||||
)
|
)
|
||||||
while True:
|
while True:
|
||||||
try:
|
try:
|
||||||
@@ -939,8 +948,14 @@ class NetworkMonitor:
|
|||||||
# 2. Fetch UniFi devices once — used by both snapshot and alert processing
|
# 2. Fetch UniFi devices once — used by both snapshot and alert processing
|
||||||
unifi_devices = self.unifi.get_devices()
|
unifi_devices = self.unifi.get_devices()
|
||||||
|
|
||||||
# 3. Collect and store snapshot for dashboard
|
# 3a. Ping-only hosts once — shared by snapshot and alert processing
|
||||||
snapshot = self._collect_snapshot(iface_states, unifi_devices)
|
ping_states: Dict[str, bool] = {
|
||||||
|
h['name']: self.pulse.ping(h['ip'])
|
||||||
|
for h in self.cfg.get('monitor', {}).get('ping_hosts', [])
|
||||||
|
}
|
||||||
|
|
||||||
|
# 3b. Collect and store snapshot for dashboard
|
||||||
|
snapshot = self._collect_snapshot(iface_states, unifi_devices, ping_states)
|
||||||
db.set_state('network_snapshot', snapshot)
|
db.set_state('network_snapshot', snapshot)
|
||||||
db.set_state('last_check', _now_utc())
|
db.set_state('last_check', _now_utc())
|
||||||
|
|
||||||
@@ -956,7 +971,7 @@ class NetworkMonitor:
|
|||||||
self._process_interfaces(iface_states, suppressions)
|
self._process_interfaces(iface_states, suppressions)
|
||||||
self._process_unifi(unifi_devices, suppressions)
|
self._process_unifi(unifi_devices, suppressions)
|
||||||
|
|
||||||
self._process_ping_hosts(suppressions)
|
self._process_ping_hosts(suppressions, ping_states)
|
||||||
|
|
||||||
# Housekeeping: deactivate expired suppressions and purge old resolved events
|
# Housekeeping: deactivate expired suppressions and purge old resolved events
|
||||||
db.cleanup_expired_suppressions()
|
db.cleanup_expired_suppressions()
|
||||||
@@ -967,6 +982,7 @@ class NetworkMonitor:
|
|||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f'Monitor loop error: {e}', exc_info=True)
|
logger.error(f'Monitor loop error: {e}', exc_info=True)
|
||||||
time.sleep(30)
|
time.sleep(30)
|
||||||
|
continue
|
||||||
|
|
||||||
time.sleep(self.poll_interval)
|
time.sleep(self.poll_interval)
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,19 @@
|
|||||||
|
# Demo environment (for README screenshots)
|
||||||
|
|
||||||
|
Fictional data only (RFC 5737 documentation addresses). A throwaway MariaDB on loopback, the Flask app, and a tiny mock of the PULSE internal API so the **Run Diagnostics** button works.
|
||||||
|
|
||||||
|
```bash
|
||||||
|
D=$(mktemp -d); rsync -a --exclude .git . $D/gandalf && cd $D
|
||||||
|
mariadb-install-db --no-defaults --datadir=$D/db --auth-root-authentication-method=normal --skip-test-db
|
||||||
|
mariadbd --no-defaults --datadir=$D/db --socket=$D/sock --port=3398 --bind-address=127.0.0.1 --user=root &
|
||||||
|
mariadb --no-defaults -S $D/sock -e "CREATE USER 'gandalf'@'127.0.0.1' IDENTIFIED BY 'demo-only'" < /dev/null
|
||||||
|
mariadb --no-defaults -S $D/sock < gandalf/schema.sql
|
||||||
|
mariadb --no-defaults -S $D/sock -e "GRANT ALL ON gandalf.* TO 'gandalf'@'127.0.0.1'"
|
||||||
|
# gandalf/config.json: database -> 127.0.0.1:3398 gandalf/demo-only; pulse.url -> http://127.0.0.1:8650 (api_key/worker_id any value)
|
||||||
|
python3 gandalf/scripts/demo/seed.py # events, suppressions, snapshot and link stats
|
||||||
|
python3 gandalf/scripts/demo/mock_pulse.py & # canned diagnostics output
|
||||||
|
(cd gandalf && gunicorn -b 127.0.0.1:8500 app:app) &
|
||||||
|
python3 gandalf/scripts/demo/capture.py # screenshots + GIF -> docs/img/
|
||||||
|
```
|
||||||
|
|
||||||
|
The app trusts Authelia `Remote-User` / `Remote-Groups` headers, so `capture.py` sends them directly. Requires `playwright` and `pillow`.
|
||||||
@@ -0,0 +1,164 @@
|
|||||||
|
"""Regenerates docs/img/* from a LOCAL demo instance (fictional data). See README.md in this folder."""
|
||||||
|
|
||||||
|
# flake8: noqa: E501 (demo data and canned output contain long literal lines)
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
import io
|
||||||
|
import pathlib
|
||||||
|
import sys
|
||||||
|
|
||||||
|
from PIL import Image
|
||||||
|
from playwright.async_api import async_playwright
|
||||||
|
|
||||||
|
OUT = str(pathlib.Path(__file__).resolve().parents[2] / "docs" / "img")
|
||||||
|
B = "http://127.0.0.1:8500"
|
||||||
|
H = {
|
||||||
|
"Remote-User": "alex.rivera",
|
||||||
|
"Remote-Name": "Alex Rivera",
|
||||||
|
"Remote-Email": "alex@example.com",
|
||||||
|
"Remote-Groups": "admin",
|
||||||
|
}
|
||||||
|
TOP = (
|
||||||
|
"document.activeElement&&document.activeElement.blur();"
|
||||||
|
"document.documentElement.style.scrollBehavior='auto';window.scrollTo(0,0)"
|
||||||
|
)
|
||||||
|
AV = (
|
||||||
|
'<svg xmlns="http://www.w3.org/2000/svg" width="64" height="64">'
|
||||||
|
'<rect width="64" height="64" fill="#050a10"/>'
|
||||||
|
'<text x="32" y="40" font-family="monospace" font-size="24" font-weight="700" fill="#00d4ff" text-anchor="middle">AR</text></svg>'
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
async def mk(p, theme="dark", w=1440, h=900):
|
||||||
|
b = await p.chromium.launch()
|
||||||
|
c = await b.new_context(viewport={"width": w, "height": h}, extra_http_headers=H)
|
||||||
|
await c.add_init_script(f"try{{localStorage.setItem('lt_theme','{theme}')}}catch(e){{}}")
|
||||||
|
|
||||||
|
async def av(route):
|
||||||
|
await route.fulfill(status=200, content_type="image/svg+xml", body=AV)
|
||||||
|
|
||||||
|
await c.route("**/api/avatar*", av)
|
||||||
|
return b, c
|
||||||
|
|
||||||
|
|
||||||
|
async def go(pg, path, wait=5500):
|
||||||
|
await pg.goto(B + path)
|
||||||
|
await pg.wait_for_timeout(wait)
|
||||||
|
await pg.evaluate(TOP)
|
||||||
|
await pg.wait_for_timeout(300)
|
||||||
|
|
||||||
|
|
||||||
|
async def el(pg, sel, name, idx=0):
|
||||||
|
loc = pg.locator(sel).nth(idx)
|
||||||
|
await loc.scroll_into_view_if_needed()
|
||||||
|
await pg.wait_for_timeout(300)
|
||||||
|
await loc.screenshot(path=f"{OUT}/{name}.png")
|
||||||
|
print(name, "ok")
|
||||||
|
|
||||||
|
|
||||||
|
async def select_port(pg, sw, idx):
|
||||||
|
await pg.locator(f'[data-action="select-port"][data-switch="{sw}"][data-port-idx="{idx}"]').first.click()
|
||||||
|
await pg.wait_for_timeout(1200)
|
||||||
|
|
||||||
|
|
||||||
|
async def shots():
|
||||||
|
async with async_playwright() as p:
|
||||||
|
b, c = await mk(p)
|
||||||
|
pg = await c.new_page()
|
||||||
|
errs = []
|
||||||
|
pg.on("pageerror", lambda e: errs.append(str(e)[:100]))
|
||||||
|
await go(pg, "/")
|
||||||
|
await pg.screenshot(path=f"{OUT}/dashboard.png")
|
||||||
|
print("dashboard ok")
|
||||||
|
await el(pg, "#topology-diagram", "topology")
|
||||||
|
await el(pg, ".lt-section-header >> text=ALERT QUEUE", "alerts-tmp") if False else None
|
||||||
|
# host cards + unifi table via element crops
|
||||||
|
hb = (
|
||||||
|
await pg.locator("#hosts-grid, .host-grid, .hosts-grid").first.bounding_box()
|
||||||
|
if await pg.locator("#hosts-grid, .host-grid, .hosts-grid").count()
|
||||||
|
else None
|
||||||
|
)
|
||||||
|
if hb:
|
||||||
|
await pg.locator("#hosts-grid, .host-grid, .hosts-grid").first.screenshot(path=f"{OUT}/host-cards.png")
|
||||||
|
print("host-cards ok")
|
||||||
|
await go(pg, "/links")
|
||||||
|
await pg.screenshot(path=f"{OUT}/link-debug.png")
|
||||||
|
names = await pg.evaluate("[...document.querySelectorAll('.link-host-name')].map(e=>e.textContent.trim())")
|
||||||
|
print("hosts", names)
|
||||||
|
for nm, fn in (
|
||||||
|
("large1", "link-debug-sfp-warning"),
|
||||||
|
("compute-storage-gpu-01", "link-debug-sfp"),
|
||||||
|
("Pro 24 PoE", "link-debug-switch"),
|
||||||
|
):
|
||||||
|
i = next((k for k, n in enumerate(names) if nm in n), None)
|
||||||
|
if i is not None:
|
||||||
|
await el(pg, ".link-host-panel", fn, i)
|
||||||
|
await go(pg, "/inspector")
|
||||||
|
await pg.screenshot(path=f"{OUT}/inspector.png")
|
||||||
|
await select_port(pg, "USW-Aggregation", 4)
|
||||||
|
await pg.screenshot(path=f"{OUT}/inspector-port.png", full_page=False)
|
||||||
|
await pg.locator('[data-action="run-diagnostic"]').first.click()
|
||||||
|
await pg.wait_for_timeout(9000)
|
||||||
|
await pg.evaluate("document.getElementById('diag-results')?.scrollIntoView({block:'start'})")
|
||||||
|
await pg.wait_for_timeout(600)
|
||||||
|
await pg.screenshot(path=f"{OUT}/inspector-diagnostics.png")
|
||||||
|
print("diagnostics ok")
|
||||||
|
await go(pg, "/suppressions")
|
||||||
|
await pg.screenshot(path=f"{OUT}/suppressions.png", full_page=True)
|
||||||
|
print("errors", errs[:3])
|
||||||
|
await b.close()
|
||||||
|
b, c = await mk(p, "light")
|
||||||
|
pg = await c.new_page()
|
||||||
|
await go(pg, "/")
|
||||||
|
await pg.screenshot(path=f"{OUT}/dashboard-light.png")
|
||||||
|
await b.close()
|
||||||
|
|
||||||
|
|
||||||
|
async def gif():
|
||||||
|
frames = []
|
||||||
|
async with async_playwright() as p:
|
||||||
|
b, c = await mk(p, "dark", 1280, 720)
|
||||||
|
pg = await c.new_page()
|
||||||
|
|
||||||
|
async def snap(n=1, ms=0):
|
||||||
|
if ms:
|
||||||
|
await pg.wait_for_timeout(ms)
|
||||||
|
im = Image.open(io.BytesIO(await pg.screenshot())).convert("RGB").resize((960, 540), Image.LANCZOS)
|
||||||
|
for _ in range(n):
|
||||||
|
frames.append(im)
|
||||||
|
|
||||||
|
await go(pg, "/", 5000)
|
||||||
|
await snap(3)
|
||||||
|
await pg.mouse.wheel(0, 620)
|
||||||
|
await snap(3, 600)
|
||||||
|
await pg.mouse.wheel(0, 700)
|
||||||
|
await snap(2, 600)
|
||||||
|
await go(pg, "/links", 4500)
|
||||||
|
await snap(3)
|
||||||
|
await pg.mouse.wheel(0, 500)
|
||||||
|
await snap(2, 500)
|
||||||
|
await go(pg, "/inspector", 4000)
|
||||||
|
await snap(2)
|
||||||
|
await select_port(pg, "USW-Aggregation", 4)
|
||||||
|
await snap(3)
|
||||||
|
await pg.locator('[data-action="run-diagnostic"]').first.click()
|
||||||
|
await snap(2, 1500)
|
||||||
|
await snap(2, 7500)
|
||||||
|
await pg.evaluate("document.getElementById('diag-results')?.scrollIntoView({block:'start'})")
|
||||||
|
await snap(4, 600)
|
||||||
|
await go(pg, "/suppressions", 4000)
|
||||||
|
await snap(3)
|
||||||
|
await b.close()
|
||||||
|
pal = [f.quantize(colors=96, method=Image.MEDIANCUT, dither=Image.NONE) for f in frames]
|
||||||
|
pal[0].save(f"{OUT}/demo.gif", save_all=True, append_images=pal[1:], duration=1100, loop=0, optimize=True)
|
||||||
|
print("gif", len(frames))
|
||||||
|
|
||||||
|
|
||||||
|
async def main():
|
||||||
|
if "gif" not in sys.argv:
|
||||||
|
await shots()
|
||||||
|
if "shots" not in sys.argv:
|
||||||
|
await gif()
|
||||||
|
|
||||||
|
|
||||||
|
asyncio.run(main())
|
||||||
@@ -0,0 +1,133 @@
|
|||||||
|
"""Minimal stand-in for the PULSE internal API, returning canned diagnostics output (fictional data)."""
|
||||||
|
|
||||||
|
# flake8: noqa: E501 (demo data and canned output contain long literal lines)
|
||||||
|
|
||||||
|
import json
|
||||||
|
from http.server import BaseHTTPRequestHandler, HTTPServer
|
||||||
|
|
||||||
|
CANNED = """=== carrier ===
|
||||||
|
1
|
||||||
|
=== operstate ===
|
||||||
|
up
|
||||||
|
=== sysfs_stats ===
|
||||||
|
rx_bytes:9183029113
|
||||||
|
tx_bytes:7732918231
|
||||||
|
rx_errors:14
|
||||||
|
tx_errors:0
|
||||||
|
rx_dropped:0
|
||||||
|
tx_dropped:0
|
||||||
|
rx_crc_errors:14
|
||||||
|
rx_frame_errors:0
|
||||||
|
rx_fifo_errors:0
|
||||||
|
tx_carrier_errors:0
|
||||||
|
collisions:0
|
||||||
|
rx_missed_errors:0
|
||||||
|
=== carrier_changes ===
|
||||||
|
6
|
||||||
|
=== ethtool ===
|
||||||
|
Settings for enp5s0:
|
||||||
|
Supported ports: [ FIBRE ]
|
||||||
|
Supported link modes: 10000baseT/Full
|
||||||
|
Speed: 10000Mb/s
|
||||||
|
Duplex: Full
|
||||||
|
Port: FIBRE
|
||||||
|
PHYAD: 0
|
||||||
|
Auto-negotiation: off
|
||||||
|
Link detected: yes
|
||||||
|
=== ethtool_driver ===
|
||||||
|
driver: ixgbe
|
||||||
|
version: 6.8.12-4-pve
|
||||||
|
firmware-version: 0x800003e1, 1.3429.0
|
||||||
|
bus-info: 0000:05:00.0
|
||||||
|
=== ethtool_pause ===
|
||||||
|
Pause parameters for enp5s0:
|
||||||
|
Autonegotiate: off
|
||||||
|
RX: off
|
||||||
|
TX: off
|
||||||
|
=== ethtool_ring ===
|
||||||
|
Ring parameters for enp5s0:
|
||||||
|
Pre-set maximums:
|
||||||
|
RX: 8192
|
||||||
|
RX Mini: n/a
|
||||||
|
RX Jumbo: n/a
|
||||||
|
TX: 8192
|
||||||
|
Current hardware settings:
|
||||||
|
RX: 512
|
||||||
|
RX Mini: n/a
|
||||||
|
RX Jumbo: n/a
|
||||||
|
TX: 512
|
||||||
|
=== ethtool_stats ===
|
||||||
|
NIC statistics:
|
||||||
|
rx_packets: 183029113
|
||||||
|
tx_packets: 120918231
|
||||||
|
rx_errors: 14
|
||||||
|
tx_errors: 0
|
||||||
|
rx_crc_errors: 14
|
||||||
|
rx_missed_errors: 0
|
||||||
|
tx_timeout_count: 0
|
||||||
|
fdir_match: 41022
|
||||||
|
tx_flow_control_xon: 0
|
||||||
|
rx_flow_control_xon: 0
|
||||||
|
=== ethtool_dom ===
|
||||||
|
Identifier : 0x03 (SFP)
|
||||||
|
Connector : 0x07 (LC)
|
||||||
|
Vendor name : FS
|
||||||
|
Vendor PN : SFP-10GSR-85
|
||||||
|
Laser wavelength : 850nm
|
||||||
|
Laser bias current : 6.100 mA
|
||||||
|
Laser output power : 0.5530 mW / -2.57 dBm
|
||||||
|
Receiver signal average optical power : 0.0120 mW / -19.21 dBm
|
||||||
|
Module temperature : 47.50 degrees C / 117.50 degrees F
|
||||||
|
Module voltage : 3.3120 V
|
||||||
|
=== ip_link ===
|
||||||
|
5: enp5s0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 9000 qdisc mq master vmbr1 state UP mode DEFAULT group default qlen 1000
|
||||||
|
link/ether 02:00:5e:10:00:05 brd ff:ff:ff:ff:ff:ff
|
||||||
|
RX: bytes packets errors dropped missed mcast
|
||||||
|
9183029113 183029113 14 0 0 10234
|
||||||
|
TX: bytes packets errors dropped carrier collsns
|
||||||
|
7732918231 120918231 0 0 0 0
|
||||||
|
=== ip_addr ===
|
||||||
|
5: enp5s0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 9000 qdisc mq master vmbr1 state UP group default qlen 1000
|
||||||
|
link/ether 02:00:5e:10:00:05 brd ff:ff:ff:ff:ff:ff
|
||||||
|
=== ip_route ===
|
||||||
|
=== dmesg ===
|
||||||
|
[ 412.118233] ixgbe 0000:05:00.0 enp5s0: NIC Link is Up 10 Gbps, Flow Control: None
|
||||||
|
[1188221.402917] ixgbe 0000:05:00.0 enp5s0: NIC Link is Down
|
||||||
|
[1188224.190331] ixgbe 0000:05:00.0 enp5s0: NIC Link is Up 10 Gbps, Flow Control: None
|
||||||
|
[2230418.550112] ixgbe 0000:05:00.0 enp5s0: Reset adapter
|
||||||
|
[2230421.118904] ixgbe 0000:05:00.0 enp5s0: NIC Link is Up 10 Gbps, Flow Control: None
|
||||||
|
=== lldpctl ===
|
||||||
|
-------------------------------------------------------------------------------
|
||||||
|
LLDP neighbors:
|
||||||
|
-------------------------------------------------------------------------------
|
||||||
|
Interface: enp5s0, via: LLDP, RID: 1, Time: 12 days, 03:11:42
|
||||||
|
Chassis:
|
||||||
|
ChassisID: mac aa:bb:cc:00:00:03
|
||||||
|
SysName: USW-Aggregation
|
||||||
|
Port:
|
||||||
|
PortID: ifname Port 4
|
||||||
|
=== end ===
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
class H(BaseHTTPRequestHandler):
|
||||||
|
def _send(self, obj, code=200):
|
||||||
|
b = json.dumps(obj).encode()
|
||||||
|
self.send_response(code)
|
||||||
|
self.send_header("Content-Type", "application/json")
|
||||||
|
self.send_header("Content-Length", str(len(b)))
|
||||||
|
self.end_headers()
|
||||||
|
self.wfile.write(b)
|
||||||
|
|
||||||
|
def do_POST(self):
|
||||||
|
self.rfile.read(int(self.headers.get("Content-Length", 0) or 0))
|
||||||
|
self._send({"execution_id": "demo-exec-0001"})
|
||||||
|
|
||||||
|
def do_GET(self):
|
||||||
|
self._send({"status": "completed", "logs": [{"action": "command_result", "stdout": CANNED}]})
|
||||||
|
|
||||||
|
def log_message(self, *a):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
HTTPServer(("127.0.0.1", 8650), H).serve_forever()
|
||||||
@@ -0,0 +1,425 @@
|
|||||||
|
"""Seed a throwaway GANDALF database with fictional data (documentation IP ranges only)."""
|
||||||
|
|
||||||
|
# flake8: noqa: E501 (demo data and canned output contain long literal lines)
|
||||||
|
|
||||||
|
import json
|
||||||
|
import random
|
||||||
|
from datetime import datetime, timedelta, timezone
|
||||||
|
import pymysql
|
||||||
|
|
||||||
|
random.seed(11)
|
||||||
|
conn = pymysql.connect(
|
||||||
|
host="127.0.0.1",
|
||||||
|
port=3398,
|
||||||
|
user="gandalf",
|
||||||
|
password="demo-only",
|
||||||
|
database="gandalf",
|
||||||
|
autocommit=True,
|
||||||
|
charset="utf8mb4",
|
||||||
|
)
|
||||||
|
cur = conn.cursor()
|
||||||
|
now = datetime.now(timezone.utc)
|
||||||
|
|
||||||
|
|
||||||
|
def ts(h):
|
||||||
|
return (now - timedelta(hours=h)).strftime("%Y-%m-%d %H:%M:%S")
|
||||||
|
|
||||||
|
|
||||||
|
def state(k, v):
|
||||||
|
cur.execute(
|
||||||
|
"REPLACE INTO monitor_state (key_name,value) VALUES (%s,%s)", (k, v if isinstance(v, str) else json.dumps(v))
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# ---- hosts / interfaces: (name, ip, [(iface, speed_mbps, up, kind)]) kind: rj45 | fiber | dac
|
||||||
|
HOSTS = [
|
||||||
|
(
|
||||||
|
"compute-storage-01",
|
||||||
|
"192.0.2.4",
|
||||||
|
[("eno1", 1000, True, "rj45"), ("enp1s0f0", 10000, True, "dac"), ("enp1s0f1", 10000, True, "dac")],
|
||||||
|
),
|
||||||
|
("compute-storage-gpu-01", "192.0.2.10", [("enp2s0", 1000, True, "rj45"), ("enp4s0", 10000, True, "fiber")]),
|
||||||
|
("large1", "192.0.2.2", [("enp3s0", 1000, True, "rj45"), ("enp5s0", 10000, True, "fiber")]),
|
||||||
|
(
|
||||||
|
"storage-01",
|
||||||
|
"192.0.2.11",
|
||||||
|
[("enp5s0", 1000, True, "rj45"), ("enp1s0", 10000, False, "fiber")],
|
||||||
|
), # regression: link down
|
||||||
|
(
|
||||||
|
"micro1",
|
||||||
|
"192.0.2.8",
|
||||||
|
[("enp2s0", 1000, True, "rj45"), ("enp3s0", 1000, False, "rj45")],
|
||||||
|
), # unused bond leg (baseline, not alerted)
|
||||||
|
("monitor-02", "192.0.2.9", [("enp2s0", 1000, True, "rj45"), ("enp3s0", 1000, True, "rj45")]),
|
||||||
|
]
|
||||||
|
hosts = {}
|
||||||
|
for name, ip, ifs in HOSTS:
|
||||||
|
st = [u for _, _, u, _ in ifs]
|
||||||
|
status = "up" if all(st) else ("down" if not any(st) else "degraded")
|
||||||
|
hosts[name] = {
|
||||||
|
"ip": ip,
|
||||||
|
"interfaces": {i: ("up" if u else "down") for i, _, u, _ in ifs},
|
||||||
|
"status": status,
|
||||||
|
"source": "prometheus",
|
||||||
|
}
|
||||||
|
hosts["backup-01"] = {"ip": "198.51.100.20", "interfaces": {}, "status": "up", "source": "ping"}
|
||||||
|
unifi = [
|
||||||
|
{
|
||||||
|
"name": "Dream Machine Pro",
|
||||||
|
"mac": "aa:bb:cc:00:00:01",
|
||||||
|
"ip": "192.0.2.1",
|
||||||
|
"type": "udm",
|
||||||
|
"model": "UDMPRO",
|
||||||
|
"state": 1,
|
||||||
|
"connected": True,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Pro 24 PoE",
|
||||||
|
"mac": "aa:bb:cc:00:00:02",
|
||||||
|
"ip": "192.0.2.30",
|
||||||
|
"type": "usw",
|
||||||
|
"model": "US24PRO",
|
||||||
|
"state": 1,
|
||||||
|
"connected": True,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "USW-Aggregation",
|
||||||
|
"mac": "aa:bb:cc:00:00:03",
|
||||||
|
"ip": "192.0.2.31",
|
||||||
|
"type": "usw",
|
||||||
|
"model": "USL8A",
|
||||||
|
"state": 1,
|
||||||
|
"connected": True,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "UNVR Pro",
|
||||||
|
"mac": "aa:bb:cc:00:00:04",
|
||||||
|
"ip": "192.0.2.40",
|
||||||
|
"type": "unvr",
|
||||||
|
"model": "UNVRPRO",
|
||||||
|
"state": 1,
|
||||||
|
"connected": True,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Garage AP",
|
||||||
|
"mac": "aa:bb:cc:00:00:05",
|
||||||
|
"ip": "192.0.2.41",
|
||||||
|
"type": "uap",
|
||||||
|
"model": "U6LR",
|
||||||
|
"state": 0,
|
||||||
|
"connected": False,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Office AP",
|
||||||
|
"mac": "aa:bb:cc:00:00:06",
|
||||||
|
"ip": "192.0.2.42",
|
||||||
|
"type": "uap",
|
||||||
|
"model": "U6PRO",
|
||||||
|
"state": 1,
|
||||||
|
"connected": True,
|
||||||
|
},
|
||||||
|
]
|
||||||
|
state("network_snapshot", {"hosts": hosts, "unifi": unifi, "updated": now.isoformat().replace("+00:00", "Z")})
|
||||||
|
state("last_check", now.strftime("%Y-%m-%d %H:%M:%S UTC"))
|
||||||
|
|
||||||
|
|
||||||
|
# ---- per-interface link stats
|
||||||
|
def rate(speed, frac):
|
||||||
|
return speed * 1e6 / 8 * frac
|
||||||
|
|
||||||
|
|
||||||
|
link_hosts = {}
|
||||||
|
for name, ip, ifs in HOSTS:
|
||||||
|
d = {}
|
||||||
|
for iface, speed, up, kind in ifs:
|
||||||
|
e = {"host_ip": ip, "link_detected": up, "carrier_changes": random.choice([1, 2, 2, 3]) if up else 9}
|
||||||
|
if up:
|
||||||
|
f = random.uniform(0.01, 0.18) if speed == 1000 else random.uniform(0.04, 0.38)
|
||||||
|
e.update(
|
||||||
|
{
|
||||||
|
"speed_mbps": speed,
|
||||||
|
"duplex": "Full",
|
||||||
|
"auto_neg": True,
|
||||||
|
"tx_bytes_rate": rate(speed, f),
|
||||||
|
"rx_bytes_rate": rate(speed, f * random.uniform(0.5, 1.4)),
|
||||||
|
"tx_errs_rate": 0.0,
|
||||||
|
"rx_errs_rate": 0.0,
|
||||||
|
"tx_drops_rate": 0.0,
|
||||||
|
"rx_drops_rate": 0.0,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
e.update(
|
||||||
|
{
|
||||||
|
"speed_mbps": None,
|
||||||
|
"duplex": None,
|
||||||
|
"auto_neg": True,
|
||||||
|
"tx_bytes_rate": 0,
|
||||||
|
"rx_bytes_rate": 0,
|
||||||
|
"tx_errs_rate": 0,
|
||||||
|
"rx_errs_rate": 0,
|
||||||
|
"tx_drops_rate": 0,
|
||||||
|
"rx_drops_rate": 0,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
if up and kind == "fiber":
|
||||||
|
e["sfp"] = {
|
||||||
|
"sfp_type": "SFP+ 10GBASE-SR",
|
||||||
|
"vendor": "FS",
|
||||||
|
"part_no": "SFP-10GSR-85",
|
||||||
|
"temp_c": round(random.uniform(38, 47), 1),
|
||||||
|
"tx_power_dbm": round(random.uniform(-2.8, -1.6), 2),
|
||||||
|
"rx_power_dbm": round(random.uniform(-4.5, -3.0), 2),
|
||||||
|
}
|
||||||
|
if up and kind == "dac":
|
||||||
|
e["sfp"] = {"sfp_type": "SFP+ passive DAC", "vendor": "Ubiquiti", "part_no": "UACC-DAC-SFP10-1M"}
|
||||||
|
d[iface] = e
|
||||||
|
link_hosts[name] = d
|
||||||
|
# one marginal fibre (low RX power) for the warning colour
|
||||||
|
link_hosts["large1"]["enp5s0"]["sfp"].update({"rx_power_dbm": -10.9, "temp_c": 58.4})
|
||||||
|
link_hosts["large1"]["enp5s0"]["carrier_changes"] = 6
|
||||||
|
|
||||||
|
|
||||||
|
# ---- UniFi switch ports (with LLDP neighbours = the hosts above)
|
||||||
|
def port(idx, up, speed, sw_ip, uplink=False, media="GE", lldp=None, poe=None, frac=0.05):
|
||||||
|
|
||||||
|
def r():
|
||||||
|
return rate(speed, frac * random.uniform(0.5, 1.5)) if up else 0
|
||||||
|
|
||||||
|
p = {
|
||||||
|
"port_idx": idx,
|
||||||
|
"switch_ip": sw_ip,
|
||||||
|
"up": up,
|
||||||
|
"speed_mbps": speed if up else 0,
|
||||||
|
"full_duplex": up,
|
||||||
|
"autoneg": True,
|
||||||
|
"is_uplink": uplink,
|
||||||
|
"media": media,
|
||||||
|
"poe_power": poe[0] if poe else None,
|
||||||
|
"poe_class": poe[1] if poe else None,
|
||||||
|
"poe_max_power": poe[2] if poe else None,
|
||||||
|
"poe_mode": "auto" if poe else "",
|
||||||
|
"lldp": lldp,
|
||||||
|
"tx_bytes": int(random.uniform(1e11, 9e12)) if up else 0,
|
||||||
|
"rx_bytes": int(random.uniform(1e11, 9e12)) if up else 0,
|
||||||
|
"tx_errors": 0,
|
||||||
|
"rx_errors": 0,
|
||||||
|
"tx_dropped": 0,
|
||||||
|
"rx_dropped": 0,
|
||||||
|
"tx_bytes_rate": r(),
|
||||||
|
"rx_bytes_rate": r(),
|
||||||
|
"tx_errs_rate": 0.0,
|
||||||
|
"rx_errs_rate": 0.0,
|
||||||
|
"tx_drops_rate": 0.0,
|
||||||
|
"rx_drops_rate": 0.0,
|
||||||
|
}
|
||||||
|
return p
|
||||||
|
|
||||||
|
|
||||||
|
def L(host, iface):
|
||||||
|
return {
|
||||||
|
"chassis_id": "aa:bb:cc:11:22:33",
|
||||||
|
"system_name": host,
|
||||||
|
"port_id": iface,
|
||||||
|
"port_desc": iface,
|
||||||
|
"mgmt_ips": [dict(HOSTS_IP)[host]] if False else [],
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
HOSTS_IP = {n: ip for n, ip, _ in HOSTS}
|
||||||
|
pro = {}
|
||||||
|
mgmt = [
|
||||||
|
(1, "compute-storage-01", "eno1"),
|
||||||
|
(2, "compute-storage-gpu-01", "enp2s0"),
|
||||||
|
(3, "large1", "enp3s0"),
|
||||||
|
(4, "storage-01", "enp5s0"),
|
||||||
|
(5, "micro1", "enp2s0"),
|
||||||
|
(6, "monitor-02", "enp2s0"),
|
||||||
|
(7, "monitor-02", "enp3s0"),
|
||||||
|
]
|
||||||
|
for i in range(1, 25):
|
||||||
|
m = [x for x in mgmt if x[0] == i]
|
||||||
|
if m:
|
||||||
|
pro[f"Port {i}"] = port(i, True, 1000, "192.0.2.30", lldp=L(m[0][1], m[0][2]), frac=0.06)
|
||||||
|
elif i == 8:
|
||||||
|
pro[f"Port {i}"] = port(i, False, 0, "192.0.2.30") # micro1 unused bond leg
|
||||||
|
elif i in (10, 11, 12):
|
||||||
|
pro[f"Port {i}"] = port(
|
||||||
|
i,
|
||||||
|
True,
|
||||||
|
100 if i == 12 else 1000,
|
||||||
|
"192.0.2.30",
|
||||||
|
lldp=None,
|
||||||
|
poe=(round(random.uniform(3, 9), 1), 4, 30.0),
|
||||||
|
frac=0.02,
|
||||||
|
) # APs / cameras on PoE
|
||||||
|
elif i == 9:
|
||||||
|
pro[f"Port {i}"] = port(
|
||||||
|
i,
|
||||||
|
True,
|
||||||
|
1000,
|
||||||
|
"192.0.2.30",
|
||||||
|
lldp={
|
||||||
|
"chassis_id": "aa:bb:cc:00:00:04",
|
||||||
|
"system_name": "UNVR Pro",
|
||||||
|
"port_id": "eth0",
|
||||||
|
"port_desc": "eth0",
|
||||||
|
"mgmt_ips": [],
|
||||||
|
},
|
||||||
|
poe=(0.0, 0, 30.0),
|
||||||
|
frac=0.2,
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
pro[f"Port {i}"] = port(i, False, 0, "192.0.2.30")
|
||||||
|
pro["Port 25"] = port(
|
||||||
|
25,
|
||||||
|
True,
|
||||||
|
10000,
|
||||||
|
"192.0.2.30",
|
||||||
|
uplink=True,
|
||||||
|
media="SFP+",
|
||||||
|
lldp={
|
||||||
|
"chassis_id": "aa:bb:cc:00:00:03",
|
||||||
|
"system_name": "USW-Aggregation",
|
||||||
|
"port_id": "Port 7",
|
||||||
|
"port_desc": "Port 7",
|
||||||
|
"mgmt_ips": [],
|
||||||
|
},
|
||||||
|
frac=0.08,
|
||||||
|
)
|
||||||
|
pro["Port 26"] = port(26, False, 0, "192.0.2.30", media="SFP+")
|
||||||
|
agg = {}
|
||||||
|
aggmap = [
|
||||||
|
(1, "compute-storage-01", "enp1s0f0"),
|
||||||
|
(2, "compute-storage-01", "enp1s0f1"),
|
||||||
|
(3, "compute-storage-gpu-01", "enp4s0"),
|
||||||
|
(4, "large1", "enp5s0"),
|
||||||
|
(5, "storage-01", "enp1s0"),
|
||||||
|
]
|
||||||
|
for i in range(1, 9):
|
||||||
|
m = [x for x in aggmap if x[0] == i]
|
||||||
|
if i == 5:
|
||||||
|
agg[f"Port {i}"] = port(i, False, 0, "192.0.2.31", media="SFP+", lldp=None) # storage-01 10G link is down
|
||||||
|
elif m:
|
||||||
|
agg[f"Port {i}"] = port(i, True, 10000, "192.0.2.31", media="SFP+", lldp=L(m[0][1], m[0][2]), frac=0.15)
|
||||||
|
elif i == 7:
|
||||||
|
agg[f"Port {i}"] = port(
|
||||||
|
i,
|
||||||
|
True,
|
||||||
|
10000,
|
||||||
|
"192.0.2.31",
|
||||||
|
uplink=True,
|
||||||
|
media="SFP+",
|
||||||
|
lldp={
|
||||||
|
"chassis_id": "aa:bb:cc:00:00:02",
|
||||||
|
"system_name": "Pro 24 PoE",
|
||||||
|
"port_id": "Port 25",
|
||||||
|
"port_desc": "Port 25",
|
||||||
|
"mgmt_ips": [],
|
||||||
|
},
|
||||||
|
frac=0.08,
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
agg[f"Port {i}"] = port(i, False, 0, "192.0.2.31", media="SFP+")
|
||||||
|
state(
|
||||||
|
"link_stats",
|
||||||
|
{
|
||||||
|
"hosts": link_hosts,
|
||||||
|
"unifi_switches": {
|
||||||
|
"Pro 24 PoE": {"ip": "192.0.2.30", "model": "US24PRO", "ports": pro},
|
||||||
|
"USW-Aggregation": {"ip": "192.0.2.31", "model": "USL8A", "ports": agg},
|
||||||
|
},
|
||||||
|
"updated": now.strftime("%Y-%m-%d %H:%M:%S UTC"),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
# ---- events
|
||||||
|
EV = [ # type, sev, source, target, detail, desc, first_seen_h, last_seen_h, resolved_h|None, fails, ticket
|
||||||
|
(
|
||||||
|
"interface_down",
|
||||||
|
"warning",
|
||||||
|
"prometheus",
|
||||||
|
"storage-01",
|
||||||
|
"enp1s0",
|
||||||
|
"Interface enp1s0 on storage-01 went down (UP -> DOWN regression).",
|
||||||
|
3.2,
|
||||||
|
0.0,
|
||||||
|
None,
|
||||||
|
5,
|
||||||
|
"731905284",
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"unifi_device_offline",
|
||||||
|
"warning",
|
||||||
|
"unifi",
|
||||||
|
"Garage AP",
|
||||||
|
"uap / 192.0.2.41",
|
||||||
|
'UniFi device "Garage AP" has been offline for 3 consecutive checks.',
|
||||||
|
1.1,
|
||||||
|
0.0,
|
||||||
|
None,
|
||||||
|
3,
|
||||||
|
"731905301",
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"interface_down",
|
||||||
|
"warning",
|
||||||
|
"prometheus",
|
||||||
|
"compute-storage-01",
|
||||||
|
"enp1s0f1",
|
||||||
|
"Interface enp1s0f1 on compute-storage-01 went down.",
|
||||||
|
9.0,
|
||||||
|
7.5,
|
||||||
|
7.4,
|
||||||
|
2,
|
||||||
|
None,
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"host_unreachable",
|
||||||
|
"warning",
|
||||||
|
"ping",
|
||||||
|
"backup-01",
|
||||||
|
"198.51.100.20",
|
||||||
|
"Host backup-01 stopped answering ping for 2 consecutive checks.",
|
||||||
|
30.0,
|
||||||
|
28.0,
|
||||||
|
27.9,
|
||||||
|
2,
|
||||||
|
"731881772",
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"unifi_device_offline",
|
||||||
|
"warning",
|
||||||
|
"unifi",
|
||||||
|
"Office AP",
|
||||||
|
"uap / 192.0.2.42",
|
||||||
|
'UniFi device "Office AP" was offline for 3 consecutive checks.',
|
||||||
|
19.0,
|
||||||
|
18.0,
|
||||||
|
17.8,
|
||||||
|
3,
|
||||||
|
"731890014",
|
||||||
|
),
|
||||||
|
]
|
||||||
|
for t, sev, src, tgt, det, desc, f, l, r, fails, tid in EV:
|
||||||
|
cur.execute(
|
||||||
|
"INSERT INTO network_events (event_type,severity,source_type,target_name,target_detail,description,first_seen,last_seen,resolved_at,consecutive_failures,ticket_id) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)",
|
||||||
|
(t, sev, src, tgt, det, desc, ts(f), ts(l), ts(r) if r is not None else None, fails, tid),
|
||||||
|
)
|
||||||
|
# ---- suppressions (one active timed, one manual, one expired in history)
|
||||||
|
SUP = [
|
||||||
|
("interface", "micro1", "enp3s0", "Unused bond leg, cable not connected", "alex.rivera", 48, None, 1),
|
||||||
|
("host", "monitor-02", "", "Planned firmware update window", "alex.rivera", 0.4, 0.6, 1),
|
||||||
|
("unifi_device", "Office AP", "", "Replacing the access point", "sam.chen", 30, 29, 0),
|
||||||
|
]
|
||||||
|
for tt, tn, td, reason, by, created_h, exp_in, active in SUP:
|
||||||
|
exp = (
|
||||||
|
(now + timedelta(hours=exp_in)).strftime("%Y-%m-%d %H:%M:%S")
|
||||||
|
if exp_in and exp_in > 0
|
||||||
|
else (ts(-exp_in) if exp_in else None)
|
||||||
|
)
|
||||||
|
cur.execute(
|
||||||
|
"INSERT INTO suppression_rules (target_type,target_name,target_detail,reason,suppressed_by,created_at,expires_at,active) VALUES (%s,%s,%s,%s,%s,%s,%s,%s)",
|
||||||
|
(tt, tn, td, reason, by, ts(created_h), exp, active),
|
||||||
|
)
|
||||||
|
print("seeded: hosts", len(hosts), "unifi", len(unifi), "events", len(EV), "suppressions", len(SUP))
|
||||||
@@ -220,7 +220,7 @@ function updateEventsTable(events, totalActive) {
|
|||||||
? GANDALF_CONFIG.ticket_web_url : 'http://t.lotusguild.org/ticket/';
|
? GANDALF_CONFIG.ticket_web_url : 'http://t.lotusguild.org/ticket/';
|
||||||
const ticket = e.ticket_id
|
const ticket = e.ticket_id
|
||||||
? `<a href="${lt.escHtml(ticketBase)}${lt.escHtml(String(e.ticket_id))}" target="_blank"
|
? `<a href="${lt.escHtml(ticketBase)}${lt.escHtml(String(e.ticket_id))}" target="_blank"
|
||||||
class="ticket-link">#${e.ticket_id}</a>`
|
class="ticket-link">#${lt.escHtml(String(e.ticket_id))}</a>`
|
||||||
: '–';
|
: '–';
|
||||||
const supBadge = e.is_suppressed
|
const supBadge = e.is_suppressed
|
||||||
? `<span class="lt-badge badge-suppressed" title="Alert suppressed">🔕 sup</span>`
|
? `<span class="lt-badge badge-suppressed" title="Alert suppressed">🔕 sup</span>`
|
||||||
@@ -294,18 +294,33 @@ function updateSuppressForm() {
|
|||||||
const type = document.getElementById('sup-type').value;
|
const type = document.getElementById('sup-type').value;
|
||||||
const nameGrp = document.getElementById('sup-name-group');
|
const nameGrp = document.getElementById('sup-name-group');
|
||||||
const detailGrp = document.getElementById('sup-detail-group');
|
const detailGrp = document.getElementById('sup-detail-group');
|
||||||
|
const nameInput = document.getElementById('sup-name');
|
||||||
|
const detailInput = document.getElementById('sup-detail');
|
||||||
if (nameGrp) nameGrp.style.display = (type === 'all') ? 'none' : '';
|
if (nameGrp) nameGrp.style.display = (type === 'all') ? 'none' : '';
|
||||||
if (detailGrp) detailGrp.style.display = (type === 'interface') ? '' : 'none';
|
if (detailGrp) detailGrp.style.display = (type === 'interface') ? '' : 'none';
|
||||||
|
if (nameInput) {
|
||||||
|
const req = (type !== 'all');
|
||||||
|
nameInput.required = req;
|
||||||
|
nameInput.setAttribute('aria-required', String(req));
|
||||||
|
}
|
||||||
|
if (detailInput) {
|
||||||
|
const req = (type === 'interface');
|
||||||
|
detailInput.required = req;
|
||||||
|
detailInput.setAttribute('aria-required', String(req));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
function setDuration(mins, el) {
|
function setDuration(mins, el, opts) {
|
||||||
document.getElementById('sup-expires').value = mins || '';
|
const o = opts || {};
|
||||||
document.querySelectorAll('#suppress-modal .pill').forEach(p => {
|
const expiresEl = document.getElementById(o.expiresId || 'sup-expires');
|
||||||
|
const pillSel = o.pillSel || '#suppress-modal .pill';
|
||||||
|
const hint = document.getElementById(o.hintId || 'duration-hint');
|
||||||
|
if (expiresEl) expiresEl.value = mins || '';
|
||||||
|
document.querySelectorAll(pillSel).forEach(p => {
|
||||||
p.classList.remove('active');
|
p.classList.remove('active');
|
||||||
p.setAttribute('aria-pressed', 'false');
|
p.setAttribute('aria-pressed', 'false');
|
||||||
});
|
});
|
||||||
if (el) { el.classList.add('active'); el.setAttribute('aria-pressed', 'true'); }
|
if (el) { el.classList.add('active'); el.setAttribute('aria-pressed', 'true'); }
|
||||||
const hint = document.getElementById('duration-hint');
|
|
||||||
if (hint) {
|
if (hint) {
|
||||||
if (mins) {
|
if (mins) {
|
||||||
const h = Math.floor(mins / 60), m = mins % 60;
|
const h = Math.floor(mins / 60), m = mins % 60;
|
||||||
|
|||||||
@@ -227,16 +227,16 @@
|
|||||||
<div class="lt-form-group">
|
<div class="lt-form-group">
|
||||||
<label class="lt-label" for="sup-reason">Reason <span class="required">*</span></label>
|
<label class="lt-label" for="sup-reason">Reason <span class="required">*</span></label>
|
||||||
<input type="text" class="lt-input" id="sup-reason" name="reason"
|
<input type="text" class="lt-input" id="sup-reason" name="reason"
|
||||||
placeholder="e.g. Planned switch reboot" required>
|
placeholder="e.g. Planned switch reboot" required aria-required="true">
|
||||||
</div>
|
</div>
|
||||||
<div class="lt-form-group lt-form-group--last">
|
<div class="lt-form-group lt-form-group--last">
|
||||||
<label class="lt-label">Duration</label>
|
<label class="lt-label">Duration</label>
|
||||||
<div class="duration-pills" role="group" aria-label="Select suppression duration">
|
<div class="duration-pills" role="group" aria-label="Select suppression duration">
|
||||||
<button type="button" class="pill" data-duration="30" aria-pressed="false">30 min</button>
|
<button type="button" class="pill" data-duration="30" aria-pressed="false" aria-label="30 minutes">30 min</button>
|
||||||
<button type="button" class="pill" data-duration="60" aria-pressed="false">1 hr</button>
|
<button type="button" class="pill" data-duration="60" aria-pressed="false" aria-label="1 hour">1 hr</button>
|
||||||
<button type="button" class="pill" data-duration="240" aria-pressed="false">4 hr</button>
|
<button type="button" class="pill" data-duration="240" aria-pressed="false" aria-label="4 hours">4 hr</button>
|
||||||
<button type="button" class="pill" data-duration="480" aria-pressed="false">8 hr</button>
|
<button type="button" class="pill" data-duration="480" aria-pressed="false" aria-label="8 hours">8 hr</button>
|
||||||
<button type="button" class="pill pill-manual active" data-duration="" aria-pressed="true">Manual ∞</button>
|
<button type="button" class="pill pill-manual active" data-duration="" aria-pressed="true" aria-label="Manual, no expiry">Manual ∞</button>
|
||||||
</div>
|
</div>
|
||||||
<input type="hidden" id="sup-expires" name="expires_minutes" value="">
|
<input type="hidden" id="sup-expires" name="expires_minutes" value="">
|
||||||
<div class="lt-field-hint" id="duration-hint">Persists until manually removed.</div>
|
<div class="lt-field-hint" id="duration-hint">Persists until manually removed.</div>
|
||||||
|
|||||||
@@ -324,6 +324,7 @@
|
|||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
<div class="host-grid" id="host-grid">
|
<div class="host-grid" id="host-grid">
|
||||||
|
{%- set has_global_sup = suppressions | selectattr('target_type', 'equalto', 'all') | list | length > 0 -%}
|
||||||
{% for name, host in snapshot.hosts.items() %}
|
{% for name, host in snapshot.hosts.items() %}
|
||||||
{% set suppressed = suppressions | selectattr('target_name', 'equalto', name) | list %}
|
{% set suppressed = suppressions | selectattr('target_name', 'equalto', name) | list %}
|
||||||
<div class="host-card host-card-{{ host.status }}" data-host="{{ name }}">
|
<div class="host-card host-card-{{ host.status }}" data-host="{{ name }}">
|
||||||
@@ -331,7 +332,7 @@
|
|||||||
<div class="host-name-row">
|
<div class="host-name-row">
|
||||||
<span class="host-status-dot dot-{{ host.status }}"></span>
|
<span class="host-status-dot dot-{{ host.status }}"></span>
|
||||||
<span class="host-name">{{ name }}</span>
|
<span class="host-name">{{ name }}</span>
|
||||||
{% if suppressed %}
|
{% if suppressed or has_global_sup %}
|
||||||
<span class="badge-suppressed" title="Suppressed">🔕</span>
|
<span class="badge-suppressed" title="Suppressed">🔕</span>
|
||||||
{% endif %}
|
{% endif %}
|
||||||
</div>
|
</div>
|
||||||
@@ -468,7 +469,7 @@
|
|||||||
{% block scripts %}
|
{% block scripts %}
|
||||||
<script>
|
<script>
|
||||||
// Start auto-refresh using saved settings interval (default 30 s)
|
// Start auto-refresh using saved settings interval (default 30 s)
|
||||||
const _savedInterval = (window.gandalfSettings && window.gandalfSettings.refreshInterval) || 30;
|
const _savedInterval = window.gandalfSettings?.refreshInterval ?? 30;
|
||||||
if (_savedInterval > 0) lt.autoRefresh.start(refreshAll, _savedInterval * 1000);
|
if (_savedInterval > 0) lt.autoRefresh.start(refreshAll, _savedInterval * 1000);
|
||||||
|
|
||||||
// When settings change, restart auto-refresh with new interval
|
// When settings change, restart auto-refresh with new interval
|
||||||
|
|||||||
@@ -218,6 +218,7 @@ let _apiData = null;
|
|||||||
function selectPort(el) {
|
function selectPort(el) {
|
||||||
const swName = el.dataset.switch;
|
const swName = el.dataset.switch;
|
||||||
const idx = parseInt(el.dataset.portIdx, 10);
|
const idx = parseInt(el.dataset.portIdx, 10);
|
||||||
|
if (_diagPollTimer) { clearInterval(_diagPollTimer); _diagPollTimer = null; }
|
||||||
document.querySelectorAll('.switch-port-block.selected')
|
document.querySelectorAll('.switch-port-block.selected')
|
||||||
.forEach(e => e.classList.remove('selected'));
|
.forEach(e => e.classList.remove('selected'));
|
||||||
el.classList.add('selected');
|
el.classList.add('selected');
|
||||||
@@ -428,7 +429,7 @@ function renderInspector(data) {
|
|||||||
|
|
||||||
const updEl = document.getElementById('inspector-updated');
|
const updEl = document.getElementById('inspector-updated');
|
||||||
if (updEl && data.updated) {
|
if (updEl && data.updated) {
|
||||||
const updMs = new Date(data.updated + (data.updated.includes('Z') ? '' : 'Z'));
|
const updMs = new Date(_toIso(data.updated));
|
||||||
const ageMin = (Date.now() - updMs) / 60000;
|
const ageMin = (Date.now() - updMs) / 60000;
|
||||||
const timeStr = updMs.toLocaleTimeString();
|
const timeStr = updMs.toLocaleTimeString();
|
||||||
if (ageMin > 15) {
|
if (ageMin > 15) {
|
||||||
@@ -472,7 +473,7 @@ async function loadInspector() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
loadInspector();
|
loadInspector();
|
||||||
const _inspInterval = (window.gandalfSettings && window.gandalfSettings.refreshInterval) || 60;
|
const _inspInterval = window.gandalfSettings?.refreshInterval ?? 60;
|
||||||
if (_inspInterval > 0) lt.autoRefresh.start(loadInspector, Math.max(_inspInterval, 15) * 1000);
|
if (_inspInterval > 0) lt.autoRefresh.start(loadInspector, Math.max(_inspInterval, 15) * 1000);
|
||||||
|
|
||||||
window.onGandalfSettingsChanged = function(s) {
|
window.onGandalfSettingsChanged = function(s) {
|
||||||
|
|||||||
@@ -36,7 +36,6 @@
|
|||||||
{% block scripts %}
|
{% block scripts %}
|
||||||
<script>
|
<script>
|
||||||
const escHtml = s => lt.escHtml(s);
|
const escHtml = s => lt.escHtml(s);
|
||||||
const _toIso = s => s ? s.replace(' UTC', 'Z').replace(' ', 'T') : s;
|
|
||||||
|
|
||||||
// ── Formatting helpers ────────────────────────────────────────────
|
// ── Formatting helpers ────────────────────────────────────────────
|
||||||
function fmtRate(bytesPerSec) {
|
function fmtRate(bytesPerSec) {
|
||||||
@@ -373,14 +372,16 @@ function togglePanel(panel) {
|
|||||||
if (title) title.setAttribute('aria-expanded', isCollapsed ? 'false' : 'true');
|
if (title) title.setAttribute('aria-expanded', isCollapsed ? 'false' : 'true');
|
||||||
const id = panel.id;
|
const id = panel.id;
|
||||||
if (id) {
|
if (id) {
|
||||||
const collapsed = JSON.parse(sessionStorage.getItem('linksCollapsed') || '{}');
|
let collapsed = {};
|
||||||
|
try { collapsed = JSON.parse(sessionStorage.getItem('linksCollapsed') || '{}'); } catch(_) {}
|
||||||
collapsed[id] = panel.classList.contains('collapsed');
|
collapsed[id] = panel.classList.contains('collapsed');
|
||||||
sessionStorage.setItem('linksCollapsed', JSON.stringify(collapsed));
|
try { sessionStorage.setItem('linksCollapsed', JSON.stringify(collapsed)); } catch(_) {}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
function restoreCollapseState() {
|
function restoreCollapseState() {
|
||||||
const collapsed = JSON.parse(sessionStorage.getItem('linksCollapsed') || '{}');
|
let collapsed = {};
|
||||||
|
try { collapsed = JSON.parse(sessionStorage.getItem('linksCollapsed') || '{}'); } catch(_) {}
|
||||||
for (const [id, isCollapsed] of Object.entries(collapsed)) {
|
for (const [id, isCollapsed] of Object.entries(collapsed)) {
|
||||||
const panel = document.getElementById(id);
|
const panel = document.getElementById(id);
|
||||||
if (!panel) continue;
|
if (!panel) continue;
|
||||||
@@ -508,9 +509,11 @@ function collapseAll() {
|
|||||||
if (btn) btn.textContent = '[+]';
|
if (btn) btn.textContent = '[+]';
|
||||||
if (title) title.setAttribute('aria-expanded', 'false');
|
if (title) title.setAttribute('aria-expanded', 'false');
|
||||||
});
|
});
|
||||||
sessionStorage.setItem('linksCollapsed', JSON.stringify(
|
try {
|
||||||
Object.fromEntries([...document.querySelectorAll('.link-host-panel')].map(p => [p.id, true]))
|
sessionStorage.setItem('linksCollapsed', JSON.stringify(
|
||||||
));
|
Object.fromEntries([...document.querySelectorAll('.link-host-panel')].map(p => [p.id, true]))
|
||||||
|
));
|
||||||
|
} catch(_) {}
|
||||||
}
|
}
|
||||||
|
|
||||||
function expandAll() {
|
function expandAll() {
|
||||||
@@ -521,13 +524,13 @@ function expandAll() {
|
|||||||
if (btn) btn.textContent = '[–]';
|
if (btn) btn.textContent = '[–]';
|
||||||
if (title) title.setAttribute('aria-expanded', 'true');
|
if (title) title.setAttribute('aria-expanded', 'true');
|
||||||
});
|
});
|
||||||
sessionStorage.setItem('linksCollapsed', '{}');
|
try { sessionStorage.setItem('linksCollapsed', '{}'); } catch(_) {}
|
||||||
}
|
}
|
||||||
|
|
||||||
// ── Stale data warning ────────────────────────────────────────────
|
// ── Stale data warning ────────────────────────────────────────────
|
||||||
function checkLinksStale(updatedStr) {
|
function checkLinksStale(updatedStr) {
|
||||||
if (!updatedStr) return;
|
if (!updatedStr) return;
|
||||||
const age = (Date.now() - new Date(updatedStr + (updatedStr.includes('Z') ? '' : 'Z'))) / 1000;
|
const age = (Date.now() - new Date(_toIso(updatedStr))) / 1000;
|
||||||
let banner = document.getElementById('links-stale-banner');
|
let banner = document.getElementById('links-stale-banner');
|
||||||
if (age > 120) {
|
if (age > 120) {
|
||||||
if (!banner) {
|
if (!banner) {
|
||||||
@@ -549,14 +552,14 @@ function checkLinksStale(updatedStr) {
|
|||||||
async function loadLinks() {
|
async function loadLinks() {
|
||||||
try {
|
try {
|
||||||
const data = await lt.api.get('/api/links');
|
const data = await lt.api.get('/api/links');
|
||||||
if (!data.hosts && !data.unifi_switches) {
|
if ((!data.hosts || !Object.keys(data.hosts).length) && (!data.unifi_switches || !Object.keys(data.unifi_switches).length)) {
|
||||||
document.getElementById('links-container').innerHTML =
|
document.getElementById('links-container').innerHTML =
|
||||||
'<div class="link-no-data">No link data yet — monitor has not completed a full cycle.</div>';
|
'<div class="link-no-data">No link data yet — monitor has not completed a full cycle.</div>';
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
const updEl = document.getElementById('links-updated');
|
const updEl = document.getElementById('links-updated');
|
||||||
if (updEl && data.updated) {
|
if (updEl && data.updated) {
|
||||||
updEl.textContent = 'Updated: ' + new Date(data.updated + (data.updated.includes('Z') ? '' : 'Z')).toLocaleTimeString();
|
updEl.textContent = 'Updated: ' + new Date(_toIso(data.updated)).toLocaleTimeString();
|
||||||
}
|
}
|
||||||
renderLinks(data);
|
renderLinks(data);
|
||||||
checkLinksStale(data.updated);
|
checkLinksStale(data.updated);
|
||||||
@@ -568,7 +571,7 @@ async function loadLinks() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
loadLinks();
|
loadLinks();
|
||||||
const _linksInterval = (window.gandalfSettings && window.gandalfSettings.refreshInterval) || 60;
|
const _linksInterval = window.gandalfSettings?.refreshInterval ?? 60;
|
||||||
if (_linksInterval > 0) lt.autoRefresh.start(loadLinks, Math.max(_linksInterval, 15) * 1000);
|
if (_linksInterval > 0) lt.autoRefresh.start(loadLinks, Math.max(_linksInterval, 15) * 1000);
|
||||||
|
|
||||||
window.onGandalfSettingsChanged = function(s) {
|
window.onGandalfSettingsChanged = function(s) {
|
||||||
|
|||||||
@@ -32,7 +32,7 @@
|
|||||||
<label class="lt-label" for="s-name">Target Name <span class="required">*</span></label>
|
<label class="lt-label" for="s-name">Target Name <span class="required">*</span></label>
|
||||||
<input type="text" class="lt-input" id="s-name" name="target_name"
|
<input type="text" class="lt-input" id="s-name" name="target_name"
|
||||||
placeholder="hostname or device name" autocomplete="off"
|
placeholder="hostname or device name" autocomplete="off"
|
||||||
list="target-name-list">
|
required aria-required="true" list="target-name-list">
|
||||||
<datalist id="target-name-list">
|
<datalist id="target-name-list">
|
||||||
{% for name in snapshot.hosts.keys() | sort %}
|
{% for name in snapshot.hosts.keys() | sort %}
|
||||||
<option value="{{ name }}">
|
<option value="{{ name }}">
|
||||||
@@ -51,7 +51,7 @@
|
|||||||
<label class="lt-label" for="s-reason">Reason <span class="required">*</span></label>
|
<label class="lt-label" for="s-reason">Reason <span class="required">*</span></label>
|
||||||
<input type="text" class="lt-input" id="s-reason" name="reason"
|
<input type="text" class="lt-input" id="s-reason" name="reason"
|
||||||
placeholder="e.g. Planned switch maintenance, replacing SFP on large1/enp43s0"
|
placeholder="e.g. Planned switch maintenance, replacing SFP on large1/enp43s0"
|
||||||
required>
|
required aria-required="true">
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
@@ -59,11 +59,11 @@
|
|||||||
<div class="lt-form-group">
|
<div class="lt-form-group">
|
||||||
<label class="lt-label">Duration</label>
|
<label class="lt-label">Duration</label>
|
||||||
<div class="duration-pills" role="group" aria-label="Select suppression duration">
|
<div class="duration-pills" role="group" aria-label="Select suppression duration">
|
||||||
<button type="button" class="pill" data-duration="30" aria-pressed="false">30 min</button>
|
<button type="button" class="pill" data-duration="30" aria-pressed="false" aria-label="30 minutes">30 min</button>
|
||||||
<button type="button" class="pill" data-duration="60" aria-pressed="false">1 hr</button>
|
<button type="button" class="pill" data-duration="60" aria-pressed="false" aria-label="1 hour">1 hr</button>
|
||||||
<button type="button" class="pill" data-duration="240" aria-pressed="false">4 hr</button>
|
<button type="button" class="pill" data-duration="240" aria-pressed="false" aria-label="4 hours">4 hr</button>
|
||||||
<button type="button" class="pill" data-duration="480" aria-pressed="false">8 hr</button>
|
<button type="button" class="pill" data-duration="480" aria-pressed="false" aria-label="8 hours">8 hr</button>
|
||||||
<button type="button" class="pill pill-manual active" data-duration="" aria-pressed="true">Manual ∞</button>
|
<button type="button" class="pill pill-manual active" data-duration="" aria-pressed="true" aria-label="Manual, no expiry">Manual ∞</button>
|
||||||
</div>
|
</div>
|
||||||
<input type="hidden" id="s-expires" name="expires_minutes" value="">
|
<input type="hidden" id="s-expires" name="expires_minutes" value="">
|
||||||
<div class="lt-field-hint" id="s-dur-hint">Persists until manually removed.</div>
|
<div class="lt-field-hint" id="s-dur-hint">Persists until manually removed.</div>
|
||||||
@@ -217,23 +217,16 @@
|
|||||||
const t = document.getElementById('s-type').value;
|
const t = document.getElementById('s-type').value;
|
||||||
document.getElementById('name-group').style.display = (t==='all') ? 'none' : '';
|
document.getElementById('name-group').style.display = (t==='all') ? 'none' : '';
|
||||||
document.getElementById('detail-group').style.display = (t==='interface') ? '' : 'none';
|
document.getElementById('detail-group').style.display = (t==='interface') ? '' : 'none';
|
||||||
document.getElementById('s-name').required = (t!=='all');
|
const nameInput = document.getElementById('s-name');
|
||||||
|
if (nameInput) {
|
||||||
|
const req = (t !== 'all');
|
||||||
|
nameInput.required = req;
|
||||||
|
nameInput.setAttribute('aria-required', String(req));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
function setDur(mins, el) {
|
function setDur(mins, el) {
|
||||||
document.getElementById('s-expires').value = mins || '';
|
setDuration(mins, el, { expiresId: 's-expires', pillSel: '#create-suppression-form .pill', hintId: 's-dur-hint' });
|
||||||
document.querySelectorAll('.duration-pills .pill').forEach(p => {
|
|
||||||
p.classList.remove('active');
|
|
||||||
p.setAttribute('aria-pressed', 'false');
|
|
||||||
});
|
|
||||||
if (el) { el.classList.add('active'); el.setAttribute('aria-pressed', 'true'); }
|
|
||||||
const hint = document.getElementById('s-dur-hint');
|
|
||||||
if (mins) {
|
|
||||||
const h = Math.floor(mins/60), m = mins%60;
|
|
||||||
hint.textContent = `Expires in ${h?h+'h ':''}${m?m+'m':''}`.trim()+'.';
|
|
||||||
} else {
|
|
||||||
hint.textContent = 'Persists until manually removed.';
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
function renderActiveRows(rows) {
|
function renderActiveRows(rows) {
|
||||||
@@ -302,9 +295,7 @@
|
|||||||
showToast('Suppression applied', 'success');
|
showToast('Suppression applied', 'success');
|
||||||
form.reset();
|
form.reset();
|
||||||
onTypeChange();
|
onTypeChange();
|
||||||
document.querySelectorAll('.duration-pills .pill').forEach(p => p.classList.remove('active'));
|
setDur(null, document.querySelector('#create-suppression-form .pill-manual'));
|
||||||
document.querySelector('.duration-pills .pill-manual')?.classList.add('active');
|
|
||||||
document.getElementById('s-dur-hint').textContent = 'Persists until manually removed.';
|
|
||||||
await refreshActive();
|
await refreshActive();
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
showToast(err.message || 'Error', 'error');
|
showToast(err.message || 'Error', 'error');
|
||||||
|
|||||||
@@ -36,6 +36,12 @@ class TestBuildSshCommand:
|
|||||||
cmd = DiagnosticsRunner.build_ssh_command('10.0.0.1', 'eth0')
|
cmd = DiagnosticsRunner.build_ssh_command('10.0.0.1', 'eth0')
|
||||||
assert 'ethtool' in cmd
|
assert 'ethtool' in cmd
|
||||||
|
|
||||||
|
def test_dmesg_uses_fixed_string_grep(self):
|
||||||
|
# grep -F prevents iface names with dots (e.g. eth0.1) being treated as
|
||||||
|
# regex wildcards; -- prevents leading - from being parsed as a flag
|
||||||
|
cmd = DiagnosticsRunner.build_ssh_command('10.0.0.1', 'eth0')
|
||||||
|
assert 'grep -F --' in cmd
|
||||||
|
|
||||||
|
|
||||||
# ── parse_output ─────────────────────────────────────────────────────────────
|
# ── parse_output ─────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
@@ -467,12 +473,27 @@ class TestAnalyze:
|
|||||||
assert 'LLDP_MISSING' in codes
|
assert 'LLDP_MISSING' in codes
|
||||||
|
|
||||||
def test_lldp_mismatch_is_warning(self):
|
def test_lldp_mismatch_is_warning(self):
|
||||||
|
# The server's lldpd sees a different switch than the one this port belongs to.
|
||||||
sections = self._sections(lldpctl={'available': True, 'neighbor_system': 'wrong-switch'})
|
sections = self._sections(lldpctl={'available': True, 'neighbor_system': 'wrong-switch'})
|
||||||
switch_data = {'speed_mbps': 1000, 'lldp': {'system_name': 'core-sw-01'}}
|
switch_data = {'speed_mbps': 1000, 'switch_name': 'core-sw-01', 'lldp': {'system_name': 'server-a'}}
|
||||||
result = DiagnosticsRunner.analyze(sections, switch_data)
|
result = DiagnosticsRunner.analyze(sections, switch_data)
|
||||||
codes = [w['code'] for w in result['warnings']]
|
codes = [w['code'] for w in result['warnings']]
|
||||||
assert 'LLDP_MISMATCH' in codes
|
assert 'LLDP_MISMATCH' in codes
|
||||||
|
|
||||||
|
def test_lldp_match_is_not_a_warning(self):
|
||||||
|
# Correct cabling: the switch sees the server, the server sees the switch.
|
||||||
|
sections = self._sections(lldpctl={'available': True, 'neighbor_system': 'core-sw-01'})
|
||||||
|
switch_data = {'speed_mbps': 1000, 'switch_name': 'core-sw-01', 'lldp': {'system_name': 'server-a'}}
|
||||||
|
result = DiagnosticsRunner.analyze(sections, switch_data)
|
||||||
|
codes = [w['code'] for w in result['warnings']]
|
||||||
|
assert 'LLDP_MISMATCH' not in codes
|
||||||
|
|
||||||
|
def test_lldp_without_switch_name_skips_check(self):
|
||||||
|
sections = self._sections(lldpctl={'available': True, 'neighbor_system': 'anything'})
|
||||||
|
result = DiagnosticsRunner.analyze(sections, {'speed_mbps': 1000, 'lldp': {'system_name': 'server-a'}})
|
||||||
|
codes = [w['code'] for w in result['warnings']]
|
||||||
|
assert 'LLDP_MISMATCH' not in codes
|
||||||
|
|
||||||
def test_healthy_link_no_issues(self):
|
def test_healthy_link_no_issues(self):
|
||||||
result = DiagnosticsRunner.analyze(self._sections(), {})
|
result = DiagnosticsRunner.analyze(self._sections(), {})
|
||||||
assert result['issues'] == []
|
assert result['issues'] == []
|
||||||
|
|||||||