Fix CI: pass --break-system-packages to pip3 installs

The runner's Debian image update enforces PEP 668, so system-wide pip3
installs (flake8, bandit, pytest, requirements.txt) were failing before
any job logic ran, blocking lint/test/security and the deploy job.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01UB2D82eYNzD1bnMHhFPjRr

.eslintrc.json

@@ -0,0 +1,21 @@
+{
+  "env": {
+    "browser": true,
+    "es2021": true
+  },
+  "globals": {
+    "lt": "readonly",
+    "GANDALF_CONFIG": "readonly",
+    "CSS": "readonly"
+  },
+  "rules": {
+    "no-undef": "error",
+    "no-unused-vars": ["warn", { "argsIgnorePattern": "^_", "varsIgnorePattern": "^_" }],
+    "no-console": "off",
+    "eqeqeq": ["error", "always", { "null": "ignore" }]
+  },
+  "parserOptions": {
+    "ecmaVersion": 2021,
+    "sourceType": "script"
+  }
+}

.gitea/workflows/lint.yml

@@ -17,7 +17,7 @@ jobs:
         run: |
           apt-get update -qq
           apt-get install -y -qq python3 python3-pip
-          pip3 install flake8
+          pip3 install --break-system-packages flake8
 
       - name: Run flake8
         run: flake8 . --exclude=__pycache__,.git

.gitea/workflows/security.yml

@@ -19,7 +19,7 @@ jobs:
         run: |
           apt-get update -qq
           apt-get install -y -qq python3 python3-pip
-          pip3 install bandit
+          pip3 install --break-system-packages bandit
 
       - name: Run bandit
         run: bandit -r . --exclude .git,__pycache__,node_modules -ll

.gitea/workflows/test.yml

@@ -17,8 +17,8 @@ jobs:
         run: |
           apt-get update -qq
           apt-get install -y -qq python3 python3-pip
-          pip3 install pytest pytest-cov
+          pip3 install --break-system-packages pytest pytest-cov
-          pip3 install -r requirements.txt --quiet
+          pip3 install --break-system-packages -r requirements.txt --quiet
 
       - name: Run pytest with coverage
         run: python3 -m pytest tests/ -v --cov=. --cov-report=term-missing --cov-config=.coveragerc

README.md

@@ -148,7 +148,7 @@ returned by the UniFi API for that device.
 
 | Condition | Priority |
 |---|---|
-| UniFi device offline (≥2 consecutive checks) | P2 High |
+| UniFi device offline (≥`unifi_failure_threshold` consecutive checks) | P2 High |
 | Proxmox host NIC link-down regression (≥2 consecutive checks) | P2 High |
 | Host unreachable via ping (≥2 consecutive checks) | P2 High |
 | ≥3 hosts simultaneously reporting interface failures | P1 Critical |
@@ -218,6 +218,7 @@ Shared by both processes. Located in the working directory (`/var/www/html/prod/
   "monitor": {
     "poll_interval": 120,
     "failure_threshold": 2,
+    "unifi_failure_threshold": 5,
     "cluster_threshold": 3,
     "ping_hosts": [
       { "name": "pbs", "ip": "10.10.10.3" }
@@ -243,6 +244,7 @@ Shared by both processes. Located in the working directory (`/var/www/html/prod/
 | `hosts` | Maps Prometheus instance labels → display hostnames |
 | `monitor.poll_interval` | Seconds between full check cycles (default: 120) |
 | `monitor.failure_threshold` | Consecutive failures before creating ticket (default: 2) |
+| `monitor.unifi_failure_threshold` | Consecutive failures before ticketing a UniFi device as offline (default: same as `failure_threshold`). Set higher than `failure_threshold` — UniFi switches/APs can take several minutes to rejoin after a firmware-update reboot, so a low threshold cuts tickets for momentary maintenance reboots instead of real failures. |
 | `monitor.cluster_threshold` | Hosts with failures to trigger cluster-wide P1 (default: 3) |
 | `monitor.ping_hosts` | Hosts checked only by ping (no node_exporter) |
 

app.py

@@ -64,7 +64,7 @@ _diag_rate: dict = {}
 
 
 def _purge_old_jobs_loop():
-    """Background thread: remove stale diag jobs and run daily event purge."""
+    """Background thread: remove stale diagnostic jobs and mark stuck ones done."""
     while True:
         time.sleep(120)
         cutoff = time.time() - 600
@@ -174,17 +174,26 @@ _PAGE_LIMIT = 200  # max events returned per request
 
 
 def _annotate_suppressions(events: list, suppressions: list) -> None:
-    """Annotate each event dict in-place with an is_suppressed bool."""
+    """Annotate each event dict in-place with an is_suppressed bool.
+
+    Mirrors the suppression check order in monitor.py exactly:
+      interface_down  → interface OR host
+      unifi_device_*  → unifi_device
+      everything else → host
+    """
     for ev in events:
-        sup_type = (
+        etype = ev.get('event_type', '')
-            'unifi_device' if ev.get('event_type') == 'unifi_device_offline'
+        name = ev.get('target_name', '')
-            else 'interface' if ev.get('event_type') == 'interface_down'
+        detail = ev.get('target_detail', '') or ''
-            else 'host'
+        if etype == 'interface_down':
-        )
+            ev['is_suppressed'] = (
-        ev['is_suppressed'] = db.check_suppressed(
+                db.check_suppressed(suppressions, 'interface', name, detail) or
-            suppressions, sup_type,
+                db.check_suppressed(suppressions, 'host', name)
-            ev.get('target_name', ''), ev.get('target_detail', '') or '',
+            )
-        )
+        elif etype == 'unifi_device_offline':
+            ev['is_suppressed'] = db.check_suppressed(suppressions, 'unifi_device', name, detail)
+        else:
+            ev['is_suppressed'] = db.check_suppressed(suppressions, 'host', name, detail)
 
 
 # ---------------------------------------------------------------------------

db.py

@@ -3,7 +3,7 @@ import json
 import logging
 import threading
 from contextlib import contextmanager
-from datetime import datetime, timedelta
+from datetime import datetime, timedelta, timezone
 from typing import Optional
 
 import pymysql
@@ -114,12 +114,12 @@ def upsert_event(
     target_detail: str,
     description: str,
 ) -> tuple:
-    """Insert or update a network event. Returns (id, is_new, consecutive_failures)."""
+    """Insert or update a network event. Returns (id, is_new, consecutive_failures, ticket_id)."""
     detail = target_detail or ''
     with get_conn() as conn:
         with conn.cursor() as cur:
             cur.execute(
-                """SELECT id, consecutive_failures FROM network_events
+                """SELECT id, consecutive_failures, ticket_id FROM network_events
                    WHERE event_type=%s AND target_name=%s AND target_detail=%s
                    AND resolved_at IS NULL LIMIT 1""",
                 (event_type, target_name, detail),
@@ -134,7 +134,7 @@ def upsert_event(
                        WHERE id=%s""",
                     (new_count, description, existing['id']),
                 )
-                return existing['id'], False, new_count
+                return existing['id'], False, new_count, existing.get('ticket_id')
             else:
                 cur.execute(
                     """INSERT INTO network_events
@@ -142,7 +142,7 @@ def upsert_event(
                        VALUES (%s, %s, %s, %s, %s, %s)""",
                     (event_type, severity, source_type, target_name, detail, description),
                 )
-                return cur.lastrowid, True, 1
+                return cur.lastrowid, True, 1, None
 
 
 def resolve_event(event_type: str, target_name: str, target_detail: str = '') -> None:
@@ -281,7 +281,7 @@ def create_suppression(
 ) -> int:
     expires_at = None
     if expires_minutes:
-        expires_at = datetime.utcnow() + timedelta(minutes=int(expires_minutes))
+        expires_at = datetime.now(timezone.utc) + timedelta(minutes=int(expires_minutes))
     with get_conn() as conn:
         with conn.cursor() as cur:
             cur.execute(

diagnose.py

@@ -68,7 +68,7 @@ class DiagnosticsRunner:
             f' echo "=== ip_route ===";'
             f' ip route show dev {q} 2>/dev/null;'
             f' echo "=== dmesg ===";'
-            f' dmesg 2>/dev/null | grep {q} | tail -50;'
+            f' dmesg 2>/dev/null | grep -F -- {q} | tail -50;'
             f' echo "=== lldpctl ===";'
             f' lldpctl 2>/dev/null || echo "lldpd not running";'
             f' echo "=== end ==="'

monitor.py

@@ -12,7 +12,7 @@ import logging
 import re
 import shlex
 import time
-from datetime import datetime
+from datetime import datetime, timezone
 from typing import Dict, List, Optional
 
 import requests
@@ -618,7 +618,7 @@ class LinkStatsCollector:
         return {
             'hosts':          result_hosts,
             'unifi_switches': unifi_switches,
-            'updated':        datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S UTC'),
+            'updated':        datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M:%S UTC'),
         }
 
     def _compute_unifi_rates(self, raw: Dict[str, dict], now: float) -> Dict[str, dict]:
@@ -653,7 +653,7 @@ class LinkStatsCollector:
 # Helpers
 # --------------------------------------------------------------------------
 def _now_utc() -> str:
-    return datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S UTC')
+    return datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M:%S UTC')
 
 
 # --------------------------------------------------------------------------
@@ -677,6 +677,11 @@ class NetworkMonitor:
         mon = self.cfg.get('monitor', {})
         self.poll_interval = mon.get('poll_interval', 120)
         self.fail_thresh = mon.get('failure_threshold', 2)
+        # UniFi gear (switches/APs) can take several minutes to rejoin after a
+        # firmware-update reboot, much longer than a Proxmox host link flap.
+        # Use a separate, more forgiving threshold so those momentary reboots
+        # don't cut tickets, while still catching genuine device failures.
+        self.unifi_fail_thresh = mon.get('unifi_failure_threshold', self.fail_thresh)
         self.cluster_thresh = mon.get('cluster_threshold', 3)
         self.cluster_name = mon.get('cluster_name', CLUSTER_NAME)
 
@@ -728,13 +733,13 @@ class NetworkMonitor:
                             db.check_suppressed(suppressions, 'interface', host, iface) or
                             db.check_suppressed(suppressions, 'host', host)
                         )
-                        event_id, is_new, consec = db.upsert_event(
+                        event_id, is_new, consec, ticket_id = db.upsert_event(
                             'interface_down', 'critical', 'prometheus',
                             host, iface,
                             f'Interface {iface} on {host} went link-down ({_now_utc()})',
                         )
-                        if not sup and consec >= self.fail_thresh:
+                        if not sup and consec >= self.fail_thresh and not ticket_id:
-                            self._ticket_interface(event_id, is_new, host, iface, consec)
+                            self._ticket_interface(event_id, host, iface, consec)
 
             if host_has_regression:
                 hosts_with_regression.append(host)
@@ -744,13 +749,13 @@ class NetworkMonitor:
         # Cluster-wide check – only genuine regressions count
         if len(hosts_with_regression) >= self.cluster_thresh:
             sup = db.check_suppressed(suppressions, 'all', '')
-            event_id, is_new, consec = db.upsert_event(
+            event_id, is_new, consec, ticket_id = db.upsert_event(
                 'cluster_network_issue', 'critical', 'prometheus',
                 self.cluster_name, '',
                 f'{len(hosts_with_regression)} hosts reporting simultaneous interface failures: '
                 f'{", ".join(hosts_with_regression)}',
             )
-            if not sup and is_new:
+            if not sup and (is_new or not ticket_id):
                 title = (
                     f'[{self.cluster_name}][auto][production][issue][network][cluster-wide] '
                     f'Multiple hosts reporting interface failures'
@@ -771,7 +776,7 @@ class NetworkMonitor:
             db.resolve_event('cluster_network_issue', self.cluster_name, '')
 
     def _ticket_interface(
-        self, event_id: int, is_new: bool, host: str, iface: str, consec: int
+        self, event_id: int, host: str, iface: str, consec: int
     ) -> None:
         title = (
             f'[{host}][auto][production][issue][network][single-node] '
@@ -804,17 +809,17 @@ class NetworkMonitor:
             name = d['name']
             if not d['connected']:
                 sup = db.check_suppressed(suppressions, 'unifi_device', name)
-                event_id, is_new, consec = db.upsert_event(
+                event_id, is_new, consec, ticket_id = db.upsert_event(
                     'unifi_device_offline', 'critical', 'unifi',
                     name, d.get('type', ''),
                     f'UniFi {name} ({d.get("ip","")}) offline ({_now_utc()})',
                 )
-                if not sup and consec >= self.fail_thresh:
+                if not sup and consec >= self.unifi_fail_thresh and not ticket_id:
-                    self._ticket_unifi(event_id, is_new, d)
+                    self._ticket_unifi(event_id, d)
             else:
                 db.resolve_event('unifi_device_offline', name, d.get('type', ''))
 
-    def _ticket_unifi(self, event_id: int, is_new: bool, device: dict) -> None:
+    def _ticket_unifi(self, event_id: int, device: dict) -> None:
         name = device['name']
         title = (
             f'[{name}][auto][production][issue][network][single-node] '
@@ -837,25 +842,25 @@ class NetworkMonitor:
     # ------------------------------------------------------------------
     # Ping-only hosts (no node_exporter)
     # ------------------------------------------------------------------
-    def _process_ping_hosts(self, suppressions: list) -> None:
+    def _process_ping_hosts(self, suppressions: list, ping_states: Dict[str, bool]) -> None:
         for h in self.cfg.get('monitor', {}).get('ping_hosts', []):
             name, ip = h['name'], h['ip']
-            reachable = self.pulse.ping(ip)
+            reachable = ping_states.get(name, False)
 
             if not reachable:
                 sup = db.check_suppressed(suppressions, 'host', name)
-                event_id, is_new, consec = db.upsert_event(
+                event_id, is_new, consec, ticket_id = db.upsert_event(
                     'host_unreachable', 'critical', 'ping',
                     name, ip,
                     f'Host {name} ({ip}) unreachable via ping ({_now_utc()})',
                 )
-                if not sup and consec >= self.fail_thresh:
+                if not sup and consec >= self.fail_thresh and not ticket_id:
-                    self._ticket_unreachable(event_id, is_new, name, ip, consec)
+                    self._ticket_unreachable(event_id, name, ip, consec)
             else:
                 db.resolve_event('host_unreachable', name, ip)
 
     def _ticket_unreachable(
-        self, event_id: int, is_new: bool, name: str, ip: str, consec: int
+        self, event_id: int, name: str, ip: str, consec: int
     ) -> None:
         title = (
             f'[{name}][auto][production][issue][network][single-node] '
@@ -882,6 +887,7 @@ class NetworkMonitor:
     def _collect_snapshot(
         self, iface_states: Dict[str, Dict[str, bool]],
         unifi_devices: Optional[List[dict]] = None,
+        ping_states: Optional[Dict[str, bool]] = None,
     ) -> dict:
         # Accept pre-fetched devices; fall back to empty list if unavailable
         display_unifi = unifi_devices if unifi_devices is not None else []
@@ -910,7 +916,7 @@ class NetworkMonitor:
 
         for h in self.cfg.get('monitor', {}).get('ping_hosts', []):
             name, ip = h['name'], h['ip']
-            reachable = self.pulse.ping(ip, count=1, timeout=2)
+            reachable = (ping_states or {}).get(name, False)
             hosts[name] = {
                 'ip': ip,
                 'interfaces': {},
@@ -921,7 +927,7 @@ class NetworkMonitor:
         return {
             'hosts': hosts,
             'unifi': display_unifi,
-            'updated': datetime.utcnow().isoformat() + 'Z',
+            'updated': datetime.now(timezone.utc).isoformat().replace('+00:00', 'Z'),
         }
 
     # ------------------------------------------------------------------
@@ -930,7 +936,7 @@ class NetworkMonitor:
     def run(self) -> None:
         logger.info(
             f'Gandalf monitor started – poll_interval={self.poll_interval}s '
-            f'fail_thresh={self.fail_thresh}'
+            f'fail_thresh={self.fail_thresh} unifi_fail_thresh={self.unifi_fail_thresh}'
         )
         while True:
             try:
@@ -942,8 +948,14 @@ class NetworkMonitor:
                 # 2. Fetch UniFi devices once — used by both snapshot and alert processing
                 unifi_devices = self.unifi.get_devices()
 
-                # 3. Collect and store snapshot for dashboard
+                # 3a. Ping-only hosts once — shared by snapshot and alert processing
-                snapshot = self._collect_snapshot(iface_states, unifi_devices)
+                ping_states: Dict[str, bool] = {
+                    h['name']: self.pulse.ping(h['ip'])
+                    for h in self.cfg.get('monitor', {}).get('ping_hosts', [])
+                }
+
+                # 3b. Collect and store snapshot for dashboard
+                snapshot = self._collect_snapshot(iface_states, unifi_devices, ping_states)
                 db.set_state('network_snapshot', snapshot)
                 db.set_state('last_check', _now_utc())
 
@@ -959,7 +971,7 @@ class NetworkMonitor:
                 self._process_interfaces(iface_states, suppressions)
                 self._process_unifi(unifi_devices, suppressions)
 
-                self._process_ping_hosts(suppressions)
+                self._process_ping_hosts(suppressions, ping_states)
 
                 # Housekeeping: deactivate expired suppressions and purge old resolved events
                 db.cleanup_expired_suppressions()
@@ -970,6 +982,7 @@ class NetworkMonitor:
             except Exception as e:
                 logger.error(f'Monitor loop error: {e}', exc_info=True)
                 time.sleep(30)
+                continue
 
             time.sleep(self.poll_interval)
 

templates/index.html

@@ -324,6 +324,7 @@
     </div>
   </div>
   <div class="host-grid" id="host-grid">
+    {%- set has_global_sup = suppressions | selectattr('target_type', 'equalto', 'all') | list | length > 0 -%}
     {% for name, host in snapshot.hosts.items() %}
     {% set suppressed = suppressions | selectattr('target_name', 'equalto', name) | list %}
     <div class="host-card host-card-{{ host.status }}" data-host="{{ name }}">
@@ -331,7 +332,7 @@
         <div class="host-name-row">
           <span class="host-status-dot dot-{{ host.status }}"></span>
           <span class="host-name">{{ name }}</span>
-          {% if suppressed %}
+          {% if suppressed or has_global_sup %}
           <span class="badge-suppressed" title="Suppressed">🔕</span>
           {% endif %}
         </div>
@@ -468,7 +469,7 @@
 {% block scripts %}
 <script>
   // Start auto-refresh using saved settings interval (default 30 s)
-  const _savedInterval = (window.gandalfSettings && window.gandalfSettings.refreshInterval) || 30;
+  const _savedInterval = window.gandalfSettings?.refreshInterval ?? 30;
   if (_savedInterval > 0) lt.autoRefresh.start(refreshAll, _savedInterval * 1000);
 
   // When settings change, restart auto-refresh with new interval

templates/inspector.html

@@ -473,7 +473,7 @@ async function loadInspector() {
 }
 
 loadInspector();
-const _inspInterval = (window.gandalfSettings && window.gandalfSettings.refreshInterval) || 60;
+const _inspInterval = window.gandalfSettings?.refreshInterval ?? 60;
 if (_inspInterval > 0) lt.autoRefresh.start(loadInspector, Math.max(_inspInterval, 15) * 1000);
 
 window.onGandalfSettingsChanged = function(s) {

templates/links.html

@@ -571,7 +571,7 @@ async function loadLinks() {
 }
 
 loadLinks();
-const _linksInterval = (window.gandalfSettings && window.gandalfSettings.refreshInterval) || 60;
+const _linksInterval = window.gandalfSettings?.refreshInterval ?? 60;
 if (_linksInterval > 0) lt.autoRefresh.start(loadLinks, Math.max(_linksInterval, 15) * 1000);
 
 window.onGandalfSettingsChanged = function(s) {

tests/test_diagnose.py

@@ -36,6 +36,12 @@ class TestBuildSshCommand:
         cmd = DiagnosticsRunner.build_ssh_command('10.0.0.1', 'eth0')
         assert 'ethtool' in cmd
 
+    def test_dmesg_uses_fixed_string_grep(self):
+        # grep -F prevents iface names with dots (e.g. eth0.1) being treated as
+        # regex wildcards; -- prevents leading - from being parsed as a flag
+        cmd = DiagnosticsRunner.build_ssh_command('10.0.0.1', 'eth0')
+        assert 'grep -F --' in cmd
+
 
 # ── parse_output ─────────────────────────────────────────────────────────────