Key network dedup tickets on the problem, not just the host (#116)

Network tickets never set issue_subtype, so every automated network alert
for a host hashed alike: Gandalf's "Interface enp3s0 link-down" for micro1
matched closed reboot ticket 993441773, got noted there as a recurrence,
and never got its own ticket.

Automated network tickets now key on the optional `check_code` payload
field when sent (it must name the instance, e.g. iface_down:enp3s0),
otherwise on the title text outside the [tags], normalised with digits
kept. Manual tickets and every other category hash exactly as before.

The hash logic moves to helpers/TicketHash.php so the new
scripts/rehash_network_tickets.php can re-key open network tickets once
after deploy (--dry-run first). Closed tickets keep their old hash so they
stop catching unrelated alerts.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

README.md

@@ -225,6 +225,7 @@ Content-Type: application/json
 - Authenticated via `Authorization: Bearer` header — API key stored in `/etc/hwmonDaemon/.env`
 - **Deduplication**: Generates a SHA-256 hash from the issue category, hostname, and device (no time window). A repeat alert matching an existing **open** ticket updates its title/description and escalates the priority if the condition worsened; if the matching ticket was already **closed**, it is reopened instead of creating a new one
 - Cluster-wide issues (Ceph health, etc.) deduplicate across all nodes (hostname excluded from hash)
+- Automated network tickets are keyed per problem, not just per host: on the optional `check_code` payload field when sent (it must name the instance, e.g. `iface_down:enp3s0`), otherwise on the title text outside the `[tags]`. After deploying this change (#116), run `php scripts/rehash_network_tickets.php --dry-run`, then without `--dry-run`, so open network tickets keep matching their alerts. The hash logic lives in `helpers/TicketHash.php`
 - Matrix notification sent automatically on ticket creation, priority escalation, and reopen
 - API key must be generated at `/admin/api-keys`; the key goes in hwmonDaemon's `/etc/hwmonDaemon/.env` as `TICKET_API_KEY`
 

create_ticket_api.php

@@ -45,6 +45,7 @@ require_once __DIR__ . '/models/StatsModel.php';
 require_once __DIR__ . '/models/TicketModel.php';
 require_once __DIR__ . '/models/WorkflowModel.php';
 require_once __DIR__ . '/helpers/UrlHelper.php';
+require_once __DIR__ . '/helpers/TicketHash.php';
 
 $apiKeyAuth = new ApiKeyAuth($conn);
 
@@ -80,148 +81,6 @@ if (!is_array($data) || empty($data['title'])) {
     }
 }
 
-// Generate hash from stable components
-function generateTicketHash($data)
-{
-    $title = (string)($data['title'] ?? '');
-
-    // Prefer explicit serial from payload; fall back to extracting device path from title
-    // for backwards compatibility with older hwmonDaemon versions.
-    $serial = isset($data['serial']) && $data['serial'] !== null && $data['serial'] !== ''
-        ? (string)$data['serial']
-        : null;
-
-    // Extract device name if present (matches /dev/sdX, /dev/nvmeXnY patterns)
-    preg_match('/\/dev\/(sd[a-z]+|nvme\d+n\d+)/', $title, $deviceMatches);
-    $isDriveTicket = !empty($deviceMatches) || $serial !== null;
-
-    // Extract first bracketed tag as hostname/source
-    preg_match('/^\[([^\]]+)\]/', $title, $hostMatches);
-    $hostname = $hostMatches[1] ?? '';
-
-    // Detect issue category and optional sub-type
-    $issueCategory = '';
-    $issueSubtype  = '';
-    $isClusterWide = false;
-
-    if (stripos($title, 'SMART issues') !== false) {
-        $issueCategory = 'smart';
-    } elseif (stripos($title, 'ZFS pool') !== false) {
-        $issueCategory = 'zfs';
-        // Extract pool name so each pool gets its own ticket
-        if (preg_match("/ZFS pool '([^']+)'/i", $title, $poolMatch)) {
-            $poolName = strtolower(preg_replace('/[^a-z0-9_]/i', '_', $poolMatch[1]));
-            if (stripos($title, 'state:') !== false || preg_match('/DEGRADED|FAULTED|UNAVAIL|OFFLINE/i', $title)) {
-                $issueSubtype = 'pool_state_' . $poolName;
-            } elseif (stripos($title, 'usage') !== false) {
-                $issueSubtype = 'pool_usage_' . $poolName;
-            } elseif (stripos($title, 'errors') !== false) {
-                $issueSubtype = 'pool_errors_' . $poolName;
-            } else {
-                $issueSubtype = 'pool_' . $poolName;
-            }
-        }
-    } elseif (stripos($title, 'LXC') !== false || stripos($title, 'storage usage') !== false) {
-        $issueCategory = 'storage';
-        // Include the LXC container ID so each container gets its own ticket
-        if (preg_match('/LXC\s+(\d+)/i', $title, $lxcMatch)) {
-            $issueSubtype = 'lxc_' . $lxcMatch[1];
-        }
-    } elseif (stripos($title, 'memory') !== false) {
-        $issueCategory = 'memory';
-    } elseif (stripos($title, 'cpu') !== false) {
-        $issueCategory = 'cpu';
-    } elseif (stripos($title, 'network') !== false) {
-        $issueCategory = 'network';
-    } elseif (stripos($title, 'Ceph') !== false || stripos($title, '[ceph]') !== false) {
-        $issueCategory = 'ceph';
-        if (
-            stripos($title, '[cluster-wide]') !== false ||
-            stripos($title, 'HEALTH_ERR') !== false ||
-            stripos($title, 'HEALTH_WARN') !== false ||
-            stripos($title, 'cluster usage') !== false
-        ) {
-            $isClusterWide = true;
-        }
-        // Normalize the specific Ceph warning type so different warnings get distinct tickets
-        if (stripos($title, 'slow') !== false && stripos($title, 'BlueStore') !== false) {
-            $issueSubtype = 'bluestore_slow';
-        } elseif (stripos($title, 'clock skew') !== false) {
-            $issueSubtype = 'clock_skew';
-        } elseif (stripos($title, 'cluster usage') !== false) {
-            $issueSubtype = 'usage';
-        } elseif (stripos($title, 'OSD down') !== false || preg_match('/osd\.\d+\s+is\s+DOWN/i', $title)) {
-            // Include the specific OSD ID so each individual OSD gets its own ticket
-            if (preg_match('/osd\.(\d+)/i', $title, $osdMatch)) {
-                $issueSubtype = 'osd_down_' . $osdMatch[1];
-            } else {
-                $issueSubtype = 'osd_down';
-            }
-        } elseif (stripos($title, 'HEALTH_ERR') !== false) {
-            $issueSubtype = 'health_err';
-        } else {
-            // Any other warning gets its own ticket. Previously every other
-            // HEALTH_WARN shared one empty subtype, so unrelated warnings
-            // (noout, nearfull, insecure keys, ...) collapsed onto a single
-            // ticket and overwrote its title in turn every run (#112).
-            // Prefer Ceph's own health-check code when the reporter sends
-            // it; otherwise key on the warning text with counts stripped.
-            $checkCode = isset($data['check_code']) && is_string($data['check_code']) ? $data['check_code'] : '';
-            if ($checkCode !== '' && preg_match('/^[A-Z0-9_]{1,64}$/', $checkCode)) {
-                $issueSubtype = 'check_' . strtolower($checkCode);
-            } elseif (preg_match('/HEALTH_WARN:\s*([^\[]+)/i', $title, $warnMatch)) {
-                $warnText = strtolower(explode(':', $warnMatch[1])[0]);
-                $warnText = trim(preg_replace('/[^a-z]+/', '_', $warnText), '_');
-                if ($warnText !== '') {
-                    $issueSubtype = 'warn_' . $warnText;
-                }
-            }
-        }
-    }
-
-    // Include source type so automated tickets never collide with manual ones
-    $sourceType = stripos($title, '[auto]') !== false ? 'auto' : 'manual';
-
-    // Build stable components
-    $stableComponents = [
-        'source_type'    => $sourceType,
-        'issue_category' => $issueCategory,
-        'issue_subtype'  => $issueSubtype,
-        'environment_tags' => (function () use ($title) {
-            // Extract each [bracketed] tag, then keep the known environment ones.
-            // (explode('][') leaves brackets stuck to the first/last tag, so e.g.
-            // "[production] ..." never matched and the env tag was dropped from the
-            // dedup hash — letting prod and staging issues collide onto one ticket.)
-            preg_match_all('/\[([^\]]+)\]/', $title, $m);
-            return array_values(array_filter(
-                $m[1],
-                fn($tag) => in_array($tag, ['production', 'development', 'staging', 'single-node', 'cluster-wide'], true)
-            ));
-        })(),
-    ];
-
-    // Manual tickets should be unique by title (so different software installs don't collide)
-    if ($sourceType === 'manual') {
-        $stableComponents['title'] = $title;
-    }
-
-    // Include hostname for node-specific issues
-    if (!$isClusterWide) {
-        $stableComponents['hostname'] = $hostname;
-    }
-
-    // Include drive identifier for drive-specific tickets.
-    // Use serial when available (stable across reboots/reshuffles); fall back to
-    // device path for tickets created before serial was added to the payload.
-    if ($isDriveTicket) {
-        $stableComponents['drive'] = $serial ?? ($deviceMatches[0] ?? '');
-    }
-
-    sort($stableComponents['environment_tags']);
-
-    return hash('sha256', json_encode($stableComponents, JSON_UNESCAPED_SLASHES));
-}
-
 // Shared ticket data
 $title       = (string)($data['title']       ?? '');
 $description = (string)($data['description'] ?? '');
@@ -246,7 +105,7 @@ if (!is_numeric($priority) || (int)$priority < 1 || (int)$priority > 5) {
 }
 $priority = (int)$priority;
 
-$ticketHash = generateTicketHash($data);
+$ticketHash = TicketHash::generate($data);
 $auditLog   = new AuditLogModel($conn);
 
 // Everything from here through either updating/reopening the matched ticket

helpers/TicketHash.php

@@ -0,0 +1,186 @@
+<?php
+
+/**
+ * Dedup hash for tickets created through create_ticket_api.php.
+ *
+ * Two reports with the same hash land on the same ticket (updated while open,
+ * reopened when closed), so the components below decide what counts as "the
+ * same problem". Changing them re-keys existing tickets: see
+ * scripts/rehash_network_tickets.php for how #116 handled that.
+ */
+class TicketHash
+{
+    /**
+     * The stable components the hash is built from.
+     *
+     * @param array $data Request payload: title, plus optional serial / check_code.
+     */
+    public static function components(array $data): array
+    {
+        $title = (string)($data['title'] ?? '');
+
+        // Prefer explicit serial from payload; fall back to extracting device path from title
+        // for backwards compatibility with older hwmonDaemon versions.
+        $serial = isset($data['serial']) && $data['serial'] !== null && $data['serial'] !== ''
+            ? (string)$data['serial']
+            : null;
+
+        // Extract device name if present (matches /dev/sdX, /dev/nvmeXnY patterns)
+        preg_match('/\/dev\/(sd[a-z]+|nvme\d+n\d+)/', $title, $deviceMatches);
+        $isDriveTicket = !empty($deviceMatches) || $serial !== null;
+
+        // Extract first bracketed tag as hostname/source
+        preg_match('/^\[([^\]]+)\]/', $title, $hostMatches);
+        $hostname = $hostMatches[1] ?? '';
+
+        // Detect issue category and optional sub-type
+        $issueCategory = '';
+        $issueSubtype  = '';
+        $isClusterWide = false;
+
+        if (stripos($title, 'SMART issues') !== false) {
+            $issueCategory = 'smart';
+        } elseif (stripos($title, 'ZFS pool') !== false) {
+            $issueCategory = 'zfs';
+            // Extract pool name so each pool gets its own ticket
+            if (preg_match("/ZFS pool '([^']+)'/i", $title, $poolMatch)) {
+                $poolName = strtolower(preg_replace('/[^a-z0-9_]/i', '_', $poolMatch[1]));
+                if (stripos($title, 'state:') !== false || preg_match('/DEGRADED|FAULTED|UNAVAIL|OFFLINE/i', $title)) {
+                    $issueSubtype = 'pool_state_' . $poolName;
+                } elseif (stripos($title, 'usage') !== false) {
+                    $issueSubtype = 'pool_usage_' . $poolName;
+                } elseif (stripos($title, 'errors') !== false) {
+                    $issueSubtype = 'pool_errors_' . $poolName;
+                } else {
+                    $issueSubtype = 'pool_' . $poolName;
+                }
+            }
+        } elseif (stripos($title, 'LXC') !== false || stripos($title, 'storage usage') !== false) {
+            $issueCategory = 'storage';
+            // Include the LXC container ID so each container gets its own ticket
+            if (preg_match('/LXC\s+(\d+)/i', $title, $lxcMatch)) {
+                $issueSubtype = 'lxc_' . $lxcMatch[1];
+            }
+        } elseif (stripos($title, 'memory') !== false) {
+            $issueCategory = 'memory';
+        } elseif (stripos($title, 'cpu') !== false) {
+            $issueCategory = 'cpu';
+        } elseif (stripos($title, 'network') !== false) {
+            $issueCategory = 'network';
+            // One ticket per kind of network problem per host, not one per
+            // host. With an empty subtype every network alert for a host
+            // (reboot, NIC link-down, cable mismatch, ...) hashed alike, so an
+            // interface alert landed on an unrelated closed reboot ticket
+            // (#116). Prefer the reporter's check_code when sent; it must name
+            // the instance too (e.g. "iface_down:enp3s0"), not just the alert
+            // type. Otherwise key on the title text outside the [tags]; the
+            // reporters' headings are stable per problem, and digits are kept
+            // because they identify it (enp3s0, cable C07). Manual tickets
+            // are already keyed on their full title, so they are left as-is.
+            $checkCode = isset($data['check_code']) && is_string($data['check_code']) ? $data['check_code'] : '';
+            if (stripos($title, '[auto]') === false) {
+                // manual: unchanged hash
+            } elseif ($checkCode !== '' && preg_match('/^[A-Za-z0-9_.:-]{1,80}$/', $checkCode)) {
+                $issueSubtype = 'net_check_' . strtolower($checkCode);
+            } else {
+                $heading = trim(preg_replace('/\[[^\]]*\]/', '', $title), " -\t");
+                $slug = trim(preg_replace('/[^a-z0-9]+/', '_', strtolower($heading)), '_');
+                if ($slug !== '') {
+                    $issueSubtype = 'net_' . substr($slug, 0, 80);
+                }
+            }
+        } elseif (stripos($title, 'Ceph') !== false || stripos($title, '[ceph]') !== false) {
+            $issueCategory = 'ceph';
+            if (
+                stripos($title, '[cluster-wide]') !== false ||
+                stripos($title, 'HEALTH_ERR') !== false ||
+                stripos($title, 'HEALTH_WARN') !== false ||
+                stripos($title, 'cluster usage') !== false
+            ) {
+                $isClusterWide = true;
+            }
+            // Normalize the specific Ceph warning type so different warnings get distinct tickets
+            if (stripos($title, 'slow') !== false && stripos($title, 'BlueStore') !== false) {
+                $issueSubtype = 'bluestore_slow';
+            } elseif (stripos($title, 'clock skew') !== false) {
+                $issueSubtype = 'clock_skew';
+            } elseif (stripos($title, 'cluster usage') !== false) {
+                $issueSubtype = 'usage';
+            } elseif (stripos($title, 'OSD down') !== false || preg_match('/osd\.\d+\s+is\s+DOWN/i', $title)) {
+                // Include the specific OSD ID so each individual OSD gets its own ticket
+                if (preg_match('/osd\.(\d+)/i', $title, $osdMatch)) {
+                    $issueSubtype = 'osd_down_' . $osdMatch[1];
+                } else {
+                    $issueSubtype = 'osd_down';
+                }
+            } elseif (stripos($title, 'HEALTH_ERR') !== false) {
+                $issueSubtype = 'health_err';
+            } else {
+                // Any other warning gets its own ticket. Previously every other
+                // HEALTH_WARN shared one empty subtype, so unrelated warnings
+                // (noout, nearfull, insecure keys, ...) collapsed onto a single
+                // ticket and overwrote its title in turn every run (#112).
+                // Prefer Ceph's own health-check code when the reporter sends
+                // it; otherwise key on the warning text with counts stripped.
+                $checkCode = isset($data['check_code']) && is_string($data['check_code']) ? $data['check_code'] : '';
+                if ($checkCode !== '' && preg_match('/^[A-Z0-9_]{1,64}$/', $checkCode)) {
+                    $issueSubtype = 'check_' . strtolower($checkCode);
+                } elseif (preg_match('/HEALTH_WARN:\s*([^\[]+)/i', $title, $warnMatch)) {
+                    $warnText = strtolower(explode(':', $warnMatch[1])[0]);
+                    $warnText = trim(preg_replace('/[^a-z]+/', '_', $warnText), '_');
+                    if ($warnText !== '') {
+                        $issueSubtype = 'warn_' . $warnText;
+                    }
+                }
+            }
+        }
+
+        // Include source type so automated tickets never collide with manual ones
+        $sourceType = stripos($title, '[auto]') !== false ? 'auto' : 'manual';
+
+        // Build stable components
+        $stableComponents = [
+            'source_type'    => $sourceType,
+            'issue_category' => $issueCategory,
+            'issue_subtype'  => $issueSubtype,
+            'environment_tags' => (function () use ($title) {
+                // Extract each [bracketed] tag, then keep the known environment ones.
+                // (explode('][') leaves brackets stuck to the first/last tag, so e.g.
+                // "[production] ..." never matched and the env tag was dropped from the
+                // dedup hash — letting prod and staging issues collide onto one ticket.)
+                preg_match_all('/\[([^\]]+)\]/', $title, $m);
+                return array_values(array_filter(
+                    $m[1],
+                    fn($tag) => in_array($tag, ['production', 'development', 'staging', 'single-node', 'cluster-wide'], true)
+                ));
+            })(),
+        ];
+
+        // Manual tickets should be unique by title (so different software installs don't collide)
+        if ($sourceType === 'manual') {
+            $stableComponents['title'] = $title;
+        }
+
+        // Include hostname for node-specific issues
+        if (!$isClusterWide) {
+            $stableComponents['hostname'] = $hostname;
+        }
+
+        // Include drive identifier for drive-specific tickets.
+        // Use serial when available (stable across reboots/reshuffles); fall back to
+        // device path for tickets created before serial was added to the payload.
+        if ($isDriveTicket) {
+            $stableComponents['drive'] = $serial ?? ($deviceMatches[0] ?? '');
+        }
+
+        sort($stableComponents['environment_tags']);
+
+        return $stableComponents;
+    }
+
+    /** SHA-256 dedup hash of the payload's stable components. */
+    public static function generate(array $data): string
+    {
+        return hash('sha256', json_encode(self::components($data), JSON_UNESCAPED_SLASHES));
+    }
+}

scripts/rehash_network_tickets.php

@@ -0,0 +1,78 @@
+#!/usr/bin/env php
+<?php
+
+/**
+ * Re-key open automated network tickets after #116.
+ *
+ * Network tickets used to hash on host alone, so every network alert for a
+ * host shared one ticket. They now hash on host + the problem's heading (see
+ * helpers/TicketHash.php). Without this script, each open network ticket
+ * keeps its old hash and its next alert opens a duplicate instead of
+ * updating it. Run once after deploying #116:
+ *
+ *   php scripts/rehash_network_tickets.php --dry-run   # report only
+ *   php scripts/rehash_network_tickets.php
+ *
+ * Closed tickets keep their old hash on purpose: they stop catching
+ * unrelated alerts. A ticket whose new hash already belongs to another ticket
+ * is skipped and reported (resolve it by hand). Safe to re-run.
+ */
+
+if (php_sapi_name() !== 'cli') {
+    http_response_code(403);
+    exit('CLI access only');
+}
+
+require_once dirname(__DIR__) . '/config/config.php';
+require_once dirname(__DIR__) . '/helpers/Database.php';
+require_once dirname(__DIR__) . '/helpers/TicketHash.php';
+
+$dryRun = in_array('--dry-run', $argv, true);
+$conn = Database::getConnection();
+
+$rows = $conn->query("SELECT ticket_id, title, hash FROM tickets WHERE status <> 'Closed' ORDER BY ticket_id")
+    ->fetch_all(MYSQLI_ASSOC);
+
+$counts = ['updated' => 0, 'unchanged' => 0, 'conflict' => 0];
+$lookup = $conn->prepare("SELECT ticket_id FROM tickets WHERE hash = ? AND ticket_id <> ? LIMIT 1");
+$update = $conn->prepare("UPDATE tickets SET hash = ? WHERE ticket_id = ?");
+
+foreach ($rows as $row) {
+    $data = ['title' => (string)$row['title']];
+    $components = TicketHash::components($data);
+    if ($components['issue_category'] !== 'network' || $components['source_type'] !== 'auto') {
+        continue;
+    }
+    $newHash = TicketHash::generate($data);
+    $id = $row['ticket_id'];
+
+    if ($newHash === $row['hash']) {
+        $counts['unchanged']++;
+        echo "unchanged  $id  {$row['title']}\n";
+        continue;
+    }
+
+    $lookup->bind_param('ss', $newHash, $id);
+    $lookup->execute();
+    $other = $lookup->get_result()->fetch_assoc();
+    if ($other) {
+        $counts['conflict']++;
+        echo "CONFLICT   $id  {$row['title']}  (new hash already on ticket {$other['ticket_id']})\n";
+        continue;
+    }
+
+    if (!$dryRun) {
+        $update->bind_param('ss', $newHash, $id);
+        $update->execute();
+    }
+    $counts['updated']++;
+    echo ($dryRun ? 'would update' : 'updated   ') . " $id  {$row['title']}\n";
+}
+
+printf(
+    "%s%d updated, %d already current, %d conflicts\n",
+    $dryRun ? '[dry run] ' : '',
+    $counts['updated'],
+    $counts['unchanged'],
+    $counts['conflict']
+);