diff --git a/README.md b/README.md index edd1f16..32420b1 100644 --- a/README.md +++ b/README.md @@ -281,6 +281,7 @@ Content-Type: application/json - Authenticated via `Authorization: Bearer` header — API key stored in `/etc/hwmonDaemon/.env` - **Deduplication**: Generates a SHA-256 hash from the issue category, hostname, and device (no time window). A repeat alert matching an existing **open** ticket updates its title/description and escalates the priority if the condition worsened; if the matching ticket was already **closed**, it is reopened instead of creating a new one - Cluster-wide issues (Ceph health, etc.) deduplicate across all nodes (hostname excluded from hash) +- Automated network tickets are keyed per problem, not just per host: on the optional `check_code` payload field when sent (it must name the instance, e.g. `iface_down:enp3s0`), otherwise on the title text outside the `[tags]`. After deploying this change (#116), run `php scripts/rehash_network_tickets.php --dry-run`, then without `--dry-run`, so open network tickets keep matching their alerts. The hash logic lives in `helpers/TicketHash.php` - Matrix notification sent automatically on ticket creation, priority escalation, and reopen - API key must be generated at `/admin/api-keys`; the key goes in hwmonDaemon's `/etc/hwmonDaemon/.env` as `TICKET_API_KEY` diff --git a/create_ticket_api.php b/create_ticket_api.php index 290f972..ba2a3c2 100644 --- a/create_ticket_api.php +++ b/create_ticket_api.php @@ -45,6 +45,7 @@ require_once __DIR__ . '/models/StatsModel.php'; require_once __DIR__ . '/models/TicketModel.php'; require_once __DIR__ . '/models/WorkflowModel.php'; require_once __DIR__ . '/helpers/UrlHelper.php'; +require_once __DIR__ . '/helpers/TicketHash.php'; $apiKeyAuth = new ApiKeyAuth($conn); @@ -80,148 +81,6 @@ if (!is_array($data) || empty($data['title'])) { } } -// Generate hash from stable components -function generateTicketHash($data) -{ - $title = (string)($data['title'] ?? ''); - - // Prefer explicit serial from payload; fall back to extracting device path from title - // for backwards compatibility with older hwmonDaemon versions. - $serial = isset($data['serial']) && $data['serial'] !== null && $data['serial'] !== '' - ? (string)$data['serial'] - : null; - - // Extract device name if present (matches /dev/sdX, /dev/nvmeXnY patterns) - preg_match('/\/dev\/(sd[a-z]+|nvme\d+n\d+)/', $title, $deviceMatches); - $isDriveTicket = !empty($deviceMatches) || $serial !== null; - - // Extract first bracketed tag as hostname/source - preg_match('/^\[([^\]]+)\]/', $title, $hostMatches); - $hostname = $hostMatches[1] ?? ''; - - // Detect issue category and optional sub-type - $issueCategory = ''; - $issueSubtype = ''; - $isClusterWide = false; - - if (stripos($title, 'SMART issues') !== false) { - $issueCategory = 'smart'; - } elseif (stripos($title, 'ZFS pool') !== false) { - $issueCategory = 'zfs'; - // Extract pool name so each pool gets its own ticket - if (preg_match("/ZFS pool '([^']+)'/i", $title, $poolMatch)) { - $poolName = strtolower(preg_replace('/[^a-z0-9_]/i', '_', $poolMatch[1])); - if (stripos($title, 'state:') !== false || preg_match('/DEGRADED|FAULTED|UNAVAIL|OFFLINE/i', $title)) { - $issueSubtype = 'pool_state_' . $poolName; - } elseif (stripos($title, 'usage') !== false) { - $issueSubtype = 'pool_usage_' . $poolName; - } elseif (stripos($title, 'errors') !== false) { - $issueSubtype = 'pool_errors_' . $poolName; - } else { - $issueSubtype = 'pool_' . $poolName; - } - } - } elseif (stripos($title, 'LXC') !== false || stripos($title, 'storage usage') !== false) { - $issueCategory = 'storage'; - // Include the LXC container ID so each container gets its own ticket - if (preg_match('/LXC\s+(\d+)/i', $title, $lxcMatch)) { - $issueSubtype = 'lxc_' . $lxcMatch[1]; - } - } elseif (stripos($title, 'memory') !== false) { - $issueCategory = 'memory'; - } elseif (stripos($title, 'cpu') !== false) { - $issueCategory = 'cpu'; - } elseif (stripos($title, 'network') !== false) { - $issueCategory = 'network'; - } elseif (stripos($title, 'Ceph') !== false || stripos($title, '[ceph]') !== false) { - $issueCategory = 'ceph'; - if ( - stripos($title, '[cluster-wide]') !== false || - stripos($title, 'HEALTH_ERR') !== false || - stripos($title, 'HEALTH_WARN') !== false || - stripos($title, 'cluster usage') !== false - ) { - $isClusterWide = true; - } - // Normalize the specific Ceph warning type so different warnings get distinct tickets - if (stripos($title, 'slow') !== false && stripos($title, 'BlueStore') !== false) { - $issueSubtype = 'bluestore_slow'; - } elseif (stripos($title, 'clock skew') !== false) { - $issueSubtype = 'clock_skew'; - } elseif (stripos($title, 'cluster usage') !== false) { - $issueSubtype = 'usage'; - } elseif (stripos($title, 'OSD down') !== false || preg_match('/osd\.\d+\s+is\s+DOWN/i', $title)) { - // Include the specific OSD ID so each individual OSD gets its own ticket - if (preg_match('/osd\.(\d+)/i', $title, $osdMatch)) { - $issueSubtype = 'osd_down_' . $osdMatch[1]; - } else { - $issueSubtype = 'osd_down'; - } - } elseif (stripos($title, 'HEALTH_ERR') !== false) { - $issueSubtype = 'health_err'; - } else { - // Any other warning gets its own ticket. Previously every other - // HEALTH_WARN shared one empty subtype, so unrelated warnings - // (noout, nearfull, insecure keys, ...) collapsed onto a single - // ticket and overwrote its title in turn every run (#112). - // Prefer Ceph's own health-check code when the reporter sends - // it; otherwise key on the warning text with counts stripped. - $checkCode = isset($data['check_code']) && is_string($data['check_code']) ? $data['check_code'] : ''; - if ($checkCode !== '' && preg_match('/^[A-Z0-9_]{1,64}$/', $checkCode)) { - $issueSubtype = 'check_' . strtolower($checkCode); - } elseif (preg_match('/HEALTH_WARN:\s*([^\[]+)/i', $title, $warnMatch)) { - $warnText = strtolower(explode(':', $warnMatch[1])[0]); - $warnText = trim(preg_replace('/[^a-z]+/', '_', $warnText), '_'); - if ($warnText !== '') { - $issueSubtype = 'warn_' . $warnText; - } - } - } - } - - // Include source type so automated tickets never collide with manual ones - $sourceType = stripos($title, '[auto]') !== false ? 'auto' : 'manual'; - - // Build stable components - $stableComponents = [ - 'source_type' => $sourceType, - 'issue_category' => $issueCategory, - 'issue_subtype' => $issueSubtype, - 'environment_tags' => (function () use ($title) { - // Extract each [bracketed] tag, then keep the known environment ones. - // (explode('][') leaves brackets stuck to the first/last tag, so e.g. - // "[production] ..." never matched and the env tag was dropped from the - // dedup hash — letting prod and staging issues collide onto one ticket.) - preg_match_all('/\[([^\]]+)\]/', $title, $m); - return array_values(array_filter( - $m[1], - fn($tag) => in_array($tag, ['production', 'development', 'staging', 'single-node', 'cluster-wide'], true) - )); - })(), - ]; - - // Manual tickets should be unique by title (so different software installs don't collide) - if ($sourceType === 'manual') { - $stableComponents['title'] = $title; - } - - // Include hostname for node-specific issues - if (!$isClusterWide) { - $stableComponents['hostname'] = $hostname; - } - - // Include drive identifier for drive-specific tickets. - // Use serial when available (stable across reboots/reshuffles); fall back to - // device path for tickets created before serial was added to the payload. - if ($isDriveTicket) { - $stableComponents['drive'] = $serial ?? ($deviceMatches[0] ?? ''); - } - - sort($stableComponents['environment_tags']); - - return hash('sha256', json_encode($stableComponents, JSON_UNESCAPED_SLASHES)); -} - // Shared ticket data $title = (string)($data['title'] ?? ''); $description = (string)($data['description'] ?? ''); @@ -246,7 +105,7 @@ if (!is_numeric($priority) || (int)$priority < 1 || (int)$priority > 5) { } $priority = (int)$priority; -$ticketHash = generateTicketHash($data); +$ticketHash = TicketHash::generate($data); $auditLog = new AuditLogModel($conn); // Everything from here through either updating/reopening the matched ticket diff --git a/helpers/TicketHash.php b/helpers/TicketHash.php new file mode 100644 index 0000000..2b0a2a9 --- /dev/null +++ b/helpers/TicketHash.php @@ -0,0 +1,186 @@ + $sourceType, + 'issue_category' => $issueCategory, + 'issue_subtype' => $issueSubtype, + 'environment_tags' => (function () use ($title) { + // Extract each [bracketed] tag, then keep the known environment ones. + // (explode('][') leaves brackets stuck to the first/last tag, so e.g. + // "[production] ..." never matched and the env tag was dropped from the + // dedup hash — letting prod and staging issues collide onto one ticket.) + preg_match_all('/\[([^\]]+)\]/', $title, $m); + return array_values(array_filter( + $m[1], + fn($tag) => in_array($tag, ['production', 'development', 'staging', 'single-node', 'cluster-wide'], true) + )); + })(), + ]; + + // Manual tickets should be unique by title (so different software installs don't collide) + if ($sourceType === 'manual') { + $stableComponents['title'] = $title; + } + + // Include hostname for node-specific issues + if (!$isClusterWide) { + $stableComponents['hostname'] = $hostname; + } + + // Include drive identifier for drive-specific tickets. + // Use serial when available (stable across reboots/reshuffles); fall back to + // device path for tickets created before serial was added to the payload. + if ($isDriveTicket) { + $stableComponents['drive'] = $serial ?? ($deviceMatches[0] ?? ''); + } + + sort($stableComponents['environment_tags']); + + return $stableComponents; + } + + /** SHA-256 dedup hash of the payload's stable components. */ + public static function generate(array $data): string + { + return hash('sha256', json_encode(self::components($data), JSON_UNESCAPED_SLASHES)); + } +} diff --git a/scripts/rehash_network_tickets.php b/scripts/rehash_network_tickets.php new file mode 100644 index 0000000..cde01aa --- /dev/null +++ b/scripts/rehash_network_tickets.php @@ -0,0 +1,78 @@ +#!/usr/bin/env php +query("SELECT ticket_id, title, hash FROM tickets WHERE status <> 'Closed' ORDER BY ticket_id") + ->fetch_all(MYSQLI_ASSOC); + +$counts = ['updated' => 0, 'unchanged' => 0, 'conflict' => 0]; +$lookup = $conn->prepare("SELECT ticket_id FROM tickets WHERE hash = ? AND ticket_id <> ? LIMIT 1"); +$update = $conn->prepare("UPDATE tickets SET hash = ? WHERE ticket_id = ?"); + +foreach ($rows as $row) { + $data = ['title' => (string)$row['title']]; + $components = TicketHash::components($data); + if ($components['issue_category'] !== 'network' || $components['source_type'] !== 'auto') { + continue; + } + $newHash = TicketHash::generate($data); + $id = $row['ticket_id']; + + if ($newHash === $row['hash']) { + $counts['unchanged']++; + echo "unchanged $id {$row['title']}\n"; + continue; + } + + $lookup->bind_param('ss', $newHash, $id); + $lookup->execute(); + $other = $lookup->get_result()->fetch_assoc(); + if ($other) { + $counts['conflict']++; + echo "CONFLICT $id {$row['title']} (new hash already on ticket {$other['ticket_id']})\n"; + continue; + } + + if (!$dryRun) { + $update->bind_param('ss', $newHash, $id); + $update->execute(); + } + $counts['updated']++; + echo ($dryRun ? 'would update' : 'updated ') . " $id {$row['title']}\n"; +} + +printf( + "%s%d updated, %d already current, %d conflicts\n", + $dryRun ? '[dry run] ' : '', + $counts['updated'], + $counts['unchanged'], + $counts['conflict'] +);