From b15fa58cd556d7d7c9a4288c2554f406699d7332 Mon Sep 17 00:00:00 2001 From: Jared Vititoe Date: Tue, 30 Jun 2026 18:10:15 -0400 Subject: [PATCH] fix: stop multi-critical P1 escalation from dragging up unrelated low issues _determine_ticket_priority returned P1 for EVERY issue in a scan whenever the report had >= PRIORITY_ESCALATION_THRESHOLD criticals, because the critical count is global to the health report. A benign P4 "high storage usage" or "CPU usage" alert on a node that also had 3+ critical drive failures was created as P1, flooding P1s. Split the severity cascade into _base_issue_priority() and only apply the multi-critical "cluster risk" P1 bump to issues that are themselves critical/high-severity (base P1/P2). Routine warnings (P3) and normal alerts (P4/P5) keep their base priority. Critical issues still escalate to P1 under multi-critical conditions as before. Co-Authored-By: Claude Opus 4.8 --- hwmonDaemon.py | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/hwmonDaemon.py b/hwmonDaemon.py index 9269a69..36167c7 100644 --- a/hwmonDaemon.py +++ b/hwmonDaemon.py @@ -1782,16 +1782,30 @@ class SystemHealthMonitor: P5 = Informational/minimal impact """ issue_lower = issue.lower() + base = self._base_issue_priority(issue_lower) - # Count total critical issues for escalation logic + # Multi-critical "cluster risk" escalation: when several critical failures + # are occurring at once, raise the SERIOUS issues to P1. This must only + # apply to issues that are themselves critical/high-severity (base P1/P2); + # it must NOT drag routine warnings (P3) or normal alerts (P4/P5) up to P1 + # just because unrelated criticals exist elsewhere on the node. critical_count = self._count_critical_issues(health_report) escalation_threshold = self.CONFIG.get('PRIORITY_ESCALATION_THRESHOLD', 3) - - # P1 - Multiple simultaneous critical failures (cluster risk) - if critical_count >= escalation_threshold: - logger.info(f"P1 escalation triggered: {critical_count} critical issues detected") + if (critical_count >= escalation_threshold + and base in (self.PRIORITIES['CRITICAL'], self.PRIORITIES['HIGH'])): + logger.info( + f"P1 escalation: {critical_count} simultaneous critical issues; " + f"raising P{base} issue to P1" + ) return self.PRIORITIES['CRITICAL'] # P1 + return base + + def _base_issue_priority(self, issue_lower: str) -> str: + """ + Severity-based priority for a single issue, with no cross-issue escalation. + _determine_ticket_priority layers the multi-critical P1 escalation on top. + """ # P1 - Specific cluster-affecting scenarios if any(keyword in issue_lower for keyword in [ 'raid degraded', 'multiple drive',