Merge pull request 'fix: gate CPU tickets on sustained 15-min load, not transient spikes' (#26) from fix/cpu-sustained-load-gating into main

Reviewed-on: #26

hwmonDaemon.py

@@ -1012,7 +1012,10 @@ class SystemHealthMonitor:
                 if health_report['memory_health'].get('ecc_errors'):
                     logger.info(f"ECC Errors: {len(health_report['memory_health']['ecc_errors'])} found")
 
-            logger.info(f"\nCPU Usage: {health_report['cpu_health']['cpu_usage_percent']}%")
+            _cpu = health_report['cpu_health']
+            _load15 = _cpu.get('load15_per_core_percent')
+            _load_str = f", 15-min load {_load15}%/core (sustained)" if _load15 is not None else ""
+            logger.info(f"\nCPU Usage: {_cpu['cpu_usage_percent']}% now{_load_str} [{_cpu.get('status', 'N/A')}]")
 
             logger.info("\nNetwork Status:")
             logger.info(f"Management: {health_report['network_health']['management_network']['status']}")
@@ -1631,10 +1634,13 @@ class SystemHealthMonitor:
             cpu_threshold = self.CONFIG['THRESHOLDS']['CPU_WARNING']
             cpu_status = cpu_health.get('status', 'N/A')
             cpu_usage_str = f"{cpu_usage}%" if isinstance(cpu_usage, (int, float)) else cpu_usage
+            load15 = cpu_health.get('load15_per_core_percent')
+            load15_str = f"{load15}% per core (sustained)" if load15 is not None else 'N/A'
 
             description += f"""
 ┏━ CPU STATUS {'━' * (box_width - 13)}┓
-┃  Usage        │ {cpu_usage_str:<61}┃
+┃  Usage (now)  │ {cpu_usage_str:<61}┃
+┃  15-min load  │ {load15_str:<61}┃
 ┃  Threshold    │ {str(cpu_threshold) + '%':<61}┃
 ┃  Status       │ {cpu_status:<61}┃
 ┗{'━' * box_width}┛
@@ -2207,10 +2213,14 @@ class SystemHealthMonitor:
         if memory_health.get('has_ecc') and memory_health.get('ecc_errors'):
             issues.extend(memory_health['ecc_errors'])
 
-        # Check for CPU-related issues
+        # Check for CPU-related issues. Gate on the sustained signal (15-min load, set in
+        # _check_cpu_usage) so transient spikes don't raise/reopen a CPU ticket. The message
+        # is kept stable (threshold constant, no measured value) so the title doesn't churn
+        # every run; the measured load is recorded in the ticket description instead.
         cpu_health = health_report.get('cpu_health', {})
-        if cpu_health and cpu_health.get('cpu_usage_percent', 0) > self.CONFIG['THRESHOLDS']['CPU_WARNING']:
-            issues.append("CPU usage is above threshold of 95%")
+        if cpu_health.get('sustained_high'):
+            threshold = self.CONFIG['THRESHOLDS']['CPU_WARNING']
+            issues.append(f"CPU usage sustained above threshold of {threshold}%")
 
         # Check for network-related issues
         network_health = health_report.get('network_health', {})
@@ -3311,14 +3321,44 @@ class SystemHealthMonitor:
         """
         Check CPU usage and return health metrics.
 
+        Ticketing is gated on a *sustained* signal — the 15-minute load average
+        normalized per core — not the instantaneous sample. A transient spike barely
+        moves the 15-minute load, so a momentary burst no longer raises (or, via the
+        API's reopen-on-match dedup, repeatedly reopens) a CPU ticket every hourly run.
+        The instantaneous percentage is still reported (dry-run summary, Prometheus,
+        ticket description) for context.
+
         :return: Dictionary with CPU health metrics.
         """
-        cpu_usage_percent = psutil.cpu_percent(interval=1)
-        cpu_health = {
+        threshold = self.CONFIG['THRESHOLDS']['CPU_WARNING']
+        cpu_usage_percent = psutil.cpu_percent(interval=1)   # instantaneous, informational
+        cpu_count = psutil.cpu_count() or 1
+
+        try:
+            load1, load5, load15 = os.getloadavg()
+        except (OSError, AttributeError):
+            load1 = load5 = load15 = None
+
+        if load15 is not None:
+            # Normalize per core to a 0-100% scale comparable to CPU_WARNING
+            # (per-core load of 1.0 == 100% of one core).
+            load15_per_core = round(load15 / cpu_count * 100, 1)
+            sustained_high = load15_per_core >= threshold
+        else:
+            # No load average available (non-Linux / unsupported): fall back to the
+            # instantaneous sample so we still alert rather than silently going quiet.
+            load15_per_core = None
+            sustained_high = cpu_usage_percent >= threshold
+
+        return {
             'cpu_usage_percent': cpu_usage_percent,
-            'status': 'OK' if cpu_usage_percent < self.CONFIG['THRESHOLDS']['CPU_WARNING'] else 'WARNING'
+            'cpu_count': cpu_count,
+            'load_average': ({'1min': load1, '5min': load5, '15min': load15}
+                             if load15 is not None else None),
+            'load15_per_core_percent': load15_per_core,
+            'sustained_high': sustained_high,
+            'status': 'WARNING' if sustained_high else 'OK',
         }
-        return cpu_health
 
     def _check_network_status(self) -> Dict[str, Any]:
         """

tests/test_hwmon.py

@@ -536,3 +536,72 @@ class TestLxcDryRunSummary:
     def test_error_status_is_reported(self, monitor):
         lines = monitor._format_lxc_dry_run({'status': 'ERROR', 'containers': [], 'issues': ['boom']})
         assert any('check error' in ln for ln in lines)
+
+
+# ── _check_cpu_usage (ticket on sustained load, not transient spikes) ──────────
+
+def _cpu(monitor, instantaneous, load15, cpu_count=4):
+    """Run _check_cpu_usage with mocked psutil + load average."""
+    with patch('psutil.cpu_percent', return_value=instantaneous), \
+         patch('psutil.cpu_count', return_value=cpu_count), \
+         patch('os.getloadavg', return_value=(load15, load15, load15)):
+        return monitor._check_cpu_usage()
+
+
+def _report_with_cpu(cpu_health):
+    """Minimal health_report skeleton (all sections empty) with cpu_health overridden,
+    sufficient for _detect_issues which direct-indexes several sections for logging."""
+    return {
+        'drives_health': {'drives': [], 'overall_status': 'HEALTHY'},
+        'memory_health': {'status': 'OK'},
+        'cpu_health': cpu_health,
+        'network_health': {'management_network': {}, 'ceph_network': {}},
+        'ceph_health': {},
+        'lxc_health': {'status': 'OK', 'issues': []},
+        'system_health': {'status': 'OK', 'issues': []},
+        'pbs_health': {},
+    }
+
+
+class TestCpuSustainedGating:
+    def test_transient_spike_does_not_warn(self, monitor):
+        """Instantaneous 100% but low 15-min load (e.g. 0.4/core = 10%) -> OK, no ticket."""
+        h = _cpu(monitor, instantaneous=100.0, load15=1.6, cpu_count=4)   # 1.6/4 = 40%/core
+        assert h['status'] == 'OK'
+        assert h['sustained_high'] is False
+        assert h['cpu_usage_percent'] == 100.0          # still reported for context
+        assert h['load15_per_core_percent'] == 40.0
+
+    def test_sustained_high_load_warns(self, monitor):
+        """15-min load 3.9/4 cores = 97.5%/core >= 95% threshold -> WARNING."""
+        h = _cpu(monitor, instantaneous=20.0, load15=3.9, cpu_count=4)
+        assert h['status'] == 'WARNING'
+        assert h['sustained_high'] is True
+        assert h['load15_per_core_percent'] == 97.5
+
+    def test_detect_issues_ignores_transient_spike(self, monitor):
+        """A high instantaneous sample with sustained_high False must not create a CPU issue."""
+        report = _report_with_cpu({'cpu_usage_percent': 99.0, 'sustained_high': False,
+                                   'load15_per_core_percent': 30.0, 'status': 'OK'})
+        issues = monitor._detect_issues(report)
+        assert not any('CPU' in i for i in issues)
+
+    def test_detect_issues_flags_sustained(self, monitor):
+        """sustained_high True must create exactly one stable, value-free CPU issue."""
+        report = _report_with_cpu({'cpu_usage_percent': 96.0, 'sustained_high': True,
+                                   'load15_per_core_percent': 98.0, 'status': 'WARNING'})
+        issues = monitor._detect_issues(report)
+        cpu_issues = [i for i in issues if 'CPU' in i]
+        assert len(cpu_issues) == 1
+        # Title-stable: no measured value baked into the message (only the threshold constant)
+        assert '98' not in cpu_issues[0]
+        assert 'sustained' in cpu_issues[0].lower()
+
+    def test_falls_back_to_instantaneous_without_loadavg(self, monitor):
+        """If os.getloadavg is unavailable, don't go silent — use the instantaneous sample."""
+        with patch('psutil.cpu_percent', return_value=97.0), \
+             patch('psutil.cpu_count', return_value=4), \
+             patch('os.getloadavg', side_effect=OSError):
+            h = monitor._check_cpu_usage()
+        assert h['status'] == 'WARNING'
+        assert h['load15_per_core_percent'] is None