From 48d4474c201f4538df984b5e8ed469333b168fcf Mon Sep 17 00:00:00 2001 From: Jared Vititoe Date: Wed, 15 Jul 2026 17:31:23 -0400 Subject: [PATCH] fix: gate CPU tickets on sustained 15-min load, not transient spikes _check_cpu_usage took a single 1-second psutil sample per run, so a momentary spike raised a CPU ticket -- and because the ticket API reopens a closed ticket on a matching alert, the same CPU ticket flapped open/closed on every hourly run. Ticketing is now gated on the 15-minute load average normalized per core (a true "sustained" signal): a transient spike barely moves the 15-min load, so it no longer raises or reopens a ticket. The instantaneous percentage is still reported (dry-run summary + description) and the measured sustained load is shown in the ticket description; the title stays value-free so it doesn't churn each run. Falls back to the instantaneous sample only if os.getloadavg() is unavailable. Adds 5 regression tests. Co-Authored-By: Claude Opus 4.8 --- hwmonDaemon.py | 58 +++++++++++++++++++++++++++++++------ tests/test_hwmon.py | 69 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 118 insertions(+), 9 deletions(-) diff --git a/hwmonDaemon.py b/hwmonDaemon.py index 9cedd92..f47a229 100644 --- a/hwmonDaemon.py +++ b/hwmonDaemon.py @@ -1012,7 +1012,10 @@ class SystemHealthMonitor: if health_report['memory_health'].get('ecc_errors'): logger.info(f"ECC Errors: {len(health_report['memory_health']['ecc_errors'])} found") - logger.info(f"\nCPU Usage: {health_report['cpu_health']['cpu_usage_percent']}%") + _cpu = health_report['cpu_health'] + _load15 = _cpu.get('load15_per_core_percent') + _load_str = f", 15-min load {_load15}%/core (sustained)" if _load15 is not None else "" + logger.info(f"\nCPU Usage: {_cpu['cpu_usage_percent']}% now{_load_str} [{_cpu.get('status', 'N/A')}]") logger.info("\nNetwork Status:") logger.info(f"Management: {health_report['network_health']['management_network']['status']}") @@ -1631,10 +1634,13 @@ class SystemHealthMonitor: cpu_threshold = self.CONFIG['THRESHOLDS']['CPU_WARNING'] cpu_status = cpu_health.get('status', 'N/A') cpu_usage_str = f"{cpu_usage}%" if isinstance(cpu_usage, (int, float)) else cpu_usage + load15 = cpu_health.get('load15_per_core_percent') + load15_str = f"{load15}% per core (sustained)" if load15 is not None else 'N/A' description += f""" ┏━ CPU STATUS {'━' * (box_width - 13)}┓ -┃ Usage │ {cpu_usage_str:<61}┃ +┃ Usage (now) │ {cpu_usage_str:<61}┃ +┃ 15-min load │ {load15_str:<61}┃ ┃ Threshold │ {str(cpu_threshold) + '%':<61}┃ ┃ Status │ {cpu_status:<61}┃ ┗{'━' * box_width}┛ @@ -2207,10 +2213,14 @@ class SystemHealthMonitor: if memory_health.get('has_ecc') and memory_health.get('ecc_errors'): issues.extend(memory_health['ecc_errors']) - # Check for CPU-related issues + # Check for CPU-related issues. Gate on the sustained signal (15-min load, set in + # _check_cpu_usage) so transient spikes don't raise/reopen a CPU ticket. The message + # is kept stable (threshold constant, no measured value) so the title doesn't churn + # every run; the measured load is recorded in the ticket description instead. cpu_health = health_report.get('cpu_health', {}) - if cpu_health and cpu_health.get('cpu_usage_percent', 0) > self.CONFIG['THRESHOLDS']['CPU_WARNING']: - issues.append("CPU usage is above threshold of 95%") + if cpu_health.get('sustained_high'): + threshold = self.CONFIG['THRESHOLDS']['CPU_WARNING'] + issues.append(f"CPU usage sustained above threshold of {threshold}%") # Check for network-related issues network_health = health_report.get('network_health', {}) @@ -3311,14 +3321,44 @@ class SystemHealthMonitor: """ Check CPU usage and return health metrics. + Ticketing is gated on a *sustained* signal — the 15-minute load average + normalized per core — not the instantaneous sample. A transient spike barely + moves the 15-minute load, so a momentary burst no longer raises (or, via the + API's reopen-on-match dedup, repeatedly reopens) a CPU ticket every hourly run. + The instantaneous percentage is still reported (dry-run summary, Prometheus, + ticket description) for context. + :return: Dictionary with CPU health metrics. """ - cpu_usage_percent = psutil.cpu_percent(interval=1) - cpu_health = { + threshold = self.CONFIG['THRESHOLDS']['CPU_WARNING'] + cpu_usage_percent = psutil.cpu_percent(interval=1) # instantaneous, informational + cpu_count = psutil.cpu_count() or 1 + + try: + load1, load5, load15 = os.getloadavg() + except (OSError, AttributeError): + load1 = load5 = load15 = None + + if load15 is not None: + # Normalize per core to a 0-100% scale comparable to CPU_WARNING + # (per-core load of 1.0 == 100% of one core). + load15_per_core = round(load15 / cpu_count * 100, 1) + sustained_high = load15_per_core >= threshold + else: + # No load average available (non-Linux / unsupported): fall back to the + # instantaneous sample so we still alert rather than silently going quiet. + load15_per_core = None + sustained_high = cpu_usage_percent >= threshold + + return { 'cpu_usage_percent': cpu_usage_percent, - 'status': 'OK' if cpu_usage_percent < self.CONFIG['THRESHOLDS']['CPU_WARNING'] else 'WARNING' + 'cpu_count': cpu_count, + 'load_average': ({'1min': load1, '5min': load5, '15min': load15} + if load15 is not None else None), + 'load15_per_core_percent': load15_per_core, + 'sustained_high': sustained_high, + 'status': 'WARNING' if sustained_high else 'OK', } - return cpu_health def _check_network_status(self) -> Dict[str, Any]: """ diff --git a/tests/test_hwmon.py b/tests/test_hwmon.py index 62261e1..0147dc7 100644 --- a/tests/test_hwmon.py +++ b/tests/test_hwmon.py @@ -536,3 +536,72 @@ class TestLxcDryRunSummary: def test_error_status_is_reported(self, monitor): lines = monitor._format_lxc_dry_run({'status': 'ERROR', 'containers': [], 'issues': ['boom']}) assert any('check error' in ln for ln in lines) + + +# ── _check_cpu_usage (ticket on sustained load, not transient spikes) ────────── + +def _cpu(monitor, instantaneous, load15, cpu_count=4): + """Run _check_cpu_usage with mocked psutil + load average.""" + with patch('psutil.cpu_percent', return_value=instantaneous), \ + patch('psutil.cpu_count', return_value=cpu_count), \ + patch('os.getloadavg', return_value=(load15, load15, load15)): + return monitor._check_cpu_usage() + + +def _report_with_cpu(cpu_health): + """Minimal health_report skeleton (all sections empty) with cpu_health overridden, + sufficient for _detect_issues which direct-indexes several sections for logging.""" + return { + 'drives_health': {'drives': [], 'overall_status': 'HEALTHY'}, + 'memory_health': {'status': 'OK'}, + 'cpu_health': cpu_health, + 'network_health': {'management_network': {}, 'ceph_network': {}}, + 'ceph_health': {}, + 'lxc_health': {'status': 'OK', 'issues': []}, + 'system_health': {'status': 'OK', 'issues': []}, + 'pbs_health': {}, + } + + +class TestCpuSustainedGating: + def test_transient_spike_does_not_warn(self, monitor): + """Instantaneous 100% but low 15-min load (e.g. 0.4/core = 10%) -> OK, no ticket.""" + h = _cpu(monitor, instantaneous=100.0, load15=1.6, cpu_count=4) # 1.6/4 = 40%/core + assert h['status'] == 'OK' + assert h['sustained_high'] is False + assert h['cpu_usage_percent'] == 100.0 # still reported for context + assert h['load15_per_core_percent'] == 40.0 + + def test_sustained_high_load_warns(self, monitor): + """15-min load 3.9/4 cores = 97.5%/core >= 95% threshold -> WARNING.""" + h = _cpu(monitor, instantaneous=20.0, load15=3.9, cpu_count=4) + assert h['status'] == 'WARNING' + assert h['sustained_high'] is True + assert h['load15_per_core_percent'] == 97.5 + + def test_detect_issues_ignores_transient_spike(self, monitor): + """A high instantaneous sample with sustained_high False must not create a CPU issue.""" + report = _report_with_cpu({'cpu_usage_percent': 99.0, 'sustained_high': False, + 'load15_per_core_percent': 30.0, 'status': 'OK'}) + issues = monitor._detect_issues(report) + assert not any('CPU' in i for i in issues) + + def test_detect_issues_flags_sustained(self, monitor): + """sustained_high True must create exactly one stable, value-free CPU issue.""" + report = _report_with_cpu({'cpu_usage_percent': 96.0, 'sustained_high': True, + 'load15_per_core_percent': 98.0, 'status': 'WARNING'}) + issues = monitor._detect_issues(report) + cpu_issues = [i for i in issues if 'CPU' in i] + assert len(cpu_issues) == 1 + # Title-stable: no measured value baked into the message (only the threshold constant) + assert '98' not in cpu_issues[0] + assert 'sustained' in cpu_issues[0].lower() + + def test_falls_back_to_instantaneous_without_loadavg(self, monitor): + """If os.getloadavg is unavailable, don't go silent — use the instantaneous sample.""" + with patch('psutil.cpu_percent', return_value=97.0), \ + patch('psutil.cpu_count', return_value=4), \ + patch('os.getloadavg', side_effect=OSError): + h = monitor._check_cpu_usage() + assert h['status'] == 'WARNING' + assert h['load15_per_core_percent'] is None