fix: gate CPU tickets on sustained 15-min load, not transient spikes
Lint / Python (flake8) (push) Successful in 1m13s
Security / Python Security (bandit) (push) Successful in 37s
Test / Python Tests (pytest) (push) Successful in 41s
Lint / Notify on failure (push) Has been skipped
Lint / Python (flake8) (pull_request) Successful in 30s
Security / Python Security (bandit) (pull_request) Successful in 1m14s
Test / Python Tests (pytest) (pull_request) Successful in 43s
Lint / Notify on failure (pull_request) Has been skipped

_check_cpu_usage took a single 1-second psutil sample per run, so a momentary
spike raised a CPU ticket -- and because the ticket API reopens a closed ticket
on a matching alert, the same CPU ticket flapped open/closed on every hourly run.

Ticketing is now gated on the 15-minute load average normalized per core (a true
"sustained" signal): a transient spike barely moves the 15-min load, so it no
longer raises or reopens a ticket. The instantaneous percentage is still reported
(dry-run summary + description) and the measured sustained load is shown in the
ticket description; the title stays value-free so it doesn't churn each run. Falls
back to the instantaneous sample only if os.getloadavg() is unavailable.
Adds 5 regression tests.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
2026-07-15 17:32:34 -04:00
co-authored by Claude Opus 4.8
parent 26aef22b47
commit 48d4474c20
2 changed files with 118 additions and 9 deletions
+69
View File
@@ -536,3 +536,72 @@ class TestLxcDryRunSummary:
def test_error_status_is_reported(self, monitor):
lines = monitor._format_lxc_dry_run({'status': 'ERROR', 'containers': [], 'issues': ['boom']})
assert any('check error' in ln for ln in lines)
# ── _check_cpu_usage (ticket on sustained load, not transient spikes) ──────────
def _cpu(monitor, instantaneous, load15, cpu_count=4):
"""Run _check_cpu_usage with mocked psutil + load average."""
with patch('psutil.cpu_percent', return_value=instantaneous), \
patch('psutil.cpu_count', return_value=cpu_count), \
patch('os.getloadavg', return_value=(load15, load15, load15)):
return monitor._check_cpu_usage()
def _report_with_cpu(cpu_health):
"""Minimal health_report skeleton (all sections empty) with cpu_health overridden,
sufficient for _detect_issues which direct-indexes several sections for logging."""
return {
'drives_health': {'drives': [], 'overall_status': 'HEALTHY'},
'memory_health': {'status': 'OK'},
'cpu_health': cpu_health,
'network_health': {'management_network': {}, 'ceph_network': {}},
'ceph_health': {},
'lxc_health': {'status': 'OK', 'issues': []},
'system_health': {'status': 'OK', 'issues': []},
'pbs_health': {},
}
class TestCpuSustainedGating:
def test_transient_spike_does_not_warn(self, monitor):
"""Instantaneous 100% but low 15-min load (e.g. 0.4/core = 10%) -> OK, no ticket."""
h = _cpu(monitor, instantaneous=100.0, load15=1.6, cpu_count=4) # 1.6/4 = 40%/core
assert h['status'] == 'OK'
assert h['sustained_high'] is False
assert h['cpu_usage_percent'] == 100.0 # still reported for context
assert h['load15_per_core_percent'] == 40.0
def test_sustained_high_load_warns(self, monitor):
"""15-min load 3.9/4 cores = 97.5%/core >= 95% threshold -> WARNING."""
h = _cpu(monitor, instantaneous=20.0, load15=3.9, cpu_count=4)
assert h['status'] == 'WARNING'
assert h['sustained_high'] is True
assert h['load15_per_core_percent'] == 97.5
def test_detect_issues_ignores_transient_spike(self, monitor):
"""A high instantaneous sample with sustained_high False must not create a CPU issue."""
report = _report_with_cpu({'cpu_usage_percent': 99.0, 'sustained_high': False,
'load15_per_core_percent': 30.0, 'status': 'OK'})
issues = monitor._detect_issues(report)
assert not any('CPU' in i for i in issues)
def test_detect_issues_flags_sustained(self, monitor):
"""sustained_high True must create exactly one stable, value-free CPU issue."""
report = _report_with_cpu({'cpu_usage_percent': 96.0, 'sustained_high': True,
'load15_per_core_percent': 98.0, 'status': 'WARNING'})
issues = monitor._detect_issues(report)
cpu_issues = [i for i in issues if 'CPU' in i]
assert len(cpu_issues) == 1
# Title-stable: no measured value baked into the message (only the threshold constant)
assert '98' not in cpu_issues[0]
assert 'sustained' in cpu_issues[0].lower()
def test_falls_back_to_instantaneous_without_loadavg(self, monitor):
"""If os.getloadavg is unavailable, don't go silent — use the instantaneous sample."""
with patch('psutil.cpu_percent', return_value=97.0), \
patch('psutil.cpu_count', return_value=4), \
patch('os.getloadavg', side_effect=OSError):
h = monitor._check_cpu_usage()
assert h['status'] == 'WARNING'
assert h['load15_per_core_percent'] is None