fix: gate CPU tickets on sustained 15-min load, not transient spikes
Lint / Python (flake8) (push) Successful in 1m13s
Security / Python Security (bandit) (push) Successful in 37s
Test / Python Tests (pytest) (push) Successful in 41s
Lint / Notify on failure (push) Has been skipped
Lint / Python (flake8) (pull_request) Successful in 30s
Security / Python Security (bandit) (pull_request) Successful in 1m14s
Test / Python Tests (pytest) (pull_request) Successful in 43s
Lint / Notify on failure (pull_request) Has been skipped
Lint / Python (flake8) (push) Successful in 1m13s
Security / Python Security (bandit) (push) Successful in 37s
Test / Python Tests (pytest) (push) Successful in 41s
Lint / Notify on failure (push) Has been skipped
Lint / Python (flake8) (pull_request) Successful in 30s
Security / Python Security (bandit) (pull_request) Successful in 1m14s
Test / Python Tests (pytest) (pull_request) Successful in 43s
Lint / Notify on failure (pull_request) Has been skipped
_check_cpu_usage took a single 1-second psutil sample per run, so a momentary spike raised a CPU ticket -- and because the ticket API reopens a closed ticket on a matching alert, the same CPU ticket flapped open/closed on every hourly run. Ticketing is now gated on the 15-minute load average normalized per core (a true "sustained" signal): a transient spike barely moves the 15-min load, so it no longer raises or reopens a ticket. The instantaneous percentage is still reported (dry-run summary + description) and the measured sustained load is shown in the ticket description; the title stays value-free so it doesn't churn each run. Falls back to the instantaneous sample only if os.getloadavg() is unavailable. Adds 5 regression tests. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -536,3 +536,72 @@ class TestLxcDryRunSummary:
|
||||
def test_error_status_is_reported(self, monitor):
|
||||
lines = monitor._format_lxc_dry_run({'status': 'ERROR', 'containers': [], 'issues': ['boom']})
|
||||
assert any('check error' in ln for ln in lines)
|
||||
|
||||
|
||||
# ── _check_cpu_usage (ticket on sustained load, not transient spikes) ──────────
|
||||
|
||||
def _cpu(monitor, instantaneous, load15, cpu_count=4):
|
||||
"""Run _check_cpu_usage with mocked psutil + load average."""
|
||||
with patch('psutil.cpu_percent', return_value=instantaneous), \
|
||||
patch('psutil.cpu_count', return_value=cpu_count), \
|
||||
patch('os.getloadavg', return_value=(load15, load15, load15)):
|
||||
return monitor._check_cpu_usage()
|
||||
|
||||
|
||||
def _report_with_cpu(cpu_health):
|
||||
"""Minimal health_report skeleton (all sections empty) with cpu_health overridden,
|
||||
sufficient for _detect_issues which direct-indexes several sections for logging."""
|
||||
return {
|
||||
'drives_health': {'drives': [], 'overall_status': 'HEALTHY'},
|
||||
'memory_health': {'status': 'OK'},
|
||||
'cpu_health': cpu_health,
|
||||
'network_health': {'management_network': {}, 'ceph_network': {}},
|
||||
'ceph_health': {},
|
||||
'lxc_health': {'status': 'OK', 'issues': []},
|
||||
'system_health': {'status': 'OK', 'issues': []},
|
||||
'pbs_health': {},
|
||||
}
|
||||
|
||||
|
||||
class TestCpuSustainedGating:
|
||||
def test_transient_spike_does_not_warn(self, monitor):
|
||||
"""Instantaneous 100% but low 15-min load (e.g. 0.4/core = 10%) -> OK, no ticket."""
|
||||
h = _cpu(monitor, instantaneous=100.0, load15=1.6, cpu_count=4) # 1.6/4 = 40%/core
|
||||
assert h['status'] == 'OK'
|
||||
assert h['sustained_high'] is False
|
||||
assert h['cpu_usage_percent'] == 100.0 # still reported for context
|
||||
assert h['load15_per_core_percent'] == 40.0
|
||||
|
||||
def test_sustained_high_load_warns(self, monitor):
|
||||
"""15-min load 3.9/4 cores = 97.5%/core >= 95% threshold -> WARNING."""
|
||||
h = _cpu(monitor, instantaneous=20.0, load15=3.9, cpu_count=4)
|
||||
assert h['status'] == 'WARNING'
|
||||
assert h['sustained_high'] is True
|
||||
assert h['load15_per_core_percent'] == 97.5
|
||||
|
||||
def test_detect_issues_ignores_transient_spike(self, monitor):
|
||||
"""A high instantaneous sample with sustained_high False must not create a CPU issue."""
|
||||
report = _report_with_cpu({'cpu_usage_percent': 99.0, 'sustained_high': False,
|
||||
'load15_per_core_percent': 30.0, 'status': 'OK'})
|
||||
issues = monitor._detect_issues(report)
|
||||
assert not any('CPU' in i for i in issues)
|
||||
|
||||
def test_detect_issues_flags_sustained(self, monitor):
|
||||
"""sustained_high True must create exactly one stable, value-free CPU issue."""
|
||||
report = _report_with_cpu({'cpu_usage_percent': 96.0, 'sustained_high': True,
|
||||
'load15_per_core_percent': 98.0, 'status': 'WARNING'})
|
||||
issues = monitor._detect_issues(report)
|
||||
cpu_issues = [i for i in issues if 'CPU' in i]
|
||||
assert len(cpu_issues) == 1
|
||||
# Title-stable: no measured value baked into the message (only the threshold constant)
|
||||
assert '98' not in cpu_issues[0]
|
||||
assert 'sustained' in cpu_issues[0].lower()
|
||||
|
||||
def test_falls_back_to_instantaneous_without_loadavg(self, monitor):
|
||||
"""If os.getloadavg is unavailable, don't go silent — use the instantaneous sample."""
|
||||
with patch('psutil.cpu_percent', return_value=97.0), \
|
||||
patch('psutil.cpu_count', return_value=4), \
|
||||
patch('os.getloadavg', side_effect=OSError):
|
||||
h = monitor._check_cpu_usage()
|
||||
assert h['status'] == 'WARNING'
|
||||
assert h['load15_per_core_percent'] is None
|
||||
|
||||
Reference in New Issue
Block a user