fix: gate CPU tickets on sustained 15-min load, not transient spikes
Lint / Python (flake8) (push) Successful in 1m13s
Security / Python Security (bandit) (push) Successful in 37s
Test / Python Tests (pytest) (push) Successful in 41s
Lint / Notify on failure (push) Has been skipped
Lint / Python (flake8) (pull_request) Successful in 30s
Security / Python Security (bandit) (pull_request) Successful in 1m14s
Test / Python Tests (pytest) (pull_request) Successful in 43s
Lint / Notify on failure (pull_request) Has been skipped

_check_cpu_usage took a single 1-second psutil sample per run, so a momentary
spike raised a CPU ticket -- and because the ticket API reopens a closed ticket
on a matching alert, the same CPU ticket flapped open/closed on every hourly run.

Ticketing is now gated on the 15-minute load average normalized per core (a true
"sustained" signal): a transient spike barely moves the 15-min load, so it no
longer raises or reopens a ticket. The instantaneous percentage is still reported
(dry-run summary + description) and the measured sustained load is shown in the
ticket description; the title stays value-free so it doesn't churn each run. Falls
back to the instantaneous sample only if os.getloadavg() is unavailable.
Adds 5 regression tests.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
2026-07-15 17:32:34 -04:00
co-authored by Claude Opus 4.8
parent 26aef22b47
commit 48d4474c20
2 changed files with 118 additions and 9 deletions
+49 -9
View File
@@ -1012,7 +1012,10 @@ class SystemHealthMonitor:
if health_report['memory_health'].get('ecc_errors'):
logger.info(f"ECC Errors: {len(health_report['memory_health']['ecc_errors'])} found")
logger.info(f"\nCPU Usage: {health_report['cpu_health']['cpu_usage_percent']}%")
_cpu = health_report['cpu_health']
_load15 = _cpu.get('load15_per_core_percent')
_load_str = f", 15-min load {_load15}%/core (sustained)" if _load15 is not None else ""
logger.info(f"\nCPU Usage: {_cpu['cpu_usage_percent']}% now{_load_str} [{_cpu.get('status', 'N/A')}]")
logger.info("\nNetwork Status:")
logger.info(f"Management: {health_report['network_health']['management_network']['status']}")
@@ -1631,10 +1634,13 @@ class SystemHealthMonitor:
cpu_threshold = self.CONFIG['THRESHOLDS']['CPU_WARNING']
cpu_status = cpu_health.get('status', 'N/A')
cpu_usage_str = f"{cpu_usage}%" if isinstance(cpu_usage, (int, float)) else cpu_usage
load15 = cpu_health.get('load15_per_core_percent')
load15_str = f"{load15}% per core (sustained)" if load15 is not None else 'N/A'
description += f"""
┏━ CPU STATUS {'' * (box_width - 13)}
┃ Usage {cpu_usage_str:<61}
┃ Usage (now){cpu_usage_str:<61}
┃ 15-min load │ {load15_str:<61}
┃ Threshold │ {str(cpu_threshold) + '%':<61}
┃ Status │ {cpu_status:<61}
{'' * box_width}
@@ -2207,10 +2213,14 @@ class SystemHealthMonitor:
if memory_health.get('has_ecc') and memory_health.get('ecc_errors'):
issues.extend(memory_health['ecc_errors'])
# Check for CPU-related issues
# Check for CPU-related issues. Gate on the sustained signal (15-min load, set in
# _check_cpu_usage) so transient spikes don't raise/reopen a CPU ticket. The message
# is kept stable (threshold constant, no measured value) so the title doesn't churn
# every run; the measured load is recorded in the ticket description instead.
cpu_health = health_report.get('cpu_health', {})
if cpu_health and cpu_health.get('cpu_usage_percent', 0) > self.CONFIG['THRESHOLDS']['CPU_WARNING']:
issues.append("CPU usage is above threshold of 95%")
if cpu_health.get('sustained_high'):
threshold = self.CONFIG['THRESHOLDS']['CPU_WARNING']
issues.append(f"CPU usage sustained above threshold of {threshold}%")
# Check for network-related issues
network_health = health_report.get('network_health', {})
@@ -3311,14 +3321,44 @@ class SystemHealthMonitor:
"""
Check CPU usage and return health metrics.
Ticketing is gated on a *sustained* signal — the 15-minute load average
normalized per core — not the instantaneous sample. A transient spike barely
moves the 15-minute load, so a momentary burst no longer raises (or, via the
API's reopen-on-match dedup, repeatedly reopens) a CPU ticket every hourly run.
The instantaneous percentage is still reported (dry-run summary, Prometheus,
ticket description) for context.
:return: Dictionary with CPU health metrics.
"""
cpu_usage_percent = psutil.cpu_percent(interval=1)
cpu_health = {
threshold = self.CONFIG['THRESHOLDS']['CPU_WARNING']
cpu_usage_percent = psutil.cpu_percent(interval=1) # instantaneous, informational
cpu_count = psutil.cpu_count() or 1
try:
load1, load5, load15 = os.getloadavg()
except (OSError, AttributeError):
load1 = load5 = load15 = None
if load15 is not None:
# Normalize per core to a 0-100% scale comparable to CPU_WARNING
# (per-core load of 1.0 == 100% of one core).
load15_per_core = round(load15 / cpu_count * 100, 1)
sustained_high = load15_per_core >= threshold
else:
# No load average available (non-Linux / unsupported): fall back to the
# instantaneous sample so we still alert rather than silently going quiet.
load15_per_core = None
sustained_high = cpu_usage_percent >= threshold
return {
'cpu_usage_percent': cpu_usage_percent,
'status': 'OK' if cpu_usage_percent < self.CONFIG['THRESHOLDS']['CPU_WARNING'] else 'WARNING'
'cpu_count': cpu_count,
'load_average': ({'1min': load1, '5min': load5, '15min': load15}
if load15 is not None else None),
'load15_per_core_percent': load15_per_core,
'sustained_high': sustained_high,
'status': 'WARNING' if sustained_high else 'OK',
}
return cpu_health
def _check_network_status(self) -> Dict[str, Any]:
"""