fix: gate CPU tickets on sustained 15-min load, not transient spikes
Lint / Python (flake8) (push) Successful in 1m13s
Security / Python Security (bandit) (push) Successful in 37s
Test / Python Tests (pytest) (push) Successful in 41s
Lint / Notify on failure (push) Has been skipped
Lint / Python (flake8) (pull_request) Successful in 30s
Security / Python Security (bandit) (pull_request) Successful in 1m14s
Test / Python Tests (pytest) (pull_request) Successful in 43s
Lint / Notify on failure (pull_request) Has been skipped
Lint / Python (flake8) (push) Successful in 1m13s
Security / Python Security (bandit) (push) Successful in 37s
Test / Python Tests (pytest) (push) Successful in 41s
Lint / Notify on failure (push) Has been skipped
Lint / Python (flake8) (pull_request) Successful in 30s
Security / Python Security (bandit) (pull_request) Successful in 1m14s
Test / Python Tests (pytest) (pull_request) Successful in 43s
Lint / Notify on failure (pull_request) Has been skipped
_check_cpu_usage took a single 1-second psutil sample per run, so a momentary spike raised a CPU ticket -- and because the ticket API reopens a closed ticket on a matching alert, the same CPU ticket flapped open/closed on every hourly run. Ticketing is now gated on the 15-minute load average normalized per core (a true "sustained" signal): a transient spike barely moves the 15-min load, so it no longer raises or reopens a ticket. The instantaneous percentage is still reported (dry-run summary + description) and the measured sustained load is shown in the ticket description; the title stays value-free so it doesn't churn each run. Falls back to the instantaneous sample only if os.getloadavg() is unavailable. Adds 5 regression tests. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
+49
-9
@@ -1012,7 +1012,10 @@ class SystemHealthMonitor:
|
||||
if health_report['memory_health'].get('ecc_errors'):
|
||||
logger.info(f"ECC Errors: {len(health_report['memory_health']['ecc_errors'])} found")
|
||||
|
||||
logger.info(f"\nCPU Usage: {health_report['cpu_health']['cpu_usage_percent']}%")
|
||||
_cpu = health_report['cpu_health']
|
||||
_load15 = _cpu.get('load15_per_core_percent')
|
||||
_load_str = f", 15-min load {_load15}%/core (sustained)" if _load15 is not None else ""
|
||||
logger.info(f"\nCPU Usage: {_cpu['cpu_usage_percent']}% now{_load_str} [{_cpu.get('status', 'N/A')}]")
|
||||
|
||||
logger.info("\nNetwork Status:")
|
||||
logger.info(f"Management: {health_report['network_health']['management_network']['status']}")
|
||||
@@ -1631,10 +1634,13 @@ class SystemHealthMonitor:
|
||||
cpu_threshold = self.CONFIG['THRESHOLDS']['CPU_WARNING']
|
||||
cpu_status = cpu_health.get('status', 'N/A')
|
||||
cpu_usage_str = f"{cpu_usage}%" if isinstance(cpu_usage, (int, float)) else cpu_usage
|
||||
load15 = cpu_health.get('load15_per_core_percent')
|
||||
load15_str = f"{load15}% per core (sustained)" if load15 is not None else 'N/A'
|
||||
|
||||
description += f"""
|
||||
┏━ CPU STATUS {'━' * (box_width - 13)}┓
|
||||
┃ Usage │ {cpu_usage_str:<61}┃
|
||||
┃ Usage (now) │ {cpu_usage_str:<61}┃
|
||||
┃ 15-min load │ {load15_str:<61}┃
|
||||
┃ Threshold │ {str(cpu_threshold) + '%':<61}┃
|
||||
┃ Status │ {cpu_status:<61}┃
|
||||
┗{'━' * box_width}┛
|
||||
@@ -2207,10 +2213,14 @@ class SystemHealthMonitor:
|
||||
if memory_health.get('has_ecc') and memory_health.get('ecc_errors'):
|
||||
issues.extend(memory_health['ecc_errors'])
|
||||
|
||||
# Check for CPU-related issues
|
||||
# Check for CPU-related issues. Gate on the sustained signal (15-min load, set in
|
||||
# _check_cpu_usage) so transient spikes don't raise/reopen a CPU ticket. The message
|
||||
# is kept stable (threshold constant, no measured value) so the title doesn't churn
|
||||
# every run; the measured load is recorded in the ticket description instead.
|
||||
cpu_health = health_report.get('cpu_health', {})
|
||||
if cpu_health and cpu_health.get('cpu_usage_percent', 0) > self.CONFIG['THRESHOLDS']['CPU_WARNING']:
|
||||
issues.append("CPU usage is above threshold of 95%")
|
||||
if cpu_health.get('sustained_high'):
|
||||
threshold = self.CONFIG['THRESHOLDS']['CPU_WARNING']
|
||||
issues.append(f"CPU usage sustained above threshold of {threshold}%")
|
||||
|
||||
# Check for network-related issues
|
||||
network_health = health_report.get('network_health', {})
|
||||
@@ -3311,14 +3321,44 @@ class SystemHealthMonitor:
|
||||
"""
|
||||
Check CPU usage and return health metrics.
|
||||
|
||||
Ticketing is gated on a *sustained* signal — the 15-minute load average
|
||||
normalized per core — not the instantaneous sample. A transient spike barely
|
||||
moves the 15-minute load, so a momentary burst no longer raises (or, via the
|
||||
API's reopen-on-match dedup, repeatedly reopens) a CPU ticket every hourly run.
|
||||
The instantaneous percentage is still reported (dry-run summary, Prometheus,
|
||||
ticket description) for context.
|
||||
|
||||
:return: Dictionary with CPU health metrics.
|
||||
"""
|
||||
cpu_usage_percent = psutil.cpu_percent(interval=1)
|
||||
cpu_health = {
|
||||
threshold = self.CONFIG['THRESHOLDS']['CPU_WARNING']
|
||||
cpu_usage_percent = psutil.cpu_percent(interval=1) # instantaneous, informational
|
||||
cpu_count = psutil.cpu_count() or 1
|
||||
|
||||
try:
|
||||
load1, load5, load15 = os.getloadavg()
|
||||
except (OSError, AttributeError):
|
||||
load1 = load5 = load15 = None
|
||||
|
||||
if load15 is not None:
|
||||
# Normalize per core to a 0-100% scale comparable to CPU_WARNING
|
||||
# (per-core load of 1.0 == 100% of one core).
|
||||
load15_per_core = round(load15 / cpu_count * 100, 1)
|
||||
sustained_high = load15_per_core >= threshold
|
||||
else:
|
||||
# No load average available (non-Linux / unsupported): fall back to the
|
||||
# instantaneous sample so we still alert rather than silently going quiet.
|
||||
load15_per_core = None
|
||||
sustained_high = cpu_usage_percent >= threshold
|
||||
|
||||
return {
|
||||
'cpu_usage_percent': cpu_usage_percent,
|
||||
'status': 'OK' if cpu_usage_percent < self.CONFIG['THRESHOLDS']['CPU_WARNING'] else 'WARNING'
|
||||
'cpu_count': cpu_count,
|
||||
'load_average': ({'1min': load1, '5min': load5, '15min': load15}
|
||||
if load15 is not None else None),
|
||||
'load15_per_core_percent': load15_per_core,
|
||||
'sustained_high': sustained_high,
|
||||
'status': 'WARNING' if sustained_high else 'OK',
|
||||
}
|
||||
return cpu_health
|
||||
|
||||
def _check_network_status(self) -> Dict[str, Any]:
|
||||
"""
|
||||
|
||||
Reference in New Issue
Block a user