fix: stop false-positive [hardware] tickets in a redundant Ceph fleet #24
+18
-4
@@ -1296,10 +1296,20 @@ class SystemHealthMonitor:
|
||||
(r'nvme\d+.*I/O error', 'NVMe I/O errors')
|
||||
]
|
||||
|
||||
# Kernel-log references to non-physical / network-backed block devices.
|
||||
# Errors on these are NOT local hardware faults: e.g. "Buffer I/O error on
|
||||
# dev rbdN" is caused by Ceph connectivity blips, not a failing drive. Ceph
|
||||
# RBD is likewise excluded from physical-disk monitoring (see _is_physical_disk).
|
||||
non_physical_dev = re.compile(r'\b(?:rbd\d+|dm-\d+|loop\d+)\b', re.IGNORECASE)
|
||||
|
||||
for pattern, description in error_patterns:
|
||||
matches = re.findall(pattern, result.stdout, re.IGNORECASE)
|
||||
if matches:
|
||||
count = len(matches)
|
||||
compiled = re.compile(pattern, re.IGNORECASE)
|
||||
# Count matching lines, skipping those for non-physical devices.
|
||||
count = sum(
|
||||
1 for line in result.stdout.splitlines()
|
||||
if compiled.search(line) and not non_physical_dev.search(line)
|
||||
)
|
||||
if count:
|
||||
if count >= 5:
|
||||
system_health['status'] = 'CRITICAL'
|
||||
system_health['issues'].append(f"CRITICAL: {description} in system logs ({count} occurrences)")
|
||||
@@ -2610,7 +2620,11 @@ class SystemHealthMonitor:
|
||||
'Reported_Uncorrect': {'warning': 1, 'critical': 10},
|
||||
'Spin_Retry_Count': {'warning': 1, 'critical': 5},
|
||||
'Power_Cycle_Count': {'warning': 5000, 'critical': 10000},
|
||||
'Power_On_Hours': {'warning': 61320, 'critical': 70080},
|
||||
# Power_On_Hours (drive age) intentionally has NO threshold: age alone is not a
|
||||
# failure and we run drives to hard-failure rather than replace on age. With Ceph
|
||||
# (2-3x redundancy) + PBS backups, a dead drive just drops an OSD and self-heals.
|
||||
# The value is still recorded (attributes/history/description); only real
|
||||
# failure-predictors (reallocated/pending/uncorrectable/CRC, self-test) raise tickets.
|
||||
'Temperature_Celsius': {'warning': 65, 'critical': 75},
|
||||
'Available_Spare': {'warning': 30, 'critical': 10},
|
||||
'Program_Fail_Count': {'warning': 10, 'critical': 20},
|
||||
|
||||
+81
-1
@@ -6,7 +6,7 @@ import pytest
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(__file__)))
|
||||
|
||||
from unittest.mock import patch # noqa: E402
|
||||
from unittest.mock import patch, MagicMock # noqa: E402
|
||||
from hwmonDaemon import SystemHealthMonitor # noqa: E402
|
||||
|
||||
|
||||
@@ -423,3 +423,83 @@ class TestCategorizeIssue:
|
||||
def test_nvme_is_hardware(self, monitor):
|
||||
cat, _, _, _ = monitor._categorize_issue('NVMe drive warning on /dev/nvme0')
|
||||
assert cat == monitor.TICKET_CATEGORIES['HARDWARE']
|
||||
|
||||
|
||||
# ── _check_system_drive_indicators (dmesg scan) ───────────────────────────────
|
||||
|
||||
def _dmesg(monitor, lines):
|
||||
"""Run _check_system_drive_indicators with a faked `dmesg` output."""
|
||||
fake = MagicMock(returncode=0, stdout='\n'.join(lines), stderr='')
|
||||
with patch('subprocess.run', return_value=fake):
|
||||
return monitor._check_system_drive_indicators()
|
||||
|
||||
|
||||
class TestSystemDriveIndicators:
|
||||
def test_rbd_buffer_io_errors_are_ignored(self, monitor):
|
||||
"""Ceph RBD Buffer I/O errors are storage-connectivity, not local hardware —
|
||||
they must not raise a CRITICAL drive alert (regression for the false
|
||||
'[hardware] CRITICAL: Buffer I/O errors' ticket)."""
|
||||
lines = [
|
||||
f'[Wed Jul 15 03:2{i}:51 2026] Buffer I/O error on dev rbd2, '
|
||||
'logical block 8999, lost sync page write'
|
||||
for i in range(75)
|
||||
]
|
||||
res = _dmesg(monitor, lines)
|
||||
assert res['status'] == 'OK'
|
||||
assert res['issues'] == []
|
||||
|
||||
def test_local_disk_buffer_io_errors_still_counted(self, monitor):
|
||||
"""Genuine local-disk Buffer I/O errors must still be reported."""
|
||||
lines = [
|
||||
f'[Wed Jul 15 03:30:0{i} 2026] Buffer I/O error on dev sda1, '
|
||||
'logical block 42, lost async page write'
|
||||
for i in range(5)
|
||||
]
|
||||
res = _dmesg(monitor, lines)
|
||||
assert res['status'] == 'CRITICAL'
|
||||
assert any('Buffer I/O errors' in i and '5 occurrences' in i for i in res['issues'])
|
||||
|
||||
def test_rbd_noise_does_not_inflate_real_errors(self, monitor):
|
||||
"""RBD errors mixed with real disk errors: only the real ones count."""
|
||||
lines = [f'Buffer I/O error on dev rbd4, logical block {i}, lost sync page write'
|
||||
for i in range(75)]
|
||||
lines += [
|
||||
'Buffer I/O error on dev sda1, logical block 1, lost async page write',
|
||||
'Buffer I/O error on dev sda1, logical block 2, lost async page write',
|
||||
]
|
||||
res = _dmesg(monitor, lines)
|
||||
# 2 real occurrences → WARNING (>=2), not CRITICAL (>=5)
|
||||
assert res['status'] == 'WARNING'
|
||||
assert any('Buffer I/O errors' in i and '2 occurrences' in i for i in res['issues'])
|
||||
|
||||
def test_device_mapper_and_loop_ignored(self, monitor):
|
||||
"""Device-mapper and loop devices are also non-physical and ignored."""
|
||||
lines = [
|
||||
'Buffer I/O error on dev dm-3, logical block 5, lost sync page write',
|
||||
'Buffer I/O error on dev loop0, logical block 6, lost sync page write',
|
||||
]
|
||||
res = _dmesg(monitor, lines)
|
||||
assert res['status'] == 'OK'
|
||||
assert res['issues'] == []
|
||||
|
||||
|
||||
# ── _get_attribute_thresholds (drive age is not ticketed) ─────────────────────
|
||||
|
||||
class TestAttributeThresholds:
|
||||
def test_power_on_hours_has_no_threshold(self, monitor):
|
||||
"""Drive age must not raise a ticket: we run to hard-failure (Ceph redundancy +
|
||||
PBS backups), so Power_On_Hours has no warning/critical threshold. Regression for
|
||||
the '[hardware] Drive ... has SMART issues: Warning Power_On_Hours' age tickets."""
|
||||
assert monitor._get_attribute_thresholds('Power_On_Hours', {}) is None
|
||||
|
||||
def test_real_failure_predictors_still_alert(self, monitor):
|
||||
"""Genuine failure-predictor attributes must keep their thresholds."""
|
||||
for attr in ('Reallocated_Sector_Ct', 'Current_Pending_Sector',
|
||||
'Offline_Uncorrectable', 'Reported_Uncorrect'):
|
||||
th = monitor._get_attribute_thresholds(attr, {})
|
||||
assert th is not None and 'warning' in th and 'critical' in th
|
||||
|
||||
def test_power_on_hours_still_recorded_for_new_drive_logic(self, monitor):
|
||||
"""Age is still *tracked* (just not ticketed) — new-drive detection must work."""
|
||||
assert monitor._is_new_drive(100) is True # ~4 days
|
||||
assert monitor._is_new_drive(64860) is False # ~7.4 years
|
||||
|
||||
Reference in New Issue
Block a user