From 2c6b8c12945b6d644764dd5f36be24fb32fd9331 Mon Sep 17 00:00:00 2001 From: Jared Vititoe Date: Wed, 15 Jul 2026 15:20:49 -0400 Subject: [PATCH] fix: stop false-positive [hardware] tickets in a redundant Ceph fleet Two monitoring-logic fixes so hwmonDaemon stops raising spurious hardware tickets on a Ceph-backed cluster with PBS backups: - _check_system_drive_indicators: count dmesg Buffer I/O / block errors per-line and skip non-physical / network-backed devices (Ceph RBD, dm-, loop). RBD Buffer I/O errors are storage-connectivity events (e.g. a Ceph mon-session blip), not local drive faults, so they must not raise a CRITICAL drive alert. Consistent with the existing _is_physical_disk exclusion. (Fired a false "CRITICAL: Buffer I/O errors (75 occurrences)" ticket where all 75 were on rbd devices.) - _get_attribute_thresholds: drop the Power_On_Hours warning/critical threshold. Drive age alone is not a failure; with 2-3x Ceph redundancy and PBS backups we run drives to hard-failure rather than replace on age. The value is still recorded (history/description/new-drive logic) but no longer generates a ticket. Real failure-predictors (reallocated/pending/ uncorrectable/CRC, self-test, trends) are unchanged. Adds 7 regression tests (TestSystemDriveIndicators, TestAttributeThresholds). All 98 tests pass; flake8 clean. Co-Authored-By: Claude Opus 4.8 --- hwmonDaemon.py | 22 +++++++++--- tests/test_hwmon.py | 82 ++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 99 insertions(+), 5 deletions(-) diff --git a/hwmonDaemon.py b/hwmonDaemon.py index 36167c7..6c74768 100644 --- a/hwmonDaemon.py +++ b/hwmonDaemon.py @@ -1296,10 +1296,20 @@ class SystemHealthMonitor: (r'nvme\d+.*I/O error', 'NVMe I/O errors') ] + # Kernel-log references to non-physical / network-backed block devices. + # Errors on these are NOT local hardware faults: e.g. "Buffer I/O error on + # dev rbdN" is caused by Ceph connectivity blips, not a failing drive. Ceph + # RBD is likewise excluded from physical-disk monitoring (see _is_physical_disk). + non_physical_dev = re.compile(r'\b(?:rbd\d+|dm-\d+|loop\d+)\b', re.IGNORECASE) + for pattern, description in error_patterns: - matches = re.findall(pattern, result.stdout, re.IGNORECASE) - if matches: - count = len(matches) + compiled = re.compile(pattern, re.IGNORECASE) + # Count matching lines, skipping those for non-physical devices. + count = sum( + 1 for line in result.stdout.splitlines() + if compiled.search(line) and not non_physical_dev.search(line) + ) + if count: if count >= 5: system_health['status'] = 'CRITICAL' system_health['issues'].append(f"CRITICAL: {description} in system logs ({count} occurrences)") @@ -2610,7 +2620,11 @@ class SystemHealthMonitor: 'Reported_Uncorrect': {'warning': 1, 'critical': 10}, 'Spin_Retry_Count': {'warning': 1, 'critical': 5}, 'Power_Cycle_Count': {'warning': 5000, 'critical': 10000}, - 'Power_On_Hours': {'warning': 61320, 'critical': 70080}, + # Power_On_Hours (drive age) intentionally has NO threshold: age alone is not a + # failure and we run drives to hard-failure rather than replace on age. With Ceph + # (2-3x redundancy) + PBS backups, a dead drive just drops an OSD and self-heals. + # The value is still recorded (attributes/history/description); only real + # failure-predictors (reallocated/pending/uncorrectable/CRC, self-test) raise tickets. 'Temperature_Celsius': {'warning': 65, 'critical': 75}, 'Available_Spare': {'warning': 30, 'critical': 10}, 'Program_Fail_Count': {'warning': 10, 'critical': 20}, diff --git a/tests/test_hwmon.py b/tests/test_hwmon.py index a75b9fe..d6e127b 100644 --- a/tests/test_hwmon.py +++ b/tests/test_hwmon.py @@ -6,7 +6,7 @@ import pytest sys.path.insert(0, os.path.dirname(os.path.dirname(__file__))) -from unittest.mock import patch # noqa: E402 +from unittest.mock import patch, MagicMock # noqa: E402 from hwmonDaemon import SystemHealthMonitor # noqa: E402 @@ -423,3 +423,83 @@ class TestCategorizeIssue: def test_nvme_is_hardware(self, monitor): cat, _, _, _ = monitor._categorize_issue('NVMe drive warning on /dev/nvme0') assert cat == monitor.TICKET_CATEGORIES['HARDWARE'] + + +# ── _check_system_drive_indicators (dmesg scan) ─────────────────────────────── + +def _dmesg(monitor, lines): + """Run _check_system_drive_indicators with a faked `dmesg` output.""" + fake = MagicMock(returncode=0, stdout='\n'.join(lines), stderr='') + with patch('subprocess.run', return_value=fake): + return monitor._check_system_drive_indicators() + + +class TestSystemDriveIndicators: + def test_rbd_buffer_io_errors_are_ignored(self, monitor): + """Ceph RBD Buffer I/O errors are storage-connectivity, not local hardware — + they must not raise a CRITICAL drive alert (regression for the false + '[hardware] CRITICAL: Buffer I/O errors' ticket).""" + lines = [ + f'[Wed Jul 15 03:2{i}:51 2026] Buffer I/O error on dev rbd2, ' + 'logical block 8999, lost sync page write' + for i in range(75) + ] + res = _dmesg(monitor, lines) + assert res['status'] == 'OK' + assert res['issues'] == [] + + def test_local_disk_buffer_io_errors_still_counted(self, monitor): + """Genuine local-disk Buffer I/O errors must still be reported.""" + lines = [ + f'[Wed Jul 15 03:30:0{i} 2026] Buffer I/O error on dev sda1, ' + 'logical block 42, lost async page write' + for i in range(5) + ] + res = _dmesg(monitor, lines) + assert res['status'] == 'CRITICAL' + assert any('Buffer I/O errors' in i and '5 occurrences' in i for i in res['issues']) + + def test_rbd_noise_does_not_inflate_real_errors(self, monitor): + """RBD errors mixed with real disk errors: only the real ones count.""" + lines = [f'Buffer I/O error on dev rbd4, logical block {i}, lost sync page write' + for i in range(75)] + lines += [ + 'Buffer I/O error on dev sda1, logical block 1, lost async page write', + 'Buffer I/O error on dev sda1, logical block 2, lost async page write', + ] + res = _dmesg(monitor, lines) + # 2 real occurrences → WARNING (>=2), not CRITICAL (>=5) + assert res['status'] == 'WARNING' + assert any('Buffer I/O errors' in i and '2 occurrences' in i for i in res['issues']) + + def test_device_mapper_and_loop_ignored(self, monitor): + """Device-mapper and loop devices are also non-physical and ignored.""" + lines = [ + 'Buffer I/O error on dev dm-3, logical block 5, lost sync page write', + 'Buffer I/O error on dev loop0, logical block 6, lost sync page write', + ] + res = _dmesg(monitor, lines) + assert res['status'] == 'OK' + assert res['issues'] == [] + + +# ── _get_attribute_thresholds (drive age is not ticketed) ───────────────────── + +class TestAttributeThresholds: + def test_power_on_hours_has_no_threshold(self, monitor): + """Drive age must not raise a ticket: we run to hard-failure (Ceph redundancy + + PBS backups), so Power_On_Hours has no warning/critical threshold. Regression for + the '[hardware] Drive ... has SMART issues: Warning Power_On_Hours' age tickets.""" + assert monitor._get_attribute_thresholds('Power_On_Hours', {}) is None + + def test_real_failure_predictors_still_alert(self, monitor): + """Genuine failure-predictor attributes must keep their thresholds.""" + for attr in ('Reallocated_Sector_Ct', 'Current_Pending_Sector', + 'Offline_Uncorrectable', 'Reported_Uncorrect'): + th = monitor._get_attribute_thresholds(attr, {}) + assert th is not None and 'warning' in th and 'critical' in th + + def test_power_on_hours_still_recorded_for_new_drive_logic(self, monitor): + """Age is still *tracked* (just not ticketed) — new-drive detection must work.""" + assert monitor._is_new_drive(100) is True # ~4 days + assert monitor._is_new_drive(64860) is False # ~7.4 years