fix: stop false-positive [hardware] tickets in a redundant Ceph fleet

Two monitoring-logic fixes so hwmonDaemon stops raising spurious hardware
tickets on a Ceph-backed cluster with PBS backups:

- _check_system_drive_indicators: count dmesg Buffer I/O / block errors
  per-line and skip non-physical / network-backed devices (Ceph RBD, dm-,
  loop). RBD Buffer I/O errors are storage-connectivity events (e.g. a Ceph
  mon-session blip), not local drive faults, so they must not raise a
  CRITICAL drive alert. Consistent with the existing _is_physical_disk
  exclusion. (Fired a false "CRITICAL: Buffer I/O errors (75 occurrences)"
  ticket where all 75 were on rbd devices.)

- _get_attribute_thresholds: drop the Power_On_Hours warning/critical
  threshold. Drive age alone is not a failure; with 2-3x Ceph redundancy
  and PBS backups we run drives to hard-failure rather than replace on age.
  The value is still recorded (history/description/new-drive logic) but no
  longer generates a ticket. Real failure-predictors (reallocated/pending/
  uncorrectable/CRC, self-test, trends) are unchanged.

Adds 7 regression tests (TestSystemDriveIndicators, TestAttributeThresholds).
All 98 tests pass; flake8 clean.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

hwmonDaemon.py

@@ -1296,10 +1296,20 @@ class SystemHealthMonitor:
                     (r'nvme\d+.*I/O error', 'NVMe I/O errors')
                 ]
 
+                # Kernel-log references to non-physical / network-backed block devices.
+                # Errors on these are NOT local hardware faults: e.g. "Buffer I/O error on
+                # dev rbdN" is caused by Ceph connectivity blips, not a failing drive. Ceph
+                # RBD is likewise excluded from physical-disk monitoring (see _is_physical_disk).
+                non_physical_dev = re.compile(r'\b(?:rbd\d+|dm-\d+|loop\d+)\b', re.IGNORECASE)
+
                 for pattern, description in error_patterns:
-                    matches = re.findall(pattern, result.stdout, re.IGNORECASE)
-                    if matches:
-                        count = len(matches)
+                    compiled = re.compile(pattern, re.IGNORECASE)
+                    # Count matching lines, skipping those for non-physical devices.
+                    count = sum(
+                        1 for line in result.stdout.splitlines()
+                        if compiled.search(line) and not non_physical_dev.search(line)
+                    )
+                    if count:
                         if count >= 5:
                             system_health['status'] = 'CRITICAL'
                             system_health['issues'].append(f"CRITICAL: {description} in system logs ({count} occurrences)")
@@ -2610,7 +2620,11 @@ class SystemHealthMonitor:
             'Reported_Uncorrect': {'warning': 1, 'critical': 10},
             'Spin_Retry_Count': {'warning': 1, 'critical': 5},
             'Power_Cycle_Count': {'warning': 5000, 'critical': 10000},
-            'Power_On_Hours': {'warning': 61320, 'critical': 70080},
+            # Power_On_Hours (drive age) intentionally has NO threshold: age alone is not a
+            # failure and we run drives to hard-failure rather than replace on age. With Ceph
+            # (2-3x redundancy) + PBS backups, a dead drive just drops an OSD and self-heals.
+            # The value is still recorded (attributes/history/description); only real
+            # failure-predictors (reallocated/pending/uncorrectable/CRC, self-test) raise tickets.
             'Temperature_Celsius': {'warning': 65, 'critical': 75},
             'Available_Spare': {'warning': 30, 'critical': 10},
             'Program_Fail_Count': {'warning': 10, 'critical': 20},

tests/test_hwmon.py

@@ -6,7 +6,7 @@ import pytest
 
 sys.path.insert(0, os.path.dirname(os.path.dirname(__file__)))
 
-from unittest.mock import patch  # noqa: E402
+from unittest.mock import patch, MagicMock  # noqa: E402
 from hwmonDaemon import SystemHealthMonitor  # noqa: E402
 
 
@@ -423,3 +423,83 @@ class TestCategorizeIssue:
     def test_nvme_is_hardware(self, monitor):
         cat, _, _, _ = monitor._categorize_issue('NVMe drive warning on /dev/nvme0')
         assert cat == monitor.TICKET_CATEGORIES['HARDWARE']
+
+
+# ── _check_system_drive_indicators (dmesg scan) ───────────────────────────────
+
+def _dmesg(monitor, lines):
+    """Run _check_system_drive_indicators with a faked `dmesg` output."""
+    fake = MagicMock(returncode=0, stdout='\n'.join(lines), stderr='')
+    with patch('subprocess.run', return_value=fake):
+        return monitor._check_system_drive_indicators()
+
+
+class TestSystemDriveIndicators:
+    def test_rbd_buffer_io_errors_are_ignored(self, monitor):
+        """Ceph RBD Buffer I/O errors are storage-connectivity, not local hardware —
+        they must not raise a CRITICAL drive alert (regression for the false
+        '[hardware] CRITICAL: Buffer I/O errors' ticket)."""
+        lines = [
+            f'[Wed Jul 15 03:2{i}:51 2026] Buffer I/O error on dev rbd2, '
+            'logical block 8999, lost sync page write'
+            for i in range(75)
+        ]
+        res = _dmesg(monitor, lines)
+        assert res['status'] == 'OK'
+        assert res['issues'] == []
+
+    def test_local_disk_buffer_io_errors_still_counted(self, monitor):
+        """Genuine local-disk Buffer I/O errors must still be reported."""
+        lines = [
+            f'[Wed Jul 15 03:30:0{i} 2026] Buffer I/O error on dev sda1, '
+            'logical block 42, lost async page write'
+            for i in range(5)
+        ]
+        res = _dmesg(monitor, lines)
+        assert res['status'] == 'CRITICAL'
+        assert any('Buffer I/O errors' in i and '5 occurrences' in i for i in res['issues'])
+
+    def test_rbd_noise_does_not_inflate_real_errors(self, monitor):
+        """RBD errors mixed with real disk errors: only the real ones count."""
+        lines = [f'Buffer I/O error on dev rbd4, logical block {i}, lost sync page write'
+                 for i in range(75)]
+        lines += [
+            'Buffer I/O error on dev sda1, logical block 1, lost async page write',
+            'Buffer I/O error on dev sda1, logical block 2, lost async page write',
+        ]
+        res = _dmesg(monitor, lines)
+        # 2 real occurrences → WARNING (>=2), not CRITICAL (>=5)
+        assert res['status'] == 'WARNING'
+        assert any('Buffer I/O errors' in i and '2 occurrences' in i for i in res['issues'])
+
+    def test_device_mapper_and_loop_ignored(self, monitor):
+        """Device-mapper and loop devices are also non-physical and ignored."""
+        lines = [
+            'Buffer I/O error on dev dm-3, logical block 5, lost sync page write',
+            'Buffer I/O error on dev loop0, logical block 6, lost sync page write',
+        ]
+        res = _dmesg(monitor, lines)
+        assert res['status'] == 'OK'
+        assert res['issues'] == []
+
+
+# ── _get_attribute_thresholds (drive age is not ticketed) ─────────────────────
+
+class TestAttributeThresholds:
+    def test_power_on_hours_has_no_threshold(self, monitor):
+        """Drive age must not raise a ticket: we run to hard-failure (Ceph redundancy +
+        PBS backups), so Power_On_Hours has no warning/critical threshold. Regression for
+        the '[hardware] Drive ... has SMART issues: Warning Power_On_Hours' age tickets."""
+        assert monitor._get_attribute_thresholds('Power_On_Hours', {}) is None
+
+    def test_real_failure_predictors_still_alert(self, monitor):
+        """Genuine failure-predictor attributes must keep their thresholds."""
+        for attr in ('Reallocated_Sector_Ct', 'Current_Pending_Sector',
+                     'Offline_Uncorrectable', 'Reported_Uncorrect'):
+            th = monitor._get_attribute_thresholds(attr, {})
+            assert th is not None and 'warning' in th and 'critical' in th
+
+    def test_power_on_hours_still_recorded_for_new_drive_logic(self, monitor):
+        """Age is still *tracked* (just not ticketed) — new-drive detection must work."""
+        assert monitor._is_new_drive(100) is True      # ~4 days
+        assert monitor._is_new_drive(64860) is False   # ~7.4 years