From 286112aa8135545f1a4ee407f4e83ebc7c635acb Mon Sep 17 00:00:00 2001 From: Jared Vititoe Date: Wed, 4 Dec 2024 21:55:33 -0500 Subject: [PATCH] Added comments and adjusted ticket creation to not just critical events. --- hwmonDaemon.py | 332 ++++++++++++++++++++++--------------------------- 1 file changed, 150 insertions(+), 182 deletions(-) diff --git a/hwmonDaemon.py b/hwmonDaemon.py index 0da3138..0ce4ea6 100644 --- a/hwmonDaemon.py +++ b/hwmonDaemon.py @@ -2,7 +2,6 @@ import os import sys import json -import datetime import requests import psutil import socket @@ -14,136 +13,154 @@ class SystemHealthMonitor: ticket_api_url: str = 'http://10.10.10.45/create_ticket_api.php', state_file: str = '/tmp/last_health_check.json'): """ - Initialize the system health monitor + Initialize the system health monitor. - :param ticket_api_url: URL for ticket creation API - :param state_file: File to track last health check + :param ticket_api_url: URL for the ticket creation API. + :param state_file: File path to track the last health check results. """ self.ticket_api_url = ticket_api_url self.state_file = state_file def run(self): """ - Perform a one-shot health check + Perform a one-shot health check of the system. """ try: - # Perform health checks + # Perform health checks and gather the report health_report = self.perform_health_checks() - # Create tickets for critical issues + # Create tickets for any detected critical issues self._create_tickets_for_issues(health_report) except Exception as e: print(f"Unexpected error during health check: {e}") def perform_health_checks(self) -> Dict[str, Any]: """ - Perform comprehensive system health checks + Perform comprehensive system health checks and return a report. - :return: Dictionary with health check results + :return: Dictionary containing results of various health checks. """ health_report = { 'drives_health': self._check_drives_health(), 'memory_health': self._check_memory_usage(), 'cpu_health': self._check_cpu_usage(), 'network_health': self._check_network_status() - #'temperature_health': self._check_system_temperatures() + # 'temperature_health': self._check_system_temperatures() # Optional temperature check } return health_report def _create_tickets_for_issues(self, health_report: Dict[str, Any]): """ - Create tickets for critical issues with dynamic parameters. + Create tickets for detected issues with dynamic parameters based on severity. + + :param health_report: The comprehensive health report from the checks. """ - critical_issues = self._detect_critical_issues(health_report) - if not critical_issues: - print("No critical issues detected.") + issues = self._detect_issues(health_report) + if not issues: + print("No issues detected.") return # Initialize default ticket fields - priority = "P4" # Default to low priority - categories = set() # To accumulate unique categories - issue_types = set() # To accumulate unique issue types - hostname = socket.gethostname() - action_type = "[auto]" - scope = "[cluster-wide]" - environment = "[production]" - ticket_type = "[maintenance]" - - # Analyze critical issues to determine ticket parameters - for issue in critical_issues: - if "disk" in issue.lower(): + hostname = socket.gethostname() # Get the current hostname + action_type = "[auto]" # Default action type for automatic checks + scope = "[cluster-wide]" # Scope of the issues + environment = "[production]" # Environment where the issues were found + ticket_type = "[maintenance]" # Type of the ticket being created + + for issue in issues: + # Determine priority, category, and type based on the issue detected + priority = "P4" # Default to low priority + category = "Other" + issue_type = "Task" + + if "Disk" in issue: priority = "P3" # Medium priority for disk issues - categories.add("Hardware") - issue_types.add("Incident") - elif "memory" in issue.lower(): + category = "Hardware" + issue_type = "Incident" + elif "Memory" in issue: priority = "P4" # Low priority for memory issues - categories.add("Hardware") - issue_types.add("Incident") - elif "cpu" in issue.lower(): + category = "Hardware" + issue_type = "Incident" + elif "CPU" in issue: priority = "P4" # Low priority for CPU issues - categories.add("Hardware") - issue_types.add("Incident") - elif "internet connectivity" in issue.lower(): + category = "Hardware" + issue_type = "Incident" + elif "issues" in issue: # Any network issues priority = "P2" # High priority for network issues - categories.add("Network") - issue_types.add("Problem") - elif "health issues" in issue.lower(): - priority = "P1" # Critical priority for health issues - categories.add("Hardware") - issue_types.add("Problem") - - # Create a list from the set to get unique values - category = list(categories)[0] if categories else "Other" - issue_type = list(issue_types)[0] if issue_types else "Task" - - ticket_title = f"[{hostname}]{action_type}[{issue_type}] System Health Issues Detected {scope}{environment}{ticket_type}" - ticket_description = "Multiple system health issues detected:\n\n" + "\n".join(critical_issues) - - ticket_payload = { - "title": ticket_title, - "description": ticket_description, - "priority": priority, - "status": "Open", - "category": category, - "type": issue_type - } - - try: - response = requests.post( - self.ticket_api_url, - json=ticket_payload, - headers={'Content-Type': 'application/json'} - ) - if response.status_code in [200, 201]: - print(f"Ticket created successfully: {ticket_title}") - else: - print(f"Failed to create ticket. Status code: {response.status_code}") - print(f"Response: {response.text}") - except Exception as e: - print(f"Error creating ticket: {e}") + category = "Network" + issue_type = "Problem" + # Create the ticket title with relevant details + ticket_title = f"[{hostname}]{action_type}[{issue_type}] {issue} {scope}{environment}{ticket_type}" + ticket_payload = { + "title": ticket_title, + "description": issue, + "priority": priority, + "status": "Open", + "category": category, + "type": issue_type + } + # Attempt to create the ticket via the API + try: + response = requests.post( + self.ticket_api_url, + json=ticket_payload, + headers={'Content-Type': 'application/json'} + ) + if response.status_code in [200, 201]: + print(f"Ticket created successfully: {ticket_title}") + else: + print(f"Failed to create ticket. Status code: {response.status_code}") + print(f"Response: {response.text}") + except Exception as e: + print(f"Error creating ticket: {e}") - def _detect_critical_issues(self, health_report: Dict[str, Any]) -> List[str]: + def _detect_issues(self, health_report: Dict[str, Any]) -> List[str]: """ - Detect critical issues in the health report + Detect issues in the health report including non-critical issues. - :param health_report: Comprehensive health report - :return: List of critical issue descriptions + :param health_report: The comprehensive health report from the checks. + :return: List of issue descriptions detected during checks. """ - critical_issues = [] - for partition in health_report.get('disk_health', {}).get('partitions', []): - if partition.get('status') == 'CRITICAL_HIGH_USAGE': - critical_issues.append( + issues = [] + + # Check for drive-related issues + for partition in health_report.get('drives_health', {}).get('drives', []): + if partition.get('usage_status') == 'CRITICAL_HIGH_USAGE': + issues.append( f"Disk {partition['mountpoint']} is {partition['usage_percent']}% full" ) - return critical_issues + elif partition.get('usage_status') == 'WARNING_HIGH_USAGE': + issues.append( + f"Disk {partition['mountpoint']} is {partition['usage_percent']}% full (Warning)" + ) + if partition.get('smart_status') == 'UNHEALTHY': + issues.append(f"Disk {partition['mountpoint']} has an unhealthy SMART status") + + # Check for memory-related issues + memory_health = health_report.get('memory_health', {}) + if memory_health and memory_health.get('memory_percent', 0) > 80: + issues.append("Memory usage is above 80%") + + # Check for CPU-related issues + cpu_health = health_report.get('cpu_health', {}) + if cpu_health and cpu_health.get('cpu_usage_percent', 0) > 80: + issues.append("CPU usage is above 80%") + + # Check for network-related issues + network_health = health_report.get('network_health', {}) + for network in ['management_network', 'ceph_network']: + if network_health.get(network, {}).get('issues'): + issues.extend(network_health[network]['issues']) + + return issues def _check_drives_health(self) -> Dict[str, Any]: """ Check overall health of drives including disk usage and SMART status. - :return: Combined health report of all drives + :return: Combined health report of all drives and their status. """ drives_health = {'overall_status': 'NORMAL', 'drives': []} try: @@ -156,7 +173,7 @@ class SystemHealthMonitor: 'mountpoint': partition.mountpoint } try: - # Disk usage + # Check disk usage usage = psutil.disk_usage(partition.mountpoint) usage_status = 'NORMAL' if usage.percent > 90: @@ -172,13 +189,13 @@ class SystemHealthMonitor: 'usage_status': usage_status }) - # Update overall status + # Update overall status based on usage if usage_status == 'CRITICAL_HIGH_USAGE': overall_status = 'CRITICAL_HIGH_USAGE' elif usage_status == 'WARNING_HIGH_USAGE' and overall_status != 'CRITICAL_HIGH_USAGE': overall_status = 'WARNING_HIGH_USAGE' - # SMART status + # Check SMART status of the drive try: result = subprocess.run( ['smartctl', '-H', partition.device], @@ -190,11 +207,12 @@ class SystemHealthMonitor: smart_status = 'HEALTHY' if 'PASSED' in output else 'UNHEALTHY' drive_report['smart_status'] = smart_status + # Update overall status if SMART status is unhealthy if smart_status == 'UNHEALTHY' and overall_status != 'CRITICAL_HIGH_USAGE': overall_status = 'UNHEALTHY' - except Exception as e: - drive_report['smart_status'] = f"ERROR: {str(e)}" + print(f"Error checking SMART status for {partition.device}: {str(e)}") + drive_report['smart_status'] = 'ERROR' except Exception as e: drive_report['error'] = f"Could not check drive: {str(e)}" @@ -209,14 +227,13 @@ class SystemHealthMonitor: return {'error': str(e)} - def _convert_bytes(self, bytes_value: int, suffix: str = 'B') -> str: """ - Convert bytes to human-readable format + Convert bytes to a human-readable format. - :param bytes_value: Number of bytes - :param suffix: Suffix to append (default 'B' for bytes) - :return: Formatted string with size + :param bytes_value: Number of bytes to convert. + :param suffix: Suffix to append (default is 'B' for bytes). + :return: Formatted string with the size in human-readable form. """ for unit in ['', 'K', 'M', 'G', 'T', 'P', 'E', 'Z']: if abs(bytes_value) < 1024.0: @@ -226,112 +243,63 @@ class SystemHealthMonitor: def _check_memory_usage(self) -> Dict[str, Any]: """ - Check memory usage and return health metrics + Check memory usage and return health metrics. - :return: Memory health metrics + :return: Dictionary with memory health metrics. """ - try: - memory = psutil.virtual_memory() - return { - 'total_memory': self._convert_bytes(memory.total), - 'used_memory': self._convert_bytes(memory.used), - 'free_memory': self._convert_bytes(memory.available), - 'memory_percent': memory.percent - } - except Exception as e: - print(f"Memory health check failed: {e}") - return {'error': str(e)} + memory_info = psutil.virtual_memory() + memory_health = { + 'total_memory': self._convert_bytes(memory_info.total), + 'used_memory': self._convert_bytes(memory_info.used), + 'memory_percent': memory_info.percent, + 'status': 'OK' if memory_info.percent < 80 else 'WARNING' + } + return memory_health def _check_cpu_usage(self) -> Dict[str, Any]: """ - Check CPU usage and return health metrics + Check CPU usage and return health metrics. - :return: CPU health metrics + :return: Dictionary with CPU health metrics. """ - try: - cpu_usage = psutil.cpu_percent(interval=1) - return { - 'cpu_usage_percent': cpu_usage - } - except Exception as e: - print(f"CPU health check failed: {e}") - return {'error': str(e)} + cpu_usage_percent = psutil.cpu_percent(interval=1) + cpu_health = { + 'cpu_usage_percent': cpu_usage_percent, + 'status': 'OK' if cpu_usage_percent < 80 else 'WARNING' + } + return cpu_health def _check_network_status(self) -> Dict[str, Any]: """ - Check network connectivity between nodes and include detailed identifiers. + Check the status of network interfaces and report any issues. - :return: Network health report + :return: Dictionary containing network health metrics and any issues found. """ network_health = { - 'management_network': {'status': 'UNKNOWN', 'issues': []}, - 'ceph_network': {'status': 'UNKNOWN', 'issues': []} + 'management_network': {'issues': []}, + 'ceph_network': {'issues': []} } - - # IP-to-hostname mapping - management_mapping = { - '10.10.10.2': 'large1', - '10.10.10.10': 'medium1', - '10.10.10.4': 'medium2', - '10.10.10.8': 'micro1', - '10.10.10.9': 'micro2' - } - ceph_mapping = { - '10.10.90.10': 'large1', - '10.10.90.4': 'medium1', - '10.10.90.3': 'medium2', - '10.10.90.2': 'micro1', - '10.10.90.6': 'micro2' - } - - def _ping_device(ip: str) -> bool: - try: - result = subprocess.run(['ping', '-c', '1', ip], stdout=subprocess.PIPE, stderr=subprocess.PIPE) - return result.returncode == 0 - except Exception as e: - print(f"Error pinging {ip}: {e}") - return False - - # Check management network - management_ips = list(management_mapping.keys()) - for source_ip in management_ips: - for target_ip in management_ips: - if source_ip != target_ip and not _ping_device(target_ip): - source_host = management_mapping[source_ip] - target_host = management_mapping[target_ip] - issue = f"{source_host} ({source_ip}) cannot reach {target_host} ({target_ip}) in Management Network" - network_health['management_network']['issues'].append(issue) - - # Check Ceph network - ceph_ips = list(ceph_mapping.keys()) - for source_ip in ceph_ips: - for target_ip in ceph_ips: - if source_ip != target_ip and not _ping_device(target_ip): - source_host = ceph_mapping[source_ip] - target_host = ceph_mapping[target_ip] - issue = f"{source_host} ({source_ip}) cannot reach {target_host} ({target_ip}) in Ceph Network" - network_health['ceph_network']['issues'].append(issue) - - # Update statuses - network_health['management_network']['status'] = 'HEALTHY' if not network_health['management_network']['issues'] else 'ISSUES_DETECTED' - network_health['ceph_network']['status'] = 'HEALTHY' if not network_health['ceph_network']['issues'] else 'ISSUES_DETECTED' - - return network_health + try: + # Check management network connectivity + management_check = os.system("ping -c 1 10.10.10.1") + if management_check != 0: + network_health['management_network']['issues'].append( + "Management network is unreachable." + ) + # Check Ceph network connectivity + ceph_check = os.system("ping -c 1 10.10.90.1") + if ceph_check != 0: + network_health['ceph_network']['issues'].append( + "Ceph network is unreachable." + ) -def main(): - # Initialize the monitor - monitor = SystemHealthMonitor( - ticket_api_url='http://10.10.10.45/create_ticket_api.php' - ) - - # Run the monitor - monitor.run() + return network_health + + except Exception as e: + print(f"Network health check failed: {e}") + return {'error': str(e)} if __name__ == '__main__': - # Require root/sudo for full system access - if os.geteuid() != 0: - print("This script must be run with sudo/root privileges") - sys.exit(1) - - main() \ No newline at end of file + monitor = SystemHealthMonitor() + monitor.run()