LotusGuild/hwmonDaemon
2
0
Fork 0
Code Issues 3 Pull Requests Actions Packages Projects Releases Wiki Activity

Added comments and adjusted ticket creation to not just critical events.

Browse Source
This commit is contained in:
Jared Vititoe
2024-12-04 21:55:33 -05:00
parent 34075225bb
commit 286112aa81
1 changed files with 150 additions and 182 deletions
Download Patch File Download Diff File Expand all files Collapse all files

278
hwmonDaemon.py
View File

@ -2,7 +2,6 @@
import os import os
import sys import sys
import json import json
import datetime
import requests import requests
import psutil import psutil
import socket import socket
@ -14,100 +13,95 @@ class SystemHealthMonitor:
ticket_api_url: str = 'http://10.10.10.45/create_ticket_api.php', ticket_api_url: str = 'http://10.10.10.45/create_ticket_api.php',
state_file: str = '/tmp/last_health_check.json'): state_file: str = '/tmp/last_health_check.json'):
""" """
Initialize the system health monitor Initialize the system health monitor.
:param ticket_api_url: URL for ticket creation API :param ticket_api_url: URL for the ticket creation API.
:param state_file: File to track last health check :param state_file: File path to track the last health check results.
""" """
self.ticket_api_url = ticket_api_url self.ticket_api_url = ticket_api_url
self.state_file = state_file self.state_file = state_file
def run(self): def run(self):
""" """
Perform a one-shot health check Perform a one-shot health check of the system.
""" """
try: try:
# Perform health checks # Perform health checks and gather the report
health_report = self.perform_health_checks() health_report = self.perform_health_checks()
# Create tickets for critical issues # Create tickets for any detected critical issues
self._create_tickets_for_issues(health_report) self._create_tickets_for_issues(health_report)
except Exception as e: except Exception as e:
print(f"Unexpected error during health check: {e}") print(f"Unexpected error during health check: {e}")
def perform_health_checks(self) -> Dict[str, Any]: def perform_health_checks(self) -> Dict[str, Any]:
""" """
Perform comprehensive system health checks Perform comprehensive system health checks and return a report.
:return: Dictionary with health check results :return: Dictionary containing results of various health checks.
""" """
health_report = { health_report = {
'drives_health': self._check_drives_health(), 'drives_health': self._check_drives_health(),
'memory_health': self._check_memory_usage(), 'memory_health': self._check_memory_usage(),
'cpu_health': self._check_cpu_usage(), 'cpu_health': self._check_cpu_usage(),
'network_health': self._check_network_status() 'network_health': self._check_network_status()
#'temperature_health': self._check_system_temperatures() # 'temperature_health': self._check_system_temperatures() # Optional temperature check
} }
return health_report return health_report
def _create_tickets_for_issues(self, health_report: Dict[str, Any]): def _create_tickets_for_issues(self, health_report: Dict[str, Any]):
""" """
Create tickets for critical issues with dynamic parameters. Create tickets for detected issues with dynamic parameters based on severity.
:param health_report: The comprehensive health report from the checks.
""" """
critical_issues = self._detect_critical_issues(health_report) issues = self._detect_issues(health_report)
if not critical_issues: if not issues:
print("No critical issues detected.") print("No issues detected.")
return return
# Initialize default ticket fields # Initialize default ticket fields
hostname = socket.gethostname() # Get the current hostname
action_type = "[auto]" # Default action type for automatic checks
scope = "[cluster-wide]" # Scope of the issues
environment = "[production]" # Environment where the issues were found
ticket_type = "[maintenance]" # Type of the ticket being created
for issue in issues:
# Determine priority, category, and type based on the issue detected
priority = "P4" # Default to low priority priority = "P4" # Default to low priority
categories = set() # To accumulate unique categories category = "Other"
issue_types = set() # To accumulate unique issue types issue_type = "Task"
hostname = socket.gethostname()
action_type = "[auto]"
scope = "[cluster-wide]"
environment = "[production]"
ticket_type = "[maintenance]"
# Analyze critical issues to determine ticket parameters if "Disk" in issue:
for issue in critical_issues:
if "disk" in issue.lower():
priority = "P3" # Medium priority for disk issues priority = "P3" # Medium priority for disk issues
categories.add("Hardware") category = "Hardware"
issue_types.add("Incident") issue_type = "Incident"
elif "memory" in issue.lower(): elif "Memory" in issue:
priority = "P4" # Low priority for memory issues priority = "P4" # Low priority for memory issues
categories.add("Hardware") category = "Hardware"
issue_types.add("Incident") issue_type = "Incident"
elif "cpu" in issue.lower(): elif "CPU" in issue:
priority = "P4" # Low priority for CPU issues priority = "P4" # Low priority for CPU issues
categories.add("Hardware") category = "Hardware"
issue_types.add("Incident") issue_type = "Incident"
elif "internet connectivity" in issue.lower(): elif "issues" in issue: # Any network issues
priority = "P2" # High priority for network issues priority = "P2" # High priority for network issues
categories.add("Network") category = "Network"
issue_types.add("Problem") issue_type = "Problem"
elif "health issues" in issue.lower():
priority = "P1" # Critical priority for health issues
categories.add("Hardware")
issue_types.add("Problem")
# Create a list from the set to get unique values
category = list(categories)[0] if categories else "Other"
issue_type = list(issue_types)[0] if issue_types else "Task"
ticket_title = f"[{hostname}]{action_type}[{issue_type}] System Health Issues Detected {scope}{environment}{ticket_type}"
ticket_description = "Multiple system health issues detected:\n\n" + "\n".join(critical_issues)
# Create the ticket title with relevant details
ticket_title = f"[{hostname}]{action_type}[{issue_type}] {issue} {scope}{environment}{ticket_type}"
ticket_payload = { ticket_payload = {
"title": ticket_title, "title": ticket_title,
"description": ticket_description, "description": issue,
"priority": priority, "priority": priority,
"status": "Open", "status": "Open",
"category": category, "category": category,
"type": issue_type "type": issue_type
} }
# Attempt to create the ticket via the API
try: try:
response = requests.post( response = requests.post(
self.ticket_api_url, self.ticket_api_url,
@ -122,28 +116,51 @@ class SystemHealthMonitor:
except Exception as e: except Exception as e:
print(f"Error creating ticket: {e}") print(f"Error creating ticket: {e}")
def _detect_issues(self, health_report: Dict[str, Any]) -> List[str]:
def _detect_critical_issues(self, health_report: Dict[str, Any]) -> List[str]:
""" """
Detect critical issues in the health report Detect issues in the health report including non-critical issues.
:param health_report: Comprehensive health report :param health_report: The comprehensive health report from the checks.
:return: List of critical issue descriptions :return: List of issue descriptions detected during checks.
""" """
critical_issues = [] issues = []
for partition in health_report.get('disk_health', {}).get('partitions', []):
if partition.get('status') == 'CRITICAL_HIGH_USAGE': # Check for drive-related issues
critical_issues.append( for partition in health_report.get('drives_health', {}).get('drives', []):
if partition.get('usage_status') == 'CRITICAL_HIGH_USAGE':
issues.append(
f"Disk {partition['mountpoint']} is {partition['usage_percent']}% full" f"Disk {partition['mountpoint']} is {partition['usage_percent']}% full"
) )
return critical_issues elif partition.get('usage_status') == 'WARNING_HIGH_USAGE':
issues.append(
f"Disk {partition['mountpoint']} is {partition['usage_percent']}% full (Warning)"
)
if partition.get('smart_status') == 'UNHEALTHY':
issues.append(f"Disk {partition['mountpoint']} has an unhealthy SMART status")
# Check for memory-related issues
memory_health = health_report.get('memory_health', {})
if memory_health and memory_health.get('memory_percent', 0) > 80:
issues.append("Memory usage is above 80%")
# Check for CPU-related issues
cpu_health = health_report.get('cpu_health', {})
if cpu_health and cpu_health.get('cpu_usage_percent', 0) > 80:
issues.append("CPU usage is above 80%")
# Check for network-related issues
network_health = health_report.get('network_health', {})
for network in ['management_network', 'ceph_network']:
if network_health.get(network, {}).get('issues'):
issues.extend(network_health[network]['issues'])
return issues
def _check_drives_health(self) -> Dict[str, Any]: def _check_drives_health(self) -> Dict[str, Any]:
""" """
Check overall health of drives including disk usage and SMART status. Check overall health of drives including disk usage and SMART status.
:return: Combined health report of all drives :return: Combined health report of all drives and their status.
""" """
drives_health = {'overall_status': 'NORMAL', 'drives': []} drives_health = {'overall_status': 'NORMAL', 'drives': []}
try: try:
@ -156,7 +173,7 @@ class SystemHealthMonitor:
'mountpoint': partition.mountpoint 'mountpoint': partition.mountpoint
} }
try: try:
# Disk usage # Check disk usage
usage = psutil.disk_usage(partition.mountpoint) usage = psutil.disk_usage(partition.mountpoint)
usage_status = 'NORMAL' usage_status = 'NORMAL'
if usage.percent > 90: if usage.percent > 90:
@ -172,13 +189,13 @@ class SystemHealthMonitor:
'usage_status': usage_status 'usage_status': usage_status
}) })
# Update overall status # Update overall status based on usage
if usage_status == 'CRITICAL_HIGH_USAGE': if usage_status == 'CRITICAL_HIGH_USAGE':
overall_status = 'CRITICAL_HIGH_USAGE' overall_status = 'CRITICAL_HIGH_USAGE'
elif usage_status == 'WARNING_HIGH_USAGE' and overall_status != 'CRITICAL_HIGH_USAGE': elif usage_status == 'WARNING_HIGH_USAGE' and overall_status != 'CRITICAL_HIGH_USAGE':
overall_status = 'WARNING_HIGH_USAGE' overall_status = 'WARNING_HIGH_USAGE'
# SMART status # Check SMART status of the drive
try: try:
result = subprocess.run( result = subprocess.run(
['smartctl', '-H', partition.device], ['smartctl', '-H', partition.device],
@ -190,11 +207,12 @@ class SystemHealthMonitor:
smart_status = 'HEALTHY' if 'PASSED' in output else 'UNHEALTHY' smart_status = 'HEALTHY' if 'PASSED' in output else 'UNHEALTHY'
drive_report['smart_status'] = smart_status drive_report['smart_status'] = smart_status
# Update overall status if SMART status is unhealthy
if smart_status == 'UNHEALTHY' and overall_status != 'CRITICAL_HIGH_USAGE': if smart_status == 'UNHEALTHY' and overall_status != 'CRITICAL_HIGH_USAGE':
overall_status = 'UNHEALTHY' overall_status = 'UNHEALTHY'
except Exception as e: except Exception as e:
drive_report['smart_status'] = f"ERROR: {str(e)}" print(f"Error checking SMART status for {partition.device}: {str(e)}")
drive_report['smart_status'] = 'ERROR'
except Exception as e: except Exception as e:
drive_report['error'] = f"Could not check drive: {str(e)}" drive_report['error'] = f"Could not check drive: {str(e)}"
@ -209,14 +227,13 @@ class SystemHealthMonitor:
return {'error': str(e)} return {'error': str(e)}
def _convert_bytes(self, bytes_value: int, suffix: str = 'B') -> str: def _convert_bytes(self, bytes_value: int, suffix: str = 'B') -> str:
""" """
Convert bytes to human-readable format Convert bytes to a human-readable format.
:param bytes_value: Number of bytes :param bytes_value: Number of bytes to convert.
:param suffix: Suffix to append (default 'B' for bytes) :param suffix: Suffix to append (default is 'B' for bytes).
:return: Formatted string with size :return: Formatted string with the size in human-readable form.
""" """
for unit in ['', 'K', 'M', 'G', 'T', 'P', 'E', 'Z']: for unit in ['', 'K', 'M', 'G', 'T', 'P', 'E', 'Z']:
if abs(bytes_value) < 1024.0: if abs(bytes_value) < 1024.0:
@ -226,112 +243,63 @@ class SystemHealthMonitor:
def _check_memory_usage(self) -> Dict[str, Any]: def _check_memory_usage(self) -> Dict[str, Any]:
""" """
Check memory usage and return health metrics Check memory usage and return health metrics.
:return: Memory health metrics :return: Dictionary with memory health metrics.
""" """
try: memory_info = psutil.virtual_memory()
memory = psutil.virtual_memory() memory_health = {
return { 'total_memory': self._convert_bytes(memory_info.total),
'total_memory': self._convert_bytes(memory.total), 'used_memory': self._convert_bytes(memory_info.used),
'used_memory': self._convert_bytes(memory.used), 'memory_percent': memory_info.percent,
'free_memory': self._convert_bytes(memory.available), 'status': 'OK' if memory_info.percent < 80 else 'WARNING'
'memory_percent': memory.percent
} }
except Exception as e: return memory_health
print(f"Memory health check failed: {e}")
return {'error': str(e)}
def _check_cpu_usage(self) -> Dict[str, Any]: def _check_cpu_usage(self) -> Dict[str, Any]:
""" """
Check CPU usage and return health metrics Check CPU usage and return health metrics.
:return: CPU health metrics :return: Dictionary with CPU health metrics.
""" """
try: cpu_usage_percent = psutil.cpu_percent(interval=1)
cpu_usage = psutil.cpu_percent(interval=1) cpu_health = {
return { 'cpu_usage_percent': cpu_usage_percent,
'cpu_usage_percent': cpu_usage 'status': 'OK' if cpu_usage_percent < 80 else 'WARNING'
} }
except Exception as e: return cpu_health
print(f"CPU health check failed: {e}")
return {'error': str(e)}
def _check_network_status(self) -> Dict[str, Any]: def _check_network_status(self) -> Dict[str, Any]:
""" """
Check network connectivity between nodes and include detailed identifiers. Check the status of network interfaces and report any issues.
:return: Network health report :return: Dictionary containing network health metrics and any issues found.
""" """
network_health = { network_health = {
'management_network': {'status': 'UNKNOWN', 'issues': []}, 'management_network': {'issues': []},
'ceph_network': {'status': 'UNKNOWN', 'issues': []} 'ceph_network': {'issues': []}
} }
# IP-to-hostname mapping
management_mapping = {
'10.10.10.2': 'large1',
'10.10.10.10': 'medium1',
'10.10.10.4': 'medium2',
'10.10.10.8': 'micro1',
'10.10.10.9': 'micro2'
}
ceph_mapping = {
'10.10.90.10': 'large1',
'10.10.90.4': 'medium1',
'10.10.90.3': 'medium2',
'10.10.90.2': 'micro1',
'10.10.90.6': 'micro2'
}
def _ping_device(ip: str) -> bool:
try: try:
result = subprocess.run(['ping', '-c', '1', ip], stdout=subprocess.PIPE, stderr=subprocess.PIPE) # Check management network connectivity
return result.returncode == 0 management_check = os.system("ping -c 1 10.10.10.1")
except Exception as e: if management_check != 0:
print(f"Error pinging {ip}: {e}") network_health['management_network']['issues'].append(
return False "Management network is unreachable."
)
# Check management network # Check Ceph network connectivity
management_ips = list(management_mapping.keys()) ceph_check = os.system("ping -c 1 10.10.90.1")
for source_ip in management_ips: if ceph_check != 0:
for target_ip in management_ips: network_health['ceph_network']['issues'].append(
if source_ip != target_ip and not _ping_device(target_ip): "Ceph network is unreachable."
source_host = management_mapping[source_ip] )
target_host = management_mapping[target_ip]
issue = f"{source_host} ({source_ip}) cannot reach {target_host} ({target_ip}) in Management Network"
network_health['management_network']['issues'].append(issue)
# Check Ceph network
ceph_ips = list(ceph_mapping.keys())
for source_ip in ceph_ips:
for target_ip in ceph_ips:
if source_ip != target_ip and not _ping_device(target_ip):
source_host = ceph_mapping[source_ip]
target_host = ceph_mapping[target_ip]
issue = f"{source_host} ({source_ip}) cannot reach {target_host} ({target_ip}) in Ceph Network"
network_health['ceph_network']['issues'].append(issue)
# Update statuses
network_health['management_network']['status'] = 'HEALTHY' if not network_health['management_network']['issues'] else 'ISSUES_DETECTED'
network_health['ceph_network']['status'] = 'HEALTHY' if not network_health['ceph_network']['issues'] else 'ISSUES_DETECTED'
return network_health return network_health
except Exception as e:
def main(): print(f"Network health check failed: {e}")
# Initialize the monitor return {'error': str(e)}
monitor = SystemHealthMonitor(
ticket_api_url='http://10.10.10.45/create_ticket_api.php'
)
# Run the monitor
monitor.run()
if __name__ == '__main__': if __name__ == '__main__':
# Require root/sudo for full system access monitor = SystemHealthMonitor()
if os.geteuid() != 0: monitor.run()
print("This script must be run with sudo/root privileges")
sys.exit(1)
main()