diff --git a/README.md b/README.md index 5ac03aa..16bed7e 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,61 @@ A robust system health monitoring daemon that tracks hardware status and automatically creates tickets for detected issues. +![hwmonDaemon dry-run: health summary followed by the ticket it would create](docs/img/demo.gif) + +## ⚡ Quick run + +Copy and paste on any Proxmox node (as root). No clone needed. + +```bash +# Test it: prints the health summary and the tickets it WOULD create (nothing is sent) +python3 -c "import urllib.request; exec(urllib.request.urlopen('https://code.lotusguild.org/LotusGuild/hwmonDaemon/raw/branch/main/hwmonDaemon.py').read().decode('utf-8'))" --dry-run + +# Run for real (creates tickets) +python3 -c "import urllib.request; exec(urllib.request.urlopen('https://code.lotusguild.org/LotusGuild/hwmonDaemon/raw/branch/main/hwmonDaemon.py').read().decode('utf-8'))" + +# Install the systemd timer (hourly check) +curl -o /etc/systemd/system/hwmon.service https://code.lotusguild.org/LotusGuild/hwmonDaemon/raw/branch/main/hwmon.service && curl -o /etc/systemd/system/hwmon.timer https://code.lotusguild.org/LotusGuild/hwmonDaemon/raw/branch/main/hwmon.timer && systemctl daemon-reload && systemctl enable --now hwmon.timer +``` + +## Why this exists + +A six-node Proxmox/Ceph cluster fails in small, boring ways: a drive starts reallocating sectors, a node runs hot, a pool fills up, Ceph reports slow operations. Nobody watches a dashboard all day, and a naive alert that fires every hour buries the one that matters. hwmonDaemon runs on every node, checks the hardware, and files **one ticket per real problem** in [Tinker Tickets](https://code.lotusguild.org/LotusGuild/tinker_tickets), with the diagnosis already written up. + +```mermaid +flowchart LR + subgraph node["each Proxmox node (systemd timer)"] + smart["SMART + drive usage"] --> detect + sys["memory / CPU load / ECC"] --> detect + net["management + Ceph network"] --> detect + ceph["Ceph health + OSDs"] --> detect + lxc["LXC storage"] --> detect + detect{{"Detect + filter
sustained load, standing flags"}} + end + detect -->|"dedup key: category + host + device"| api["Tinker Tickets API"] + api -->|"open ticket exists"| upd["update it"] + api -->|"closed, problem recurs"| reopen["reopen it"] + api -->|"new problem"| new["create ticket"] +``` + +Design choices worth calling out: +- **Dedup by hash** of category + hostname + device, so a repeating alert updates the existing ticket and a recurrence reopens a closed one. +- **Sustained, not spiky:** CPU tickets are gated on the 15-minute load average per core, not a transient spike. +- **Knows the cluster:** a standing Ceph `noout` flag (set on purpose for a node without a UPS) is not ticketed. +- **Dry-run first:** `--dry-run` prints exactly what would be filed, which is how the screenshots below were produced (on a live node, with nothing sent). + +## Gallery + +Output from a live node (`micro1`) in dry-run mode: + +![Health summary](docs/img/dry-run-summary.png) + +An issue was detected (Ceph reporting slow BlueStore operations), so the daemon renders the ticket it would create, including an executive summary and the cluster status: + +![Simulated ticket](docs/img/dry-run-ticket.png) + +Rendered from real command output with `scripts/render_terminal.py`. + ## Features - Comprehensive system health monitoring: diff --git a/docs/img/demo.gif b/docs/img/demo.gif new file mode 100644 index 0000000..ff72c27 Binary files /dev/null and b/docs/img/demo.gif differ diff --git a/docs/img/dry-run-summary.png b/docs/img/dry-run-summary.png new file mode 100644 index 0000000..2b60929 Binary files /dev/null and b/docs/img/dry-run-summary.png differ diff --git a/docs/img/dry-run-ticket.png b/docs/img/dry-run-ticket.png new file mode 100644 index 0000000..89c1b6d Binary files /dev/null and b/docs/img/dry-run-ticket.png differ diff --git a/scripts/render_terminal.py b/scripts/render_terminal.py new file mode 100644 index 0000000..ce731f3 --- /dev/null +++ b/scripts/render_terminal.py @@ -0,0 +1,72 @@ +"""Render CLI output as a styled terminal screenshot / animated GIF with Playwright + Pillow. + +Usage (as a module): + from term_render import render + render(title, command, lines, png='out.png', gif='out.gif', width=1000, rules=[(regex, css_color), ...]) +`lines` is a list of plain-text output lines. No real terminal is needed, so output can be sanitized first. +""" +import asyncio, html, io, re +from PIL import Image +from playwright.async_api import async_playwright + +DEFAULT_RULES=[(r'✓|\bOK\b|\bNORMAL\b|\bPASS(ED)?\b|HEALTH_OK','#3ddc84'),(r'⚠️?|\bWARN(ING)?\b|HEALTH_WARN|\[WARN\]','#ffb454'),(r'\bERROR\b|\bFAIL(ED)?\b|\bCRITICAL\b|✗|HEALTH_ERR','#ff5c6c'),(r'^=+.*=+$|^┏.*|^┃.*|^┣.*|^┗.*','#7fdbff')] + +PAGE=''' + + +
%(t)s
'''
+
+def colorize(line, rules):
+    s=html.escape(line)
+    for pat,col in rules:
+        if pat.startswith('^'):
+            if re.search(pat,line): return f'{s}'
+        else:
+            s=re.sub('('+pat+')',lambda m:f'{m.group(0)}',s)
+    return s
+
+async def _render(title,command,lines,png,gif,width,rules,min_h):
+    rules=rules or DEFAULT_RULES
+    prompt='root@node:~# '
+    async with async_playwright() as p:
+        b=await p.chromium.launch(); pg=await b.new_page(viewport={'width':width+2,'height':600})
+        await pg.set_content(PAGE%{'w':width,'h':min_h,'t':html.escape(title)}); await pg.wait_for_timeout(1200)
+        async def show(body,cursor=False):
+            await pg.evaluate("(h)=>{document.getElementById('t').innerHTML=h}",body+(' ' if cursor else ''))
+        async def shot():
+            el=pg.locator('#win'); return await el.screenshot()
+        full=prompt+''+html.escape(command)+'\n'+'\n'.join(colorize(l,rules) for l in lines)
+        await show(full)
+        if png: open(png,'wb').write(await shot()); print('png',png)
+        if gif:
+            frames=[]; dur=[]
+            def add(img,d): frames.append(Image.open(io.BytesIO(img)).convert('RGB')); dur.append(d)
+            await show(prompt,True); add(await shot(),500)
+            for i in range(1,len(command)+1,3):
+                await show(prompt+''+html.escape(command[:i])+'',True); add(await shot(),45)
+            await show(prompt+''+html.escape(command)+'',True); add(await shot(),500)
+            head=prompt+''+html.escape(command)+'\n'; out=[]
+            step=max(1,len(lines)//20)
+            for i in range(0,len(lines),step):
+                out=lines[:i+step]; await show(head+'\n'.join(colorize(l,rules) for l in out)); add(await shot(),120)
+            await show(full); add(await shot(),3500)
+            w0=frames[0].width
+            scale=min(1.0,780/w0); frames=[f.resize((int(f.width*scale),int(f.height*scale)),Image.LANCZOS) for f in frames]
+            # all frames must share size: pad to the tallest
+            H=max(f.height for f in frames); W=frames[0].width
+            norm=[]
+            for f in frames:
+                c=Image.new('RGB',(W,H),(13,17,23)); c.paste(f,(0,0)); norm.append(c.quantize(colors=32,method=Image.MEDIANCUT,dither=Image.NONE))
+            norm[0].save(gif,save_all=True,append_images=norm[1:],duration=dur,loop=0,optimize=True); print('gif',gif,len(norm))
+        await b.close()
+
+def render(title,command,lines,png=None,gif=None,width=1000,rules=None,min_h=200):
+    asyncio.run(_render(title,command,lines,png,gif,width,rules,min_h))