README: add Quick run block, architecture diagram, dry-run gallery and demo GIF; add terminal renderer script
This commit is contained in:
@@ -5,6 +5,61 @@
|
||||
|
||||
A robust system health monitoring daemon that tracks hardware status and automatically creates tickets for detected issues.
|
||||
|
||||

|
||||
|
||||
## ⚡ Quick run
|
||||
|
||||
Copy and paste on any Proxmox node (as root). No clone needed.
|
||||
|
||||
```bash
|
||||
# Test it: prints the health summary and the tickets it WOULD create (nothing is sent)
|
||||
python3 -c "import urllib.request; exec(urllib.request.urlopen('https://code.lotusguild.org/LotusGuild/hwmonDaemon/raw/branch/main/hwmonDaemon.py').read().decode('utf-8'))" --dry-run
|
||||
|
||||
# Run for real (creates tickets)
|
||||
python3 -c "import urllib.request; exec(urllib.request.urlopen('https://code.lotusguild.org/LotusGuild/hwmonDaemon/raw/branch/main/hwmonDaemon.py').read().decode('utf-8'))"
|
||||
|
||||
# Install the systemd timer (hourly check)
|
||||
curl -o /etc/systemd/system/hwmon.service https://code.lotusguild.org/LotusGuild/hwmonDaemon/raw/branch/main/hwmon.service && curl -o /etc/systemd/system/hwmon.timer https://code.lotusguild.org/LotusGuild/hwmonDaemon/raw/branch/main/hwmon.timer && systemctl daemon-reload && systemctl enable --now hwmon.timer
|
||||
```
|
||||
|
||||
## Why this exists
|
||||
|
||||
A six-node Proxmox/Ceph cluster fails in small, boring ways: a drive starts reallocating sectors, a node runs hot, a pool fills up, Ceph reports slow operations. Nobody watches a dashboard all day, and a naive alert that fires every hour buries the one that matters. hwmonDaemon runs on every node, checks the hardware, and files **one ticket per real problem** in [Tinker Tickets](https://code.lotusguild.org/LotusGuild/tinker_tickets), with the diagnosis already written up.
|
||||
|
||||
```mermaid
|
||||
flowchart LR
|
||||
subgraph node["each Proxmox node (systemd timer)"]
|
||||
smart["SMART + drive usage"] --> detect
|
||||
sys["memory / CPU load / ECC"] --> detect
|
||||
net["management + Ceph network"] --> detect
|
||||
ceph["Ceph health + OSDs"] --> detect
|
||||
lxc["LXC storage"] --> detect
|
||||
detect{{"Detect + filter<br/>sustained load, standing flags"}}
|
||||
end
|
||||
detect -->|"dedup key: category + host + device"| api["Tinker Tickets API"]
|
||||
api -->|"open ticket exists"| upd["update it"]
|
||||
api -->|"closed, problem recurs"| reopen["reopen it"]
|
||||
api -->|"new problem"| new["create ticket"]
|
||||
```
|
||||
|
||||
Design choices worth calling out:
|
||||
- **Dedup by hash** of category + hostname + device, so a repeating alert updates the existing ticket and a recurrence reopens a closed one.
|
||||
- **Sustained, not spiky:** CPU tickets are gated on the 15-minute load average per core, not a transient spike.
|
||||
- **Knows the cluster:** a standing Ceph `noout` flag (set on purpose for a node without a UPS) is not ticketed.
|
||||
- **Dry-run first:** `--dry-run` prints exactly what would be filed, which is how the screenshots below were produced (on a live node, with nothing sent).
|
||||
|
||||
## Gallery
|
||||
|
||||
Output from a live node (`micro1`) in dry-run mode:
|
||||
|
||||

|
||||
|
||||
An issue was detected (Ceph reporting slow BlueStore operations), so the daemon renders the ticket it would create, including an executive summary and the cluster status:
|
||||
|
||||

|
||||
|
||||
<sub>Rendered from real command output with `scripts/render_terminal.py`.</sub>
|
||||
|
||||
## Features
|
||||
|
||||
- Comprehensive system health monitoring:
|
||||
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 1.1 MiB |
Binary file not shown.
|
After Width: | Height: | Size: 84 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 65 KiB |
@@ -0,0 +1,72 @@
|
||||
"""Render CLI output as a styled terminal screenshot / animated GIF with Playwright + Pillow.
|
||||
|
||||
Usage (as a module):
|
||||
from term_render import render
|
||||
render(title, command, lines, png='out.png', gif='out.gif', width=1000, rules=[(regex, css_color), ...])
|
||||
`lines` is a list of plain-text output lines. No real terminal is needed, so output can be sanitized first.
|
||||
"""
|
||||
import asyncio, html, io, re
|
||||
from PIL import Image
|
||||
from playwright.async_api import async_playwright
|
||||
|
||||
DEFAULT_RULES=[(r'✓|\bOK\b|\bNORMAL\b|\bPASS(ED)?\b|HEALTH_OK','#3ddc84'),(r'⚠️?|\bWARN(ING)?\b|HEALTH_WARN|\[WARN\]','#ffb454'),(r'\bERROR\b|\bFAIL(ED)?\b|\bCRITICAL\b|✗|HEALTH_ERR','#ff5c6c'),(r'^=+.*=+$|^┏.*|^┃.*|^┣.*|^┗.*','#7fdbff')]
|
||||
|
||||
PAGE='''<!doctype html><meta charset=utf-8>
|
||||
<link href="https://fonts.googleapis.com/css2?family=JetBrains+Mono:wght@400;700&display=swap" rel="stylesheet">
|
||||
<style>
|
||||
body{margin:0;background:#0b0f14;font-family:'JetBrains Mono','DejaVu Sans Mono',monospace}
|
||||
#win{width:%(w)dpx;margin:0;background:#0d1117;border:1px solid #1f2a37;border-radius:10px;overflow:hidden}
|
||||
#bar{height:34px;background:#161b22;display:flex;align-items:center;padding:0 12px;gap:8px;border-bottom:1px solid #1f2a37}
|
||||
.dot{width:12px;height:12px;border-radius:50%%}
|
||||
#ttl{margin-left:10px;color:#8b949e;font-size:13px}
|
||||
pre{margin:0;padding:16px 18px;color:#c9d1d9;font-size:14px;line-height:1.42;white-space:pre-wrap;min-height:%(h)dpx}
|
||||
.p{color:#ff8c1a;font-weight:700}.c{color:#e6edf3}.cur{background:#c9d1d9;color:#0d1117}
|
||||
</style>
|
||||
<div id=win><div id=bar><span class=dot style=background:#ff5f56></span><span class=dot style=background:#ffbd2e></span><span class=dot style=background:#27c93f></span><span id=ttl>%(t)s</span></div><pre id=t></pre></div>'''
|
||||
|
||||
def colorize(line, rules):
|
||||
s=html.escape(line)
|
||||
for pat,col in rules:
|
||||
if pat.startswith('^'):
|
||||
if re.search(pat,line): return f'<span style="color:{col}">{s}</span>'
|
||||
else:
|
||||
s=re.sub('('+pat+')',lambda m:f'<span style="color:{col}">{m.group(0)}</span>',s)
|
||||
return s
|
||||
|
||||
async def _render(title,command,lines,png,gif,width,rules,min_h):
|
||||
rules=rules or DEFAULT_RULES
|
||||
prompt='<span class=p>root@node</span><span style="color:#8b949e">:~#</span> '
|
||||
async with async_playwright() as p:
|
||||
b=await p.chromium.launch(); pg=await b.new_page(viewport={'width':width+2,'height':600})
|
||||
await pg.set_content(PAGE%{'w':width,'h':min_h,'t':html.escape(title)}); await pg.wait_for_timeout(1200)
|
||||
async def show(body,cursor=False):
|
||||
await pg.evaluate("(h)=>{document.getElementById('t').innerHTML=h}",body+('<span class=cur> </span>' if cursor else ''))
|
||||
async def shot():
|
||||
el=pg.locator('#win'); return await el.screenshot()
|
||||
full=prompt+'<span class=c>'+html.escape(command)+'</span>\n'+'\n'.join(colorize(l,rules) for l in lines)
|
||||
await show(full)
|
||||
if png: open(png,'wb').write(await shot()); print('png',png)
|
||||
if gif:
|
||||
frames=[]; dur=[]
|
||||
def add(img,d): frames.append(Image.open(io.BytesIO(img)).convert('RGB')); dur.append(d)
|
||||
await show(prompt,True); add(await shot(),500)
|
||||
for i in range(1,len(command)+1,3):
|
||||
await show(prompt+'<span class=c>'+html.escape(command[:i])+'</span>',True); add(await shot(),45)
|
||||
await show(prompt+'<span class=c>'+html.escape(command)+'</span>',True); add(await shot(),500)
|
||||
head=prompt+'<span class=c>'+html.escape(command)+'</span>\n'; out=[]
|
||||
step=max(1,len(lines)//20)
|
||||
for i in range(0,len(lines),step):
|
||||
out=lines[:i+step]; await show(head+'\n'.join(colorize(l,rules) for l in out)); add(await shot(),120)
|
||||
await show(full); add(await shot(),3500)
|
||||
w0=frames[0].width
|
||||
scale=min(1.0,780/w0); frames=[f.resize((int(f.width*scale),int(f.height*scale)),Image.LANCZOS) for f in frames]
|
||||
# all frames must share size: pad to the tallest
|
||||
H=max(f.height for f in frames); W=frames[0].width
|
||||
norm=[]
|
||||
for f in frames:
|
||||
c=Image.new('RGB',(W,H),(13,17,23)); c.paste(f,(0,0)); norm.append(c.quantize(colors=32,method=Image.MEDIANCUT,dither=Image.NONE))
|
||||
norm[0].save(gif,save_all=True,append_images=norm[1:],duration=dur,loop=0,optimize=True); print('gif',gif,len(norm))
|
||||
await b.close()
|
||||
|
||||
def render(title,command,lines,png=None,gif=None,width=1000,rules=None,min_h=200):
|
||||
asyncio.run(_render(title,command,lines,png,gif,width,rules,min_h))
|
||||
Reference in New Issue
Block a user