jaredandClaude Opus 5.5
c0f475bffe
- Ops Alerts → Ask Claude now calls `devdesk attach --investigate SESSION UID`. The server looks the alert up, writes the prompt and starts Claude in a new alert-<name> session with it as the first prompt, already sent. Prompts are stored in prompts/<session>.txt and consumed once by devdesk-claude-run. A session that is already running is just opened. - The prompt has the rule, query, summary, details and labels, and where it runs: the CT/VM id, the node it is on now, status and HA state, and `ssh <node> pct exec <id> --`. It reminds Claude that HA guests move (`devdesk guests <id>`), and gives aggregate rules a hint. - `devdesk guests`: Proxmox inventory over SSH (DEVDESK_PVE_NODES). pvesh gives the cluster view; each node gives lxc-info and guest-agent IPs. Cached in guests.json, keeping last-seen IPs for stopped guests; --alerts refreshes it in the background every 5 minutes. - Alerts carry uid, query, description and targets, matched by instance IP/host or by name; each card shows the target. - cmd_alerts moves to libexec/devdesk-ops.sh. Closes #39 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
301 lines
16 KiB
Bash
301 lines
16 KiB
Bash
# shellcheck shell=bash disable=SC2016 # single-quoted jq filters use jq variables
|
|
# Ops (devdesk#39): Grafana alerts, the Proxmox guests they are about, and
|
|
# "Ask Claude" investigations in a session of their own. Sourced by bin/devdesk.
|
|
|
|
DEVDESK_GUESTS="$DEVDESK_STATE_DIR/guests.json"
|
|
|
|
# ---------------------------------------------------------------- Proxmox guests
|
|
# Which CT/VM has which IP, and on which node it runs right now: guests are HA
|
|
# and move between nodes, so this is read live from the cluster, over SSH to the
|
|
# nodes in DEVDESK_PVE_NODES (empty = off). Cached in guests.json, where a
|
|
# stopped guest keeps the IPs it had when it was last seen running.
|
|
|
|
# Cluster view, from any one node: guests (id, name, node, status, HA) and nodes.
|
|
DD_PVE_INVENTORY='pvesh get /cluster/resources --type vm --output-format json && echo && pvesh get /cluster/status --output-format json'
|
|
# Per node: its own addresses, then those of its running guests ("<id>\t<IPs>").
|
|
IFS= read -r -d '' DD_PVE_NODE_SCAN <<'EOF' || true
|
|
printf '%s\t%s\n' "$(hostname)" "$(hostname -I)"
|
|
for id in $(lxc-ls --running 2>/dev/null); do
|
|
printf '%s\t%s\n' "$id" "$(lxc-info -n "$id" -iH 2>/dev/null | tr '\n' ' ')"
|
|
done
|
|
# VMs only through the QEMU guest agent, where one is configured; slow, so all at once.
|
|
for f in /var/run/qemu-server/*.pid; do
|
|
id=${f##*/}
|
|
id=${id%.pid}
|
|
case $id in '' | *[!0-9]*) continue ;; esac
|
|
grep -qE '^agent: *(1|enabled=1)' "/etc/pve/qemu-server/$id.conf" 2>/dev/null || continue
|
|
printf '%s\t%s\n' "$id" "$(timeout 3 qm guest cmd "$id" network-get-interfaces 2>/dev/null |
|
|
grep -oE '"ip-address" *: *"[0-9.]+"' | grep -oE '[0-9.]+' | tr '\n' ' ')" &
|
|
done
|
|
wait
|
|
EOF
|
|
|
|
dd_pve_ssh() {
|
|
local host=$1
|
|
shift
|
|
timeout 12 "${DEVDESK_PVE_SSH:-ssh}" -o BatchMode=yes -o ConnectTimeout=4 "$host" "$@"
|
|
}
|
|
|
|
# Refresh guests.json. Returns non-zero if no node answered (the cache stays).
|
|
dd_pve_scan() {
|
|
[ -n "${DEVDESK_PVE_NODES:-}" ] || return 1
|
|
local host inv="" tmp node ip pids=() rc=0
|
|
for host in $DEVDESK_PVE_NODES; do
|
|
inv=$(dd_pve_ssh "$host" "$DD_PVE_INVENTORY" 2>/dev/null) && jq -es 'length == 2' >/dev/null 2>&1 <<<"$inv" && break
|
|
inv=""
|
|
done
|
|
[ -n "$inv" ] || return 1
|
|
tmp=$(mktemp -d) || return 1
|
|
# Every online node at once: by name, or by its cluster address if the name doesn't answer.
|
|
while IFS=$'\t' read -r node ip; do
|
|
{ dd_pve_ssh "$node" "$DD_PVE_NODE_SCAN" || dd_pve_ssh "$ip" "$DD_PVE_NODE_SCAN"; } >"$tmp/$node" 2>/dev/null </dev/null &
|
|
pids+=($!)
|
|
done < <(jq -rs '.[1][] | select(.type == "node" and .online == 1) | [.name, .ip] | @tsv' <<<"$inv")
|
|
[ ${#pids[@]} -eq 0 ] || wait "${pids[@]}" 2>/dev/null || true
|
|
mkdir -p "$DEVDESK_STATE_DIR"
|
|
# Scan lines: the first is "<node>\t<its IPs>", then "<CT id>\t<its IPs>".
|
|
if { cat "$tmp"/* 2>/dev/null || true; } | jq -cMRs --slurpfile inv <(jq -s . <<<"$inv") \
|
|
--slurpfile old <(cat "$DEVDESK_GUESTS" 2>/dev/null || echo '{}') --arg src "$host" '
|
|
def v4: [splits("[[:space:]]+") | select(test("^[0-9]{1,3}(\\.[0-9]{1,3}){3}$") and (startswith("127.") | not))];
|
|
(split("\n") | map(select(length > 0) | split("\t"))) as $rows
|
|
| ([$rows[] | select(.[0] | test("^[0-9]+$")) | {key: .[0], value: (.[1] // "" | v4)}] | from_entries) as $live
|
|
| ([$rows[] | select(.[0] | test("^[0-9]+$") | not) | {key: .[0], value: (.[1] // "" | v4)}] | from_entries) as $nodeips
|
|
| ([$old[0].guests[]? | {key: (.id | tostring), value: .ips}] | from_entries) as $prev
|
|
| {updated: (now | floor), source: $src,
|
|
nodes: [$inv[0][1][] | select(.type == "node")
|
|
| {name, online: (.online == 1), ips: (([.ip] + ($nodeips[.name] // [])) | map(select(. != null)) | unique)}],
|
|
guests: [$inv[0][0][] | select(.vmid != null) | (.vmid | tostring) as $id
|
|
| ($live[$id] // []) as $now
|
|
| {id: .vmid, type, name, node, status, ha: (.hastate // null),
|
|
ips: (if ($now | length) > 0 then $now else ($prev[$id] // []) end),
|
|
ips_live: (($now | length) > 0)}]
|
|
| sort_by(.id)}' >"$tmp/out" && [ -s "$tmp/out" ]; then
|
|
chmod 644 "$tmp/out"
|
|
mv -f "$tmp/out" "$DEVDESK_GUESTS"
|
|
else
|
|
rc=1
|
|
fi
|
|
rm -rf "$tmp"
|
|
return $rc
|
|
}
|
|
|
|
# Refresh, one scan at a time: "wait" for a running one (then scan again), else
|
|
# skip when one is already running.
|
|
dd_pve_refresh() {
|
|
mkdir -p "$DEVDESK_STATE_DIR"
|
|
if [ "${1:-}" = wait ]; then
|
|
(flock -w 30 9 || exit 1; dd_pve_scan) 9>"$DEVDESK_GUESTS.lock"
|
|
else
|
|
(flock -n 9 || exit 0; dd_pve_scan) 9>"$DEVDESK_GUESTS.lock"
|
|
fi
|
|
}
|
|
|
|
# Seconds since guests.json was written (large when there is none).
|
|
dd_guests_age() {
|
|
local m
|
|
m=$(stat -c %Y "$DEVDESK_GUESTS" 2>/dev/null) || { echo 999999; return; }
|
|
echo $(($(date +%s) - m))
|
|
}
|
|
|
|
# jq: locate the guest or node a host/label is about, given the guests.json
|
|
# document as $g. A target: {kind: "ct"|"vm"|"node", id, name, node, status, ha,
|
|
# ip, by: "ip"|"host"|"name"}.
|
|
DD_LOCATE_JQ='
|
|
def kindof: if .type == "qemu" then "vm" else "ct" end;
|
|
def as_guest: {kind: kindof, id, name, node, status, ha};
|
|
def as_node: {kind: "node", name, node: .name, status: (if .online then "online" else "offline" end)};
|
|
def isv4: test("^[0-9]{1,3}(\\.[0-9]{1,3}){3}$");
|
|
# "http://10.0.0.1:8080/x", "10.0.0.1:9100", "host.example.org:9100" -> the host
|
|
def hostpart: sub("^[A-Za-z][A-Za-z0-9+.-]*://"; "") | sub("/.*$"; "") | sub("^[^@]*@"; "") | sub(":[0-9]+$"; "") | ascii_downcase;
|
|
def norm: ascii_downcase | gsub("[^a-z0-9]+"; "-");
|
|
def by_host($g; $h):
|
|
if ($h | isv4) then
|
|
([$g.nodes[]? | select(.ips | index([$h])) | as_node + {ip: $h, by: "ip"}]
|
|
+ [$g.guests[]? | select(.ips | index([$h])) | as_guest + {ip: $h, by: "ip", ip_seen: (.ips_live | not)}]) | first
|
|
else
|
|
($h | split(".")[0]) as $s
|
|
| ([$g.nodes[]? | select((.name | ascii_downcase) == $s) | as_node + {by: "host"}]
|
|
+ [$g.guests[]? | select((.name | ascii_downcase) == $s) | as_guest + {by: "host"}]) | first
|
|
end;
|
|
# A guest or node whose name appears as whole words in $v ("draupnir-node",
|
|
# "compute-storage-01-ceph", "Draupnir Down"); the longest name wins.
|
|
def by_name($g; $v):
|
|
("-" + ($v | norm) + "-") as $t
|
|
| [($g.nodes[]? | as_node), ($g.guests[]? | as_guest)]
|
|
| map(select((.name // "" | norm) as $n | ($n | length) >= 4 and ($t | contains("-" + $n + "-"))))
|
|
| sort_by(-(.name | length)) | first | if . then . + {by: "name"} else empty end;
|
|
# Targets of one alert (an element of the alerts list below): by each label
|
|
# set: its instance (IP or host name), else its job; with no labels at all, by the
|
|
# name of the alert itself ("Draupnir Down").
|
|
def targets($g):
|
|
if ($g | type) != "object" then [] else
|
|
.name as $alert
|
|
| [.labelsets[] | (.instance // "" | hostpart) as $h
|
|
| if $h != "" then (by_host($g; $h) // by_name($g; .job // "") // {kind: "unknown", ip: $h})
|
|
elif .job then by_name($g; .job)
|
|
else empty end]
|
|
| if length == 0 then [by_name($g; $alert)] else . end
|
|
| map(select(. != null)) | unique_by([.kind, .id, .name, .ip]) | .[:6]
|
|
end;
|
|
'
|
|
|
|
# ---------------------------------------------------------------- alerts
|
|
|
|
# Firing and pending Grafana alert rules, most severe first, each with the
|
|
# guests/nodes it is about (from the guest cache, refreshed in the background).
|
|
cmd_alerts() {
|
|
if [ -z "$DEVDESK_GRAFANA_URL" ]; then
|
|
jq -cMn '{configured: false, alerts: []}'
|
|
return
|
|
fi
|
|
local raw
|
|
if ! raw=$({
|
|
if [ -r "$DEVDESK_GRAFANA_TOKEN_FILE" ]; then
|
|
# shellcheck disable=SC1003 # tr deletes whitespace, quotes and backslashes from the token
|
|
printf 'header = "Authorization: Bearer %s"\n' "$(tr -d '[:space:]"\\' <"$DEVDESK_GRAFANA_TOKEN_FILE")"
|
|
fi
|
|
} | curl -fsS --max-time 6 -K - "$DEVDESK_GRAFANA_URL/api/prometheus/grafana/api/v1/rules" 2>&1); then
|
|
jq -cMn --arg e "${raw:-no response}" '{configured: true, error: $e, alerts: []}'
|
|
return
|
|
fi
|
|
if [ -n "${DEVDESK_PVE_NODES:-}" ] && [ "$(dd_guests_age)" -gt 300 ]; then
|
|
setsid -f "$0" guests --refresh </dev/null >/dev/null 2>&1 3>&- || true
|
|
fi
|
|
jq -cM --arg pub "$DEVDESK_GRAFANA_PUBLIC_URL" --slurpfile g <(cat "$DEVDESK_GUESTS" 2>/dev/null || echo null) "$DD_LOCATE_JQ"'
|
|
def num: (tonumber? // .) | tostring;
|
|
($g[0]) as $guests
|
|
| {configured: true, checked: (now | floor), pve: ($guests != null), alerts: [
|
|
.data.groups[] | .name as $group | .file as $folder | .rules[]
|
|
| select(.state == "firing" or .state == "pending")
|
|
| ([.alerts[]? | select(.state == "Alerting" or .state == "Pending")]) as $active
|
|
| {uid, name, state, severity: (.labels.severity // null), group: $group, folder: $folder,
|
|
query: (.query // null),
|
|
since: (.activeAt // $active[0].activeAt // null),
|
|
summary: ((.annotations.summary // .annotations.description // "")
|
|
| gsub("\\{\\{ *\\$value *\\}\\}"; ($active[0].value // "?" | num))),
|
|
description: (if .annotations.summary then (.annotations.description // null) else null end),
|
|
labelsets: [$active[] | .labels
|
|
| del(.__alert_rule_uid__, .__grafana_origin__, .alertname, .grafana_folder, .__name__, .severity)
|
|
| select(length > 0)],
|
|
url: "\($pub)/alerting/grafana/\(.uid)/view"}
|
|
| .instances = [.labelsets[] | to_entries | map("\(.key)=\(.value)") | join(" ")]
|
|
| .targets = targets($guests)
|
|
] | sort_by(if .severity == "critical" then 0 elif .severity == "warning" then 1 else 2 end)}' <<<"$raw"
|
|
}
|
|
|
|
# ---------------------------------------------------------------- guests command
|
|
|
|
# devdesk guests [--json] [--cached] [FILTER...]: Proxmox guests with their node,
|
|
# state and IPs, fresh from the cluster (the cache if no node answers).
|
|
# FILTER matches an id, an IP, or part of a name or node.
|
|
cmd_guests() {
|
|
local json="" cached="" filters=()
|
|
while [ $# -gt 0 ]; do
|
|
case $1 in
|
|
--json) json=1 ;;
|
|
--cached) cached=1 ;;
|
|
--refresh)
|
|
dd_pve_refresh
|
|
return 0
|
|
;;
|
|
-*) dd_die "usage: devdesk guests [--json] [--cached] [ID|IP|NAME...]" ;;
|
|
*) filters+=("$1") ;;
|
|
esac
|
|
shift
|
|
done
|
|
[ -n "${DEVDESK_PVE_NODES:-}" ] || dd_die "no Proxmox nodes configured (DEVDESK_PVE_NODES in /etc/devdesk/env.local)"
|
|
if [ -z "$cached" ] && ! dd_pve_refresh wait; then
|
|
[ -s "$DEVDESK_GUESTS" ] || dd_die "no Proxmox node answered ($DEVDESK_PVE_NODES)"
|
|
printf 'devdesk: no Proxmox node answered; showing what was known %s minutes ago\n' "$(($(dd_guests_age) / 60))" >&2
|
|
fi
|
|
[ -s "$DEVDESK_GUESTS" ] || dd_die "nothing known yet: run devdesk guests without --cached"
|
|
local select='[.guests[] | . as $x | select(($f | length) == 0 or any($f[]; . as $q
|
|
| ($x.id | tostring) == $q or ($x.ips | index([$q])) or ($x.name | ascii_downcase | contains($q | ascii_downcase))
|
|
or ($x.node | ascii_downcase | contains($q | ascii_downcase))))]'
|
|
local f
|
|
f=$(printf '%s\n' "${filters[@]}" | jq -R . | jq -cs 'map(select(length > 0))')
|
|
if [ -n "$json" ]; then
|
|
jq -cM --argjson f "$f" "{updated, nodes, guests: $select}" "$DEVDESK_GUESTS"
|
|
return
|
|
fi
|
|
jq -rM --argjson f "$f" "$select"' as $g
|
|
| if ($g | length) == 0 then "no matching guests" else
|
|
(["ID", "TYPE", "NAME", "NODE", "STATUS", "HA", "IP"] | @tsv),
|
|
($g[] | [(.id | tostring), (if .type == "qemu" then "vm" else "ct" end), .name, .node, .status, (.ha // "-"),
|
|
((.ips | join(",")) + (if (.ips | length) > 0 and (.ips_live | not) then " (last seen)" else "" end) | if . == "" then "-" else . end)] | @tsv)
|
|
end' "$DEVDESK_GUESTS" | column -t -s $'\t'
|
|
}
|
|
|
|
# ---------------------------------------------------------------- investigate
|
|
|
|
# The opening prompt for an investigation of alert $a (one element of the
|
|
# alerts output), with $since already readable.
|
|
DD_PROMPT_JQ='
|
|
def ref: if .kind == "ct" then "CT \(.id) \"\(.name)\"" elif .kind == "vm" then "VM \(.id) \"\(.name)\"" else "Proxmox node \(.name)" end;
|
|
def state: [(if .status then .status else empty end), (if .ha then "HA state \(.ha)" else empty end)] | join(", ");
|
|
def how:
|
|
if .kind == "ct" then "commands inside it: `ssh \(.node) pct exec \(.id) -- <command>`"
|
|
elif .kind == "vm" then "commands inside it (needs the guest agent): `ssh \(.node) qm guest exec \(.id) -- <command>`"
|
|
else "reach it with `ssh \(.name)`" end;
|
|
def where:
|
|
if .kind == "unknown" then "- \(.ip) is not a Proxmox guest or node that DevDesk knows (`devdesk guests` lists them all)."
|
|
elif .kind == "node" then "- \(if .ip then "\(.ip) is " else "" end)\(ref)\(if .by == "name" then " (probably: matched by name)" else "" end), \(.status); \(how)"
|
|
else "- \(if .by == "name" then "Probably " else "" end)\(if .ip then "\(.ip) is " else "" end)\(ref) on node \(.node)"
|
|
+ "\(if .by == "name" then " (matched by name, not by IP)" elif .ip_seen then " (its IP when last seen running)" else "" end), \(state); \(how)"
|
|
end;
|
|
$a
|
|
| ([.targets[] | select(.kind == "ct" or .kind == "vm")] | first) as $guest
|
|
| [
|
|
"Investigate this Grafana alert and find the cause. Look first and change nothing yet: tell me what you found and what you would change, then wait for my go-ahead.",
|
|
"",
|
|
"Alert: \(.name) (\(.severity // "no severity"), \(.state) since \($since))",
|
|
"Rule: \(.folder) / \(.group), uid \(.uid). Look at it with the grafana MCP server: \(.url)",
|
|
(if .query then "Query: `\(.query)`" else empty end),
|
|
(if .summary != "" then "Summary: \(.summary)" else empty end),
|
|
(if .description then "Details: \(.description)" else empty end),
|
|
(if (.instances | length) > 0 then "Firing for: \(.instances | join("; "))"
|
|
else "Firing for: no labels. The rule aggregates, so it does not say what it is about: run the inner query to find the series"
|
|
+ (if $pve then ", then `devdesk guests <IP>` shows which Proxmox guest each one is and where it runs." else "." end) end),
|
|
(if $pve and (.targets | length) > 0 then
|
|
("", "Where it runs (Proxmox, looked up just now):"),
|
|
(.targets[] | where),
|
|
(if $guest then "Guests are HA and can migrate between nodes, so check again before acting: `devdesk guests \($guest.id)` shows where it is now." else empty end)
|
|
else empty end)
|
|
] | join("\n")'
|
|
|
|
# --investigate SESSION RULE_UID: start Claude in a new session SESSION with an
|
|
# investigation of that alert as its first prompt. An existing session is left as
|
|
# it is: {"ok":true,"session":…,"started":false}.
|
|
cmd_investigate() {
|
|
[ $# -eq 2 ] || dd_die "usage: devdesk attach --investigate SESSION ALERT_RULE_UID"
|
|
local name=$1 uid=$2 alerts alert since prompt dir pve=false
|
|
dd_valid_name "$name" || dd_die "invalid session name: $name"
|
|
[[ $uid =~ ^[A-Za-z0-9_-]{1,64}$ ]] || dd_die "invalid alert rule id: $uid"
|
|
if dd_server_up && dd_tmux has-session -t "=$name" 2>/dev/null; then
|
|
jq -cMn --arg s "$name" '{ok: true, session: $s, started: false}'
|
|
return
|
|
fi
|
|
[ -n "$DEVDESK_GRAFANA_URL" ] || dd_die "Grafana is not configured"
|
|
# Where the guests are right now (HA moves them); the cache if no node answers.
|
|
[ -z "${DEVDESK_PVE_NODES:-}" ] || dd_pve_refresh wait || true
|
|
alerts=$(cmd_alerts)
|
|
alert=$(jq -cM --arg uid "$uid" '.alerts[] | select(.uid == $uid)' <<<"$alerts" | head -n1)
|
|
if [ -z "$alert" ]; then
|
|
dd_die "$(jq -rM 'if .error then "Grafana unreachable: \(.error)" else "that alert is no longer firing" end' <<<"$alerts")"
|
|
fi
|
|
since=$(jq -rM '.since // empty' <<<"$alert")
|
|
since=$(date -d "$since" '+%a %-d %b %H:%M %Z' 2>/dev/null || echo "${since:-an unknown time}")
|
|
[ -s "$DEVDESK_GUESTS" ] && [ -n "${DEVDESK_PVE_NODES:-}" ] && pve=true
|
|
prompt=$(jq -rnM --argjson a "$alert" --arg since "$since" --argjson pve "$pve" "$DD_PROMPT_JQ") || dd_die "could not write the prompt"
|
|
# devdesk-claude-run sends this once, as the new conversation's first prompt.
|
|
mkdir -p "$DEVDESK_STATE_DIR/prompts"
|
|
(umask 077 && printf '%s\n' "$prompt" >"$DEVDESK_STATE_DIR/prompts/$name.txt")
|
|
# A new conversation, even if an earlier investigation used this name (that
|
|
# one stays in Search past conversations).
|
|
dd_registry_update 'if .sessions[$n] then .sessions[$n] |= del(.claude_session_id, .claude_source, .fork) else . end' --arg n "$name"
|
|
dir=$(dd_resolve_dir "$name")
|
|
dd_ensure_server
|
|
create_detached "$name" "$dir"
|
|
jq -cMn --arg s "$name" '{ok: true, session: $s, started: true}'
|
|
}
|