devdesk/libexec/devdesk-ops.sh

# shellcheck shell=bash disable=SC2016 # single-quoted jq filters use jq variables
# Ops (devdesk#39): Grafana alerts, the Proxmox guests they are about, and
# "Ask Claude" investigations in a session of their own. Sourced by bin/devdesk.

DEVDESK_GUESTS="$DEVDESK_STATE_DIR/guests.json"

# ---------------------------------------------------------------- Proxmox guests
# Which CT/VM has which IP, and on which node it runs right now: guests are HA
# and move between nodes, so this is read live from the cluster, over SSH to the
# nodes in DEVDESK_PVE_NODES (empty = off). Cached in guests.json, where a
# stopped guest keeps the IPs it had when it was last seen running.

# Cluster view, from any one node: guests (id, name, node, status, HA) and nodes.
DD_PVE_INVENTORY='pvesh get /cluster/resources --type vm --output-format json && echo && pvesh get /cluster/status --output-format json'
# Per node: its own addresses, then those of its running guests ("<id>\t<IPs>").
IFS= read -r -d '' DD_PVE_NODE_SCAN <<'EOF' || true
printf '%s\t%s\n' "$(hostname)" "$(hostname -I)"
for id in $(lxc-ls --running 2>/dev/null); do
    printf '%s\t%s\n' "$id" "$(lxc-info -n "$id" -iH 2>/dev/null | tr '\n' ' ')"
done
# VMs only through the QEMU guest agent, where one is configured; slow, so all at once.
for f in /var/run/qemu-server/*.pid; do
    id=${f##*/}
    id=${id%.pid}
    case $id in '' | *[!0-9]*) continue ;; esac
    grep -qE '^agent: *(1|enabled=1)' "/etc/pve/qemu-server/$id.conf" 2>/dev/null || continue
    printf '%s\t%s\n' "$id" "$(timeout 3 qm guest cmd "$id" network-get-interfaces 2>/dev/null |
        grep -oE '"ip-address" *: *"[0-9.]+"' | grep -oE '[0-9.]+' | tr '\n' ' ')" &
done
wait
EOF

dd_pve_ssh() {
    local host=$1
    shift
    timeout 12 "${DEVDESK_PVE_SSH:-ssh}" -o BatchMode=yes -o ConnectTimeout=4 "$host" "$@"
}

# Refresh guests.json. Returns non-zero if no node answered (the cache stays).
dd_pve_scan() {
    [ -n "${DEVDESK_PVE_NODES:-}" ] || return 1
    local host inv="" tmp node ip pids=() rc=0
    for host in $DEVDESK_PVE_NODES; do
        inv=$(dd_pve_ssh "$host" "$DD_PVE_INVENTORY" 2>/dev/null) && jq -es 'length == 2' >/dev/null 2>&1 <<<"$inv" && break
        inv=""
    done
    [ -n "$inv" ] || return 1
    tmp=$(mktemp -d) || return 1
    # Every online node at once: by name, or by its cluster address if the name doesn't answer.
    while IFS=$'\t' read -r node ip; do
        { dd_pve_ssh "$node" "$DD_PVE_NODE_SCAN" || dd_pve_ssh "$ip" "$DD_PVE_NODE_SCAN"; } >"$tmp/$node" 2>/dev/null </dev/null &
        pids+=($!)
    done < <(jq -rs '.[1][] | select(.type == "node" and .online == 1) | [.name, .ip] | @tsv' <<<"$inv")
    [ ${#pids[@]} -eq 0 ] || wait "${pids[@]}" 2>/dev/null || true
    mkdir -p "$DEVDESK_STATE_DIR"
    # Scan lines: the first is "<node>\t<its IPs>", then "<CT id>\t<its IPs>".
    if { cat "$tmp"/* 2>/dev/null || true; } | jq -cMRs --slurpfile inv <(jq -s . <<<"$inv") \
        --slurpfile old <(cat "$DEVDESK_GUESTS" 2>/dev/null || echo '{}') --arg src "$host" '
        def v4: [splits("[[:space:]]+") | select(test("^[0-9]{1,3}(\\.[0-9]{1,3}){3}$") and (startswith("127.") | not))];
        (split("\n") | map(select(length > 0) | split("\t"))) as $rows
        | ([$rows[] | select(.[0] | test("^[0-9]+$")) | {key: .[0], value: (.[1] // "" | v4)}] | from_entries) as $live
        | ([$rows[] | select(.[0] | test("^[0-9]+$") | not) | {key: .[0], value: (.[1] // "" | v4)}] | from_entries) as $nodeips
        | ([$old[0].guests[]? | {key: (.id | tostring), value: .ips}] | from_entries) as $prev
        | {updated: (now | floor), source: $src,
           nodes: [$inv[0][1][] | select(.type == "node")
                   | {name, online: (.online == 1), ips: (([.ip] + ($nodeips[.name] // [])) | map(select(. != null)) | unique)}],
           guests: [$inv[0][0][] | select(.vmid != null) | (.vmid | tostring) as $id
                    | ($live[$id] // []) as $now
                    | {id: .vmid, type, name, node, status, ha: (.hastate // null),
                       ips: (if ($now | length) > 0 then $now else ($prev[$id] // []) end),
                       ips_live: (($now | length) > 0)}]
                  | sort_by(.id)}' >"$tmp/out" && [ -s "$tmp/out" ]; then
        chmod 644 "$tmp/out"
        mv -f "$tmp/out" "$DEVDESK_GUESTS"
    else
        rc=1
    fi
    rm -rf "$tmp"
    return $rc
}

# Refresh, one scan at a time: "wait" for a running one (then scan again), else
# skip when one is already running.
dd_pve_refresh() {
    mkdir -p "$DEVDESK_STATE_DIR"
    if [ "${1:-}" = wait ]; then
        (flock -w 30 9 || exit 1; dd_pve_scan) 9>"$DEVDESK_GUESTS.lock"
    else
        (flock -n 9 || exit 0; dd_pve_scan) 9>"$DEVDESK_GUESTS.lock"
    fi
}

# Seconds since guests.json was written (large when there is none).
dd_guests_age() {
    local m
    m=$(stat -c %Y "$DEVDESK_GUESTS" 2>/dev/null) || { echo 999999; return; }
    echo $(($(date +%s) - m))
}

# jq: locate the guest or node a host/label is about, given the guests.json
# document as $g. A target: {kind: "ct"|"vm"|"node", id, name, node, status, ha,
# ip, by: "ip"|"host"|"name"}.
DD_LOCATE_JQ='
def kindof: if .type == "qemu" then "vm" else "ct" end;
def as_guest: {kind: kindof, id, name, node, status, ha};
def as_node: {kind: "node", name, node: .name, status: (if .online then "online" else "offline" end)};
def isv4: test("^[0-9]{1,3}(\\.[0-9]{1,3}){3}$");
# "http://10.0.0.1:8080/x", "10.0.0.1:9100", "host.example.org:9100" -> the host
def hostpart: sub("^[A-Za-z][A-Za-z0-9+.-]*://"; "") | sub("/.*$"; "") | sub("^[^@]*@"; "") | sub(":[0-9]+$"; "") | ascii_downcase;
def norm: ascii_downcase | gsub("[^a-z0-9]+"; "-");
def by_host($g; $h):
  if ($h | isv4) then
    ([$g.nodes[]? | select(.ips | index([$h])) | as_node + {ip: $h, by: "ip"}]
     + [$g.guests[]? | select(.ips | index([$h])) | as_guest + {ip: $h, by: "ip", ip_seen: (.ips_live | not)}]) | first
  else
    ($h | split(".")[0]) as $s
    | ([$g.nodes[]? | select((.name | ascii_downcase) == $s) | as_node + {by: "host"}]
       + [$g.guests[]? | select((.name | ascii_downcase) == $s) | as_guest + {by: "host"}]) | first
  end;
# A guest or node whose name appears as whole words in $v ("draupnir-node",
# "compute-storage-01-ceph", "Draupnir Down"); the longest name wins.
def by_name($g; $v):
  ("-" + ($v | norm) + "-") as $t
  | [($g.nodes[]? | as_node), ($g.guests[]? | as_guest)]
  | map(select((.name // "" | norm) as $n | ($n | length) >= 4 and ($t | contains("-" + $n + "-"))))
  | sort_by(-(.name | length)) | first | if . then . + {by: "name"} else empty end;
# Targets of one alert (an element of the alerts list below): by each label
# set: its instance (IP or host name), else its job; with no labels at all, by the
# name of the alert itself ("Draupnir Down").
def targets($g):
  if ($g | type) != "object" then [] else
    .name as $alert
    | [.labelsets[] | (.instance // "" | hostpart) as $h
       | if $h != "" then (by_host($g; $h) // by_name($g; .job // "") // {kind: "unknown", ip: $h})
         elif .job then by_name($g; .job)
         else empty end]
    | if length == 0 then [by_name($g; $alert)] else . end
    | map(select(. != null)) | unique_by([.kind, .id, .name, .ip]) | .[:6]
  end;
'

# ---------------------------------------------------------------- alerts

# Firing and pending Grafana alert rules, most severe first, each with the
# guests/nodes it is about (from the guest cache, refreshed in the background).
cmd_alerts() {
    if [ -z "$DEVDESK_GRAFANA_URL" ]; then
        jq -cMn '{configured: false, alerts: []}'
        return
    fi
    local raw
    if ! raw=$({
        if [ -r "$DEVDESK_GRAFANA_TOKEN_FILE" ]; then
            # shellcheck disable=SC1003 # tr deletes whitespace, quotes and backslashes from the token
            printf 'header = "Authorization: Bearer %s"\n' "$(tr -d '[:space:]"\\' <"$DEVDESK_GRAFANA_TOKEN_FILE")"
        fi
    } | curl -fsS --max-time 6 -K - "$DEVDESK_GRAFANA_URL/api/prometheus/grafana/api/v1/rules" 2>&1); then
        jq -cMn --arg e "${raw:-no response}" '{configured: true, error: $e, alerts: []}'
        return
    fi
    if [ -n "${DEVDESK_PVE_NODES:-}" ] && [ "$(dd_guests_age)" -gt 300 ]; then
        setsid -f "$0" guests --refresh </dev/null >/dev/null 2>&1 3>&- || true
    fi
    jq -cM --arg pub "$DEVDESK_GRAFANA_PUBLIC_URL" --slurpfile g <(cat "$DEVDESK_GUESTS" 2>/dev/null || echo null) "$DD_LOCATE_JQ"'
        def num: (tonumber? // .) | tostring;
        ($g[0]) as $guests
        | {configured: true, checked: (now | floor), pve: ($guests != null), alerts: [
            .data.groups[] | .name as $group | .file as $folder | .rules[]
            | select(.state == "firing" or .state == "pending")
            | ([.alerts[]? | select(.state == "Alerting" or .state == "Pending")]) as $active
            | {uid, name, state, severity: (.labels.severity // null), group: $group, folder: $folder,
               query: (.query // null),
               since: (.activeAt // $active[0].activeAt // null),
               summary: ((.annotations.summary // .annotations.description // "")
                         | gsub("\\{\\{ *\\$value *\\}\\}"; ($active[0].value // "?" | num))),
               description: (if .annotations.summary then (.annotations.description // null) else null end),
               labelsets: [$active[] | .labels
                            | del(.__alert_rule_uid__, .__grafana_origin__, .alertname, .grafana_folder, .__name__, .severity)
                            | select(length > 0)],
               url: "\($pub)/alerting/grafana/\(.uid)/view"}
            | .instances = [.labelsets[] | to_entries | map("\(.key)=\(.value)") | join(" ")]
            | .targets = targets($guests)
        ] | sort_by(if .severity == "critical" then 0 elif .severity == "warning" then 1 else 2 end)}' <<<"$raw"
}

# ---------------------------------------------------------------- guests command

# devdesk guests [--json] [--cached] [FILTER...]: Proxmox guests with their node,
# state and IPs, fresh from the cluster (the cache if no node answers).
# FILTER matches an id, an IP, or part of a name or node.
cmd_guests() {
    local json="" cached="" filters=()
    while [ $# -gt 0 ]; do
        case $1 in
        --json) json=1 ;;
        --cached) cached=1 ;;
        --refresh)
            dd_pve_refresh
            return 0
            ;;
        -*) dd_die "usage: devdesk guests [--json] [--cached] [ID|IP|NAME...]" ;;
        *) filters+=("$1") ;;
        esac
        shift
    done
    [ -n "${DEVDESK_PVE_NODES:-}" ] || dd_die "no Proxmox nodes configured (DEVDESK_PVE_NODES in /etc/devdesk/env.local)"
    if [ -z "$cached" ] && ! dd_pve_refresh wait; then
        [ -s "$DEVDESK_GUESTS" ] || dd_die "no Proxmox node answered ($DEVDESK_PVE_NODES)"
        printf 'devdesk: no Proxmox node answered; showing what was known %s minutes ago\n' "$(($(dd_guests_age) / 60))" >&2
    fi
    [ -s "$DEVDESK_GUESTS" ] || dd_die "nothing known yet: run devdesk guests without --cached"
    local select='[.guests[] | . as $x | select(($f | length) == 0 or any($f[]; . as $q
        | ($x.id | tostring) == $q or ($x.ips | index([$q])) or ($x.name | ascii_downcase | contains($q | ascii_downcase))
          or ($x.node | ascii_downcase | contains($q | ascii_downcase))))]'
    local f
    f=$(printf '%s\n' "${filters[@]}" | jq -R . | jq -cs 'map(select(length > 0))')
    if [ -n "$json" ]; then
        jq -cM --argjson f "$f" "{updated, nodes, guests: $select}" "$DEVDESK_GUESTS"
        return
    fi
    jq -rM --argjson f "$f" "$select"' as $g
        | if ($g | length) == 0 then "no matching guests" else
          (["ID", "TYPE", "NAME", "NODE", "STATUS", "HA", "IP"] | @tsv),
          ($g[] | [(.id | tostring), (if .type == "qemu" then "vm" else "ct" end), .name, .node, .status, (.ha // "-"),
                   ((.ips | join(",")) + (if (.ips | length) > 0 and (.ips_live | not) then " (last seen)" else "" end) | if . == "" then "-" else . end)] | @tsv)
          end' "$DEVDESK_GUESTS" | column -t -s $'\t'
}

# ---------------------------------------------------------------- investigate

# The opening prompt for an investigation of alert $a (one element of the
# alerts output), with $since already readable.
DD_PROMPT_JQ='
def ref: if .kind == "ct" then "CT \(.id) \"\(.name)\"" elif .kind == "vm" then "VM \(.id) \"\(.name)\"" else "Proxmox node \(.name)" end;
def state: [(if .status then .status else empty end), (if .ha then "HA state \(.ha)" else empty end)] | join(", ");
def how:
  if .kind == "ct" then "commands inside it: `ssh \(.node) pct exec \(.id) -- <command>`"
  elif .kind == "vm" then "commands inside it (needs the guest agent): `ssh \(.node) qm guest exec \(.id) -- <command>`"
  else "reach it with `ssh \(.name)`" end;
def where:
  if .kind == "unknown" then "- \(.ip) is not a Proxmox guest or node that DevDesk knows (`devdesk guests` lists them all)."
  elif .kind == "node" then "- \(if .ip then "\(.ip) is " else "" end)\(ref)\(if .by == "name" then " (probably: matched by name)" else "" end), \(.status); \(how)"
  else "- \(if .by == "name" then "Probably " else "" end)\(if .ip then "\(.ip) is " else "" end)\(ref) on node \(.node)"
       + "\(if .by == "name" then " (matched by name, not by IP)" elif .ip_seen then " (its IP when last seen running)" else "" end), \(state); \(how)"
  end;
$a
| ([.targets[] | select(.kind == "ct" or .kind == "vm")] | first) as $guest
| [
  "Investigate this Grafana alert and find the cause. Look first and change nothing yet: tell me what you found and what you would change, then wait for my go-ahead.",
  "",
  "Alert: \(.name) (\(.severity // "no severity"), \(.state) since \($since))",
  "Rule: \(.folder) / \(.group), uid \(.uid). Look at it with the grafana MCP server: \(.url)",
  (if .query then "Query: `\(.query)`" else empty end),
  (if .summary != "" then "Summary: \(.summary)" else empty end),
  (if .description then "Details: \(.description)" else empty end),
  (if (.instances | length) > 0 then "Firing for: \(.instances | join("; "))"
   else "Firing for: no labels. The rule aggregates, so it does not say what it is about: run the inner query to find the series"
        + (if $pve then ", then `devdesk guests <IP>` shows which Proxmox guest each one is and where it runs." else "." end) end),
  (if $pve and (.targets | length) > 0 then
     ("", "Where it runs (Proxmox, looked up just now):"),
     (.targets[] | where),
     (if $guest then "Guests are HA and can migrate between nodes, so check again before acting: `devdesk guests \($guest.id)` shows where it is now." else empty end)
   else empty end)
  ] | join("\n")'

# --investigate SESSION RULE_UID: start Claude in a new session SESSION with an
# investigation of that alert as its first prompt. An existing session is left as
# it is: {"ok":true,"session":…,"started":false}.
cmd_investigate() {
    [ $# -eq 2 ] || dd_die "usage: devdesk attach --investigate SESSION ALERT_RULE_UID"
    local name=$1 uid=$2 alerts alert since prompt dir pve=false
    dd_valid_name "$name" || dd_die "invalid session name: $name"
    [[ $uid =~ ^[A-Za-z0-9_-]{1,64}$ ]] || dd_die "invalid alert rule id: $uid"
    if dd_server_up && dd_tmux has-session -t "=$name" 2>/dev/null; then
        jq -cMn --arg s "$name" '{ok: true, session: $s, started: false}'
        return
    fi
    [ -n "$DEVDESK_GRAFANA_URL" ] || dd_die "Grafana is not configured"
    # Where the guests are right now (HA moves them); the cache if no node answers.
    [ -z "${DEVDESK_PVE_NODES:-}" ] || dd_pve_refresh wait || true
    alerts=$(cmd_alerts)
    alert=$(jq -cM --arg uid "$uid" '.alerts[] | select(.uid == $uid)' <<<"$alerts" | head -n1)
    if [ -z "$alert" ]; then
        dd_die "$(jq -rM 'if .error then "Grafana unreachable: \(.error)" else "that alert is no longer firing" end' <<<"$alerts")"
    fi
    since=$(jq -rM '.since // empty' <<<"$alert")
    since=$(date -d "$since" '+%a %-d %b %H:%M %Z' 2>/dev/null || echo "${since:-an unknown time}")
    [ -s "$DEVDESK_GUESTS" ] && [ -n "${DEVDESK_PVE_NODES:-}" ] && pve=true
    prompt=$(jq -rnM --argjson a "$alert" --arg since "$since" --argjson pve "$pve" "$DD_PROMPT_JQ") || dd_die "could not write the prompt"
    # devdesk-claude-run sends this once, as the new conversation's first prompt.
    mkdir -p "$DEVDESK_STATE_DIR/prompts"
    (umask 077 && printf '%s\n' "$prompt" >"$DEVDESK_STATE_DIR/prompts/$name.txt")
    # A new conversation, even if an earlier investigation used this name (that
    # one stays in Search past conversations).
    dd_registry_update 'if .sessions[$n] then .sessions[$n] |= del(.claude_session_id, .claude_source, .fork) else . end' --arg n "$name"
    dir=$(dd_resolve_dir "$name")
    dd_ensure_server
    create_detached "$name" "$dir"
    jq -cMn --arg s "$name" '{ok: true, session: $s, started: true}'
}