From d9f023762e9721693122146c164a6164335cd3b8 Mon Sep 17 00:00:00 2001
From: Lotus CI <ci@lotusguild.org>
Date: Tue, 3 Nov 2026 00:25:58 -0500
Subject: [PATCH] fix(livekit): graceful restart could restart LiveKit in the
 middle of a call

livekit-graceful-restart.sh decided "no active calls" by counting
established TCP connections on :7881 with `ss ... state established |
grep -c ESTAB`. Two bugs:

- With a `state` filter ss omits the State column, so `grep -c ESTAB`
  is always 0: the check never saw anything.
- :7881 is only LiveKit's ICE-TCP media fallback. Signalling is on
  :7880 and media is normally UDP.

Reproduced in a two-person call on the local stack: the old check
counted 0 and would have restarted. Now it waits while LiveKit's own
livekit_participant_total is above 0 or any signalling WebSocket on
:7880 is open, and also waits if the metrics can't be read.

The deploy hook now installs the script when it changes (it never
did, so edits to it never reached the server).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_017PCb59uY3UKqDqt7B6hzd5
---
 deploy/livekit-graceful-restart.sh | 25 +++++++++++++++++++------
 deploy/lxc151-hookshot.sh          |  7 +++++++
 2 files changed, 26 insertions(+), 6 deletions(-)

diff --git a/deploy/livekit-graceful-restart.sh b/deploy/livekit-graceful-restart.sh
index 8e6a6d0..7732e0a 100644
--- a/deploy/livekit-graceful-restart.sh
+++ b/deploy/livekit-graceful-restart.sh
@@ -1,18 +1,31 @@
 #!/bin/bash
 # Checks every 5 minutes if a livekit restart is pending.
-# Only restarts when there are no active WebSocket connections on port 7881
-# (established connections = active call participants).
+# Only restarts when LiveKit reports no participants AND nobody holds a
+# signalling connection, so a restart never drops a call.
 # Run by: livekit-graceful-restart.timer (systemd)
 
 if [ ! -f /run/livekit-restart-pending ]; then
     exit 0
 fi
 
-# Count established WebSocket signaling connections on livekit port 7881
-ACTIVE=$(ss -tn state established '( dport = :7881 or sport = :7881 )' | grep -c ESTAB || true)
+# Participants LiveKit itself counts, across all rooms. Counting TCP connections
+# on :7881 (the old check) only saw clients on the ICE-TCP fallback: signalling
+# is on :7880 and media is normally UDP, so most calls looked idle.
+METRICS_URL="${LIVEKIT_METRICS_URL:-http://127.0.0.1:6789/metrics}"
+if ! METRICS=$(curl -sf --max-time 5 "$METRICS_URL") \
+    || ! printf '%s\n' "$METRICS" | grep -q '^livekit_participant_total'; then
+    echo "$(date): Livekit restart pending but its metrics are unreadable — assuming a call is active, waiting."
+    exit 0
+fi
+PARTICIPANTS=$(printf '%s\n' "$METRICS" | awk '/^livekit_participant_total/ {s += $NF} END {printf "%d", s}')
 
-if [ "$ACTIVE" -gt 0 ]; then
-    echo "$(date): Livekit restart pending but $ACTIVE active connection(s) — waiting."
+# Belt and braces: open signalling WebSockets (a client mid-join may not be a
+# participant yet). Count lines (-H drops the header): with a `state` filter ss
+# omits the State column, so the old `grep -c ESTAB` was always 0.
+SIGNALLING=$(ss -Htn state established '( sport = :7880 )' | wc -l)
+
+if [ "$PARTICIPANTS" -gt 0 ] || [ "$SIGNALLING" -gt 0 ]; then
+    echo "$(date): Livekit restart pending but $PARTICIPANTS participant(s), $SIGNALLING signalling connection(s) — waiting."
     exit 0
 fi
 
diff --git a/deploy/lxc151-hookshot.sh b/deploy/lxc151-hookshot.sh
index b03c6cf..90d065a 100644
--- a/deploy/lxc151-hookshot.sh
+++ b/deploy/lxc151-hookshot.sh
@@ -47,6 +47,13 @@ else
         echo "Restart pending — will apply when no active calls."
     fi
 
+    # The graceful-restart check itself (run by livekit-graceful-restart.timer)
+    if echo "$CHANGED" | grep -q '^deploy/livekit-graceful-restart\.sh'; then
+        echo "Deploying livekit-graceful-restart.sh..."
+        bash -n "$REPO_DIR/deploy/livekit-graceful-restart.sh" \
+            && install -m755 "$REPO_DIR/deploy/livekit-graceful-restart.sh" /usr/local/bin/livekit-graceful-restart.sh
+    fi
+
     # Voice-limit / quality guard (fronts lk-jwt-service on :8070)
     if echo "$CHANGED" | grep -qE '^livekit/voice-limit-guard\.py|^systemd/voice-limit-guard\.service'; then
         echo "Deploying voice-limit-guard..."
-- 
2.47.3