fix(livekit): graceful restart could restart LiveKit in the middle of a call

livekit-graceful-restart.sh decided "no active calls" by counting established TCP connections on :7881 with `ss ... state established | grep -c ESTAB`. Two bugs: - With a `state` filter ss omits the State column, so `grep -c ESTAB` is always 0: the check never saw anything. - :7881 is only LiveKit's ICE-TCP media fallback. Signalling is on :7880 and media is normally UDP. Reproduced in a two-person call on the local stack: the old check counted 0 and would have restarted. Now it waits while LiveKit's own livekit_participant_total is above 0 or any signalling WebSocket on :7880 is open, and also waits if the metrics can't be read. The deploy hook now installs the script when it changes (it never did, so edits to it never reached the server). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_017PCb59uY3UKqDqt7B6hzd5
2026-11-03 00:25:58 -05:00
parent d32c74052c
commit d9f023762e
2 changed files with 26 additions and 6 deletions
@@ -1,18 +1,31 @@
 #!/bin/bash
 # Checks every 5 minutes if a livekit restart is pending.
-# Only restarts when there are no active WebSocket connections on port 7881
-# (established connections = active call participants).
+# Only restarts when LiveKit reports no participants AND nobody holds a
+# signalling connection, so a restart never drops a call.
 # Run by: livekit-graceful-restart.timer (systemd)

 if [ ! -f /run/livekit-restart-pending ]; then
    exit 0
 fi

-# Count established WebSocket signaling connections on livekit port 7881
-ACTIVE=$(ss -tn state established '( dport = :7881 or sport = :7881 )' | grep -c ESTAB || true)
+# Participants LiveKit itself counts, across all rooms. Counting TCP connections
+# on :7881 (the old check) only saw clients on the ICE-TCP fallback: signalling
+# is on :7880 and media is normally UDP, so most calls looked idle.
+METRICS_URL="${LIVEKIT_METRICS_URL:-http://127.0.0.1:6789/metrics}"
+if ! METRICS=$(curl -sf --max-time 5 "$METRICS_URL") \
+    || ! printf '%s\n' "$METRICS" | grep -q '^livekit_participant_total'; then
+    echo "$(date): Livekit restart pending but its metrics are unreadable — assuming a call is active, waiting."
+    exit 0
+fi
+PARTICIPANTS=$(printf '%s\n' "$METRICS" | awk '/^livekit_participant_total/ {s += $NF} END {printf "%d", s}')

-if [ "$ACTIVE" -gt 0 ]; then
-    echo "$(date): Livekit restart pending but $ACTIVE active connection(s) — waiting."
+# Belt and braces: open signalling WebSockets (a client mid-join may not be a
+# participant yet). Count lines (-H drops the header): with a `state` filter ss
+# omits the State column, so the old `grep -c ESTAB` was always 0.
+SIGNALLING=$(ss -Htn state established '( sport = :7880 )' | wc -l)
+
+if [ "$PARTICIPANTS" -gt 0 ] || [ "$SIGNALLING" -gt 0 ]; then
+    echo "$(date): Livekit restart pending but $PARTICIPANTS participant(s), $SIGNALLING signalling connection(s) — waiting."
    exit 0
 fi

@@ -47,6 +47,13 @@ else
        echo "Restart pending — will apply when no active calls."
    fi

+    # The graceful-restart check itself (run by livekit-graceful-restart.timer)
+    if echo "$CHANGED" | grep -q '^deploy/livekit-graceful-restart\.sh'; then
+        echo "Deploying livekit-graceful-restart.sh..."
+        bash -n "$REPO_DIR/deploy/livekit-graceful-restart.sh" \
+            && install -m755 "$REPO_DIR/deploy/livekit-graceful-restart.sh" /usr/local/bin/livekit-graceful-restart.sh
+    fi
+
    # Voice-limit / quality guard (fronts lk-jwt-service on :8070)
    if echo "$CHANGED" | grep -qE '^livekit/voice-limit-guard\.py|^systemd/voice-limit-guard\.service'; then
        echo "Deploying voice-limit-guard..."