deploy/livekit-graceful-restart.sh

@@ -1,18 +1,31 @@
 #!/bin/bash
 # Checks every 5 minutes if a livekit restart is pending.
-# Only restarts when there are no active WebSocket connections on port 7881
-# (established connections = active call participants).
+# Only restarts when LiveKit reports no participants AND nobody holds a
+# signalling connection, so a restart never drops a call.
 # Run by: livekit-graceful-restart.timer (systemd)
 
 if [ ! -f /run/livekit-restart-pending ]; then
     exit 0
 fi
 
-# Count established WebSocket signaling connections on livekit port 7881
-ACTIVE=$(ss -tn state established '( dport = :7881 or sport = :7881 )' | grep -c ESTAB || true)
+# Participants LiveKit itself counts, across all rooms. Counting TCP connections
+# on :7881 (the old check) only saw clients on the ICE-TCP fallback: signalling
+# is on :7880 and media is normally UDP, so most calls looked idle.
+METRICS_URL="${LIVEKIT_METRICS_URL:-http://127.0.0.1:6789/metrics}"
+if ! METRICS=$(curl -sf --max-time 5 "$METRICS_URL") \
+    || ! printf '%s\n' "$METRICS" | grep -q '^livekit_participant_total'; then
+    echo "$(date): Livekit restart pending but its metrics are unreadable — assuming a call is active, waiting."
+    exit 0
+fi
+PARTICIPANTS=$(printf '%s\n' "$METRICS" | awk '/^livekit_participant_total/ {s += $NF} END {printf "%d", s}')
 
-if [ "$ACTIVE" -gt 0 ]; then
-    echo "$(date): Livekit restart pending but $ACTIVE active connection(s) — waiting."
+# Belt and braces: open signalling WebSockets (a client mid-join may not be a
+# participant yet). Count lines (-H drops the header): with a `state` filter ss
+# omits the State column, so the old `grep -c ESTAB` was always 0.
+SIGNALLING=$(ss -Htn state established '( sport = :7880 )' | wc -l)
+
+if [ "$PARTICIPANTS" -gt 0 ] || [ "$SIGNALLING" -gt 0 ]; then
+    echo "$(date): Livekit restart pending but $PARTICIPANTS participant(s), $SIGNALLING signalling connection(s) — waiting."
     exit 0
 fi
 

deploy/lxc151-hookshot.sh

@@ -47,6 +47,13 @@ else
         echo "Restart pending — will apply when no active calls."
     fi
 
+    # The graceful-restart check itself (run by livekit-graceful-restart.timer)
+    if echo "$CHANGED" | grep -q '^deploy/livekit-graceful-restart\.sh'; then
+        echo "Deploying livekit-graceful-restart.sh..."
+        bash -n "$REPO_DIR/deploy/livekit-graceful-restart.sh" \
+            && install -m755 "$REPO_DIR/deploy/livekit-graceful-restart.sh" /usr/local/bin/livekit-graceful-restart.sh
+    fi
+
     # Voice-limit / quality guard (fronts lk-jwt-service on :8070)
     if echo "$CHANGED" | grep -qE '^livekit/voice-limit-guard\.py|^systemd/voice-limit-guard\.service'; then
         echo "Deploying voice-limit-guard..."