diff --git a/deploy/livekit-graceful-restart.sh b/deploy/livekit-graceful-restart.sh index 8e6a6d0..7732e0a 100644 --- a/deploy/livekit-graceful-restart.sh +++ b/deploy/livekit-graceful-restart.sh @@ -1,18 +1,31 @@ #!/bin/bash # Checks every 5 minutes if a livekit restart is pending. -# Only restarts when there are no active WebSocket connections on port 7881 -# (established connections = active call participants). +# Only restarts when LiveKit reports no participants AND nobody holds a +# signalling connection, so a restart never drops a call. # Run by: livekit-graceful-restart.timer (systemd) if [ ! -f /run/livekit-restart-pending ]; then exit 0 fi -# Count established WebSocket signaling connections on livekit port 7881 -ACTIVE=$(ss -tn state established '( dport = :7881 or sport = :7881 )' | grep -c ESTAB || true) +# Participants LiveKit itself counts, across all rooms. Counting TCP connections +# on :7881 (the old check) only saw clients on the ICE-TCP fallback: signalling +# is on :7880 and media is normally UDP, so most calls looked idle. +METRICS_URL="${LIVEKIT_METRICS_URL:-http://127.0.0.1:6789/metrics}" +if ! METRICS=$(curl -sf --max-time 5 "$METRICS_URL") \ + || ! printf '%s\n' "$METRICS" | grep -q '^livekit_participant_total'; then + echo "$(date): Livekit restart pending but its metrics are unreadable — assuming a call is active, waiting." + exit 0 +fi +PARTICIPANTS=$(printf '%s\n' "$METRICS" | awk '/^livekit_participant_total/ {s += $NF} END {printf "%d", s}') -if [ "$ACTIVE" -gt 0 ]; then - echo "$(date): Livekit restart pending but $ACTIVE active connection(s) — waiting." +# Belt and braces: open signalling WebSockets (a client mid-join may not be a +# participant yet). Count lines (-H drops the header): with a `state` filter ss +# omits the State column, so the old `grep -c ESTAB` was always 0. +SIGNALLING=$(ss -Htn state established '( sport = :7880 )' | wc -l) + +if [ "$PARTICIPANTS" -gt 0 ] || [ "$SIGNALLING" -gt 0 ]; then + echo "$(date): Livekit restart pending but $PARTICIPANTS participant(s), $SIGNALLING signalling connection(s) — waiting." exit 0 fi diff --git a/deploy/lxc151-hookshot.sh b/deploy/lxc151-hookshot.sh index b03c6cf..90d065a 100644 --- a/deploy/lxc151-hookshot.sh +++ b/deploy/lxc151-hookshot.sh @@ -47,6 +47,13 @@ else echo "Restart pending — will apply when no active calls." fi + # The graceful-restart check itself (run by livekit-graceful-restart.timer) + if echo "$CHANGED" | grep -q '^deploy/livekit-graceful-restart\.sh'; then + echo "Deploying livekit-graceful-restart.sh..." + bash -n "$REPO_DIR/deploy/livekit-graceful-restart.sh" \ + && install -m755 "$REPO_DIR/deploy/livekit-graceful-restart.sh" /usr/local/bin/livekit-graceful-restart.sh + fi + # Voice-limit / quality guard (fronts lk-jwt-service on :8070) if echo "$CHANGED" | grep -qE '^livekit/voice-limit-guard\.py|^systemd/voice-limit-guard\.service'; then echo "Deploying voice-limit-guard..."