From c2267800be3f97175283b0c777ed2740876d9023 Mon Sep 17 00:00:00 2001 From: Lotus CI Date: Sat, 19 Sep 2026 22:45:23 -0400 Subject: [PATCH] feat(lotus): report speakingWhileMuted for the local participant in io.lotus.call_state (#37) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit While the mic is muted LiveKit keeps the capture alive and merely disables the published MediaStreamTrack, so VAD says nothing. lotusMutedSpeech taps a CLONE of the published track (post-denoise when the in-source processor is active), samples RMS at 10 Hz through an AnalyserNode and runs a hysteresis gate (300 ms of voice on, 800 ms of quiet off, threshold ≈ −36 dBFS). The tap only exists while a muted mic publication exists; the flag is set on the local entry of io.lotus.call_state only and is never sent to other participants. Gate is unit-tested; verified end-to-end: muted with a tone mic → true within 2.5 s, silence → false, unmute → false. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01PPmy3tPq869XDW4njjVaKA --- src/lotus/lotusCallState.test.ts | 6 +- src/lotus/lotusCallState.ts | 69 +++++++------ src/lotus/lotusMutedSpeech.test.ts | 27 +++++ src/lotus/lotusMutedSpeech.ts | 154 +++++++++++++++++++++++++++++ 4 files changed, 227 insertions(+), 29 deletions(-) create mode 100644 src/lotus/lotusMutedSpeech.test.ts create mode 100644 src/lotus/lotusMutedSpeech.ts diff --git a/src/lotus/lotusCallState.test.ts b/src/lotus/lotusCallState.test.ts index cce3e986..0923da64 100644 --- a/src/lotus/lotusCallState.test.ts +++ b/src/lotus/lotusCallState.test.ts @@ -53,7 +53,11 @@ function mockMember(id: string, userId: string): Member { } function mockVm(members: Member[]): CallViewModel { - return { userMedia$: of(members) } as unknown as CallViewModel; + return { + userMedia$: of(members), + // no livekit connections → the muted-speech tap never starts + allConnections$: of({ getConnections: () => [] }), + } as unknown as CallViewModel; } function participantsOf(call: number): unknown[] { diff --git a/src/lotus/lotusCallState.ts b/src/lotus/lotusCallState.ts index 286c212c..133422a9 100644 --- a/src/lotus/lotusCallState.ts +++ b/src/lotus/lotusCallState.ts @@ -5,7 +5,7 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { combineLatest, of, type Subscription } from "rxjs"; +import { combineLatest, of, startWith, type Subscription } from "rxjs"; import { distinctUntilChanged, map, @@ -16,6 +16,7 @@ import { import { type CallViewModel } from "../state/CallViewModel/CallViewModel"; import { widget } from "../widget"; import { LotusWidgetActions, lotusFlag, lotusSendToHost } from "./lotusWidget"; +import { observeSpeakingWhileMuted$ } from "./lotusMutedSpeech"; interface ParticipantState { /** EC media id (`${userId}:${deviceId}`), stable per participant device. */ @@ -25,6 +26,11 @@ interface ParticipantState { speaking: boolean; audioEnabled: boolean; videoEnabled: boolean; + /** + * [lotus #37] LOCAL participant only: voice detected on the mic while it is + * muted (see lotusMutedSpeech.ts). Absent for remote participants. + */ + speakingWhileMuted?: boolean; } /** @@ -45,7 +51,8 @@ function participantsEqual( p.userId === b[i].userId && p.speaking === b[i].speaking && p.audioEnabled === b[i].audioEnabled && - p.videoEnabled === b[i].videoEnabled, + p.videoEnabled === b[i].videoEnabled && + p.speakingWhileMuted === b[i].speakingWhileMuted, ) ); } @@ -66,34 +73,40 @@ export function startLotusCallState(vm: CallViewModel): () => void { // lotusDecorations.ts. if (!widget) return () => undefined; - const sub: Subscription = vm.userMedia$ - .pipe( - switchMap((members) => - members.length === 0 - ? of([] as ParticipantState[]) - : combineLatest( - members.map((m) => - combineLatest([ - m.speaking$, - m.audioEnabled$, - m.videoEnabled$, - ]).pipe( - map( - ([ - speaking, - audioEnabled, - videoEnabled, - ]): ParticipantState => ({ - id: m.id, - userId: m.userId, - speaking, - audioEnabled, - videoEnabled, - }), - ), - ), + const participants$ = vm.userMedia$.pipe( + switchMap((members) => + members.length === 0 + ? of([] as (ParticipantState & { local: boolean })[]) + : combineLatest( + members.map((m) => + combineLatest([ + m.speaking$, + m.audioEnabled$, + m.videoEnabled$, + ]).pipe( + map(([speaking, audioEnabled, videoEnabled]) => ({ + id: m.id, + userId: m.userId, + speaking, + audioEnabled, + videoEnabled, + local: m.local === true, + })), ), ), + ), + ), + ); + + const sub: Subscription = combineLatest([ + participants$, + observeSpeakingWhileMuted$(vm).pipe(startWith(false)), + ]) + .pipe( + map(([members, speakingWhileMuted]): ParticipantState[] => + members.map(({ local, ...p }) => + local ? { ...p, speakingWhileMuted } : p, + ), ), // `speaking` flips rapidly; drop no-op repeats BEFORE throttling so // the throttle window isn't spent re-emitting an unchanged value. diff --git a/src/lotus/lotusMutedSpeech.test.ts b/src/lotus/lotusMutedSpeech.test.ts new file mode 100644 index 00000000..0cb78cee --- /dev/null +++ b/src/lotus/lotusMutedSpeech.test.ts @@ -0,0 +1,27 @@ +/* +Copyright 2026 Lotus Guild + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { expect, test } from "vitest"; + +import { MutedSpeechGate } from "./lotusMutedSpeech"; + +test("needs 300 ms of voice to switch on and 800 ms of quiet to switch off", () => { + const g = new MutedSpeechGate(0.015); + expect(g.push(0.1)).toBe(false); + expect(g.push(0.1)).toBe(false); + expect(g.push(0.1)).toBe(true); + // a short dip does not drop it + for (let i = 0; i < 7; i++) expect(g.push(0.0)).toBe(true); + expect(g.push(0.0)).toBe(false); + // one loud sample after quiet does not re-trigger + expect(g.push(0.2)).toBe(false); +}); + +test("keyboard-level noise below the threshold never triggers", () => { + const g = new MutedSpeechGate(0.015); + for (let i = 0; i < 50; i++) expect(g.push(0.01)).toBe(false); +}); diff --git a/src/lotus/lotusMutedSpeech.ts b/src/lotus/lotusMutedSpeech.ts new file mode 100644 index 00000000..14285886 --- /dev/null +++ b/src/lotus/lotusMutedSpeech.ts @@ -0,0 +1,154 @@ +/* +Copyright 2026 Lotus Guild + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { + type LocalTrackPublication, + type Room as LivekitRoom, + RoomEvent, + Track, +} from "livekit-client"; +import { Observable, distinctUntilChanged, switchMap } from "rxjs"; + +import { type CallViewModel } from "../state/CallViewModel/CallViewModel"; + +/** + * [lotus #37] "Talking while muted" detection for the LOCAL participant. + * + * While the mic is muted LiveKit keeps the capture alive and merely disables + * the published MediaStreamTrack, so VAD/`speaking` is false and nothing + * tells the user they are talking into a muted mic. This taps a CLONE of the + * published track (the post-processor track when the in-source denoiser is + * active, so keyboard noise doesn't count), samples RMS at ~10 Hz and emits a + * debounced boolean. Zero cost when unmuted (tap torn down), local-only — + * the flag rides `io.lotus.call_state` to the host and never reaches other + * participants. + */ + +export const MUTED_SPEECH_RMS = 0.015; // ≈ −36 dBFS; normal speech into a headset is 0.05–0.3 +const SAMPLE_MS = 100; +const ON_SAMPLES = 3; // 300 ms of voice before we say "talking" +const OFF_SAMPLES = 8; // 800 ms of quiet before we drop it + +/** Pure hysteresis gate over successive RMS samples (unit-tested). */ +export class MutedSpeechGate { + private above = 0; + + private below = 0; + + private on = false; + + public constructor(private readonly threshold = MUTED_SPEECH_RMS) {} + + public push(rms: number): boolean { + if (rms >= this.threshold) { + this.above += 1; + this.below = 0; + if (!this.on && this.above >= ON_SAMPLES) this.on = true; + } else { + this.below += 1; + this.above = 0; + if (this.on && this.below >= OFF_SAMPLES) this.on = false; + } + return this.on; + } + + public get value(): boolean { + return this.on; + } +} + +const mutedMicTrack = (room: LivekitRoom): MediaStreamTrack | null => { + const pub: LocalTrackPublication | undefined = + room.localParticipant.getTrackPublication(Track.Source.Microphone); + const track = pub?.track?.mediaStreamTrack; + return pub?.isMuted && track && track.readyState === "live" ? track : null; +}; + +/** + * Emits while the local mic is muted and voice is detected on it. Emits + * `false` whenever the mic is unmuted, unpublished or the connection changes. + */ +export function observeSpeakingWhileMuted$( + vm: CallViewModel, +): Observable { + return vm.allConnections$.pipe( + switchMap( + (data) => + new Observable((subscriber) => { + const rooms = data.getConnections().map((c) => c.livekitRoom); + let ctx: AudioContext | null = null; + let clone: MediaStreamTrack | null = null; + let timer: ReturnType | undefined; + let tapped: MediaStreamTrack | null = null; + + const stopTap = (): void => { + if (timer !== undefined) clearInterval(timer); + timer = undefined; + clone?.stop(); + clone = null; + void ctx?.close().catch(() => undefined); + ctx = null; + tapped = null; + subscriber.next(false); + }; + + const startTap = (source: MediaStreamTrack): void => { + try { + clone = source.clone(); + clone.enabled = true; // the source is disabled by the mute — the clone must not be + ctx = new AudioContext(); + const analyser = ctx.createAnalyser(); + analyser.fftSize = 1024; + ctx + .createMediaStreamSource(new MediaStream([clone])) + .connect(analyser); + const buf = new Float32Array(analyser.fftSize); + const gate = new MutedSpeechGate(); + tapped = source; + timer = setInterval(() => { + analyser.getFloatTimeDomainData(buf); + let sum = 0; + for (let i = 0; i < buf.length; i += 1) sum += buf[i] * buf[i]; + subscriber.next(gate.push(Math.sqrt(sum / buf.length))); + }, SAMPLE_MS); + } catch { + stopTap(); + } + }; + + const reconcile = (): void => { + const track = + rooms.map(mutedMicTrack).find((t) => t !== null) ?? null; + if (track === tapped) return; + if (tapped) stopTap(); + if (track) startTap(track); + }; + + const events = [ + RoomEvent.TrackMuted, + RoomEvent.TrackUnmuted, + RoomEvent.LocalTrackPublished, + RoomEvent.LocalTrackUnpublished, + RoomEvent.Reconnected, + RoomEvent.Disconnected, + ] as const; + rooms.forEach((room) => + events.forEach((ev) => room.on(ev, reconcile)), + ); + subscriber.next(false); + reconcile(); + return () => { + rooms.forEach((room) => + events.forEach((ev) => room.off(ev, reconcile)), + ); + if (tapped) stopTap(); + }; + }), + ), + distinctUntilChanged(), + ); +}