diff --git a/src/lotus/lotusActions.test.ts b/src/lotus/lotusActions.test.ts index d4b9401e..e1b8a9a9 100644 --- a/src/lotus/lotusActions.test.ts +++ b/src/lotus/lotusActions.test.ts @@ -50,5 +50,6 @@ describe("LotusWidgetActions", () => { expect(LOTUS_TO_WIDGET_ACTIONS).not.toContain( LotusWidgetActions.ControlsState, ); + expect(LOTUS_TO_WIDGET_ACTIONS).not.toContain(LotusWidgetActions.MicLevel); }); }); diff --git a/src/lotus/lotusActions.ts b/src/lotus/lotusActions.ts index 1ffed38f..67e31bdc 100644 --- a/src/lotus/lotusActions.ts +++ b/src/lotus/lotusActions.ts @@ -80,6 +80,11 @@ export enum LotusWidgetActions { * them. Its arrival also tells the host this fork supports the actions above. */ ControlsState = "io.lotus.controls_state", + /** + * fromWidget: local mic level `{ bars: 0 | 1 | 2 | 3 }` (cinny #146), sent + * only when it changes (≤ 10 Hz); 0 while muted or with no mic. + */ + MicLevel = "io.lotus.mic_level", } /** toWidget Lotus actions that must be allow-listed in `initializeWidget`. */ diff --git a/src/lotus/lotusMicLevel.test.ts b/src/lotus/lotusMicLevel.test.ts new file mode 100644 index 00000000..239ad608 --- /dev/null +++ b/src/lotus/lotusMicLevel.test.ts @@ -0,0 +1,37 @@ +/* +Copyright 2026 Lotus Guild + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { describe, expect, it } from "vitest"; + +import { MicLevelQuantizer } from "./lotusMicLevel"; + +describe("MicLevelQuantizer", () => { + it("maps RMS to 0–3 bars", () => { + expect(new MicLevelQuantizer().push(0.001)).toBe(0); + expect(new MicLevelQuantizer().push(0.008)).toBe(1); + expect(new MicLevelQuantizer().push(0.02)).toBe(2); + expect(new MicLevelQuantizer().push(0.2)).toBe(3); + }); + + it("rises at once and falls one bar per sample", () => { + const q = new MicLevelQuantizer(); + expect(q.push(0.2)).toBe(3); + expect(q.push(0)).toBe(2); + expect(q.push(0)).toBe(1); + expect(q.push(0.02)).toBe(2); + expect(q.push(0)).toBe(1); + expect(q.push(0)).toBe(0); + expect(q.push(0)).toBe(0); + }); + + it("reset drops straight to 0", () => { + const q = new MicLevelQuantizer(); + q.push(0.2); + q.reset(); + expect(q.push(0)).toBe(0); + }); +}); diff --git a/src/lotus/lotusMicLevel.ts b/src/lotus/lotusMicLevel.ts new file mode 100644 index 00000000..6fc2f298 --- /dev/null +++ b/src/lotus/lotusMicLevel.ts @@ -0,0 +1,192 @@ +/* +Copyright 2026 Lotus Guild + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { + type LocalTrackPublication, + type Room as LivekitRoom, + RoomEvent, + Track, +} from "livekit-client"; +import { Observable, type Subscription, share, switchMap } from "rxjs"; +import { distinctUntilChanged, map } from "rxjs/operators"; + +import { type CallViewModel } from "../state/CallViewModel/CallViewModel"; +import { widget } from "../widget"; +import { LotusWidgetActions, lotusFlag, lotusSendToHost } from "./lotusWidget"; + +/** + * [lotus #146] One local mic sampler shared by the host's mic level meter + * (`io.lotus.mic_level`) and "talking while muted" (#37, lotusMutedSpeech). + * It taps a CLONE of the published mic track (the post-processor track when + * the in-source denoiser is active, so what's measured is what's sent) and + * reads RMS at ~10 Hz while a mic track is published, muted or not. + * Local only: nothing here reaches other participants. + */ + +const SAMPLE_MS = 100; + +export interface LocalMicSample { + rms: number; + /** The mic is published but muted (the clone still hears it). */ + muted: boolean; +} + +const micPublication = ( + room: LivekitRoom, +): { track: MediaStreamTrack; muted: boolean } | null => { + const pub: LocalTrackPublication | undefined = + room.localParticipant.getTrackPublication(Track.Source.Microphone); + const track = pub?.track?.mediaStreamTrack; + return track && track.readyState === "live" + ? { track, muted: pub?.isMuted ?? false } + : null; +}; + +const samplers = new WeakMap< + CallViewModel, + Observable +>(); + +/** + * RMS samples of the local mic, or `null` while no mic track is published. + * Shared per call view model, so the meter and the muted-speech detector use + * one AudioContext between them. + */ +export function observeLocalMicSample$( + vm: CallViewModel, +): Observable { + const cached = samplers.get(vm); + if (cached) return cached; + const sampler$ = vm.allConnections$.pipe( + switchMap( + (data) => + new Observable((subscriber) => { + const rooms = data.getConnections().map((c) => c.livekitRoom); + let ctx: AudioContext | null = null; + let clone: MediaStreamTrack | null = null; + let timer: ReturnType | undefined; + let tapped: MediaStreamTrack | null = null; + let muted = false; + + const stopTap = (): void => { + if (timer !== undefined) clearInterval(timer); + timer = undefined; + clone?.stop(); + clone = null; + void ctx?.close().catch(() => undefined); + ctx = null; + tapped = null; + subscriber.next(null); + }; + + const startTap = (source: MediaStreamTrack): void => { + try { + clone = source.clone(); + // Muting disables the published track; the clone must still hear. + clone.enabled = true; + ctx = new AudioContext(); + const analyser = ctx.createAnalyser(); + analyser.fftSize = 1024; + ctx + .createMediaStreamSource(new MediaStream([clone])) + .connect(analyser); + const buf = new Float32Array(analyser.fftSize); + tapped = source; + timer = setInterval(() => { + analyser.getFloatTimeDomainData(buf); + let sum = 0; + for (let i = 0; i < buf.length; i += 1) sum += buf[i] * buf[i]; + subscriber.next({ rms: Math.sqrt(sum / buf.length), muted }); + }, SAMPLE_MS); + } catch { + stopTap(); + } + }; + + const reconcile = (): void => { + const pub = + rooms.map(micPublication).find((p) => p !== null) ?? null; + muted = pub?.muted ?? false; + const track = pub?.track ?? null; + if (track === tapped) return; + if (tapped) stopTap(); + if (track) startTap(track); + }; + + const events = [ + RoomEvent.TrackMuted, + RoomEvent.TrackUnmuted, + RoomEvent.LocalTrackPublished, + RoomEvent.LocalTrackUnpublished, + RoomEvent.Reconnected, + RoomEvent.Disconnected, + ] as const; + rooms.forEach((room) => + events.forEach((ev) => room.on(ev, reconcile)), + ); + subscriber.next(null); + reconcile(); + return (): void => { + rooms.forEach((room) => + events.forEach((ev) => room.off(ev, reconcile)), + ); + if (tapped) stopTap(); + }; + }), + ), + share(), + ); + samplers.set(vm, sampler$); + return sampler$; +} + +/** RMS at which each bar lights: ≈ −46, −36 and −26 dBFS. */ +export const BAR_THRESHOLDS = [0.005, 0.015, 0.05] as const; + +/** + * Quantise RMS to 0–3 bars with a little hysteresis: rises at once, falls one + * bar per sample, so the meter doesn't flicker between words. Unit-tested. + */ +export class MicLevelQuantizer { + private bars = 0; + + public reset(): void { + this.bars = 0; + } + + public push(rms: number): number { + const target = BAR_THRESHOLDS.filter((t) => rms >= t).length; + this.bars = target >= this.bars ? target : this.bars - 1; + return this.bars; + } +} + +/** + * Send the host `io.lotus.mic_level { bars }` (0–3) whenever the quantised + * level changes; 0 while muted or with no mic. At most one message per sample + * (10 Hz) and none during steady silence. Opt-in with the rest of the host + * state stream (`lotusCallState=1`). Returns a teardown function. + */ +export function startLotusMicLevel(vm: CallViewModel): () => void { + if (!lotusFlag("lotusCallState") || !widget) return (): void => undefined; + const quantizer = new MicLevelQuantizer(); + const sub: Subscription = observeLocalMicSample$(vm) + .pipe( + map((sample) => { + if (!sample || sample.muted) { + quantizer.reset(); + return 0; + } + return quantizer.push(sample.rms); + }), + distinctUntilChanged(), + ) + .subscribe((bars) => { + lotusSendToHost(LotusWidgetActions.MicLevel, { bars }); + }); + return (): void => sub.unsubscribe(); +} diff --git a/src/lotus/lotusMutedSpeech.ts b/src/lotus/lotusMutedSpeech.ts index 14285886..938ba648 100644 --- a/src/lotus/lotusMutedSpeech.ts +++ b/src/lotus/lotusMutedSpeech.ts @@ -6,14 +6,15 @@ Please see LICENSE in the repository root for full details. */ import { - type LocalTrackPublication, - type Room as LivekitRoom, - RoomEvent, - Track, -} from "livekit-client"; -import { Observable, distinctUntilChanged, switchMap } from "rxjs"; + type Observable, + distinctUntilChanged, + map, + scan, + startWith, +} from "rxjs"; import { type CallViewModel } from "../state/CallViewModel/CallViewModel"; +import { type LocalMicSample, observeLocalMicSample$ } from "./lotusMicLevel"; /** * [lotus #37] "Talking while muted" detection for the LOCAL participant. @@ -23,13 +24,12 @@ import { type CallViewModel } from "../state/CallViewModel/CallViewModel"; * tells the user they are talking into a muted mic. This taps a CLONE of the * published track (the post-processor track when the in-source denoiser is * active, so keyboard noise doesn't count), samples RMS at ~10 Hz and emits a - * debounced boolean. Zero cost when unmuted (tap torn down), local-only — + * debounced boolean. Local-only — * the flag rides `io.lotus.call_state` to the host and never reaches other * participants. */ export const MUTED_SPEECH_RMS = 0.015; // ≈ −36 dBFS; normal speech into a headset is 0.05–0.3 -const SAMPLE_MS = 100; const ON_SAMPLES = 3; // 300 ms of voice before we say "talking" const OFF_SAMPLES = 8; // 800 ms of quiet before we drop it @@ -61,94 +61,24 @@ export class MutedSpeechGate { } } -const mutedMicTrack = (room: LivekitRoom): MediaStreamTrack | null => { - const pub: LocalTrackPublication | undefined = - room.localParticipant.getTrackPublication(Track.Source.Microphone); - const track = pub?.track?.mediaStreamTrack; - return pub?.isMuted && track && track.readyState === "live" ? track : null; -}; - /** * Emits while the local mic is muted and voice is detected on it. Emits * `false` whenever the mic is unmuted, unpublished or the connection changes. + * [lotus #146] Fed by the shared local mic sampler (lotusMicLevel.ts), which + * also drives the host's mic level meter while unmuted. */ export function observeSpeakingWhileMuted$( vm: CallViewModel, ): Observable { - return vm.allConnections$.pipe( - switchMap( - (data) => - new Observable((subscriber) => { - const rooms = data.getConnections().map((c) => c.livekitRoom); - let ctx: AudioContext | null = null; - let clone: MediaStreamTrack | null = null; - let timer: ReturnType | undefined; - let tapped: MediaStreamTrack | null = null; - - const stopTap = (): void => { - if (timer !== undefined) clearInterval(timer); - timer = undefined; - clone?.stop(); - clone = null; - void ctx?.close().catch(() => undefined); - ctx = null; - tapped = null; - subscriber.next(false); - }; - - const startTap = (source: MediaStreamTrack): void => { - try { - clone = source.clone(); - clone.enabled = true; // the source is disabled by the mute — the clone must not be - ctx = new AudioContext(); - const analyser = ctx.createAnalyser(); - analyser.fftSize = 1024; - ctx - .createMediaStreamSource(new MediaStream([clone])) - .connect(analyser); - const buf = new Float32Array(analyser.fftSize); - const gate = new MutedSpeechGate(); - tapped = source; - timer = setInterval(() => { - analyser.getFloatTimeDomainData(buf); - let sum = 0; - for (let i = 0; i < buf.length; i += 1) sum += buf[i] * buf[i]; - subscriber.next(gate.push(Math.sqrt(sum / buf.length))); - }, SAMPLE_MS); - } catch { - stopTap(); - } - }; - - const reconcile = (): void => { - const track = - rooms.map(mutedMicTrack).find((t) => t !== null) ?? null; - if (track === tapped) return; - if (tapped) stopTap(); - if (track) startTap(track); - }; - - const events = [ - RoomEvent.TrackMuted, - RoomEvent.TrackUnmuted, - RoomEvent.LocalTrackPublished, - RoomEvent.LocalTrackUnpublished, - RoomEvent.Reconnected, - RoomEvent.Disconnected, - ] as const; - rooms.forEach((room) => - events.forEach((ev) => room.on(ev, reconcile)), - ); - subscriber.next(false); - reconcile(); - return () => { - rooms.forEach((room) => - events.forEach((ev) => room.off(ev, reconcile)), - ); - if (tapped) stopTap(); - }; - }), - ), + return observeLocalMicSample$(vm).pipe( + scan((gate: MutedSpeechGate | null, sample: LocalMicSample | null) => { + if (!sample?.muted) return null; + const g = gate ?? new MutedSpeechGate(); + g.push(sample.rms); + return g; + }, null), + map((gate) => gate?.value ?? false), + startWith(false), distinctUntilChanged(), ); } diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index c669bc97..8b4199a0 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -32,6 +32,7 @@ import { widget } from "../widget"; import { startLotusCallState } from "../lotus/lotusCallState"; import { startLotusFocus } from "../lotus/lotusFocus"; import { startLotusControls } from "../lotus/lotusControls"; +import { startLotusMicLevel } from "../lotus/lotusMicLevel"; import { startLotusAudioInject } from "../lotus/lotusAudioInject"; import { startLotusQuality } from "../lotus/lotusQuality"; import { startLotusDecorations } from "../lotus/lotusDecorations"; @@ -304,6 +305,8 @@ export const InCallView: FC = ({ // [cinny #43] layout / settings / reactions over the widget API, plus a // screensharing + layout report, replacing the host's DOM access. useEffect(() => startLotusControls(vm), [vm]); + // [cinny #146] Local mic level for the host's mute-button meter. + useEffect(() => startLotusMicLevel(vm), [vm]); // [lotus] Handle the host's io.lotus.inject_audio action to mix a soundboard // clip into the call as a separate track (#3). No-op unless the host sends it. useEffect(() => startLotusAudioInject(vm), [vm]);