From f1cfcc73773776593ca1d68bc7494675b7056fb3 Mon Sep 17 00:00:00 2001 From: Lotus CI Date: Sat, 12 Sep 2026 11:34:08 -0400 Subject: [PATCH 1/6] fix(lotus): implement deafen via EC's global output mute, not setVolume MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Deafen wrote RemoteParticipant.setVolume(0, Microphone), which EC's own createVolumeControls overwrote with the tile volume every time sink$ re-emitted (every join / re-resolution) — so anyone joining while you were deafened was audible — and undeafen blanket-wrote 1 to every participant, clobbering per-tile volume/mute. The ParticipantConnected listeners also hung off livekitRoomItems$, which is empty while alone, so the first joiner could be heard briefly. Deafen now drives setAudioEnabled$ -> muteAllAudio$, the `muted` prop InCallView already passes to every audio renderer, which also silences Track.Source.Unknown soundboard audio (the known P6-2 gap). Undeafen restores the user's own output state and never touches the mute-all setting. Re-applying the same state is a no-op so resendForkState() is safe. Screenshare-audio-only mute keeps setVolume(ScreenShareAudio) but only restores participants it muted itself. Rooms come from allConnections$ like the sibling modules. Unit-tested. Fixes #1 Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01PPmy3tPq869XDW4njjVaKA --- src/lotus/lotusDeafen.test.ts | 154 ++++++++++++++++++++++++++++++++++ src/lotus/lotusDeafen.ts | 134 +++++++++++++++++++++-------- 2 files changed, 255 insertions(+), 33 deletions(-) create mode 100644 src/lotus/lotusDeafen.test.ts diff --git a/src/lotus/lotusDeafen.test.ts b/src/lotus/lotusDeafen.test.ts new file mode 100644 index 00000000..b9a11ee6 --- /dev/null +++ b/src/lotus/lotusDeafen.test.ts @@ -0,0 +1,154 @@ +/* +Copyright 2026 Lotus Guild + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { EventEmitter } from "events"; +import { afterEach, beforeEach, expect, test, vi } from "vitest"; +import { of } from "rxjs"; +import { Track } from "livekit-client"; + +import { type CallViewModel } from "../state/CallViewModel/CallViewModel"; +import { setAudioEnabled$ } from "../controls"; +import { startLotusDeafen } from "./lotusDeafen"; +import { LotusWidgetActions } from "./lotusActions"; + +const lazyActions = new EventEmitter(); + +vi.mock("../widget", () => ({ + widget: { + api: { transport: { reply: vi.fn().mockResolvedValue(undefined) } }, + // Getter: `vi.mock` factories run at import time, before the const above. + get lazyActions(): EventEmitter { + return lazyActions; + }, + }, +})); + +/** Minimal CallViewModel stub: no connections, so no livekit rooms. */ +function mockVm(participants: unknown[] = []): CallViewModel { + const livekitRoom = { + remoteParticipants: new Map(participants.map((p, i) => [String(i), p])), + on: vi.fn(), + off: vi.fn(), + }; + return { + allConnections$: of({ getConnections: () => [{ livekitRoom }] }), + } as unknown as CallViewModel; +} + +function send(data: unknown): void { + lazyActions.emit(LotusWidgetActions.SetDeafen, { + detail: { data }, + }); +} + +/** Records everything pushed to the global audio-output subject. */ +let emissions: boolean[]; +let sub: { unsubscribe: () => void }; + +beforeEach(() => { + emissions = []; + sub = setAudioEnabled$.subscribe((v) => emissions.push(v)); +}); + +afterEach(() => { + sub.unsubscribe(); + lazyActions.removeAllListeners(); +}); + +test("deafen mutes, and undeafen unmutes, EC's global audio output", () => { + const stop = startLotusDeafen(mockVm()); + + send({ deafened: true, screenshareAudioMuted: false }); + expect(emissions).toEqual([false]); + + send({ deafened: false, screenshareAudioMuted: false }); + expect(emissions).toEqual([false, true]); + + stop(); + expect(emissions).toEqual([false, true]); +}); + +test("re-sending the same state is idempotent (host resendForkState)", () => { + const stop = startLotusDeafen(mockVm()); + + send({ deafened: true, screenshareAudioMuted: false }); + send({ deafened: true, screenshareAudioMuted: false }); + send({ deafened: true, screenshareAudioMuted: false }); + expect(emissions).toEqual([false]); + + stop(); + // Teardown restores the user's own state, which was "audio enabled". + expect(emissions).toEqual([false, true]); +}); + +test("undeafen does not re-enable audio the user had muted themselves", () => { + const stop = startLotusDeafen(mockVm()); + + // The user mutes all audio through EC's own control first. + setAudioEnabled$.next(false); + expect(emissions).toEqual([false]); + + send({ deafened: true, screenshareAudioMuted: false }); + send({ deafened: false, screenshareAudioMuted: false }); + // Nothing further was pushed: audio was already off and stays off. + expect(emissions).toEqual([false]); + + stop(); + expect(emissions).toEqual([false]); +}); + +test("deafen never touches per-participant volume", () => { + const participant = { setVolume: vi.fn() }; + const stop = startLotusDeafen(mockVm([participant])); + + send({ deafened: true, screenshareAudioMuted: false }); + send({ deafened: false, screenshareAudioMuted: false }); + expect(participant.setVolume).not.toHaveBeenCalled(); + + stop(); +}); + +test("screenshare-audio mute is applied per source and only undone for participants we muted", () => { + const participant = { setVolume: vi.fn() }; + const stop = startLotusDeafen(mockVm([participant])); + + // Not muted yet: no volume writes at all. + send({ deafened: false, screenshareAudioMuted: false }); + expect(participant.setVolume).not.toHaveBeenCalled(); + + send({ deafened: false, screenshareAudioMuted: true }); + expect(participant.setVolume).toHaveBeenCalledWith( + 0, + Track.Source.ScreenShareAudio, + ); + + participant.setVolume.mockClear(); + send({ deafened: false, screenshareAudioMuted: false }); + expect(participant.setVolume).toHaveBeenCalledWith( + 1, + Track.Source.ScreenShareAudio, + ); + + // Releasing again writes nothing: we no longer own that participant. + participant.setVolume.mockClear(); + send({ deafened: false, screenshareAudioMuted: false }); + expect(participant.setVolume).not.toHaveBeenCalled(); + + stop(); +}); + +test("a partial payload only moves the flag it names", () => { + const stop = startLotusDeafen(mockVm()); + + send({ deafened: true }); + expect(emissions).toEqual([false]); + send({ screenshareAudioMuted: true }); + // Still deafened: no change to the global mute. + expect(emissions).toEqual([false]); + + stop(); +}); diff --git a/src/lotus/lotusDeafen.ts b/src/lotus/lotusDeafen.ts index 2d0bc5b2..e5bec430 100644 --- a/src/lotus/lotusDeafen.ts +++ b/src/lotus/lotusDeafen.ts @@ -15,29 +15,52 @@ import { logger } from "matrix-js-sdk/lib/logger"; import { type IWidgetApiRequest } from "matrix-widget-api"; import { type CallViewModel } from "../state/CallViewModel/CallViewModel"; +import { setAudioEnabled$ } from "../controls"; import { widget } from "../widget"; import { LotusWidgetActions } from "./lotusActions"; /** - * Handle the host's `io.lotus.set_deafen` toWidget action: silence remote audio - * at the LiveKit source, replacing cinny's brittle iframe-DOM `.muted` hack - * (which fought MatrixAudioRenderer and broke on re-render / late tracks). + * Handle the host's `io.lotus.set_deafen` toWidget action, replacing cinny's + * brittle iframe-DOM `.muted` hack (which fought MatrixAudioRenderer and broke + * on re-render / late tracks). * - * `deafened` mutes every remote participant's microphone AND screenshare audio; - * `screenshareAudioMuted` mutes only the screenshare audio (so the host can - * drop shared-tab/game audio while still hearing voices). Volume is set PER - * SOURCE via `RemoteParticipant.setVolume(volume, source)` — whose verified - * signature in livekit-client ^2.18.1 is - * `setVolume(volume, source?: Track.Source.Microphone | Track.Source.ScreenShareAudio)` - * and DEFAULTS `source` to `Microphone` (NOT "all audio"), so each source must - * be set explicitly. setVolume records the value in the participant's - * `volumeMap`, so a track that (re)subscribes later re-applies it automatically. + * `deafened` drives EC's OWN global audio output mute: `setAudioEnabled$` feeds + * `muteAllAudio$` (`src/state/MuteAllAudioModel.ts`), which `InCallView` already + * passes as `muted` to every `LivekitRoomAudioRenderer` (and hence + * `MatrixAudioRenderer`), plus `CallEventAudioRenderer` and + * `ReactionsAudioRenderer`. That silences EVERY remote source — microphone, + * screenshare audio and `Track.Source.Unknown` soundboard clips — and needs no + * per-participant bookkeeping for late joiners or reconnects. * - * State is closure-scoped (per invocation, matching the sibling lotus modules) - * and re-applied to every current room's participants on change, and to LATE - * JOINERS via `RoomEvent.ParticipantConnected`. The cinny host re-sends the - * current state on every call join (CallControl.forceState), so a fresh call - * never inherits a previous call's deafen state. + * This deliberately does NOT use `RemoteParticipant.setVolume` for deafen any + * more: EC's per-participant volume slider / per-tile mute writes the same + * `volumeMap` from `createVolumeControls` (`src/state/VolumeControls.ts`) + * whenever its `sink$` re-emits (every join, every participant re-resolution), + * so a `setVolume(0)` deafen was silently undone for anyone joining while + * deafened, and an undeafen `setVolume(1)` clobbered the user's own per-tile + * volume/mute state. + * + * `screenshareAudioMuted` is a NARROWER, independent host control (drop shared + * tab/game audio while still hearing voices). EC has no global per-source mute, + * so that one still has to go through + * `RemoteParticipant.setVolume(volume, Track.Source.ScreenShareAudio)` — whose + * verified signature in livekit-client ^2.18.1 is + * `setVolume(volume, source?: Track.Source.Microphone | Track.Source.ScreenShareAudio)`. + * It is re-applied to late joiners via `RoomEvent.ParticipantConnected`, and we + * only ever restore participants we muted ourselves. Known limitation: EC's own + * screenshare volume slider writes the same `volumeMap`, so a user who moves + * that slider while screenshare audio is host-muted wins; the mic path (the + * actual deafen) is no longer affected by that race at all. + * + * Undeafen restores the user's OWN output-enabled state as it was before the + * deafen (and never touches the `mute-all-audio` setting), so a user who had + * already muted all audio themselves stays muted. + * + * State is closure-scoped (per invocation, matching the sibling lotus modules). + * Applying the same state twice is a no-op, so the host's + * `CallControl.resendForkState()` after a reconnect is safe. The host re-sends + * the current state on every call join (CallControl.forceState), so a fresh + * call never inherits a previous call's deafen state. * * No effect unless the host sends the action. Returns a teardown function. */ @@ -48,32 +71,69 @@ export function startLotusDeafen(vm: CallViewModel): () => void { let deafened = false; let screenshareAudioMuted = false; + // The user's own audio-output state, tracked from `setAudioEnabled$` so an + // undeafen restores it rather than blindly enabling output. Defaults to + // `true`, matching `muteAllAudio$`'s `startWith(true)`. + let userAudioEnabled = true; + // Ignore our own (synchronous) emissions while tracking the user's state, and + // skip pushes that would not change anything so re-applying the same state + // (the host's `resendForkState()` after a reconnect) is a no-op. + let selfEmitting = false; + let lastApplied: boolean | null = null; + + const applyGlobalMute = (): void => { + const desired = deafened ? false : userAudioEnabled; + // `lastApplied` is our own override, if any; otherwise the live state is + // whatever the user last set. + const current = lastApplied ?? userAudioEnabled; + lastApplied = desired; + if (desired === current) return; + selfEmitting = true; + try { + setAudioEnabled$.next(desired); + } finally { + selfEmitting = false; + } + }; + + const audioSub = setAudioEnabled$.subscribe((enabled) => { + if (selfEmitting) return; + userAudioEnabled = enabled; + if (deafened && enabled) { + // Something else (the native output controls) re-enabled audio while + // deafened — re-assert the mute instead of silently losing deafen. + lastApplied = true; + applyGlobalMute(); + } + }); + + // Participants whose screenshare audio WE turned down, so undoing the host's + // screenshare mute never writes a volume to anyone else. + const screenshareMuted = new Set(); + const applyToParticipant = (p: RemoteParticipant): void => { - p.setVolume(deafened ? 0 : 1, Track.Source.Microphone); - p.setVolume( - deafened || screenshareAudioMuted ? 0 : 1, - Track.Source.ScreenShareAudio, - ); - // NOTE: injected/soundboard audio (published as `Track.Source.Unknown`) is - // deliberately NOT silenced here. The verified `setVolume` type signature - // only accepts `Track.Source.Microphone | Track.Source.ScreenShareAudio`, - // so passing `Unknown` would fail the fork's `tsc` gate and require an - // unsafe cast. Soundboard clips are short, host-triggered content the host - // already controls at the inject source, so leaving them audible is the - // type-clean, safe choice (a full-parity "mute everything" is not needed - // for the deafen semantics: don't-hear-other-people's-voices). + if (screenshareAudioMuted) { + p.setVolume(0, Track.Source.ScreenShareAudio); + screenshareMuted.add(p); + } else if (screenshareMuted.delete(p)) { + p.setVolume(1, Track.Source.ScreenShareAudio); + } }; const applyToRoom = (room: LivekitRoom): void => room.remoteParticipants.forEach(applyToParticipant); // Per-room ParticipantConnected listeners, so LATE JOINERS pick up the - // current deafen state the moment they connect. + // current screenshare-audio mute the moment they connect. Drive off the + // local participant's connection(s), not `livekitRoomItems$` — that stream is + // empty until a matrix-validated REMOTE member resolves, so listeners would + // be attached too late for the first joiner (the sibling lotus modules all + // use `allConnections$` for the same reason). const roomListeners = new Map void>(); let rooms: LivekitRoom[] = []; - const sub = vm.livekitRoomItems$.subscribe((items) => { - const next = items.map((i) => i.livekitRoom); + const sub = vm.allConnections$.subscribe((data) => { + const next = data.getConnections().map((c) => c.livekitRoom); rooms = next; // Detach listeners for rooms that went away. for (const [room, off] of roomListeners) { @@ -107,6 +167,7 @@ export function startLotusDeafen(vm: CallViewModel): () => void { logger.debug( `[lotus] set_deafen: deafened=${deafened} screenshareAudioMuted=${screenshareAudioMuted}`, ); + applyGlobalMute(); rooms.forEach(applyToRoom); }; @@ -115,6 +176,13 @@ export function startLotusDeafen(vm: CallViewModel): () => void { sub.unsubscribe(); for (const off of roomListeners.values()) off(); roomListeners.clear(); + // Leave the user's own output state as they had it before deafen. + if (deafened) { + deafened = false; + applyGlobalMute(); + } + screenshareMuted.clear(); + audioSub.unsubscribe(); w.lazyActions.off(LotusWidgetActions.SetDeafen, handler); }; } From 936a083533037991e1669fc9f70a4573797b081c Mon Sep 17 00:00:00 2001 From: Lotus CI Date: Sat, 12 Sep 2026 11:34:08 -0400 Subject: [PATCH 2/6] fix(lotus): denoise restart fallback must not hand LiveKit its own raw track MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On a failed restart() the processor set processedTrack to the LiveKit-owned input track. LiveKit's internalStopProcessor() stops processedTrack and then republishes _mediaStreamTrack — the same object — so any later stopProcessor()/teardown killed the live mic for the rest of the session. Leave processedTrack undefined so LiveKit falls through to its own track. Unit-tested. Fixes #2 Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01PPmy3tPq869XDW4njjVaKA --- src/lotus/lotusDenoiseProcessor.test.ts | 55 +++++++++++++++++++++++++ src/lotus/lotusDenoiseProcessor.ts | 54 ++++++++++++++++-------- 2 files changed, 92 insertions(+), 17 deletions(-) create mode 100644 src/lotus/lotusDenoiseProcessor.test.ts diff --git a/src/lotus/lotusDenoiseProcessor.test.ts b/src/lotus/lotusDenoiseProcessor.test.ts new file mode 100644 index 00000000..35573d8f --- /dev/null +++ b/src/lotus/lotusDenoiseProcessor.test.ts @@ -0,0 +1,55 @@ +/* +Copyright 2026 Lotus Guild + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { describe, expect, test } from "vitest"; +import { type AudioProcessorOptions } from "livekit-client"; + +import { LotusDenoiseProcessor } from "./lotusDenoiseProcessor"; + +function makeProcessor(): LotusDenoiseProcessor { + return new LotusDenoiseProcessor({ + model: "rnnoise", + assetBase: "https://example.invalid/denoise/", + gate: false, + gateThreshold: -45, + floor: 0.15, + }); +} + +describe("LotusDenoiseProcessor.restart", () => { + test("falls back to processedTrack = undefined (never the raw LiveKit track) when graph rebuild fails", async () => { + const processor = makeProcessor(); + + // Stub out the AudioContext/graph plumbing: pretend the context is fine + // but the graph rebuild (wasm load / worklet construction) throws, which + // is the path this test targets. + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (processor as any).ensureContext = async (): Promise => { + await Promise.resolve(); + }; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (processor as any).buildGraph = async (): Promise => { + await Promise.resolve(); + throw new Error("simulated graph build failure"); + }; + + const rawTrack = { + stop: (): void => undefined, + } as unknown as MediaStreamTrack; + + await processor.restart({ + track: rawTrack, + } as unknown as AudioProcessorOptions); + + // Must NOT be the raw, LiveKit-owned track: LiveKit's + // internalStopProcessor() calls `processor.processedTrack?.stop()` then + // re-publishes the same `_mediaStreamTrack` object, which would kill the + // live mic on the next stopProcessor()/teardown if we handed it back here. + expect(processor.processedTrack).toBeUndefined(); + expect(processor.processedTrack).not.toBe(rawTrack); + }); +}); diff --git a/src/lotus/lotusDenoiseProcessor.ts b/src/lotus/lotusDenoiseProcessor.ts index 71f09275..eeca521f 100644 --- a/src/lotus/lotusDenoiseProcessor.ts +++ b/src/lotus/lotusDenoiseProcessor.ts @@ -12,11 +12,7 @@ import { } from "livekit-client"; import { logger } from "matrix-js-sdk/lib/logger"; -export type LotusDenoiseModel = - | "rnnoise" - | "speex" - | "dtln" - | "deepfilternet"; +export type LotusDenoiseModel = "rnnoise" | "speex" | "dtln" | "deepfilternet"; export interface LotusDenoiseConfig { model: LotusDenoiseModel; @@ -111,8 +107,8 @@ function supportsSimd(): boolean { // Minimal SIMD module (v128) — validates only where SIMD is supported. return WebAssembly.validate( new Uint8Array([ - 0, 97, 115, 109, 1, 0, 0, 0, 1, 5, 1, 96, 0, 1, 123, 3, 2, 1, 0, 10, - 10, 1, 8, 0, 65, 0, 253, 15, 253, 98, 11, + 0, 97, 115, 109, 1, 0, 0, 0, 1, 5, 1, 96, 0, 1, 123, 3, 2, 1, 0, 10, 10, + 1, 8, 0, 65, 0, 253, 15, 253, 98, 11, ]), ); } catch { @@ -145,9 +141,10 @@ interface Graph { * stopped track on the sender: on failure it degrades to the RAW mic track * rather than silence. */ -export class LotusDenoiseProcessor - implements TrackProcessor -{ +export class LotusDenoiseProcessor implements TrackProcessor< + Track.Kind.Audio, + AudioProcessorOptions +> { public readonly name = "lotus-denoise"; public processedTrack?: MediaStreamTrack; @@ -180,10 +177,16 @@ export class LotusDenoiseProcessor this.processedTrack = next.track; } catch (e) { // Never go silent on the A7/device-switch path: fall back to raw audio. + // [lotus] IMPORTANT: never assign `opts.track` (LiveKit-owned) here. + // LiveKit's internalStopProcessor() does `processor.processedTrack?.stop()` + // then re-publishes `_mediaStreamTrack` — the SAME object if we set it as + // processedTrack — which kills the live mic on the next stopProcessor()/ + // teardown. Leaving processedTrack undefined makes LiveKit fall through to + // its own `_mediaStreamTrack` instead. logger.warn("[lotus] denoise restart failed; using raw mic", e); this.disposeGraph(this.graph); this.graph = undefined; - this.processedTrack = opts.track; + this.processedTrack = undefined; } } @@ -209,7 +212,11 @@ export class LotusDenoiseProcessor /** Create (once) the model-rate context + register the flat worklet modules. */ private async ensureContext(): Promise { const rate = sampleRateFor(this.config.model); - if (this.ctx && this.ctx.state !== "closed" && this.ctx.sampleRate === rate) { + if ( + this.ctx && + this.ctx.state !== "closed" && + this.ctx.sampleRate === rate + ) { if (this.ctx.state === "suspended") await resumeCtx(this.ctx); return; } @@ -316,7 +323,12 @@ export class LotusDenoiseProcessor logger.info( `[lotus] denoise processor active (${this.config.model}, floor=${floor})`, ); - return { source, nodes, disposes, track: dest.stream.getAudioTracks()[0] }; + return { + source, + nodes, + disposes, + track: dest.stream.getAudioTracks()[0], + }; } catch (e) { // A node constructor / model load can throw mid-build; clean up the // partially-built graph so it doesn't leak (init/restart still fall back @@ -338,15 +350,20 @@ export class LotusDenoiseProcessor if (model === "dtln") { // Self-contained ESM that resolves its own processor + LiteRT wasm + // TFLite models. bypassUntilReady passes raw audio until the model loads. - const mod = await import(/* @vite-ignore */ `${base}workadventure/audio-worklet.js`); + const mod = await import( + /* @vite-ignore */ `${base}workadventure/audio-worklet.js` + ); return (await mod.createNoiseSuppressionAudioWorklet(ctx, { bypassUntilReady: true, })) as MlNode; } if (model === "deepfilternet") { - const dfnBase = new URL(`${base}deepfilternet`, window.location.href).href; - const mod = await import(/* @vite-ignore */ `${base}deepfilternet/index.esm.js`); + const dfnBase = new URL(`${base}deepfilternet`, window.location.href) + .href; + const mod = await import( + /* @vite-ignore */ `${base}deepfilternet/index.esm.js` + ); const core = new mod.DeepFilterNet3Core({ sampleRate: 48_000, // 60, not 80: full-strength suppression is the main source of the @@ -379,7 +396,10 @@ export class LotusDenoiseProcessor numberOfOutputs: 1, processorOptions: { maxChannels: 1, wasmBinary }, }); - return { node, dispose: () => void safeCall(() => node.port.postMessage("destroy")) }; + return { + node, + dispose: () => void safeCall(() => node.port.postMessage("destroy")), + }; } private disposeGraph(graph: Graph | undefined): void { From 59e0c852eed22760ff5325ee16f86dfe2a50e917 Mon Sep 17 00:00:00 2001 From: Lotus CI Date: Sat, 12 Sep 2026 11:34:08 -0400 Subject: [PATCH 3/6] fix(lotus): don't start two denoise processors on racing LocalTrackPublished MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit apply() guarded on mic.getProcessor(), which is only set after setProcessor() resolves (after the whole wasm/model load), so mic-published followed by camera-published constructed two processors — two AudioContexts, two model loads, double lock hold time. Track the in-flight processor per room, skip apply() while one is pending, and destroy a pending processor if the module is torn down before setProcessor resolves. Unit-tested. Fixes #10 Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01PPmy3tPq869XDW4njjVaKA --- src/lotus/lotusDenoise.test.ts | 141 +++++++++++++++++++++++++++++++++ src/lotus/lotusDenoise.ts | 35 ++++++-- 2 files changed, 171 insertions(+), 5 deletions(-) create mode 100644 src/lotus/lotusDenoise.test.ts diff --git a/src/lotus/lotusDenoise.test.ts b/src/lotus/lotusDenoise.test.ts new file mode 100644 index 00000000..8006c8a2 --- /dev/null +++ b/src/lotus/lotusDenoise.test.ts @@ -0,0 +1,141 @@ +/* +Copyright 2026 Lotus Guild + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { afterEach, beforeEach, describe, expect, test, vi } from "vitest"; +import { ParticipantEvent, Track } from "livekit-client"; + +import { type CallViewModel } from "../state/CallViewModel/CallViewModel"; +import { startLotusDenoise } from "./lotusDenoise"; + +// Track constructed instances so tests can assert exactly one processor was +// built across racing apply() calls, and can assert the pending one is +// destroyed on early teardown. `vi.hoisted` is required because `vi.mock` +// factories are hoisted above this file's other top-level statements. +const instances = vi.hoisted( + () => [] as { destroy: ReturnType }[], +); +vi.mock("./lotusDenoiseProcessor", () => ({ + LotusDenoiseProcessor: class { + public destroy = vi.fn().mockResolvedValue(undefined); + public constructor() { + instances.push(this); + } + }, +})); + +/** A promise plus externally-callable resolve, for controlling ordering. */ +function deferred(): { + promise: Promise; + resolve: (v: T) => void; +} { + let resolve!: (v: T) => void; + const promise = new Promise((r) => { + resolve = r; + }); + return { promise, resolve }; +} + +function makeRoomAndVm(mic: { + getProcessor: () => unknown; + setProcessor: ReturnType; +}): { + vm: CallViewModel; + room: { localParticipant: Record }; + firePublished: () => void; +} { + const handlers = new Map void>(); + const localParticipant = { + getTrackPublication: ( + source: Track.Source, + ): { track: typeof mic } | undefined => + source === Track.Source.Microphone ? { track: mic } : undefined, + on: (event: string, cb: () => void): Map void> => + handlers.set(event, cb), + off: (event: string): boolean => handlers.delete(event), + }; + const room = { localParticipant }; + const vm = { + allConnections$: { + subscribe: ( + cb: (data: { + getConnections: () => { livekitRoom: unknown }[]; + }) => void, + ) => { + cb({ getConnections: () => [{ livekitRoom: room }] }); + return { unsubscribe: (): void => undefined }; + }, + }, + } as unknown as CallViewModel; + return { + vm, + room, + firePublished: () => handlers.get(ParticipantEvent.LocalTrackPublished)?.(), + }; +} + +beforeEach(() => { + instances.length = 0; + // `lotusParam`/`lotusFlag` cache the URL params at first read; seed the hash + // before startLotusDenoise() so the dedicated `lotusDenoiseSource` flag reads on. + window.location.hash = "#/room?lotusDenoiseSource=1"; +}); + +afterEach(() => { + vi.resetModules(); +}); + +describe("startLotusDenoise", () => { + test("racing LocalTrackPublished events only construct one processor", async () => { + const setProcessorDeferred = deferred(); + let attached: unknown; + const mic = { + getProcessor: (): unknown => attached, + setProcessor: vi.fn(async (p: unknown) => { + await setProcessorDeferred.promise; + attached = p; + }), + }; + const { vm, firePublished } = makeRoomAndVm(mic); + + startLotusDenoise(vm); + // Simulate a second LocalTrackPublished (e.g. camera) firing before the + // first setProcessor() has resolved. + firePublished(); + + expect(mic.setProcessor).toHaveBeenCalledTimes(1); + expect(instances).toHaveLength(1); + + setProcessorDeferred.resolve(); + await Promise.resolve(); + await Promise.resolve(); + + // Once attached, a further publish should be a no-op (mic.getProcessor() + // is now set). + firePublished(); + expect(mic.setProcessor).toHaveBeenCalledTimes(1); + }); + + test("destroys the in-flight processor if torn down before setProcessor resolves", async () => { + const setProcessorDeferred = deferred(); + const mic = { + getProcessor: (): unknown => undefined, + setProcessor: vi.fn(async () => setProcessorDeferred.promise), + }; + const { vm } = makeRoomAndVm(mic); + + const teardown = startLotusDenoise(vm); + expect(instances).toHaveLength(1); + + teardown(); + + expect(instances[0].destroy).toHaveBeenCalledTimes(1); + + // Resolving afterwards must not throw/reject unhandled. + setProcessorDeferred.resolve(); + await Promise.resolve(); + }); +}); diff --git a/src/lotus/lotusDenoise.ts b/src/lotus/lotusDenoise.ts index 8f06aa41..7c87e6d0 100644 --- a/src/lotus/lotusDenoise.ts +++ b/src/lotus/lotusDenoise.ts @@ -97,13 +97,29 @@ export function startLotusDenoise(vm: CallViewModel): () => void { room.localParticipant.getTrackPublication(Track.Source.Microphone) ?.track as LocalAudioTrack | undefined; + // [lotus] `mic.getProcessor()` only becomes set once `setProcessor()` + // resolves — i.e. after the whole wasm/model load. LiveKit fires + // `LocalTrackPublished` once per local track (mic, then camera on join with + // video), so two calls to `apply()` can both observe `!mic.getProcessor()` + // and race to construct a second `LotusDenoiseProcessor` (a second + // AudioContext + model load) before the first has attached. Track an + // in-flight setProcessor per room and skip `apply()` while one is pending. + const pendingProcessors = new Map(); + const apply = (room: LivekitRoom): void => { const mic = micOf(room); - if (mic && !mic.getProcessor()) { - void mic - .setProcessor(new LotusDenoiseProcessor(config)) - .catch((e) => logger.warn("[lotus] denoise setProcessor failed", e)); - } + if (!mic || mic.getProcessor() || pendingProcessors.has(room)) return; + const processor = new LotusDenoiseProcessor(config); + pendingProcessors.set(room, processor); + void mic + .setProcessor(processor) + .catch((e) => logger.warn("[lotus] denoise setProcessor failed", e)) + .finally(() => { + // Only clear if we're still the pending entry (a teardown that ran + // while this was in flight may have already replaced/removed it). + if (pendingProcessors.get(room) === processor) + pendingProcessors.delete(room); + }); }; const roomListeners = new Map void>(); @@ -149,6 +165,15 @@ export function startLotusDenoise(vm: CallViewModel): () => void { for (const room of rooms) { const mic = micOf(room); if (mic?.getProcessor()) void mic.stopProcessor(); + else { + // [lotus] A setProcessor() call may still be in flight (mid wasm/model + // load) when teardown runs, in which case `mic.getProcessor()` is + // still undefined and `stopProcessor()` above is a no-op. Destroy the + // pending processor directly so its AudioContext/graph don't leak. + const pending = pendingProcessors.get(room); + if (pending) void pending.destroy().catch(() => undefined); + } } + pendingProcessors.clear(); }; } From 37c9348ee869776eef1a81d0ebb8a45b2d62e05e Mon Sep 17 00:00:00 2001 From: Lotus CI Date: Sat, 12 Sep 2026 11:34:08 -0400 Subject: [PATCH 4/6] fix(lotus): enable strictOriginCheck on the widget transport matrix-widget-api's PostmessageTransport accepts toWidget actions from any origin by default; the fork added actions that inject audio, deafen, retune the encoder and render images. EC is served same-origin with the cinny host, so globalThis.origin === parentOrigin and the strict check passes. Fixes #15 Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01PPmy3tPq869XDW4njjVaKA --- src/widget.ts | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/widget.ts b/src/widget.ts index 4c4e9d1d..2511376a 100644 --- a/src/widget.ts +++ b/src/widget.ts @@ -93,6 +93,16 @@ export const initializeWidget = ( const parentOrigin = new URL(parentUrl).origin; logger.info("Widget API is available"); const api = new WidgetApi(widgetId, parentOrigin); + // [lotus] matrix-widget-api's PostmessageTransport defaults + // strictOriginCheck to false, which would let any frame holding a + // handle to our window post toWidget actions (including the + // io.lotus.* actions below). The Lotus deployment serves EC + // same-origin with the host (cinny loads /public/element-call/index.html), + // so globalThis.origin === parentOrigin and this check passes safely. + // A cross-origin deployment would need to compare ev.origin to + // parentOrigin instead, since strictOriginCheck compares against + // globalThis.origin. + api.transport.strictOriginCheck = true; api.requestCapability(MatrixCapabilities.AlwaysOnScreen); api.requestCapability(MatrixCapabilities.MSC4039DownloadFile); From fd957badfff8b3aa8be0a7b20ffb497b79d9ecc0 Mon Sep 17 00:00:00 2001 From: Lotus CI Date: Sat, 12 Sep 2026 11:34:08 -0400 Subject: [PATCH 5/6] fix(lotus): call_state skips standalone mode; dedupe before throttle startLotusCallState gated only on the URL flag, so a standalone (non-widget) load built the whole per-member pipeline and stringified it at up to 8 Hz for a host that doesn't exist. Add the same `if (!widget)` guard the sibling modules use. Also move distinctUntilChanged ahead of throttleTime so an unchanged value no longer spends a throttle window (first half of #20). Fixes #31 Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01PPmy3tPq869XDW4njjVaKA --- src/lotus/lotusCallState.ts | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/src/lotus/lotusCallState.ts b/src/lotus/lotusCallState.ts index 86c7ea88..11ca6d78 100644 --- a/src/lotus/lotusCallState.ts +++ b/src/lotus/lotusCallState.ts @@ -6,9 +6,15 @@ Please see LICENSE in the repository root for full details. */ import { combineLatest, of, type Subscription } from "rxjs"; -import { distinctUntilChanged, map, switchMap, throttleTime } from "rxjs/operators"; +import { + distinctUntilChanged, + map, + switchMap, + throttleTime, +} from "rxjs/operators"; import { type CallViewModel } from "../state/CallViewModel/CallViewModel"; +import { widget } from "../widget"; import { LotusWidgetActions, lotusFlag, lotusSendToHost } from "./lotusWidget"; interface ParticipantState { @@ -32,6 +38,10 @@ interface ParticipantState { */ export function startLotusCallState(vm: CallViewModel): () => void { if (!lotusFlag("lotusCallState")) return () => undefined; + // [lotus] Standalone (non-widget) mode has no host to send state to; + // skip building the whole stream pipeline, mirroring lotusFocus.ts / + // lotusDecorations.ts. + if (!widget) return () => undefined; const sub: Subscription = vm.userMedia$ .pipe( @@ -46,7 +56,11 @@ export function startLotusCallState(vm: CallViewModel): () => void { m.videoEnabled$, ]).pipe( map( - ([speaking, audioEnabled, videoEnabled]): ParticipantState => ({ + ([ + speaking, + audioEnabled, + videoEnabled, + ]): ParticipantState => ({ id: m.id, userId: m.userId, speaking, @@ -58,11 +72,12 @@ export function startLotusCallState(vm: CallViewModel): () => void { ), ), ), - // `speaking` flips rapidly; cap the send rate and drop no-op repeats. - // 250ms is plenty for speaking rings / mute badges and keeps the - // request/response widget traffic modest. - throttleTime(250, undefined, { leading: true, trailing: true }), + // `speaking` flips rapidly; drop no-op repeats BEFORE throttling so + // the throttle window isn't spent re-emitting an unchanged value, then + // cap the send rate. 250ms is plenty for speaking rings / mute badges + // and keeps the request/response widget traffic modest. distinctUntilChanged((a, b) => JSON.stringify(a) === JSON.stringify(b)), + throttleTime(250, undefined, { leading: true, trailing: true }), ) .subscribe((participants) => { lotusSendToHost(LotusWidgetActions.CallState, { participants }); From cbd42bf9759380a657169250ad1204b295aebdbf Mon Sep 17 00:00:00 2001 From: Lotus CI Date: Sat, 12 Sep 2026 11:34:08 -0400 Subject: [PATCH 6/6] fix(lotus): set_quality null actually clears a cap on the sender MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A null cap only removed the key from the sticky settings and applyToRoom then skipped that sender, leaving the previously written maxBitrate / maxFramerate live on the RTCRtpSender — contrary to the host contract documented in cinny's CallControl. Track which keys this module wrote per sender and write them back as undefined on clear. Unit-tested. Fixes #11 Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01PPmy3tPq869XDW4njjVaKA --- src/lotus/lotusQuality.test.ts | 93 ++++++++++++++++++++++++++++++++++ src/lotus/lotusQuality.ts | 78 ++++++++++++++++++++++------ 2 files changed, 155 insertions(+), 16 deletions(-) create mode 100644 src/lotus/lotusQuality.test.ts diff --git a/src/lotus/lotusQuality.test.ts b/src/lotus/lotusQuality.test.ts new file mode 100644 index 00000000..e8dbce03 --- /dev/null +++ b/src/lotus/lotusQuality.test.ts @@ -0,0 +1,93 @@ +/* +Copyright 2026 Lotus Guild + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { describe, expect, test, vi } from "vitest"; + +import { buildPatch, patchSender } from "./lotusQuality"; + +function makeSender( + initialEncodings: RTCRtpEncodingParameters[] = [{}], +): RTCRtpSender { + let encodings = initialEncodings; + return { + getParameters: vi.fn(() => ({ encodings })), + setParameters: vi.fn(async (params: RTCRtpSendParameters) => { + await Promise.resolve(); + encodings = params.encodings ?? []; + }), + } as unknown as RTCRtpSender; +} + +describe("lotusQuality set_quality -> clear (#11)", () => { + test("clearing a previously-set cap actively unsets it on the sender", async () => { + const sender = makeSender(); + const writtenKeys = new WeakMap< + RTCRtpSender, + Set + >(); + + // Set: audioMaxBitrate = 64000. + const setPatch = buildPatch(sender, { maxBitrate: 64_000 }, writtenKeys); + expect(setPatch).toEqual({ maxBitrate: 64_000 }); + await patchSender(sender, setPatch, writtenKeys); + + expect(sender.getParameters().encodings[0].maxBitrate).toBe(64_000); + expect(writtenKeys.get(sender)).toEqual(new Set(["maxBitrate"])); + + // Clear: host sends `null`, so the caller now wants an empty desired + // patch. buildPatch must still emit an explicit `undefined` for the key + // it previously wrote, instead of an empty patch that leaves the stale + // cap on the sender. + const clearPatch = buildPatch(sender, {}, writtenKeys); + expect(clearPatch).toEqual({ maxBitrate: undefined }); + await patchSender(sender, clearPatch, writtenKeys); + + const finalEncoding = sender.getParameters().encodings[0]; + expect(finalEncoding.maxBitrate).toBeUndefined(); + expect("maxBitrate" in finalEncoding).toBe(true); // explicitly cleared, not merely absent + expect(writtenKeys.has(sender)).toBe(false); + }); + + test("clearing one of several caps leaves the others (and their tracking) intact", async () => { + const sender = makeSender(); + const writtenKeys = new WeakMap< + RTCRtpSender, + Set + >(); + + const setPatch = buildPatch( + sender, + { maxBitrate: 500_000, maxFramerate: 24 }, + writtenKeys, + ); + await patchSender(sender, setPatch, writtenKeys); + expect(writtenKeys.get(sender)).toEqual( + new Set(["maxBitrate", "maxFramerate"]), + ); + + // Only maxFramerate is still desired; maxBitrate should be actively + // cleared. + const clearPatch = buildPatch(sender, { maxFramerate: 24 }, writtenKeys); + expect(clearPatch).toEqual({ maxFramerate: 24, maxBitrate: undefined }); + await patchSender(sender, clearPatch, writtenKeys); + + const finalEncoding = sender.getParameters().encodings[0]; + expect(finalEncoding.maxBitrate).toBeUndefined(); + expect(finalEncoding.maxFramerate).toBe(24); + expect(writtenKeys.get(sender)).toEqual(new Set(["maxFramerate"])); + }); + + test("no sender means an empty patch and no call", () => { + const writtenKeys = new WeakMap< + RTCRtpSender, + Set + >(); + expect(buildPatch(undefined, { maxBitrate: 1000 }, writtenKeys)).toEqual( + {}, + ); + }); +}); diff --git a/src/lotus/lotusQuality.ts b/src/lotus/lotusQuality.ts index 8c182358..4917fd6f 100644 --- a/src/lotus/lotusQuality.ts +++ b/src/lotus/lotusQuality.ts @@ -45,6 +45,14 @@ export function startLotusQuality(vm: CallViewModel): () => void { if (!w) return () => undefined; const settings: QualitySettings = {}; + // [lotus] Tracks which RTCRtpEncodingParameters keys this module has + // actively written on each sender, so a later `null` (clear) can write + // `undefined` into those same keys instead of just dropping the sticky + // setting and leaving the stale cap live on the sender (#11). + const writtenKeys = new WeakMap< + RTCRtpSender, + Set + >(); // Per-room LocalTrackPublished listeners, so sticky settings re-apply on // every (re)publish. const roomListeners = new Map void>(); @@ -57,24 +65,27 @@ export function startLotusQuality(vm: CallViewModel): () => void { const applyToRoom = (room: LivekitRoom): void => { const lp = room.localParticipant; - if (settings.audioMaxBitrate !== undefined) { - const mic = lp.getTrackPublication(Track.Source.Microphone)?.track as - | LocalTrack - | undefined; - void patchSender(mic?.sender, { maxBitrate: settings.audioMaxBitrate }); - } + const mic = lp.getTrackPublication(Track.Source.Microphone)?.track as + | LocalTrack + | undefined; + const micDesired: Partial = {}; + if (settings.audioMaxBitrate !== undefined) + micDesired.maxBitrate = settings.audioMaxBitrate; + const micPatch = buildPatch(mic?.sender, micDesired, writtenKeys); + if (Object.keys(micPatch).length > 0) + void patchSender(mic?.sender, micPatch, writtenKeys); - const ssPatch: Partial = {}; + const ssDesired: Partial = {}; if (settings.screenshareMaxBitrate !== undefined) - ssPatch.maxBitrate = settings.screenshareMaxBitrate; + ssDesired.maxBitrate = settings.screenshareMaxBitrate; if (settings.screenshareMaxFramerate !== undefined) - ssPatch.maxFramerate = settings.screenshareMaxFramerate; - if (Object.keys(ssPatch).length > 0) { - const ss = lp.getTrackPublication(Track.Source.ScreenShare)?.track as - | LocalTrack - | undefined; - void patchSender(ss?.sender, ssPatch); - } + ssDesired.maxFramerate = settings.screenshareMaxFramerate; + const ss = lp.getTrackPublication(Track.Source.ScreenShare)?.track as + | LocalTrack + | undefined; + const ssPatch = buildPatch(ss?.sender, ssDesired, writtenKeys); + if (Object.keys(ssPatch).length > 0) + void patchSender(ss?.sender, ssPatch, writtenKeys); }; const applyToAll = (): void => rooms.forEach(applyToRoom); @@ -164,9 +175,35 @@ export function startLotusQuality(vm: CallViewModel): () => void { }; } -async function patchSender( +// [lotus] Build the patch to actually send to a sender: the desired caps, +// plus an explicit `undefined` for any key this module previously wrote to +// this sender but no longer wants (see writtenKeys / #11) — otherwise a +// cleared sticky setting would simply be skipped here and the stale +// maxBitrate/maxFramerate would stay live on the RTCRtpSender. +// Exported for unit testing only (see lotusQuality.test.ts) — not part of +// the module's public surface used by callers. +export function buildPatch( + sender: RTCRtpSender | undefined, + desired: Partial, + writtenKeys: WeakMap>, +): Partial { + if (!sender) return {}; + const patch: Partial = { ...desired }; + const prev = writtenKeys.get(sender); + if (prev) { + for (const key of prev) { + if (!(key in patch)) patch[key] = undefined; + } + } + return patch; +} + +// Exported for unit testing only (see lotusQuality.test.ts) — not part of +// the module's public surface used by callers. +export async function patchSender( sender: RTCRtpSender | undefined, patch: Partial, + writtenKeys: WeakMap>, ): Promise { if (!sender) return; try { @@ -178,6 +215,15 @@ async function patchSender( // full-resolution layer — the real bandwidth hog — is a later encoding. for (const enc of params.encodings) Object.assign(enc, patch); await sender.setParameters(params); + // Remember only the caps that are still active (defined) after this + // write, so a later clear knows exactly which keys to unset. + const active = new Set(); + for (const [key, value] of Object.entries(patch)) { + if (value !== undefined) + active.add(key as keyof RTCRtpEncodingParameters); + } + if (active.size > 0) writtenKeys.set(sender, active); + else writtenKeys.delete(sender); } catch (e) { logger.warn("[lotus] set_quality: setParameters failed", e); }