From 949103d6491d120b4d1e9da12ae41c452ecf9644 Mon Sep 17 00:00:00 2001 From: Lotus CI Date: Fri, 30 Oct 2026 21:13:40 -0400 Subject: [PATCH] feat(denoise): switch noise suppression mid-call (io.lotus.set_denoise) The host can now change noise suppression on the live mic without a rejoin: off / browser-native / ML, the ML model, the gate and stacking browser NS in front of the model. - Model swaps preload the new model while the old one keeps running, then swap the processor (no mic re-capture). - Browser NS on/off re-captures the mic via restartTrack with the new constraint (passing the deviceId, without which LiveKit drops every constraint). The choice is kept in lotusCaptureOverride, which ConnectionFactory reads, so a later SFU reconnect captures the same way. - Switches are serialized; a burst collapses into the last one. Each one reports io.lotus.denoise_state (now with `mode`). Fixes found while testing in Chromium: - The mic level meter (and muted-speech nudge) kept a clone of the ORIGINAL mic track and never noticed LiveKit's restartTrack. Chrome makes all open tracks of a device share audio processing, so every re-capture silently kept the old noiseSuppression (asked false, got true). The meter now drops its clone around our re-capture and re-taps on TrackEvent.Restarted, which also fixes it listening to the old device after a mic switch. - LiveKit defaults voiceIsolation ON, a further voice filter in "off" and ML modes where the platform implements it. It now follows browser noise suppression. Verified in a two-browser call: joined as browser and as ML(dtln), then switched dtln -> deepfilternet -> rnnoise -> browser -> off; each step applied within ~0.5 s and every re-capture got the constraints it asked for. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_017PCb59uY3UKqDqt7B6hzd5 --- src/lotus/lotusActions.test.ts | 1 + src/lotus/lotusActions.ts | 11 +- src/lotus/lotusAudioConstraints.test.ts | 15 ++ src/lotus/lotusCaptureOverride.ts | 26 +++ src/lotus/lotusDenoise.test.ts | 183 ++++++++++++++- src/lotus/lotusDenoise.ts | 213 +++++++++++++++--- src/lotus/lotusMicLevel.ts | 55 ++++- .../remoteMembers/ConnectionFactory.ts | 10 +- 8 files changed, 474 insertions(+), 40 deletions(-) create mode 100644 src/lotus/lotusCaptureOverride.ts diff --git a/src/lotus/lotusActions.test.ts b/src/lotus/lotusActions.test.ts index a0dc93bb..02ced805 100644 --- a/src/lotus/lotusActions.test.ts +++ b/src/lotus/lotusActions.test.ts @@ -36,6 +36,7 @@ describe("LotusWidgetActions", () => { LotusWidgetActions.PromptScreenshare, LotusWidgetActions.Reconnect, LotusWidgetActions.DeviceDecision, + LotusWidgetActions.SetDenoise, ]; expect(new Set(LOTUS_TO_WIDGET_ACTIONS)).toEqual(new Set(expectedToWidget)); diff --git a/src/lotus/lotusActions.ts b/src/lotus/lotusActions.ts index 75cf559b..e5f1ad37 100644 --- a/src/lotus/lotusActions.ts +++ b/src/lotus/lotusActions.ts @@ -41,11 +41,19 @@ export enum LotusWidgetActions { RequestState = "io.lotus.request_state", /** * fromWidget: real state of the in-source denoise engine — - * `{ active: boolean, model: string, error?: string }` — sent once the + * `{ active: boolean, model: string, mode: "ml", error?: string }` (or + * `{ active: false, mode }` after a switch away from ML) — sent once the * processor attaches (or after the rnnoise fallback also fails), so the host * toggle can reflect reality rather than the requested state. */ DenoiseState = "io.lotus.denoise_state", + /** + * toWidget: switch noise suppression mid-call — + * `{ mode: "off" | "browser" | "ml", model?, nativeNS?, gate?, gateThreshold? }`. + * `model`/`gate`/`gateThreshold` only matter for "ml"; `nativeNS` stacks the + * browser suppressor in front of the model. Omitted fields keep their value. + */ + SetDenoise = "io.lotus.set_denoise", /** * fromWidget: one-shot end-of-call readout for the local participant — * `{ durationMs, reconnects, poorMs, verdict }` (#143) — sent on SFU @@ -163,4 +171,5 @@ export const LOTUS_TO_WIDGET_ACTIONS: LotusWidgetActions[] = [ LotusWidgetActions.PromptScreenshare, LotusWidgetActions.Reconnect, LotusWidgetActions.DeviceDecision, + LotusWidgetActions.SetDenoise, ]; diff --git a/src/lotus/lotusAudioConstraints.test.ts b/src/lotus/lotusAudioConstraints.test.ts index effac8d0..27667653 100644 --- a/src/lotus/lotusAudioConstraints.test.ts +++ b/src/lotus/lotusAudioConstraints.test.ts @@ -25,6 +25,7 @@ import { echoCancellationSetting, noiseSuppressionSetting, } from "../settings/settings.ts"; +import { setLotusNoiseSuppression } from "./lotusCaptureOverride.ts"; // [lotus] Upstream v0.25.0 moved the audio-capture constraints to Settings. // The Lotus host still drives them per-call via URL params (it turns browser @@ -83,6 +84,7 @@ beforeEach(() => { }); afterEach(() => { + setLotusNoiseSuppression(null); testScope.end(); vi.mocked(LivekitRoom).mockClear(); }); @@ -123,6 +125,7 @@ describe("[lotus] audio-capture URL param overrides", () => { expect(capturedAudioDefaults()).toMatchObject({ echoCancellation: true, noiseSuppression: false, + voiceIsolation: false, autoGainControl: false, }); }); @@ -137,4 +140,16 @@ describe("[lotus] audio-capture URL param overrides", () => { createRoom(); expect(capturedAudioDefaults()).toMatchObject({ echoCancellation: false }); }); + + test("a mid-call set_denoise choice beats the noiseSuppression the call joined with", () => { + getUrlParams.mockReturnValue({ noiseSuppression: false }); + setLotusNoiseSuppression(true); + createRoom(); + expect(capturedAudioDefaults()).toMatchObject({ noiseSuppression: true }); + + getUrlParams.mockReturnValue({ noiseSuppression: true }); + setLotusNoiseSuppression(false); + createRoom(); + expect(capturedAudioDefaults()).toMatchObject({ noiseSuppression: false }); + }); }); diff --git a/src/lotus/lotusCaptureOverride.ts b/src/lotus/lotusCaptureOverride.ts new file mode 100644 index 00000000..1381469b --- /dev/null +++ b/src/lotus/lotusCaptureOverride.ts @@ -0,0 +1,26 @@ +/* +Copyright 2026 Lotus Guild + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +/** + * Browser noise suppression chosen live by the host (`io.lotus.set_denoise`), + * overriding the `noiseSuppression` URL param the call was opened with. Read by + * ConnectionFactory, so a connection created after a mid-call switch (e.g. an + * SFU reconnect) captures the mic the way the user last picked, not the way + * they joined. Dependency-free so ConnectionFactory can import it cheaply. + */ +let noiseSuppressionOverride: boolean | null = null; + +export function setLotusNoiseSuppression(on: boolean | null): void { + noiseSuppressionOverride = on; +} + +/** The override if the host set one, else the URL param (default on). */ +export function lotusNoiseSuppressionAllowed( + urlParam: boolean | undefined, +): boolean { + return noiseSuppressionOverride ?? urlParam !== false; +} diff --git a/src/lotus/lotusDenoise.test.ts b/src/lotus/lotusDenoise.test.ts index 5b43599c..65944462 100644 --- a/src/lotus/lotusDenoise.test.ts +++ b/src/lotus/lotusDenoise.test.ts @@ -9,7 +9,12 @@ import { afterEach, beforeEach, describe, expect, test, vi } from "vitest"; import { ParticipantEvent, Track } from "livekit-client"; import { type CallViewModel } from "../state/CallViewModel/CallViewModel"; -import { startLotusDenoise } from "./lotusDenoise"; +import { + applySetDenoise, + browserNsFor, + type DenoiseTarget, + startLotusDenoise, +} from "./lotusDenoise"; import { LotusWidgetActions } from "./lotusActions"; // Track constructed instances so tests can assert exactly one processor was @@ -42,9 +47,25 @@ vi.mock("./lotusDenoiseProcessor", () => ({ releasePreparedDenoiseAssets: vi.fn().mockResolvedValue(undefined), })); const hostSend = vi.hoisted(() => vi.fn().mockResolvedValue({})); +// Host → widget actions: tests call `sendAction()` to deliver one. +const actionHandlers = vi.hoisted( + () => new Map void>(), +); vi.mock("../widget", () => ({ - widget: { api: { transport: { send: hostSend } } }, + widget: { + api: { transport: { send: hostSend, reply: vi.fn() } }, + lazyActions: { + on: (name: string, cb: (ev: CustomEvent) => void): void => { + actionHandlers.set(name, cb); + }, + off: (name: string): void => { + actionHandlers.delete(name); + }, + }, + }, })); +const sendAction = (name: string, data: unknown): void => + actionHandlers.get(name)?.(new CustomEvent(name, { detail: { data } })); /** A promise plus externally-callable resolve, for controlling ordering. */ function deferred(): { @@ -110,6 +131,7 @@ beforeEach(() => { instances.length = 0; calls.length = 0; hostSend.mockClear(); + actionHandlers.clear(); // `lotusParam`/`lotusFlag` cache the URL params at first read; seed the hash // before startLotusDenoise() so the dedicated `lotusDenoiseSource` flag reads on. window.location.hash = "#/room?lotusDenoiseSource=1"; @@ -188,6 +210,7 @@ describe("startLotusDenoise", () => { expect(hostSend).toHaveBeenCalledWith(LotusWidgetActions.DenoiseState, { active: true, model: "rnnoise", + mode: "ml", }); }); @@ -217,6 +240,7 @@ describe("startLotusDenoise", () => { expect(hostSend).toHaveBeenCalledWith(LotusWidgetActions.DenoiseState, { active: true, model: "rnnoise", + mode: "ml", }); }); @@ -239,6 +263,7 @@ describe("startLotusDenoise", () => { active: false, error: "no worklet for you", model: "rnnoise", + mode: "ml", }); }); @@ -272,3 +297,157 @@ describe("startLotusDenoise", () => { expect(instances[0].setMicMuted).not.toHaveBeenCalled(); }); }); + +/** A mic whose processor and capture constraints behave like LiveKit's. */ +function liveMic(): { + getProcessor: () => { config: { model: string } } | undefined; + setProcessor: ReturnType; + stopProcessor: ReturnType; + restartTrack: ReturnType; + constraints: MediaTrackConstraints; + mediaStreamTrack: { getSettings: () => MediaTrackSettings }; + isMuted: boolean; +} { + let attached: { config: { model: string } } | undefined; + return { + isMuted: false, + getProcessor: () => attached, + setProcessor: vi.fn(async (p: { config: { model: string } }) => { + await Promise.resolve(); + attached = p; + }), + stopProcessor: vi.fn(async () => { + await Promise.resolve(); + attached = undefined; + }), + restartTrack: vi.fn().mockResolvedValue(undefined), + constraints: { echoCancellation: true, autoGainControl: true }, + mediaStreamTrack: { getSettings: () => ({ deviceId: "mic-1" }) }, + }; +} + +describe("io.lotus.set_denoise (switch mid-call)", () => { + test("swaps the ML model on the live mic without re-capturing it", async () => { + window.location.hash = + "#/room?lotusDenoiseSource=1&lotusModel=dtln&noiseSuppression=false"; + const mic = liveMic(); + const { vm } = makeRoomAndVm(mic); + startLotusDenoise(vm); + await flush(); + expect(mic.getProcessor()?.config.model).toBe("dtln"); + + sendAction(LotusWidgetActions.SetDenoise, { + mode: "ml", + model: "deepfilternet", + }); + await flush(); + await flush(); + + expect(mic.stopProcessor).toHaveBeenCalledTimes(1); + expect(mic.getProcessor()?.config.model).toBe("deepfilternet"); + expect(mic.restartTrack).not.toHaveBeenCalled(); + expect(hostSend).toHaveBeenLastCalledWith(LotusWidgetActions.DenoiseState, { + active: true, + model: "deepfilternet", + mode: "ml", + }); + }); + + test("ML → browser-native drops the model and re-captures with browser NS on", async () => { + window.location.hash = + "#/room?lotusDenoiseSource=1&lotusModel=rnnoise&noiseSuppression=false"; + const mic = liveMic(); + const { vm } = makeRoomAndVm(mic); + startLotusDenoise(vm); + await flush(); + + sendAction(LotusWidgetActions.SetDenoise, { mode: "browser" }); + await flush(); + await flush(); + + expect(mic.stopProcessor).toHaveBeenCalledTimes(1); + expect(mic.getProcessor()).toBeUndefined(); + expect(mic.restartTrack).toHaveBeenCalledWith({ + deviceId: "mic-1", + channelCount: 1, + echoCancellation: true, + autoGainControl: true, + noiseSuppression: true, + voiceIsolation: true, + }); + expect(hostSend).toHaveBeenLastCalledWith(LotusWidgetActions.DenoiseState, { + active: false, + mode: "browser", + }); + }); + + test("browser-native → ML works even when the call joined without ML", async () => { + window.location.hash = "#/room?noiseSuppression=true"; + const mic = liveMic(); + const { vm } = makeRoomAndVm(mic); + startLotusDenoise(vm); + await flush(); + expect(mic.setProcessor).not.toHaveBeenCalled(); + + sendAction(LotusWidgetActions.SetDenoise, { mode: "ml", model: "speex" }); + await flush(); + await flush(); + + expect(mic.restartTrack).toHaveBeenCalledWith( + expect.objectContaining({ noiseSuppression: false }), + ); + expect(mic.getProcessor()?.config.model).toBe("speex"); + }); + + test("ignores a payload without a valid mode", async () => { + window.location.hash = "#/room?lotusDenoiseSource=1"; + const mic = liveMic(); + const { vm } = makeRoomAndVm(mic); + startLotusDenoise(vm); + await flush(); + sendAction(LotusWidgetActions.SetDenoise, { mode: "loud" }); + await flush(); + expect(mic.stopProcessor).not.toHaveBeenCalled(); + expect(mic.restartTrack).not.toHaveBeenCalled(); + }); +}); + +describe("applySetDenoise", () => { + const base: DenoiseTarget = { + mode: "ml", + nativeNS: false, + config: { + model: "rnnoise", + assetBase: "https://x/denoise/", + gate: false, + gateThreshold: -45, + floor: 0.15, + }, + }; + + test("keeps fields the payload doesn't name or gets wrong", () => { + expect( + applySetDenoise(base, { mode: "ml", model: "nope", gate: "yes" }), + ).toEqual(base); + }); + + test("moves only the named fields", () => { + const next = applySetDenoise(base, { + mode: "ml", + model: "dtln", + gateThreshold: -50, + }); + expect(next?.config).toEqual({ + ...base.config, + model: "dtln", + gateThreshold: -50, + }); + }); + + test("browser NS is on for browser mode and for ML with nativeNS", () => { + expect(browserNsFor({ ...base, mode: "browser" })).toBe(true); + expect(browserNsFor({ ...base, mode: "off" })).toBe(false); + expect(browserNsFor(base)).toBe(false); + expect(browserNsFor({ ...base, nativeNS: true })).toBe(true); + }); +}); diff --git a/src/lotus/lotusDenoise.ts b/src/lotus/lotusDenoise.ts index c9c00ff3..836487df 100644 --- a/src/lotus/lotusDenoise.ts +++ b/src/lotus/lotusDenoise.ts @@ -13,8 +13,12 @@ import { type TrackPublication, } from "livekit-client"; import { logger } from "matrix-js-sdk/lib/logger"; +import { type IWidgetApiRequest } from "matrix-widget-api"; import { type CallViewModel } from "../state/CallViewModel/CallViewModel"; +import { widget } from "../widget"; +import { setLotusNoiseSuppression } from "./lotusCaptureOverride"; +import { withMicTapsReleased } from "./lotusMicLevel"; import { LotusWidgetActions, lotusFlag, @@ -64,49 +68,135 @@ function safeAssetBase(raw: string | null): string { } } +export type LotusNoiseMode = "off" | "browser" | "ml"; + +/** What the host wants: browser suppression and/or an ML model. */ +export interface DenoiseTarget { + mode: LotusNoiseMode; + /** Stack the browser's suppressor in front of the model (ML only). */ + nativeNS: boolean; + config: LotusDenoiseConfig; +} + +const MODELS: ReadonlySet = new Set([ + "rnnoise", + "speex", + "dtln", + "deepfilternet", +]); +const isModel = (v: unknown): v is LotusDenoiseModel => + typeof v === "string" && MODELS.has(v); + +/** Whether the mic should be captured with the browser's noise suppression. */ +export const browserNsFor = (t: DenoiseTarget): boolean => + t.mode === "browser" || (t.mode === "ml" && t.nativeNS); + +/** + * Apply an `io.lotus.set_denoise` payload to the current target. Unknown or + * malformed fields keep their current value, so a partial payload only moves + * what it names; a payload with no valid `mode` is ignored (returns null). + */ +export function applySetDenoise( + current: DenoiseTarget, + data: unknown, +): DenoiseTarget | null { + const d = (data ?? {}) as Record; + if (d.mode !== "off" && d.mode !== "browser" && d.mode !== "ml") return null; + const model = isModel(d.model) ? d.model : current.config.model; + return { + mode: d.mode, + nativeNS: typeof d.nativeNS === "boolean" ? d.nativeNS : current.nativeNS, + config: { + ...current.config, + model, + gate: typeof d.gate === "boolean" ? d.gate : current.config.gate, + gateThreshold: + typeof d.gateThreshold === "number" && Number.isFinite(d.gateThreshold) + ? d.gateThreshold + : current.config.gateThreshold, + }, + }; +} + +const sameMlConfig = (a: LotusDenoiseConfig, b: LotusDenoiseConfig): boolean => + a.model === b.model && + a.gate === b.gate && + a.gateThreshold === b.gateThreshold && + a.floor === b.floor; + +/** + * Re-acquire the mic with browser noise suppression on or off, keeping the + * rest of its constraints. LiveKit re-attaches any processor to the new track; + * the level-meter clone is dropped first so the new constraint actually takes. + * The deviceId must be passed: without one LiveKit requests `audio: true` and + * silently drops every other constraint. + */ +async function restartCapture( + mic: LocalAudioTrack, + noiseSuppression: boolean, +): Promise { + const c = mic.constraints; + await withMicTapsReleased(async () => { + await mic.restartTrack({ + deviceId: c.deviceId ?? mic.mediaStreamTrack.getSettings().deviceId, + channelCount: c.channelCount ?? 1, + echoCancellation: c.echoCancellation, + autoGainControl: c.autoGainControl, + noiseSuppression, + voiceIsolation: noiseSuppression, + }); + }); +} + export function startLotusDenoise(vm: CallViewModel): () => void { // Gate on a DEDICATED flag — NOT the existing `lotusDenoise=ml` that the // host's build-time getUserMedia shim already uses. Otherwise simply shipping // this fork (while the host still injects its shim and still sets // lotusDenoise=ml) would denoise twice. The host opts into the in-source // engine with `lotusDenoiseSource=1` AND stops injecting the shim at the same - // time. Default off ⇒ the fork is inert and behaviour is unchanged. - if (!lotusFlag("lotusDenoiseSource")) return () => undefined; + // time. Without the flag the engine stays idle unless the host switches to ML + // mid-call (io.lotus.set_denoise), which needs the widget API. + const w = widget; + const mlAtJoin = lotusFlag("lotusDenoiseSource"); + if (!mlAtJoin && !w) return () => undefined; const requested = lotusParam("lotusModel"); - const model: LotusDenoiseConfig["model"] = - requested === "speex" || - requested === "dtln" || - requested === "deepfilternet" - ? requested - : "rnnoise"; + const model: LotusDenoiseConfig["model"] = isModel(requested) + ? requested + : "rnnoise"; const rawThreshold = lotusParam("lotusGateThreshold"); const rawFloor = lotusParam("lotusDenoiseFloor"); - const config: LotusDenoiseConfig = { - model, - assetBase: safeAssetBase(lotusParam("lotusDenoiseBase")), - gate: lotusFlag("lotusGate"), - // Default -45 (matches the reference shim); accept an explicit 0 (don't - // coerce it away via `|| default`). - gateThreshold: - rawThreshold !== null && Number.isFinite(Number(rawThreshold)) - ? Number(rawThreshold) - : -45, - // Dry/wet attenuation floor. Default 0.15 (~-16 dB) tames the - // over-suppression "underwater"/pumping artifact; host can tune via - // `lotusDenoiseFloor` (0 = full suppression, no floor). - floor: - rawFloor !== null && Number.isFinite(Number(rawFloor)) - ? Math.min(0.5, Math.max(0, Number(rawFloor))) - : 0.15, + const browserNsAtJoin = lotusParam("noiseSuppression") !== "false"; + // `let`: io.lotus.set_denoise replaces it mid-call. + let target: DenoiseTarget = { + mode: mlAtJoin ? "ml" : browserNsAtJoin ? "browser" : "off", + nativeNS: mlAtJoin && browserNsAtJoin, + config: { + model, + assetBase: safeAssetBase(lotusParam("lotusDenoiseBase")), + gate: lotusFlag("lotusGate"), + // Default -45 (matches the reference shim); accept an explicit 0 (don't + // coerce it away via `|| default`). + gateThreshold: + rawThreshold !== null && Number.isFinite(Number(rawThreshold)) + ? Number(rawThreshold) + : -45, + // Dry/wet attenuation floor. Default 0.15 (~-16 dB) tames the + // over-suppression "underwater"/pumping artifact; host can tune via + // `lotusDenoiseFloor` (0 = full suppression, no floor). + floor: + rawFloor !== null && Number.isFinite(Number(rawFloor)) + ? Math.min(0.5, Math.max(0, Number(rawFloor))) + : 0.15, + }, }; // [lotus #7] Start fetching wasm / creating the AudioContext / addModule-ing // the worklet (and loading the DFN model) NOW, before any mic track exists. // `setProcessor()` holds LiveKit's trackChangeLock while awaiting `init()`, // so anything still loading there freezes mute/unmute/device-switch. - void prepareDenoiseAssets(config); + if (target.mode === "ml") void prepareDenoiseAssets(target.config); const micOf = (room: LivekitRoom): LocalAudioTrack | undefined => room.localParticipant.getTrackPublication(Track.Source.Microphone) @@ -127,7 +217,7 @@ export function startLotusDenoise(vm: CallViewModel): () => void { mic: LocalAudioTrack, model: LotusDenoiseModel, ): Promise => { - const processor = new LotusDenoiseProcessor({ ...config, model }); + const processor = new LotusDenoiseProcessor({ ...target.config, model }); pendingProcessors.set(room, processor); // [lotus #9] Seed the mute state so a processor attached while already // muted starts suspended rather than burning inference on silence. @@ -142,11 +232,15 @@ export function startLotusDenoise(vm: CallViewModel): () => void { } }; + // In-flight apply() per room, so a mid-call switch waits for it to settle. + const inflight = new Map>(); + const apply = (room: LivekitRoom): void => { + if (target.mode !== "ml") return; const mic = micOf(room); if (!mic || mic.getProcessor() || pendingProcessors.has(room)) return; - void (async () => { - let model = config.model; + const run = (async (): Promise => { + let model = target.config.model; try { try { await attach(room, mic, model); @@ -164,6 +258,7 @@ export function startLotusDenoise(vm: CallViewModel): () => void { lotusSendToHost(LotusWidgetActions.DenoiseState, { active: true, model, + mode: "ml", }); } catch (e) { logger.warn("[lotus] denoise unavailable; publishing raw mic", e); @@ -171,11 +266,69 @@ export function startLotusDenoise(vm: CallViewModel): () => void { active: false, error: e instanceof Error ? e.message : String(e), model, + mode: "ml", }); } })(); + inflight.set(room, run); + void run.finally(() => { + if (inflight.get(room) === run) inflight.delete(room); + }); }; + // ── Mid-call switching (io.lotus.set_denoise) ─────────────────────────── + // `applied` is what the mic is actually set up for. Switches run one at a + // time; each works from `applied` to the latest `target`, so a burst of + // changes collapses into the last one. + let applied: DenoiseTarget = target; + let queue: Promise = Promise.resolve(); + + const reconfigure = async (): Promise => { + const prev = applied; + const next = target; + if (prev === next || stopped) return; + const nsChanged = browserNsFor(prev) !== browserNsFor(next); + const swapProcessor = + prev.mode === "ml" && + (next.mode !== "ml" || !sameMlConfig(prev.config, next.config)); + setLotusNoiseSuppression(browserNsFor(next)); + // Load the new model while the old one keeps running, so the switch + // itself only re-wires the graph instead of sending raw audio meanwhile. + if (next.mode === "ml") await prepareDenoiseAssets(next.config); + for (const room of rooms) { + await inflight.get(room)?.catch(() => undefined); + const mic = micOf(room); + if (!mic) continue; + try { + if (swapProcessor && mic.getProcessor()) await mic.stopProcessor(); + if (nsChanged) await restartCapture(mic, browserNsFor(next)); + } catch (e) { + logger.warn("[lotus] set_denoise: re-capturing the mic failed", e); + } + apply(room); + } + applied = next; + if (next.mode !== "ml") + lotusSendToHost(LotusWidgetActions.DenoiseState, { + active: false, + mode: next.mode, + }); + }; + + const onSetDenoise = (ev: CustomEvent): void => { + w?.api.transport.reply(ev.detail, {}); + const next = applySetDenoise(target, ev.detail.data); + if (!next) return; + logger.info( + `[lotus] set_denoise: ${next.mode}${next.mode === "ml" ? ` (${next.config.model})` : ""}`, + ); + target = next; + queue = queue.then(reconfigure).catch((e) => { + logger.warn("[lotus] set_denoise failed", e); + }); + }; + w?.lazyActions.on(LotusWidgetActions.SetDenoise, onSetDenoise); + // [lotus #9] Suspend/resume the processor's own AudioContext with the mic's // mute state (LiveKit keeps the muted track flowing silent frames, so the // worklet would otherwise run inference the whole time the user is muted). @@ -231,6 +384,8 @@ export function startLotusDenoise(vm: CallViewModel): () => void { return () => { stopped = true; + w?.lazyActions.off(LotusWidgetActions.SetDenoise, onSetDenoise); + setLotusNoiseSuppression(null); sub.unsubscribe(); for (const off of roomListeners.values()) off(); roomListeners.clear(); diff --git a/src/lotus/lotusMicLevel.ts b/src/lotus/lotusMicLevel.ts index 3e148532..53b4852d 100644 --- a/src/lotus/lotusMicLevel.ts +++ b/src/lotus/lotusMicLevel.ts @@ -6,10 +6,12 @@ Please see LICENSE in the repository root for full details. */ import { + type LocalTrack, type LocalTrackPublication, type Room as LivekitRoom, RoomEvent, Track, + TrackEvent, } from "livekit-client"; import { Observable, type Subscription, share, switchMap } from "rxjs"; import { distinctUntilChanged, map } from "rxjs/operators"; @@ -37,15 +39,35 @@ export interface LocalMicSample { const micPublication = ( room: LivekitRoom, -): { track: MediaStreamTrack; muted: boolean } | null => { +): { track: MediaStreamTrack; local: LocalTrack; muted: boolean } | null => { const pub: LocalTrackPublication | undefined = room.localParticipant.getTrackPublication(Track.Source.Microphone); - const track = pub?.track?.mediaStreamTrack; - return track && track.readyState === "live" - ? { track, muted: pub?.isMuted ?? false } + const local = pub?.track; + const track = local?.mediaStreamTrack; + return local && track && track.readyState === "live" + ? { track, local, muted: pub?.isMuted ?? false } : null; }; +// Every live tap, so a mic re-capture can drop the clones first. +const liveTaps = new Set<{ release: () => void; reconcile: () => void }>(); + +/** + * Run a mic re-capture (`restartTrack`) with every level-meter clone stopped, + * then re-tap the new track. Chrome makes all open tracks of a device share + * one set of audio processing, so a clone still holding the old track makes + * the new capture ignore its `noiseSuppression` (or AGC/EC) constraint. + */ +export async function withMicTapsReleased(fn: () => Promise): Promise { + const taps = [...liveTaps]; + taps.forEach((t) => t.release()); + try { + return await fn(); + } finally { + taps.forEach((t) => t.reconcile()); + } +} + /** Pure: the same LiveKit rooms, in the same order (by identity). */ export const sameRooms = (a: readonly T[], b: readonly T[]): boolean => a.length === b.length && a.every((r, i) => r === b[i]); @@ -79,11 +101,17 @@ export function observeLocalMicSample$( let clone: MediaStreamTrack | null = null; let timer: ReturnType | undefined; let tapped: MediaStreamTrack | null = null; + // LiveKit swaps the MediaStreamTrack inside the same LocalTrack on + // restartTrack (device switch, re-capture) without any room event, + // so follow its Restarted event or the clone keeps the old one open. + let tappedLocal: LocalTrack | null = null; let muted = false; const stopTap = (): void => { if (timer !== undefined) clearInterval(timer); timer = undefined; + tappedLocal?.off(TrackEvent.Restarted, reconcile); + tappedLocal = null; clone?.stop(); clone = null; void ctx?.close().catch(() => undefined); @@ -92,7 +120,12 @@ export function observeLocalMicSample$( subscriber.next(null); }; - const startTap = (source: MediaStreamTrack): void => { + const startTap = ( + source: MediaStreamTrack, + local: LocalTrack, + ): void => { + tappedLocal = local; + local.on(TrackEvent.Restarted, reconcile); try { clone = source.clone(); // Muting disables the published track; the clone must still hear. @@ -116,15 +149,22 @@ export function observeLocalMicSample$( } }; - const reconcile = (): void => { + function reconcile(): void { const pub = rooms.map(micPublication).find((p) => p !== null) ?? null; muted = pub?.muted ?? false; const track = pub?.track ?? null; if (track === tapped) return; if (tapped) stopTap(); - if (track) startTap(track); + if (track && pub) startTap(track, pub.local); + } + const control = { + release: (): void => { + if (tapped) stopTap(); + }, + reconcile, }; + liveTaps.add(control); const events = [ RoomEvent.TrackMuted, @@ -143,6 +183,7 @@ export function observeLocalMicSample$( rooms.forEach((room) => events.forEach((ev) => room.off(ev, reconcile)), ); + liveTaps.delete(control); if (tapped) stopTap(); }; }), diff --git a/src/state/CallViewModel/remoteMembers/ConnectionFactory.ts b/src/state/CallViewModel/remoteMembers/ConnectionFactory.ts index 393c736e..7e1d18c0 100644 --- a/src/state/CallViewModel/remoteMembers/ConnectionFactory.ts +++ b/src/state/CallViewModel/remoteMembers/ConnectionFactory.ts @@ -47,6 +47,7 @@ import { // default to true, so with no params this is byte-for-byte upstream behaviour; // `?autoGainControl=false` forces AGC off regardless of the stored Setting. import { getUrlParams } from "../../../UrlParams.ts"; +import { lotusNoiseSuppressionAllowed } from "../../../lotus/lotusCaptureOverride.ts"; // TODO evaluate if this should be done like the Publisher Factory export interface ConnectionFactory { @@ -193,9 +194,16 @@ function generateRoomOption({ echoCancellation: echoCancellationSetting.getValue() && getUrlParams().echoCancellation !== false, + // [lotus] The host can switch this mid-call (io.lotus.set_denoise). noiseSuppression: noiseSuppressionSetting.getValue() && - getUrlParams().noiseSuppression !== false, + lotusNoiseSuppressionAllowed(getUrlParams().noiseSuppression), + // [lotus] LiveKit defaults voiceIsolation ON, which (where the platform + // implements it) is one more voice filter in front of the ML model or + // in "off" mode. Tie it to browser noise suppression. + voiceIsolation: + noiseSuppressionSetting.getValue() && + lotusNoiseSuppressionAllowed(getUrlParams().noiseSuppression), autoGainControl: autoGainControlSetting.getValue() && getUrlParams().autoGainControl !== false, -- 2.47.3