feat(denoise): switch noise suppression mid-call (+ fix re-captures ignoring noiseSuppression) #66
@@ -36,6 +36,7 @@ describe("LotusWidgetActions", () => {
|
||||
LotusWidgetActions.PromptScreenshare,
|
||||
LotusWidgetActions.Reconnect,
|
||||
LotusWidgetActions.DeviceDecision,
|
||||
LotusWidgetActions.SetDenoise,
|
||||
];
|
||||
|
||||
expect(new Set(LOTUS_TO_WIDGET_ACTIONS)).toEqual(new Set(expectedToWidget));
|
||||
|
||||
@@ -41,11 +41,19 @@ export enum LotusWidgetActions {
|
||||
RequestState = "io.lotus.request_state",
|
||||
/**
|
||||
* fromWidget: real state of the in-source denoise engine —
|
||||
* `{ active: boolean, model: string, error?: string }` — sent once the
|
||||
* `{ active: boolean, model: string, mode: "ml", error?: string }` (or
|
||||
* `{ active: false, mode }` after a switch away from ML) — sent once the
|
||||
* processor attaches (or after the rnnoise fallback also fails), so the host
|
||||
* toggle can reflect reality rather than the requested state.
|
||||
*/
|
||||
DenoiseState = "io.lotus.denoise_state",
|
||||
/**
|
||||
* toWidget: switch noise suppression mid-call —
|
||||
* `{ mode: "off" | "browser" | "ml", model?, nativeNS?, gate?, gateThreshold? }`.
|
||||
* `model`/`gate`/`gateThreshold` only matter for "ml"; `nativeNS` stacks the
|
||||
* browser suppressor in front of the model. Omitted fields keep their value.
|
||||
*/
|
||||
SetDenoise = "io.lotus.set_denoise",
|
||||
/**
|
||||
* fromWidget: one-shot end-of-call readout for the local participant —
|
||||
* `{ durationMs, reconnects, poorMs, verdict }` (#143) — sent on SFU
|
||||
@@ -163,4 +171,5 @@ export const LOTUS_TO_WIDGET_ACTIONS: LotusWidgetActions[] = [
|
||||
LotusWidgetActions.PromptScreenshare,
|
||||
LotusWidgetActions.Reconnect,
|
||||
LotusWidgetActions.DeviceDecision,
|
||||
LotusWidgetActions.SetDenoise,
|
||||
];
|
||||
|
||||
@@ -25,6 +25,7 @@ import {
|
||||
echoCancellationSetting,
|
||||
noiseSuppressionSetting,
|
||||
} from "../settings/settings.ts";
|
||||
import { setLotusNoiseSuppression } from "./lotusCaptureOverride.ts";
|
||||
|
||||
// [lotus] Upstream v0.25.0 moved the audio-capture constraints to Settings.
|
||||
// The Lotus host still drives them per-call via URL params (it turns browser
|
||||
@@ -83,6 +84,7 @@ beforeEach(() => {
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
setLotusNoiseSuppression(null);
|
||||
testScope.end();
|
||||
vi.mocked(LivekitRoom).mockClear();
|
||||
});
|
||||
@@ -123,6 +125,7 @@ describe("[lotus] audio-capture URL param overrides", () => {
|
||||
expect(capturedAudioDefaults()).toMatchObject({
|
||||
echoCancellation: true,
|
||||
noiseSuppression: false,
|
||||
voiceIsolation: false,
|
||||
autoGainControl: false,
|
||||
});
|
||||
});
|
||||
@@ -137,4 +140,16 @@ describe("[lotus] audio-capture URL param overrides", () => {
|
||||
createRoom();
|
||||
expect(capturedAudioDefaults()).toMatchObject({ echoCancellation: false });
|
||||
});
|
||||
|
||||
test("a mid-call set_denoise choice beats the noiseSuppression the call joined with", () => {
|
||||
getUrlParams.mockReturnValue({ noiseSuppression: false });
|
||||
setLotusNoiseSuppression(true);
|
||||
createRoom();
|
||||
expect(capturedAudioDefaults()).toMatchObject({ noiseSuppression: true });
|
||||
|
||||
getUrlParams.mockReturnValue({ noiseSuppression: true });
|
||||
setLotusNoiseSuppression(false);
|
||||
createRoom();
|
||||
expect(capturedAudioDefaults()).toMatchObject({ noiseSuppression: false });
|
||||
});
|
||||
});
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
/*
|
||||
Copyright 2026 Lotus Guild
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
/**
|
||||
* Browser noise suppression chosen live by the host (`io.lotus.set_denoise`),
|
||||
* overriding the `noiseSuppression` URL param the call was opened with. Read by
|
||||
* ConnectionFactory, so a connection created after a mid-call switch (e.g. an
|
||||
* SFU reconnect) captures the mic the way the user last picked, not the way
|
||||
* they joined. Dependency-free so ConnectionFactory can import it cheaply.
|
||||
*/
|
||||
let noiseSuppressionOverride: boolean | null = null;
|
||||
|
||||
export function setLotusNoiseSuppression(on: boolean | null): void {
|
||||
noiseSuppressionOverride = on;
|
||||
}
|
||||
|
||||
/** The override if the host set one, else the URL param (default on). */
|
||||
export function lotusNoiseSuppressionAllowed(
|
||||
urlParam: boolean | undefined,
|
||||
): boolean {
|
||||
return noiseSuppressionOverride ?? urlParam !== false;
|
||||
}
|
||||
@@ -9,7 +9,12 @@ import { afterEach, beforeEach, describe, expect, test, vi } from "vitest";
|
||||
import { ParticipantEvent, Track } from "livekit-client";
|
||||
|
||||
import { type CallViewModel } from "../state/CallViewModel/CallViewModel";
|
||||
import { startLotusDenoise } from "./lotusDenoise";
|
||||
import {
|
||||
applySetDenoise,
|
||||
browserNsFor,
|
||||
type DenoiseTarget,
|
||||
startLotusDenoise,
|
||||
} from "./lotusDenoise";
|
||||
import { LotusWidgetActions } from "./lotusActions";
|
||||
|
||||
// Track constructed instances so tests can assert exactly one processor was
|
||||
@@ -42,9 +47,25 @@ vi.mock("./lotusDenoiseProcessor", () => ({
|
||||
releasePreparedDenoiseAssets: vi.fn().mockResolvedValue(undefined),
|
||||
}));
|
||||
const hostSend = vi.hoisted(() => vi.fn().mockResolvedValue({}));
|
||||
// Host → widget actions: tests call `sendAction()` to deliver one.
|
||||
const actionHandlers = vi.hoisted(
|
||||
() => new Map<string, (ev: CustomEvent) => void>(),
|
||||
);
|
||||
vi.mock("../widget", () => ({
|
||||
widget: { api: { transport: { send: hostSend } } },
|
||||
widget: {
|
||||
api: { transport: { send: hostSend, reply: vi.fn() } },
|
||||
lazyActions: {
|
||||
on: (name: string, cb: (ev: CustomEvent) => void): void => {
|
||||
actionHandlers.set(name, cb);
|
||||
},
|
||||
off: (name: string): void => {
|
||||
actionHandlers.delete(name);
|
||||
},
|
||||
},
|
||||
},
|
||||
}));
|
||||
const sendAction = (name: string, data: unknown): void =>
|
||||
actionHandlers.get(name)?.(new CustomEvent(name, { detail: { data } }));
|
||||
|
||||
/** A promise plus externally-callable resolve, for controlling ordering. */
|
||||
function deferred<T>(): {
|
||||
@@ -110,6 +131,7 @@ beforeEach(() => {
|
||||
instances.length = 0;
|
||||
calls.length = 0;
|
||||
hostSend.mockClear();
|
||||
actionHandlers.clear();
|
||||
// `lotusParam`/`lotusFlag` cache the URL params at first read; seed the hash
|
||||
// before startLotusDenoise() so the dedicated `lotusDenoiseSource` flag reads on.
|
||||
window.location.hash = "#/room?lotusDenoiseSource=1";
|
||||
@@ -188,6 +210,7 @@ describe("startLotusDenoise", () => {
|
||||
expect(hostSend).toHaveBeenCalledWith(LotusWidgetActions.DenoiseState, {
|
||||
active: true,
|
||||
model: "rnnoise",
|
||||
mode: "ml",
|
||||
});
|
||||
});
|
||||
|
||||
@@ -217,6 +240,7 @@ describe("startLotusDenoise", () => {
|
||||
expect(hostSend).toHaveBeenCalledWith(LotusWidgetActions.DenoiseState, {
|
||||
active: true,
|
||||
model: "rnnoise",
|
||||
mode: "ml",
|
||||
});
|
||||
});
|
||||
|
||||
@@ -239,6 +263,7 @@ describe("startLotusDenoise", () => {
|
||||
active: false,
|
||||
error: "no worklet for you",
|
||||
model: "rnnoise",
|
||||
mode: "ml",
|
||||
});
|
||||
});
|
||||
|
||||
@@ -272,3 +297,157 @@ describe("startLotusDenoise", () => {
|
||||
expect(instances[0].setMicMuted).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
/** A mic whose processor and capture constraints behave like LiveKit's. */
|
||||
function liveMic(): {
|
||||
getProcessor: () => { config: { model: string } } | undefined;
|
||||
setProcessor: ReturnType<typeof vi.fn>;
|
||||
stopProcessor: ReturnType<typeof vi.fn>;
|
||||
restartTrack: ReturnType<typeof vi.fn>;
|
||||
constraints: MediaTrackConstraints;
|
||||
mediaStreamTrack: { getSettings: () => MediaTrackSettings };
|
||||
isMuted: boolean;
|
||||
} {
|
||||
let attached: { config: { model: string } } | undefined;
|
||||
return {
|
||||
isMuted: false,
|
||||
getProcessor: () => attached,
|
||||
setProcessor: vi.fn(async (p: { config: { model: string } }) => {
|
||||
await Promise.resolve();
|
||||
attached = p;
|
||||
}),
|
||||
stopProcessor: vi.fn(async () => {
|
||||
await Promise.resolve();
|
||||
attached = undefined;
|
||||
}),
|
||||
restartTrack: vi.fn().mockResolvedValue(undefined),
|
||||
constraints: { echoCancellation: true, autoGainControl: true },
|
||||
mediaStreamTrack: { getSettings: () => ({ deviceId: "mic-1" }) },
|
||||
};
|
||||
}
|
||||
|
||||
describe("io.lotus.set_denoise (switch mid-call)", () => {
|
||||
test("swaps the ML model on the live mic without re-capturing it", async () => {
|
||||
window.location.hash =
|
||||
"#/room?lotusDenoiseSource=1&lotusModel=dtln&noiseSuppression=false";
|
||||
const mic = liveMic();
|
||||
const { vm } = makeRoomAndVm(mic);
|
||||
startLotusDenoise(vm);
|
||||
await flush();
|
||||
expect(mic.getProcessor()?.config.model).toBe("dtln");
|
||||
|
||||
sendAction(LotusWidgetActions.SetDenoise, {
|
||||
mode: "ml",
|
||||
model: "deepfilternet",
|
||||
});
|
||||
await flush();
|
||||
await flush();
|
||||
|
||||
expect(mic.stopProcessor).toHaveBeenCalledTimes(1);
|
||||
expect(mic.getProcessor()?.config.model).toBe("deepfilternet");
|
||||
expect(mic.restartTrack).not.toHaveBeenCalled();
|
||||
expect(hostSend).toHaveBeenLastCalledWith(LotusWidgetActions.DenoiseState, {
|
||||
active: true,
|
||||
model: "deepfilternet",
|
||||
mode: "ml",
|
||||
});
|
||||
});
|
||||
|
||||
test("ML → browser-native drops the model and re-captures with browser NS on", async () => {
|
||||
window.location.hash =
|
||||
"#/room?lotusDenoiseSource=1&lotusModel=rnnoise&noiseSuppression=false";
|
||||
const mic = liveMic();
|
||||
const { vm } = makeRoomAndVm(mic);
|
||||
startLotusDenoise(vm);
|
||||
await flush();
|
||||
|
||||
sendAction(LotusWidgetActions.SetDenoise, { mode: "browser" });
|
||||
await flush();
|
||||
await flush();
|
||||
|
||||
expect(mic.stopProcessor).toHaveBeenCalledTimes(1);
|
||||
expect(mic.getProcessor()).toBeUndefined();
|
||||
expect(mic.restartTrack).toHaveBeenCalledWith({
|
||||
deviceId: "mic-1",
|
||||
channelCount: 1,
|
||||
echoCancellation: true,
|
||||
autoGainControl: true,
|
||||
noiseSuppression: true,
|
||||
voiceIsolation: true,
|
||||
});
|
||||
expect(hostSend).toHaveBeenLastCalledWith(LotusWidgetActions.DenoiseState, {
|
||||
active: false,
|
||||
mode: "browser",
|
||||
});
|
||||
});
|
||||
|
||||
test("browser-native → ML works even when the call joined without ML", async () => {
|
||||
window.location.hash = "#/room?noiseSuppression=true";
|
||||
const mic = liveMic();
|
||||
const { vm } = makeRoomAndVm(mic);
|
||||
startLotusDenoise(vm);
|
||||
await flush();
|
||||
expect(mic.setProcessor).not.toHaveBeenCalled();
|
||||
|
||||
sendAction(LotusWidgetActions.SetDenoise, { mode: "ml", model: "speex" });
|
||||
await flush();
|
||||
await flush();
|
||||
|
||||
expect(mic.restartTrack).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ noiseSuppression: false }),
|
||||
);
|
||||
expect(mic.getProcessor()?.config.model).toBe("speex");
|
||||
});
|
||||
|
||||
test("ignores a payload without a valid mode", async () => {
|
||||
window.location.hash = "#/room?lotusDenoiseSource=1";
|
||||
const mic = liveMic();
|
||||
const { vm } = makeRoomAndVm(mic);
|
||||
startLotusDenoise(vm);
|
||||
await flush();
|
||||
sendAction(LotusWidgetActions.SetDenoise, { mode: "loud" });
|
||||
await flush();
|
||||
expect(mic.stopProcessor).not.toHaveBeenCalled();
|
||||
expect(mic.restartTrack).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe("applySetDenoise", () => {
|
||||
const base: DenoiseTarget = {
|
||||
mode: "ml",
|
||||
nativeNS: false,
|
||||
config: {
|
||||
model: "rnnoise",
|
||||
assetBase: "https://x/denoise/",
|
||||
gate: false,
|
||||
gateThreshold: -45,
|
||||
floor: 0.15,
|
||||
},
|
||||
};
|
||||
|
||||
test("keeps fields the payload doesn't name or gets wrong", () => {
|
||||
expect(
|
||||
applySetDenoise(base, { mode: "ml", model: "nope", gate: "yes" }),
|
||||
).toEqual(base);
|
||||
});
|
||||
|
||||
test("moves only the named fields", () => {
|
||||
const next = applySetDenoise(base, {
|
||||
mode: "ml",
|
||||
model: "dtln",
|
||||
gateThreshold: -50,
|
||||
});
|
||||
expect(next?.config).toEqual({
|
||||
...base.config,
|
||||
model: "dtln",
|
||||
gateThreshold: -50,
|
||||
});
|
||||
});
|
||||
|
||||
test("browser NS is on for browser mode and for ML with nativeNS", () => {
|
||||
expect(browserNsFor({ ...base, mode: "browser" })).toBe(true);
|
||||
expect(browserNsFor({ ...base, mode: "off" })).toBe(false);
|
||||
expect(browserNsFor(base)).toBe(false);
|
||||
expect(browserNsFor({ ...base, nativeNS: true })).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
+184
-29
@@ -13,8 +13,12 @@ import {
|
||||
type TrackPublication,
|
||||
} from "livekit-client";
|
||||
import { logger } from "matrix-js-sdk/lib/logger";
|
||||
import { type IWidgetApiRequest } from "matrix-widget-api";
|
||||
|
||||
import { type CallViewModel } from "../state/CallViewModel/CallViewModel";
|
||||
import { widget } from "../widget";
|
||||
import { setLotusNoiseSuppression } from "./lotusCaptureOverride";
|
||||
import { withMicTapsReleased } from "./lotusMicLevel";
|
||||
import {
|
||||
LotusWidgetActions,
|
||||
lotusFlag,
|
||||
@@ -64,49 +68,135 @@ function safeAssetBase(raw: string | null): string {
|
||||
}
|
||||
}
|
||||
|
||||
export type LotusNoiseMode = "off" | "browser" | "ml";
|
||||
|
||||
/** What the host wants: browser suppression and/or an ML model. */
|
||||
export interface DenoiseTarget {
|
||||
mode: LotusNoiseMode;
|
||||
/** Stack the browser's suppressor in front of the model (ML only). */
|
||||
nativeNS: boolean;
|
||||
config: LotusDenoiseConfig;
|
||||
}
|
||||
|
||||
const MODELS: ReadonlySet<string> = new Set<LotusDenoiseModel>([
|
||||
"rnnoise",
|
||||
"speex",
|
||||
"dtln",
|
||||
"deepfilternet",
|
||||
]);
|
||||
const isModel = (v: unknown): v is LotusDenoiseModel =>
|
||||
typeof v === "string" && MODELS.has(v);
|
||||
|
||||
/** Whether the mic should be captured with the browser's noise suppression. */
|
||||
export const browserNsFor = (t: DenoiseTarget): boolean =>
|
||||
t.mode === "browser" || (t.mode === "ml" && t.nativeNS);
|
||||
|
||||
/**
|
||||
* Apply an `io.lotus.set_denoise` payload to the current target. Unknown or
|
||||
* malformed fields keep their current value, so a partial payload only moves
|
||||
* what it names; a payload with no valid `mode` is ignored (returns null).
|
||||
*/
|
||||
export function applySetDenoise(
|
||||
current: DenoiseTarget,
|
||||
data: unknown,
|
||||
): DenoiseTarget | null {
|
||||
const d = (data ?? {}) as Record<string, unknown>;
|
||||
if (d.mode !== "off" && d.mode !== "browser" && d.mode !== "ml") return null;
|
||||
const model = isModel(d.model) ? d.model : current.config.model;
|
||||
return {
|
||||
mode: d.mode,
|
||||
nativeNS: typeof d.nativeNS === "boolean" ? d.nativeNS : current.nativeNS,
|
||||
config: {
|
||||
...current.config,
|
||||
model,
|
||||
gate: typeof d.gate === "boolean" ? d.gate : current.config.gate,
|
||||
gateThreshold:
|
||||
typeof d.gateThreshold === "number" && Number.isFinite(d.gateThreshold)
|
||||
? d.gateThreshold
|
||||
: current.config.gateThreshold,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
const sameMlConfig = (a: LotusDenoiseConfig, b: LotusDenoiseConfig): boolean =>
|
||||
a.model === b.model &&
|
||||
a.gate === b.gate &&
|
||||
a.gateThreshold === b.gateThreshold &&
|
||||
a.floor === b.floor;
|
||||
|
||||
/**
|
||||
* Re-acquire the mic with browser noise suppression on or off, keeping the
|
||||
* rest of its constraints. LiveKit re-attaches any processor to the new track;
|
||||
* the level-meter clone is dropped first so the new constraint actually takes.
|
||||
* The deviceId must be passed: without one LiveKit requests `audio: true` and
|
||||
* silently drops every other constraint.
|
||||
*/
|
||||
async function restartCapture(
|
||||
mic: LocalAudioTrack,
|
||||
noiseSuppression: boolean,
|
||||
): Promise<void> {
|
||||
const c = mic.constraints;
|
||||
await withMicTapsReleased(async () => {
|
||||
await mic.restartTrack({
|
||||
deviceId: c.deviceId ?? mic.mediaStreamTrack.getSettings().deviceId,
|
||||
channelCount: c.channelCount ?? 1,
|
||||
echoCancellation: c.echoCancellation,
|
||||
autoGainControl: c.autoGainControl,
|
||||
noiseSuppression,
|
||||
voiceIsolation: noiseSuppression,
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
export function startLotusDenoise(vm: CallViewModel): () => void {
|
||||
// Gate on a DEDICATED flag — NOT the existing `lotusDenoise=ml` that the
|
||||
// host's build-time getUserMedia shim already uses. Otherwise simply shipping
|
||||
// this fork (while the host still injects its shim and still sets
|
||||
// lotusDenoise=ml) would denoise twice. The host opts into the in-source
|
||||
// engine with `lotusDenoiseSource=1` AND stops injecting the shim at the same
|
||||
// time. Default off ⇒ the fork is inert and behaviour is unchanged.
|
||||
if (!lotusFlag("lotusDenoiseSource")) return () => undefined;
|
||||
// time. Without the flag the engine stays idle unless the host switches to ML
|
||||
// mid-call (io.lotus.set_denoise), which needs the widget API.
|
||||
const w = widget;
|
||||
const mlAtJoin = lotusFlag("lotusDenoiseSource");
|
||||
if (!mlAtJoin && !w) return () => undefined;
|
||||
|
||||
const requested = lotusParam("lotusModel");
|
||||
const model: LotusDenoiseConfig["model"] =
|
||||
requested === "speex" ||
|
||||
requested === "dtln" ||
|
||||
requested === "deepfilternet"
|
||||
? requested
|
||||
: "rnnoise";
|
||||
const model: LotusDenoiseConfig["model"] = isModel(requested)
|
||||
? requested
|
||||
: "rnnoise";
|
||||
|
||||
const rawThreshold = lotusParam("lotusGateThreshold");
|
||||
const rawFloor = lotusParam("lotusDenoiseFloor");
|
||||
const config: LotusDenoiseConfig = {
|
||||
model,
|
||||
assetBase: safeAssetBase(lotusParam("lotusDenoiseBase")),
|
||||
gate: lotusFlag("lotusGate"),
|
||||
// Default -45 (matches the reference shim); accept an explicit 0 (don't
|
||||
// coerce it away via `|| default`).
|
||||
gateThreshold:
|
||||
rawThreshold !== null && Number.isFinite(Number(rawThreshold))
|
||||
? Number(rawThreshold)
|
||||
: -45,
|
||||
// Dry/wet attenuation floor. Default 0.15 (~-16 dB) tames the
|
||||
// over-suppression "underwater"/pumping artifact; host can tune via
|
||||
// `lotusDenoiseFloor` (0 = full suppression, no floor).
|
||||
floor:
|
||||
rawFloor !== null && Number.isFinite(Number(rawFloor))
|
||||
? Math.min(0.5, Math.max(0, Number(rawFloor)))
|
||||
: 0.15,
|
||||
const browserNsAtJoin = lotusParam("noiseSuppression") !== "false";
|
||||
// `let`: io.lotus.set_denoise replaces it mid-call.
|
||||
let target: DenoiseTarget = {
|
||||
mode: mlAtJoin ? "ml" : browserNsAtJoin ? "browser" : "off",
|
||||
nativeNS: mlAtJoin && browserNsAtJoin,
|
||||
config: {
|
||||
model,
|
||||
assetBase: safeAssetBase(lotusParam("lotusDenoiseBase")),
|
||||
gate: lotusFlag("lotusGate"),
|
||||
// Default -45 (matches the reference shim); accept an explicit 0 (don't
|
||||
// coerce it away via `|| default`).
|
||||
gateThreshold:
|
||||
rawThreshold !== null && Number.isFinite(Number(rawThreshold))
|
||||
? Number(rawThreshold)
|
||||
: -45,
|
||||
// Dry/wet attenuation floor. Default 0.15 (~-16 dB) tames the
|
||||
// over-suppression "underwater"/pumping artifact; host can tune via
|
||||
// `lotusDenoiseFloor` (0 = full suppression, no floor).
|
||||
floor:
|
||||
rawFloor !== null && Number.isFinite(Number(rawFloor))
|
||||
? Math.min(0.5, Math.max(0, Number(rawFloor)))
|
||||
: 0.15,
|
||||
},
|
||||
};
|
||||
|
||||
// [lotus #7] Start fetching wasm / creating the AudioContext / addModule-ing
|
||||
// the worklet (and loading the DFN model) NOW, before any mic track exists.
|
||||
// `setProcessor()` holds LiveKit's trackChangeLock while awaiting `init()`,
|
||||
// so anything still loading there freezes mute/unmute/device-switch.
|
||||
void prepareDenoiseAssets(config);
|
||||
if (target.mode === "ml") void prepareDenoiseAssets(target.config);
|
||||
|
||||
const micOf = (room: LivekitRoom): LocalAudioTrack | undefined =>
|
||||
room.localParticipant.getTrackPublication(Track.Source.Microphone)
|
||||
@@ -127,7 +217,7 @@ export function startLotusDenoise(vm: CallViewModel): () => void {
|
||||
mic: LocalAudioTrack,
|
||||
model: LotusDenoiseModel,
|
||||
): Promise<void> => {
|
||||
const processor = new LotusDenoiseProcessor({ ...config, model });
|
||||
const processor = new LotusDenoiseProcessor({ ...target.config, model });
|
||||
pendingProcessors.set(room, processor);
|
||||
// [lotus #9] Seed the mute state so a processor attached while already
|
||||
// muted starts suspended rather than burning inference on silence.
|
||||
@@ -142,11 +232,15 @@ export function startLotusDenoise(vm: CallViewModel): () => void {
|
||||
}
|
||||
};
|
||||
|
||||
// In-flight apply() per room, so a mid-call switch waits for it to settle.
|
||||
const inflight = new Map<LivekitRoom, Promise<void>>();
|
||||
|
||||
const apply = (room: LivekitRoom): void => {
|
||||
if (target.mode !== "ml") return;
|
||||
const mic = micOf(room);
|
||||
if (!mic || mic.getProcessor() || pendingProcessors.has(room)) return;
|
||||
void (async () => {
|
||||
let model = config.model;
|
||||
const run = (async (): Promise<void> => {
|
||||
let model = target.config.model;
|
||||
try {
|
||||
try {
|
||||
await attach(room, mic, model);
|
||||
@@ -164,6 +258,7 @@ export function startLotusDenoise(vm: CallViewModel): () => void {
|
||||
lotusSendToHost(LotusWidgetActions.DenoiseState, {
|
||||
active: true,
|
||||
model,
|
||||
mode: "ml",
|
||||
});
|
||||
} catch (e) {
|
||||
logger.warn("[lotus] denoise unavailable; publishing raw mic", e);
|
||||
@@ -171,11 +266,69 @@ export function startLotusDenoise(vm: CallViewModel): () => void {
|
||||
active: false,
|
||||
error: e instanceof Error ? e.message : String(e),
|
||||
model,
|
||||
mode: "ml",
|
||||
});
|
||||
}
|
||||
})();
|
||||
inflight.set(room, run);
|
||||
void run.finally(() => {
|
||||
if (inflight.get(room) === run) inflight.delete(room);
|
||||
});
|
||||
};
|
||||
|
||||
// ── Mid-call switching (io.lotus.set_denoise) ───────────────────────────
|
||||
// `applied` is what the mic is actually set up for. Switches run one at a
|
||||
// time; each works from `applied` to the latest `target`, so a burst of
|
||||
// changes collapses into the last one.
|
||||
let applied: DenoiseTarget = target;
|
||||
let queue: Promise<void> = Promise.resolve();
|
||||
|
||||
const reconfigure = async (): Promise<void> => {
|
||||
const prev = applied;
|
||||
const next = target;
|
||||
if (prev === next || stopped) return;
|
||||
const nsChanged = browserNsFor(prev) !== browserNsFor(next);
|
||||
const swapProcessor =
|
||||
prev.mode === "ml" &&
|
||||
(next.mode !== "ml" || !sameMlConfig(prev.config, next.config));
|
||||
setLotusNoiseSuppression(browserNsFor(next));
|
||||
// Load the new model while the old one keeps running, so the switch
|
||||
// itself only re-wires the graph instead of sending raw audio meanwhile.
|
||||
if (next.mode === "ml") await prepareDenoiseAssets(next.config);
|
||||
for (const room of rooms) {
|
||||
await inflight.get(room)?.catch(() => undefined);
|
||||
const mic = micOf(room);
|
||||
if (!mic) continue;
|
||||
try {
|
||||
if (swapProcessor && mic.getProcessor()) await mic.stopProcessor();
|
||||
if (nsChanged) await restartCapture(mic, browserNsFor(next));
|
||||
} catch (e) {
|
||||
logger.warn("[lotus] set_denoise: re-capturing the mic failed", e);
|
||||
}
|
||||
apply(room);
|
||||
}
|
||||
applied = next;
|
||||
if (next.mode !== "ml")
|
||||
lotusSendToHost(LotusWidgetActions.DenoiseState, {
|
||||
active: false,
|
||||
mode: next.mode,
|
||||
});
|
||||
};
|
||||
|
||||
const onSetDenoise = (ev: CustomEvent<IWidgetApiRequest>): void => {
|
||||
w?.api.transport.reply(ev.detail, {});
|
||||
const next = applySetDenoise(target, ev.detail.data);
|
||||
if (!next) return;
|
||||
logger.info(
|
||||
`[lotus] set_denoise: ${next.mode}${next.mode === "ml" ? ` (${next.config.model})` : ""}`,
|
||||
);
|
||||
target = next;
|
||||
queue = queue.then(reconfigure).catch((e) => {
|
||||
logger.warn("[lotus] set_denoise failed", e);
|
||||
});
|
||||
};
|
||||
w?.lazyActions.on(LotusWidgetActions.SetDenoise, onSetDenoise);
|
||||
|
||||
// [lotus #9] Suspend/resume the processor's own AudioContext with the mic's
|
||||
// mute state (LiveKit keeps the muted track flowing silent frames, so the
|
||||
// worklet would otherwise run inference the whole time the user is muted).
|
||||
@@ -231,6 +384,8 @@ export function startLotusDenoise(vm: CallViewModel): () => void {
|
||||
|
||||
return () => {
|
||||
stopped = true;
|
||||
w?.lazyActions.off(LotusWidgetActions.SetDenoise, onSetDenoise);
|
||||
setLotusNoiseSuppression(null);
|
||||
sub.unsubscribe();
|
||||
for (const off of roomListeners.values()) off();
|
||||
roomListeners.clear();
|
||||
|
||||
@@ -6,10 +6,12 @@ Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import {
|
||||
type LocalTrack,
|
||||
type LocalTrackPublication,
|
||||
type Room as LivekitRoom,
|
||||
RoomEvent,
|
||||
Track,
|
||||
TrackEvent,
|
||||
} from "livekit-client";
|
||||
import { Observable, type Subscription, share, switchMap } from "rxjs";
|
||||
import { distinctUntilChanged, map } from "rxjs/operators";
|
||||
@@ -37,15 +39,35 @@ export interface LocalMicSample {
|
||||
|
||||
const micPublication = (
|
||||
room: LivekitRoom,
|
||||
): { track: MediaStreamTrack; muted: boolean } | null => {
|
||||
): { track: MediaStreamTrack; local: LocalTrack; muted: boolean } | null => {
|
||||
const pub: LocalTrackPublication | undefined =
|
||||
room.localParticipant.getTrackPublication(Track.Source.Microphone);
|
||||
const track = pub?.track?.mediaStreamTrack;
|
||||
return track && track.readyState === "live"
|
||||
? { track, muted: pub?.isMuted ?? false }
|
||||
const local = pub?.track;
|
||||
const track = local?.mediaStreamTrack;
|
||||
return local && track && track.readyState === "live"
|
||||
? { track, local, muted: pub?.isMuted ?? false }
|
||||
: null;
|
||||
};
|
||||
|
||||
// Every live tap, so a mic re-capture can drop the clones first.
|
||||
const liveTaps = new Set<{ release: () => void; reconcile: () => void }>();
|
||||
|
||||
/**
|
||||
* Run a mic re-capture (`restartTrack`) with every level-meter clone stopped,
|
||||
* then re-tap the new track. Chrome makes all open tracks of a device share
|
||||
* one set of audio processing, so a clone still holding the old track makes
|
||||
* the new capture ignore its `noiseSuppression` (or AGC/EC) constraint.
|
||||
*/
|
||||
export async function withMicTapsReleased<T>(fn: () => Promise<T>): Promise<T> {
|
||||
const taps = [...liveTaps];
|
||||
taps.forEach((t) => t.release());
|
||||
try {
|
||||
return await fn();
|
||||
} finally {
|
||||
taps.forEach((t) => t.reconcile());
|
||||
}
|
||||
}
|
||||
|
||||
/** Pure: the same LiveKit rooms, in the same order (by identity). */
|
||||
export const sameRooms = <T>(a: readonly T[], b: readonly T[]): boolean =>
|
||||
a.length === b.length && a.every((r, i) => r === b[i]);
|
||||
@@ -79,11 +101,17 @@ export function observeLocalMicSample$(
|
||||
let clone: MediaStreamTrack | null = null;
|
||||
let timer: ReturnType<typeof setInterval> | undefined;
|
||||
let tapped: MediaStreamTrack | null = null;
|
||||
// LiveKit swaps the MediaStreamTrack inside the same LocalTrack on
|
||||
// restartTrack (device switch, re-capture) without any room event,
|
||||
// so follow its Restarted event or the clone keeps the old one open.
|
||||
let tappedLocal: LocalTrack | null = null;
|
||||
let muted = false;
|
||||
|
||||
const stopTap = (): void => {
|
||||
if (timer !== undefined) clearInterval(timer);
|
||||
timer = undefined;
|
||||
tappedLocal?.off(TrackEvent.Restarted, reconcile);
|
||||
tappedLocal = null;
|
||||
clone?.stop();
|
||||
clone = null;
|
||||
void ctx?.close().catch(() => undefined);
|
||||
@@ -92,7 +120,12 @@ export function observeLocalMicSample$(
|
||||
subscriber.next(null);
|
||||
};
|
||||
|
||||
const startTap = (source: MediaStreamTrack): void => {
|
||||
const startTap = (
|
||||
source: MediaStreamTrack,
|
||||
local: LocalTrack,
|
||||
): void => {
|
||||
tappedLocal = local;
|
||||
local.on(TrackEvent.Restarted, reconcile);
|
||||
try {
|
||||
clone = source.clone();
|
||||
// Muting disables the published track; the clone must still hear.
|
||||
@@ -116,15 +149,22 @@ export function observeLocalMicSample$(
|
||||
}
|
||||
};
|
||||
|
||||
const reconcile = (): void => {
|
||||
function reconcile(): void {
|
||||
const pub =
|
||||
rooms.map(micPublication).find((p) => p !== null) ?? null;
|
||||
muted = pub?.muted ?? false;
|
||||
const track = pub?.track ?? null;
|
||||
if (track === tapped) return;
|
||||
if (tapped) stopTap();
|
||||
if (track) startTap(track);
|
||||
if (track && pub) startTap(track, pub.local);
|
||||
}
|
||||
const control = {
|
||||
release: (): void => {
|
||||
if (tapped) stopTap();
|
||||
},
|
||||
reconcile,
|
||||
};
|
||||
liveTaps.add(control);
|
||||
|
||||
const events = [
|
||||
RoomEvent.TrackMuted,
|
||||
@@ -143,6 +183,7 @@ export function observeLocalMicSample$(
|
||||
rooms.forEach((room) =>
|
||||
events.forEach((ev) => room.off(ev, reconcile)),
|
||||
);
|
||||
liveTaps.delete(control);
|
||||
if (tapped) stopTap();
|
||||
};
|
||||
}),
|
||||
|
||||
@@ -47,6 +47,7 @@ import {
|
||||
// default to true, so with no params this is byte-for-byte upstream behaviour;
|
||||
// `?autoGainControl=false` forces AGC off regardless of the stored Setting.
|
||||
import { getUrlParams } from "../../../UrlParams.ts";
|
||||
import { lotusNoiseSuppressionAllowed } from "../../../lotus/lotusCaptureOverride.ts";
|
||||
|
||||
// TODO evaluate if this should be done like the Publisher Factory
|
||||
export interface ConnectionFactory {
|
||||
@@ -193,9 +194,16 @@ function generateRoomOption({
|
||||
echoCancellation:
|
||||
echoCancellationSetting.getValue() &&
|
||||
getUrlParams().echoCancellation !== false,
|
||||
// [lotus] The host can switch this mid-call (io.lotus.set_denoise).
|
||||
noiseSuppression:
|
||||
noiseSuppressionSetting.getValue() &&
|
||||
getUrlParams().noiseSuppression !== false,
|
||||
lotusNoiseSuppressionAllowed(getUrlParams().noiseSuppression),
|
||||
// [lotus] LiveKit defaults voiceIsolation ON, which (where the platform
|
||||
// implements it) is one more voice filter in front of the ML model or
|
||||
// in "off" mode. Tie it to browser noise suppression.
|
||||
voiceIsolation:
|
||||
noiseSuppressionSetting.getValue() &&
|
||||
lotusNoiseSuppressionAllowed(getUrlParams().noiseSuppression),
|
||||
autoGainControl:
|
||||
autoGainControlSetting.getValue() &&
|
||||
getUrlParams().autoGainControl !== false,
|
||||
|
||||
Reference in New Issue
Block a user