feat(denoise): switch noise suppression mid-call (io.lotus.set_denoise)
The host can now change noise suppression on the live mic without a rejoin: off / browser-native / ML, the ML model, the gate and stacking browser NS in front of the model. - Model swaps preload the new model while the old one keeps running, then swap the processor (no mic re-capture). - Browser NS on/off re-captures the mic via restartTrack with the new constraint (passing the deviceId, without which LiveKit drops every constraint). The choice is kept in lotusCaptureOverride, which ConnectionFactory reads, so a later SFU reconnect captures the same way. - Switches are serialized; a burst collapses into the last one. Each one reports io.lotus.denoise_state (now with `mode`). Fixes found while testing in Chromium: - The mic level meter (and muted-speech nudge) kept a clone of the ORIGINAL mic track and never noticed LiveKit's restartTrack. Chrome makes all open tracks of a device share audio processing, so every re-capture silently kept the old noiseSuppression (asked false, got true). The meter now drops its clone around our re-capture and re-taps on TrackEvent.Restarted, which also fixes it listening to the old device after a mic switch. - LiveKit defaults voiceIsolation ON, a further voice filter in "off" and ML modes where the platform implements it. It now follows browser noise suppression. Verified in a two-browser call: joined as browser and as ML(dtln), then switched dtln -> deepfilternet -> rnnoise -> browser -> off; each step applied within ~0.5 s and every re-capture got the constraints it asked for. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_017PCb59uY3UKqDqt7B6hzd5
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
9e6fe75f14
commit
949103d649
@@ -9,7 +9,12 @@ import { afterEach, beforeEach, describe, expect, test, vi } from "vitest";
|
||||
import { ParticipantEvent, Track } from "livekit-client";
|
||||
|
||||
import { type CallViewModel } from "../state/CallViewModel/CallViewModel";
|
||||
import { startLotusDenoise } from "./lotusDenoise";
|
||||
import {
|
||||
applySetDenoise,
|
||||
browserNsFor,
|
||||
type DenoiseTarget,
|
||||
startLotusDenoise,
|
||||
} from "./lotusDenoise";
|
||||
import { LotusWidgetActions } from "./lotusActions";
|
||||
|
||||
// Track constructed instances so tests can assert exactly one processor was
|
||||
@@ -42,9 +47,25 @@ vi.mock("./lotusDenoiseProcessor", () => ({
|
||||
releasePreparedDenoiseAssets: vi.fn().mockResolvedValue(undefined),
|
||||
}));
|
||||
const hostSend = vi.hoisted(() => vi.fn().mockResolvedValue({}));
|
||||
// Host → widget actions: tests call `sendAction()` to deliver one.
|
||||
const actionHandlers = vi.hoisted(
|
||||
() => new Map<string, (ev: CustomEvent) => void>(),
|
||||
);
|
||||
vi.mock("../widget", () => ({
|
||||
widget: { api: { transport: { send: hostSend } } },
|
||||
widget: {
|
||||
api: { transport: { send: hostSend, reply: vi.fn() } },
|
||||
lazyActions: {
|
||||
on: (name: string, cb: (ev: CustomEvent) => void): void => {
|
||||
actionHandlers.set(name, cb);
|
||||
},
|
||||
off: (name: string): void => {
|
||||
actionHandlers.delete(name);
|
||||
},
|
||||
},
|
||||
},
|
||||
}));
|
||||
const sendAction = (name: string, data: unknown): void =>
|
||||
actionHandlers.get(name)?.(new CustomEvent(name, { detail: { data } }));
|
||||
|
||||
/** A promise plus externally-callable resolve, for controlling ordering. */
|
||||
function deferred<T>(): {
|
||||
@@ -110,6 +131,7 @@ beforeEach(() => {
|
||||
instances.length = 0;
|
||||
calls.length = 0;
|
||||
hostSend.mockClear();
|
||||
actionHandlers.clear();
|
||||
// `lotusParam`/`lotusFlag` cache the URL params at first read; seed the hash
|
||||
// before startLotusDenoise() so the dedicated `lotusDenoiseSource` flag reads on.
|
||||
window.location.hash = "#/room?lotusDenoiseSource=1";
|
||||
@@ -188,6 +210,7 @@ describe("startLotusDenoise", () => {
|
||||
expect(hostSend).toHaveBeenCalledWith(LotusWidgetActions.DenoiseState, {
|
||||
active: true,
|
||||
model: "rnnoise",
|
||||
mode: "ml",
|
||||
});
|
||||
});
|
||||
|
||||
@@ -217,6 +240,7 @@ describe("startLotusDenoise", () => {
|
||||
expect(hostSend).toHaveBeenCalledWith(LotusWidgetActions.DenoiseState, {
|
||||
active: true,
|
||||
model: "rnnoise",
|
||||
mode: "ml",
|
||||
});
|
||||
});
|
||||
|
||||
@@ -239,6 +263,7 @@ describe("startLotusDenoise", () => {
|
||||
active: false,
|
||||
error: "no worklet for you",
|
||||
model: "rnnoise",
|
||||
mode: "ml",
|
||||
});
|
||||
});
|
||||
|
||||
@@ -272,3 +297,157 @@ describe("startLotusDenoise", () => {
|
||||
expect(instances[0].setMicMuted).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
/** A mic whose processor and capture constraints behave like LiveKit's. */
|
||||
function liveMic(): {
|
||||
getProcessor: () => { config: { model: string } } | undefined;
|
||||
setProcessor: ReturnType<typeof vi.fn>;
|
||||
stopProcessor: ReturnType<typeof vi.fn>;
|
||||
restartTrack: ReturnType<typeof vi.fn>;
|
||||
constraints: MediaTrackConstraints;
|
||||
mediaStreamTrack: { getSettings: () => MediaTrackSettings };
|
||||
isMuted: boolean;
|
||||
} {
|
||||
let attached: { config: { model: string } } | undefined;
|
||||
return {
|
||||
isMuted: false,
|
||||
getProcessor: () => attached,
|
||||
setProcessor: vi.fn(async (p: { config: { model: string } }) => {
|
||||
await Promise.resolve();
|
||||
attached = p;
|
||||
}),
|
||||
stopProcessor: vi.fn(async () => {
|
||||
await Promise.resolve();
|
||||
attached = undefined;
|
||||
}),
|
||||
restartTrack: vi.fn().mockResolvedValue(undefined),
|
||||
constraints: { echoCancellation: true, autoGainControl: true },
|
||||
mediaStreamTrack: { getSettings: () => ({ deviceId: "mic-1" }) },
|
||||
};
|
||||
}
|
||||
|
||||
describe("io.lotus.set_denoise (switch mid-call)", () => {
|
||||
test("swaps the ML model on the live mic without re-capturing it", async () => {
|
||||
window.location.hash =
|
||||
"#/room?lotusDenoiseSource=1&lotusModel=dtln&noiseSuppression=false";
|
||||
const mic = liveMic();
|
||||
const { vm } = makeRoomAndVm(mic);
|
||||
startLotusDenoise(vm);
|
||||
await flush();
|
||||
expect(mic.getProcessor()?.config.model).toBe("dtln");
|
||||
|
||||
sendAction(LotusWidgetActions.SetDenoise, {
|
||||
mode: "ml",
|
||||
model: "deepfilternet",
|
||||
});
|
||||
await flush();
|
||||
await flush();
|
||||
|
||||
expect(mic.stopProcessor).toHaveBeenCalledTimes(1);
|
||||
expect(mic.getProcessor()?.config.model).toBe("deepfilternet");
|
||||
expect(mic.restartTrack).not.toHaveBeenCalled();
|
||||
expect(hostSend).toHaveBeenLastCalledWith(LotusWidgetActions.DenoiseState, {
|
||||
active: true,
|
||||
model: "deepfilternet",
|
||||
mode: "ml",
|
||||
});
|
||||
});
|
||||
|
||||
test("ML → browser-native drops the model and re-captures with browser NS on", async () => {
|
||||
window.location.hash =
|
||||
"#/room?lotusDenoiseSource=1&lotusModel=rnnoise&noiseSuppression=false";
|
||||
const mic = liveMic();
|
||||
const { vm } = makeRoomAndVm(mic);
|
||||
startLotusDenoise(vm);
|
||||
await flush();
|
||||
|
||||
sendAction(LotusWidgetActions.SetDenoise, { mode: "browser" });
|
||||
await flush();
|
||||
await flush();
|
||||
|
||||
expect(mic.stopProcessor).toHaveBeenCalledTimes(1);
|
||||
expect(mic.getProcessor()).toBeUndefined();
|
||||
expect(mic.restartTrack).toHaveBeenCalledWith({
|
||||
deviceId: "mic-1",
|
||||
channelCount: 1,
|
||||
echoCancellation: true,
|
||||
autoGainControl: true,
|
||||
noiseSuppression: true,
|
||||
voiceIsolation: true,
|
||||
});
|
||||
expect(hostSend).toHaveBeenLastCalledWith(LotusWidgetActions.DenoiseState, {
|
||||
active: false,
|
||||
mode: "browser",
|
||||
});
|
||||
});
|
||||
|
||||
test("browser-native → ML works even when the call joined without ML", async () => {
|
||||
window.location.hash = "#/room?noiseSuppression=true";
|
||||
const mic = liveMic();
|
||||
const { vm } = makeRoomAndVm(mic);
|
||||
startLotusDenoise(vm);
|
||||
await flush();
|
||||
expect(mic.setProcessor).not.toHaveBeenCalled();
|
||||
|
||||
sendAction(LotusWidgetActions.SetDenoise, { mode: "ml", model: "speex" });
|
||||
await flush();
|
||||
await flush();
|
||||
|
||||
expect(mic.restartTrack).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ noiseSuppression: false }),
|
||||
);
|
||||
expect(mic.getProcessor()?.config.model).toBe("speex");
|
||||
});
|
||||
|
||||
test("ignores a payload without a valid mode", async () => {
|
||||
window.location.hash = "#/room?lotusDenoiseSource=1";
|
||||
const mic = liveMic();
|
||||
const { vm } = makeRoomAndVm(mic);
|
||||
startLotusDenoise(vm);
|
||||
await flush();
|
||||
sendAction(LotusWidgetActions.SetDenoise, { mode: "loud" });
|
||||
await flush();
|
||||
expect(mic.stopProcessor).not.toHaveBeenCalled();
|
||||
expect(mic.restartTrack).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe("applySetDenoise", () => {
|
||||
const base: DenoiseTarget = {
|
||||
mode: "ml",
|
||||
nativeNS: false,
|
||||
config: {
|
||||
model: "rnnoise",
|
||||
assetBase: "https://x/denoise/",
|
||||
gate: false,
|
||||
gateThreshold: -45,
|
||||
floor: 0.15,
|
||||
},
|
||||
};
|
||||
|
||||
test("keeps fields the payload doesn't name or gets wrong", () => {
|
||||
expect(
|
||||
applySetDenoise(base, { mode: "ml", model: "nope", gate: "yes" }),
|
||||
).toEqual(base);
|
||||
});
|
||||
|
||||
test("moves only the named fields", () => {
|
||||
const next = applySetDenoise(base, {
|
||||
mode: "ml",
|
||||
model: "dtln",
|
||||
gateThreshold: -50,
|
||||
});
|
||||
expect(next?.config).toEqual({
|
||||
...base.config,
|
||||
model: "dtln",
|
||||
gateThreshold: -50,
|
||||
});
|
||||
});
|
||||
|
||||
test("browser NS is on for browser mode and for ML with nativeNS", () => {
|
||||
expect(browserNsFor({ ...base, mode: "browser" })).toBe(true);
|
||||
expect(browserNsFor({ ...base, mode: "off" })).toBe(false);
|
||||
expect(browserNsFor(base)).toBe(false);
|
||||
expect(browserNsFor({ ...base, nativeNS: true })).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user