feat(denoise): switch noise suppression mid-call (io.lotus.set_denoise)
CI / Build embedded bundle (push) Skipped
CI / Build embedded bundle (pull_request) Successful in 1m47s
CI / Publish to Gitea npm registry (pull_request) Skipped

The host can now change noise suppression on the live mic without a
rejoin: off / browser-native / ML, the ML model, the gate and stacking
browser NS in front of the model.

- Model swaps preload the new model while the old one keeps running,
  then swap the processor (no mic re-capture).
- Browser NS on/off re-captures the mic via restartTrack with the new
  constraint (passing the deviceId, without which LiveKit drops every
  constraint). The choice is kept in lotusCaptureOverride, which
  ConnectionFactory reads, so a later SFU reconnect captures the same way.
- Switches are serialized; a burst collapses into the last one. Each one
  reports io.lotus.denoise_state (now with `mode`).

Fixes found while testing in Chromium:
- The mic level meter (and muted-speech nudge) kept a clone of the
  ORIGINAL mic track and never noticed LiveKit's restartTrack. Chrome
  makes all open tracks of a device share audio processing, so every
  re-capture silently kept the old noiseSuppression (asked false, got
  true). The meter now drops its clone around our re-capture and
  re-taps on TrackEvent.Restarted, which also fixes it listening to the
  old device after a mic switch.
- LiveKit defaults voiceIsolation ON, a further voice filter in "off"
  and ML modes where the platform implements it. It now follows browser
  noise suppression.

Verified in a two-browser call: joined as browser and as ML(dtln), then
switched dtln -> deepfilternet -> rnnoise -> browser -> off; each step
applied within ~0.5 s and every re-capture got the constraints it asked
for.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_017PCb59uY3UKqDqt7B6hzd5
This commit is contained in:
Lotus CI
2026-10-30 21:13:40 -04:00
co-authored by Claude Opus 5.5
parent 9e6fe75f14
commit 949103d649
8 changed files with 474 additions and 40 deletions
+181 -2
View File
@@ -9,7 +9,12 @@ import { afterEach, beforeEach, describe, expect, test, vi } from "vitest";
import { ParticipantEvent, Track } from "livekit-client";
import { type CallViewModel } from "../state/CallViewModel/CallViewModel";
import { startLotusDenoise } from "./lotusDenoise";
import {
applySetDenoise,
browserNsFor,
type DenoiseTarget,
startLotusDenoise,
} from "./lotusDenoise";
import { LotusWidgetActions } from "./lotusActions";
// Track constructed instances so tests can assert exactly one processor was
@@ -42,9 +47,25 @@ vi.mock("./lotusDenoiseProcessor", () => ({
releasePreparedDenoiseAssets: vi.fn().mockResolvedValue(undefined),
}));
const hostSend = vi.hoisted(() => vi.fn().mockResolvedValue({}));
// Host → widget actions: tests call `sendAction()` to deliver one.
const actionHandlers = vi.hoisted(
() => new Map<string, (ev: CustomEvent) => void>(),
);
vi.mock("../widget", () => ({
widget: { api: { transport: { send: hostSend } } },
widget: {
api: { transport: { send: hostSend, reply: vi.fn() } },
lazyActions: {
on: (name: string, cb: (ev: CustomEvent) => void): void => {
actionHandlers.set(name, cb);
},
off: (name: string): void => {
actionHandlers.delete(name);
},
},
},
}));
const sendAction = (name: string, data: unknown): void =>
actionHandlers.get(name)?.(new CustomEvent(name, { detail: { data } }));
/** A promise plus externally-callable resolve, for controlling ordering. */
function deferred<T>(): {
@@ -110,6 +131,7 @@ beforeEach(() => {
instances.length = 0;
calls.length = 0;
hostSend.mockClear();
actionHandlers.clear();
// `lotusParam`/`lotusFlag` cache the URL params at first read; seed the hash
// before startLotusDenoise() so the dedicated `lotusDenoiseSource` flag reads on.
window.location.hash = "#/room?lotusDenoiseSource=1";
@@ -188,6 +210,7 @@ describe("startLotusDenoise", () => {
expect(hostSend).toHaveBeenCalledWith(LotusWidgetActions.DenoiseState, {
active: true,
model: "rnnoise",
mode: "ml",
});
});
@@ -217,6 +240,7 @@ describe("startLotusDenoise", () => {
expect(hostSend).toHaveBeenCalledWith(LotusWidgetActions.DenoiseState, {
active: true,
model: "rnnoise",
mode: "ml",
});
});
@@ -239,6 +263,7 @@ describe("startLotusDenoise", () => {
active: false,
error: "no worklet for you",
model: "rnnoise",
mode: "ml",
});
});
@@ -272,3 +297,157 @@ describe("startLotusDenoise", () => {
expect(instances[0].setMicMuted).not.toHaveBeenCalled();
});
});
/** A mic whose processor and capture constraints behave like LiveKit's. */
function liveMic(): {
getProcessor: () => { config: { model: string } } | undefined;
setProcessor: ReturnType<typeof vi.fn>;
stopProcessor: ReturnType<typeof vi.fn>;
restartTrack: ReturnType<typeof vi.fn>;
constraints: MediaTrackConstraints;
mediaStreamTrack: { getSettings: () => MediaTrackSettings };
isMuted: boolean;
} {
let attached: { config: { model: string } } | undefined;
return {
isMuted: false,
getProcessor: () => attached,
setProcessor: vi.fn(async (p: { config: { model: string } }) => {
await Promise.resolve();
attached = p;
}),
stopProcessor: vi.fn(async () => {
await Promise.resolve();
attached = undefined;
}),
restartTrack: vi.fn().mockResolvedValue(undefined),
constraints: { echoCancellation: true, autoGainControl: true },
mediaStreamTrack: { getSettings: () => ({ deviceId: "mic-1" }) },
};
}
describe("io.lotus.set_denoise (switch mid-call)", () => {
test("swaps the ML model on the live mic without re-capturing it", async () => {
window.location.hash =
"#/room?lotusDenoiseSource=1&lotusModel=dtln&noiseSuppression=false";
const mic = liveMic();
const { vm } = makeRoomAndVm(mic);
startLotusDenoise(vm);
await flush();
expect(mic.getProcessor()?.config.model).toBe("dtln");
sendAction(LotusWidgetActions.SetDenoise, {
mode: "ml",
model: "deepfilternet",
});
await flush();
await flush();
expect(mic.stopProcessor).toHaveBeenCalledTimes(1);
expect(mic.getProcessor()?.config.model).toBe("deepfilternet");
expect(mic.restartTrack).not.toHaveBeenCalled();
expect(hostSend).toHaveBeenLastCalledWith(LotusWidgetActions.DenoiseState, {
active: true,
model: "deepfilternet",
mode: "ml",
});
});
test("ML → browser-native drops the model and re-captures with browser NS on", async () => {
window.location.hash =
"#/room?lotusDenoiseSource=1&lotusModel=rnnoise&noiseSuppression=false";
const mic = liveMic();
const { vm } = makeRoomAndVm(mic);
startLotusDenoise(vm);
await flush();
sendAction(LotusWidgetActions.SetDenoise, { mode: "browser" });
await flush();
await flush();
expect(mic.stopProcessor).toHaveBeenCalledTimes(1);
expect(mic.getProcessor()).toBeUndefined();
expect(mic.restartTrack).toHaveBeenCalledWith({
deviceId: "mic-1",
channelCount: 1,
echoCancellation: true,
autoGainControl: true,
noiseSuppression: true,
voiceIsolation: true,
});
expect(hostSend).toHaveBeenLastCalledWith(LotusWidgetActions.DenoiseState, {
active: false,
mode: "browser",
});
});
test("browser-native → ML works even when the call joined without ML", async () => {
window.location.hash = "#/room?noiseSuppression=true";
const mic = liveMic();
const { vm } = makeRoomAndVm(mic);
startLotusDenoise(vm);
await flush();
expect(mic.setProcessor).not.toHaveBeenCalled();
sendAction(LotusWidgetActions.SetDenoise, { mode: "ml", model: "speex" });
await flush();
await flush();
expect(mic.restartTrack).toHaveBeenCalledWith(
expect.objectContaining({ noiseSuppression: false }),
);
expect(mic.getProcessor()?.config.model).toBe("speex");
});
test("ignores a payload without a valid mode", async () => {
window.location.hash = "#/room?lotusDenoiseSource=1";
const mic = liveMic();
const { vm } = makeRoomAndVm(mic);
startLotusDenoise(vm);
await flush();
sendAction(LotusWidgetActions.SetDenoise, { mode: "loud" });
await flush();
expect(mic.stopProcessor).not.toHaveBeenCalled();
expect(mic.restartTrack).not.toHaveBeenCalled();
});
});
describe("applySetDenoise", () => {
const base: DenoiseTarget = {
mode: "ml",
nativeNS: false,
config: {
model: "rnnoise",
assetBase: "https://x/denoise/",
gate: false,
gateThreshold: -45,
floor: 0.15,
},
};
test("keeps fields the payload doesn't name or gets wrong", () => {
expect(
applySetDenoise(base, { mode: "ml", model: "nope", gate: "yes" }),
).toEqual(base);
});
test("moves only the named fields", () => {
const next = applySetDenoise(base, {
mode: "ml",
model: "dtln",
gateThreshold: -50,
});
expect(next?.config).toEqual({
...base.config,
model: "dtln",
gateThreshold: -50,
});
});
test("browser NS is on for browser mode and for ML with nativeNS", () => {
expect(browserNsFor({ ...base, mode: "browser" })).toBe(true);
expect(browserNsFor({ ...base, mode: "off" })).toBe(false);
expect(browserNsFor(base)).toBe(false);
expect(browserNsFor({ ...base, nativeNS: true })).toBe(true);
});
});