feat(denoise): switch noise suppression mid-call (+ fix re-captures ignoring noiseSuppression) #66

Merged
jared merged 1 commits from feat/live-denoise-switch into lotus 2026-10-30 21:15:44 -04:00
8 changed files with 474 additions and 40 deletions
+1
View File
@@ -36,6 +36,7 @@ describe("LotusWidgetActions", () => {
LotusWidgetActions.PromptScreenshare,
LotusWidgetActions.Reconnect,
LotusWidgetActions.DeviceDecision,
LotusWidgetActions.SetDenoise,
];
expect(new Set(LOTUS_TO_WIDGET_ACTIONS)).toEqual(new Set(expectedToWidget));
+10 -1
View File
@@ -41,11 +41,19 @@ export enum LotusWidgetActions {
RequestState = "io.lotus.request_state",
/**
* fromWidget: real state of the in-source denoise engine —
* `{ active: boolean, model: string, error?: string }` — sent once the
* `{ active: boolean, model: string, mode: "ml", error?: string }` (or
* `{ active: false, mode }` after a switch away from ML) — sent once the
* processor attaches (or after the rnnoise fallback also fails), so the host
* toggle can reflect reality rather than the requested state.
*/
DenoiseState = "io.lotus.denoise_state",
/**
* toWidget: switch noise suppression mid-call —
* `{ mode: "off" | "browser" | "ml", model?, nativeNS?, gate?, gateThreshold? }`.
* `model`/`gate`/`gateThreshold` only matter for "ml"; `nativeNS` stacks the
* browser suppressor in front of the model. Omitted fields keep their value.
*/
SetDenoise = "io.lotus.set_denoise",
/**
* fromWidget: one-shot end-of-call readout for the local participant —
* `{ durationMs, reconnects, poorMs, verdict }` (#143) — sent on SFU
@@ -163,4 +171,5 @@ export const LOTUS_TO_WIDGET_ACTIONS: LotusWidgetActions[] = [
LotusWidgetActions.PromptScreenshare,
LotusWidgetActions.Reconnect,
LotusWidgetActions.DeviceDecision,
LotusWidgetActions.SetDenoise,
];
+15
View File
@@ -25,6 +25,7 @@ import {
echoCancellationSetting,
noiseSuppressionSetting,
} from "../settings/settings.ts";
import { setLotusNoiseSuppression } from "./lotusCaptureOverride.ts";
// [lotus] Upstream v0.25.0 moved the audio-capture constraints to Settings.
// The Lotus host still drives them per-call via URL params (it turns browser
@@ -83,6 +84,7 @@ beforeEach(() => {
});
afterEach(() => {
setLotusNoiseSuppression(null);
testScope.end();
vi.mocked(LivekitRoom).mockClear();
});
@@ -123,6 +125,7 @@ describe("[lotus] audio-capture URL param overrides", () => {
expect(capturedAudioDefaults()).toMatchObject({
echoCancellation: true,
noiseSuppression: false,
voiceIsolation: false,
autoGainControl: false,
});
});
@@ -137,4 +140,16 @@ describe("[lotus] audio-capture URL param overrides", () => {
createRoom();
expect(capturedAudioDefaults()).toMatchObject({ echoCancellation: false });
});
test("a mid-call set_denoise choice beats the noiseSuppression the call joined with", () => {
getUrlParams.mockReturnValue({ noiseSuppression: false });
setLotusNoiseSuppression(true);
createRoom();
expect(capturedAudioDefaults()).toMatchObject({ noiseSuppression: true });
getUrlParams.mockReturnValue({ noiseSuppression: true });
setLotusNoiseSuppression(false);
createRoom();
expect(capturedAudioDefaults()).toMatchObject({ noiseSuppression: false });
});
});
+26
View File
@@ -0,0 +1,26 @@
/*
Copyright 2026 Lotus Guild
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
/**
* Browser noise suppression chosen live by the host (`io.lotus.set_denoise`),
* overriding the `noiseSuppression` URL param the call was opened with. Read by
* ConnectionFactory, so a connection created after a mid-call switch (e.g. an
* SFU reconnect) captures the mic the way the user last picked, not the way
* they joined. Dependency-free so ConnectionFactory can import it cheaply.
*/
let noiseSuppressionOverride: boolean | null = null;
export function setLotusNoiseSuppression(on: boolean | null): void {
noiseSuppressionOverride = on;
}
/** The override if the host set one, else the URL param (default on). */
export function lotusNoiseSuppressionAllowed(
urlParam: boolean | undefined,
): boolean {
return noiseSuppressionOverride ?? urlParam !== false;
}
+181 -2
View File
@@ -9,7 +9,12 @@ import { afterEach, beforeEach, describe, expect, test, vi } from "vitest";
import { ParticipantEvent, Track } from "livekit-client";
import { type CallViewModel } from "../state/CallViewModel/CallViewModel";
import { startLotusDenoise } from "./lotusDenoise";
import {
applySetDenoise,
browserNsFor,
type DenoiseTarget,
startLotusDenoise,
} from "./lotusDenoise";
import { LotusWidgetActions } from "./lotusActions";
// Track constructed instances so tests can assert exactly one processor was
@@ -42,9 +47,25 @@ vi.mock("./lotusDenoiseProcessor", () => ({
releasePreparedDenoiseAssets: vi.fn().mockResolvedValue(undefined),
}));
const hostSend = vi.hoisted(() => vi.fn().mockResolvedValue({}));
// Host → widget actions: tests call `sendAction()` to deliver one.
const actionHandlers = vi.hoisted(
() => new Map<string, (ev: CustomEvent) => void>(),
);
vi.mock("../widget", () => ({
widget: { api: { transport: { send: hostSend } } },
widget: {
api: { transport: { send: hostSend, reply: vi.fn() } },
lazyActions: {
on: (name: string, cb: (ev: CustomEvent) => void): void => {
actionHandlers.set(name, cb);
},
off: (name: string): void => {
actionHandlers.delete(name);
},
},
},
}));
const sendAction = (name: string, data: unknown): void =>
actionHandlers.get(name)?.(new CustomEvent(name, { detail: { data } }));
/** A promise plus externally-callable resolve, for controlling ordering. */
function deferred<T>(): {
@@ -110,6 +131,7 @@ beforeEach(() => {
instances.length = 0;
calls.length = 0;
hostSend.mockClear();
actionHandlers.clear();
// `lotusParam`/`lotusFlag` cache the URL params at first read; seed the hash
// before startLotusDenoise() so the dedicated `lotusDenoiseSource` flag reads on.
window.location.hash = "#/room?lotusDenoiseSource=1";
@@ -188,6 +210,7 @@ describe("startLotusDenoise", () => {
expect(hostSend).toHaveBeenCalledWith(LotusWidgetActions.DenoiseState, {
active: true,
model: "rnnoise",
mode: "ml",
});
});
@@ -217,6 +240,7 @@ describe("startLotusDenoise", () => {
expect(hostSend).toHaveBeenCalledWith(LotusWidgetActions.DenoiseState, {
active: true,
model: "rnnoise",
mode: "ml",
});
});
@@ -239,6 +263,7 @@ describe("startLotusDenoise", () => {
active: false,
error: "no worklet for you",
model: "rnnoise",
mode: "ml",
});
});
@@ -272,3 +297,157 @@ describe("startLotusDenoise", () => {
expect(instances[0].setMicMuted).not.toHaveBeenCalled();
});
});
/** A mic whose processor and capture constraints behave like LiveKit's. */
function liveMic(): {
getProcessor: () => { config: { model: string } } | undefined;
setProcessor: ReturnType<typeof vi.fn>;
stopProcessor: ReturnType<typeof vi.fn>;
restartTrack: ReturnType<typeof vi.fn>;
constraints: MediaTrackConstraints;
mediaStreamTrack: { getSettings: () => MediaTrackSettings };
isMuted: boolean;
} {
let attached: { config: { model: string } } | undefined;
return {
isMuted: false,
getProcessor: () => attached,
setProcessor: vi.fn(async (p: { config: { model: string } }) => {
await Promise.resolve();
attached = p;
}),
stopProcessor: vi.fn(async () => {
await Promise.resolve();
attached = undefined;
}),
restartTrack: vi.fn().mockResolvedValue(undefined),
constraints: { echoCancellation: true, autoGainControl: true },
mediaStreamTrack: { getSettings: () => ({ deviceId: "mic-1" }) },
};
}
describe("io.lotus.set_denoise (switch mid-call)", () => {
test("swaps the ML model on the live mic without re-capturing it", async () => {
window.location.hash =
"#/room?lotusDenoiseSource=1&lotusModel=dtln&noiseSuppression=false";
const mic = liveMic();
const { vm } = makeRoomAndVm(mic);
startLotusDenoise(vm);
await flush();
expect(mic.getProcessor()?.config.model).toBe("dtln");
sendAction(LotusWidgetActions.SetDenoise, {
mode: "ml",
model: "deepfilternet",
});
await flush();
await flush();
expect(mic.stopProcessor).toHaveBeenCalledTimes(1);
expect(mic.getProcessor()?.config.model).toBe("deepfilternet");
expect(mic.restartTrack).not.toHaveBeenCalled();
expect(hostSend).toHaveBeenLastCalledWith(LotusWidgetActions.DenoiseState, {
active: true,
model: "deepfilternet",
mode: "ml",
});
});
test("ML → browser-native drops the model and re-captures with browser NS on", async () => {
window.location.hash =
"#/room?lotusDenoiseSource=1&lotusModel=rnnoise&noiseSuppression=false";
const mic = liveMic();
const { vm } = makeRoomAndVm(mic);
startLotusDenoise(vm);
await flush();
sendAction(LotusWidgetActions.SetDenoise, { mode: "browser" });
await flush();
await flush();
expect(mic.stopProcessor).toHaveBeenCalledTimes(1);
expect(mic.getProcessor()).toBeUndefined();
expect(mic.restartTrack).toHaveBeenCalledWith({
deviceId: "mic-1",
channelCount: 1,
echoCancellation: true,
autoGainControl: true,
noiseSuppression: true,
voiceIsolation: true,
});
expect(hostSend).toHaveBeenLastCalledWith(LotusWidgetActions.DenoiseState, {
active: false,
mode: "browser",
});
});
test("browser-native → ML works even when the call joined without ML", async () => {
window.location.hash = "#/room?noiseSuppression=true";
const mic = liveMic();
const { vm } = makeRoomAndVm(mic);
startLotusDenoise(vm);
await flush();
expect(mic.setProcessor).not.toHaveBeenCalled();
sendAction(LotusWidgetActions.SetDenoise, { mode: "ml", model: "speex" });
await flush();
await flush();
expect(mic.restartTrack).toHaveBeenCalledWith(
expect.objectContaining({ noiseSuppression: false }),
);
expect(mic.getProcessor()?.config.model).toBe("speex");
});
test("ignores a payload without a valid mode", async () => {
window.location.hash = "#/room?lotusDenoiseSource=1";
const mic = liveMic();
const { vm } = makeRoomAndVm(mic);
startLotusDenoise(vm);
await flush();
sendAction(LotusWidgetActions.SetDenoise, { mode: "loud" });
await flush();
expect(mic.stopProcessor).not.toHaveBeenCalled();
expect(mic.restartTrack).not.toHaveBeenCalled();
});
});
describe("applySetDenoise", () => {
const base: DenoiseTarget = {
mode: "ml",
nativeNS: false,
config: {
model: "rnnoise",
assetBase: "https://x/denoise/",
gate: false,
gateThreshold: -45,
floor: 0.15,
},
};
test("keeps fields the payload doesn't name or gets wrong", () => {
expect(
applySetDenoise(base, { mode: "ml", model: "nope", gate: "yes" }),
).toEqual(base);
});
test("moves only the named fields", () => {
const next = applySetDenoise(base, {
mode: "ml",
model: "dtln",
gateThreshold: -50,
});
expect(next?.config).toEqual({
...base.config,
model: "dtln",
gateThreshold: -50,
});
});
test("browser NS is on for browser mode and for ML with nativeNS", () => {
expect(browserNsFor({ ...base, mode: "browser" })).toBe(true);
expect(browserNsFor({ ...base, mode: "off" })).toBe(false);
expect(browserNsFor(base)).toBe(false);
expect(browserNsFor({ ...base, nativeNS: true })).toBe(true);
});
});
+184 -29
View File
@@ -13,8 +13,12 @@ import {
type TrackPublication,
} from "livekit-client";
import { logger } from "matrix-js-sdk/lib/logger";
import { type IWidgetApiRequest } from "matrix-widget-api";
import { type CallViewModel } from "../state/CallViewModel/CallViewModel";
import { widget } from "../widget";
import { setLotusNoiseSuppression } from "./lotusCaptureOverride";
import { withMicTapsReleased } from "./lotusMicLevel";
import {
LotusWidgetActions,
lotusFlag,
@@ -64,49 +68,135 @@ function safeAssetBase(raw: string | null): string {
}
}
export type LotusNoiseMode = "off" | "browser" | "ml";
/** What the host wants: browser suppression and/or an ML model. */
export interface DenoiseTarget {
mode: LotusNoiseMode;
/** Stack the browser's suppressor in front of the model (ML only). */
nativeNS: boolean;
config: LotusDenoiseConfig;
}
const MODELS: ReadonlySet<string> = new Set<LotusDenoiseModel>([
"rnnoise",
"speex",
"dtln",
"deepfilternet",
]);
const isModel = (v: unknown): v is LotusDenoiseModel =>
typeof v === "string" && MODELS.has(v);
/** Whether the mic should be captured with the browser's noise suppression. */
export const browserNsFor = (t: DenoiseTarget): boolean =>
t.mode === "browser" || (t.mode === "ml" && t.nativeNS);
/**
* Apply an `io.lotus.set_denoise` payload to the current target. Unknown or
* malformed fields keep their current value, so a partial payload only moves
* what it names; a payload with no valid `mode` is ignored (returns null).
*/
export function applySetDenoise(
current: DenoiseTarget,
data: unknown,
): DenoiseTarget | null {
const d = (data ?? {}) as Record<string, unknown>;
if (d.mode !== "off" && d.mode !== "browser" && d.mode !== "ml") return null;
const model = isModel(d.model) ? d.model : current.config.model;
return {
mode: d.mode,
nativeNS: typeof d.nativeNS === "boolean" ? d.nativeNS : current.nativeNS,
config: {
...current.config,
model,
gate: typeof d.gate === "boolean" ? d.gate : current.config.gate,
gateThreshold:
typeof d.gateThreshold === "number" && Number.isFinite(d.gateThreshold)
? d.gateThreshold
: current.config.gateThreshold,
},
};
}
const sameMlConfig = (a: LotusDenoiseConfig, b: LotusDenoiseConfig): boolean =>
a.model === b.model &&
a.gate === b.gate &&
a.gateThreshold === b.gateThreshold &&
a.floor === b.floor;
/**
* Re-acquire the mic with browser noise suppression on or off, keeping the
* rest of its constraints. LiveKit re-attaches any processor to the new track;
* the level-meter clone is dropped first so the new constraint actually takes.
* The deviceId must be passed: without one LiveKit requests `audio: true` and
* silently drops every other constraint.
*/
async function restartCapture(
mic: LocalAudioTrack,
noiseSuppression: boolean,
): Promise<void> {
const c = mic.constraints;
await withMicTapsReleased(async () => {
await mic.restartTrack({
deviceId: c.deviceId ?? mic.mediaStreamTrack.getSettings().deviceId,
channelCount: c.channelCount ?? 1,
echoCancellation: c.echoCancellation,
autoGainControl: c.autoGainControl,
noiseSuppression,
voiceIsolation: noiseSuppression,
});
});
}
export function startLotusDenoise(vm: CallViewModel): () => void {
// Gate on a DEDICATED flag — NOT the existing `lotusDenoise=ml` that the
// host's build-time getUserMedia shim already uses. Otherwise simply shipping
// this fork (while the host still injects its shim and still sets
// lotusDenoise=ml) would denoise twice. The host opts into the in-source
// engine with `lotusDenoiseSource=1` AND stops injecting the shim at the same
// time. Default off ⇒ the fork is inert and behaviour is unchanged.
if (!lotusFlag("lotusDenoiseSource")) return () => undefined;
// time. Without the flag the engine stays idle unless the host switches to ML
// mid-call (io.lotus.set_denoise), which needs the widget API.
const w = widget;
const mlAtJoin = lotusFlag("lotusDenoiseSource");
if (!mlAtJoin && !w) return () => undefined;
const requested = lotusParam("lotusModel");
const model: LotusDenoiseConfig["model"] =
requested === "speex" ||
requested === "dtln" ||
requested === "deepfilternet"
? requested
: "rnnoise";
const model: LotusDenoiseConfig["model"] = isModel(requested)
? requested
: "rnnoise";
const rawThreshold = lotusParam("lotusGateThreshold");
const rawFloor = lotusParam("lotusDenoiseFloor");
const config: LotusDenoiseConfig = {
model,
assetBase: safeAssetBase(lotusParam("lotusDenoiseBase")),
gate: lotusFlag("lotusGate"),
// Default -45 (matches the reference shim); accept an explicit 0 (don't
// coerce it away via `|| default`).
gateThreshold:
rawThreshold !== null && Number.isFinite(Number(rawThreshold))
? Number(rawThreshold)
: -45,
// Dry/wet attenuation floor. Default 0.15 (~-16 dB) tames the
// over-suppression "underwater"/pumping artifact; host can tune via
// `lotusDenoiseFloor` (0 = full suppression, no floor).
floor:
rawFloor !== null && Number.isFinite(Number(rawFloor))
? Math.min(0.5, Math.max(0, Number(rawFloor)))
: 0.15,
const browserNsAtJoin = lotusParam("noiseSuppression") !== "false";
// `let`: io.lotus.set_denoise replaces it mid-call.
let target: DenoiseTarget = {
mode: mlAtJoin ? "ml" : browserNsAtJoin ? "browser" : "off",
nativeNS: mlAtJoin && browserNsAtJoin,
config: {
model,
assetBase: safeAssetBase(lotusParam("lotusDenoiseBase")),
gate: lotusFlag("lotusGate"),
// Default -45 (matches the reference shim); accept an explicit 0 (don't
// coerce it away via `|| default`).
gateThreshold:
rawThreshold !== null && Number.isFinite(Number(rawThreshold))
? Number(rawThreshold)
: -45,
// Dry/wet attenuation floor. Default 0.15 (~-16 dB) tames the
// over-suppression "underwater"/pumping artifact; host can tune via
// `lotusDenoiseFloor` (0 = full suppression, no floor).
floor:
rawFloor !== null && Number.isFinite(Number(rawFloor))
? Math.min(0.5, Math.max(0, Number(rawFloor)))
: 0.15,
},
};
// [lotus #7] Start fetching wasm / creating the AudioContext / addModule-ing
// the worklet (and loading the DFN model) NOW, before any mic track exists.
// `setProcessor()` holds LiveKit's trackChangeLock while awaiting `init()`,
// so anything still loading there freezes mute/unmute/device-switch.
void prepareDenoiseAssets(config);
if (target.mode === "ml") void prepareDenoiseAssets(target.config);
const micOf = (room: LivekitRoom): LocalAudioTrack | undefined =>
room.localParticipant.getTrackPublication(Track.Source.Microphone)
@@ -127,7 +217,7 @@ export function startLotusDenoise(vm: CallViewModel): () => void {
mic: LocalAudioTrack,
model: LotusDenoiseModel,
): Promise<void> => {
const processor = new LotusDenoiseProcessor({ ...config, model });
const processor = new LotusDenoiseProcessor({ ...target.config, model });
pendingProcessors.set(room, processor);
// [lotus #9] Seed the mute state so a processor attached while already
// muted starts suspended rather than burning inference on silence.
@@ -142,11 +232,15 @@ export function startLotusDenoise(vm: CallViewModel): () => void {
}
};
// In-flight apply() per room, so a mid-call switch waits for it to settle.
const inflight = new Map<LivekitRoom, Promise<void>>();
const apply = (room: LivekitRoom): void => {
if (target.mode !== "ml") return;
const mic = micOf(room);
if (!mic || mic.getProcessor() || pendingProcessors.has(room)) return;
void (async () => {
let model = config.model;
const run = (async (): Promise<void> => {
let model = target.config.model;
try {
try {
await attach(room, mic, model);
@@ -164,6 +258,7 @@ export function startLotusDenoise(vm: CallViewModel): () => void {
lotusSendToHost(LotusWidgetActions.DenoiseState, {
active: true,
model,
mode: "ml",
});
} catch (e) {
logger.warn("[lotus] denoise unavailable; publishing raw mic", e);
@@ -171,11 +266,69 @@ export function startLotusDenoise(vm: CallViewModel): () => void {
active: false,
error: e instanceof Error ? e.message : String(e),
model,
mode: "ml",
});
}
})();
inflight.set(room, run);
void run.finally(() => {
if (inflight.get(room) === run) inflight.delete(room);
});
};
// ── Mid-call switching (io.lotus.set_denoise) ───────────────────────────
// `applied` is what the mic is actually set up for. Switches run one at a
// time; each works from `applied` to the latest `target`, so a burst of
// changes collapses into the last one.
let applied: DenoiseTarget = target;
let queue: Promise<void> = Promise.resolve();
const reconfigure = async (): Promise<void> => {
const prev = applied;
const next = target;
if (prev === next || stopped) return;
const nsChanged = browserNsFor(prev) !== browserNsFor(next);
const swapProcessor =
prev.mode === "ml" &&
(next.mode !== "ml" || !sameMlConfig(prev.config, next.config));
setLotusNoiseSuppression(browserNsFor(next));
// Load the new model while the old one keeps running, so the switch
// itself only re-wires the graph instead of sending raw audio meanwhile.
if (next.mode === "ml") await prepareDenoiseAssets(next.config);
for (const room of rooms) {
await inflight.get(room)?.catch(() => undefined);
const mic = micOf(room);
if (!mic) continue;
try {
if (swapProcessor && mic.getProcessor()) await mic.stopProcessor();
if (nsChanged) await restartCapture(mic, browserNsFor(next));
} catch (e) {
logger.warn("[lotus] set_denoise: re-capturing the mic failed", e);
}
apply(room);
}
applied = next;
if (next.mode !== "ml")
lotusSendToHost(LotusWidgetActions.DenoiseState, {
active: false,
mode: next.mode,
});
};
const onSetDenoise = (ev: CustomEvent<IWidgetApiRequest>): void => {
w?.api.transport.reply(ev.detail, {});
const next = applySetDenoise(target, ev.detail.data);
if (!next) return;
logger.info(
`[lotus] set_denoise: ${next.mode}${next.mode === "ml" ? ` (${next.config.model})` : ""}`,
);
target = next;
queue = queue.then(reconfigure).catch((e) => {
logger.warn("[lotus] set_denoise failed", e);
});
};
w?.lazyActions.on(LotusWidgetActions.SetDenoise, onSetDenoise);
// [lotus #9] Suspend/resume the processor's own AudioContext with the mic's
// mute state (LiveKit keeps the muted track flowing silent frames, so the
// worklet would otherwise run inference the whole time the user is muted).
@@ -231,6 +384,8 @@ export function startLotusDenoise(vm: CallViewModel): () => void {
return () => {
stopped = true;
w?.lazyActions.off(LotusWidgetActions.SetDenoise, onSetDenoise);
setLotusNoiseSuppression(null);
sub.unsubscribe();
for (const off of roomListeners.values()) off();
roomListeners.clear();
+48 -7
View File
@@ -6,10 +6,12 @@ Please see LICENSE in the repository root for full details.
*/
import {
type LocalTrack,
type LocalTrackPublication,
type Room as LivekitRoom,
RoomEvent,
Track,
TrackEvent,
} from "livekit-client";
import { Observable, type Subscription, share, switchMap } from "rxjs";
import { distinctUntilChanged, map } from "rxjs/operators";
@@ -37,15 +39,35 @@ export interface LocalMicSample {
const micPublication = (
room: LivekitRoom,
): { track: MediaStreamTrack; muted: boolean } | null => {
): { track: MediaStreamTrack; local: LocalTrack; muted: boolean } | null => {
const pub: LocalTrackPublication | undefined =
room.localParticipant.getTrackPublication(Track.Source.Microphone);
const track = pub?.track?.mediaStreamTrack;
return track && track.readyState === "live"
? { track, muted: pub?.isMuted ?? false }
const local = pub?.track;
const track = local?.mediaStreamTrack;
return local && track && track.readyState === "live"
? { track, local, muted: pub?.isMuted ?? false }
: null;
};
// Every live tap, so a mic re-capture can drop the clones first.
const liveTaps = new Set<{ release: () => void; reconcile: () => void }>();
/**
* Run a mic re-capture (`restartTrack`) with every level-meter clone stopped,
* then re-tap the new track. Chrome makes all open tracks of a device share
* one set of audio processing, so a clone still holding the old track makes
* the new capture ignore its `noiseSuppression` (or AGC/EC) constraint.
*/
export async function withMicTapsReleased<T>(fn: () => Promise<T>): Promise<T> {
const taps = [...liveTaps];
taps.forEach((t) => t.release());
try {
return await fn();
} finally {
taps.forEach((t) => t.reconcile());
}
}
/** Pure: the same LiveKit rooms, in the same order (by identity). */
export const sameRooms = <T>(a: readonly T[], b: readonly T[]): boolean =>
a.length === b.length && a.every((r, i) => r === b[i]);
@@ -79,11 +101,17 @@ export function observeLocalMicSample$(
let clone: MediaStreamTrack | null = null;
let timer: ReturnType<typeof setInterval> | undefined;
let tapped: MediaStreamTrack | null = null;
// LiveKit swaps the MediaStreamTrack inside the same LocalTrack on
// restartTrack (device switch, re-capture) without any room event,
// so follow its Restarted event or the clone keeps the old one open.
let tappedLocal: LocalTrack | null = null;
let muted = false;
const stopTap = (): void => {
if (timer !== undefined) clearInterval(timer);
timer = undefined;
tappedLocal?.off(TrackEvent.Restarted, reconcile);
tappedLocal = null;
clone?.stop();
clone = null;
void ctx?.close().catch(() => undefined);
@@ -92,7 +120,12 @@ export function observeLocalMicSample$(
subscriber.next(null);
};
const startTap = (source: MediaStreamTrack): void => {
const startTap = (
source: MediaStreamTrack,
local: LocalTrack,
): void => {
tappedLocal = local;
local.on(TrackEvent.Restarted, reconcile);
try {
clone = source.clone();
// Muting disables the published track; the clone must still hear.
@@ -116,15 +149,22 @@ export function observeLocalMicSample$(
}
};
const reconcile = (): void => {
function reconcile(): void {
const pub =
rooms.map(micPublication).find((p) => p !== null) ?? null;
muted = pub?.muted ?? false;
const track = pub?.track ?? null;
if (track === tapped) return;
if (tapped) stopTap();
if (track) startTap(track);
if (track && pub) startTap(track, pub.local);
}
const control = {
release: (): void => {
if (tapped) stopTap();
},
reconcile,
};
liveTaps.add(control);
const events = [
RoomEvent.TrackMuted,
@@ -143,6 +183,7 @@ export function observeLocalMicSample$(
rooms.forEach((room) =>
events.forEach((ev) => room.off(ev, reconcile)),
);
liveTaps.delete(control);
if (tapped) stopTap();
};
}),
@@ -47,6 +47,7 @@ import {
// default to true, so with no params this is byte-for-byte upstream behaviour;
// `?autoGainControl=false` forces AGC off regardless of the stored Setting.
import { getUrlParams } from "../../../UrlParams.ts";
import { lotusNoiseSuppressionAllowed } from "../../../lotus/lotusCaptureOverride.ts";
// TODO evaluate if this should be done like the Publisher Factory
export interface ConnectionFactory {
@@ -193,9 +194,16 @@ function generateRoomOption({
echoCancellation:
echoCancellationSetting.getValue() &&
getUrlParams().echoCancellation !== false,
// [lotus] The host can switch this mid-call (io.lotus.set_denoise).
noiseSuppression:
noiseSuppressionSetting.getValue() &&
getUrlParams().noiseSuppression !== false,
lotusNoiseSuppressionAllowed(getUrlParams().noiseSuppression),
// [lotus] LiveKit defaults voiceIsolation ON, which (where the platform
// implements it) is one more voice filter in front of the ML model or
// in "off" mode. Tie it to browser noise suppression.
voiceIsolation:
noiseSuppressionSetting.getValue() &&
lotusNoiseSuppressionAllowed(getUrlParams().noiseSuppression),
autoGainControl:
autoGainControlSetting.getValue() &&
getUrlParams().autoGainControl !== false,