feat(lotus-denoise): AGC off for ML tier + init/build leak hardening
AEC/AGC audit fix + two hardening items from the engine review. - Add an `autoGainControl` capture param (UrlParams -> CallViewModel -> ConnectionFactory audioCaptureDefaults), mirroring echoCancellation/ noiseSuppression. Defaults true (unchanged); the host sets it false only for the ML tier so the browser's auto gain control doesn't fight the in-source ML denoiser (pumping). Echo cancellation stays on. Tests cover the URL parse and the audioCaptureDefaults wiring. - L1: init() now closes the owned AudioContext on a build failure (was orphaned; browsers cap live contexts, so repeated failures could exhaust them). - L2: buildGraph() disposes its partially-built nodes on failure (disposeGraph previously only cleaned the prior graph). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
parent
6ab52d9926
commit
940d71da92
@@ -391,6 +391,21 @@ describe("UrlParams", () => {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe("autoGainControl", () => {
|
||||||
|
it("defaults to true", () => {
|
||||||
|
expect(computeUrlParams().autoGainControl).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("is parsed", () => {
|
||||||
|
expect(computeUrlParams("?autoGainControl=true").autoGainControl).toBe(
|
||||||
|
true,
|
||||||
|
);
|
||||||
|
expect(computeUrlParams("?autoGainControl=false").autoGainControl).toBe(
|
||||||
|
false,
|
||||||
|
);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
describe("header", () => {
|
describe("header", () => {
|
||||||
it("uses header if provided", () => {
|
it("uses header if provided", () => {
|
||||||
expect(computeUrlParams("?header=app_bar&hideHeader=true").header).toBe(
|
expect(computeUrlParams("?header=app_bar&hideHeader=true").header).toBe(
|
||||||
|
|||||||
@@ -239,6 +239,12 @@ export interface UrlConfiguration {
|
|||||||
* Defaults to true.
|
* Defaults to true.
|
||||||
*/
|
*/
|
||||||
noiseSuppression?: boolean;
|
noiseSuppression?: boolean;
|
||||||
|
/**
|
||||||
|
* Whether to enable auto gain control for audio capture.
|
||||||
|
* Defaults to true. Lotus turns this OFF for the in-source ML denoise tier so
|
||||||
|
* the browser's dynamic gain doesn't fight the ML model (pumping artifacts).
|
||||||
|
*/
|
||||||
|
autoGainControl?: boolean;
|
||||||
|
|
||||||
callIntent?: RTCCallIntent;
|
callIntent?: RTCCallIntent;
|
||||||
}
|
}
|
||||||
@@ -485,6 +491,7 @@ export const computeUrlParams = (search = "", hash = ""): UrlParams => {
|
|||||||
autoLeaveWhenOthersLeft: parser.getFlag("autoLeave"),
|
autoLeaveWhenOthersLeft: parser.getFlag("autoLeave"),
|
||||||
noiseSuppression: parser.getFlagParam("noiseSuppression", true),
|
noiseSuppression: parser.getFlagParam("noiseSuppression", true),
|
||||||
echoCancellation: parser.getFlagParam("echoCancellation", true),
|
echoCancellation: parser.getFlagParam("echoCancellation", true),
|
||||||
|
autoGainControl: parser.getFlagParam("autoGainControl", true),
|
||||||
};
|
};
|
||||||
|
|
||||||
// Log the final configuration for debugging purposes.
|
// Log the final configuration for debugging purposes.
|
||||||
|
|||||||
@@ -158,9 +158,17 @@ export class LotusDenoiseProcessor
|
|||||||
public constructor(private readonly config: LotusDenoiseConfig) {}
|
public constructor(private readonly config: LotusDenoiseConfig) {}
|
||||||
|
|
||||||
public async init(_opts: AudioProcessorOptions): Promise<void> {
|
public async init(_opts: AudioProcessorOptions): Promise<void> {
|
||||||
await this.ensureContext();
|
try {
|
||||||
this.graph = await this.buildGraph(_opts.track);
|
await this.ensureContext();
|
||||||
this.processedTrack = this.graph.track;
|
this.graph = await this.buildGraph(_opts.track);
|
||||||
|
this.processedTrack = this.graph.track;
|
||||||
|
} catch (e) {
|
||||||
|
// Don't orphan the owned context if graph construction fails (browsers
|
||||||
|
// cap live AudioContexts, so repeated failed inits could exhaust them).
|
||||||
|
// The caller degrades to the raw mic; we just release our resources.
|
||||||
|
await this.closeContext();
|
||||||
|
throw e;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public async restart(opts: AudioProcessorOptions): Promise<void> {
|
public async restart(opts: AudioProcessorOptions): Promise<void> {
|
||||||
@@ -249,63 +257,78 @@ export class LotusDenoiseProcessor
|
|||||||
const nodes: AudioNode[] = [];
|
const nodes: AudioNode[] = [];
|
||||||
const disposes: (() => void)[] = [];
|
const disposes: (() => void)[] = [];
|
||||||
|
|
||||||
// Wet (denoised) path: source → ml → [gate] → wetGain.
|
try {
|
||||||
const ml = await this.buildMlNode(ctx);
|
// Wet (denoised) path: source → ml → [gate] → wetGain.
|
||||||
source.connect(ml.node);
|
const ml = await this.buildMlNode(ctx);
|
||||||
nodes.push(ml.node);
|
source.connect(ml.node);
|
||||||
if (ml.dispose) disposes.push(ml.dispose);
|
nodes.push(ml.node);
|
||||||
let wetHead: AudioNode = ml.node;
|
if (ml.dispose) disposes.push(ml.dispose);
|
||||||
|
let wetHead: AudioNode = ml.node;
|
||||||
|
|
||||||
// Gate AFTER the ML model, not before: gating the raw noisy signal fed hard
|
// Gate AFTER the ML model, not before: gating the raw noisy signal fed
|
||||||
// zeroed frames into the model (discontinuities it must fight) and made the
|
// hard-zeroed frames into the model (discontinuities it must fight) and
|
||||||
// threshold operate on pre-denoise levels. Gate the residual instead.
|
// made the threshold operate on pre-denoise levels. Gate the residual.
|
||||||
if (this.config.gate) {
|
if (this.config.gate) {
|
||||||
const gate = new AudioWorkletNode(ctx, GATE.name, {
|
const gate = new AudioWorkletNode(ctx, GATE.name, {
|
||||||
processorOptions: {
|
processorOptions: {
|
||||||
openThreshold: this.config.gateThreshold,
|
openThreshold: this.config.gateThreshold,
|
||||||
closeThreshold: this.config.gateThreshold - 5,
|
closeThreshold: this.config.gateThreshold - 5,
|
||||||
holdMs: 150,
|
holdMs: 150,
|
||||||
maxChannels: 1,
|
maxChannels: 1,
|
||||||
},
|
},
|
||||||
|
});
|
||||||
|
wetHead.connect(gate);
|
||||||
|
wetHead = gate;
|
||||||
|
nodes.push(gate);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Only mix a dry floor for the LOW-LATENCY flat models (RNNoise/Speex).
|
||||||
|
// DTLN/DeepFilterNet add tens of ms of algorithmic latency, so summing an
|
||||||
|
// undelayed dry copy would comb-filter the voice — for those we rely on
|
||||||
|
// the model's own level (e.g. DFN noiseReductionLevel) instead. RNNoise is
|
||||||
|
// also where the "robotic/underwater" reports come from, so this targets it.
|
||||||
|
const lowLatency =
|
||||||
|
this.config.model === "rnnoise" || this.config.model === "speex";
|
||||||
|
const floor = lowLatency
|
||||||
|
? Math.min(0.5, Math.max(0, this.config.floor))
|
||||||
|
: 0;
|
||||||
|
if (floor > 0) {
|
||||||
|
// Dry/wet mix: blend a small amount of the ORIGINAL mic under the
|
||||||
|
// denoised signal so suppression can't fully collapse the noise floor
|
||||||
|
// (kills the "underwater"/pumping artifact). During speech (denoised ≈
|
||||||
|
// original) the two sum back to ~unity; in noise-only gaps the output
|
||||||
|
// floors at `floor` × original instead of digital silence.
|
||||||
|
const wetGain = ctx.createGain();
|
||||||
|
wetGain.gain.value = 1 - floor;
|
||||||
|
wetHead.connect(wetGain);
|
||||||
|
wetGain.connect(dest);
|
||||||
|
nodes.push(wetGain);
|
||||||
|
|
||||||
|
const dryGain = ctx.createGain();
|
||||||
|
dryGain.gain.value = floor;
|
||||||
|
source.connect(dryGain);
|
||||||
|
dryGain.connect(dest);
|
||||||
|
nodes.push(dryGain);
|
||||||
|
} else {
|
||||||
|
wetHead.connect(dest);
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.info(
|
||||||
|
`[lotus] denoise processor active (${this.config.model}, floor=${floor})`,
|
||||||
|
);
|
||||||
|
return { source, nodes, disposes, track: dest.stream.getAudioTracks()[0] };
|
||||||
|
} catch (e) {
|
||||||
|
// A node constructor / model load can throw mid-build; clean up the
|
||||||
|
// partially-built graph so it doesn't leak (init/restart still fall back
|
||||||
|
// to the raw mic on the rejection).
|
||||||
|
this.disposeGraph({
|
||||||
|
source,
|
||||||
|
nodes,
|
||||||
|
disposes,
|
||||||
|
track: dest.stream.getAudioTracks()[0],
|
||||||
});
|
});
|
||||||
wetHead.connect(gate);
|
throw e;
|
||||||
wetHead = gate;
|
|
||||||
nodes.push(gate);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Only mix a dry floor for the LOW-LATENCY flat models (RNNoise/Speex).
|
|
||||||
// DTLN/DeepFilterNet add tens of ms of algorithmic latency, so summing an
|
|
||||||
// undelayed dry copy would comb-filter the voice — for those we rely on the
|
|
||||||
// model's own level (e.g. DFN noiseReductionLevel) instead. RNNoise is also
|
|
||||||
// where the "robotic/underwater" reports come from, so this targets it.
|
|
||||||
const lowLatency =
|
|
||||||
this.config.model === "rnnoise" || this.config.model === "speex";
|
|
||||||
const floor = lowLatency ? Math.min(0.5, Math.max(0, this.config.floor)) : 0;
|
|
||||||
if (floor > 0) {
|
|
||||||
// Dry/wet mix: blend a small amount of the ORIGINAL mic under the
|
|
||||||
// denoised signal so suppression can't fully collapse the noise floor
|
|
||||||
// (kills the "underwater"/pumping artifact). During speech (denoised ≈
|
|
||||||
// original) the two sum back to ~unity; in noise-only gaps the output
|
|
||||||
// floors at `floor` × original instead of digital silence.
|
|
||||||
const wetGain = ctx.createGain();
|
|
||||||
wetGain.gain.value = 1 - floor;
|
|
||||||
wetHead.connect(wetGain);
|
|
||||||
wetGain.connect(dest);
|
|
||||||
nodes.push(wetGain);
|
|
||||||
|
|
||||||
const dryGain = ctx.createGain();
|
|
||||||
dryGain.gain.value = floor;
|
|
||||||
source.connect(dryGain);
|
|
||||||
dryGain.connect(dest);
|
|
||||||
nodes.push(dryGain);
|
|
||||||
} else {
|
|
||||||
wetHead.connect(dest);
|
|
||||||
}
|
|
||||||
|
|
||||||
logger.info(
|
|
||||||
`[lotus] denoise processor active (${this.config.model}, floor=${floor})`,
|
|
||||||
);
|
|
||||||
return { source, nodes, disposes, track: dest.stream.getAudioTracks()[0] };
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private async buildMlNode(ctx: AudioContext): Promise<MlNode> {
|
private async buildMlNode(ctx: AudioContext): Promise<MlNode> {
|
||||||
|
|||||||
@@ -504,6 +504,7 @@ export function createCallViewModel$(
|
|||||||
options.livekitRoomFactory,
|
options.livekitRoomFactory,
|
||||||
getUrlParams().echoCancellation,
|
getUrlParams().echoCancellation,
|
||||||
getUrlParams().noiseSuppression,
|
getUrlParams().noiseSuppression,
|
||||||
|
getUrlParams().autoGainControl,
|
||||||
);
|
);
|
||||||
|
|
||||||
const connectionManager = createConnectionManager$({
|
const connectionManager = createConnectionManager$({
|
||||||
|
|||||||
@@ -55,6 +55,7 @@ export class ECConnectionFactory implements ConnectionFactory {
|
|||||||
* @param livekitRoomFactory - Optional factory function (for testing) to create LivekitRoom instances. If not provided, a default factory is used.
|
* @param livekitRoomFactory - Optional factory function (for testing) to create LivekitRoom instances. If not provided, a default factory is used.
|
||||||
* @param echoCancellation - Whether to enable echo cancellation for audio capture.
|
* @param echoCancellation - Whether to enable echo cancellation for audio capture.
|
||||||
* @param noiseSuppression - Whether to enable noise suppression for audio capture.
|
* @param noiseSuppression - Whether to enable noise suppression for audio capture.
|
||||||
|
* @param autoGainControl - Whether to enable auto gain control for audio capture.
|
||||||
*/
|
*/
|
||||||
public constructor(
|
public constructor(
|
||||||
private client: OpenIDClientParts,
|
private client: OpenIDClientParts,
|
||||||
@@ -66,6 +67,7 @@ export class ECConnectionFactory implements ConnectionFactory {
|
|||||||
livekitRoomFactory?: () => LivekitRoom,
|
livekitRoomFactory?: () => LivekitRoom,
|
||||||
echoCancellation: boolean = true,
|
echoCancellation: boolean = true,
|
||||||
noiseSuppression: boolean = true,
|
noiseSuppression: boolean = true,
|
||||||
|
autoGainControl: boolean = true,
|
||||||
) {
|
) {
|
||||||
const defaultFactory = (): LivekitRoom =>
|
const defaultFactory = (): LivekitRoom =>
|
||||||
new LivekitRoom(
|
new LivekitRoom(
|
||||||
@@ -81,6 +83,7 @@ export class ECConnectionFactory implements ConnectionFactory {
|
|||||||
controlledAudioDevices: this.controlledAudioDevices,
|
controlledAudioDevices: this.controlledAudioDevices,
|
||||||
echoCancellation,
|
echoCancellation,
|
||||||
noiseSuppression,
|
noiseSuppression,
|
||||||
|
autoGainControl,
|
||||||
}),
|
}),
|
||||||
);
|
);
|
||||||
this.livekitRoomFactory = livekitRoomFactory ?? defaultFactory;
|
this.livekitRoomFactory = livekitRoomFactory ?? defaultFactory;
|
||||||
@@ -127,6 +130,7 @@ function generateRoomOption({
|
|||||||
controlledAudioDevices,
|
controlledAudioDevices,
|
||||||
echoCancellation,
|
echoCancellation,
|
||||||
noiseSuppression,
|
noiseSuppression,
|
||||||
|
autoGainControl,
|
||||||
}: {
|
}: {
|
||||||
devices: MediaDevices;
|
devices: MediaDevices;
|
||||||
processorState: ProcessorState;
|
processorState: ProcessorState;
|
||||||
@@ -137,6 +141,7 @@ function generateRoomOption({
|
|||||||
controlledAudioDevices: boolean;
|
controlledAudioDevices: boolean;
|
||||||
echoCancellation: boolean;
|
echoCancellation: boolean;
|
||||||
noiseSuppression: boolean;
|
noiseSuppression: boolean;
|
||||||
|
autoGainControl: boolean;
|
||||||
}): RoomOptions {
|
}): RoomOptions {
|
||||||
return {
|
return {
|
||||||
...defaultLiveKitOptions,
|
...defaultLiveKitOptions,
|
||||||
@@ -150,6 +155,7 @@ function generateRoomOption({
|
|||||||
deviceId: devices.audioInput.selected$.value?.id,
|
deviceId: devices.audioInput.selected$.value?.id,
|
||||||
echoCancellation,
|
echoCancellation,
|
||||||
noiseSuppression,
|
noiseSuppression,
|
||||||
|
autoGainControl,
|
||||||
},
|
},
|
||||||
audioOutput: {
|
audioOutput: {
|
||||||
// When using controlled audio devices, we don't want to set the
|
// When using controlled audio devices, we don't want to set the
|
||||||
|
|||||||
@@ -53,14 +53,13 @@ beforeEach(() => {
|
|||||||
|
|
||||||
describe("ECConnectionFactory - Audio inputs options", () => {
|
describe("ECConnectionFactory - Audio inputs options", () => {
|
||||||
test.each([
|
test.each([
|
||||||
{ echo: true, noise: true },
|
{ echo: true, noise: true, agc: true },
|
||||||
{ echo: true, noise: false },
|
{ echo: true, noise: false, agc: false },
|
||||||
{ echo: false, noise: true },
|
{ echo: false, noise: true, agc: false },
|
||||||
{ echo: false, noise: false },
|
{ echo: false, noise: false, agc: true },
|
||||||
])(
|
])(
|
||||||
"it sets echoCancellation=$echo and noiseSuppression=$noise based on constructor parameters",
|
"it sets echoCancellation=$echo, noiseSuppression=$noise, autoGainControl=$agc based on constructor parameters",
|
||||||
({ echo, noise }) => {
|
({ echo, noise, agc }) => {
|
||||||
// test("it sets echoCancellation and noiseSuppression based on constructor parameters", () => {
|
|
||||||
const RoomConstructor = vi.mocked(LivekitRoom);
|
const RoomConstructor = vi.mocked(LivekitRoom);
|
||||||
|
|
||||||
const ecConnectionFactory = new ECConnectionFactory(
|
const ecConnectionFactory = new ECConnectionFactory(
|
||||||
@@ -76,6 +75,7 @@ describe("ECConnectionFactory - Audio inputs options", () => {
|
|||||||
undefined,
|
undefined,
|
||||||
echo,
|
echo,
|
||||||
noise,
|
noise,
|
||||||
|
agc,
|
||||||
);
|
);
|
||||||
ecConnectionFactory.createConnection(
|
ecConnectionFactory.createConnection(
|
||||||
testScope,
|
testScope,
|
||||||
@@ -90,6 +90,7 @@ describe("ECConnectionFactory - Audio inputs options", () => {
|
|||||||
audioCaptureDefaults: expect.objectContaining({
|
audioCaptureDefaults: expect.objectContaining({
|
||||||
echoCancellation: echo,
|
echoCancellation: echo,
|
||||||
noiseSuppression: noise,
|
noiseSuppression: noise,
|
||||||
|
autoGainControl: agc,
|
||||||
}),
|
}),
|
||||||
}),
|
}),
|
||||||
);
|
);
|
||||||
|
|||||||
Reference in New Issue
Block a user