fix(lotus): denoise — prefetch assets, RNNoise fallback + host notify, suspend while muted, mono/delay graph, typed modules
- Assets (context, worklets, wasm, DFN core) are prepared as soon as the flag is seen, so init() under LiveKit's trackChangeLock only wires already-loaded pieces; resume timeout 3 s -> 500 ms (#7). - init failure retries once with rnnoise; success/failure is reported to the host as io.lotus.denoise_state so the UI can reflect reality (#8). - Mic TrackMuted/TrackUnmuted suspend/resume the processor's context so no inference runs on silence (#9). - Every node is explicit mono; the dry path gets a per-model DelayNode so the floor mix no longer comb-filters (#24, #25). - DTLN/DFN dynamic imports are typed and their exports asserted at load, feeding the #8 fallback instead of failing silently (#26). Unit-tested (13 tests across the two files). Fixes #7 Fixes #8 Fixes #9 Fixes #24 Fixes #25 Fixes #26 Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01PPmy3tPq869XDW4njjVaKA
This commit is contained in:
Lotus CI
co-authored by
Claude Opus 5
Claude Opus 5
parent
872b877248
commit
e504a31efd
@@ -5,19 +5,97 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { describe, expect, test } from "vitest";
|
||||
import { afterEach, beforeEach, describe, expect, test, vi } from "vitest";
|
||||
import { type AudioProcessorOptions } from "livekit-client";
|
||||
|
||||
import { LotusDenoiseProcessor } from "./lotusDenoiseProcessor";
|
||||
import {
|
||||
assertModuleExport,
|
||||
type LotusDenoiseConfig,
|
||||
LotusDenoiseProcessor,
|
||||
prepareDenoiseAssets,
|
||||
releasePreparedDenoiseAssets,
|
||||
} from "./lotusDenoiseProcessor";
|
||||
|
||||
function makeProcessor(): LotusDenoiseProcessor {
|
||||
return new LotusDenoiseProcessor({
|
||||
model: "rnnoise",
|
||||
assetBase: "https://example.invalid/denoise/",
|
||||
gate: false,
|
||||
gateThreshold: -45,
|
||||
floor: 0.15,
|
||||
const baseConfig: LotusDenoiseConfig = {
|
||||
model: "rnnoise",
|
||||
assetBase: "https://example.invalid/denoise/",
|
||||
gate: false,
|
||||
gateThreshold: -45,
|
||||
floor: 0.15,
|
||||
};
|
||||
|
||||
function makeProcessor(
|
||||
overrides: Partial<LotusDenoiseConfig> = {},
|
||||
): LotusDenoiseProcessor {
|
||||
return new LotusDenoiseProcessor({ ...baseConfig, ...overrides });
|
||||
}
|
||||
|
||||
/**
|
||||
* Minimal fake AudioContext: tracks state, records suspend/resume, and lets a
|
||||
* test fire `statechange` like the browser would after an OS interruption.
|
||||
*/
|
||||
class FakeAudioContext extends EventTarget {
|
||||
public static created: FakeAudioContext[] = [];
|
||||
public state: AudioContextState = "running";
|
||||
public readonly sampleRate: number;
|
||||
public readonly audioWorklet = {
|
||||
addModule: vi.fn().mockResolvedValue(undefined),
|
||||
};
|
||||
public suspend = vi.fn(async (): Promise<void> => {
|
||||
await Promise.resolve();
|
||||
this.state = "suspended";
|
||||
});
|
||||
public resume = vi.fn(async (): Promise<void> => {
|
||||
await Promise.resolve();
|
||||
this.state = "running";
|
||||
});
|
||||
public close = vi.fn(async (): Promise<void> => {
|
||||
await Promise.resolve();
|
||||
this.state = "closed";
|
||||
});
|
||||
public constructor(opts?: { sampleRate?: number }) {
|
||||
super();
|
||||
this.sampleRate = opts?.sampleRate ?? 48_000;
|
||||
FakeAudioContext.created.push(this);
|
||||
}
|
||||
/** Simulate an external (OS/browser) suspension. */
|
||||
public externallySuspend(): void {
|
||||
this.state = "suspended";
|
||||
this.dispatchEvent(new Event("statechange"));
|
||||
}
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
FakeAudioContext.created = [];
|
||||
vi.stubGlobal("AudioContext", FakeAudioContext);
|
||||
// The flat wasm prefetch just warms a cache; give it a 200 so prepare()
|
||||
// succeeds without a network.
|
||||
vi.stubGlobal(
|
||||
"fetch",
|
||||
vi.fn().mockResolvedValue({
|
||||
ok: true,
|
||||
arrayBuffer: (): ArrayBuffer => new ArrayBuffer(8),
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await releasePreparedDenoiseAssets();
|
||||
vi.unstubAllGlobals();
|
||||
});
|
||||
|
||||
/** Skip the real graph (needs worklets) — init() only needs a track back. */
|
||||
function stubGraph(processor: LotusDenoiseProcessor): void {
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
(processor as any).buildGraph = async (): Promise<unknown> => {
|
||||
await Promise.resolve();
|
||||
return {
|
||||
source: { disconnect: vi.fn() },
|
||||
nodes: [],
|
||||
disposes: [],
|
||||
track: { stop: vi.fn() },
|
||||
};
|
||||
};
|
||||
}
|
||||
|
||||
describe("LotusDenoiseProcessor.restart", () => {
|
||||
@@ -53,3 +131,117 @@ describe("LotusDenoiseProcessor.restart", () => {
|
||||
expect(processor.processedTrack).not.toBe(rawTrack);
|
||||
});
|
||||
});
|
||||
|
||||
describe("prepareDenoiseAssets (#7)", () => {
|
||||
test("init() adopts the context/worklet prepared before setProcessor instead of loading inline", async () => {
|
||||
await prepareDenoiseAssets(baseConfig);
|
||||
// The heavy parts ran up front: one context, worklet module added, wasm fetched.
|
||||
expect(FakeAudioContext.created).toHaveLength(1);
|
||||
const prepared = FakeAudioContext.created[0];
|
||||
expect(prepared.audioWorklet.addModule).toHaveBeenCalledWith(
|
||||
`${baseConfig.assetBase}rnnoiseWorklet.js`,
|
||||
);
|
||||
expect(fetch).toHaveBeenCalled();
|
||||
|
||||
const processor = makeProcessor();
|
||||
stubGraph(processor);
|
||||
await processor.init({
|
||||
track: {} as MediaStreamTrack,
|
||||
} as unknown as AudioProcessorOptions);
|
||||
|
||||
// No second AudioContext / addModule inside init (i.e. under the lock).
|
||||
expect(FakeAudioContext.created).toHaveLength(1);
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
expect((processor as any).ctx).toBe(prepared);
|
||||
|
||||
// The claim is one-shot: a later processor prepares its own.
|
||||
const second = makeProcessor();
|
||||
stubGraph(second);
|
||||
await second.init({
|
||||
track: {} as MediaStreamTrack,
|
||||
} as unknown as AudioProcessorOptions);
|
||||
expect(FakeAudioContext.created).toHaveLength(2);
|
||||
|
||||
await processor.destroy();
|
||||
await second.destroy();
|
||||
});
|
||||
|
||||
test("releasePreparedDenoiseAssets closes an unclaimed prepared context", async () => {
|
||||
await prepareDenoiseAssets(baseConfig);
|
||||
const prepared = FakeAudioContext.created[0];
|
||||
await releasePreparedDenoiseAssets();
|
||||
expect(prepared.close).toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe("LotusDenoiseProcessor.setMicMuted (#9)", () => {
|
||||
test("suspends the owned context on mute, resumes on unmute, and the statechange watcher leaves the mute suspension alone", async () => {
|
||||
const processor = makeProcessor();
|
||||
stubGraph(processor);
|
||||
await processor.init({
|
||||
track: {} as MediaStreamTrack,
|
||||
} as unknown as AudioProcessorOptions);
|
||||
const ctx = FakeAudioContext.created[0];
|
||||
expect(ctx.state).toBe("running");
|
||||
|
||||
processor.setMicMuted(true);
|
||||
await Promise.resolve();
|
||||
expect(ctx.suspend).toHaveBeenCalledTimes(1);
|
||||
expect(ctx.state).toBe("suspended");
|
||||
|
||||
// The browser fires statechange for our own suspend too; the watcher must
|
||||
// NOT undo it while muted.
|
||||
ctx.resume.mockClear();
|
||||
ctx.dispatchEvent(new Event("statechange"));
|
||||
await Promise.resolve();
|
||||
expect(ctx.resume).not.toHaveBeenCalled();
|
||||
expect(ctx.state).toBe("suspended");
|
||||
|
||||
processor.setMicMuted(false);
|
||||
await Promise.resolve();
|
||||
expect(ctx.resume).toHaveBeenCalledTimes(1);
|
||||
expect(ctx.state).toBe("running");
|
||||
|
||||
// An EXTERNAL suspension while unmuted is still healed by the watcher.
|
||||
ctx.resume.mockClear();
|
||||
ctx.externallySuspend();
|
||||
await Promise.resolve();
|
||||
expect(ctx.resume).toHaveBeenCalledTimes(1);
|
||||
|
||||
await processor.destroy();
|
||||
});
|
||||
|
||||
test("a processor attached while already muted starts suspended", async () => {
|
||||
const processor = makeProcessor();
|
||||
stubGraph(processor);
|
||||
processor.setMicMuted(true);
|
||||
await processor.init({
|
||||
track: {} as MediaStreamTrack,
|
||||
} as unknown as AudioProcessorOptions);
|
||||
const ctx = FakeAudioContext.created[0];
|
||||
expect(ctx.state).toBe("suspended");
|
||||
await processor.destroy();
|
||||
});
|
||||
});
|
||||
|
||||
describe("assertModuleExport (#26)", () => {
|
||||
test("returns the module when the export is a function", () => {
|
||||
const mod = { createNoiseSuppressionAudioWorklet: (): void => undefined };
|
||||
expect(
|
||||
assertModuleExport(mod, "createNoiseSuppressionAudioWorklet", "x.js"),
|
||||
).toBe(mod);
|
||||
});
|
||||
|
||||
test("throws a clear error naming the module and export when missing", () => {
|
||||
expect(() =>
|
||||
assertModuleExport(
|
||||
{ other: 1 },
|
||||
"DeepFilterNet3Core",
|
||||
"dfn/index.esm.js",
|
||||
),
|
||||
).toThrow(/dfn\/index\.esm\.js does not export DeepFilterNet3Core/);
|
||||
expect(() => assertModuleExport(undefined, "X", "y.js")).toThrow(
|
||||
/does not export X/,
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user