fix(lotus): denoise restart fallback must not hand LiveKit its own raw track
On a failed restart() the processor set processedTrack to the LiveKit-owned input track. LiveKit's internalStopProcessor() stops processedTrack and then republishes _mediaStreamTrack — the same object — so any later stopProcessor()/teardown killed the live mic for the rest of the session. Leave processedTrack undefined so LiveKit falls through to its own track. Unit-tested. Fixes #2 Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01PPmy3tPq869XDW4njjVaKA
This commit is contained in:
co-authored by
Claude Opus 5
parent
f1cfcc7377
commit
936a083533
@@ -0,0 +1,55 @@
|
||||
/*
|
||||
Copyright 2026 Lotus Guild
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { describe, expect, test } from "vitest";
|
||||
import { type AudioProcessorOptions } from "livekit-client";
|
||||
|
||||
import { LotusDenoiseProcessor } from "./lotusDenoiseProcessor";
|
||||
|
||||
function makeProcessor(): LotusDenoiseProcessor {
|
||||
return new LotusDenoiseProcessor({
|
||||
model: "rnnoise",
|
||||
assetBase: "https://example.invalid/denoise/",
|
||||
gate: false,
|
||||
gateThreshold: -45,
|
||||
floor: 0.15,
|
||||
});
|
||||
}
|
||||
|
||||
describe("LotusDenoiseProcessor.restart", () => {
|
||||
test("falls back to processedTrack = undefined (never the raw LiveKit track) when graph rebuild fails", async () => {
|
||||
const processor = makeProcessor();
|
||||
|
||||
// Stub out the AudioContext/graph plumbing: pretend the context is fine
|
||||
// but the graph rebuild (wasm load / worklet construction) throws, which
|
||||
// is the path this test targets.
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
(processor as any).ensureContext = async (): Promise<void> => {
|
||||
await Promise.resolve();
|
||||
};
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
(processor as any).buildGraph = async (): Promise<never> => {
|
||||
await Promise.resolve();
|
||||
throw new Error("simulated graph build failure");
|
||||
};
|
||||
|
||||
const rawTrack = {
|
||||
stop: (): void => undefined,
|
||||
} as unknown as MediaStreamTrack;
|
||||
|
||||
await processor.restart({
|
||||
track: rawTrack,
|
||||
} as unknown as AudioProcessorOptions);
|
||||
|
||||
// Must NOT be the raw, LiveKit-owned track: LiveKit's
|
||||
// internalStopProcessor() calls `processor.processedTrack?.stop()` then
|
||||
// re-publishes the same `_mediaStreamTrack` object, which would kill the
|
||||
// live mic on the next stopProcessor()/teardown if we handed it back here.
|
||||
expect(processor.processedTrack).toBeUndefined();
|
||||
expect(processor.processedTrack).not.toBe(rawTrack);
|
||||
});
|
||||
});
|
||||
@@ -12,11 +12,7 @@ import {
|
||||
} from "livekit-client";
|
||||
import { logger } from "matrix-js-sdk/lib/logger";
|
||||
|
||||
export type LotusDenoiseModel =
|
||||
| "rnnoise"
|
||||
| "speex"
|
||||
| "dtln"
|
||||
| "deepfilternet";
|
||||
export type LotusDenoiseModel = "rnnoise" | "speex" | "dtln" | "deepfilternet";
|
||||
|
||||
export interface LotusDenoiseConfig {
|
||||
model: LotusDenoiseModel;
|
||||
@@ -111,8 +107,8 @@ function supportsSimd(): boolean {
|
||||
// Minimal SIMD module (v128) — validates only where SIMD is supported.
|
||||
return WebAssembly.validate(
|
||||
new Uint8Array([
|
||||
0, 97, 115, 109, 1, 0, 0, 0, 1, 5, 1, 96, 0, 1, 123, 3, 2, 1, 0, 10,
|
||||
10, 1, 8, 0, 65, 0, 253, 15, 253, 98, 11,
|
||||
0, 97, 115, 109, 1, 0, 0, 0, 1, 5, 1, 96, 0, 1, 123, 3, 2, 1, 0, 10, 10,
|
||||
1, 8, 0, 65, 0, 253, 15, 253, 98, 11,
|
||||
]),
|
||||
);
|
||||
} catch {
|
||||
@@ -145,9 +141,10 @@ interface Graph {
|
||||
* stopped track on the sender: on failure it degrades to the RAW mic track
|
||||
* rather than silence.
|
||||
*/
|
||||
export class LotusDenoiseProcessor
|
||||
implements TrackProcessor<Track.Kind.Audio, AudioProcessorOptions>
|
||||
{
|
||||
export class LotusDenoiseProcessor implements TrackProcessor<
|
||||
Track.Kind.Audio,
|
||||
AudioProcessorOptions
|
||||
> {
|
||||
public readonly name = "lotus-denoise";
|
||||
public processedTrack?: MediaStreamTrack;
|
||||
|
||||
@@ -180,10 +177,16 @@ export class LotusDenoiseProcessor
|
||||
this.processedTrack = next.track;
|
||||
} catch (e) {
|
||||
// Never go silent on the A7/device-switch path: fall back to raw audio.
|
||||
// [lotus] IMPORTANT: never assign `opts.track` (LiveKit-owned) here.
|
||||
// LiveKit's internalStopProcessor() does `processor.processedTrack?.stop()`
|
||||
// then re-publishes `_mediaStreamTrack` — the SAME object if we set it as
|
||||
// processedTrack — which kills the live mic on the next stopProcessor()/
|
||||
// teardown. Leaving processedTrack undefined makes LiveKit fall through to
|
||||
// its own `_mediaStreamTrack` instead.
|
||||
logger.warn("[lotus] denoise restart failed; using raw mic", e);
|
||||
this.disposeGraph(this.graph);
|
||||
this.graph = undefined;
|
||||
this.processedTrack = opts.track;
|
||||
this.processedTrack = undefined;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -209,7 +212,11 @@ export class LotusDenoiseProcessor
|
||||
/** Create (once) the model-rate context + register the flat worklet modules. */
|
||||
private async ensureContext(): Promise<void> {
|
||||
const rate = sampleRateFor(this.config.model);
|
||||
if (this.ctx && this.ctx.state !== "closed" && this.ctx.sampleRate === rate) {
|
||||
if (
|
||||
this.ctx &&
|
||||
this.ctx.state !== "closed" &&
|
||||
this.ctx.sampleRate === rate
|
||||
) {
|
||||
if (this.ctx.state === "suspended") await resumeCtx(this.ctx);
|
||||
return;
|
||||
}
|
||||
@@ -316,7 +323,12 @@ export class LotusDenoiseProcessor
|
||||
logger.info(
|
||||
`[lotus] denoise processor active (${this.config.model}, floor=${floor})`,
|
||||
);
|
||||
return { source, nodes, disposes, track: dest.stream.getAudioTracks()[0] };
|
||||
return {
|
||||
source,
|
||||
nodes,
|
||||
disposes,
|
||||
track: dest.stream.getAudioTracks()[0],
|
||||
};
|
||||
} catch (e) {
|
||||
// A node constructor / model load can throw mid-build; clean up the
|
||||
// partially-built graph so it doesn't leak (init/restart still fall back
|
||||
@@ -338,15 +350,20 @@ export class LotusDenoiseProcessor
|
||||
if (model === "dtln") {
|
||||
// Self-contained ESM that resolves its own processor + LiteRT wasm +
|
||||
// TFLite models. bypassUntilReady passes raw audio until the model loads.
|
||||
const mod = await import(/* @vite-ignore */ `${base}workadventure/audio-worklet.js`);
|
||||
const mod = await import(
|
||||
/* @vite-ignore */ `${base}workadventure/audio-worklet.js`
|
||||
);
|
||||
return (await mod.createNoiseSuppressionAudioWorklet(ctx, {
|
||||
bypassUntilReady: true,
|
||||
})) as MlNode;
|
||||
}
|
||||
|
||||
if (model === "deepfilternet") {
|
||||
const dfnBase = new URL(`${base}deepfilternet`, window.location.href).href;
|
||||
const mod = await import(/* @vite-ignore */ `${base}deepfilternet/index.esm.js`);
|
||||
const dfnBase = new URL(`${base}deepfilternet`, window.location.href)
|
||||
.href;
|
||||
const mod = await import(
|
||||
/* @vite-ignore */ `${base}deepfilternet/index.esm.js`
|
||||
);
|
||||
const core = new mod.DeepFilterNet3Core({
|
||||
sampleRate: 48_000,
|
||||
// 60, not 80: full-strength suppression is the main source of the
|
||||
@@ -379,7 +396,10 @@ export class LotusDenoiseProcessor
|
||||
numberOfOutputs: 1,
|
||||
processorOptions: { maxChannels: 1, wasmBinary },
|
||||
});
|
||||
return { node, dispose: () => void safeCall(() => node.port.postMessage("destroy")) };
|
||||
return {
|
||||
node,
|
||||
dispose: () => void safeCall(() => node.port.postMessage("destroy")),
|
||||
};
|
||||
}
|
||||
|
||||
private disposeGraph(graph: Graph | undefined): void {
|
||||
|
||||
Reference in New Issue
Block a user