fix(lotus): denoise restart fallback must not hand LiveKit its own raw track
On a failed restart() the processor set processedTrack to the LiveKit-owned input track. LiveKit's internalStopProcessor() stops processedTrack and then republishes _mediaStreamTrack — the same object — so any later stopProcessor()/teardown killed the live mic for the rest of the session. Leave processedTrack undefined so LiveKit falls through to its own track. Unit-tested. Fixes #2 Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01PPmy3tPq869XDW4njjVaKA
This commit is contained in:
co-authored by
Claude Opus 5
parent
f1cfcc7377
commit
936a083533
@@ -0,0 +1,55 @@
|
|||||||
|
/*
|
||||||
|
Copyright 2026 Lotus Guild
|
||||||
|
|
||||||
|
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||||
|
Please see LICENSE in the repository root for full details.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { describe, expect, test } from "vitest";
|
||||||
|
import { type AudioProcessorOptions } from "livekit-client";
|
||||||
|
|
||||||
|
import { LotusDenoiseProcessor } from "./lotusDenoiseProcessor";
|
||||||
|
|
||||||
|
function makeProcessor(): LotusDenoiseProcessor {
|
||||||
|
return new LotusDenoiseProcessor({
|
||||||
|
model: "rnnoise",
|
||||||
|
assetBase: "https://example.invalid/denoise/",
|
||||||
|
gate: false,
|
||||||
|
gateThreshold: -45,
|
||||||
|
floor: 0.15,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
describe("LotusDenoiseProcessor.restart", () => {
|
||||||
|
test("falls back to processedTrack = undefined (never the raw LiveKit track) when graph rebuild fails", async () => {
|
||||||
|
const processor = makeProcessor();
|
||||||
|
|
||||||
|
// Stub out the AudioContext/graph plumbing: pretend the context is fine
|
||||||
|
// but the graph rebuild (wasm load / worklet construction) throws, which
|
||||||
|
// is the path this test targets.
|
||||||
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||||
|
(processor as any).ensureContext = async (): Promise<void> => {
|
||||||
|
await Promise.resolve();
|
||||||
|
};
|
||||||
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||||
|
(processor as any).buildGraph = async (): Promise<never> => {
|
||||||
|
await Promise.resolve();
|
||||||
|
throw new Error("simulated graph build failure");
|
||||||
|
};
|
||||||
|
|
||||||
|
const rawTrack = {
|
||||||
|
stop: (): void => undefined,
|
||||||
|
} as unknown as MediaStreamTrack;
|
||||||
|
|
||||||
|
await processor.restart({
|
||||||
|
track: rawTrack,
|
||||||
|
} as unknown as AudioProcessorOptions);
|
||||||
|
|
||||||
|
// Must NOT be the raw, LiveKit-owned track: LiveKit's
|
||||||
|
// internalStopProcessor() calls `processor.processedTrack?.stop()` then
|
||||||
|
// re-publishes the same `_mediaStreamTrack` object, which would kill the
|
||||||
|
// live mic on the next stopProcessor()/teardown if we handed it back here.
|
||||||
|
expect(processor.processedTrack).toBeUndefined();
|
||||||
|
expect(processor.processedTrack).not.toBe(rawTrack);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -12,11 +12,7 @@ import {
|
|||||||
} from "livekit-client";
|
} from "livekit-client";
|
||||||
import { logger } from "matrix-js-sdk/lib/logger";
|
import { logger } from "matrix-js-sdk/lib/logger";
|
||||||
|
|
||||||
export type LotusDenoiseModel =
|
export type LotusDenoiseModel = "rnnoise" | "speex" | "dtln" | "deepfilternet";
|
||||||
| "rnnoise"
|
|
||||||
| "speex"
|
|
||||||
| "dtln"
|
|
||||||
| "deepfilternet";
|
|
||||||
|
|
||||||
export interface LotusDenoiseConfig {
|
export interface LotusDenoiseConfig {
|
||||||
model: LotusDenoiseModel;
|
model: LotusDenoiseModel;
|
||||||
@@ -111,8 +107,8 @@ function supportsSimd(): boolean {
|
|||||||
// Minimal SIMD module (v128) — validates only where SIMD is supported.
|
// Minimal SIMD module (v128) — validates only where SIMD is supported.
|
||||||
return WebAssembly.validate(
|
return WebAssembly.validate(
|
||||||
new Uint8Array([
|
new Uint8Array([
|
||||||
0, 97, 115, 109, 1, 0, 0, 0, 1, 5, 1, 96, 0, 1, 123, 3, 2, 1, 0, 10,
|
0, 97, 115, 109, 1, 0, 0, 0, 1, 5, 1, 96, 0, 1, 123, 3, 2, 1, 0, 10, 10,
|
||||||
10, 1, 8, 0, 65, 0, 253, 15, 253, 98, 11,
|
1, 8, 0, 65, 0, 253, 15, 253, 98, 11,
|
||||||
]),
|
]),
|
||||||
);
|
);
|
||||||
} catch {
|
} catch {
|
||||||
@@ -145,9 +141,10 @@ interface Graph {
|
|||||||
* stopped track on the sender: on failure it degrades to the RAW mic track
|
* stopped track on the sender: on failure it degrades to the RAW mic track
|
||||||
* rather than silence.
|
* rather than silence.
|
||||||
*/
|
*/
|
||||||
export class LotusDenoiseProcessor
|
export class LotusDenoiseProcessor implements TrackProcessor<
|
||||||
implements TrackProcessor<Track.Kind.Audio, AudioProcessorOptions>
|
Track.Kind.Audio,
|
||||||
{
|
AudioProcessorOptions
|
||||||
|
> {
|
||||||
public readonly name = "lotus-denoise";
|
public readonly name = "lotus-denoise";
|
||||||
public processedTrack?: MediaStreamTrack;
|
public processedTrack?: MediaStreamTrack;
|
||||||
|
|
||||||
@@ -180,10 +177,16 @@ export class LotusDenoiseProcessor
|
|||||||
this.processedTrack = next.track;
|
this.processedTrack = next.track;
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
// Never go silent on the A7/device-switch path: fall back to raw audio.
|
// Never go silent on the A7/device-switch path: fall back to raw audio.
|
||||||
|
// [lotus] IMPORTANT: never assign `opts.track` (LiveKit-owned) here.
|
||||||
|
// LiveKit's internalStopProcessor() does `processor.processedTrack?.stop()`
|
||||||
|
// then re-publishes `_mediaStreamTrack` — the SAME object if we set it as
|
||||||
|
// processedTrack — which kills the live mic on the next stopProcessor()/
|
||||||
|
// teardown. Leaving processedTrack undefined makes LiveKit fall through to
|
||||||
|
// its own `_mediaStreamTrack` instead.
|
||||||
logger.warn("[lotus] denoise restart failed; using raw mic", e);
|
logger.warn("[lotus] denoise restart failed; using raw mic", e);
|
||||||
this.disposeGraph(this.graph);
|
this.disposeGraph(this.graph);
|
||||||
this.graph = undefined;
|
this.graph = undefined;
|
||||||
this.processedTrack = opts.track;
|
this.processedTrack = undefined;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -209,7 +212,11 @@ export class LotusDenoiseProcessor
|
|||||||
/** Create (once) the model-rate context + register the flat worklet modules. */
|
/** Create (once) the model-rate context + register the flat worklet modules. */
|
||||||
private async ensureContext(): Promise<void> {
|
private async ensureContext(): Promise<void> {
|
||||||
const rate = sampleRateFor(this.config.model);
|
const rate = sampleRateFor(this.config.model);
|
||||||
if (this.ctx && this.ctx.state !== "closed" && this.ctx.sampleRate === rate) {
|
if (
|
||||||
|
this.ctx &&
|
||||||
|
this.ctx.state !== "closed" &&
|
||||||
|
this.ctx.sampleRate === rate
|
||||||
|
) {
|
||||||
if (this.ctx.state === "suspended") await resumeCtx(this.ctx);
|
if (this.ctx.state === "suspended") await resumeCtx(this.ctx);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
@@ -316,7 +323,12 @@ export class LotusDenoiseProcessor
|
|||||||
logger.info(
|
logger.info(
|
||||||
`[lotus] denoise processor active (${this.config.model}, floor=${floor})`,
|
`[lotus] denoise processor active (${this.config.model}, floor=${floor})`,
|
||||||
);
|
);
|
||||||
return { source, nodes, disposes, track: dest.stream.getAudioTracks()[0] };
|
return {
|
||||||
|
source,
|
||||||
|
nodes,
|
||||||
|
disposes,
|
||||||
|
track: dest.stream.getAudioTracks()[0],
|
||||||
|
};
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
// A node constructor / model load can throw mid-build; clean up the
|
// A node constructor / model load can throw mid-build; clean up the
|
||||||
// partially-built graph so it doesn't leak (init/restart still fall back
|
// partially-built graph so it doesn't leak (init/restart still fall back
|
||||||
@@ -338,15 +350,20 @@ export class LotusDenoiseProcessor
|
|||||||
if (model === "dtln") {
|
if (model === "dtln") {
|
||||||
// Self-contained ESM that resolves its own processor + LiteRT wasm +
|
// Self-contained ESM that resolves its own processor + LiteRT wasm +
|
||||||
// TFLite models. bypassUntilReady passes raw audio until the model loads.
|
// TFLite models. bypassUntilReady passes raw audio until the model loads.
|
||||||
const mod = await import(/* @vite-ignore */ `${base}workadventure/audio-worklet.js`);
|
const mod = await import(
|
||||||
|
/* @vite-ignore */ `${base}workadventure/audio-worklet.js`
|
||||||
|
);
|
||||||
return (await mod.createNoiseSuppressionAudioWorklet(ctx, {
|
return (await mod.createNoiseSuppressionAudioWorklet(ctx, {
|
||||||
bypassUntilReady: true,
|
bypassUntilReady: true,
|
||||||
})) as MlNode;
|
})) as MlNode;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (model === "deepfilternet") {
|
if (model === "deepfilternet") {
|
||||||
const dfnBase = new URL(`${base}deepfilternet`, window.location.href).href;
|
const dfnBase = new URL(`${base}deepfilternet`, window.location.href)
|
||||||
const mod = await import(/* @vite-ignore */ `${base}deepfilternet/index.esm.js`);
|
.href;
|
||||||
|
const mod = await import(
|
||||||
|
/* @vite-ignore */ `${base}deepfilternet/index.esm.js`
|
||||||
|
);
|
||||||
const core = new mod.DeepFilterNet3Core({
|
const core = new mod.DeepFilterNet3Core({
|
||||||
sampleRate: 48_000,
|
sampleRate: 48_000,
|
||||||
// 60, not 80: full-strength suppression is the main source of the
|
// 60, not 80: full-strength suppression is the main source of the
|
||||||
@@ -379,7 +396,10 @@ export class LotusDenoiseProcessor
|
|||||||
numberOfOutputs: 1,
|
numberOfOutputs: 1,
|
||||||
processorOptions: { maxChannels: 1, wasmBinary },
|
processorOptions: { maxChannels: 1, wasmBinary },
|
||||||
});
|
});
|
||||||
return { node, dispose: () => void safeCall(() => node.port.postMessage("destroy")) };
|
return {
|
||||||
|
node,
|
||||||
|
dispose: () => void safeCall(() => node.port.postMessage("destroy")),
|
||||||
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
private disposeGraph(graph: Graph | undefined): void {
|
private disposeGraph(graph: Graph | undefined): void {
|
||||||
|
|||||||
Reference in New Issue
Block a user