fix(lotus-denoise): reliability — never-silent watchdog, resume timeout, cache + activation
Track-A robustness fixes from the engine review; no quality/model changes. - H1: auto-resume the AudioContext on `statechange` if it suspends mid-call (mobile backgrounding / audio interruption). Previously the dest node emitted digital silence with no recovery — a silent mute of the sender. - H2: `resumeCtx()` races `resume()` against a timeout. A suspended context can only resume on a user gesture; the action can arrive via postMessage, so a bare `await resume()` inside LiveKit's track-change lock could hang and deadlock all later mute/unmute/device-switch. Now it proceeds and the H1 watcher heals it. - M1: don't cache a REJECTED wasm fetch — a transient blip during a reconnect used to permanently disable denoise for the session. Evict on failure. - M2: activate denoise off `allConnections$` (local participant's connections) instead of `livekitRoomItems$`, which excludes the local participant and only surfaces rooms with a remote member — so denoise now also runs when you're alone and no longer couples to a remote-render concern. - Context lifecycle: `closeContext()` removes the state watcher before closing; `ensureContext()` closes a half-initialised context on any failure (no leak). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
parent
a4309f3e0c
commit
9a4e9bf7da
@@ -101,8 +101,12 @@ export function startLotusDenoise(vm: CallViewModel): () => void {
|
|||||||
const roomListeners = new Map<LivekitRoom, () => void>();
|
const roomListeners = new Map<LivekitRoom, () => void>();
|
||||||
let rooms: LivekitRoom[] = [];
|
let rooms: LivekitRoom[] = [];
|
||||||
|
|
||||||
const sub = vm.livekitRoomItems$.subscribe((items) => {
|
// Drive activation off the LOCAL participant's connection(s), not
|
||||||
const next = items.map((i) => i.livekitRoom);
|
// `livekitRoomItems$` — that stream excludes the local participant and only
|
||||||
|
// surfaces rooms with ≥1 remote member, so it wouldn't denoise you while
|
||||||
|
// you're alone and is a fragile coupling to a remote-render concern.
|
||||||
|
const sub = vm.allConnections$.subscribe((data) => {
|
||||||
|
const next = data.getConnections().map((c) => c.livekitRoom);
|
||||||
rooms = next;
|
rooms = next;
|
||||||
for (const [room, off] of roomListeners) {
|
for (const [room, off] of roomListeners) {
|
||||||
if (!next.includes(room)) {
|
if (!next.includes(room)) {
|
||||||
|
|||||||
@@ -64,18 +64,41 @@ const sampleRateFor = (model: LotusDenoiseModel): number =>
|
|||||||
|
|
||||||
// Cache fetched wasm per URL so a reconnect/device-switch doesn't re-download.
|
// Cache fetched wasm per URL so a reconnect/device-switch doesn't re-download.
|
||||||
const wasmCache = new Map<string, Promise<ArrayBuffer>>();
|
const wasmCache = new Map<string, Promise<ArrayBuffer>>();
|
||||||
function fetchWasm(url: string): Promise<ArrayBuffer> {
|
async function fetchWasmUncached(url: string): Promise<ArrayBuffer> {
|
||||||
|
const r = await fetch(url);
|
||||||
|
if (!r.ok) throw new Error(`denoise wasm ${url} -> ${r.status}`);
|
||||||
|
return r.arrayBuffer();
|
||||||
|
}
|
||||||
|
async function fetchWasm(url: string): Promise<ArrayBuffer> {
|
||||||
let p = wasmCache.get(url);
|
let p = wasmCache.get(url);
|
||||||
if (!p) {
|
if (!p) {
|
||||||
p = fetch(url).then((r) => {
|
p = fetchWasmUncached(url);
|
||||||
if (!r.ok) throw new Error(`denoise wasm ${url} -> ${r.status}`);
|
// Never cache a REJECTED fetch: a transient failure (e.g. a blip during a
|
||||||
return r.arrayBuffer();
|
// reconnect) must not permanently disable denoise for the whole session.
|
||||||
});
|
// Evict on failure so the next restart/device-switch retries.
|
||||||
|
void p.catch(() => wasmCache.delete(url));
|
||||||
wasmCache.set(url, p);
|
wasmCache.set(url, p);
|
||||||
}
|
}
|
||||||
return p;
|
return p;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Resume an AudioContext, but never block indefinitely. A suspended context can
|
||||||
|
* only resume after a user gesture; the denoise action can arrive via host
|
||||||
|
* postMessage (no gesture), so `resume()` may stay pending forever. Since this
|
||||||
|
* runs inside LiveKit's per-track change lock, a hung resume() would deadlock
|
||||||
|
* every later mute/unmute/device-switch. Race it against a timeout and proceed
|
||||||
|
* either way — the processor's `statechange` watcher resumes it once a gesture
|
||||||
|
* lands, and a still-suspended context degrades to (temporary) silence that the
|
||||||
|
* watcher heals, not a hang.
|
||||||
|
*/
|
||||||
|
async function resumeCtx(ctx: AudioContext, timeoutMs = 3_000): Promise<void> {
|
||||||
|
await Promise.race([
|
||||||
|
ctx.resume().catch(() => undefined),
|
||||||
|
new Promise<void>((resolve) => setTimeout(resolve, timeoutMs)),
|
||||||
|
]);
|
||||||
|
}
|
||||||
|
|
||||||
function supportsSimd(): boolean {
|
function supportsSimd(): boolean {
|
||||||
try {
|
try {
|
||||||
// Minimal SIMD module (v128) — validates only where SIMD is supported.
|
// Minimal SIMD module (v128) — validates only where SIMD is supported.
|
||||||
@@ -123,6 +146,7 @@ export class LotusDenoiseProcessor
|
|||||||
|
|
||||||
private ctx?: AudioContext;
|
private ctx?: AudioContext;
|
||||||
private graph?: Graph;
|
private graph?: Graph;
|
||||||
|
private ctxStateHandler?: () => void;
|
||||||
|
|
||||||
public constructor(private readonly config: LotusDenoiseConfig) {}
|
public constructor(private readonly config: LotusDenoiseConfig) {}
|
||||||
|
|
||||||
@@ -152,38 +176,63 @@ export class LotusDenoiseProcessor
|
|||||||
this.disposeGraph(this.graph);
|
this.disposeGraph(this.graph);
|
||||||
this.graph = undefined;
|
this.graph = undefined;
|
||||||
this.processedTrack = undefined;
|
this.processedTrack = undefined;
|
||||||
if (this.ctx && this.ctx.state !== "closed")
|
await this.closeContext();
|
||||||
await this.ctx.close().catch(() => undefined);
|
}
|
||||||
|
|
||||||
|
/** Remove the state watcher and close the owned context, if any. */
|
||||||
|
private async closeContext(): Promise<void> {
|
||||||
|
const ctx = this.ctx;
|
||||||
|
if (!ctx) return;
|
||||||
|
if (this.ctxStateHandler) {
|
||||||
|
ctx.removeEventListener("statechange", this.ctxStateHandler);
|
||||||
|
this.ctxStateHandler = undefined;
|
||||||
|
}
|
||||||
this.ctx = undefined;
|
this.ctx = undefined;
|
||||||
|
if (ctx.state !== "closed") await ctx.close().catch(() => undefined);
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Create (once) the model-rate context + register the flat worklet modules. */
|
/** Create (once) the model-rate context + register the flat worklet modules. */
|
||||||
private async ensureContext(): Promise<void> {
|
private async ensureContext(): Promise<void> {
|
||||||
const rate = sampleRateFor(this.config.model);
|
const rate = sampleRateFor(this.config.model);
|
||||||
if (this.ctx && this.ctx.state !== "closed" && this.ctx.sampleRate === rate) {
|
if (this.ctx && this.ctx.state !== "closed" && this.ctx.sampleRate === rate) {
|
||||||
if (this.ctx.state === "suspended") await this.ctx.resume();
|
if (this.ctx.state === "suspended") await resumeCtx(this.ctx);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
if (this.ctx && this.ctx.state !== "closed")
|
await this.closeContext();
|
||||||
await this.ctx.close().catch(() => undefined);
|
|
||||||
|
|
||||||
const ctx = new AudioContext({ sampleRate: rate });
|
const ctx = new AudioContext({ sampleRate: rate });
|
||||||
if (ctx.sampleRate !== rate) {
|
try {
|
||||||
|
if (ctx.sampleRate !== rate)
|
||||||
|
throw new Error(`denoise: got ${ctx.sampleRate}Hz, need ${rate}Hz`);
|
||||||
|
|
||||||
|
// Auto-resume if the OS/browser suspends the context mid-call (mobile
|
||||||
|
// backgrounding, audio interruption): the dest node otherwise emits
|
||||||
|
// silence with no recovery. Only resume while a graph is live.
|
||||||
|
const onStateChange = (): void => {
|
||||||
|
if (ctx.state === "suspended" && this.graph)
|
||||||
|
void ctx.resume().catch(() => undefined);
|
||||||
|
};
|
||||||
|
ctx.addEventListener("statechange", onStateChange);
|
||||||
|
|
||||||
|
// Flat models register via addModule here; DTLN/DeepFilterNet bring their
|
||||||
|
// own processor via the dynamic-imported helper (see buildMlNode).
|
||||||
|
if (this.config.model === "rnnoise" || this.config.model === "speex")
|
||||||
|
await ctx.audioWorklet.addModule(
|
||||||
|
this.config.assetBase + FLAT[this.config.model].script,
|
||||||
|
);
|
||||||
|
if (this.config.gate)
|
||||||
|
await ctx.audioWorklet.addModule(this.config.assetBase + GATE.script);
|
||||||
|
// The action can arrive via host postMessage, not a gesture in this
|
||||||
|
// iframe, so the context can start suspended — resume without hanging.
|
||||||
|
if (ctx.state === "suspended") await resumeCtx(ctx);
|
||||||
|
|
||||||
|
this.ctx = ctx;
|
||||||
|
this.ctxStateHandler = onStateChange;
|
||||||
|
} catch (e) {
|
||||||
|
// Don't leak a half-initialised context on any failure path.
|
||||||
await ctx.close().catch(() => undefined);
|
await ctx.close().catch(() => undefined);
|
||||||
throw new Error(`denoise: got ${ctx.sampleRate}Hz, need ${rate}Hz`);
|
throw e;
|
||||||
}
|
}
|
||||||
// Flat models register via addModule here; DTLN/DeepFilterNet bring their
|
|
||||||
// own processor via the dynamic-imported helper (see buildMlNode).
|
|
||||||
if (this.config.model === "rnnoise" || this.config.model === "speex")
|
|
||||||
await ctx.audioWorklet.addModule(
|
|
||||||
this.config.assetBase + FLAT[this.config.model].script,
|
|
||||||
);
|
|
||||||
if (this.config.gate)
|
|
||||||
await ctx.audioWorklet.addModule(this.config.assetBase + GATE.script);
|
|
||||||
// The action arrives via host postMessage, not a gesture in this iframe, so
|
|
||||||
// the context can start suspended.
|
|
||||||
if (ctx.state === "suspended") await ctx.resume();
|
|
||||||
this.ctx = ctx;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private async buildGraph(track: MediaStreamTrack): Promise<Graph> {
|
private async buildGraph(track: MediaStreamTrack): Promise<Graph> {
|
||||||
|
|||||||
Reference in New Issue
Block a user