diff --git a/src/lotus/lotusDenoise.ts b/src/lotus/lotusDenoise.ts index af916664..d785060e 100644 --- a/src/lotus/lotusDenoise.ts +++ b/src/lotus/lotusDenoise.ts @@ -53,16 +53,13 @@ function safeAssetBase(raw: string | null): string { export function startLotusDenoise(vm: CallViewModel): () => void { if (lotusParam("lotusDenoise") !== "ml") return () => undefined; - const requestedModel = lotusParam("lotusModel"); - // Only RNNoise/Speex are implemented in-source so far. Don't silently - // degrade an unsupported model (e.g. the host's dtln/deepfilternet) to - // rnnoise — log it so the mismatch is visible. - let model: LotusDenoiseConfig["model"] = "rnnoise"; - if (requestedModel === "speex") model = "speex"; - else if (requestedModel && requestedModel !== "rnnoise") - logger.warn( - `[lotus] denoise model "${requestedModel}" not implemented in-source; using rnnoise`, - ); + const requested = lotusParam("lotusModel"); + const model: LotusDenoiseConfig["model"] = + requested === "speex" || + requested === "dtln" || + requested === "deepfilternet" + ? requested + : "rnnoise"; const rawThreshold = lotusParam("lotusGateThreshold"); const config: LotusDenoiseConfig = { diff --git a/src/lotus/lotusDenoiseProcessor.ts b/src/lotus/lotusDenoiseProcessor.ts index f6f55fbd..3600cd2b 100644 --- a/src/lotus/lotusDenoiseProcessor.ts +++ b/src/lotus/lotusDenoiseProcessor.ts @@ -12,35 +12,37 @@ import { } from "livekit-client"; import { logger } from "matrix-js-sdk/lib/logger"; -export type LotusDenoiseModel = "rnnoise" | "speex"; +export type LotusDenoiseModel = + | "rnnoise" + | "speex" + | "dtln" + | "deepfilternet"; export interface LotusDenoiseConfig { model: LotusDenoiseModel; - /** Base URL the worklet scripts + wasm are served from (e.g. "./denoise/"). */ + /** Base URL the worklet scripts/wasm/ESM are served from (e.g. "./denoise/"). */ assetBase: string; gate: boolean; gateThreshold: number; } -// Flat sapphi worklets: each registers a processor under these names when its -// script module is added. Same assets the Lotus host already ships under -// public/element-call/denoise/, so no new EC dependency is needed. -// -// ⚠️ CONTRACT: this is an undeclared, cross-repo asset dependency. The embedded -// fork build does NOT bundle these — they are copied in by cinny's -// vite.config.js `lotusDenoise()` plugin from `@sapphi-red/web-noise-suppressor`. -// The worklet/wasm version must match what this processor expects (sapphi -// rnnoise/speex, 48kHz). If the fork ever wants to own them, add -// `@sapphi-red/web-noise-suppressor` as a dep + a copy step here. Until then, -// an integration smoke-check should assert GET .../denoise/rnnoise.wasm == 200. -const PROCESSORS: Record< - LotusDenoiseModel, - { name: string; script: string; wasm: string } +// Flat sapphi worklets (RNNoise/Speex): each registers a processor under `name` +// when its `script` module is added; we feed it the fetched wasm binary. +// ⚠️ CONTRACT: these assets are NOT bundled by the fork build — cinny's +// vite.config.js `lotusDenoise()` plugin copies them from +// `@sapphi-red/web-noise-suppressor` / `@workadventure/noise-suppression` / +// `deepfilternet3-noise-filter` into public/element-call/denoise/. The +// worklet/wasm/ESM versions must match what this processor expects. An +// integration smoke-check should assert GET .../denoise/rnnoise.wasm == 200. +const FLAT: Record< + "rnnoise" | "speex", + { name: string; script: string; wasm: string; simdWasm?: string } > = { rnnoise: { name: "@sapphi-red/web-noise-suppressor/rnnoise", script: "rnnoiseWorklet.js", wasm: "rnnoise.wasm", + simdWasm: "rnnoise_simd.wasm", }, speex: { name: "@sapphi-red/web-noise-suppressor/speex", @@ -48,21 +50,25 @@ const PROCESSORS: Record< wasm: "speex.wasm", }, }; -const GATE_NAME = "@sapphi-red/web-noise-suppressor/noiseGate"; -const GATE_SCRIPT = "noiseGateWorklet.js"; +// The sapphi gate worklet registers under "noise-gate" (hyphenated). +const GATE = { + name: "@sapphi-red/web-noise-suppressor/noise-gate", + script: "noiseGateWorklet.js", +}; -// The sapphi rnnoise/speex worklets are 48kHz-fullband and don't resample, so -// the graph MUST run at 48kHz regardless of the hardware default. -const SAMPLE_RATE = 48_000; +// DTLN (@workadventure) targets 16kHz and doesn't resample; RNNoise/Speex and +// DeepFilterNet are 48kHz fullband. The worklets don't resample, so the whole +// graph must run at the model's native rate. +const sampleRateFor = (model: LotusDenoiseModel): number => + model === "dtln" ? 16_000 : 48_000; -// Cache fetched wasm per URL so a reconnect/device-switch (which rebuilds the -// graph) doesn't re-download it. +// Cache fetched wasm per URL so a reconnect/device-switch doesn't re-download. const wasmCache = new Map>(); -function loadWasm(url: string): Promise { +function fetchWasm(url: string): Promise { let p = wasmCache.get(url); if (!p) { p = fetch(url).then((r) => { - if (!r.ok) throw new Error(`denoise wasm ${r.status}`); + if (!r.ok) throw new Error(`denoise wasm ${url} -> ${r.status}`); return r.arrayBuffer(); }); wasmCache.set(url, p); @@ -70,27 +76,44 @@ function loadWasm(url: string): Promise { return p; } +function supportsSimd(): boolean { + try { + // Minimal SIMD module (v128) — validates only where SIMD is supported. + return WebAssembly.validate( + new Uint8Array([ + 0, 97, 115, 109, 1, 0, 0, 0, 1, 5, 1, 96, 0, 1, 123, 3, 2, 1, 0, 10, + 10, 1, 8, 0, 65, 0, 253, 15, 253, 98, 11, + ]), + ); + } catch { + return false; + } +} + +interface MlNode { + node: AudioNode; + dispose?: () => void; +} interface Graph { source: MediaStreamAudioSourceNode; - nodes: AudioWorkletNode[]; + nodes: AudioNode[]; + disposes: (() => void)[]; track: MediaStreamTrack; } /** * A LiveKit audio TrackProcessor that runs Lotus ML noise suppression - * (RNNoise/Speex) on the local microphone track, as a first-class stage in - * Element Call's publish pipeline. + * (RNNoise / Speex / DTLN / DeepFilterNet) on the local microphone track, as a + * first-class stage in Element Call's publish pipeline — replacing the host's + * `getUserMedia` monkeypatch. * - * This replaces the host's `getUserMedia` monkeypatch: because it's a real - * LiveKit processor, EC re-applies it on every (re)publish/restart, so denoise - * survives EC's own mid-call reconnect — the root cause of the A7 "mic dead - * after reconnect" bug. - * - * It owns a dedicated 48kHz AudioContext (the worklets require it, and LiveKit - * does NOT pass an audioContext to restart()), reused across restarts and - * closed on destroy. restart() never throws and never leaves a stopped track - * on the sender: on failure it degrades to the raw (un-denoised) track rather - * than silence. + * Because it's a real LiveKit processor, EC re-applies it on every + * (re)publish/restart, so denoise survives EC's mid-call reconnect — the root + * cause of A7. It owns a dedicated AudioContext at the model's required sample + * rate (LiveKit does NOT pass an audioContext to restart()), reused across + * restarts and closed on destroy. restart() never throws and never leaves a + * stopped track on the sender: on failure it degrades to the RAW mic track + * rather than silence. */ export class LotusDenoiseProcessor implements TrackProcessor @@ -104,18 +127,14 @@ export class LotusDenoiseProcessor public constructor(private readonly config: LotusDenoiseConfig) {} public async init(_opts: AudioProcessorOptions): Promise { - // Own context at the required rate — do NOT use opts.audioContext (it's the - // room's hardware-rate context, and is undefined on restart()). - this.ctx = new AudioContext({ sampleRate: SAMPLE_RATE }); - const graph = await this.buildGraph(_opts.track); - this.graph = graph; - this.processedTrack = graph.track; + await this.ensureContext(); + this.graph = await this.buildGraph(_opts.track); + this.processedTrack = this.graph.track; } public async restart(opts: AudioProcessorOptions): Promise { - if (!this.ctx || this.ctx.state === "closed") - this.ctx = new AudioContext({ sampleRate: SAMPLE_RATE }); try { + await this.ensureContext(); const next = await this.buildGraph(opts.track); this.disposeGraph(this.graph); this.graph = next; @@ -138,22 +157,45 @@ export class LotusDenoiseProcessor this.ctx = undefined; } + /** Create (once) the model-rate context + register the flat worklet modules. */ + private async ensureContext(): Promise { + const rate = sampleRateFor(this.config.model); + if (this.ctx && this.ctx.state !== "closed" && this.ctx.sampleRate === rate) { + if (this.ctx.state === "suspended") await this.ctx.resume(); + return; + } + if (this.ctx && this.ctx.state !== "closed") + await this.ctx.close().catch(() => undefined); + + const ctx = new AudioContext({ sampleRate: rate }); + if (ctx.sampleRate !== rate) { + await ctx.close().catch(() => undefined); + throw new Error(`denoise: got ${ctx.sampleRate}Hz, need ${rate}Hz`); + } + // Flat models register via addModule here; DTLN/DeepFilterNet bring their + // own processor via the dynamic-imported helper (see buildMlNode). + if (this.config.model === "rnnoise" || this.config.model === "speex") + await ctx.audioWorklet.addModule( + this.config.assetBase + FLAT[this.config.model].script, + ); + if (this.config.gate) + await ctx.audioWorklet.addModule(this.config.assetBase + GATE.script); + // The action arrives via host postMessage, not a gesture in this iframe, so + // the context can start suspended. + if (ctx.state === "suspended") await ctx.resume(); + this.ctx = ctx; + } + private async buildGraph(track: MediaStreamTrack): Promise { const ctx = this.ctx!; - const base = this.config.assetBase; - const proc = PROCESSORS[this.config.model]; - - await ctx.audioWorklet.addModule(base + proc.script); - if (this.config.gate) await ctx.audioWorklet.addModule(base + GATE_SCRIPT); - const wasmBinary = await loadWasm(base + proc.wasm); - const source = ctx.createMediaStreamSource(new MediaStream([track])); const dest = ctx.createMediaStreamDestination(); - const nodes: AudioWorkletNode[] = []; + const nodes: AudioNode[] = []; + const disposes: (() => void)[] = []; let head: AudioNode = source; if (this.config.gate) { - const gate = new AudioWorkletNode(ctx, GATE_NAME, { + const gate = new AudioWorkletNode(ctx, GATE.name, { processorOptions: { openThreshold: this.config.gateThreshold, closeThreshold: this.config.gateThreshold - 5, @@ -166,39 +208,77 @@ export class LotusDenoiseProcessor nodes.push(gate); } - const ml = new AudioWorkletNode(ctx, proc.name, { + const ml = await this.buildMlNode(ctx); + head.connect(ml.node); + ml.node.connect(dest); + nodes.push(ml.node); + if (ml.dispose) disposes.push(ml.dispose); + + logger.info(`[lotus] denoise processor active (${this.config.model})`); + return { source, nodes, disposes, track: dest.stream.getAudioTracks()[0] }; + } + + private async buildMlNode(ctx: AudioContext): Promise { + const base = this.config.assetBase; + const model = this.config.model; + + if (model === "dtln") { + // Self-contained ESM that resolves its own processor + LiteRT wasm + + // TFLite models. bypassUntilReady passes raw audio until the model loads. + const mod = await import(/* @vite-ignore */ `${base}workadventure/audio-worklet.js`); + return (await mod.createNoiseSuppressionAudioWorklet(ctx, { + bypassUntilReady: true, + })) as MlNode; + } + + if (model === "deepfilternet") { + const dfnBase = new URL(`${base}deepfilternet`, window.location.href).href; + const mod = await import(/* @vite-ignore */ `${base}deepfilternet/index.esm.js`); + const core = new mod.DeepFilterNet3Core({ + sampleRate: 48_000, + noiseReductionLevel: 80, + assetConfig: { cdnUrl: dfnBase }, + }); + await core.initialize(); + const node = (await core.createAudioWorkletNode(ctx)) as AudioNode; + return { node, dispose: () => void safeCall(() => core.destroy()) }; + } + + // Flat sapphi worklet (rnnoise/speex). + const flat = FLAT[model]; + const useSimd = model === "rnnoise" && !!flat.simdWasm && supportsSimd(); + const wasmFile = useSimd ? flat.simdWasm! : flat.wasm; + let wasmBinary: ArrayBuffer; + try { + wasmBinary = await fetchWasm(base + wasmFile); + } catch (e) { + if (useSimd) { + wasmCache.delete(base + wasmFile); + wasmBinary = await fetchWasm(base + flat.wasm); // fall back to non-SIMD + } else throw e; + } + const node = new AudioWorkletNode(ctx, flat.name, { channelCount: 1, numberOfInputs: 1, numberOfOutputs: 1, - processorOptions: { maxChannels: 1, wasmBinary: wasmBinary.slice(0) }, + processorOptions: { maxChannels: 1, wasmBinary }, }); - head.connect(ml); - ml.connect(dest); - nodes.push(ml); - - logger.info(`[lotus] denoise processor active (${this.config.model})`); - return { source, nodes, track: dest.stream.getAudioTracks()[0] }; + return { node, dispose: () => void safeCall(() => node.port.postMessage("destroy")) }; } private disposeGraph(graph: Graph | undefined): void { if (!graph) return; - for (const node of graph.nodes) { - try { - node.port.postMessage("destroy"); - } catch { - /* ignore */ - } - try { - node.disconnect(); - } catch { - /* ignore */ - } - } - try { - graph.source.disconnect(); - } catch { - /* ignore */ - } + for (const dispose of graph.disposes) safeCall(dispose); + for (const node of graph.nodes) safeCall(() => node.disconnect()); + safeCall(() => graph.source.disconnect()); graph.track.stop(); } } + +function safeCall(fn: () => void): void { + try { + fn(); + } catch { + /* ignore */ + } +}