/* Copyright 2026 Lotus Guild SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ import { type AudioProcessorOptions, type Track, type TrackProcessor, } from "livekit-client"; import { logger } from "matrix-js-sdk/lib/logger"; export type LotusDenoiseModel = "rnnoise" | "speex" | "dtln" | "deepfilternet"; export interface LotusDenoiseConfig { model: LotusDenoiseModel; /** Base URL the worklet scripts/wasm/ESM are served from (e.g. "./denoise/"). */ assetBase: string; gate: boolean; gateThreshold: number; /** * Attenuation floor as a dry/wet mix: the fraction (0..~0.3) of the ORIGINAL * mic blended back under the denoised signal so full suppression never fully * collapses the noise floor — this is what kills the "underwater"/pumping * artifact. 0.15 ≈ a -16 dB floor. 0 = full suppression (no floor). */ floor: number; } // Flat sapphi worklets (RNNoise/Speex): each registers a processor under `name` // when its `script` module is added; we feed it the fetched wasm binary. // ⚠️ CONTRACT: these assets are NOT bundled by the fork build — cinny's // vite.config.js `lotusDenoise()` plugin copies them from // `@sapphi-red/web-noise-suppressor` / `@workadventure/noise-suppression` / // `deepfilternet3-noise-filter` into public/element-call/denoise/. The // worklet/wasm/ESM versions must match what this processor expects. An // integration smoke-check should assert GET .../denoise/rnnoise.wasm == 200. const FLAT: Record< "rnnoise" | "speex", { name: string; script: string; wasm: string; simdWasm?: string } > = { rnnoise: { name: "@sapphi-red/web-noise-suppressor/rnnoise", script: "rnnoiseWorklet.js", wasm: "rnnoise.wasm", simdWasm: "rnnoise_simd.wasm", }, speex: { name: "@sapphi-red/web-noise-suppressor/speex", script: "speexWorklet.js", wasm: "speex.wasm", }, }; // The sapphi gate worklet registers under "noise-gate" (hyphenated). const GATE = { name: "@sapphi-red/web-noise-suppressor/noise-gate", script: "noiseGateWorklet.js", }; // DTLN (@workadventure) targets 16kHz and doesn't resample; RNNoise/Speex and // DeepFilterNet are 48kHz fullband. The worklets don't resample, so the whole // graph must run at the model's native rate. const sampleRateFor = (model: LotusDenoiseModel): number => model === "dtln" ? 16_000 : 48_000; // Cache fetched wasm per URL so a reconnect/device-switch doesn't re-download. const wasmCache = new Map>(); async function fetchWasmUncached(url: string): Promise { const r = await fetch(url); if (!r.ok) throw new Error(`denoise wasm ${url} -> ${r.status}`); return r.arrayBuffer(); } async function fetchWasm(url: string): Promise { let p = wasmCache.get(url); if (!p) { p = fetchWasmUncached(url); // Never cache a REJECTED fetch: a transient failure (e.g. a blip during a // reconnect) must not permanently disable denoise for the whole session. // Evict on failure so the next restart/device-switch retries. void p.catch(() => wasmCache.delete(url)); wasmCache.set(url, p); } return p; } /** * Resume an AudioContext, but never block indefinitely. A suspended context can * only resume after a user gesture; the denoise action can arrive via host * postMessage (no gesture), so `resume()` may stay pending forever. Since this * runs inside LiveKit's per-track change lock, a hung resume() would deadlock * every later mute/unmute/device-switch. Race it against a timeout and proceed * either way — the processor's `statechange` watcher resumes it once a gesture * lands, and a still-suspended context degrades to (temporary) silence that the * watcher heals, not a hang. */ // [lotus] 500 ms, not 3 s: this still runs under LiveKit's trackChangeLock (#7), // and the statechange watcher heals a still-suspended context later anyway. async function resumeCtx(ctx: AudioContext, timeoutMs = 500): Promise { await Promise.race([ ctx.resume().catch(() => undefined), new Promise((resolve) => setTimeout(resolve, timeoutMs)), ]); } function supportsSimd(): boolean { try { // Minimal SIMD module (v128) — validates only where SIMD is supported. return WebAssembly.validate( new Uint8Array([ 0, 97, 115, 109, 1, 0, 0, 0, 1, 5, 1, 96, 0, 1, 123, 3, 2, 1, 0, 10, 10, 1, 8, 0, 65, 0, 253, 15, 253, 98, 11, ]), ); } catch { return false; } } interface MlNode { node: AudioNode; dispose?: () => void; } interface Graph { source: MediaStreamAudioSourceNode; nodes: AudioNode[]; disposes: (() => void)[]; track: MediaStreamTrack; } // [lotus #26] Minimal local contracts for the two dynamically-imported ESM // helpers (not bundled here — see the CONTRACT note above). Their exports are // asserted at runtime so an asset bump that renames/removes one fails loudly // (and flows into the rnnoise fallback in lotusDenoise.ts) instead of as a // vague TypeError deep inside `init()`. interface DtlnModule { createNoiseSuppressionAudioWorklet: ( ctx: AudioContext, opts: { bypassUntilReady: boolean }, ) => Promise; } interface DfnCore { initialize: () => Promise; createAudioWorkletNode: (ctx: AudioContext) => Promise; destroy: () => void; } interface DfnModule { DeepFilterNet3Core: new (opts: { sampleRate: number; noiseReductionLevel: number; assetConfig: { cdnUrl: string }; }) => DfnCore; } /** Throw a clear error if a dynamic-import module lacks an expected export. */ export function assertModuleExport( mod: unknown, name: string, url: string, ): T { const exp = (mod as Record | null | undefined)?.[name]; if (typeof exp !== "function") throw new Error( `denoise: ${url} does not export ${name} (got ${typeof exp}) — asset/version mismatch`, ); return mod as T; } async function loadDfnCore(config: LotusDenoiseConfig): Promise { const base = config.assetBase; const url = `${base}deepfilternet/index.esm.js`; const dfnBase = new URL(`${base}deepfilternet`, window.location.href).href; const mod = assertModuleExport( await import(/* @vite-ignore */ url), "DeepFilterNet3Core", url, ); const core = new mod.DeepFilterNet3Core({ sampleRate: 48_000, // 60, not 80: full-strength suppression is the main source of the // "over-processed" character; a lower level keeps voice natural while // the dry/wet floor handles the noise tail. noiseReductionLevel: 60, assetConfig: { cdnUrl: dfnBase }, }); await core.initialize(); return core; } async function loadDtlnModule(config: LotusDenoiseConfig): Promise { const url = `${config.assetBase}workadventure/audio-worklet.js`; return assertModuleExport( await import(/* @vite-ignore */ url), "createNoiseSuppressionAudioWorklet", url, ); } /** Which wasm file a flat model uses (SIMD build when supported). */ function flatWasmFiles(model: "rnnoise" | "speex"): { primary: string; fallback?: string; } { const flat = FLAT[model]; const useSimd = model === "rnnoise" && !!flat.simdWasm && supportsSimd(); return useSimd ? { primary: flat.simdWasm!, fallback: flat.wasm } : { primary: flat.wasm }; } // [lotus #24] Force every node in the graph to a single, explicitly-downmixed // channel. Without `channelCountMode: "explicit"` the default ("max") IGNORES // `channelCount`, so a stereo capture device would feed 2 channels into a // worklet configured with `maxChannels: 1` and sum a stereo dry copy against a // mono wet one at the destination. const MONO: AudioNodeOptions = { channelCount: 1, channelCountMode: "explicit", channelInterpretation: "speakers", }; // [lotus #25] Algorithmic latency of each model in samples at its native rate, // used to delay the DRY copy of the floor mix so it lines up with the wet path // (otherwise the sum comb-filters — a hollow/phasey colouration on voice). // - rnnoise: 480-sample (10 ms @ 48 kHz) frames; the sapphi worklet buffers // 128-sample quanta up to one frame, so the wet path lags by one frame. // - speex: the sapphi speex worklet uses the same 480-sample framing. // - dtln: 512-sample block / 128 hop @ 16 kHz (~32 ms) per the DTLN paper — // best-known, unmeasured (the floor is not mixed for dtln, see buildGraph). // - deepfilternet: 480-sample hop + 2-frame lookahead @ 48 kHz (~30 ms) per // DeepFilterNet3 — best-known, unmeasured (floor not mixed for dfn either). const DRY_DELAY_SAMPLES: Record = { rnnoise: 480, speex: 480, dtln: 512, deepfilternet: 1440, }; /** * Create the model-rate context and register the flat/gate worklet modules. * Closes the context (and rethrows) on any failure so nothing half-built leaks. */ async function createModelContext( config: LotusDenoiseConfig, ): Promise { const rate = sampleRateFor(config.model); const ctx = new AudioContext({ sampleRate: rate }); try { if (ctx.sampleRate !== rate) throw new Error(`denoise: got ${ctx.sampleRate}Hz, need ${rate}Hz`); // Flat models register via addModule here; DTLN/DeepFilterNet bring their // own processor via the dynamic-imported helper (see buildMlNode). if (config.model === "rnnoise" || config.model === "speex") await ctx.audioWorklet.addModule( config.assetBase + FLAT[config.model].script, ); if (config.gate) await ctx.audioWorklet.addModule(config.assetBase + GATE.script); return ctx; } catch (e) { await ctx.close().catch(() => undefined); throw e; } } // [lotus #7] Everything heavy that `init()` needs but that does NOT depend on // the mic track: the AudioContext + worklet modules, the flat wasm binary, and // (DFN) the fully-initialised model core. `LocalAudioTrack.setProcessor()` // holds LiveKit's `trackChangeLock` while awaiting `init()`, so every // mute/unmute/device-switch queues behind it — prepare these as soon as the // flag is seen (before any track exists) so `init()` only wires them up. interface PreparedAssets { ctx: AudioContext; dfnCore?: DfnCore; } const preparedAssets = new Map>(); const preparedKey = (c: LotusDenoiseConfig): string => `${c.model}|${c.gate ? 1 : 0}|${c.assetBase}`; async function prepareUncached( config: LotusDenoiseConfig, ): Promise { const ctx = await createModelContext(config); try { let dfnCore: DfnCore | undefined; if (config.model === "rnnoise" || config.model === "speex") { const { primary, fallback } = flatWasmFiles(config.model); // Warm the wasm cache; a SIMD miss is fine — buildMlNode falls back. await fetchWasm(config.assetBase + primary).catch(async () => fallback ? fetchWasm(config.assetBase + fallback) : undefined, ); } else if (config.model === "dtln") { await loadDtlnModule(config); // warms the browser's module map } else { dfnCore = await loadDfnCore(config); } return { ctx, dfnCore }; } catch (e) { await ctx.close().catch(() => undefined); throw e; } } /** * Prefetch/prepare the assets for `config` (idempotent per model). Never * rejects: a failed prepare is evicted so `init()` simply loads inline and * surfaces the real error there. */ export async function prepareDenoiseAssets( config: LotusDenoiseConfig, ): Promise { const key = preparedKey(config); let p = preparedAssets.get(key); if (!p) { p = prepareUncached(config); void p.catch((e) => { if (preparedAssets.get(key) === p) preparedAssets.delete(key); logger.warn(`[lotus] denoise prepare failed (${config.model})`, e); }); preparedAssets.set(key, p); } await p.catch(() => undefined); } /** Take (one-shot) the prepared assets for `config`, if any were prepared. */ function claimPreparedAssets( config: LotusDenoiseConfig, ): Promise | undefined { const key = preparedKey(config); const p = preparedAssets.get(key); if (p) preparedAssets.delete(key); return p; } /** Close any prepared-but-unclaimed contexts (call on feature teardown). */ export async function releasePreparedDenoiseAssets(): Promise { const all = [...preparedAssets.values()]; preparedAssets.clear(); await Promise.all( all.map(async (p) => p .then(async (a) => { safeCall(() => a.dfnCore?.destroy()); if (a.ctx.state !== "closed") await a.ctx.close(); }) .catch(() => undefined), ), ); } /** * A LiveKit audio TrackProcessor that runs Lotus ML noise suppression * (RNNoise / Speex / DTLN / DeepFilterNet) on the local microphone track, as a * first-class stage in Element Call's publish pipeline — replacing the host's * `getUserMedia` monkeypatch. * * Because it's a real LiveKit processor, EC re-applies it on every * (re)publish/restart, so denoise survives EC's mid-call reconnect — the root * cause of A7. It owns a dedicated AudioContext at the model's required sample * rate (LiveKit does NOT pass an audioContext to restart()), reused across * restarts and closed on destroy. restart() never throws and never leaves a * stopped track on the sender: on failure it degrades to the RAW mic track * rather than silence. */ export class LotusDenoiseProcessor implements TrackProcessor< Track.Kind.Audio, AudioProcessorOptions > { public readonly name = "lotus-denoise"; public processedTrack?: MediaStreamTrack; private ctx?: AudioContext; private graph?: Graph; private ctxStateHandler?: () => void; private preparedDfnCore?: DfnCore; // [lotus #9] True while the mic is muted: we suspend our own context so the // worklet stops running inference on silence, and the statechange watcher // must not "heal" that intentional suspension. private micMuted = false; public constructor(private readonly config: LotusDenoiseConfig) {} /** * [lotus #9] Mirror the mic's mute state onto the owned context. EC uses * `stopMicTrackOnMute: false`, so a muted mic keeps producing (silent) frames * and the ML worklet would otherwise keep running full inference for the * whole time the user is muted. */ public setMicMuted(muted: boolean): void { this.micMuted = muted; const ctx = this.ctx; if (!ctx || ctx.state === "closed") return; if (muted) { if (ctx.state === "running") void ctx.suspend().catch(() => undefined); } else if (ctx.state === "suspended" && this.graph) { void ctx.resume().catch(() => undefined); } } public async init(_opts: AudioProcessorOptions): Promise { try { await this.ensureContext(); this.graph = await this.buildGraph(_opts.track); this.processedTrack = this.graph.track; } catch (e) { // Don't orphan the owned context if graph construction fails (browsers // cap live AudioContexts, so repeated failed inits could exhaust them). // The caller degrades to the raw mic; we just release our resources. const core = this.preparedDfnCore; this.preparedDfnCore = undefined; if (core) safeCall(() => core.destroy()); await this.closeContext(); throw e; } } public async restart(opts: AudioProcessorOptions): Promise { try { await this.ensureContext(); const next = await this.buildGraph(opts.track); this.disposeGraph(this.graph); this.graph = next; this.processedTrack = next.track; } catch (e) { // Never go silent on the A7/device-switch path: fall back to raw audio. // [lotus] IMPORTANT: never assign `opts.track` (LiveKit-owned) here. // LiveKit's internalStopProcessor() does `processor.processedTrack?.stop()` // then re-publishes `_mediaStreamTrack` — the SAME object if we set it as // processedTrack — which kills the live mic on the next stopProcessor()/ // teardown. Leaving processedTrack undefined makes LiveKit fall through to // its own `_mediaStreamTrack` instead. logger.warn("[lotus] denoise restart failed; using raw mic", e); this.disposeGraph(this.graph); this.graph = undefined; this.processedTrack = undefined; } } public async destroy(): Promise { this.disposeGraph(this.graph); this.graph = undefined; this.processedTrack = undefined; const core = this.preparedDfnCore; this.preparedDfnCore = undefined; if (core) safeCall(() => core.destroy()); await this.closeContext(); } /** Remove the state watcher and close the owned context, if any. */ private async closeContext(): Promise { const ctx = this.ctx; if (!ctx) return; if (this.ctxStateHandler) { ctx.removeEventListener("statechange", this.ctxStateHandler); this.ctxStateHandler = undefined; } this.ctx = undefined; if (ctx.state !== "closed") await ctx.close().catch(() => undefined); } /** Adopt the prepared context (or create one) + install the state watcher. */ private async ensureContext(): Promise { const rate = sampleRateFor(this.config.model); if ( this.ctx && this.ctx.state !== "closed" && this.ctx.sampleRate === rate ) { if (this.ctx.state === "suspended" && !this.micMuted) await resumeCtx(this.ctx); return; } await this.closeContext(); // [lotus #7] Prefer the context/modules/model prepared before // setProcessor() was called; only load inline if nothing was prepared // (e.g. the rnnoise fallback path, or a second processor after a // republish). const claimed = await claimPreparedAssets(this.config)?.catch( () => undefined, ); let ctx: AudioContext; if (claimed && claimed.ctx.state !== "closed") { ctx = claimed.ctx; this.preparedDfnCore = claimed.dfnCore; } else { ctx = await createModelContext(this.config); } try { // Auto-resume if the OS/browser suspends the context mid-call (mobile // backgrounding, audio interruption): the dest node otherwise emits // silence with no recovery. Only resume while a graph is live and the // suspension isn't our own mute suspension (#9). const onStateChange = (): void => { if (ctx.state === "suspended" && this.graph && !this.micMuted) void ctx.resume().catch(() => undefined); }; ctx.addEventListener("statechange", onStateChange); // The action can arrive via host postMessage, not a gesture in this // iframe, so the context can start suspended — resume without hanging. if (ctx.state === "suspended" && !this.micMuted) await resumeCtx(ctx); // Attached while already muted (#9): don't let a prepared, running // context burn inference until the first unmute. else if (ctx.state === "running" && this.micMuted) await ctx.suspend().catch(() => undefined); this.ctx = ctx; this.ctxStateHandler = onStateChange; } catch (e) { // Don't leak a half-initialised context on any failure path. await ctx.close().catch(() => undefined); throw e; } } private async buildGraph(track: MediaStreamTrack): Promise { const ctx = this.ctx!; const source = ctx.createMediaStreamSource(new MediaStream([track])); const dest = new MediaStreamAudioDestinationNode(ctx, MONO); const nodes: AudioNode[] = []; const disposes: (() => void)[] = []; try { // Wet (denoised) path: source → ml → [gate] → wetGain. const ml = await this.buildMlNode(ctx); source.connect(ml.node); nodes.push(ml.node); if (ml.dispose) disposes.push(ml.dispose); let wetHead: AudioNode = ml.node; // Gate AFTER the ML model, not before: gating the raw noisy signal fed // hard-zeroed frames into the model (discontinuities it must fight) and // made the threshold operate on pre-denoise levels. Gate the residual. if (this.config.gate) { const gate = new AudioWorkletNode(ctx, GATE.name, { ...MONO, processorOptions: { openThreshold: this.config.gateThreshold, closeThreshold: this.config.gateThreshold - 5, holdMs: 150, maxChannels: 1, }, }); wetHead.connect(gate); wetHead = gate; nodes.push(gate); } // Only mix a dry floor for the flat models (RNNoise/Speex), whose // framing latency is known exactly (DRY_DELAY_SAMPLES); the DTLN/DFN // figures are best-known estimates, so for those we rely on the model's // own level (e.g. DFN noiseReductionLevel) instead. RNNoise is also where // the "robotic/underwater" reports come from, so this targets it. const lowLatency = this.config.model === "rnnoise" || this.config.model === "speex"; const floor = lowLatency ? Math.min(0.5, Math.max(0, this.config.floor)) : 0; if (floor > 0) { // Dry/wet mix: blend a small amount of the ORIGINAL mic under the // denoised signal so suppression can't fully collapse the noise floor // (kills the "underwater"/pumping artifact). During speech (denoised ≈ // original) the two sum back to ~unity; in noise-only gaps the output // floors at `floor` × original instead of digital silence. const wetGain = new GainNode(ctx, { ...MONO, gain: 1 - floor }); wetHead.connect(wetGain); wetGain.connect(dest); nodes.push(wetGain); // [lotus #25] Delay the dry copy by the model's algorithmic latency so // it sums in phase with the (framed, hence delayed) wet path instead // of comb-filtering against it. const delaySec = DRY_DELAY_SAMPLES[this.config.model] / ctx.sampleRate; const dryDelay = new DelayNode(ctx, { ...MONO, maxDelayTime: Math.max(delaySec, 1 / ctx.sampleRate), delayTime: delaySec, }); const dryGain = new GainNode(ctx, { ...MONO, gain: floor }); source.connect(dryDelay); dryDelay.connect(dryGain); dryGain.connect(dest); nodes.push(dryDelay, dryGain); } else { wetHead.connect(dest); } logger.info( `[lotus] denoise processor active (${this.config.model}, floor=${floor})`, ); return { source, nodes, disposes, track: dest.stream.getAudioTracks()[0], }; } catch (e) { // A node constructor / model load can throw mid-build; clean up the // partially-built graph so it doesn't leak (init/restart still fall back // to the raw mic on the rejection). this.disposeGraph({ source, nodes, disposes, track: dest.stream.getAudioTracks()[0], }); throw e; } } private async buildMlNode(ctx: AudioContext): Promise { const base = this.config.assetBase; const model = this.config.model; if (model === "dtln") { // Self-contained ESM that resolves its own processor + LiteRT wasm + // TFLite models. bypassUntilReady passes raw audio until the model loads. const mod = await loadDtlnModule(this.config); return await mod.createNoiseSuppressionAudioWorklet(ctx, { bypassUntilReady: true, }); } if (model === "deepfilternet") { // [lotus #7] Use the core initialised by prepareDenoiseAssets() if we // have one (first graph); later rebuilds (restart) load a fresh core. const prepared = this.preparedDfnCore; this.preparedDfnCore = undefined; const core = prepared ?? (await loadDfnCore(this.config)); const node = await core.createAudioWorkletNode(ctx); return { node, dispose: () => safeCall(() => core.destroy()) }; } // Flat sapphi worklet (rnnoise/speex). const flat = FLAT[model]; const { primary, fallback } = flatWasmFiles(model); let wasmBinary: ArrayBuffer; try { wasmBinary = await fetchWasm(base + primary); } catch (e) { if (fallback) { wasmCache.delete(base + primary); wasmBinary = await fetchWasm(base + fallback); // fall back to non-SIMD } else throw e; } const node = new AudioWorkletNode(ctx, flat.name, { ...MONO, numberOfInputs: 1, numberOfOutputs: 1, processorOptions: { maxChannels: 1, wasmBinary }, }); return { node, dispose: () => safeCall(() => node.port.postMessage("destroy")), }; } private disposeGraph(graph: Graph | undefined): void { if (!graph) return; for (const dispose of graph.disposes) safeCall(dispose); for (const node of graph.nodes) safeCall(() => node.disconnect()); safeCall(() => graph.source.disconnect()); graph.track.stop(); } } function safeCall(fn: () => void): void { try { fn(); } catch { /* ignore */ } }