Files
element-call/src/lotus/lotusDenoiseProcessor.ts
T

675 lines
25 KiB
TypeScript
Raw Normal View History

/*
Copyright 2026 Lotus Guild
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import {
type AudioProcessorOptions,
type Track,
type TrackProcessor,
} from "livekit-client";
import { logger } from "matrix-js-sdk/lib/logger";
2026-09-12 11:35:57 -04:00
export type LotusDenoiseModel = "rnnoise" | "speex" | "dtln" | "deepfilternet";
export interface LotusDenoiseConfig {
model: LotusDenoiseModel;
/** Base URL the worklet scripts/wasm/ESM are served from (e.g. "./denoise/"). */
assetBase: string;
gate: boolean;
gateThreshold: number;
/**
* Attenuation floor as a dry/wet mix: the fraction (0..~0.3) of the ORIGINAL
* mic blended back under the denoised signal so full suppression never fully
* collapses the noise floor — this is what kills the "underwater"/pumping
* artifact. 0.15 ≈ a -16 dB floor. 0 = full suppression (no floor).
*/
floor: number;
}
// Flat sapphi worklets (RNNoise/Speex): each registers a processor under `name`
// when its `script` module is added; we feed it the fetched wasm binary.
// ⚠️ CONTRACT: these assets are NOT bundled by the fork build — cinny's
// vite.config.js `lotusDenoise()` plugin copies them from
// `@sapphi-red/web-noise-suppressor` / `@workadventure/noise-suppression` /
// `deepfilternet3-noise-filter` into public/element-call/denoise/. The
// worklet/wasm/ESM versions must match what this processor expects. An
// integration smoke-check should assert GET .../denoise/rnnoise.wasm == 200.
const FLAT: Record<
"rnnoise" | "speex",
{ name: string; script: string; wasm: string; simdWasm?: string }
> = {
rnnoise: {
name: "@sapphi-red/web-noise-suppressor/rnnoise",
script: "rnnoiseWorklet.js",
wasm: "rnnoise.wasm",
simdWasm: "rnnoise_simd.wasm",
},
speex: {
name: "@sapphi-red/web-noise-suppressor/speex",
script: "speexWorklet.js",
wasm: "speex.wasm",
},
};
// The sapphi gate worklet registers under "noise-gate" (hyphenated).
const GATE = {
name: "@sapphi-red/web-noise-suppressor/noise-gate",
script: "noiseGateWorklet.js",
};
// DTLN (@workadventure) targets 16kHz and doesn't resample; RNNoise/Speex and
// DeepFilterNet are 48kHz fullband. The worklets don't resample, so the whole
// graph must run at the model's native rate.
const sampleRateFor = (model: LotusDenoiseModel): number =>
model === "dtln" ? 16_000 : 48_000;
// Cache fetched wasm per URL so a reconnect/device-switch doesn't re-download.
const wasmCache = new Map<string, Promise<ArrayBuffer>>();
async function fetchWasmUncached(url: string): Promise<ArrayBuffer> {
const r = await fetch(url);
if (!r.ok) throw new Error(`denoise wasm ${url} -> ${r.status}`);
return r.arrayBuffer();
}
async function fetchWasm(url: string): Promise<ArrayBuffer> {
let p = wasmCache.get(url);
if (!p) {
p = fetchWasmUncached(url);
// Never cache a REJECTED fetch: a transient failure (e.g. a blip during a
// reconnect) must not permanently disable denoise for the whole session.
// Evict on failure so the next restart/device-switch retries.
void p.catch(() => wasmCache.delete(url));
wasmCache.set(url, p);
}
return p;
}
/**
* Resume an AudioContext, but never block indefinitely. A suspended context can
* only resume after a user gesture; the denoise action can arrive via host
* postMessage (no gesture), so `resume()` may stay pending forever. Since this
* runs inside LiveKit's per-track change lock, a hung resume() would deadlock
* every later mute/unmute/device-switch. Race it against a timeout and proceed
* either way — the processor's `statechange` watcher resumes it once a gesture
* lands, and a still-suspended context degrades to (temporary) silence that the
* watcher heals, not a hang.
*/
// [lotus] 500 ms, not 3 s: this still runs under LiveKit's trackChangeLock (#7),
// and the statechange watcher heals a still-suspended context later anyway.
async function resumeCtx(ctx: AudioContext, timeoutMs = 500): Promise<void> {
await Promise.race([
ctx.resume().catch(() => undefined),
new Promise<void>((resolve) => setTimeout(resolve, timeoutMs)),
]);
}
function supportsSimd(): boolean {
try {
// Minimal SIMD module (v128) — validates only where SIMD is supported.
return WebAssembly.validate(
new Uint8Array([
2026-09-12 11:35:57 -04:00
0, 97, 115, 109, 1, 0, 0, 0, 1, 5, 1, 96, 0, 1, 123, 3, 2, 1, 0, 10, 10,
1, 8, 0, 65, 0, 253, 15, 253, 98, 11,
]),
);
} catch {
return false;
}
}
interface MlNode {
node: AudioNode;
dispose?: () => void;
}
interface Graph {
source: MediaStreamAudioSourceNode;
nodes: AudioNode[];
disposes: (() => void)[];
track: MediaStreamTrack;
}
// [lotus #26] Minimal local contracts for the two dynamically-imported ESM
// helpers (not bundled here — see the CONTRACT note above). Their exports are
// asserted at runtime so an asset bump that renames/removes one fails loudly
// (and flows into the rnnoise fallback in lotusDenoise.ts) instead of as a
// vague TypeError deep inside `init()`.
interface DtlnModule {
createNoiseSuppressionAudioWorklet: (
ctx: AudioContext,
opts: { bypassUntilReady: boolean },
) => Promise<MlNode>;
}
interface DfnCore {
initialize: () => Promise<void>;
createAudioWorkletNode: (ctx: AudioContext) => Promise<AudioNode>;
destroy: () => void;
}
interface DfnModule {
DeepFilterNet3Core: new (opts: {
sampleRate: number;
noiseReductionLevel: number;
assetConfig: { cdnUrl: string };
}) => DfnCore;
}
/** Throw a clear error if a dynamic-import module lacks an expected export. */
export function assertModuleExport<T>(
mod: unknown,
name: string,
url: string,
): T {
const exp = (mod as Record<string, unknown> | null | undefined)?.[name];
if (typeof exp !== "function")
throw new Error(
`denoise: ${url} does not export ${name} (got ${typeof exp}) — asset/version mismatch`,
);
return mod as T;
}
async function loadDfnCore(config: LotusDenoiseConfig): Promise<DfnCore> {
const base = config.assetBase;
const url = `${base}deepfilternet/index.esm.js`;
const dfnBase = new URL(`${base}deepfilternet`, window.location.href).href;
const mod = assertModuleExport<DfnModule>(
await import(/* @vite-ignore */ url),
"DeepFilterNet3Core",
url,
);
const core = new mod.DeepFilterNet3Core({
sampleRate: 48_000,
// 60, not 80: full-strength suppression is the main source of the
// "over-processed" character; a lower level keeps voice natural while
// the dry/wet floor handles the noise tail.
noiseReductionLevel: 60,
assetConfig: { cdnUrl: dfnBase },
});
await core.initialize();
return core;
}
async function loadDtlnModule(config: LotusDenoiseConfig): Promise<DtlnModule> {
const url = `${config.assetBase}workadventure/audio-worklet.js`;
return assertModuleExport<DtlnModule>(
await import(/* @vite-ignore */ url),
"createNoiseSuppressionAudioWorklet",
url,
);
}
/** Which wasm file a flat model uses (SIMD build when supported). */
function flatWasmFiles(model: "rnnoise" | "speex"): {
primary: string;
fallback?: string;
} {
const flat = FLAT[model];
const useSimd = model === "rnnoise" && !!flat.simdWasm && supportsSimd();
return useSimd
? { primary: flat.simdWasm!, fallback: flat.wasm }
: { primary: flat.wasm };
}
// [lotus #24] Force every node in the graph to a single, explicitly-downmixed
// channel. Without `channelCountMode: "explicit"` the default ("max") IGNORES
// `channelCount`, so a stereo capture device would feed 2 channels into a
// worklet configured with `maxChannels: 1` and sum a stereo dry copy against a
// mono wet one at the destination.
const MONO: AudioNodeOptions = {
channelCount: 1,
channelCountMode: "explicit",
channelInterpretation: "speakers",
};
// [lotus #25] Algorithmic latency of each model in samples at its native rate,
// used to delay the DRY copy of the floor mix so it lines up with the wet path
// (otherwise the sum comb-filters — a hollow/phasey colouration on voice).
// - rnnoise: 480-sample (10 ms @ 48 kHz) frames; the sapphi worklet buffers
// 128-sample quanta up to one frame, so the wet path lags by one frame.
// - speex: the sapphi speex worklet uses the same 480-sample framing.
// - dtln: 512-sample block / 128 hop @ 16 kHz (~32 ms) per the DTLN paper —
// best-known, unmeasured (the floor is not mixed for dtln, see buildGraph).
// - deepfilternet: 480-sample hop + 2-frame lookahead @ 48 kHz (~30 ms) per
// DeepFilterNet3 — best-known, unmeasured (floor not mixed for dfn either).
const DRY_DELAY_SAMPLES: Record<LotusDenoiseModel, number> = {
rnnoise: 480,
speex: 480,
dtln: 512,
deepfilternet: 1440,
};
/**
* Create the model-rate context and register the flat/gate worklet modules.
* Closes the context (and rethrows) on any failure so nothing half-built leaks.
*/
async function createModelContext(
config: LotusDenoiseConfig,
): Promise<AudioContext> {
const rate = sampleRateFor(config.model);
const ctx = new AudioContext({ sampleRate: rate });
try {
if (ctx.sampleRate !== rate)
throw new Error(`denoise: got ${ctx.sampleRate}Hz, need ${rate}Hz`);
// Flat models register via addModule here; DTLN/DeepFilterNet bring their
// own processor via the dynamic-imported helper (see buildMlNode).
if (config.model === "rnnoise" || config.model === "speex")
await ctx.audioWorklet.addModule(
config.assetBase + FLAT[config.model].script,
);
if (config.gate)
await ctx.audioWorklet.addModule(config.assetBase + GATE.script);
return ctx;
} catch (e) {
await ctx.close().catch(() => undefined);
throw e;
}
}
// [lotus #7] Everything heavy that `init()` needs but that does NOT depend on
// the mic track: the AudioContext + worklet modules, the flat wasm binary, and
// (DFN) the fully-initialised model core. `LocalAudioTrack.setProcessor()`
// holds LiveKit's `trackChangeLock` while awaiting `init()`, so every
// mute/unmute/device-switch queues behind it — prepare these as soon as the
// flag is seen (before any track exists) so `init()` only wires them up.
interface PreparedAssets {
ctx: AudioContext;
dfnCore?: DfnCore;
}
const preparedAssets = new Map<string, Promise<PreparedAssets>>();
const preparedKey = (c: LotusDenoiseConfig): string =>
`${c.model}|${c.gate ? 1 : 0}|${c.assetBase}`;
async function prepareUncached(
config: LotusDenoiseConfig,
): Promise<PreparedAssets> {
const ctx = await createModelContext(config);
try {
let dfnCore: DfnCore | undefined;
if (config.model === "rnnoise" || config.model === "speex") {
const { primary, fallback } = flatWasmFiles(config.model);
// Warm the wasm cache; a SIMD miss is fine — buildMlNode falls back.
await fetchWasm(config.assetBase + primary).catch(async () =>
fallback ? fetchWasm(config.assetBase + fallback) : undefined,
);
} else if (config.model === "dtln") {
await loadDtlnModule(config); // warms the browser's module map
} else {
dfnCore = await loadDfnCore(config);
}
return { ctx, dfnCore };
} catch (e) {
await ctx.close().catch(() => undefined);
throw e;
}
}
/**
* Prefetch/prepare the assets for `config` (idempotent per model). Never
* rejects: a failed prepare is evicted so `init()` simply loads inline and
* surfaces the real error there.
*/
export async function prepareDenoiseAssets(
config: LotusDenoiseConfig,
): Promise<void> {
const key = preparedKey(config);
let p = preparedAssets.get(key);
if (!p) {
p = prepareUncached(config);
void p.catch((e) => {
if (preparedAssets.get(key) === p) preparedAssets.delete(key);
logger.warn(`[lotus] denoise prepare failed (${config.model})`, e);
});
preparedAssets.set(key, p);
}
await p.catch(() => undefined);
}
/** Take (one-shot) the prepared assets for `config`, if any were prepared. */
function claimPreparedAssets(
config: LotusDenoiseConfig,
): Promise<PreparedAssets> | undefined {
const key = preparedKey(config);
const p = preparedAssets.get(key);
if (p) preparedAssets.delete(key);
return p;
}
/** Close any prepared-but-unclaimed contexts (call on feature teardown). */
export async function releasePreparedDenoiseAssets(): Promise<void> {
const all = [...preparedAssets.values()];
preparedAssets.clear();
await Promise.all(
all.map(async (p) =>
p
.then(async (a) => {
safeCall(() => a.dfnCore?.destroy());
if (a.ctx.state !== "closed") await a.ctx.close();
})
.catch(() => undefined),
),
);
}
/**
* A LiveKit audio TrackProcessor that runs Lotus ML noise suppression
* (RNNoise / Speex / DTLN / DeepFilterNet) on the local microphone track, as a
* first-class stage in Element Call's publish pipeline — replacing the host's
* `getUserMedia` monkeypatch.
*
* Because it's a real LiveKit processor, EC re-applies it on every
* (re)publish/restart, so denoise survives EC's mid-call reconnect — the root
* cause of A7. It owns a dedicated AudioContext at the model's required sample
* rate (LiveKit does NOT pass an audioContext to restart()), reused across
* restarts and closed on destroy. restart() never throws and never leaves a
* stopped track on the sender: on failure it degrades to the RAW mic track
* rather than silence.
*/
2026-09-12 11:35:57 -04:00
export class LotusDenoiseProcessor implements TrackProcessor<
Track.Kind.Audio,
AudioProcessorOptions
> {
public readonly name = "lotus-denoise";
public processedTrack?: MediaStreamTrack;
private ctx?: AudioContext;
private graph?: Graph;
private ctxStateHandler?: () => void;
private preparedDfnCore?: DfnCore;
// [lotus #9] True while the mic is muted: we suspend our own context so the
// worklet stops running inference on silence, and the statechange watcher
// must not "heal" that intentional suspension.
private micMuted = false;
public constructor(private readonly config: LotusDenoiseConfig) {}
/**
* [lotus #9] Mirror the mic's mute state onto the owned context. EC uses
* `stopMicTrackOnMute: false`, so a muted mic keeps producing (silent) frames
* and the ML worklet would otherwise keep running full inference for the
* whole time the user is muted.
*/
public setMicMuted(muted: boolean): void {
this.micMuted = muted;
const ctx = this.ctx;
if (!ctx || ctx.state === "closed") return;
if (muted) {
if (ctx.state === "running") void ctx.suspend().catch(() => undefined);
} else if (ctx.state === "suspended" && this.graph) {
void ctx.resume().catch(() => undefined);
}
}
public async init(_opts: AudioProcessorOptions): Promise<void> {
try {
await this.ensureContext();
this.graph = await this.buildGraph(_opts.track);
this.processedTrack = this.graph.track;
} catch (e) {
// Don't orphan the owned context if graph construction fails (browsers
// cap live AudioContexts, so repeated failed inits could exhaust them).
// The caller degrades to the raw mic; we just release our resources.
const core = this.preparedDfnCore;
this.preparedDfnCore = undefined;
if (core) safeCall(() => core.destroy());
await this.closeContext();
throw e;
}
}
public async restart(opts: AudioProcessorOptions): Promise<void> {
try {
await this.ensureContext();
const next = await this.buildGraph(opts.track);
this.disposeGraph(this.graph);
this.graph = next;
this.processedTrack = next.track;
} catch (e) {
// Never go silent on the A7/device-switch path: fall back to raw audio.
// [lotus] IMPORTANT: never assign `opts.track` (LiveKit-owned) here.
// LiveKit's internalStopProcessor() does `processor.processedTrack?.stop()`
// then re-publishes `_mediaStreamTrack` — the SAME object if we set it as
// processedTrack — which kills the live mic on the next stopProcessor()/
// teardown. Leaving processedTrack undefined makes LiveKit fall through to
// its own `_mediaStreamTrack` instead.
logger.warn("[lotus] denoise restart failed; using raw mic", e);
this.disposeGraph(this.graph);
this.graph = undefined;
this.processedTrack = undefined;
}
}
public async destroy(): Promise<void> {
this.disposeGraph(this.graph);
this.graph = undefined;
this.processedTrack = undefined;
const core = this.preparedDfnCore;
this.preparedDfnCore = undefined;
if (core) safeCall(() => core.destroy());
await this.closeContext();
}
/** Remove the state watcher and close the owned context, if any. */
private async closeContext(): Promise<void> {
const ctx = this.ctx;
if (!ctx) return;
if (this.ctxStateHandler) {
ctx.removeEventListener("statechange", this.ctxStateHandler);
this.ctxStateHandler = undefined;
}
this.ctx = undefined;
if (ctx.state !== "closed") await ctx.close().catch(() => undefined);
}
/** Adopt the prepared context (or create one) + install the state watcher. */
private async ensureContext(): Promise<void> {
const rate = sampleRateFor(this.config.model);
2026-09-12 11:35:57 -04:00
if (
this.ctx &&
this.ctx.state !== "closed" &&
this.ctx.sampleRate === rate
) {
if (this.ctx.state === "suspended" && !this.micMuted)
await resumeCtx(this.ctx);
return;
}
await this.closeContext();
// [lotus #7] Prefer the context/modules/model prepared before
// setProcessor() was called; only load inline if nothing was prepared
// (e.g. the rnnoise fallback path, or a second processor after a
// republish).
const claimed = await claimPreparedAssets(this.config)?.catch(
() => undefined,
);
let ctx: AudioContext;
if (claimed && claimed.ctx.state !== "closed") {
ctx = claimed.ctx;
this.preparedDfnCore = claimed.dfnCore;
} else {
ctx = await createModelContext(this.config);
}
try {
// Auto-resume if the OS/browser suspends the context mid-call (mobile
// backgrounding, audio interruption): the dest node otherwise emits
// silence with no recovery. Only resume while a graph is live and the
// suspension isn't our own mute suspension (#9).
const onStateChange = (): void => {
if (ctx.state === "suspended" && this.graph && !this.micMuted)
void ctx.resume().catch(() => undefined);
};
ctx.addEventListener("statechange", onStateChange);
// The action can arrive via host postMessage, not a gesture in this
// iframe, so the context can start suspended — resume without hanging.
if (ctx.state === "suspended" && !this.micMuted) await resumeCtx(ctx);
// Attached while already muted (#9): don't let a prepared, running
// context burn inference until the first unmute.
else if (ctx.state === "running" && this.micMuted)
await ctx.suspend().catch(() => undefined);
this.ctx = ctx;
this.ctxStateHandler = onStateChange;
} catch (e) {
// Don't leak a half-initialised context on any failure path.
await ctx.close().catch(() => undefined);
throw e;
}
}
private async buildGraph(track: MediaStreamTrack): Promise<Graph> {
const ctx = this.ctx!;
const source = ctx.createMediaStreamSource(new MediaStream([track]));
const dest = new MediaStreamAudioDestinationNode(ctx, MONO);
const nodes: AudioNode[] = [];
const disposes: (() => void)[] = [];
try {
// Wet (denoised) path: source → ml → [gate] → wetGain.
const ml = await this.buildMlNode(ctx);
source.connect(ml.node);
nodes.push(ml.node);
if (ml.dispose) disposes.push(ml.dispose);
let wetHead: AudioNode = ml.node;
// Gate AFTER the ML model, not before: gating the raw noisy signal fed
// hard-zeroed frames into the model (discontinuities it must fight) and
// made the threshold operate on pre-denoise levels. Gate the residual.
if (this.config.gate) {
const gate = new AudioWorkletNode(ctx, GATE.name, {
...MONO,
processorOptions: {
openThreshold: this.config.gateThreshold,
closeThreshold: this.config.gateThreshold - 5,
holdMs: 150,
maxChannels: 1,
},
});
wetHead.connect(gate);
wetHead = gate;
nodes.push(gate);
}
// Only mix a dry floor for the flat models (RNNoise/Speex), whose
// framing latency is known exactly (DRY_DELAY_SAMPLES); the DTLN/DFN
// figures are best-known estimates, so for those we rely on the model's
// own level (e.g. DFN noiseReductionLevel) instead. RNNoise is also where
// the "robotic/underwater" reports come from, so this targets it.
const lowLatency =
this.config.model === "rnnoise" || this.config.model === "speex";
const floor = lowLatency
? Math.min(0.5, Math.max(0, this.config.floor))
: 0;
if (floor > 0) {
// Dry/wet mix: blend a small amount of the ORIGINAL mic under the
// denoised signal so suppression can't fully collapse the noise floor
// (kills the "underwater"/pumping artifact). During speech (denoised ≈
// original) the two sum back to ~unity; in noise-only gaps the output
// floors at `floor` × original instead of digital silence.
const wetGain = new GainNode(ctx, { ...MONO, gain: 1 - floor });
wetHead.connect(wetGain);
wetGain.connect(dest);
nodes.push(wetGain);
// [lotus #25] Delay the dry copy by the model's algorithmic latency so
// it sums in phase with the (framed, hence delayed) wet path instead
// of comb-filtering against it.
const delaySec = DRY_DELAY_SAMPLES[this.config.model] / ctx.sampleRate;
const dryDelay = new DelayNode(ctx, {
...MONO,
maxDelayTime: Math.max(delaySec, 1 / ctx.sampleRate),
delayTime: delaySec,
});
const dryGain = new GainNode(ctx, { ...MONO, gain: floor });
source.connect(dryDelay);
dryDelay.connect(dryGain);
dryGain.connect(dest);
nodes.push(dryDelay, dryGain);
} else {
wetHead.connect(dest);
}
logger.info(
`[lotus] denoise processor active (${this.config.model}, floor=${floor})`,
);
2026-09-12 11:35:57 -04:00
return {
source,
nodes,
disposes,
track: dest.stream.getAudioTracks()[0],
};
} catch (e) {
// A node constructor / model load can throw mid-build; clean up the
// partially-built graph so it doesn't leak (init/restart still fall back
// to the raw mic on the rejection).
this.disposeGraph({
source,
nodes,
disposes,
track: dest.stream.getAudioTracks()[0],
});
throw e;
}
}
private async buildMlNode(ctx: AudioContext): Promise<MlNode> {
const base = this.config.assetBase;
const model = this.config.model;
if (model === "dtln") {
// Self-contained ESM that resolves its own processor + LiteRT wasm +
// TFLite models. bypassUntilReady passes raw audio until the model loads.
const mod = await loadDtlnModule(this.config);
return await mod.createNoiseSuppressionAudioWorklet(ctx, {
bypassUntilReady: true,
});
}
if (model === "deepfilternet") {
// [lotus #7] Use the core initialised by prepareDenoiseAssets() if we
// have one (first graph); later rebuilds (restart) load a fresh core.
const prepared = this.preparedDfnCore;
this.preparedDfnCore = undefined;
const core = prepared ?? (await loadDfnCore(this.config));
const node = await core.createAudioWorkletNode(ctx);
2026-09-12 11:35:57 -04:00
return { node, dispose: () => safeCall(() => core.destroy()) };
}
// Flat sapphi worklet (rnnoise/speex).
const flat = FLAT[model];
const { primary, fallback } = flatWasmFiles(model);
let wasmBinary: ArrayBuffer;
try {
wasmBinary = await fetchWasm(base + primary);
} catch (e) {
if (fallback) {
wasmCache.delete(base + primary);
wasmBinary = await fetchWasm(base + fallback); // fall back to non-SIMD
} else throw e;
}
const node = new AudioWorkletNode(ctx, flat.name, {
...MONO,
numberOfInputs: 1,
numberOfOutputs: 1,
processorOptions: { maxChannels: 1, wasmBinary },
});
2026-09-12 11:35:57 -04:00
return {
node,
dispose: () => safeCall(() => node.port.postMessage("destroy")),
};
}
private disposeGraph(graph: Graph | undefined): void {
if (!graph) return;
for (const dispose of graph.disposes) safeCall(dispose);
for (const node of graph.nodes) safeCall(() => node.disconnect());
safeCall(() => graph.source.disconnect());
graph.track.stop();
}
}
function safeCall(fn: () => void): void {
try {
fn();
} catch {
/* ignore */
}
}