- Assets (context, worklets, wasm, DFN core) are prepared as soon as the flag is seen, so init() under LiveKit's trackChangeLock only wires already-loaded pieces; resume timeout 3 s -> 500 ms (#7). - init failure retries once with rnnoise; success/failure is reported to the host as io.lotus.denoise_state so the UI can reflect reality (#8). - Mic TrackMuted/TrackUnmuted suspend/resume the processor's context so no inference runs on silence (#9). - Every node is explicit mono; the dry path gets a per-model DelayNode so the floor mix no longer comb-filters (#24, #25). - DTLN/DFN dynamic imports are typed and their exports asserted at load, feeding the #8 fallback instead of failing silently (#26). Unit-tested (13 tests across the two files). Fixes #7 Fixes #8 Fixes #9 Fixes #24 Fixes #25 Fixes #26 Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01PPmy3tPq869XDW4njjVaKA
675 lines
25 KiB
TypeScript
675 lines
25 KiB
TypeScript
/*
|
||
Copyright 2026 Lotus Guild
|
||
|
||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||
Please see LICENSE in the repository root for full details.
|
||
*/
|
||
|
||
import {
|
||
type AudioProcessorOptions,
|
||
type Track,
|
||
type TrackProcessor,
|
||
} from "livekit-client";
|
||
import { logger } from "matrix-js-sdk/lib/logger";
|
||
|
||
export type LotusDenoiseModel = "rnnoise" | "speex" | "dtln" | "deepfilternet";
|
||
|
||
export interface LotusDenoiseConfig {
|
||
model: LotusDenoiseModel;
|
||
/** Base URL the worklet scripts/wasm/ESM are served from (e.g. "./denoise/"). */
|
||
assetBase: string;
|
||
gate: boolean;
|
||
gateThreshold: number;
|
||
/**
|
||
* Attenuation floor as a dry/wet mix: the fraction (0..~0.3) of the ORIGINAL
|
||
* mic blended back under the denoised signal so full suppression never fully
|
||
* collapses the noise floor — this is what kills the "underwater"/pumping
|
||
* artifact. 0.15 ≈ a -16 dB floor. 0 = full suppression (no floor).
|
||
*/
|
||
floor: number;
|
||
}
|
||
|
||
// Flat sapphi worklets (RNNoise/Speex): each registers a processor under `name`
|
||
// when its `script` module is added; we feed it the fetched wasm binary.
|
||
// ⚠️ CONTRACT: these assets are NOT bundled by the fork build — cinny's
|
||
// vite.config.js `lotusDenoise()` plugin copies them from
|
||
// `@sapphi-red/web-noise-suppressor` / `@workadventure/noise-suppression` /
|
||
// `deepfilternet3-noise-filter` into public/element-call/denoise/. The
|
||
// worklet/wasm/ESM versions must match what this processor expects. An
|
||
// integration smoke-check should assert GET .../denoise/rnnoise.wasm == 200.
|
||
const FLAT: Record<
|
||
"rnnoise" | "speex",
|
||
{ name: string; script: string; wasm: string; simdWasm?: string }
|
||
> = {
|
||
rnnoise: {
|
||
name: "@sapphi-red/web-noise-suppressor/rnnoise",
|
||
script: "rnnoiseWorklet.js",
|
||
wasm: "rnnoise.wasm",
|
||
simdWasm: "rnnoise_simd.wasm",
|
||
},
|
||
speex: {
|
||
name: "@sapphi-red/web-noise-suppressor/speex",
|
||
script: "speexWorklet.js",
|
||
wasm: "speex.wasm",
|
||
},
|
||
};
|
||
// The sapphi gate worklet registers under "noise-gate" (hyphenated).
|
||
const GATE = {
|
||
name: "@sapphi-red/web-noise-suppressor/noise-gate",
|
||
script: "noiseGateWorklet.js",
|
||
};
|
||
|
||
// DTLN (@workadventure) targets 16kHz and doesn't resample; RNNoise/Speex and
|
||
// DeepFilterNet are 48kHz fullband. The worklets don't resample, so the whole
|
||
// graph must run at the model's native rate.
|
||
const sampleRateFor = (model: LotusDenoiseModel): number =>
|
||
model === "dtln" ? 16_000 : 48_000;
|
||
|
||
// Cache fetched wasm per URL so a reconnect/device-switch doesn't re-download.
|
||
const wasmCache = new Map<string, Promise<ArrayBuffer>>();
|
||
async function fetchWasmUncached(url: string): Promise<ArrayBuffer> {
|
||
const r = await fetch(url);
|
||
if (!r.ok) throw new Error(`denoise wasm ${url} -> ${r.status}`);
|
||
return r.arrayBuffer();
|
||
}
|
||
async function fetchWasm(url: string): Promise<ArrayBuffer> {
|
||
let p = wasmCache.get(url);
|
||
if (!p) {
|
||
p = fetchWasmUncached(url);
|
||
// Never cache a REJECTED fetch: a transient failure (e.g. a blip during a
|
||
// reconnect) must not permanently disable denoise for the whole session.
|
||
// Evict on failure so the next restart/device-switch retries.
|
||
void p.catch(() => wasmCache.delete(url));
|
||
wasmCache.set(url, p);
|
||
}
|
||
return p;
|
||
}
|
||
|
||
/**
|
||
* Resume an AudioContext, but never block indefinitely. A suspended context can
|
||
* only resume after a user gesture; the denoise action can arrive via host
|
||
* postMessage (no gesture), so `resume()` may stay pending forever. Since this
|
||
* runs inside LiveKit's per-track change lock, a hung resume() would deadlock
|
||
* every later mute/unmute/device-switch. Race it against a timeout and proceed
|
||
* either way — the processor's `statechange` watcher resumes it once a gesture
|
||
* lands, and a still-suspended context degrades to (temporary) silence that the
|
||
* watcher heals, not a hang.
|
||
*/
|
||
// [lotus] 500 ms, not 3 s: this still runs under LiveKit's trackChangeLock (#7),
|
||
// and the statechange watcher heals a still-suspended context later anyway.
|
||
async function resumeCtx(ctx: AudioContext, timeoutMs = 500): Promise<void> {
|
||
await Promise.race([
|
||
ctx.resume().catch(() => undefined),
|
||
new Promise<void>((resolve) => setTimeout(resolve, timeoutMs)),
|
||
]);
|
||
}
|
||
|
||
function supportsSimd(): boolean {
|
||
try {
|
||
// Minimal SIMD module (v128) — validates only where SIMD is supported.
|
||
return WebAssembly.validate(
|
||
new Uint8Array([
|
||
0, 97, 115, 109, 1, 0, 0, 0, 1, 5, 1, 96, 0, 1, 123, 3, 2, 1, 0, 10, 10,
|
||
1, 8, 0, 65, 0, 253, 15, 253, 98, 11,
|
||
]),
|
||
);
|
||
} catch {
|
||
return false;
|
||
}
|
||
}
|
||
|
||
interface MlNode {
|
||
node: AudioNode;
|
||
dispose?: () => void;
|
||
}
|
||
interface Graph {
|
||
source: MediaStreamAudioSourceNode;
|
||
nodes: AudioNode[];
|
||
disposes: (() => void)[];
|
||
track: MediaStreamTrack;
|
||
}
|
||
|
||
// [lotus #26] Minimal local contracts for the two dynamically-imported ESM
|
||
// helpers (not bundled here — see the CONTRACT note above). Their exports are
|
||
// asserted at runtime so an asset bump that renames/removes one fails loudly
|
||
// (and flows into the rnnoise fallback in lotusDenoise.ts) instead of as a
|
||
// vague TypeError deep inside `init()`.
|
||
interface DtlnModule {
|
||
createNoiseSuppressionAudioWorklet: (
|
||
ctx: AudioContext,
|
||
opts: { bypassUntilReady: boolean },
|
||
) => Promise<MlNode>;
|
||
}
|
||
interface DfnCore {
|
||
initialize: () => Promise<void>;
|
||
createAudioWorkletNode: (ctx: AudioContext) => Promise<AudioNode>;
|
||
destroy: () => void;
|
||
}
|
||
interface DfnModule {
|
||
DeepFilterNet3Core: new (opts: {
|
||
sampleRate: number;
|
||
noiseReductionLevel: number;
|
||
assetConfig: { cdnUrl: string };
|
||
}) => DfnCore;
|
||
}
|
||
|
||
/** Throw a clear error if a dynamic-import module lacks an expected export. */
|
||
export function assertModuleExport<T>(
|
||
mod: unknown,
|
||
name: string,
|
||
url: string,
|
||
): T {
|
||
const exp = (mod as Record<string, unknown> | null | undefined)?.[name];
|
||
if (typeof exp !== "function")
|
||
throw new Error(
|
||
`denoise: ${url} does not export ${name} (got ${typeof exp}) — asset/version mismatch`,
|
||
);
|
||
return mod as T;
|
||
}
|
||
|
||
async function loadDfnCore(config: LotusDenoiseConfig): Promise<DfnCore> {
|
||
const base = config.assetBase;
|
||
const url = `${base}deepfilternet/index.esm.js`;
|
||
const dfnBase = new URL(`${base}deepfilternet`, window.location.href).href;
|
||
const mod = assertModuleExport<DfnModule>(
|
||
await import(/* @vite-ignore */ url),
|
||
"DeepFilterNet3Core",
|
||
url,
|
||
);
|
||
const core = new mod.DeepFilterNet3Core({
|
||
sampleRate: 48_000,
|
||
// 60, not 80: full-strength suppression is the main source of the
|
||
// "over-processed" character; a lower level keeps voice natural while
|
||
// the dry/wet floor handles the noise tail.
|
||
noiseReductionLevel: 60,
|
||
assetConfig: { cdnUrl: dfnBase },
|
||
});
|
||
await core.initialize();
|
||
return core;
|
||
}
|
||
|
||
async function loadDtlnModule(config: LotusDenoiseConfig): Promise<DtlnModule> {
|
||
const url = `${config.assetBase}workadventure/audio-worklet.js`;
|
||
return assertModuleExport<DtlnModule>(
|
||
await import(/* @vite-ignore */ url),
|
||
"createNoiseSuppressionAudioWorklet",
|
||
url,
|
||
);
|
||
}
|
||
|
||
/** Which wasm file a flat model uses (SIMD build when supported). */
|
||
function flatWasmFiles(model: "rnnoise" | "speex"): {
|
||
primary: string;
|
||
fallback?: string;
|
||
} {
|
||
const flat = FLAT[model];
|
||
const useSimd = model === "rnnoise" && !!flat.simdWasm && supportsSimd();
|
||
return useSimd
|
||
? { primary: flat.simdWasm!, fallback: flat.wasm }
|
||
: { primary: flat.wasm };
|
||
}
|
||
|
||
// [lotus #24] Force every node in the graph to a single, explicitly-downmixed
|
||
// channel. Without `channelCountMode: "explicit"` the default ("max") IGNORES
|
||
// `channelCount`, so a stereo capture device would feed 2 channels into a
|
||
// worklet configured with `maxChannels: 1` and sum a stereo dry copy against a
|
||
// mono wet one at the destination.
|
||
const MONO: AudioNodeOptions = {
|
||
channelCount: 1,
|
||
channelCountMode: "explicit",
|
||
channelInterpretation: "speakers",
|
||
};
|
||
|
||
// [lotus #25] Algorithmic latency of each model in samples at its native rate,
|
||
// used to delay the DRY copy of the floor mix so it lines up with the wet path
|
||
// (otherwise the sum comb-filters — a hollow/phasey colouration on voice).
|
||
// - rnnoise: 480-sample (10 ms @ 48 kHz) frames; the sapphi worklet buffers
|
||
// 128-sample quanta up to one frame, so the wet path lags by one frame.
|
||
// - speex: the sapphi speex worklet uses the same 480-sample framing.
|
||
// - dtln: 512-sample block / 128 hop @ 16 kHz (~32 ms) per the DTLN paper —
|
||
// best-known, unmeasured (the floor is not mixed for dtln, see buildGraph).
|
||
// - deepfilternet: 480-sample hop + 2-frame lookahead @ 48 kHz (~30 ms) per
|
||
// DeepFilterNet3 — best-known, unmeasured (floor not mixed for dfn either).
|
||
const DRY_DELAY_SAMPLES: Record<LotusDenoiseModel, number> = {
|
||
rnnoise: 480,
|
||
speex: 480,
|
||
dtln: 512,
|
||
deepfilternet: 1440,
|
||
};
|
||
|
||
/**
|
||
* Create the model-rate context and register the flat/gate worklet modules.
|
||
* Closes the context (and rethrows) on any failure so nothing half-built leaks.
|
||
*/
|
||
async function createModelContext(
|
||
config: LotusDenoiseConfig,
|
||
): Promise<AudioContext> {
|
||
const rate = sampleRateFor(config.model);
|
||
const ctx = new AudioContext({ sampleRate: rate });
|
||
try {
|
||
if (ctx.sampleRate !== rate)
|
||
throw new Error(`denoise: got ${ctx.sampleRate}Hz, need ${rate}Hz`);
|
||
// Flat models register via addModule here; DTLN/DeepFilterNet bring their
|
||
// own processor via the dynamic-imported helper (see buildMlNode).
|
||
if (config.model === "rnnoise" || config.model === "speex")
|
||
await ctx.audioWorklet.addModule(
|
||
config.assetBase + FLAT[config.model].script,
|
||
);
|
||
if (config.gate)
|
||
await ctx.audioWorklet.addModule(config.assetBase + GATE.script);
|
||
return ctx;
|
||
} catch (e) {
|
||
await ctx.close().catch(() => undefined);
|
||
throw e;
|
||
}
|
||
}
|
||
|
||
// [lotus #7] Everything heavy that `init()` needs but that does NOT depend on
|
||
// the mic track: the AudioContext + worklet modules, the flat wasm binary, and
|
||
// (DFN) the fully-initialised model core. `LocalAudioTrack.setProcessor()`
|
||
// holds LiveKit's `trackChangeLock` while awaiting `init()`, so every
|
||
// mute/unmute/device-switch queues behind it — prepare these as soon as the
|
||
// flag is seen (before any track exists) so `init()` only wires them up.
|
||
interface PreparedAssets {
|
||
ctx: AudioContext;
|
||
dfnCore?: DfnCore;
|
||
}
|
||
const preparedAssets = new Map<string, Promise<PreparedAssets>>();
|
||
const preparedKey = (c: LotusDenoiseConfig): string =>
|
||
`${c.model}|${c.gate ? 1 : 0}|${c.assetBase}`;
|
||
|
||
async function prepareUncached(
|
||
config: LotusDenoiseConfig,
|
||
): Promise<PreparedAssets> {
|
||
const ctx = await createModelContext(config);
|
||
try {
|
||
let dfnCore: DfnCore | undefined;
|
||
if (config.model === "rnnoise" || config.model === "speex") {
|
||
const { primary, fallback } = flatWasmFiles(config.model);
|
||
// Warm the wasm cache; a SIMD miss is fine — buildMlNode falls back.
|
||
await fetchWasm(config.assetBase + primary).catch(async () =>
|
||
fallback ? fetchWasm(config.assetBase + fallback) : undefined,
|
||
);
|
||
} else if (config.model === "dtln") {
|
||
await loadDtlnModule(config); // warms the browser's module map
|
||
} else {
|
||
dfnCore = await loadDfnCore(config);
|
||
}
|
||
return { ctx, dfnCore };
|
||
} catch (e) {
|
||
await ctx.close().catch(() => undefined);
|
||
throw e;
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Prefetch/prepare the assets for `config` (idempotent per model). Never
|
||
* rejects: a failed prepare is evicted so `init()` simply loads inline and
|
||
* surfaces the real error there.
|
||
*/
|
||
export async function prepareDenoiseAssets(
|
||
config: LotusDenoiseConfig,
|
||
): Promise<void> {
|
||
const key = preparedKey(config);
|
||
let p = preparedAssets.get(key);
|
||
if (!p) {
|
||
p = prepareUncached(config);
|
||
void p.catch((e) => {
|
||
if (preparedAssets.get(key) === p) preparedAssets.delete(key);
|
||
logger.warn(`[lotus] denoise prepare failed (${config.model})`, e);
|
||
});
|
||
preparedAssets.set(key, p);
|
||
}
|
||
await p.catch(() => undefined);
|
||
}
|
||
|
||
/** Take (one-shot) the prepared assets for `config`, if any were prepared. */
|
||
function claimPreparedAssets(
|
||
config: LotusDenoiseConfig,
|
||
): Promise<PreparedAssets> | undefined {
|
||
const key = preparedKey(config);
|
||
const p = preparedAssets.get(key);
|
||
if (p) preparedAssets.delete(key);
|
||
return p;
|
||
}
|
||
|
||
/** Close any prepared-but-unclaimed contexts (call on feature teardown). */
|
||
export async function releasePreparedDenoiseAssets(): Promise<void> {
|
||
const all = [...preparedAssets.values()];
|
||
preparedAssets.clear();
|
||
await Promise.all(
|
||
all.map(async (p) =>
|
||
p
|
||
.then(async (a) => {
|
||
safeCall(() => a.dfnCore?.destroy());
|
||
if (a.ctx.state !== "closed") await a.ctx.close();
|
||
})
|
||
.catch(() => undefined),
|
||
),
|
||
);
|
||
}
|
||
|
||
/**
|
||
* A LiveKit audio TrackProcessor that runs Lotus ML noise suppression
|
||
* (RNNoise / Speex / DTLN / DeepFilterNet) on the local microphone track, as a
|
||
* first-class stage in Element Call's publish pipeline — replacing the host's
|
||
* `getUserMedia` monkeypatch.
|
||
*
|
||
* Because it's a real LiveKit processor, EC re-applies it on every
|
||
* (re)publish/restart, so denoise survives EC's mid-call reconnect — the root
|
||
* cause of A7. It owns a dedicated AudioContext at the model's required sample
|
||
* rate (LiveKit does NOT pass an audioContext to restart()), reused across
|
||
* restarts and closed on destroy. restart() never throws and never leaves a
|
||
* stopped track on the sender: on failure it degrades to the RAW mic track
|
||
* rather than silence.
|
||
*/
|
||
export class LotusDenoiseProcessor implements TrackProcessor<
|
||
Track.Kind.Audio,
|
||
AudioProcessorOptions
|
||
> {
|
||
public readonly name = "lotus-denoise";
|
||
public processedTrack?: MediaStreamTrack;
|
||
|
||
private ctx?: AudioContext;
|
||
private graph?: Graph;
|
||
private ctxStateHandler?: () => void;
|
||
private preparedDfnCore?: DfnCore;
|
||
// [lotus #9] True while the mic is muted: we suspend our own context so the
|
||
// worklet stops running inference on silence, and the statechange watcher
|
||
// must not "heal" that intentional suspension.
|
||
private micMuted = false;
|
||
|
||
public constructor(private readonly config: LotusDenoiseConfig) {}
|
||
|
||
/**
|
||
* [lotus #9] Mirror the mic's mute state onto the owned context. EC uses
|
||
* `stopMicTrackOnMute: false`, so a muted mic keeps producing (silent) frames
|
||
* and the ML worklet would otherwise keep running full inference for the
|
||
* whole time the user is muted.
|
||
*/
|
||
public setMicMuted(muted: boolean): void {
|
||
this.micMuted = muted;
|
||
const ctx = this.ctx;
|
||
if (!ctx || ctx.state === "closed") return;
|
||
if (muted) {
|
||
if (ctx.state === "running") void ctx.suspend().catch(() => undefined);
|
||
} else if (ctx.state === "suspended" && this.graph) {
|
||
void ctx.resume().catch(() => undefined);
|
||
}
|
||
}
|
||
|
||
public async init(_opts: AudioProcessorOptions): Promise<void> {
|
||
try {
|
||
await this.ensureContext();
|
||
this.graph = await this.buildGraph(_opts.track);
|
||
this.processedTrack = this.graph.track;
|
||
} catch (e) {
|
||
// Don't orphan the owned context if graph construction fails (browsers
|
||
// cap live AudioContexts, so repeated failed inits could exhaust them).
|
||
// The caller degrades to the raw mic; we just release our resources.
|
||
const core = this.preparedDfnCore;
|
||
this.preparedDfnCore = undefined;
|
||
if (core) safeCall(() => core.destroy());
|
||
await this.closeContext();
|
||
throw e;
|
||
}
|
||
}
|
||
|
||
public async restart(opts: AudioProcessorOptions): Promise<void> {
|
||
try {
|
||
await this.ensureContext();
|
||
const next = await this.buildGraph(opts.track);
|
||
this.disposeGraph(this.graph);
|
||
this.graph = next;
|
||
this.processedTrack = next.track;
|
||
} catch (e) {
|
||
// Never go silent on the A7/device-switch path: fall back to raw audio.
|
||
// [lotus] IMPORTANT: never assign `opts.track` (LiveKit-owned) here.
|
||
// LiveKit's internalStopProcessor() does `processor.processedTrack?.stop()`
|
||
// then re-publishes `_mediaStreamTrack` — the SAME object if we set it as
|
||
// processedTrack — which kills the live mic on the next stopProcessor()/
|
||
// teardown. Leaving processedTrack undefined makes LiveKit fall through to
|
||
// its own `_mediaStreamTrack` instead.
|
||
logger.warn("[lotus] denoise restart failed; using raw mic", e);
|
||
this.disposeGraph(this.graph);
|
||
this.graph = undefined;
|
||
this.processedTrack = undefined;
|
||
}
|
||
}
|
||
|
||
public async destroy(): Promise<void> {
|
||
this.disposeGraph(this.graph);
|
||
this.graph = undefined;
|
||
this.processedTrack = undefined;
|
||
const core = this.preparedDfnCore;
|
||
this.preparedDfnCore = undefined;
|
||
if (core) safeCall(() => core.destroy());
|
||
await this.closeContext();
|
||
}
|
||
|
||
/** Remove the state watcher and close the owned context, if any. */
|
||
private async closeContext(): Promise<void> {
|
||
const ctx = this.ctx;
|
||
if (!ctx) return;
|
||
if (this.ctxStateHandler) {
|
||
ctx.removeEventListener("statechange", this.ctxStateHandler);
|
||
this.ctxStateHandler = undefined;
|
||
}
|
||
this.ctx = undefined;
|
||
if (ctx.state !== "closed") await ctx.close().catch(() => undefined);
|
||
}
|
||
|
||
/** Adopt the prepared context (or create one) + install the state watcher. */
|
||
private async ensureContext(): Promise<void> {
|
||
const rate = sampleRateFor(this.config.model);
|
||
if (
|
||
this.ctx &&
|
||
this.ctx.state !== "closed" &&
|
||
this.ctx.sampleRate === rate
|
||
) {
|
||
if (this.ctx.state === "suspended" && !this.micMuted)
|
||
await resumeCtx(this.ctx);
|
||
return;
|
||
}
|
||
await this.closeContext();
|
||
|
||
// [lotus #7] Prefer the context/modules/model prepared before
|
||
// setProcessor() was called; only load inline if nothing was prepared
|
||
// (e.g. the rnnoise fallback path, or a second processor after a
|
||
// republish).
|
||
const claimed = await claimPreparedAssets(this.config)?.catch(
|
||
() => undefined,
|
||
);
|
||
let ctx: AudioContext;
|
||
if (claimed && claimed.ctx.state !== "closed") {
|
||
ctx = claimed.ctx;
|
||
this.preparedDfnCore = claimed.dfnCore;
|
||
} else {
|
||
ctx = await createModelContext(this.config);
|
||
}
|
||
try {
|
||
// Auto-resume if the OS/browser suspends the context mid-call (mobile
|
||
// backgrounding, audio interruption): the dest node otherwise emits
|
||
// silence with no recovery. Only resume while a graph is live and the
|
||
// suspension isn't our own mute suspension (#9).
|
||
const onStateChange = (): void => {
|
||
if (ctx.state === "suspended" && this.graph && !this.micMuted)
|
||
void ctx.resume().catch(() => undefined);
|
||
};
|
||
ctx.addEventListener("statechange", onStateChange);
|
||
// The action can arrive via host postMessage, not a gesture in this
|
||
// iframe, so the context can start suspended — resume without hanging.
|
||
if (ctx.state === "suspended" && !this.micMuted) await resumeCtx(ctx);
|
||
// Attached while already muted (#9): don't let a prepared, running
|
||
// context burn inference until the first unmute.
|
||
else if (ctx.state === "running" && this.micMuted)
|
||
await ctx.suspend().catch(() => undefined);
|
||
|
||
this.ctx = ctx;
|
||
this.ctxStateHandler = onStateChange;
|
||
} catch (e) {
|
||
// Don't leak a half-initialised context on any failure path.
|
||
await ctx.close().catch(() => undefined);
|
||
throw e;
|
||
}
|
||
}
|
||
|
||
private async buildGraph(track: MediaStreamTrack): Promise<Graph> {
|
||
const ctx = this.ctx!;
|
||
const source = ctx.createMediaStreamSource(new MediaStream([track]));
|
||
const dest = new MediaStreamAudioDestinationNode(ctx, MONO);
|
||
const nodes: AudioNode[] = [];
|
||
const disposes: (() => void)[] = [];
|
||
|
||
try {
|
||
// Wet (denoised) path: source → ml → [gate] → wetGain.
|
||
const ml = await this.buildMlNode(ctx);
|
||
source.connect(ml.node);
|
||
nodes.push(ml.node);
|
||
if (ml.dispose) disposes.push(ml.dispose);
|
||
let wetHead: AudioNode = ml.node;
|
||
|
||
// Gate AFTER the ML model, not before: gating the raw noisy signal fed
|
||
// hard-zeroed frames into the model (discontinuities it must fight) and
|
||
// made the threshold operate on pre-denoise levels. Gate the residual.
|
||
if (this.config.gate) {
|
||
const gate = new AudioWorkletNode(ctx, GATE.name, {
|
||
...MONO,
|
||
processorOptions: {
|
||
openThreshold: this.config.gateThreshold,
|
||
closeThreshold: this.config.gateThreshold - 5,
|
||
holdMs: 150,
|
||
maxChannels: 1,
|
||
},
|
||
});
|
||
wetHead.connect(gate);
|
||
wetHead = gate;
|
||
nodes.push(gate);
|
||
}
|
||
|
||
// Only mix a dry floor for the flat models (RNNoise/Speex), whose
|
||
// framing latency is known exactly (DRY_DELAY_SAMPLES); the DTLN/DFN
|
||
// figures are best-known estimates, so for those we rely on the model's
|
||
// own level (e.g. DFN noiseReductionLevel) instead. RNNoise is also where
|
||
// the "robotic/underwater" reports come from, so this targets it.
|
||
const lowLatency =
|
||
this.config.model === "rnnoise" || this.config.model === "speex";
|
||
const floor = lowLatency
|
||
? Math.min(0.5, Math.max(0, this.config.floor))
|
||
: 0;
|
||
if (floor > 0) {
|
||
// Dry/wet mix: blend a small amount of the ORIGINAL mic under the
|
||
// denoised signal so suppression can't fully collapse the noise floor
|
||
// (kills the "underwater"/pumping artifact). During speech (denoised ≈
|
||
// original) the two sum back to ~unity; in noise-only gaps the output
|
||
// floors at `floor` × original instead of digital silence.
|
||
const wetGain = new GainNode(ctx, { ...MONO, gain: 1 - floor });
|
||
wetHead.connect(wetGain);
|
||
wetGain.connect(dest);
|
||
nodes.push(wetGain);
|
||
|
||
// [lotus #25] Delay the dry copy by the model's algorithmic latency so
|
||
// it sums in phase with the (framed, hence delayed) wet path instead
|
||
// of comb-filtering against it.
|
||
const delaySec = DRY_DELAY_SAMPLES[this.config.model] / ctx.sampleRate;
|
||
const dryDelay = new DelayNode(ctx, {
|
||
...MONO,
|
||
maxDelayTime: Math.max(delaySec, 1 / ctx.sampleRate),
|
||
delayTime: delaySec,
|
||
});
|
||
const dryGain = new GainNode(ctx, { ...MONO, gain: floor });
|
||
source.connect(dryDelay);
|
||
dryDelay.connect(dryGain);
|
||
dryGain.connect(dest);
|
||
nodes.push(dryDelay, dryGain);
|
||
} else {
|
||
wetHead.connect(dest);
|
||
}
|
||
|
||
logger.info(
|
||
`[lotus] denoise processor active (${this.config.model}, floor=${floor})`,
|
||
);
|
||
return {
|
||
source,
|
||
nodes,
|
||
disposes,
|
||
track: dest.stream.getAudioTracks()[0],
|
||
};
|
||
} catch (e) {
|
||
// A node constructor / model load can throw mid-build; clean up the
|
||
// partially-built graph so it doesn't leak (init/restart still fall back
|
||
// to the raw mic on the rejection).
|
||
this.disposeGraph({
|
||
source,
|
||
nodes,
|
||
disposes,
|
||
track: dest.stream.getAudioTracks()[0],
|
||
});
|
||
throw e;
|
||
}
|
||
}
|
||
|
||
private async buildMlNode(ctx: AudioContext): Promise<MlNode> {
|
||
const base = this.config.assetBase;
|
||
const model = this.config.model;
|
||
|
||
if (model === "dtln") {
|
||
// Self-contained ESM that resolves its own processor + LiteRT wasm +
|
||
// TFLite models. bypassUntilReady passes raw audio until the model loads.
|
||
const mod = await loadDtlnModule(this.config);
|
||
return await mod.createNoiseSuppressionAudioWorklet(ctx, {
|
||
bypassUntilReady: true,
|
||
});
|
||
}
|
||
|
||
if (model === "deepfilternet") {
|
||
// [lotus #7] Use the core initialised by prepareDenoiseAssets() if we
|
||
// have one (first graph); later rebuilds (restart) load a fresh core.
|
||
const prepared = this.preparedDfnCore;
|
||
this.preparedDfnCore = undefined;
|
||
const core = prepared ?? (await loadDfnCore(this.config));
|
||
const node = await core.createAudioWorkletNode(ctx);
|
||
return { node, dispose: () => safeCall(() => core.destroy()) };
|
||
}
|
||
|
||
// Flat sapphi worklet (rnnoise/speex).
|
||
const flat = FLAT[model];
|
||
const { primary, fallback } = flatWasmFiles(model);
|
||
let wasmBinary: ArrayBuffer;
|
||
try {
|
||
wasmBinary = await fetchWasm(base + primary);
|
||
} catch (e) {
|
||
if (fallback) {
|
||
wasmCache.delete(base + primary);
|
||
wasmBinary = await fetchWasm(base + fallback); // fall back to non-SIMD
|
||
} else throw e;
|
||
}
|
||
const node = new AudioWorkletNode(ctx, flat.name, {
|
||
...MONO,
|
||
numberOfInputs: 1,
|
||
numberOfOutputs: 1,
|
||
processorOptions: { maxChannels: 1, wasmBinary },
|
||
});
|
||
return {
|
||
node,
|
||
dispose: () => safeCall(() => node.port.postMessage("destroy")),
|
||
};
|
||
}
|
||
|
||
private disposeGraph(graph: Graph | undefined): void {
|
||
if (!graph) return;
|
||
for (const dispose of graph.disposes) safeCall(dispose);
|
||
for (const node of graph.nodes) safeCall(() => node.disconnect());
|
||
safeCall(() => graph.source.disconnect());
|
||
graph.track.stop();
|
||
}
|
||
}
|
||
|
||
function safeCall(fn: () => void): void {
|
||
try {
|
||
fn();
|
||
} catch {
|
||
/* ignore */
|
||
}
|
||
}
|