lotus(#1): implement all 4 denoise models; fix gate name + rates

Faithful port of cinny's proven pipeline into the TrackProcessor, closing
protocol gap F3 (host offers rnnoise/speex/dtln/deepfilternet; only the
first two existed in-source).

- Fix real bug: gate worklet registers as "noise-gate" (hyphenated), not
  "noiseGate" — the gated path would have failed to construct the node.
- Per-model sample rate: DTLN runs at 16kHz, others 48kHz (worklets don't
  resample); verify the context actually got the rate, else fall back.
- resume() a suspended context (host postMessage isn't a gesture).
- DTLN via dynamic-imported @workadventure helper (bypassUntilReady);
  DeepFilterNet via dynamic-imported ESM + DeepFilterNet3Core pointed at
  the self-hosted base. Same-origin base (kept from the C1 fix) makes these
  dynamic imports safe.
- Prefer SIMD rnnoise.wasm with non-SIMD fallback; cache wasm per URL.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

src/lotus/lotusDenoise.ts

@@ -53,16 +53,13 @@ function safeAssetBase(raw: string | null): string {
 export function startLotusDenoise(vm: CallViewModel): () => void {
   if (lotusParam("lotusDenoise") !== "ml") return () => undefined;
 
-  const requestedModel = lotusParam("lotusModel");
+  const requested = lotusParam("lotusModel");
-  // Only RNNoise/Speex are implemented in-source so far. Don't silently
+  const model: LotusDenoiseConfig["model"] =
-  // degrade an unsupported model (e.g. the host's dtln/deepfilternet) to
+    requested === "speex" ||
-  // rnnoise — log it so the mismatch is visible.
+    requested === "dtln" ||
-  let model: LotusDenoiseConfig["model"] = "rnnoise";
+    requested === "deepfilternet"
-  if (requestedModel === "speex") model = "speex";
+      ? requested
-  else if (requestedModel && requestedModel !== "rnnoise")
+      : "rnnoise";
-    logger.warn(
-      `[lotus] denoise model "${requestedModel}" not implemented in-source; using rnnoise`,
-    );
 
   const rawThreshold = lotusParam("lotusGateThreshold");
   const config: LotusDenoiseConfig = {

src/lotus/lotusDenoiseProcessor.ts

@@ -12,35 +12,37 @@ import {
 } from "livekit-client";
 import { logger } from "matrix-js-sdk/lib/logger";
 
-export type LotusDenoiseModel = "rnnoise" | "speex";
+export type LotusDenoiseModel =
+  | "rnnoise"
+  | "speex"
+  | "dtln"
+  | "deepfilternet";
 
 export interface LotusDenoiseConfig {
   model: LotusDenoiseModel;
-  /** Base URL the worklet scripts + wasm are served from (e.g. "./denoise/"). */
+  /** Base URL the worklet scripts/wasm/ESM are served from (e.g. "./denoise/"). */
   assetBase: string;
   gate: boolean;
   gateThreshold: number;
 }
 
-// Flat sapphi worklets: each registers a processor under these names when its
+// Flat sapphi worklets (RNNoise/Speex): each registers a processor under `name`
-// script module is added. Same assets the Lotus host already ships under
+// when its `script` module is added; we feed it the fetched wasm binary.
-// public/element-call/denoise/, so no new EC dependency is needed.
+// ⚠️ CONTRACT: these assets are NOT bundled by the fork build — cinny's
-//
+// vite.config.js `lotusDenoise()` plugin copies them from
-// ⚠️ CONTRACT: this is an undeclared, cross-repo asset dependency. The embedded
+// `@sapphi-red/web-noise-suppressor` / `@workadventure/noise-suppression` /
-// fork build does NOT bundle these — they are copied in by cinny's
+// `deepfilternet3-noise-filter` into public/element-call/denoise/. The
-// vite.config.js `lotusDenoise()` plugin from `@sapphi-red/web-noise-suppressor`.
+// worklet/wasm/ESM versions must match what this processor expects. An
-// The worklet/wasm version must match what this processor expects (sapphi
+// integration smoke-check should assert GET .../denoise/rnnoise.wasm == 200.
-// rnnoise/speex, 48kHz). If the fork ever wants to own them, add
+const FLAT: Record<
-// `@sapphi-red/web-noise-suppressor` as a dep + a copy step here. Until then,
+  "rnnoise" | "speex",
-// an integration smoke-check should assert GET .../denoise/rnnoise.wasm == 200.
+  { name: string; script: string; wasm: string; simdWasm?: string }
-const PROCESSORS: Record<
-  LotusDenoiseModel,
-  { name: string; script: string; wasm: string }
 > = {
   rnnoise: {
     name: "@sapphi-red/web-noise-suppressor/rnnoise",
     script: "rnnoiseWorklet.js",
     wasm: "rnnoise.wasm",
+    simdWasm: "rnnoise_simd.wasm",
   },
   speex: {
     name: "@sapphi-red/web-noise-suppressor/speex",
@@ -48,21 +50,25 @@ const PROCESSORS: Record<
     wasm: "speex.wasm",
   },
 };
-const GATE_NAME = "@sapphi-red/web-noise-suppressor/noiseGate";
+// The sapphi gate worklet registers under "noise-gate" (hyphenated).
-const GATE_SCRIPT = "noiseGateWorklet.js";
+const GATE = {
+  name: "@sapphi-red/web-noise-suppressor/noise-gate",
+  script: "noiseGateWorklet.js",
+};
 
-// The sapphi rnnoise/speex worklets are 48kHz-fullband and don't resample, so
+// DTLN (@workadventure) targets 16kHz and doesn't resample; RNNoise/Speex and
-// the graph MUST run at 48kHz regardless of the hardware default.
+// DeepFilterNet are 48kHz fullband. The worklets don't resample, so the whole
-const SAMPLE_RATE = 48_000;
+// graph must run at the model's native rate.
+const sampleRateFor = (model: LotusDenoiseModel): number =>
+  model === "dtln" ? 16_000 : 48_000;
 
-// Cache fetched wasm per URL so a reconnect/device-switch (which rebuilds the
+// Cache fetched wasm per URL so a reconnect/device-switch doesn't re-download.
-// graph) doesn't re-download it.
 const wasmCache = new Map<string, Promise<ArrayBuffer>>();
-function loadWasm(url: string): Promise<ArrayBuffer> {
+function fetchWasm(url: string): Promise<ArrayBuffer> {
   let p = wasmCache.get(url);
   if (!p) {
     p = fetch(url).then((r) => {
-      if (!r.ok) throw new Error(`denoise wasm ${r.status}`);
+      if (!r.ok) throw new Error(`denoise wasm ${url} -> ${r.status}`);
       return r.arrayBuffer();
     });
     wasmCache.set(url, p);
@@ -70,27 +76,44 @@ function loadWasm(url: string): Promise<ArrayBuffer> {
   return p;
 }
 
+function supportsSimd(): boolean {
+  try {
+    // Minimal SIMD module (v128) — validates only where SIMD is supported.
+    return WebAssembly.validate(
+      new Uint8Array([
+        0, 97, 115, 109, 1, 0, 0, 0, 1, 5, 1, 96, 0, 1, 123, 3, 2, 1, 0, 10,
+        10, 1, 8, 0, 65, 0, 253, 15, 253, 98, 11,
+      ]),
+    );
+  } catch {
+    return false;
+  }
+}
+
+interface MlNode {
+  node: AudioNode;
+  dispose?: () => void;
+}
 interface Graph {
   source: MediaStreamAudioSourceNode;
-  nodes: AudioWorkletNode[];
+  nodes: AudioNode[];
+  disposes: (() => void)[];
   track: MediaStreamTrack;
 }
 
 /**
  * A LiveKit audio TrackProcessor that runs Lotus ML noise suppression
- * (RNNoise/Speex) on the local microphone track, as a first-class stage in
+ * (RNNoise / Speex / DTLN / DeepFilterNet) on the local microphone track, as a
- * Element Call's publish pipeline.
+ * first-class stage in Element Call's publish pipeline — replacing the host's
+ * `getUserMedia` monkeypatch.
  *
- * This replaces the host's `getUserMedia` monkeypatch: because it's a real
+ * Because it's a real LiveKit processor, EC re-applies it on every
- * LiveKit processor, EC re-applies it on every (re)publish/restart, so denoise
+ * (re)publish/restart, so denoise survives EC's mid-call reconnect — the root
- * survives EC's own mid-call reconnect — the root cause of the A7 "mic dead
+ * cause of A7. It owns a dedicated AudioContext at the model's required sample
- * after reconnect" bug.
+ * rate (LiveKit does NOT pass an audioContext to restart()), reused across
- *
+ * restarts and closed on destroy. restart() never throws and never leaves a
- * It owns a dedicated 48kHz AudioContext (the worklets require it, and LiveKit
+ * stopped track on the sender: on failure it degrades to the RAW mic track
- * does NOT pass an audioContext to restart()), reused across restarts and
+ * rather than silence.
- * closed on destroy. restart() never throws and never leaves a stopped track
- * on the sender: on failure it degrades to the raw (un-denoised) track rather
- * than silence.
  */
 export class LotusDenoiseProcessor
   implements TrackProcessor<Track.Kind.Audio, AudioProcessorOptions>
@@ -104,18 +127,14 @@ export class LotusDenoiseProcessor
   public constructor(private readonly config: LotusDenoiseConfig) {}
 
   public async init(_opts: AudioProcessorOptions): Promise<void> {
-    // Own context at the required rate — do NOT use opts.audioContext (it's the
+    await this.ensureContext();
-    // room's hardware-rate context, and is undefined on restart()).
+    this.graph = await this.buildGraph(_opts.track);
-    this.ctx = new AudioContext({ sampleRate: SAMPLE_RATE });
+    this.processedTrack = this.graph.track;
-    const graph = await this.buildGraph(_opts.track);
-    this.graph = graph;
-    this.processedTrack = graph.track;
   }
 
   public async restart(opts: AudioProcessorOptions): Promise<void> {
-    if (!this.ctx || this.ctx.state === "closed")
-      this.ctx = new AudioContext({ sampleRate: SAMPLE_RATE });
     try {
+      await this.ensureContext();
       const next = await this.buildGraph(opts.track);
       this.disposeGraph(this.graph);
       this.graph = next;
@@ -138,22 +157,45 @@ export class LotusDenoiseProcessor
     this.ctx = undefined;
   }
 
+  /** Create (once) the model-rate context + register the flat worklet modules. */
+  private async ensureContext(): Promise<void> {
+    const rate = sampleRateFor(this.config.model);
+    if (this.ctx && this.ctx.state !== "closed" && this.ctx.sampleRate === rate) {
+      if (this.ctx.state === "suspended") await this.ctx.resume();
+      return;
+    }
+    if (this.ctx && this.ctx.state !== "closed")
+      await this.ctx.close().catch(() => undefined);
+
+    const ctx = new AudioContext({ sampleRate: rate });
+    if (ctx.sampleRate !== rate) {
+      await ctx.close().catch(() => undefined);
+      throw new Error(`denoise: got ${ctx.sampleRate}Hz, need ${rate}Hz`);
+    }
+    // Flat models register via addModule here; DTLN/DeepFilterNet bring their
+    // own processor via the dynamic-imported helper (see buildMlNode).
+    if (this.config.model === "rnnoise" || this.config.model === "speex")
+      await ctx.audioWorklet.addModule(
+        this.config.assetBase + FLAT[this.config.model].script,
+      );
+    if (this.config.gate)
+      await ctx.audioWorklet.addModule(this.config.assetBase + GATE.script);
+    // The action arrives via host postMessage, not a gesture in this iframe, so
+    // the context can start suspended.
+    if (ctx.state === "suspended") await ctx.resume();
+    this.ctx = ctx;
+  }
+
   private async buildGraph(track: MediaStreamTrack): Promise<Graph> {
     const ctx = this.ctx!;
-    const base = this.config.assetBase;
-    const proc = PROCESSORS[this.config.model];
-
-    await ctx.audioWorklet.addModule(base + proc.script);
-    if (this.config.gate) await ctx.audioWorklet.addModule(base + GATE_SCRIPT);
-    const wasmBinary = await loadWasm(base + proc.wasm);
-
     const source = ctx.createMediaStreamSource(new MediaStream([track]));
     const dest = ctx.createMediaStreamDestination();
-    const nodes: AudioWorkletNode[] = [];
+    const nodes: AudioNode[] = [];
+    const disposes: (() => void)[] = [];
     let head: AudioNode = source;
 
     if (this.config.gate) {
-      const gate = new AudioWorkletNode(ctx, GATE_NAME, {
+      const gate = new AudioWorkletNode(ctx, GATE.name, {
         processorOptions: {
           openThreshold: this.config.gateThreshold,
           closeThreshold: this.config.gateThreshold - 5,
@@ -166,39 +208,77 @@ export class LotusDenoiseProcessor
       nodes.push(gate);
     }
 
-    const ml = new AudioWorkletNode(ctx, proc.name, {
+    const ml = await this.buildMlNode(ctx);
+    head.connect(ml.node);
+    ml.node.connect(dest);
+    nodes.push(ml.node);
+    if (ml.dispose) disposes.push(ml.dispose);
+
+    logger.info(`[lotus] denoise processor active (${this.config.model})`);
+    return { source, nodes, disposes, track: dest.stream.getAudioTracks()[0] };
+  }
+
+  private async buildMlNode(ctx: AudioContext): Promise<MlNode> {
+    const base = this.config.assetBase;
+    const model = this.config.model;
+
+    if (model === "dtln") {
+      // Self-contained ESM that resolves its own processor + LiteRT wasm +
+      // TFLite models. bypassUntilReady passes raw audio until the model loads.
+      const mod = await import(/* @vite-ignore */ `${base}workadventure/audio-worklet.js`);
+      return (await mod.createNoiseSuppressionAudioWorklet(ctx, {
+        bypassUntilReady: true,
+      })) as MlNode;
+    }
+
+    if (model === "deepfilternet") {
+      const dfnBase = new URL(`${base}deepfilternet`, window.location.href).href;
+      const mod = await import(/* @vite-ignore */ `${base}deepfilternet/index.esm.js`);
+      const core = new mod.DeepFilterNet3Core({
+        sampleRate: 48_000,
+        noiseReductionLevel: 80,
+        assetConfig: { cdnUrl: dfnBase },
+      });
+      await core.initialize();
+      const node = (await core.createAudioWorkletNode(ctx)) as AudioNode;
+      return { node, dispose: () => void safeCall(() => core.destroy()) };
+    }
+
+    // Flat sapphi worklet (rnnoise/speex).
+    const flat = FLAT[model];
+    const useSimd = model === "rnnoise" && !!flat.simdWasm && supportsSimd();
+    const wasmFile = useSimd ? flat.simdWasm! : flat.wasm;
+    let wasmBinary: ArrayBuffer;
+    try {
+      wasmBinary = await fetchWasm(base + wasmFile);
+    } catch (e) {
+      if (useSimd) {
+        wasmCache.delete(base + wasmFile);
+        wasmBinary = await fetchWasm(base + flat.wasm); // fall back to non-SIMD
+      } else throw e;
+    }
+    const node = new AudioWorkletNode(ctx, flat.name, {
       channelCount: 1,
       numberOfInputs: 1,
       numberOfOutputs: 1,
-      processorOptions: { maxChannels: 1, wasmBinary: wasmBinary.slice(0) },
+      processorOptions: { maxChannels: 1, wasmBinary },
     });
-    head.connect(ml);
+    return { node, dispose: () => void safeCall(() => node.port.postMessage("destroy")) };
-    ml.connect(dest);
-    nodes.push(ml);
-
-    logger.info(`[lotus] denoise processor active (${this.config.model})`);
-    return { source, nodes, track: dest.stream.getAudioTracks()[0] };
   }
 
   private disposeGraph(graph: Graph | undefined): void {
     if (!graph) return;
-    for (const node of graph.nodes) {
+    for (const dispose of graph.disposes) safeCall(dispose);
-      try {
+    for (const node of graph.nodes) safeCall(() => node.disconnect());
-        node.port.postMessage("destroy");
+    safeCall(() => graph.source.disconnect());
-      } catch {
-        /* ignore */
-      }
-      try {
-        node.disconnect();
-      } catch {
-        /* ignore */
-      }
-    }
-    try {
-      graph.source.disconnect();
-    } catch {
-      /* ignore */
-    }
     graph.track.stop();
   }
 }
+
+function safeCall(fn: () => void): void {
+  try {
+    fn();
+  } catch {
+    /* ignore */
+  }
+}