diff --git a/src/lotus/lotusDenoise.ts b/src/lotus/lotusDenoise.ts new file mode 100644 index 00000000..623c986a --- /dev/null +++ b/src/lotus/lotusDenoise.ts @@ -0,0 +1,98 @@ +/* +Copyright 2026 Lotus Guild + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { + type LocalAudioTrack, + ParticipantEvent, + type Room as LivekitRoom, + Track, +} from "livekit-client"; +import { logger } from "matrix-js-sdk/lib/logger"; + +import { type CallViewModel } from "../state/CallViewModel/CallViewModel"; +import { lotusFlag, lotusParam } from "./lotusWidget"; +import { + type LotusDenoiseConfig, + LotusDenoiseProcessor, +} from "./lotusDenoiseProcessor"; + +/** + * Apply Lotus ML noise suppression to the local mic track as a first-class + * Element Call audio TrackProcessor (#1), replacing the host's build-time + * `getUserMedia` monkeypatch. + * + * Opt-in via `lotusDenoise=ml` (+ `lotusModel`, `lotusGate`, + * `lotusGateThreshold`, `lotusDenoiseBase`). Because the processor is attached + * to the mic publication and re-attached on every (re)publish, denoise + * survives EC's mid-call reconnect — fixing the A7 "mic dead after reconnect" + * bug. Additive: no-op without the flag. + */ +export function startLotusDenoise(vm: CallViewModel): () => void { + if (lotusParam("lotusDenoise") !== "ml") return () => undefined; + + const config: LotusDenoiseConfig = { + model: lotusParam("lotusModel") === "speex" ? "speex" : "rnnoise", + assetBase: lotusParam("lotusDenoiseBase") || "./denoise/", + gate: lotusFlag("lotusGate"), + gateThreshold: Number(lotusParam("lotusGateThreshold")) || -50, + }; + + const micOf = (room: LivekitRoom): LocalAudioTrack | undefined => + room.localParticipant.getTrackPublication(Track.Source.Microphone) + ?.track as LocalAudioTrack | undefined; + + const apply = (room: LivekitRoom): void => { + const mic = micOf(room); + if (mic && !mic.getProcessor()) { + void mic + .setProcessor(new LotusDenoiseProcessor(config)) + .catch((e) => logger.warn("[lotus] denoise setProcessor failed", e)); + } + }; + + const roomListeners = new Map void>(); + let rooms: LivekitRoom[] = []; + + const sub = vm.livekitRoomItems$.subscribe((items) => { + const next = items.map((i) => i.livekitRoom); + rooms = next; + for (const [room, off] of roomListeners) { + if (!next.includes(room)) { + off(); + roomListeners.delete(room); + } + } + for (const room of next) { + if (!roomListeners.has(room)) { + // Re-attach on every (re)publish — this is what makes denoise survive + // reconnects (A7), unlike the old getUserMedia patch. + const onPublished = (): void => apply(room); + room.localParticipant.on( + ParticipantEvent.LocalTrackPublished, + onPublished, + ); + roomListeners.set(room, () => + room.localParticipant.off( + ParticipantEvent.LocalTrackPublished, + onPublished, + ), + ); + apply(room); + } + } + }); + + return () => { + sub.unsubscribe(); + for (const off of roomListeners.values()) off(); + roomListeners.clear(); + for (const room of rooms) { + const mic = micOf(room); + if (mic?.getProcessor()) void mic.stopProcessor(); + } + }; +} diff --git a/src/lotus/lotusDenoiseProcessor.ts b/src/lotus/lotusDenoiseProcessor.ts new file mode 100644 index 00000000..f39a3273 --- /dev/null +++ b/src/lotus/lotusDenoiseProcessor.ts @@ -0,0 +1,151 @@ +/* +Copyright 2026 Lotus Guild + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { + type AudioProcessorOptions, + type Track, + type TrackProcessor, +} from "livekit-client"; +import { logger } from "matrix-js-sdk/lib/logger"; + +export type LotusDenoiseModel = "rnnoise" | "speex"; + +export interface LotusDenoiseConfig { + model: LotusDenoiseModel; + /** Base URL the worklet scripts + wasm are served from (e.g. "./denoise/"). */ + assetBase: string; + gate: boolean; + gateThreshold: number; +} + +// Flat sapphi worklets: each registers a processor under these names when its +// script module is added. Same assets the Lotus host already ships under +// public/element-call/denoise/, so no new EC dependency is needed. +const PROCESSORS: Record< + LotusDenoiseModel, + { name: string; script: string; wasm: string } +> = { + rnnoise: { + name: "@sapphi-red/web-noise-suppressor/rnnoise", + script: "rnnoiseWorklet.js", + wasm: "rnnoise.wasm", + }, + speex: { + name: "@sapphi-red/web-noise-suppressor/speex", + script: "speexWorklet.js", + wasm: "speex.wasm", + }, +}; +const GATE_NAME = "@sapphi-red/web-noise-suppressor/noiseGate"; +const GATE_SCRIPT = "noiseGateWorklet.js"; + +/** + * A LiveKit audio TrackProcessor that runs Lotus ML noise suppression + * (RNNoise/Speex) on the local microphone track, as a first-class stage in + * Element Call's publish pipeline. + * + * This replaces the host's `getUserMedia` monkeypatch: because it's a real + * LiveKit processor, EC re-applies it on every (re)publish, so denoise + * survives EC's own mid-call reconnect — the root cause of the A7 + * "mic dead after reconnect" bug. + */ +export class LotusDenoiseProcessor + implements TrackProcessor +{ + public readonly name = "lotus-denoise"; + public processedTrack?: MediaStreamTrack; + + private source?: MediaStreamAudioSourceNode; + private nodes: AudioWorkletNode[] = []; + + public constructor(private readonly config: LotusDenoiseConfig) {} + + public async init(opts: AudioProcessorOptions): Promise { + await this.build(opts.audioContext, opts.track); + } + + public async restart(opts: AudioProcessorOptions): Promise { + this.teardownGraph(); + await this.build(opts.audioContext, opts.track); + } + + public async destroy(): Promise { + this.teardownGraph(); + } + + private async build( + ctx: AudioContext, + track: MediaStreamTrack, + ): Promise { + const base = this.config.assetBase; + const proc = PROCESSORS[this.config.model]; + + // Register worklet modules (idempotent) and load the wasm binary. + await ctx.audioWorklet.addModule(base + proc.script); + if (this.config.gate) await ctx.audioWorklet.addModule(base + GATE_SCRIPT); + const wasmBinary = await fetch(base + proc.wasm).then((r) => { + if (!r.ok) throw new Error(`denoise wasm ${r.status}`); + return r.arrayBuffer(); + }); + + const source = ctx.createMediaStreamSource(new MediaStream([track])); + const dest = ctx.createMediaStreamDestination(); + let head: AudioNode = source; + + if (this.config.gate) { + const gate = new AudioWorkletNode(ctx, GATE_NAME, { + processorOptions: { + openThreshold: this.config.gateThreshold, + closeThreshold: this.config.gateThreshold - 5, + holdMs: 150, + maxChannels: 1, + }, + }); + head.connect(gate); + head = gate; + this.nodes.push(gate); + } + + const ml = new AudioWorkletNode(ctx, proc.name, { + channelCount: 1, + numberOfInputs: 1, + numberOfOutputs: 1, + processorOptions: { maxChannels: 1, wasmBinary }, + }); + head.connect(ml); + ml.connect(dest); + this.nodes.push(ml); + + this.source = source; + this.processedTrack = dest.stream.getAudioTracks()[0]; + logger.info(`[lotus] denoise processor active (${this.config.model})`); + } + + private teardownGraph(): void { + for (const node of this.nodes) { + try { + node.port.postMessage("destroy"); + } catch { + /* ignore */ + } + try { + node.disconnect(); + } catch { + /* ignore */ + } + } + this.nodes = []; + try { + this.source?.disconnect(); + } catch { + /* ignore */ + } + this.source = undefined; + this.processedTrack?.stop(); + this.processedTrack = undefined; + } +} diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index fd1078b9..50c19f44 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -34,6 +34,7 @@ import { startLotusFocus } from "../lotus/lotusFocus"; import { startLotusAudioInject } from "../lotus/lotusAudioInject"; import { startLotusQuality } from "../lotus/lotusQuality"; import { startLotusDecorations } from "../lotus/lotusDecorations"; +import { startLotusDenoise } from "../lotus/lotusDenoise"; import styles from "./InCallView.module.css"; import { GridTile } from "../tile/GridTile"; import { SettingsModal, defaultSettingsTab } from "../settings/SettingsModal"; @@ -299,6 +300,9 @@ export const InCallView: FC = ({ // [lotus] Receive per-user avatar-decoration URLs from the host and render // them on in-call tile avatars (#6). No-op unless the host sends them. useEffect(() => startLotusDecorations(), []); + // [lotus] Apply ML denoise to the mic as a first-class audio processor that + // survives reconnects (#1 / A7). No-op unless lotusDenoise=ml. + useEffect(() => startLotusDenoise(vm), [vm]); const fatalCallError = useBehavior(vm.fatalError$); // Stop the rendering and throw for the error boundary