diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index 90106bed..d7d139ca 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -21,6 +21,8 @@ on: env: # element-call's build:full sets 16384 already; keep parity for safety. NODE_OPTIONS: '--max-old-space-size=16384' + # Stamp the build so analytics/rageshakes aren't labelled "dev". + VITE_APP_VERSION: ${{ github.ref_name }} jobs: build: @@ -106,8 +108,13 @@ jobs: - name: Set published version from tag working-directory: embedded/web + # Versioning scheme: reserve bare vX.Y.Z for upstream-parity points only + # (e.g. v0.20.1 == upstream 0.20.1). For Lotus-only iterations on top of + # an upstream base, tag a semver prerelease — v0.20.1-lotus.1, -lotus.2, + # … — so we never collide with an already-published upstream-parity + # version on the registry. run: | - TAG="${GITHUB_REF_NAME#v}" # v0.20.1 -> 0.20.1 + TAG="${GITHUB_REF_NAME#v}" # v0.20.1-lotus.1 -> 0.20.1-lotus.1 npm version "$TAG" --no-git-tag-version --allow-same-version - name: Publish diff --git a/src/lotus/lotusActions.ts b/src/lotus/lotusActions.ts index de94ed00..888a8890 100644 --- a/src/lotus/lotusActions.ts +++ b/src/lotus/lotusActions.ts @@ -14,9 +14,11 @@ export enum LotusWidgetActions { /** * fromWidget: in-call per-participant speaking / mute state. * NOTE: matrix-widget-api `transport.send` is request/response — the host - * MUST reply/ack each one (cinny's `listenAction` does, replying `{}`), - * otherwise every send sits pending for the 10s transport timeout and then - * rejects, producing continuous churn + log noise for the whole call. + * MUST register a handler that replies/acks each one (cinny's `listenAction` + * does, replying `{}`). If the host has no handler, ClientWidgetApi + * immediately error-replies "unsupported from-widget action", so the data is + * silently dropped and each throttled send rejects (caught) — functional miss + * + log churn, not a hang. */ CallState = "io.lotus.call_state", /** toWidget: pin/spotlight (or clear, with userId=null) a participant. */ diff --git a/src/lotus/lotusDenoise.ts b/src/lotus/lotusDenoise.ts index a3f3ed22..af916664 100644 --- a/src/lotus/lotusDenoise.ts +++ b/src/lotus/lotusDenoise.ts @@ -53,11 +53,28 @@ function safeAssetBase(raw: string | null): string { export function startLotusDenoise(vm: CallViewModel): () => void { if (lotusParam("lotusDenoise") !== "ml") return () => undefined; + const requestedModel = lotusParam("lotusModel"); + // Only RNNoise/Speex are implemented in-source so far. Don't silently + // degrade an unsupported model (e.g. the host's dtln/deepfilternet) to + // rnnoise — log it so the mismatch is visible. + let model: LotusDenoiseConfig["model"] = "rnnoise"; + if (requestedModel === "speex") model = "speex"; + else if (requestedModel && requestedModel !== "rnnoise") + logger.warn( + `[lotus] denoise model "${requestedModel}" not implemented in-source; using rnnoise`, + ); + + const rawThreshold = lotusParam("lotusGateThreshold"); const config: LotusDenoiseConfig = { - model: lotusParam("lotusModel") === "speex" ? "speex" : "rnnoise", + model, assetBase: safeAssetBase(lotusParam("lotusDenoiseBase")), gate: lotusFlag("lotusGate"), - gateThreshold: Number(lotusParam("lotusGateThreshold")) || -50, + // Default -45 (matches the reference shim); accept an explicit 0 (don't + // coerce it away via `|| default`). + gateThreshold: + rawThreshold !== null && Number.isFinite(Number(rawThreshold)) + ? Number(rawThreshold) + : -45, }; const micOf = (room: LivekitRoom): LocalAudioTrack | undefined => diff --git a/src/lotus/lotusDenoiseProcessor.ts b/src/lotus/lotusDenoiseProcessor.ts index f39a3273..f6f55fbd 100644 --- a/src/lotus/lotusDenoiseProcessor.ts +++ b/src/lotus/lotusDenoiseProcessor.ts @@ -25,6 +25,14 @@ export interface LotusDenoiseConfig { // Flat sapphi worklets: each registers a processor under these names when its // script module is added. Same assets the Lotus host already ships under // public/element-call/denoise/, so no new EC dependency is needed. +// +// ⚠️ CONTRACT: this is an undeclared, cross-repo asset dependency. The embedded +// fork build does NOT bundle these — they are copied in by cinny's +// vite.config.js `lotusDenoise()` plugin from `@sapphi-red/web-noise-suppressor`. +// The worklet/wasm version must match what this processor expects (sapphi +// rnnoise/speex, 48kHz). If the fork ever wants to own them, add +// `@sapphi-red/web-noise-suppressor` as a dep + a copy step here. Until then, +// an integration smoke-check should assert GET .../denoise/rnnoise.wasm == 200. const PROCESSORS: Record< LotusDenoiseModel, { name: string; script: string; wasm: string } @@ -43,15 +51,46 @@ const PROCESSORS: Record< const GATE_NAME = "@sapphi-red/web-noise-suppressor/noiseGate"; const GATE_SCRIPT = "noiseGateWorklet.js"; +// The sapphi rnnoise/speex worklets are 48kHz-fullband and don't resample, so +// the graph MUST run at 48kHz regardless of the hardware default. +const SAMPLE_RATE = 48_000; + +// Cache fetched wasm per URL so a reconnect/device-switch (which rebuilds the +// graph) doesn't re-download it. +const wasmCache = new Map>(); +function loadWasm(url: string): Promise { + let p = wasmCache.get(url); + if (!p) { + p = fetch(url).then((r) => { + if (!r.ok) throw new Error(`denoise wasm ${r.status}`); + return r.arrayBuffer(); + }); + wasmCache.set(url, p); + } + return p; +} + +interface Graph { + source: MediaStreamAudioSourceNode; + nodes: AudioWorkletNode[]; + track: MediaStreamTrack; +} + /** * A LiveKit audio TrackProcessor that runs Lotus ML noise suppression * (RNNoise/Speex) on the local microphone track, as a first-class stage in * Element Call's publish pipeline. * * This replaces the host's `getUserMedia` monkeypatch: because it's a real - * LiveKit processor, EC re-applies it on every (re)publish, so denoise - * survives EC's own mid-call reconnect — the root cause of the A7 - * "mic dead after reconnect" bug. + * LiveKit processor, EC re-applies it on every (re)publish/restart, so denoise + * survives EC's own mid-call reconnect — the root cause of the A7 "mic dead + * after reconnect" bug. + * + * It owns a dedicated 48kHz AudioContext (the worklets require it, and LiveKit + * does NOT pass an audioContext to restart()), reused across restarts and + * closed on destroy. restart() never throws and never leaves a stopped track + * on the sender: on failure it degrades to the raw (un-denoised) track rather + * than silence. */ export class LotusDenoiseProcessor implements TrackProcessor @@ -59,41 +98,58 @@ export class LotusDenoiseProcessor public readonly name = "lotus-denoise"; public processedTrack?: MediaStreamTrack; - private source?: MediaStreamAudioSourceNode; - private nodes: AudioWorkletNode[] = []; + private ctx?: AudioContext; + private graph?: Graph; public constructor(private readonly config: LotusDenoiseConfig) {} - public async init(opts: AudioProcessorOptions): Promise { - await this.build(opts.audioContext, opts.track); + public async init(_opts: AudioProcessorOptions): Promise { + // Own context at the required rate — do NOT use opts.audioContext (it's the + // room's hardware-rate context, and is undefined on restart()). + this.ctx = new AudioContext({ sampleRate: SAMPLE_RATE }); + const graph = await this.buildGraph(_opts.track); + this.graph = graph; + this.processedTrack = graph.track; } public async restart(opts: AudioProcessorOptions): Promise { - this.teardownGraph(); - await this.build(opts.audioContext, opts.track); + if (!this.ctx || this.ctx.state === "closed") + this.ctx = new AudioContext({ sampleRate: SAMPLE_RATE }); + try { + const next = await this.buildGraph(opts.track); + this.disposeGraph(this.graph); + this.graph = next; + this.processedTrack = next.track; + } catch (e) { + // Never go silent on the A7/device-switch path: fall back to raw audio. + logger.warn("[lotus] denoise restart failed; using raw mic", e); + this.disposeGraph(this.graph); + this.graph = undefined; + this.processedTrack = opts.track; + } } public async destroy(): Promise { - this.teardownGraph(); + this.disposeGraph(this.graph); + this.graph = undefined; + this.processedTrack = undefined; + if (this.ctx && this.ctx.state !== "closed") + await this.ctx.close().catch(() => undefined); + this.ctx = undefined; } - private async build( - ctx: AudioContext, - track: MediaStreamTrack, - ): Promise { + private async buildGraph(track: MediaStreamTrack): Promise { + const ctx = this.ctx!; const base = this.config.assetBase; const proc = PROCESSORS[this.config.model]; - // Register worklet modules (idempotent) and load the wasm binary. await ctx.audioWorklet.addModule(base + proc.script); if (this.config.gate) await ctx.audioWorklet.addModule(base + GATE_SCRIPT); - const wasmBinary = await fetch(base + proc.wasm).then((r) => { - if (!r.ok) throw new Error(`denoise wasm ${r.status}`); - return r.arrayBuffer(); - }); + const wasmBinary = await loadWasm(base + proc.wasm); const source = ctx.createMediaStreamSource(new MediaStream([track])); const dest = ctx.createMediaStreamDestination(); + const nodes: AudioWorkletNode[] = []; let head: AudioNode = source; if (this.config.gate) { @@ -107,26 +163,26 @@ export class LotusDenoiseProcessor }); head.connect(gate); head = gate; - this.nodes.push(gate); + nodes.push(gate); } const ml = new AudioWorkletNode(ctx, proc.name, { channelCount: 1, numberOfInputs: 1, numberOfOutputs: 1, - processorOptions: { maxChannels: 1, wasmBinary }, + processorOptions: { maxChannels: 1, wasmBinary: wasmBinary.slice(0) }, }); head.connect(ml); ml.connect(dest); - this.nodes.push(ml); + nodes.push(ml); - this.source = source; - this.processedTrack = dest.stream.getAudioTracks()[0]; logger.info(`[lotus] denoise processor active (${this.config.model})`); + return { source, nodes, track: dest.stream.getAudioTracks()[0] }; } - private teardownGraph(): void { - for (const node of this.nodes) { + private disposeGraph(graph: Graph | undefined): void { + if (!graph) return; + for (const node of graph.nodes) { try { node.port.postMessage("destroy"); } catch { @@ -138,14 +194,11 @@ export class LotusDenoiseProcessor /* ignore */ } } - this.nodes = []; try { - this.source?.disconnect(); + graph.source.disconnect(); } catch { /* ignore */ } - this.source = undefined; - this.processedTrack?.stop(); - this.processedTrack = undefined; + graph.track.stop(); } }