lotus(#1): fix restart-silence (A7), 48kHz ctx; protocol + CI hardening
CI / Build embedded bundle (push) Successful in 1m20s
CI / Publish to Gitea npm registry (push) Has been skipped

Denoise deep-review (CRITICAL): restart() read opts.audioContext, which
LiveKit does NOT pass on restart — so reconnect (the A7 scenario) and mic
device-switch threw after stopping the old track, leaving the mic SILENT
(A7 reintroduced). Fix:
- Processor owns a dedicated 48kHz AudioContext (sapphi worklets require
  48kHz; H1), reused across restart, closed on destroy.
- restart() never throws and never leaves a stopped track on the sender:
  builds the new graph first, then disposes the old; on failure degrades
  to RAW mic audio rather than silence.
- Cache wasm per URL (no re-fetch each reconnect); gate threshold default
  -45 and accept an explicit 0 (M2); document the cross-repo asset contract.

Protocol audit:
- Non-silent warning when an unsupported denoise model (dtln/deepfilternet)
  is requested instead of silent rnnoise fallback (F3).
- Correct the call_state enum comment (immediate error-reply, not 10s) (F2).

Build/CI audit:
- Stamp VITE_APP_VERSION in CI; document the vX.Y.Z-lotus.N version scheme.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Lotus CI
2026-06-30 00:15:59 -04:00
co-authored by Claude Opus 4.8
parent b8543c3fe1
commit 29592fbb18
4 changed files with 116 additions and 37 deletions
+84 -31
View File
@@ -25,6 +25,14 @@ export interface LotusDenoiseConfig {
// Flat sapphi worklets: each registers a processor under these names when its
// script module is added. Same assets the Lotus host already ships under
// public/element-call/denoise/, so no new EC dependency is needed.
//
// ⚠️ CONTRACT: this is an undeclared, cross-repo asset dependency. The embedded
// fork build does NOT bundle these — they are copied in by cinny's
// vite.config.js `lotusDenoise()` plugin from `@sapphi-red/web-noise-suppressor`.
// The worklet/wasm version must match what this processor expects (sapphi
// rnnoise/speex, 48kHz). If the fork ever wants to own them, add
// `@sapphi-red/web-noise-suppressor` as a dep + a copy step here. Until then,
// an integration smoke-check should assert GET .../denoise/rnnoise.wasm == 200.
const PROCESSORS: Record<
LotusDenoiseModel,
{ name: string; script: string; wasm: string }
@@ -43,15 +51,46 @@ const PROCESSORS: Record<
const GATE_NAME = "@sapphi-red/web-noise-suppressor/noiseGate";
const GATE_SCRIPT = "noiseGateWorklet.js";
// The sapphi rnnoise/speex worklets are 48kHz-fullband and don't resample, so
// the graph MUST run at 48kHz regardless of the hardware default.
const SAMPLE_RATE = 48_000;
// Cache fetched wasm per URL so a reconnect/device-switch (which rebuilds the
// graph) doesn't re-download it.
const wasmCache = new Map<string, Promise<ArrayBuffer>>();
function loadWasm(url: string): Promise<ArrayBuffer> {
let p = wasmCache.get(url);
if (!p) {
p = fetch(url).then((r) => {
if (!r.ok) throw new Error(`denoise wasm ${r.status}`);
return r.arrayBuffer();
});
wasmCache.set(url, p);
}
return p;
}
interface Graph {
source: MediaStreamAudioSourceNode;
nodes: AudioWorkletNode[];
track: MediaStreamTrack;
}
/**
* A LiveKit audio TrackProcessor that runs Lotus ML noise suppression
* (RNNoise/Speex) on the local microphone track, as a first-class stage in
* Element Call's publish pipeline.
*
* This replaces the host's `getUserMedia` monkeypatch: because it's a real
* LiveKit processor, EC re-applies it on every (re)publish, so denoise
* survives EC's own mid-call reconnect — the root cause of the A7
* "mic dead after reconnect" bug.
* LiveKit processor, EC re-applies it on every (re)publish/restart, so denoise
* survives EC's own mid-call reconnect — the root cause of the A7 "mic dead
* after reconnect" bug.
*
* It owns a dedicated 48kHz AudioContext (the worklets require it, and LiveKit
* does NOT pass an audioContext to restart()), reused across restarts and
* closed on destroy. restart() never throws and never leaves a stopped track
* on the sender: on failure it degrades to the raw (un-denoised) track rather
* than silence.
*/
export class LotusDenoiseProcessor
implements TrackProcessor<Track.Kind.Audio, AudioProcessorOptions>
@@ -59,41 +98,58 @@ export class LotusDenoiseProcessor
public readonly name = "lotus-denoise";
public processedTrack?: MediaStreamTrack;
private source?: MediaStreamAudioSourceNode;
private nodes: AudioWorkletNode[] = [];
private ctx?: AudioContext;
private graph?: Graph;
public constructor(private readonly config: LotusDenoiseConfig) {}
public async init(opts: AudioProcessorOptions): Promise<void> {
await this.build(opts.audioContext, opts.track);
public async init(_opts: AudioProcessorOptions): Promise<void> {
// Own context at the required rate — do NOT use opts.audioContext (it's the
// room's hardware-rate context, and is undefined on restart()).
this.ctx = new AudioContext({ sampleRate: SAMPLE_RATE });
const graph = await this.buildGraph(_opts.track);
this.graph = graph;
this.processedTrack = graph.track;
}
public async restart(opts: AudioProcessorOptions): Promise<void> {
this.teardownGraph();
await this.build(opts.audioContext, opts.track);
if (!this.ctx || this.ctx.state === "closed")
this.ctx = new AudioContext({ sampleRate: SAMPLE_RATE });
try {
const next = await this.buildGraph(opts.track);
this.disposeGraph(this.graph);
this.graph = next;
this.processedTrack = next.track;
} catch (e) {
// Never go silent on the A7/device-switch path: fall back to raw audio.
logger.warn("[lotus] denoise restart failed; using raw mic", e);
this.disposeGraph(this.graph);
this.graph = undefined;
this.processedTrack = opts.track;
}
}
public async destroy(): Promise<void> {
this.teardownGraph();
this.disposeGraph(this.graph);
this.graph = undefined;
this.processedTrack = undefined;
if (this.ctx && this.ctx.state !== "closed")
await this.ctx.close().catch(() => undefined);
this.ctx = undefined;
}
private async build(
ctx: AudioContext,
track: MediaStreamTrack,
): Promise<void> {
private async buildGraph(track: MediaStreamTrack): Promise<Graph> {
const ctx = this.ctx!;
const base = this.config.assetBase;
const proc = PROCESSORS[this.config.model];
// Register worklet modules (idempotent) and load the wasm binary.
await ctx.audioWorklet.addModule(base + proc.script);
if (this.config.gate) await ctx.audioWorklet.addModule(base + GATE_SCRIPT);
const wasmBinary = await fetch(base + proc.wasm).then((r) => {
if (!r.ok) throw new Error(`denoise wasm ${r.status}`);
return r.arrayBuffer();
});
const wasmBinary = await loadWasm(base + proc.wasm);
const source = ctx.createMediaStreamSource(new MediaStream([track]));
const dest = ctx.createMediaStreamDestination();
const nodes: AudioWorkletNode[] = [];
let head: AudioNode = source;
if (this.config.gate) {
@@ -107,26 +163,26 @@ export class LotusDenoiseProcessor
});
head.connect(gate);
head = gate;
this.nodes.push(gate);
nodes.push(gate);
}
const ml = new AudioWorkletNode(ctx, proc.name, {
channelCount: 1,
numberOfInputs: 1,
numberOfOutputs: 1,
processorOptions: { maxChannels: 1, wasmBinary },
processorOptions: { maxChannels: 1, wasmBinary: wasmBinary.slice(0) },
});
head.connect(ml);
ml.connect(dest);
this.nodes.push(ml);
nodes.push(ml);
this.source = source;
this.processedTrack = dest.stream.getAudioTracks()[0];
logger.info(`[lotus] denoise processor active (${this.config.model})`);
return { source, nodes, track: dest.stream.getAudioTracks()[0] };
}
private teardownGraph(): void {
for (const node of this.nodes) {
private disposeGraph(graph: Graph | undefined): void {
if (!graph) return;
for (const node of graph.nodes) {
try {
node.port.postMessage("destroy");
} catch {
@@ -138,14 +194,11 @@ export class LotusDenoiseProcessor
/* ignore */
}
}
this.nodes = [];
try {
this.source?.disconnect();
graph.source.disconnect();
} catch {
/* ignore */
}
this.source = undefined;
this.processedTrack?.stop();
this.processedTrack = undefined;
graph.track.stop();
}
}