lotus(#1): fix restart-silence (A7), 48kHz ctx; protocol + CI hardening
Denoise deep-review (CRITICAL): restart() read opts.audioContext, which LiveKit does NOT pass on restart — so reconnect (the A7 scenario) and mic device-switch threw after stopping the old track, leaving the mic SILENT (A7 reintroduced). Fix: - Processor owns a dedicated 48kHz AudioContext (sapphi worklets require 48kHz; H1), reused across restart, closed on destroy. - restart() never throws and never leaves a stopped track on the sender: builds the new graph first, then disposes the old; on failure degrades to RAW mic audio rather than silence. - Cache wasm per URL (no re-fetch each reconnect); gate threshold default -45 and accept an explicit 0 (M2); document the cross-repo asset contract. Protocol audit: - Non-silent warning when an unsupported denoise model (dtln/deepfilternet) is requested instead of silent rnnoise fallback (F3). - Correct the call_state enum comment (immediate error-reply, not 10s) (F2). Build/CI audit: - Stamp VITE_APP_VERSION in CI; document the vX.Y.Z-lotus.N version scheme. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
parent
b8543c3fe1
commit
29592fbb18
@@ -25,6 +25,14 @@ export interface LotusDenoiseConfig {
|
||||
// Flat sapphi worklets: each registers a processor under these names when its
|
||||
// script module is added. Same assets the Lotus host already ships under
|
||||
// public/element-call/denoise/, so no new EC dependency is needed.
|
||||
//
|
||||
// ⚠️ CONTRACT: this is an undeclared, cross-repo asset dependency. The embedded
|
||||
// fork build does NOT bundle these — they are copied in by cinny's
|
||||
// vite.config.js `lotusDenoise()` plugin from `@sapphi-red/web-noise-suppressor`.
|
||||
// The worklet/wasm version must match what this processor expects (sapphi
|
||||
// rnnoise/speex, 48kHz). If the fork ever wants to own them, add
|
||||
// `@sapphi-red/web-noise-suppressor` as a dep + a copy step here. Until then,
|
||||
// an integration smoke-check should assert GET .../denoise/rnnoise.wasm == 200.
|
||||
const PROCESSORS: Record<
|
||||
LotusDenoiseModel,
|
||||
{ name: string; script: string; wasm: string }
|
||||
@@ -43,15 +51,46 @@ const PROCESSORS: Record<
|
||||
const GATE_NAME = "@sapphi-red/web-noise-suppressor/noiseGate";
|
||||
const GATE_SCRIPT = "noiseGateWorklet.js";
|
||||
|
||||
// The sapphi rnnoise/speex worklets are 48kHz-fullband and don't resample, so
|
||||
// the graph MUST run at 48kHz regardless of the hardware default.
|
||||
const SAMPLE_RATE = 48_000;
|
||||
|
||||
// Cache fetched wasm per URL so a reconnect/device-switch (which rebuilds the
|
||||
// graph) doesn't re-download it.
|
||||
const wasmCache = new Map<string, Promise<ArrayBuffer>>();
|
||||
function loadWasm(url: string): Promise<ArrayBuffer> {
|
||||
let p = wasmCache.get(url);
|
||||
if (!p) {
|
||||
p = fetch(url).then((r) => {
|
||||
if (!r.ok) throw new Error(`denoise wasm ${r.status}`);
|
||||
return r.arrayBuffer();
|
||||
});
|
||||
wasmCache.set(url, p);
|
||||
}
|
||||
return p;
|
||||
}
|
||||
|
||||
interface Graph {
|
||||
source: MediaStreamAudioSourceNode;
|
||||
nodes: AudioWorkletNode[];
|
||||
track: MediaStreamTrack;
|
||||
}
|
||||
|
||||
/**
|
||||
* A LiveKit audio TrackProcessor that runs Lotus ML noise suppression
|
||||
* (RNNoise/Speex) on the local microphone track, as a first-class stage in
|
||||
* Element Call's publish pipeline.
|
||||
*
|
||||
* This replaces the host's `getUserMedia` monkeypatch: because it's a real
|
||||
* LiveKit processor, EC re-applies it on every (re)publish, so denoise
|
||||
* survives EC's own mid-call reconnect — the root cause of the A7
|
||||
* "mic dead after reconnect" bug.
|
||||
* LiveKit processor, EC re-applies it on every (re)publish/restart, so denoise
|
||||
* survives EC's own mid-call reconnect — the root cause of the A7 "mic dead
|
||||
* after reconnect" bug.
|
||||
*
|
||||
* It owns a dedicated 48kHz AudioContext (the worklets require it, and LiveKit
|
||||
* does NOT pass an audioContext to restart()), reused across restarts and
|
||||
* closed on destroy. restart() never throws and never leaves a stopped track
|
||||
* on the sender: on failure it degrades to the raw (un-denoised) track rather
|
||||
* than silence.
|
||||
*/
|
||||
export class LotusDenoiseProcessor
|
||||
implements TrackProcessor<Track.Kind.Audio, AudioProcessorOptions>
|
||||
@@ -59,41 +98,58 @@ export class LotusDenoiseProcessor
|
||||
public readonly name = "lotus-denoise";
|
||||
public processedTrack?: MediaStreamTrack;
|
||||
|
||||
private source?: MediaStreamAudioSourceNode;
|
||||
private nodes: AudioWorkletNode[] = [];
|
||||
private ctx?: AudioContext;
|
||||
private graph?: Graph;
|
||||
|
||||
public constructor(private readonly config: LotusDenoiseConfig) {}
|
||||
|
||||
public async init(opts: AudioProcessorOptions): Promise<void> {
|
||||
await this.build(opts.audioContext, opts.track);
|
||||
public async init(_opts: AudioProcessorOptions): Promise<void> {
|
||||
// Own context at the required rate — do NOT use opts.audioContext (it's the
|
||||
// room's hardware-rate context, and is undefined on restart()).
|
||||
this.ctx = new AudioContext({ sampleRate: SAMPLE_RATE });
|
||||
const graph = await this.buildGraph(_opts.track);
|
||||
this.graph = graph;
|
||||
this.processedTrack = graph.track;
|
||||
}
|
||||
|
||||
public async restart(opts: AudioProcessorOptions): Promise<void> {
|
||||
this.teardownGraph();
|
||||
await this.build(opts.audioContext, opts.track);
|
||||
if (!this.ctx || this.ctx.state === "closed")
|
||||
this.ctx = new AudioContext({ sampleRate: SAMPLE_RATE });
|
||||
try {
|
||||
const next = await this.buildGraph(opts.track);
|
||||
this.disposeGraph(this.graph);
|
||||
this.graph = next;
|
||||
this.processedTrack = next.track;
|
||||
} catch (e) {
|
||||
// Never go silent on the A7/device-switch path: fall back to raw audio.
|
||||
logger.warn("[lotus] denoise restart failed; using raw mic", e);
|
||||
this.disposeGraph(this.graph);
|
||||
this.graph = undefined;
|
||||
this.processedTrack = opts.track;
|
||||
}
|
||||
}
|
||||
|
||||
public async destroy(): Promise<void> {
|
||||
this.teardownGraph();
|
||||
this.disposeGraph(this.graph);
|
||||
this.graph = undefined;
|
||||
this.processedTrack = undefined;
|
||||
if (this.ctx && this.ctx.state !== "closed")
|
||||
await this.ctx.close().catch(() => undefined);
|
||||
this.ctx = undefined;
|
||||
}
|
||||
|
||||
private async build(
|
||||
ctx: AudioContext,
|
||||
track: MediaStreamTrack,
|
||||
): Promise<void> {
|
||||
private async buildGraph(track: MediaStreamTrack): Promise<Graph> {
|
||||
const ctx = this.ctx!;
|
||||
const base = this.config.assetBase;
|
||||
const proc = PROCESSORS[this.config.model];
|
||||
|
||||
// Register worklet modules (idempotent) and load the wasm binary.
|
||||
await ctx.audioWorklet.addModule(base + proc.script);
|
||||
if (this.config.gate) await ctx.audioWorklet.addModule(base + GATE_SCRIPT);
|
||||
const wasmBinary = await fetch(base + proc.wasm).then((r) => {
|
||||
if (!r.ok) throw new Error(`denoise wasm ${r.status}`);
|
||||
return r.arrayBuffer();
|
||||
});
|
||||
const wasmBinary = await loadWasm(base + proc.wasm);
|
||||
|
||||
const source = ctx.createMediaStreamSource(new MediaStream([track]));
|
||||
const dest = ctx.createMediaStreamDestination();
|
||||
const nodes: AudioWorkletNode[] = [];
|
||||
let head: AudioNode = source;
|
||||
|
||||
if (this.config.gate) {
|
||||
@@ -107,26 +163,26 @@ export class LotusDenoiseProcessor
|
||||
});
|
||||
head.connect(gate);
|
||||
head = gate;
|
||||
this.nodes.push(gate);
|
||||
nodes.push(gate);
|
||||
}
|
||||
|
||||
const ml = new AudioWorkletNode(ctx, proc.name, {
|
||||
channelCount: 1,
|
||||
numberOfInputs: 1,
|
||||
numberOfOutputs: 1,
|
||||
processorOptions: { maxChannels: 1, wasmBinary },
|
||||
processorOptions: { maxChannels: 1, wasmBinary: wasmBinary.slice(0) },
|
||||
});
|
||||
head.connect(ml);
|
||||
ml.connect(dest);
|
||||
this.nodes.push(ml);
|
||||
nodes.push(ml);
|
||||
|
||||
this.source = source;
|
||||
this.processedTrack = dest.stream.getAudioTracks()[0];
|
||||
logger.info(`[lotus] denoise processor active (${this.config.model})`);
|
||||
return { source, nodes, track: dest.stream.getAudioTracks()[0] };
|
||||
}
|
||||
|
||||
private teardownGraph(): void {
|
||||
for (const node of this.nodes) {
|
||||
private disposeGraph(graph: Graph | undefined): void {
|
||||
if (!graph) return;
|
||||
for (const node of graph.nodes) {
|
||||
try {
|
||||
node.port.postMessage("destroy");
|
||||
} catch {
|
||||
@@ -138,14 +194,11 @@ export class LotusDenoiseProcessor
|
||||
/* ignore */
|
||||
}
|
||||
}
|
||||
this.nodes = [];
|
||||
try {
|
||||
this.source?.disconnect();
|
||||
graph.source.disconnect();
|
||||
} catch {
|
||||
/* ignore */
|
||||
}
|
||||
this.source = undefined;
|
||||
this.processedTrack?.stop();
|
||||
this.processedTrack = undefined;
|
||||
graph.track.stop();
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user