lotus(#1): ML denoise as a first-class audio TrackProcessor (fixes A7)
Implements RNNoise/Speex noise suppression as a LiveKit audio TrackProcessor attached to the local mic track, replacing the host's build-time getUserMedia monkeypatch. Because EC re-attaches the processor on every (re)publish (LocalTrackPublished), denoise now survives EC's mid-call reconnect — the root cause of A7 "mic dead after reconnect". Reuses the worklet/wasm assets already shipped under ./denoise/ (no new EC dependency); model/gate configured via lotusDenoise/lotusModel/lotusGate URL params. Additive: no-op unless lotusDenoise=ml. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
parent
33d0e98eb0
commit
39b57db3b2
@@ -0,0 +1,98 @@
|
||||
/*
|
||||
Copyright 2026 Lotus Guild
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import {
|
||||
type LocalAudioTrack,
|
||||
ParticipantEvent,
|
||||
type Room as LivekitRoom,
|
||||
Track,
|
||||
} from "livekit-client";
|
||||
import { logger } from "matrix-js-sdk/lib/logger";
|
||||
|
||||
import { type CallViewModel } from "../state/CallViewModel/CallViewModel";
|
||||
import { lotusFlag, lotusParam } from "./lotusWidget";
|
||||
import {
|
||||
type LotusDenoiseConfig,
|
||||
LotusDenoiseProcessor,
|
||||
} from "./lotusDenoiseProcessor";
|
||||
|
||||
/**
|
||||
* Apply Lotus ML noise suppression to the local mic track as a first-class
|
||||
* Element Call audio TrackProcessor (#1), replacing the host's build-time
|
||||
* `getUserMedia` monkeypatch.
|
||||
*
|
||||
* Opt-in via `lotusDenoise=ml` (+ `lotusModel`, `lotusGate`,
|
||||
* `lotusGateThreshold`, `lotusDenoiseBase`). Because the processor is attached
|
||||
* to the mic publication and re-attached on every (re)publish, denoise
|
||||
* survives EC's mid-call reconnect — fixing the A7 "mic dead after reconnect"
|
||||
* bug. Additive: no-op without the flag.
|
||||
*/
|
||||
export function startLotusDenoise(vm: CallViewModel): () => void {
|
||||
if (lotusParam("lotusDenoise") !== "ml") return () => undefined;
|
||||
|
||||
const config: LotusDenoiseConfig = {
|
||||
model: lotusParam("lotusModel") === "speex" ? "speex" : "rnnoise",
|
||||
assetBase: lotusParam("lotusDenoiseBase") || "./denoise/",
|
||||
gate: lotusFlag("lotusGate"),
|
||||
gateThreshold: Number(lotusParam("lotusGateThreshold")) || -50,
|
||||
};
|
||||
|
||||
const micOf = (room: LivekitRoom): LocalAudioTrack | undefined =>
|
||||
room.localParticipant.getTrackPublication(Track.Source.Microphone)
|
||||
?.track as LocalAudioTrack | undefined;
|
||||
|
||||
const apply = (room: LivekitRoom): void => {
|
||||
const mic = micOf(room);
|
||||
if (mic && !mic.getProcessor()) {
|
||||
void mic
|
||||
.setProcessor(new LotusDenoiseProcessor(config))
|
||||
.catch((e) => logger.warn("[lotus] denoise setProcessor failed", e));
|
||||
}
|
||||
};
|
||||
|
||||
const roomListeners = new Map<LivekitRoom, () => void>();
|
||||
let rooms: LivekitRoom[] = [];
|
||||
|
||||
const sub = vm.livekitRoomItems$.subscribe((items) => {
|
||||
const next = items.map((i) => i.livekitRoom);
|
||||
rooms = next;
|
||||
for (const [room, off] of roomListeners) {
|
||||
if (!next.includes(room)) {
|
||||
off();
|
||||
roomListeners.delete(room);
|
||||
}
|
||||
}
|
||||
for (const room of next) {
|
||||
if (!roomListeners.has(room)) {
|
||||
// Re-attach on every (re)publish — this is what makes denoise survive
|
||||
// reconnects (A7), unlike the old getUserMedia patch.
|
||||
const onPublished = (): void => apply(room);
|
||||
room.localParticipant.on(
|
||||
ParticipantEvent.LocalTrackPublished,
|
||||
onPublished,
|
||||
);
|
||||
roomListeners.set(room, () =>
|
||||
room.localParticipant.off(
|
||||
ParticipantEvent.LocalTrackPublished,
|
||||
onPublished,
|
||||
),
|
||||
);
|
||||
apply(room);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
return () => {
|
||||
sub.unsubscribe();
|
||||
for (const off of roomListeners.values()) off();
|
||||
roomListeners.clear();
|
||||
for (const room of rooms) {
|
||||
const mic = micOf(room);
|
||||
if (mic?.getProcessor()) void mic.stopProcessor();
|
||||
}
|
||||
};
|
||||
}
|
||||
Reference in New Issue
Block a user