feat(calls): implement advanced multi-model ML noise suppression system
Implement a flexible, multi-model noise suppression pipeline for Element Call/LiveKit integration: - ML Engines: Added support for RNNoise, Speex, DTLN, and DeepFilterNet 3 models. - Pipeline Architecture: Implemented modular audio processing in lotus-denoise.js, supporting 'Series Suppression' (running browser-native NSNet2 before ML) and a hardware-style Noise Gate. - UI & UX Enhancements: - Settings UI: Added model comparison chart with CPU/Quality metadata. - Tuning: Added Live Microphone Meter for calibrating Noise Gate thresholds. - Reporting: Added LotusToast system to alert users when ML suppression fails or falls back to raw input. - Robustness & Quality: - Capture Fidelity: Removed forced 48kHz capture constraints to allow native-rate capture (solving static issues with high-end audio interfaces). - Performance: Added WASM SIMD detection with transparent fallback. - Capability Detection: Added browser feature detection to disable unsupported ML modes. - Build Integration: Updated Vite config to self-host all model WASM/tflite assets in /denoise/ directory.
This commit is contained in:
@@ -46,6 +46,10 @@ export const createCallEmbed = (
|
||||
container: HTMLElement,
|
||||
pref?: CallPreferences,
|
||||
denoiseMode: NoiseSuppressionMode = 'browser',
|
||||
denoiseModel: string = 'rnnoise',
|
||||
denoiseNativeNS: boolean = true,
|
||||
denoiseGate: boolean = false,
|
||||
denoiseGateThreshold: number = -45,
|
||||
forceAudioOff = false,
|
||||
): CallEmbed => {
|
||||
const rtcSession = mx.matrixRTC.getRoomSession(room);
|
||||
@@ -60,6 +64,10 @@ export const createCallEmbed = (
|
||||
intent,
|
||||
themeKind,
|
||||
denoiseMode,
|
||||
denoiseModel,
|
||||
denoiseNativeNS,
|
||||
denoiseGate,
|
||||
denoiseGateThreshold,
|
||||
initialAudio,
|
||||
initialVideo,
|
||||
);
|
||||
@@ -77,6 +85,10 @@ export const useCallStart = (dm = false) => {
|
||||
const setCallEmbed = useSetAtom(callEmbedAtom);
|
||||
const callEmbedRef = useCallEmbedRef();
|
||||
const [callNoiseSuppression] = useSetting(settingsAtom, 'callNoiseSuppression');
|
||||
const [callDenoiseModel] = useSetting(settingsAtom, 'callDenoiseModel');
|
||||
const [callDenoiseNativeNS] = useSetting(settingsAtom, 'callDenoiseNativeNS');
|
||||
const [callDenoiseGate] = useSetting(settingsAtom, 'callDenoiseGate');
|
||||
const [callDenoiseGateThreshold] = useSetting(settingsAtom, 'callDenoiseGateThreshold');
|
||||
const [pttMode] = useSetting(settingsAtom, 'pttMode');
|
||||
|
||||
const startCall = useCallback(
|
||||
@@ -97,12 +109,28 @@ export const useCallStart = (dm = false) => {
|
||||
container,
|
||||
pref,
|
||||
callNoiseSuppression ?? 'browser',
|
||||
callDenoiseModel ?? 'rnnoise',
|
||||
callDenoiseNativeNS ?? true,
|
||||
callDenoiseGate ?? false,
|
||||
callDenoiseGateThreshold ?? -45,
|
||||
!!pttMode,
|
||||
);
|
||||
|
||||
setCallEmbed(callEmbed);
|
||||
},
|
||||
[mx, dm, theme, setCallEmbed, callEmbedRef, callNoiseSuppression, pttMode],
|
||||
[
|
||||
mx,
|
||||
dm,
|
||||
theme,
|
||||
setCallEmbed,
|
||||
callEmbedRef,
|
||||
callNoiseSuppression,
|
||||
callDenoiseModel,
|
||||
callDenoiseNativeNS,
|
||||
callDenoiseGate,
|
||||
callDenoiseGateThreshold,
|
||||
pttMode,
|
||||
],
|
||||
);
|
||||
|
||||
return startCall;
|
||||
|
||||
Reference in New Issue
Block a user