feat(calls): implement advanced multi-model ML noise suppression system
Implement a flexible, multi-model noise suppression pipeline for Element Call/LiveKit integration: - ML Engines: Added support for RNNoise, Speex, DTLN, and DeepFilterNet 3 models. - Pipeline Architecture: Implemented modular audio processing in lotus-denoise.js, supporting 'Series Suppression' (running browser-native NSNet2 before ML) and a hardware-style Noise Gate. - UI & UX Enhancements: - Settings UI: Added model comparison chart with CPU/Quality metadata. - Tuning: Added Live Microphone Meter for calibrating Noise Gate thresholds. - Reporting: Added LotusToast system to alert users when ML suppression fails or falls back to raw input. - Robustness & Quality: - Capture Fidelity: Removed forced 48kHz capture constraints to allow native-rate capture (solving static issues with high-end audio interfaces). - Performance: Added WASM SIMD detection with transparent fallback. - Capability Detection: Added browser feature detection to disable unsupported ML modes. - Build Integration: Updated Vite config to self-host all model WASM/tflite assets in /denoise/ directory.
This commit is contained in:
@@ -14,6 +14,7 @@ export type MessageSpacing = '0' | '100' | '200' | '300' | '400' | '500';
|
||||
// - 'browser' : WebRTC built-in suppression (Element Call noiseSuppression param)
|
||||
// - 'ml' : client-side RNNoise ML suppression (Lotus denoise shim)
|
||||
export type NoiseSuppressionMode = 'off' | 'browser' | 'ml';
|
||||
export type DenoiseModelId = 'rnnoise' | 'speex' | 'dtln' | 'deepfilternet';
|
||||
export type ChatBackground =
|
||||
| 'none'
|
||||
| 'blueprint'
|
||||
@@ -115,6 +116,10 @@ export interface Settings {
|
||||
|
||||
cameraOnJoin: boolean;
|
||||
callNoiseSuppression: NoiseSuppressionMode;
|
||||
callDenoiseModel: DenoiseModelId;
|
||||
callDenoiseNativeNS: boolean;
|
||||
callDenoiseGate: boolean;
|
||||
callDenoiseGateThreshold: number;
|
||||
pttMode: boolean;
|
||||
pttKey: string;
|
||||
|
||||
@@ -205,6 +210,10 @@ const defaultSettings: Settings = {
|
||||
|
||||
cameraOnJoin: false,
|
||||
callNoiseSuppression: 'browser',
|
||||
callDenoiseModel: 'rnnoise',
|
||||
callDenoiseNativeNS: true,
|
||||
callDenoiseGate: false,
|
||||
callDenoiseGateThreshold: -45,
|
||||
pttMode: false,
|
||||
pttKey: 'Space',
|
||||
|
||||
|
||||
Reference in New Issue
Block a user