feat(calls): implement advanced multi-model ML noise suppression system
Implement a flexible, multi-model noise suppression pipeline for Element Call/LiveKit integration: - ML Engines: Added support for RNNoise, Speex, DTLN, and DeepFilterNet 3 models. - Pipeline Architecture: Implemented modular audio processing in lotus-denoise.js, supporting 'Series Suppression' (running browser-native NSNet2 before ML) and a hardware-style Noise Gate. - UI & UX Enhancements: - Settings UI: Added model comparison chart with CPU/Quality metadata. - Tuning: Added Live Microphone Meter for calibrating Noise Gate thresholds. - Reporting: Added LotusToast system to alert users when ML suppression fails or falls back to raw input. - Robustness & Quality: - Capture Fidelity: Removed forced 48kHz capture constraints to allow native-rate capture (solving static issues with high-end audio interfaces). - Performance: Added WASM SIMD detection with transparent fallback. - Capability Detection: Added browser feature detection to disable unsupported ML modes. - Build Integration: Updated Vite config to self-host all model WASM/tflite assets in /denoise/ directory.
This commit is contained in:
@@ -382,6 +382,32 @@ function DeepLinkNavigator() {
|
||||
return null;
|
||||
}
|
||||
|
||||
function LotusDenoiseFeature() {
|
||||
const setToast = useSetAtom(toastQueueAtom);
|
||||
|
||||
useEffect(() => {
|
||||
const handleMessage = (event: MessageEvent) => {
|
||||
if (event.data?.type === 'lotus-denoise-status') {
|
||||
const { active, error } = event.data;
|
||||
if (!active) {
|
||||
setToast({
|
||||
id: `denoise-fail-${Date.now()}`,
|
||||
displayName: 'Audio Quality',
|
||||
body: `ML Noise Suppression failed: ${error || 'Unknown error'}. Falling back to raw mic.`,
|
||||
roomName: 'System',
|
||||
roomId: '',
|
||||
});
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
window.addEventListener('message', handleMessage);
|
||||
return () => window.removeEventListener('message', handleMessage);
|
||||
}, [setToast]);
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
export function ClientNonUIFeatures({ children }: ClientNonUIFeaturesProps) {
|
||||
return (
|
||||
<>
|
||||
@@ -391,6 +417,7 @@ export function ClientNonUIFeatures({ children }: ClientNonUIFeaturesProps) {
|
||||
<PresenceUpdater />
|
||||
<InviteNotifications />
|
||||
<MessageNotifications />
|
||||
<LotusDenoiseFeature />
|
||||
<DeepLinkNavigator />
|
||||
{children}
|
||||
</>
|
||||
|
||||
Reference in New Issue
Block a user