feat(calls): implement advanced multi-model ML noise suppression system
Implement a flexible, multi-model noise suppression pipeline for Element Call/LiveKit integration: - ML Engines: Added support for RNNoise, Speex, DTLN, and DeepFilterNet 3 models. - Pipeline Architecture: Implemented modular audio processing in lotus-denoise.js, supporting 'Series Suppression' (running browser-native NSNet2 before ML) and a hardware-style Noise Gate. - UI & UX Enhancements: - Settings UI: Added model comparison chart with CPU/Quality metadata. - Tuning: Added Live Microphone Meter for calibrating Noise Gate thresholds. - Reporting: Added LotusToast system to alert users when ML suppression fails or falls back to raw input. - Robustness & Quality: - Capture Fidelity: Removed forced 48kHz capture constraints to allow native-rate capture (solving static issues with high-end audio interfaces). - Performance: Added WASM SIMD detection with transparent fallback. - Capability Detection: Added browser feature detection to disable unsupported ML modes. - Build Integration: Updated Vite config to self-host all model WASM/tflite assets in /denoise/ directory.
This commit is contained in:
@@ -0,0 +1,68 @@
|
||||
/**
|
||||
* Detection utilities for Lotus ML noise suppression (RNNoise).
|
||||
*/
|
||||
|
||||
export type DenoiseModel = {
|
||||
id: string;
|
||||
name: string;
|
||||
description: string;
|
||||
cpuUsage: string;
|
||||
binarySize: string;
|
||||
transients: 'Poor' | 'Good' | 'Excellent';
|
||||
voiceQuality: 'Moderate' | 'High' | 'Very High';
|
||||
};
|
||||
|
||||
export const DENOISE_MODELS: DenoiseModel[] = [
|
||||
{
|
||||
id: 'rnnoise',
|
||||
name: 'RNNoise (Mozilla)',
|
||||
description: 'Lightweight hybrid model. Best for consistent noise like fans.',
|
||||
cpuUsage: '< 5%',
|
||||
binarySize: '< 1 MB',
|
||||
transients: 'Poor',
|
||||
voiceQuality: 'Moderate',
|
||||
},
|
||||
{
|
||||
id: 'dtln',
|
||||
name: 'DTLN (Balanced)',
|
||||
description: 'Deep learning model with a good balance of quality and CPU.',
|
||||
cpuUsage: '10-20%',
|
||||
binarySize: '3-4 MB',
|
||||
transients: 'Good',
|
||||
voiceQuality: 'High',
|
||||
},
|
||||
{
|
||||
id: 'deepfilternet',
|
||||
name: 'DeepFilterNet 3 (Pro)',
|
||||
description: 'State-of-the-art studio quality. Removes all background noise.',
|
||||
cpuUsage: '25-50%+',
|
||||
binarySize: '15-20 MB',
|
||||
transients: 'Excellent',
|
||||
voiceQuality: 'Very High',
|
||||
},
|
||||
];
|
||||
|
||||
export const isMLDenoiseSupported = (): boolean => {
|
||||
if (typeof window === 'undefined') return false;
|
||||
|
||||
// Requirements:
|
||||
// 1. AudioContext/webkitAudioContext (Web Audio API)
|
||||
// 2. AudioWorklet (Real-time processing in a background thread)
|
||||
// 3. getUserMedia (Microphone access)
|
||||
const hasAudioContext = !!(window.AudioContext || (window as any).webkitAudioContext);
|
||||
const hasAudioWorklet = hasAudioContext && !!AudioWorkletNode;
|
||||
const hasGetUserMedia = !!(navigator.mediaDevices && navigator.mediaDevices.getUserMedia);
|
||||
|
||||
return hasAudioWorklet && hasGetUserMedia;
|
||||
};
|
||||
|
||||
/**
|
||||
* EXACT requirements for ML Denoise (for UI display).
|
||||
*/
|
||||
export const ML_DENOISE_REQUIREMENTS = [
|
||||
'Modern browser with Web Audio API support',
|
||||
'AudioWorklet support (Chrome 66+, Firefox 76+, Safari 14.1+)',
|
||||
'Microphone access',
|
||||
'48kHz AudioContext capability',
|
||||
];
|
||||
|
||||
Reference in New Issue
Block a user