Files
cinny/src/app/hooks/useCallEmbed.ts
T
jared 5d5f5f4516
CI / Build & Quality Checks (push) Failing after 4m49s
Trigger Desktop Build / trigger (push) Successful in 11s
feat(calls): implement advanced multi-model ML noise suppression system
Implement a flexible, multi-model noise suppression pipeline for Element Call/LiveKit integration:

- ML Engines: Added support for RNNoise, Speex, DTLN, and DeepFilterNet 3 models.
- Pipeline Architecture: Implemented modular audio processing in lotus-denoise.js, supporting 'Series Suppression' (running browser-native NSNet2 before ML) and a hardware-style Noise Gate.
- UI & UX Enhancements:
  - Settings UI: Added model comparison chart with CPU/Quality metadata.
  - Tuning: Added Live Microphone Meter for calibrating Noise Gate thresholds.
  - Reporting: Added LotusToast system to alert users when ML suppression fails or falls back to raw input.
- Robustness & Quality:
  - Capture Fidelity: Removed forced 48kHz capture constraints to allow native-rate capture (solving static issues with high-end audio interfaces).
  - Performance: Added WASM SIMD detection with transparent fallback.
  - Capability Detection: Added browser feature detection to disable unsupported ML modes.
- Build Integration: Updated Vite config to self-host all model WASM/tflite assets in /denoise/ directory.
2026-06-16 00:50:12 -04:00

207 lines
6.3 KiB
TypeScript

import { createContext, RefObject, useCallback, useContext, useEffect, useState } from 'react';
import { MatrixClient, Room } from 'matrix-js-sdk';
import { useSetAtom } from 'jotai';
import {
CallEmbed,
ElementCallThemeKind,
ElementWidgetActions,
useClientWidgetApiEvent,
} from '../plugins/call';
import { useMatrixClient } from './useMatrixClient';
import { ThemeKind, useTheme } from './useTheme';
import { callEmbedAtom } from '../state/callEmbed';
import { useResizeObserver } from './useResizeObserver';
import { CallControlState } from '../plugins/call/CallControlState';
import { useCallMembersChange, useCallSession } from './useCall';
import { CallPreferences } from '../state/callPreferences';
import { useSetting } from '../state/hooks/settings';
import { NoiseSuppressionMode, settingsAtom } from '../state/settings';
import { unlockCallSounds } from '../utils/callSounds';
const CallEmbedContext = createContext<CallEmbed | undefined>(undefined);
export const CallEmbedContextProvider = CallEmbedContext.Provider;
export const useCallEmbed = (): CallEmbed | undefined => {
const callEmbed = useContext(CallEmbedContext);
return callEmbed;
};
const CallEmbedRefContext = createContext<RefObject<HTMLDivElement> | undefined>(undefined);
export const CallEmbedRefContextProvider = CallEmbedRefContext.Provider;
export const useCallEmbedRef = (): RefObject<HTMLDivElement> => {
const ref = useContext(CallEmbedRefContext);
if (!ref) {
throw new Error('CallEmbedRef is not provided!');
}
return ref;
};
export const createCallEmbed = (
mx: MatrixClient,
room: Room,
dm: boolean,
themeKind: ElementCallThemeKind,
container: HTMLElement,
pref?: CallPreferences,
denoiseMode: NoiseSuppressionMode = 'browser',
denoiseModel: string = 'rnnoise',
denoiseNativeNS: boolean = true,
denoiseGate: boolean = false,
denoiseGateThreshold: number = -45,
forceAudioOff = false,
): CallEmbed => {
const rtcSession = mx.matrixRTC.getRoomSession(room);
const ongoing = rtcSession.memberships.length > 0;
const intent = CallEmbed.getIntent(dm, ongoing, pref?.video);
const initialAudio = forceAudioOff ? false : (pref?.microphone ?? true);
const initialVideo = pref?.video ?? false;
const widget = CallEmbed.getWidget(
mx,
room,
intent,
themeKind,
denoiseMode,
denoiseModel,
denoiseNativeNS,
denoiseGate,
denoiseGateThreshold,
initialAudio,
initialVideo,
);
const controlState =
pref && new CallControlState(forceAudioOff ? false : pref.microphone, pref.video, pref.sound);
const embed = new CallEmbed(mx, room, widget, container, controlState, themeKind);
return embed;
};
export const useCallStart = (dm = false) => {
const mx = useMatrixClient();
const theme = useTheme();
const setCallEmbed = useSetAtom(callEmbedAtom);
const callEmbedRef = useCallEmbedRef();
const [callNoiseSuppression] = useSetting(settingsAtom, 'callNoiseSuppression');
const [callDenoiseModel] = useSetting(settingsAtom, 'callDenoiseModel');
const [callDenoiseNativeNS] = useSetting(settingsAtom, 'callDenoiseNativeNS');
const [callDenoiseGate] = useSetting(settingsAtom, 'callDenoiseGate');
const [callDenoiseGateThreshold] = useSetting(settingsAtom, 'callDenoiseGateThreshold');
const [pttMode] = useSetting(settingsAtom, 'pttMode');
const startCall = useCallback(
(room: Room, pref?: CallPreferences) => {
const container = callEmbedRef.current;
if (!container) {
throw new Error('Failed to start call, No embed container element found!');
}
// startCall is always invoked from a click/tap handler — unlock the Web
// Audio context now (within the gesture) so join/leave sounds that fire
// later, without any gesture, are audible to everyone in the call.
unlockCallSounds();
const callEmbed = createCallEmbed(
mx,
room,
dm,
theme.kind,
container,
pref,
callNoiseSuppression ?? 'browser',
callDenoiseModel ?? 'rnnoise',
callDenoiseNativeNS ?? true,
callDenoiseGate ?? false,
callDenoiseGateThreshold ?? -45,
!!pttMode,
);
setCallEmbed(callEmbed);
},
[
mx,
dm,
theme,
setCallEmbed,
callEmbedRef,
callNoiseSuppression,
callDenoiseModel,
callDenoiseNativeNS,
callDenoiseGate,
callDenoiseGateThreshold,
pttMode,
],
);
return startCall;
};
export const useCallJoined = (embed?: CallEmbed): boolean => {
const [joined, setJoined] = useState(embed?.joined ?? false);
useClientWidgetApiEvent(
embed?.call,
ElementWidgetActions.JoinCall,
useCallback(() => {
setJoined(true);
}, []),
);
useEffect(() => {
if (!embed) {
setJoined(false);
}
}, [embed]);
return joined;
};
export const useCallHangupEvent = (embed: CallEmbed, callback: () => void) => {
useClientWidgetApiEvent(embed.call, ElementWidgetActions.HangupCall, callback);
useClientWidgetApiEvent(embed.call, ElementWidgetActions.Close, callback);
};
export const useCallMemberSoundSync = (embed: CallEmbed) => {
const callSession = useCallSession(embed.room);
useCallMembersChange(
callSession,
useCallback(() => embed.control.applySound(), [embed]),
);
};
export const useCallThemeSync = (embed: CallEmbed) => {
const theme = useTheme();
useEffect(() => {
const name: ElementCallThemeKind = theme.kind === ThemeKind.Dark ? 'dark' : 'light';
embed.setTheme(name);
}, [theme.kind, embed]);
};
export const useCallEmbedPlacementSync = (containerViewRef: RefObject<HTMLDivElement>): void => {
const callEmbedRef = useCallEmbedRef();
const syncCallEmbedPlacement = useCallback(() => {
const embedEl = callEmbedRef.current;
const container = containerViewRef.current;
if (!embedEl || !container) return;
const rect = container.getBoundingClientRect();
embedEl.style.top = `${rect.top}px`;
embedEl.style.left = `${rect.left}px`;
embedEl.style.width = `${rect.width}px`;
embedEl.style.height = `${rect.height}px`;
}, [callEmbedRef, containerViewRef]);
// Sync once on mount so the embed is positioned immediately (deps are stable refs)
useEffect(() => {
syncCallEmbedPlacement();
}, [syncCallEmbedPlacement]);
useResizeObserver(
syncCallEmbedPlacement,
useCallback(() => containerViewRef.current, [containerViewRef]),
);
};