diff --git a/src/app/features/settings/general/DenoiseTester.tsx b/src/app/features/settings/general/DenoiseTester.tsx new file mode 100644 index 000000000..1f584b25b --- /dev/null +++ b/src/app/features/settings/general/DenoiseTester.tsx @@ -0,0 +1,372 @@ +import React, { useCallback, useEffect, useRef, useState } from 'react'; +import { Box, Button, Text } from 'folds'; +import { DenoiseModelId } from '../../../state/settings'; +import { + DENOISE_SAMPLE_RATE, + DenoiseNode, + buildGateNode, + buildModelNode, + readDb, +} from '../../../utils/denoisePipeline'; + +const MAX_RECORD_MS = 6000; + +const MIC_CONSTRAINTS = (nativeNS: boolean): MediaStreamConstraints => ({ + audio: { + noiseSuppression: nativeNS, + channelCount: 1, + echoCancellation: true, + autoGainControl: true, + }, +}); + +/** A -100..0 dBFS bar with optional threshold marker. */ +function DbMeter({ label, db, threshold }: { label: string; db: number; threshold?: number }) { + const pct = Math.max(0, Math.min(100, db + 100)); + const markerPct = threshold === undefined ? null : Math.max(0, Math.min(100, threshold + 100)); + return ( + + + {label} + {db <= -100 ? '−∞ dB' : `${Math.round(db)} dB`} + + + + {markerPct !== null && ( + + )} + + + ); +} + +type DenoiseTesterProps = { + model: DenoiseModelId; + useGate: boolean; + gateThreshold: number; + nativeNS: boolean; +}; + +export function DenoiseTester({ model, useGate, gateThreshold, nativeNS }: DenoiseTesterProps) { + // ── Live loopback monitor ──────────────────────────────────────────────── + const liveRef = useRef<{ + ctx: AudioContext; + stream: MediaStream; + model: DenoiseNode; + gate: AudioWorkletNode | null; + raf: number; + inAnalyser: AnalyserNode; + outAnalyser: AnalyserNode; + } | null>(null); + const [live, setLive] = useState(false); + const [inDb, setInDb] = useState(-100); + const [outDb, setOutDb] = useState(-100); + + const stopLive = useCallback(() => { + const s = liveRef.current; + liveRef.current = null; + if (s) { + cancelAnimationFrame(s.raf); + try { + s.gate?.disconnect(); + s.model.dispose(); + s.model.node.disconnect(); + } catch { + /* noop */ + } + s.stream.getTracks().forEach((t) => t.stop()); + s.ctx.close().catch(() => undefined); + } + setLive(false); + setInDb(-100); + setOutDb(-100); + }, []); + + const startLive = async () => { + try { + const ctx = new AudioContext({ sampleRate: DENOISE_SAMPLE_RATE }); + const stream = await navigator.mediaDevices.getUserMedia(MIC_CONSTRAINTS(nativeNS)); + const source = ctx.createMediaStreamSource(stream); + const inAnalyser = ctx.createAnalyser(); + inAnalyser.fftSize = 1024; + source.connect(inAnalyser); + + let head: AudioNode = source; + let gate: AudioWorkletNode | null = null; + if (useGate) { + gate = await buildGateNode(ctx, gateThreshold); + head.connect(gate); + head = gate; + } + const denoise = await buildModelNode(ctx, model); + head.connect(denoise.node); + + const outAnalyser = ctx.createAnalyser(); + outAnalyser.fftSize = 1024; + denoise.node.connect(outAnalyser); + denoise.node.connect(ctx.destination); // loopback to speakers + + const tick = () => { + setInDb(readDb(inAnalyser)); + setOutDb(readDb(outAnalyser)); + if (liveRef.current) liveRef.current.raf = requestAnimationFrame(tick); + }; + liveRef.current = { + ctx, + stream, + model: denoise, + gate, + raf: 0, + inAnalyser, + outAnalyser, + }; + setLive(true); + tick(); + } catch (e) { + console.error('[denoise-tester] live monitor failed', e); + stopLive(); + } + }; + + // ── Record & compare ───────────────────────────────────────────────────── + const recRef = useRef<{ + ctx: AudioContext; + stream: MediaStream; + recorder: MediaRecorder; + chunks: BlobPart[]; + raf: number; + analyser: AnalyserNode; + timer: number; + } | null>(null); + const clipRef = useRef(null); + const playRef = useRef<{ ctx: AudioContext; source: AudioBufferSourceNode } | null>(null); + const [recording, setRecording] = useState(false); + const [recDb, setRecDb] = useState(-100); + const [hasClip, setHasClip] = useState(false); + const [playing, setPlaying] = useState(null); + + const teardownRecorder = () => { + const r = recRef.current; + recRef.current = null; + if (r) { + cancelAnimationFrame(r.raf); + window.clearTimeout(r.timer); + r.stream.getTracks().forEach((t) => t.stop()); + r.ctx.close().catch(() => undefined); + } + }; + + const stopRecord = useCallback(() => { + const r = recRef.current; + if (r && r.recorder.state !== 'inactive') r.recorder.stop(); + setRecording(false); + }, []); + + const startRecord = async () => { + try { + const stream = await navigator.mediaDevices.getUserMedia(MIC_CONSTRAINTS(nativeNS)); + const ctx = new AudioContext({ sampleRate: DENOISE_SAMPLE_RATE }); + const source = ctx.createMediaStreamSource(stream); + const analyser = ctx.createAnalyser(); + analyser.fftSize = 1024; + source.connect(analyser); + + const recorder = new MediaRecorder(stream); + const chunks: BlobPart[] = []; + recorder.ondataavailable = (ev) => { + if (ev.data.size > 0) chunks.push(ev.data); + }; + recorder.onstop = async () => { + const blob = new Blob(chunks, { type: recorder.mimeType || 'audio/webm' }); + teardownRecorder(); + setRecDb(-100); + try { + const decodeCtx = new AudioContext({ sampleRate: DENOISE_SAMPLE_RATE }); + clipRef.current = await decodeCtx.decodeAudioData(await blob.arrayBuffer()); + decodeCtx.close().catch(() => undefined); + setHasClip(true); + } catch (e) { + console.error('[denoise-tester] decode failed', e); + } + }; + + const tick = () => { + setRecDb(readDb(analyser)); + if (recRef.current) recRef.current.raf = requestAnimationFrame(tick); + }; + const timer = window.setTimeout(stopRecord, MAX_RECORD_MS); + recRef.current = { ctx, stream, recorder, chunks, raf: 0, analyser, timer }; + recorder.start(); + setRecording(true); + tick(); + } catch (e) { + console.error('[denoise-tester] record failed', e); + teardownRecorder(); + setRecording(false); + } + }; + + const stopPlayback = useCallback(() => { + const p = playRef.current; + playRef.current = null; + if (p) { + try { + p.source.onended = null; + p.source.stop(); + } catch { + /* noop */ + } + p.ctx.close().catch(() => undefined); + } + setPlaying(null); + }, []); + + const play = async (label: string, playModel: DenoiseModelId | null) => { + stopPlayback(); + const clip = clipRef.current; + if (!clip) return; + try { + const ctx = new AudioContext({ sampleRate: DENOISE_SAMPLE_RATE }); + const source = ctx.createBufferSource(); + source.buffer = clip; + if (playModel) { + let head: AudioNode = source; + if (useGate) { + const gate = await buildGateNode(ctx, gateThreshold); + head.connect(gate); + head = gate; + } + const denoise = await buildModelNode(ctx, playModel); + head.connect(denoise.node); + denoise.node.connect(ctx.destination); + } else { + source.connect(ctx.destination); + } + source.onended = () => { + if (playRef.current?.ctx === ctx) stopPlayback(); + }; + playRef.current = { ctx, source }; + source.start(); + setPlaying(label); + } catch (e) { + console.error('[denoise-tester] playback failed', e); + stopPlayback(); + } + }; + + useEffect( + () => () => { + stopLive(); + stopPlayback(); + teardownRecorder(); + }, + [stopLive, stopPlayback], + ); + + const modelLabel = model === 'rnnoise' ? 'RNNoise' : model === 'speex' ? 'Speex' : 'DTLN'; + + return ( + + {/* Live monitor */} + + Live monitor + + Hear yourself through {modelLabel} + {useGate ? ' + gate' : ''} in real time. Use headphones to avoid feedback. Watch the In + meter while you speak and while silent — set the gate threshold (orange line) just above + your silent level. + + + + + {live && ( + + + + + )} + + + {/* Record & compare */} + + Record & compare + + Record up to {MAX_RECORD_MS / 1000}s of yourself with your usual background noise, then + play the same clip back raw vs through each model to A/B them. (Uses the gate when it's + enabled above.) + + + + {recording && Recording…} + + {recording && } + {hasClip && !recording && ( + + {( + [ + { label: 'Raw', model: null }, + { label: 'RNNoise', model: 'rnnoise' }, + { label: 'Speex', model: 'speex' }, + { label: 'DTLN', model: 'dtln' }, + ] as const + ).map((b) => ( + + ))} + + )} + + + ); +} diff --git a/src/app/features/settings/general/General.tsx b/src/app/features/settings/general/General.tsx index cd3242932..5d3482ded 100644 --- a/src/app/features/settings/general/General.tsx +++ b/src/app/features/settings/general/General.tsx @@ -75,6 +75,7 @@ import { SequenceCardStyle } from '../styles.css'; import { useTauriUpdater } from '../../../hooks/useTauriUpdater'; import { useDateFormatItems } from '../../../hooks/useDateFormat'; import { playCallJoinSound } from '../../../utils/callSounds'; +import { DenoiseTester } from './DenoiseTester'; type ThemeSelectorProps = { themeNames: Record; @@ -1203,91 +1204,6 @@ function useKeyBind(setter: (code: string) => void) { const keyLabel = (code: string) => code === 'Space' ? 'Space' : code.replace('Key', '').replace('Digit', ''); -function MicMeter() { - const [level, setLevel] = useState(0); - const [active, setActive] = useState(false); - const streamRef = useRef(null); - const ctxRef = useRef(null); - const rafRef = useRef(null); - - const stop = useCallback(() => { - if (rafRef.current !== null) cancelAnimationFrame(rafRef.current); - rafRef.current = null; - streamRef.current?.getTracks().forEach((t) => t.stop()); - streamRef.current = null; - ctxRef.current?.close(); - ctxRef.current = null; - setActive(false); - setLevel(0); - }, []); - - const start = async () => { - try { - const stream = await navigator.mediaDevices.getUserMedia({ audio: true }); - streamRef.current = stream; - const ctx = new AudioContext(); - ctxRef.current = ctx; - const source = ctx.createMediaStreamSource(stream); - const analyser = ctx.createAnalyser(); - analyser.fftSize = 256; - source.connect(analyser); - - const buffer = new Uint8Array(analyser.frequencyBinCount); - const update = () => { - analyser.getByteFrequencyData(buffer); - let sum = 0; - for (let i = 0; i < buffer.length; i += 1) sum += buffer[i]; - setLevel(sum / buffer.length); - rafRef.current = requestAnimationFrame(update); - }; - update(); - setActive(true); - } catch (e) { - // eslint-disable-next-line no-console - console.error('Mic test failed', e); - } - }; - - useEffect(() => () => stop(), [stop]); - - return ( - - - - - - - - - The green bar shows your live volume. Use this to tune the Gate Threshold. - - - ); -} - function Calls() { const [cameraOnJoin, setCameraOnJoin] = useSetting(settingsAtom, 'cameraOnJoin'); const [callNoiseSuppression, setCallNoiseSuppression] = useSetting( @@ -1480,11 +1396,28 @@ function Calls() { step="1" value={callDenoiseGateThreshold} onChange={(e) => setCallDenoiseGateThreshold(parseInt(e.target.value, 10))} - style={{ width: '100%', accentColor: 'var(--lt-accent-orange)' }} + style={{ width: '100%', accentColor: 'var(--accent-orange)' }} /> - )} + + + Test & calibrate + + Audition the selected model and tune the gate without joining a call. Changes to the + settings above apply the next time you start a monitor or play a clip. + + + )} diff --git a/src/app/utils/denoisePipeline.ts b/src/app/utils/denoisePipeline.ts new file mode 100644 index 000000000..d64b2cf76 --- /dev/null +++ b/src/app/utils/denoisePipeline.ts @@ -0,0 +1,116 @@ +/** + * Shared client-side denoise pipeline for the in-app model tester. + * + * The same RNNoise/Speex/DTLN worklets that the Element Call shim + * (build/lotus-denoise.js) injects are shipped under + * /public/element-call/denoise/. Here we load them into a normal main-app + * AudioContext so users can audition the models and calibrate the noise gate + * without joining a real call. The graph mirrors the shim: + * source -> [noise gate] -> model -> output + */ +import { DenoiseModelId } from '../state/settings'; + +// Mirror CallEmbed's widget-base resolution so assets resolve under any base. +const BASE = `${import.meta.env.BASE_URL.replace(/\/+$/, '')}/public/element-call/denoise/`; + +/** RNNoise/Speex/DTLN all assume mono 48 kHz, matching the call pipeline. */ +export const DENOISE_SAMPLE_RATE = 48000; + +export type DenoiseNode = { + node: AudioWorkletNode; + dispose: () => void; +}; + +const wasmCache: Record> = {}; +function fetchWasm(file: string): Promise { + if (!wasmCache[file]) { + wasmCache[file] = fetch(BASE + file).then((r) => { + if (!r.ok) throw new Error(`denoise asset ${file} unavailable (${r.status})`); + return r.arrayBuffer(); + }); + } + return wasmCache[file]; +} + +// addModule throws if the same module URL is added twice to one context. +const addedModules = new WeakMap>(); +async function addModuleOnce(ctx: BaseAudioContext, script: string): Promise { + let set = addedModules.get(ctx); + if (!set) { + set = new Set(); + addedModules.set(ctx, set); + } + if (set.has(script)) return; + await ctx.audioWorklet.addModule(BASE + script); + set.add(script); +} + +const SAPPHI: Record<'rnnoise' | 'speex', { proc: string; script: string; wasm: string }> = { + rnnoise: { + proc: '@sapphi-red/web-noise-suppressor/rnnoise', + script: 'rnnoiseWorklet.js', + wasm: 'rnnoise.wasm', + }, + speex: { + proc: '@sapphi-red/web-noise-suppressor/speex', + script: 'speexWorklet.js', + wasm: 'speex.wasm', + }, +}; + +/** Build the model denoise node. RNNoise/Speex are flat sapphi worklets; DTLN + * uses @workadventure's self-resolving ES-module helper. */ +export async function buildModelNode( + ctx: BaseAudioContext, + model: DenoiseModelId, +): Promise { + if (model === 'dtln') { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const mod: any = await import(/* @vite-ignore */ `${BASE}workadventure/audio-worklet.js`); + const handle = await mod.createNoiseSuppressionAudioWorklet(ctx, { bypassUntilReady: true }); + return { node: handle.node, dispose: () => handle.dispose() }; + } + const cfg = SAPPHI[model]; + const [, wasmBinary] = await Promise.all([addModuleOnce(ctx, cfg.script), fetchWasm(cfg.wasm)]); + const node = new AudioWorkletNode(ctx, cfg.proc, { + channelCount: 1, + numberOfInputs: 1, + numberOfOutputs: 1, + processorOptions: { maxChannels: 1, wasmBinary }, + }); + return { + node, + dispose: () => { + try { + node.port.postMessage('destroy'); + } catch { + /* noop */ + } + }, + }; +} + +export async function buildGateNode( + ctx: BaseAudioContext, + thresholdDb: number, +): Promise { + await addModuleOnce(ctx, 'noiseGateWorklet.js'); + return new AudioWorkletNode(ctx, '@sapphi-red/web-noise-suppressor/noise-gate', { + processorOptions: { + openThreshold: thresholdDb, + closeThreshold: thresholdDb - 5, + holdMs: 150, + maxChannels: 1, + }, + }); +} + +/** RMS level of an analyser as dBFS, clamped to [-100, 0]. */ +export function readDb(analyser: AnalyserNode): number { + const buf = new Float32Array(analyser.fftSize); + analyser.getFloatTimeDomainData(buf); + let sum = 0; + for (let i = 0; i < buf.length; i += 1) sum += buf[i] * buf[i]; + const rms = Math.sqrt(sum / buf.length); + return rms > 0 ? Math.max(-100, 20 * Math.log10(rms)) : -100; +}