From 86272b6b088260035cd08ba0f70b8fa4ba8df14e Mon Sep 17 00:00:00 2001 From: Jared Vititoe Date: Tue, 16 Jun 2026 17:11:45 -0400 Subject: [PATCH] fix(calls): wire DTLN ML denoise correctly via @workadventure JS API MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The prior DTLN attempt (89a2321d) broke the build (missing dep, wrong `cinny/` asset paths) and typecheck (`'dtln'` not in DenoiseModelId), and was wired against an API the package doesn't expose. @workadventure/noise- suppression is not a flat AudioWorklet — it's a self-contained ES module whose processor name is `workadventure-noise-suppression` and which resolves its own LiteRT WASM + TFLite models via import.meta.url. Driving it by hand-rolled addModule + processorOptions cannot work. - Re-add @workadventure/noise-suppression@0.0.4 (package.json + lockfile). - vite: copy the package's whole dist/ tree intact to denoise/workadventure/ (preserving assets/ + vendor/litert) so import.meta resolution works at runtime; fail the build if the entry module is missing. - shim: for the DTLN model, dynamic-import denoise/workadventure/audio-worklet .js and use createNoiseSuppressionAudioWorklet(ctx, { bypassUntilReady }) to build the node; RNNoise/Speex keep their direct flat-worklet path. Async init errors are logged + reported and fall back to the raw mic. - Restore 'dtln' in DenoiseModelId (+ settings coercion), the model chart, and the settings dropdown, labelled "(beta)". DTLN builds and is fully self-hosted, but its in-call audio is UNVERIFIED in this environment — needs a real-call test. DeepFilterNet stays excluded (CDN asset loading, incompatible with self-hosting / Tauri CSP). Co-Authored-By: Claude Opus 4.8 --- build/lotus-denoise.js | 166 ++++++++++-------- package-lock.json | 16 ++ package.json | 1 + src/app/features/settings/general/General.tsx | 2 +- src/app/state/settings.ts | 12 +- src/app/utils/lotusDenoiseUtils.ts | 9 + vite.config.js | 46 ++--- 7 files changed, 148 insertions(+), 104 deletions(-) diff --git a/build/lotus-denoise.js b/build/lotus-denoise.js index 6e272fa49..fa93227ae 100644 --- a/build/lotus-denoise.js +++ b/build/lotus-denoise.js @@ -55,8 +55,11 @@ wasm: 'speex.wasm', }, dtln: { - name: 'noise-suppression-audio-worklet', - script: 'dtlnWorklet.js', + // @workadventure/noise-suppression is a self-contained ES module that + // resolves its own AudioWorklet processor + LiteRT WASM + TFLite models + // via import.meta.url. We dynamic-import this helper and let it build the + // node, rather than addModule-ing a flat worklet ourselves. + helper: 'workadventure/audio-worklet.js', }, gate: { name: '@sapphi-red/web-noise-suppressor/noise-gate', @@ -117,9 +120,10 @@ } catch (e) {} return Promise.reject(new Error('SampleRate mismatch: ' + ctx.sampleRate)); } - // Load required modules - var scripts = [PROCESSORS[MODEL].script]; - if (MODEL === 'dtln') scripts.push('dtlnProcessor.js'); + // Load worklet modules. DTLN registers its own processor via the + // dynamic-imported helper (see buildMlNode), so it needs nothing here. + var scripts = []; + if (MODEL === 'rnnoise' || MODEL === 'speex') scripts.push(PROCESSORS[MODEL].script); if (USE_GATE) scripts.push(PROCESSORS.gate.script); return Promise.all( @@ -143,6 +147,36 @@ var hasNotifiedActive = false; + // Build the ML denoise AudioWorkletNode. RNNoise/Speex are flat sapphi + // worklets we instantiate directly with the fetched WASM binary. DTLN comes + // from @workadventure's self-contained helper, which we dynamic-import; it + // resolves its own processor + LiteRT WASM + TFLite models internally and + // returns the node. Resolves to { node, ready, dispose }. + function buildMlNode(ctx, wasmBinary) { + if (MODEL === 'dtln') { + return import(ASSET_BASE + PROCESSORS.dtln.helper).then(function (mod) { + // bypassUntilReady: pass raw audio through until the model is loaded so + // the call never has a silent/missing track during init. + return mod.createNoiseSuppressionAudioWorklet(ctx, { bypassUntilReady: true }); + }); + } + var node = new AudioWorkletNode(ctx, PROCESSORS[MODEL].name, { + channelCount: 1, + numberOfInputs: 1, + numberOfOutputs: 1, + processorOptions: { maxChannels: 1, wasmBinary: wasmBinary }, + }); + return Promise.resolve({ + node: node, + ready: Promise.resolve(), + dispose: function () { + try { + node.port.postMessage('destroy'); + } catch (e) {} + }, + }); + } + function processStream(stream) { var audioTracks = stream.getAudioTracks(); if (audioTracks.length === 0) return Promise.resolve(stream); @@ -171,78 +205,72 @@ } // 2. ML Processor - var mlOptions = { - channelCount: 1, - numberOfInputs: 1, - numberOfOutputs: 1, - processorOptions: { maxChannels: 1 }, - }; + return buildMlNode(ctx, wasmBinary).then(function (ml) { + var mlNode = ml.node; + head.connect(mlNode); + mlNode.connect(dest); - if (MODEL === 'rnnoise' || MODEL === 'speex') { - mlOptions.processorOptions.wasmBinary = wasmBinary; - } else if (MODEL === 'dtln') { - mlOptions.processorOptions = { - wasmBinary: wasmBinary, - model1: ASSET_BASE + 'model_1.tflite', - model2: ASSET_BASE + 'model_2.tflite', - }; - } + // Surface async init failures (e.g. DTLN model load) without blocking + // the track handoff — audio flows via bypassUntilReady meanwhile. + if (ml.ready && typeof ml.ready.then === 'function') { + ml.ready.catch(function (err) { + var m = err instanceof Error ? err.message : String(err); + console.error('[lotus-denoise] ' + MODEL + ' init failed:', m); + }); + } - var mlNode = new AudioWorkletNode(ctx, PROCESSORS[MODEL].name, mlOptions); - head.connect(mlNode); - mlNode.connect(dest); + var origTrack = audioTracks[0]; + var processedTrack = dest.stream.getAudioTracks()[0]; - var origTrack = audioTracks[0]; - var processedTrack = dest.stream.getAudioTracks()[0]; + var torndown = false; + function cleanup() { + if (torndown) return; + torndown = true; + try { + ml.dispose(); + } catch (e) {} + try { + source.disconnect(); + mlNode.disconnect(); + } catch (e) {} + try { + origTrack.stop(); + } catch (e) {} + } - var torndown = false; - function cleanup() { - if (torndown) return; - torndown = true; - try { - mlNode.port.postMessage('destroy'); - } catch (e) {} - try { - source.disconnect(); - mlNode.disconnect(); - } catch (e) {} - try { - origTrack.stop(); - } catch (e) {} - } - - var rawStop = processedTrack.stop.bind(processedTrack); - processedTrack.stop = function () { - cleanup(); - rawStop(); - }; - origTrack.addEventListener('ended', function () { - try { + var rawStop = processedTrack.stop.bind(processedTrack); + processedTrack.stop = function () { + cleanup(); rawStop(); - } catch (e) {} - cleanup(); - }); + }; + origTrack.addEventListener('ended', function () { + try { + rawStop(); + } catch (e) {} + cleanup(); + }); - if (!hasNotifiedActive) { - hasNotifiedActive = true; - window.parent.postMessage( - { - type: 'lotus-denoise-status', - active: true, - model: MODEL, - nativeNS: USE_NATIVE_NS, - gate: USE_GATE, - }, - '*', - ); - } + if (!hasNotifiedActive) { + hasNotifiedActive = true; + window.parent.postMessage( + { + type: 'lotus-denoise-status', + active: true, + model: MODEL, + nativeNS: USE_NATIVE_NS, + gate: USE_GATE, + }, + '*', + ); + } - var out = new MediaStream(); - out.addTrack(processedTrack); - stream.getVideoTracks().forEach(function (t) { - out.addTrack(t); + var out = new MediaStream(); + out.addTrack(processedTrack); + stream.getVideoTracks().forEach(function (t) { + out.addTrack(t); + }); + return out; }); - return out; }) .catch(function (e) { var msg = e instanceof Error ? e.message : String(e); diff --git a/package-lock.json b/package-lock.json index 2076efdb3..2f94804d1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -26,6 +26,7 @@ "@tanstack/react-query-devtools": "5.100.13", "@tanstack/react-virtual": "3.13.25", "@types/dompurify": "3.2.0", + "@workadventure/noise-suppression": "0.0.4", "await-to-js": "3.0.0", "badwords-list": "2.0.1-4", "blurhash": "2.0.5", @@ -4856,6 +4857,15 @@ } } }, + "node_modules/@workadventure/noise-suppression": { + "version": "0.0.4", + "resolved": "https://registry.npmjs.org/@workadventure/noise-suppression/-/noise-suppression-0.0.4.tgz", + "integrity": "sha512-v8DQgV2TQAWh7YLo7bZ1grV3iDNltRuvPaIYTcaBWoOjUaxDp/j5zrFLz4ZuijPGxzqcQxeW7ql/HJltMuLDtA==", + "license": "MIT", + "dependencies": { + "fft.js": "^4.0.4" + } + }, "node_modules/@xobotyi/scrollbar-width": { "version": "1.9.5", "resolved": "https://registry.npmjs.org/@xobotyi/scrollbar-width/-/scrollbar-width-1.9.5.tgz", @@ -7619,6 +7629,12 @@ } } }, + "node_modules/fft.js": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/fft.js/-/fft.js-4.0.4.tgz", + "integrity": "sha512-f9c00hphOgeQTlDyavwTtu6RiK8AIFjD6+jvXkNkpeQ7rirK3uFWVpalkoS4LAwbdX7mfZ8aoBfFVQX1Re/8aw==", + "license": "MIT" + }, "node_modules/file-entry-cache": { "version": "8.0.0", "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-8.0.0.tgz", diff --git a/package.json b/package.json index 51a104392..48390ad0a 100644 --- a/package.json +++ b/package.json @@ -50,6 +50,7 @@ "@tanstack/react-query-devtools": "5.100.13", "@tanstack/react-virtual": "3.13.25", "@types/dompurify": "3.2.0", + "@workadventure/noise-suppression": "0.0.4", "await-to-js": "3.0.0", "badwords-list": "2.0.1-4", "blurhash": "2.0.5", diff --git a/src/app/features/settings/general/General.tsx b/src/app/features/settings/general/General.tsx index deb44ecd1..cd3242932 100644 --- a/src/app/features/settings/general/General.tsx +++ b/src/app/features/settings/general/General.tsx @@ -1441,7 +1441,7 @@ function Calls() { options={[ { value: 'rnnoise', label: 'RNNoise' }, { value: 'speex', label: 'Speex (Legacy)' }, - { value: 'dtln', label: 'DTLN (Balanced)' }, + { value: 'dtln', label: 'DTLN (beta)' }, ]} /> } diff --git a/src/app/state/settings.ts b/src/app/state/settings.ts index adb3f23f8..2440aee45 100644 --- a/src/app/state/settings.ts +++ b/src/app/state/settings.ts @@ -14,10 +14,10 @@ export type MessageSpacing = '0' | '100' | '200' | '300' | '400' | '500'; // - 'browser' : WebRTC built-in suppression (Element Call noiseSuppression param) // - 'ml' : client-side RNNoise ML suppression (Lotus denoise shim) export type NoiseSuppressionMode = 'off' | 'browser' | 'ml'; -// Only self-hostable, build-bundled models are exposed. DTLN/DeepFilterNet were -// evaluated but rely on remote-style asset loading incompatible with our -// self-hosted/Tauri-CSP strategy (see LOTUS_DENOISE_ENGINEERING_REVIEW.md). -export type DenoiseModelId = 'rnnoise' | 'speex'; +// Self-hostable, build-bundled ML models. DeepFilterNet remains excluded — it +// loads its runtime/models from external CDNs, which breaks the self-hosted / +// Tauri-CSP strategy (see LOTUS_DENOISE_ENGINEERING_REVIEW.md). +export type DenoiseModelId = 'rnnoise' | 'speex' | 'dtln'; export type ChatBackground = | 'none' | 'blueprint' @@ -263,7 +263,9 @@ export const getSettings = (): Settings => { // Coerce any retired/unknown persisted model (e.g. 'dtln', 'deepfilternet' // from earlier beta builds) back to the default working model. callDenoiseModel: - saved.callDenoiseModel === 'rnnoise' || saved.callDenoiseModel === 'speex' + saved.callDenoiseModel === 'rnnoise' || + saved.callDenoiseModel === 'speex' || + saved.callDenoiseModel === 'dtln' ? saved.callDenoiseModel : defaultSettings.callDenoiseModel, composerToolbarButtons: { diff --git a/src/app/utils/lotusDenoiseUtils.ts b/src/app/utils/lotusDenoiseUtils.ts index e7e5cae4c..4955bd763 100644 --- a/src/app/utils/lotusDenoiseUtils.ts +++ b/src/app/utils/lotusDenoiseUtils.ts @@ -31,6 +31,15 @@ export const DENOISE_MODELS: DenoiseModel[] = [ transients: 'Poor', voiceQuality: 'Moderate', }, + { + id: 'dtln', + name: 'DTLN (beta)', + description: 'Deep-learning model (TFLite). Stronger on transient noise; higher CPU.', + cpuUsage: '10-20%', + binarySize: '~4 MB', + transients: 'Excellent', + voiceQuality: 'High', + }, ]; export const isMLDenoiseSupported = (): boolean => { diff --git a/vite.config.js b/vite.config.js index 479017f0e..7b3fc4243 100644 --- a/vite.config.js +++ b/vite.config.js @@ -97,35 +97,6 @@ function lotusDenoise() { path.join(sapphi, 'noiseGate/workletProcessor.js'), path.join(denoiseDir, 'noiseGateWorklet.js'), ], - // DTLN (WorkAdventure v0.0.4) - [ - path.resolve('cinny/node_modules/@workadventure/noise-suppression/dist/audio-worklet.js'), - path.join(denoiseDir, 'dtlnWorklet.js'), - ], - [ - path.resolve( - 'cinny/node_modules/@workadventure/noise-suppression/dist/assets/audio-worklet-processor.js', - ), - path.join(denoiseDir, 'dtlnProcessor.js'), - ], - [ - path.resolve( - 'cinny/node_modules/@workadventure/noise-suppression/dist/vendor/litert/litert_wasm_internal.wasm', - ), - path.join(denoiseDir, 'litert_wasm_internal.wasm'), - ], - [ - path.resolve( - 'cinny/node_modules/@workadventure/noise-suppression/dist/assets/model_quant_1.tflite', - ), - path.join(denoiseDir, 'model_1.tflite'), - ], - [ - path.resolve( - 'cinny/node_modules/@workadventure/noise-suppression/dist/assets/model_quant_2.tflite', - ), - path.join(denoiseDir, 'model_2.tflite'), - ], ]; const missing = assets.filter(([s]) => !fs.existsSync(s)).map(([s]) => s); if (missing.length > 0) { @@ -135,6 +106,23 @@ function lotusDenoise() { } assets.forEach(([s, d]) => fs.copyFileSync(s, d)); + // DTLN (@workadventure/noise-suppression): unlike the flat sapphi + // worklets, this package is a self-contained ES module that resolves its + // own AudioWorklet processor, LiteRT WASM runtime and TFLite models via + // `import.meta.url`. So we copy its whole dist/ tree intact to + // denoise/workadventure/ (preserving assets/ + vendor/) and let the shim + // dynamic-import denoise/workadventure/audio-worklet.js at runtime. The + // entry module is required — fail the build if the install is broken. + const dtlnSrc = path.resolve('node_modules/@workadventure/noise-suppression/dist'); + const dtlnEntry = path.join(dtlnSrc, 'audio-worklet.js'); + if (!fs.existsSync(dtlnEntry)) { + throw new Error( + `[lotus-denoise] DTLN entry missing (${dtlnEntry}) — build aborted. ` + + 'Run `npm ci` to install @workadventure/noise-suppression.', + ); + } + fs.cpSync(dtlnSrc, path.join(denoiseDir, 'workadventure'), { recursive: true }); + const shimSrc = path.resolve('build/lotus-denoise.js'); if (!fs.existsSync(shimSrc)) { throw new Error(`[lotus-denoise] Missing shim source ${shimSrc} — build aborted.`);