diff --git a/build/lotus-denoise.js b/build/lotus-denoise.js index b82640067..7266ecb8e 100644 --- a/build/lotus-denoise.js +++ b/build/lotus-denoise.js @@ -54,6 +54,14 @@ script: 'speexWorklet.js', wasm: 'speex.wasm', }, + dtln: { + name: 'dtln-processor', + script: 'dtlnWorklet.js', + }, + deepfilternet: { + name: 'deepfilter-audio-processor', + script: 'dfn3Worklet.js', + }, gate: { name: '@sapphi-red/web-noise-suppressor/noise-gate', script: 'noiseGateWorklet.js', @@ -148,14 +156,31 @@ } // 2. ML Processor - var mlNode = new AudioWorkletNode(ctx, PROCESSORS[MODEL].name, { + var mlOptions = { channelCount: 1, - channelCountMode: 'explicit', numberOfInputs: 1, numberOfOutputs: 1, - outputChannelCount: [1], - processorOptions: { maxChannels: 1, wasmBinary: wasmBinary }, - }); + processorOptions: { maxChannels: 1 } + }; + + if (MODEL === 'rnnoise' || MODEL === 'speex') { + mlOptions.processorOptions.wasmBinary = wasmBinary; + } else if (MODEL === 'dtln') { + mlOptions.processorOptions = { + wasmUrl: ASSET_BASE + 'litert_wasm_internal.wasm', + model1Url: ASSET_BASE + 'model_1.tflite', + model2Url: ASSET_BASE + 'model_2.tflite', + processorUrl: ASSET_BASE + 'dtlnProcessor.js' + }; + } else if (MODEL === 'deepfilternet') { + mlOptions.processorOptions = { + wasmModule: wasmBinary, + modelBytes: new Uint8Array(wasmBinary), + suppressionLevel: 50 + }; + } + + var mlNode = new AudioWorkletNode(ctx, PROCESSORS[MODEL].name, mlOptions); head.connect(mlNode); mlNode.connect(dest); diff --git a/package-lock.json b/package-lock.json index 2076efdb3..970749d6a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -26,6 +26,7 @@ "@tanstack/react-query-devtools": "5.100.13", "@tanstack/react-virtual": "3.13.25", "@types/dompurify": "3.2.0", + "@workadventure/noise-suppression": "0.0.4", "await-to-js": "3.0.0", "badwords-list": "2.0.1-4", "blurhash": "2.0.5", @@ -34,6 +35,7 @@ "classnames": "2.5.1", "dateformat": "5.0.3", "dayjs": "1.11.20", + "deepfilternet3-noise-filter": "1.2.1", "domhandler": "6.0.1", "dompurify": "3.4.5", "emojibase": "17.0.0", @@ -4856,6 +4858,15 @@ } } }, + "node_modules/@workadventure/noise-suppression": { + "version": "0.0.4", + "resolved": "https://registry.npmjs.org/@workadventure/noise-suppression/-/noise-suppression-0.0.4.tgz", + "integrity": "sha512-v8DQgV2TQAWh7YLo7bZ1grV3iDNltRuvPaIYTcaBWoOjUaxDp/j5zrFLz4ZuijPGxzqcQxeW7ql/HJltMuLDtA==", + "license": "MIT", + "dependencies": { + "fft.js": "^4.0.4" + } + }, "node_modules/@xobotyi/scrollbar-width": { "version": "1.9.5", "resolved": "https://registry.npmjs.org/@xobotyi/scrollbar-width/-/scrollbar-width-1.9.5.tgz", @@ -6389,6 +6400,18 @@ "integrity": "sha512-Rn+RuwkmkDwCi2/oXOFS9Gsr5lJZu/yTGpK7wAaAIE75CC+LCGEZHpY6VQJa/RoJcrmaA/docWJZvYohlNkWPA==", "dev": true }, + "node_modules/deepfilternet3-noise-filter": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/deepfilternet3-noise-filter/-/deepfilternet3-noise-filter-1.2.1.tgz", + "integrity": "sha512-OAyrHTDlUHH+AhfpVNKYEOhVqb9cZpu0fdNThplA/tB/Ts4PF/UsI+abl2n1IbSxUkhiF0OqDejEhk1n42Oqpw==", + "license": "(Apache-2.0 OR MIT)", + "engines": { + "node": ">=18.0.0" + }, + "peerDependencies": { + "livekit-client": "^2.0.0" + } + }, "node_modules/deepmerge": { "version": "4.3.1", "resolved": "https://registry.npmjs.org/deepmerge/-/deepmerge-4.3.1.tgz", @@ -7619,6 +7642,12 @@ } } }, + "node_modules/fft.js": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/fft.js/-/fft.js-4.0.4.tgz", + "integrity": "sha512-f9c00hphOgeQTlDyavwTtu6RiK8AIFjD6+jvXkNkpeQ7rirK3uFWVpalkoS4LAwbdX7mfZ8aoBfFVQX1Re/8aw==", + "license": "MIT" + }, "node_modules/file-entry-cache": { "version": "8.0.0", "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-8.0.0.tgz", diff --git a/package.json b/package.json index 51a104392..88cf54b84 100644 --- a/package.json +++ b/package.json @@ -45,6 +45,8 @@ "@giphy/js-util": "5.2.0", "@giphy/react-components": "10.1.2", "@sapphi-red/web-noise-suppressor": "0.3.5", + "@workadventure/noise-suppression": "0.0.4", + "deepfilternet3-noise-filter": "1.2.1", "@sentry/react": "10.53.1", "@tanstack/react-query": "5.100.13", "@tanstack/react-query-devtools": "5.100.13", diff --git a/vite.config.js b/vite.config.js index e1395dba2..2ed4e26c0 100644 --- a/vite.config.js +++ b/vite.config.js @@ -96,6 +96,27 @@ function lotusDenoise() { path.join(sapphi, 'noiseGate/workletProcessor.js'), path.join(denoiseDir, 'noiseGateWorklet.js'), ], + // DTLN (WorkAdventure v0.0.4) + [ + path.resolve('node_modules/@workadventure/noise-suppression/dist/audio-worklet.js'), + path.join(denoiseDir, 'dtlnWorklet.js'), + ], + [ + path.resolve('node_modules/@workadventure/noise-suppression/dist/assets/audio-worklet-processor.js'), + path.join(denoiseDir, 'dtlnProcessor.js'), + ], + [ + path.resolve('node_modules/@workadventure/noise-suppression/dist/vendor/litert/litert_wasm_internal.wasm'), + path.join(denoiseDir, 'litert_wasm_internal.wasm'), + ], + [ + path.resolve('node_modules/@workadventure/noise-suppression/dist/assets/model_quant_1.tflite'), + path.join(denoiseDir, 'model_1.tflite'), + ], + [ + path.resolve('node_modules/@workadventure/noise-suppression/dist/assets/model_quant_2.tflite'), + path.join(denoiseDir, 'model_2.tflite'), + ], ]; assets.forEach(([s, d]) => { if (fs.existsSync(s)) {