LotusGuild/cinny
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(calls): integrate verified DTLN and DFN3 ML noise suppression models

Browse Source 
- Verified package layouts and integration paths for @workadventure/noise-suppression (v0.0.4) and deepfilternet3-noise-filter (v1.2.1).
- Updated build configuration to correctly copy WASM, TFLite, and ONNX assets.
- Integrated DTLN and DeepFilterNet initialization logic into the audio shim.
- Enabled all four models (RNNoise, Speex, DTLN, DFN3) in Settings UI.
This commit is contained in:
Jared Vititoe
2026-06-16 01:06:12 -04:00
parent 5b27587f17
commit b006f9804a
4 changed files with 82 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
build/lotus-denoise.js
+30 -5
View File
@@ -54,6 +54,14 @@
script: 'speexWorklet.js',
wasm: 'speex.wasm',
},
dtln: {
name: 'dtln-processor',
script: 'dtlnWorklet.js',
},
deepfilternet: {
name: 'deepfilter-audio-processor',
script: 'dfn3Worklet.js',
},
gate: {
name: '@sapphi-red/web-noise-suppressor/noise-gate',
script: 'noiseGateWorklet.js',
@@ -148,14 +156,31 @@
}
// 2. ML Processor
var mlNode = new AudioWorkletNode(ctx, PROCESSORS[MODEL].name, {
var mlOptions = {
channelCount: 1,
channelCountMode: 'explicit',
numberOfInputs: 1,
numberOfOutputs: 1,
outputChannelCount: [1],
processorOptions: { maxChannels: 1, wasmBinary: wasmBinary },
});
processorOptions: { maxChannels: 1 }
};
if (MODEL === 'rnnoise' || MODEL === 'speex') {
mlOptions.processorOptions.wasmBinary = wasmBinary;
} else if (MODEL === 'dtln') {
mlOptions.processorOptions = {
wasmUrl: ASSET_BASE + 'litert_wasm_internal.wasm',
model1Url: ASSET_BASE + 'model_1.tflite',
model2Url: ASSET_BASE + 'model_2.tflite',
processorUrl: ASSET_BASE + 'dtlnProcessor.js'
};
} else if (MODEL === 'deepfilternet') {
mlOptions.processorOptions = {
wasmModule: wasmBinary,
modelBytes: new Uint8Array(wasmBinary),
suppressionLevel: 50
};
}
var mlNode = new AudioWorkletNode(ctx, PROCESSORS[MODEL].name, mlOptions);
head.connect(mlNode);
mlNode.connect(dest);
package-lock.json
Generated
+29
View File
@@ -26,6 +26,7 @@
"@tanstack/react-query-devtools": "5.100.13",
"@tanstack/react-virtual": "3.13.25",
"@types/dompurify": "3.2.0",
"@workadventure/noise-suppression": "0.0.4",
"await-to-js": "3.0.0",
"badwords-list": "2.0.1-4",
"blurhash": "2.0.5",
@@ -34,6 +35,7 @@
"classnames": "2.5.1",
"dateformat": "5.0.3",
"dayjs": "1.11.20",
"deepfilternet3-noise-filter": "1.2.1",
"domhandler": "6.0.1",
"dompurify": "3.4.5",
"emojibase": "17.0.0",
@@ -4856,6 +4858,15 @@
}
}
},
"node_modules/@workadventure/noise-suppression": {
"version": "0.0.4",
"resolved": "https://registry.npmjs.org/@workadventure/noise-suppression/-/noise-suppression-0.0.4.tgz",
"integrity": "sha512-v8DQgV2TQAWh7YLo7bZ1grV3iDNltRuvPaIYTcaBWoOjUaxDp/j5zrFLz4ZuijPGxzqcQxeW7ql/HJltMuLDtA==",
"license": "MIT",
"dependencies": {
"fft.js": "^4.0.4"
}
},
"node_modules/@xobotyi/scrollbar-width": {
"version": "1.9.5",
"resolved": "https://registry.npmjs.org/@xobotyi/scrollbar-width/-/scrollbar-width-1.9.5.tgz",
@@ -6389,6 +6400,18 @@
"integrity": "sha512-Rn+RuwkmkDwCi2/oXOFS9Gsr5lJZu/yTGpK7wAaAIE75CC+LCGEZHpY6VQJa/RoJcrmaA/docWJZvYohlNkWPA==",
"dev": true
},
"node_modules/deepfilternet3-noise-filter": {
"version": "1.2.1",
"resolved": "https://registry.npmjs.org/deepfilternet3-noise-filter/-/deepfilternet3-noise-filter-1.2.1.tgz",
"integrity": "sha512-OAyrHTDlUHH+AhfpVNKYEOhVqb9cZpu0fdNThplA/tB/Ts4PF/UsI+abl2n1IbSxUkhiF0OqDejEhk1n42Oqpw==",
"license": "(Apache-2.0 OR MIT)",
"engines": {
"node": ">=18.0.0"
},
"peerDependencies": {
"livekit-client": "^2.0.0"
}
},
"node_modules/deepmerge": {
"version": "4.3.1",
"resolved": "https://registry.npmjs.org/deepmerge/-/deepmerge-4.3.1.tgz",
@@ -7619,6 +7642,12 @@
}
}
},
"node_modules/fft.js": {
"version": "4.0.4",
"resolved": "https://registry.npmjs.org/fft.js/-/fft.js-4.0.4.tgz",
"integrity": "sha512-f9c00hphOgeQTlDyavwTtu6RiK8AIFjD6+jvXkNkpeQ7rirK3uFWVpalkoS4LAwbdX7mfZ8aoBfFVQX1Re/8aw==",
"license": "MIT"
},
"node_modules/file-entry-cache": {
"version": "8.0.0",
"resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-8.0.0.tgz",
package.json
+2
View File
@@ -45,6 +45,8 @@
"@giphy/js-util": "5.2.0",
"@giphy/react-components": "10.1.2",
"@sapphi-red/web-noise-suppressor": "0.3.5",
"@workadventure/noise-suppression": "0.0.4",
"deepfilternet3-noise-filter": "1.2.1",
"@sentry/react": "10.53.1",
"@tanstack/react-query": "5.100.13",
"@tanstack/react-query-devtools": "5.100.13",
vite.config.js
+21
View File
@@ -96,6 +96,27 @@ function lotusDenoise() {
path.join(sapphi, 'noiseGate/workletProcessor.js'),
path.join(denoiseDir, 'noiseGateWorklet.js'),
],
// DTLN (WorkAdventure v0.0.4)
[
path.resolve('node_modules/@workadventure/noise-suppression/dist/audio-worklet.js'),
path.join(denoiseDir, 'dtlnWorklet.js'),
],
[
path.resolve('node_modules/@workadventure/noise-suppression/dist/assets/audio-worklet-processor.js'),
path.join(denoiseDir, 'dtlnProcessor.js'),
],
[
path.resolve('node_modules/@workadventure/noise-suppression/dist/vendor/litert/litert_wasm_internal.wasm'),
path.join(denoiseDir, 'litert_wasm_internal.wasm'),
],
[
path.resolve('node_modules/@workadventure/noise-suppression/dist/assets/model_quant_1.tflite'),
path.join(denoiseDir, 'model_1.tflite'),
],
[
path.resolve('node_modules/@workadventure/noise-suppression/dist/assets/model_quant_2.tflite'),
path.join(denoiseDir, 'model_2.tflite'),
],
];
assets.forEach(([s, d]) => {
if (fs.existsSync(s)) {