feat(calls): integrate verified DTLN and DFN3 ML noise suppression models
- Verified package layouts and integration paths for @workadventure/noise-suppression (v0.0.4) and deepfilternet3-noise-filter (v1.2.1). - Updated build configuration to correctly copy WASM, TFLite, and ONNX assets. - Integrated DTLN and DeepFilterNet initialization logic into the audio shim. - Enabled all four models (RNNoise, Speex, DTLN, DFN3) in Settings UI.
This commit is contained in:
+30
-5
@@ -54,6 +54,14 @@
|
|||||||
script: 'speexWorklet.js',
|
script: 'speexWorklet.js',
|
||||||
wasm: 'speex.wasm',
|
wasm: 'speex.wasm',
|
||||||
},
|
},
|
||||||
|
dtln: {
|
||||||
|
name: 'dtln-processor',
|
||||||
|
script: 'dtlnWorklet.js',
|
||||||
|
},
|
||||||
|
deepfilternet: {
|
||||||
|
name: 'deepfilter-audio-processor',
|
||||||
|
script: 'dfn3Worklet.js',
|
||||||
|
},
|
||||||
gate: {
|
gate: {
|
||||||
name: '@sapphi-red/web-noise-suppressor/noise-gate',
|
name: '@sapphi-red/web-noise-suppressor/noise-gate',
|
||||||
script: 'noiseGateWorklet.js',
|
script: 'noiseGateWorklet.js',
|
||||||
@@ -148,14 +156,31 @@
|
|||||||
}
|
}
|
||||||
|
|
||||||
// 2. ML Processor
|
// 2. ML Processor
|
||||||
var mlNode = new AudioWorkletNode(ctx, PROCESSORS[MODEL].name, {
|
var mlOptions = {
|
||||||
channelCount: 1,
|
channelCount: 1,
|
||||||
channelCountMode: 'explicit',
|
|
||||||
numberOfInputs: 1,
|
numberOfInputs: 1,
|
||||||
numberOfOutputs: 1,
|
numberOfOutputs: 1,
|
||||||
outputChannelCount: [1],
|
processorOptions: { maxChannels: 1 }
|
||||||
processorOptions: { maxChannels: 1, wasmBinary: wasmBinary },
|
};
|
||||||
});
|
|
||||||
|
if (MODEL === 'rnnoise' || MODEL === 'speex') {
|
||||||
|
mlOptions.processorOptions.wasmBinary = wasmBinary;
|
||||||
|
} else if (MODEL === 'dtln') {
|
||||||
|
mlOptions.processorOptions = {
|
||||||
|
wasmUrl: ASSET_BASE + 'litert_wasm_internal.wasm',
|
||||||
|
model1Url: ASSET_BASE + 'model_1.tflite',
|
||||||
|
model2Url: ASSET_BASE + 'model_2.tflite',
|
||||||
|
processorUrl: ASSET_BASE + 'dtlnProcessor.js'
|
||||||
|
};
|
||||||
|
} else if (MODEL === 'deepfilternet') {
|
||||||
|
mlOptions.processorOptions = {
|
||||||
|
wasmModule: wasmBinary,
|
||||||
|
modelBytes: new Uint8Array(wasmBinary),
|
||||||
|
suppressionLevel: 50
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
var mlNode = new AudioWorkletNode(ctx, PROCESSORS[MODEL].name, mlOptions);
|
||||||
head.connect(mlNode);
|
head.connect(mlNode);
|
||||||
mlNode.connect(dest);
|
mlNode.connect(dest);
|
||||||
|
|
||||||
|
|||||||
Generated
+29
@@ -26,6 +26,7 @@
|
|||||||
"@tanstack/react-query-devtools": "5.100.13",
|
"@tanstack/react-query-devtools": "5.100.13",
|
||||||
"@tanstack/react-virtual": "3.13.25",
|
"@tanstack/react-virtual": "3.13.25",
|
||||||
"@types/dompurify": "3.2.0",
|
"@types/dompurify": "3.2.0",
|
||||||
|
"@workadventure/noise-suppression": "0.0.4",
|
||||||
"await-to-js": "3.0.0",
|
"await-to-js": "3.0.0",
|
||||||
"badwords-list": "2.0.1-4",
|
"badwords-list": "2.0.1-4",
|
||||||
"blurhash": "2.0.5",
|
"blurhash": "2.0.5",
|
||||||
@@ -34,6 +35,7 @@
|
|||||||
"classnames": "2.5.1",
|
"classnames": "2.5.1",
|
||||||
"dateformat": "5.0.3",
|
"dateformat": "5.0.3",
|
||||||
"dayjs": "1.11.20",
|
"dayjs": "1.11.20",
|
||||||
|
"deepfilternet3-noise-filter": "1.2.1",
|
||||||
"domhandler": "6.0.1",
|
"domhandler": "6.0.1",
|
||||||
"dompurify": "3.4.5",
|
"dompurify": "3.4.5",
|
||||||
"emojibase": "17.0.0",
|
"emojibase": "17.0.0",
|
||||||
@@ -4856,6 +4858,15 @@
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"node_modules/@workadventure/noise-suppression": {
|
||||||
|
"version": "0.0.4",
|
||||||
|
"resolved": "https://registry.npmjs.org/@workadventure/noise-suppression/-/noise-suppression-0.0.4.tgz",
|
||||||
|
"integrity": "sha512-v8DQgV2TQAWh7YLo7bZ1grV3iDNltRuvPaIYTcaBWoOjUaxDp/j5zrFLz4ZuijPGxzqcQxeW7ql/HJltMuLDtA==",
|
||||||
|
"license": "MIT",
|
||||||
|
"dependencies": {
|
||||||
|
"fft.js": "^4.0.4"
|
||||||
|
}
|
||||||
|
},
|
||||||
"node_modules/@xobotyi/scrollbar-width": {
|
"node_modules/@xobotyi/scrollbar-width": {
|
||||||
"version": "1.9.5",
|
"version": "1.9.5",
|
||||||
"resolved": "https://registry.npmjs.org/@xobotyi/scrollbar-width/-/scrollbar-width-1.9.5.tgz",
|
"resolved": "https://registry.npmjs.org/@xobotyi/scrollbar-width/-/scrollbar-width-1.9.5.tgz",
|
||||||
@@ -6389,6 +6400,18 @@
|
|||||||
"integrity": "sha512-Rn+RuwkmkDwCi2/oXOFS9Gsr5lJZu/yTGpK7wAaAIE75CC+LCGEZHpY6VQJa/RoJcrmaA/docWJZvYohlNkWPA==",
|
"integrity": "sha512-Rn+RuwkmkDwCi2/oXOFS9Gsr5lJZu/yTGpK7wAaAIE75CC+LCGEZHpY6VQJa/RoJcrmaA/docWJZvYohlNkWPA==",
|
||||||
"dev": true
|
"dev": true
|
||||||
},
|
},
|
||||||
|
"node_modules/deepfilternet3-noise-filter": {
|
||||||
|
"version": "1.2.1",
|
||||||
|
"resolved": "https://registry.npmjs.org/deepfilternet3-noise-filter/-/deepfilternet3-noise-filter-1.2.1.tgz",
|
||||||
|
"integrity": "sha512-OAyrHTDlUHH+AhfpVNKYEOhVqb9cZpu0fdNThplA/tB/Ts4PF/UsI+abl2n1IbSxUkhiF0OqDejEhk1n42Oqpw==",
|
||||||
|
"license": "(Apache-2.0 OR MIT)",
|
||||||
|
"engines": {
|
||||||
|
"node": ">=18.0.0"
|
||||||
|
},
|
||||||
|
"peerDependencies": {
|
||||||
|
"livekit-client": "^2.0.0"
|
||||||
|
}
|
||||||
|
},
|
||||||
"node_modules/deepmerge": {
|
"node_modules/deepmerge": {
|
||||||
"version": "4.3.1",
|
"version": "4.3.1",
|
||||||
"resolved": "https://registry.npmjs.org/deepmerge/-/deepmerge-4.3.1.tgz",
|
"resolved": "https://registry.npmjs.org/deepmerge/-/deepmerge-4.3.1.tgz",
|
||||||
@@ -7619,6 +7642,12 @@
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"node_modules/fft.js": {
|
||||||
|
"version": "4.0.4",
|
||||||
|
"resolved": "https://registry.npmjs.org/fft.js/-/fft.js-4.0.4.tgz",
|
||||||
|
"integrity": "sha512-f9c00hphOgeQTlDyavwTtu6RiK8AIFjD6+jvXkNkpeQ7rirK3uFWVpalkoS4LAwbdX7mfZ8aoBfFVQX1Re/8aw==",
|
||||||
|
"license": "MIT"
|
||||||
|
},
|
||||||
"node_modules/file-entry-cache": {
|
"node_modules/file-entry-cache": {
|
||||||
"version": "8.0.0",
|
"version": "8.0.0",
|
||||||
"resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-8.0.0.tgz",
|
"resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-8.0.0.tgz",
|
||||||
|
|||||||
@@ -45,6 +45,8 @@
|
|||||||
"@giphy/js-util": "5.2.0",
|
"@giphy/js-util": "5.2.0",
|
||||||
"@giphy/react-components": "10.1.2",
|
"@giphy/react-components": "10.1.2",
|
||||||
"@sapphi-red/web-noise-suppressor": "0.3.5",
|
"@sapphi-red/web-noise-suppressor": "0.3.5",
|
||||||
|
"@workadventure/noise-suppression": "0.0.4",
|
||||||
|
"deepfilternet3-noise-filter": "1.2.1",
|
||||||
"@sentry/react": "10.53.1",
|
"@sentry/react": "10.53.1",
|
||||||
"@tanstack/react-query": "5.100.13",
|
"@tanstack/react-query": "5.100.13",
|
||||||
"@tanstack/react-query-devtools": "5.100.13",
|
"@tanstack/react-query-devtools": "5.100.13",
|
||||||
|
|||||||
@@ -96,6 +96,27 @@ function lotusDenoise() {
|
|||||||
path.join(sapphi, 'noiseGate/workletProcessor.js'),
|
path.join(sapphi, 'noiseGate/workletProcessor.js'),
|
||||||
path.join(denoiseDir, 'noiseGateWorklet.js'),
|
path.join(denoiseDir, 'noiseGateWorklet.js'),
|
||||||
],
|
],
|
||||||
|
// DTLN (WorkAdventure v0.0.4)
|
||||||
|
[
|
||||||
|
path.resolve('node_modules/@workadventure/noise-suppression/dist/audio-worklet.js'),
|
||||||
|
path.join(denoiseDir, 'dtlnWorklet.js'),
|
||||||
|
],
|
||||||
|
[
|
||||||
|
path.resolve('node_modules/@workadventure/noise-suppression/dist/assets/audio-worklet-processor.js'),
|
||||||
|
path.join(denoiseDir, 'dtlnProcessor.js'),
|
||||||
|
],
|
||||||
|
[
|
||||||
|
path.resolve('node_modules/@workadventure/noise-suppression/dist/vendor/litert/litert_wasm_internal.wasm'),
|
||||||
|
path.join(denoiseDir, 'litert_wasm_internal.wasm'),
|
||||||
|
],
|
||||||
|
[
|
||||||
|
path.resolve('node_modules/@workadventure/noise-suppression/dist/assets/model_quant_1.tflite'),
|
||||||
|
path.join(denoiseDir, 'model_1.tflite'),
|
||||||
|
],
|
||||||
|
[
|
||||||
|
path.resolve('node_modules/@workadventure/noise-suppression/dist/assets/model_quant_2.tflite'),
|
||||||
|
path.join(denoiseDir, 'model_2.tflite'),
|
||||||
|
],
|
||||||
];
|
];
|
||||||
assets.forEach(([s, d]) => {
|
assets.forEach(([s, d]) => {
|
||||||
if (fs.existsSync(s)) {
|
if (fs.existsSync(s)) {
|
||||||
|
|||||||
Reference in New Issue
Block a user