feat(denoise): add self-hosted DeepFilterNet 3 ML noise-suppression model
Integrate DeepFilterNet 3 (deepfilternet3-noise-filter@1.2.1) as a new client-side denoise model id 'deepfilternet', mirroring the DTLN pattern. The npm package ships only an ESM whose AudioWorklet processor + wasm-bindgen glue are inlined as a string (loaded via a Blob URL — no CDN for the worklet). Its only runtime fetches are a single-threaded df_bg.wasm and an ONNX model tarball, which previously loaded from an external CDN. We now VENDOR both (build/denoise-vendor/deepfilternet/v2/...) and self-host them under denoise/deepfilternet/, overriding the package's cdnUrl so nothing hits the upstream CDN — keeping it self-hosted / Tauri-CSP safe. The wasm is single-threaded (no SharedArrayBuffer / atomics / imported shared memory), so it needs no COOP/COEP cross-origin isolation and runs fine in EC's non-isolated iframe. Runs at 48 kHz fullband. Any init/runtime failure falls back to the raw mic, like the other models. - vite.config.js: copy ESM + vendored wasm/model into the EC denoise dir with a required-asset guard that aborts the build if any entry is missing. - build/lotus-denoise.js: 'deepfilternet' branch — dynamic-import the ESM, build a DeepFilterNet3Core pointed at the self-hosted base, await init, return the worklet node; 48 kHz; raw-mic fail-safe preserved. - denoisePipeline.ts: 'deepfilternet' branch for the in-app tester + sampleRate. - settings.ts: add 'deepfilternet' to DenoiseModelId + getSettings whitelist. - lotusDenoiseUtils.ts: add the comparison-chart row. - General.tsx: add the "DeepFilterNet 3 (beta)" dropdown option. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Binary file not shown.
Binary file not shown.
+40
-3
@@ -38,9 +38,9 @@
|
||||
|
||||
var MODEL = params.get('lotusModel') || 'rnnoise';
|
||||
// DTLN (@workadventure) targets 16 kHz and does not resample internally, so
|
||||
// its whole graph runs in a 16 kHz context; RNNoise/Speex (sapphi) need
|
||||
// 48 kHz. The processed MediaStreamTrack is published to LiveKit either way
|
||||
// (WebRTC/Opus resamples as needed).
|
||||
// its whole graph runs in a 16 kHz context; RNNoise/Speex (sapphi) and
|
||||
// DeepFilterNet 3 are 48 kHz fullband. The processed MediaStreamTrack is
|
||||
// published to LiveKit either way (WebRTC/Opus resamples as needed).
|
||||
var SAMPLE_RATE = MODEL === 'dtln' ? 16000 : 48000;
|
||||
var USE_NATIVE_NS = params.get('lotusNativeNS') === 'true';
|
||||
var USE_GATE = params.get('lotusGate') === 'true';
|
||||
@@ -65,6 +65,15 @@
|
||||
// node, rather than addModule-ing a flat worklet ourselves.
|
||||
helper: 'workadventure/audio-worklet.js',
|
||||
},
|
||||
deepfilternet: {
|
||||
// deepfilternet3-noise-filter ships an ESM whose AudioWorklet processor +
|
||||
// wasm-bindgen glue are INLINED as a string (loaded via a Blob URL — no
|
||||
// CDN for the worklet). The only assets it fetches are its single-threaded
|
||||
// df_bg.wasm + ONNX model, which we vendor + self-host under
|
||||
// deepfilternet/v2/... We dynamic-import the ESM, build a DeepFilterNet3Core
|
||||
// pointed at the self-hosted base, and let it create the worklet node.
|
||||
esm: 'deepfilternet/index.esm.js',
|
||||
},
|
||||
gate: {
|
||||
name: '@sapphi-red/web-noise-suppressor/noise-gate',
|
||||
script: 'noiseGateWorklet.js',
|
||||
@@ -164,6 +173,34 @@
|
||||
return mod.createNoiseSuppressionAudioWorklet(ctx, { bypassUntilReady: true });
|
||||
});
|
||||
}
|
||||
if (MODEL === 'deepfilternet') {
|
||||
// Resolve an absolute self-hosted base so the package's cdnUrl override
|
||||
// fetches our vendored df_bg.wasm + ONNX model (never the upstream CDN).
|
||||
var dfnBase = new URL(ASSET_BASE + 'deepfilternet', window.location.href).href;
|
||||
return import(ASSET_BASE + PROCESSORS.deepfilternet.esm).then(function (mod) {
|
||||
var core = new mod.DeepFilterNet3Core({
|
||||
sampleRate: SAMPLE_RATE,
|
||||
noiseReductionLevel: 80,
|
||||
assetConfig: { cdnUrl: dfnBase },
|
||||
});
|
||||
// initialize() fetches + compiles the wasm and loads the model on the
|
||||
// main thread; the worklet node only exists once that resolves, so the
|
||||
// graph is connected with a ready model (no half-initialised passthrough).
|
||||
return core.initialize().then(function () {
|
||||
return core.createAudioWorkletNode(ctx).then(function (node) {
|
||||
return {
|
||||
node: node,
|
||||
ready: Promise.resolve(),
|
||||
dispose: function () {
|
||||
try {
|
||||
core.destroy();
|
||||
} catch (e) {}
|
||||
},
|
||||
};
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
var node = new AudioWorkletNode(ctx, PROCESSORS[MODEL].name, {
|
||||
channelCount: 1,
|
||||
numberOfInputs: 1,
|
||||
|
||||
Reference in New Issue
Block a user