fix(calls): wire DTLN ML denoise correctly via @workadventure JS API
CI / Build & Quality Checks (push) Successful in 10m25s
CI / Trigger Desktop Build (push) Successful in 6s

The prior DTLN attempt (89a2321d) broke the build (missing dep, wrong
`cinny/` asset paths) and typecheck (`'dtln'` not in DenoiseModelId), and was
wired against an API the package doesn't expose. @workadventure/noise-
suppression is not a flat AudioWorklet — it's a self-contained ES module whose
processor name is `workadventure-noise-suppression` and which resolves its own
LiteRT WASM + TFLite models via import.meta.url. Driving it by hand-rolled
addModule + processorOptions cannot work.

- Re-add @workadventure/noise-suppression@0.0.4 (package.json + lockfile).
- vite: copy the package's whole dist/ tree intact to
  denoise/workadventure/ (preserving assets/ + vendor/litert) so import.meta
  resolution works at runtime; fail the build if the entry module is missing.
- shim: for the DTLN model, dynamic-import denoise/workadventure/audio-worklet
  .js and use createNoiseSuppressionAudioWorklet(ctx, { bypassUntilReady })
  to build the node; RNNoise/Speex keep their direct flat-worklet path. Async
  init errors are logged + reported and fall back to the raw mic.
- Restore 'dtln' in DenoiseModelId (+ settings coercion), the model chart, and
  the settings dropdown, labelled "(beta)".

DTLN builds and is fully self-hosted, but its in-call audio is UNVERIFIED in
this environment — needs a real-call test. DeepFilterNet stays excluded (CDN
asset loading, incompatible with self-hosting / Tauri CSP).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
2026-06-16 17:11:45 -04:00
parent 89a2321dd4
commit 86272b6b08
7 changed files with 148 additions and 104 deletions
+97 -69
View File
@@ -55,8 +55,11 @@
wasm: 'speex.wasm',
},
dtln: {
name: 'noise-suppression-audio-worklet',
script: 'dtlnWorklet.js',
// @workadventure/noise-suppression is a self-contained ES module that
// resolves its own AudioWorklet processor + LiteRT WASM + TFLite models
// via import.meta.url. We dynamic-import this helper and let it build the
// node, rather than addModule-ing a flat worklet ourselves.
helper: 'workadventure/audio-worklet.js',
},
gate: {
name: '@sapphi-red/web-noise-suppressor/noise-gate',
@@ -117,9 +120,10 @@
} catch (e) {}
return Promise.reject(new Error('SampleRate mismatch: ' + ctx.sampleRate));
}
// Load required modules
var scripts = [PROCESSORS[MODEL].script];
if (MODEL === 'dtln') scripts.push('dtlnProcessor.js');
// Load worklet modules. DTLN registers its own processor via the
// dynamic-imported helper (see buildMlNode), so it needs nothing here.
var scripts = [];
if (MODEL === 'rnnoise' || MODEL === 'speex') scripts.push(PROCESSORS[MODEL].script);
if (USE_GATE) scripts.push(PROCESSORS.gate.script);
return Promise.all(
@@ -143,6 +147,36 @@
var hasNotifiedActive = false;
// Build the ML denoise AudioWorkletNode. RNNoise/Speex are flat sapphi
// worklets we instantiate directly with the fetched WASM binary. DTLN comes
// from @workadventure's self-contained helper, which we dynamic-import; it
// resolves its own processor + LiteRT WASM + TFLite models internally and
// returns the node. Resolves to { node, ready, dispose }.
function buildMlNode(ctx, wasmBinary) {
if (MODEL === 'dtln') {
return import(ASSET_BASE + PROCESSORS.dtln.helper).then(function (mod) {
// bypassUntilReady: pass raw audio through until the model is loaded so
// the call never has a silent/missing track during init.
return mod.createNoiseSuppressionAudioWorklet(ctx, { bypassUntilReady: true });
});
}
var node = new AudioWorkletNode(ctx, PROCESSORS[MODEL].name, {
channelCount: 1,
numberOfInputs: 1,
numberOfOutputs: 1,
processorOptions: { maxChannels: 1, wasmBinary: wasmBinary },
});
return Promise.resolve({
node: node,
ready: Promise.resolve(),
dispose: function () {
try {
node.port.postMessage('destroy');
} catch (e) {}
},
});
}
function processStream(stream) {
var audioTracks = stream.getAudioTracks();
if (audioTracks.length === 0) return Promise.resolve(stream);
@@ -171,78 +205,72 @@
}
// 2. ML Processor
var mlOptions = {
channelCount: 1,
numberOfInputs: 1,
numberOfOutputs: 1,
processorOptions: { maxChannels: 1 },
};
return buildMlNode(ctx, wasmBinary).then(function (ml) {
var mlNode = ml.node;
head.connect(mlNode);
mlNode.connect(dest);
if (MODEL === 'rnnoise' || MODEL === 'speex') {
mlOptions.processorOptions.wasmBinary = wasmBinary;
} else if (MODEL === 'dtln') {
mlOptions.processorOptions = {
wasmBinary: wasmBinary,
model1: ASSET_BASE + 'model_1.tflite',
model2: ASSET_BASE + 'model_2.tflite',
};
}
// Surface async init failures (e.g. DTLN model load) without blocking
// the track handoff — audio flows via bypassUntilReady meanwhile.
if (ml.ready && typeof ml.ready.then === 'function') {
ml.ready.catch(function (err) {
var m = err instanceof Error ? err.message : String(err);
console.error('[lotus-denoise] ' + MODEL + ' init failed:', m);
});
}
var mlNode = new AudioWorkletNode(ctx, PROCESSORS[MODEL].name, mlOptions);
head.connect(mlNode);
mlNode.connect(dest);
var origTrack = audioTracks[0];
var processedTrack = dest.stream.getAudioTracks()[0];
var origTrack = audioTracks[0];
var processedTrack = dest.stream.getAudioTracks()[0];
var torndown = false;
function cleanup() {
if (torndown) return;
torndown = true;
try {
ml.dispose();
} catch (e) {}
try {
source.disconnect();
mlNode.disconnect();
} catch (e) {}
try {
origTrack.stop();
} catch (e) {}
}
var torndown = false;
function cleanup() {
if (torndown) return;
torndown = true;
try {
mlNode.port.postMessage('destroy');
} catch (e) {}
try {
source.disconnect();
mlNode.disconnect();
} catch (e) {}
try {
origTrack.stop();
} catch (e) {}
}
var rawStop = processedTrack.stop.bind(processedTrack);
processedTrack.stop = function () {
cleanup();
rawStop();
};
origTrack.addEventListener('ended', function () {
try {
var rawStop = processedTrack.stop.bind(processedTrack);
processedTrack.stop = function () {
cleanup();
rawStop();
} catch (e) {}
cleanup();
});
};
origTrack.addEventListener('ended', function () {
try {
rawStop();
} catch (e) {}
cleanup();
});
if (!hasNotifiedActive) {
hasNotifiedActive = true;
window.parent.postMessage(
{
type: 'lotus-denoise-status',
active: true,
model: MODEL,
nativeNS: USE_NATIVE_NS,
gate: USE_GATE,
},
'*',
);
}
if (!hasNotifiedActive) {
hasNotifiedActive = true;
window.parent.postMessage(
{
type: 'lotus-denoise-status',
active: true,
model: MODEL,
nativeNS: USE_NATIVE_NS,
gate: USE_GATE,
},
'*',
);
}
var out = new MediaStream();
out.addTrack(processedTrack);
stream.getVideoTracks().forEach(function (t) {
out.addTrack(t);
var out = new MediaStream();
out.addTrack(processedTrack);
stream.getVideoTracks().forEach(function (t) {
out.addTrack(t);
});
return out;
});
return out;
})
.catch(function (e) {
var msg = e instanceof Error ? e.message : String(e);