fix(calls): wire DTLN ML denoise correctly via @workadventure JS API
The prior DTLN attempt (89a2321d) broke the build (missing dep, wrong
`cinny/` asset paths) and typecheck (`'dtln'` not in DenoiseModelId), and was
wired against an API the package doesn't expose. @workadventure/noise-
suppression is not a flat AudioWorklet — it's a self-contained ES module whose
processor name is `workadventure-noise-suppression` and which resolves its own
LiteRT WASM + TFLite models via import.meta.url. Driving it by hand-rolled
addModule + processorOptions cannot work.
- Re-add @workadventure/noise-suppression@0.0.4 (package.json + lockfile).
- vite: copy the package's whole dist/ tree intact to
denoise/workadventure/ (preserving assets/ + vendor/litert) so import.meta
resolution works at runtime; fail the build if the entry module is missing.
- shim: for the DTLN model, dynamic-import denoise/workadventure/audio-worklet
.js and use createNoiseSuppressionAudioWorklet(ctx, { bypassUntilReady })
to build the node; RNNoise/Speex keep their direct flat-worklet path. Async
init errors are logged + reported and fall back to the raw mic.
- Restore 'dtln' in DenoiseModelId (+ settings coercion), the model chart, and
the settings dropdown, labelled "(beta)".
DTLN builds and is fully self-hosted, but its in-call audio is UNVERIFIED in
this environment — needs a real-call test. DeepFilterNet stays excluded (CDN
asset loading, incompatible with self-hosting / Tauri CSP).
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
+97
-69
@@ -55,8 +55,11 @@
|
||||
wasm: 'speex.wasm',
|
||||
},
|
||||
dtln: {
|
||||
name: 'noise-suppression-audio-worklet',
|
||||
script: 'dtlnWorklet.js',
|
||||
// @workadventure/noise-suppression is a self-contained ES module that
|
||||
// resolves its own AudioWorklet processor + LiteRT WASM + TFLite models
|
||||
// via import.meta.url. We dynamic-import this helper and let it build the
|
||||
// node, rather than addModule-ing a flat worklet ourselves.
|
||||
helper: 'workadventure/audio-worklet.js',
|
||||
},
|
||||
gate: {
|
||||
name: '@sapphi-red/web-noise-suppressor/noise-gate',
|
||||
@@ -117,9 +120,10 @@
|
||||
} catch (e) {}
|
||||
return Promise.reject(new Error('SampleRate mismatch: ' + ctx.sampleRate));
|
||||
}
|
||||
// Load required modules
|
||||
var scripts = [PROCESSORS[MODEL].script];
|
||||
if (MODEL === 'dtln') scripts.push('dtlnProcessor.js');
|
||||
// Load worklet modules. DTLN registers its own processor via the
|
||||
// dynamic-imported helper (see buildMlNode), so it needs nothing here.
|
||||
var scripts = [];
|
||||
if (MODEL === 'rnnoise' || MODEL === 'speex') scripts.push(PROCESSORS[MODEL].script);
|
||||
if (USE_GATE) scripts.push(PROCESSORS.gate.script);
|
||||
|
||||
return Promise.all(
|
||||
@@ -143,6 +147,36 @@
|
||||
|
||||
var hasNotifiedActive = false;
|
||||
|
||||
// Build the ML denoise AudioWorkletNode. RNNoise/Speex are flat sapphi
|
||||
// worklets we instantiate directly with the fetched WASM binary. DTLN comes
|
||||
// from @workadventure's self-contained helper, which we dynamic-import; it
|
||||
// resolves its own processor + LiteRT WASM + TFLite models internally and
|
||||
// returns the node. Resolves to { node, ready, dispose }.
|
||||
function buildMlNode(ctx, wasmBinary) {
|
||||
if (MODEL === 'dtln') {
|
||||
return import(ASSET_BASE + PROCESSORS.dtln.helper).then(function (mod) {
|
||||
// bypassUntilReady: pass raw audio through until the model is loaded so
|
||||
// the call never has a silent/missing track during init.
|
||||
return mod.createNoiseSuppressionAudioWorklet(ctx, { bypassUntilReady: true });
|
||||
});
|
||||
}
|
||||
var node = new AudioWorkletNode(ctx, PROCESSORS[MODEL].name, {
|
||||
channelCount: 1,
|
||||
numberOfInputs: 1,
|
||||
numberOfOutputs: 1,
|
||||
processorOptions: { maxChannels: 1, wasmBinary: wasmBinary },
|
||||
});
|
||||
return Promise.resolve({
|
||||
node: node,
|
||||
ready: Promise.resolve(),
|
||||
dispose: function () {
|
||||
try {
|
||||
node.port.postMessage('destroy');
|
||||
} catch (e) {}
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
function processStream(stream) {
|
||||
var audioTracks = stream.getAudioTracks();
|
||||
if (audioTracks.length === 0) return Promise.resolve(stream);
|
||||
@@ -171,78 +205,72 @@
|
||||
}
|
||||
|
||||
// 2. ML Processor
|
||||
var mlOptions = {
|
||||
channelCount: 1,
|
||||
numberOfInputs: 1,
|
||||
numberOfOutputs: 1,
|
||||
processorOptions: { maxChannels: 1 },
|
||||
};
|
||||
return buildMlNode(ctx, wasmBinary).then(function (ml) {
|
||||
var mlNode = ml.node;
|
||||
head.connect(mlNode);
|
||||
mlNode.connect(dest);
|
||||
|
||||
if (MODEL === 'rnnoise' || MODEL === 'speex') {
|
||||
mlOptions.processorOptions.wasmBinary = wasmBinary;
|
||||
} else if (MODEL === 'dtln') {
|
||||
mlOptions.processorOptions = {
|
||||
wasmBinary: wasmBinary,
|
||||
model1: ASSET_BASE + 'model_1.tflite',
|
||||
model2: ASSET_BASE + 'model_2.tflite',
|
||||
};
|
||||
}
|
||||
// Surface async init failures (e.g. DTLN model load) without blocking
|
||||
// the track handoff — audio flows via bypassUntilReady meanwhile.
|
||||
if (ml.ready && typeof ml.ready.then === 'function') {
|
||||
ml.ready.catch(function (err) {
|
||||
var m = err instanceof Error ? err.message : String(err);
|
||||
console.error('[lotus-denoise] ' + MODEL + ' init failed:', m);
|
||||
});
|
||||
}
|
||||
|
||||
var mlNode = new AudioWorkletNode(ctx, PROCESSORS[MODEL].name, mlOptions);
|
||||
head.connect(mlNode);
|
||||
mlNode.connect(dest);
|
||||
var origTrack = audioTracks[0];
|
||||
var processedTrack = dest.stream.getAudioTracks()[0];
|
||||
|
||||
var origTrack = audioTracks[0];
|
||||
var processedTrack = dest.stream.getAudioTracks()[0];
|
||||
var torndown = false;
|
||||
function cleanup() {
|
||||
if (torndown) return;
|
||||
torndown = true;
|
||||
try {
|
||||
ml.dispose();
|
||||
} catch (e) {}
|
||||
try {
|
||||
source.disconnect();
|
||||
mlNode.disconnect();
|
||||
} catch (e) {}
|
||||
try {
|
||||
origTrack.stop();
|
||||
} catch (e) {}
|
||||
}
|
||||
|
||||
var torndown = false;
|
||||
function cleanup() {
|
||||
if (torndown) return;
|
||||
torndown = true;
|
||||
try {
|
||||
mlNode.port.postMessage('destroy');
|
||||
} catch (e) {}
|
||||
try {
|
||||
source.disconnect();
|
||||
mlNode.disconnect();
|
||||
} catch (e) {}
|
||||
try {
|
||||
origTrack.stop();
|
||||
} catch (e) {}
|
||||
}
|
||||
|
||||
var rawStop = processedTrack.stop.bind(processedTrack);
|
||||
processedTrack.stop = function () {
|
||||
cleanup();
|
||||
rawStop();
|
||||
};
|
||||
origTrack.addEventListener('ended', function () {
|
||||
try {
|
||||
var rawStop = processedTrack.stop.bind(processedTrack);
|
||||
processedTrack.stop = function () {
|
||||
cleanup();
|
||||
rawStop();
|
||||
} catch (e) {}
|
||||
cleanup();
|
||||
});
|
||||
};
|
||||
origTrack.addEventListener('ended', function () {
|
||||
try {
|
||||
rawStop();
|
||||
} catch (e) {}
|
||||
cleanup();
|
||||
});
|
||||
|
||||
if (!hasNotifiedActive) {
|
||||
hasNotifiedActive = true;
|
||||
window.parent.postMessage(
|
||||
{
|
||||
type: 'lotus-denoise-status',
|
||||
active: true,
|
||||
model: MODEL,
|
||||
nativeNS: USE_NATIVE_NS,
|
||||
gate: USE_GATE,
|
||||
},
|
||||
'*',
|
||||
);
|
||||
}
|
||||
if (!hasNotifiedActive) {
|
||||
hasNotifiedActive = true;
|
||||
window.parent.postMessage(
|
||||
{
|
||||
type: 'lotus-denoise-status',
|
||||
active: true,
|
||||
model: MODEL,
|
||||
nativeNS: USE_NATIVE_NS,
|
||||
gate: USE_GATE,
|
||||
},
|
||||
'*',
|
||||
);
|
||||
}
|
||||
|
||||
var out = new MediaStream();
|
||||
out.addTrack(processedTrack);
|
||||
stream.getVideoTracks().forEach(function (t) {
|
||||
out.addTrack(t);
|
||||
var out = new MediaStream();
|
||||
out.addTrack(processedTrack);
|
||||
stream.getVideoTracks().forEach(function (t) {
|
||||
out.addTrack(t);
|
||||
});
|
||||
return out;
|
||||
});
|
||||
return out;
|
||||
})
|
||||
.catch(function (e) {
|
||||
var msg = e instanceof Error ? e.message : String(e);
|
||||
|
||||
Reference in New Issue
Block a user