fix(denoise): stop DeepFilterNet silently passing raw audio through
If the DeepFilterNet model is served with `Content-Encoding: gzip` (vite preview does this for .tar.gz; some CDNs and nginx gzip_static setups do too), fetch() hands back the already-unpacked tar. The library's worklet then fails inside df_create, swallows the error and passes the raw mic through, while Lotus Call reports DeepFilterNet as active. Check the bytes after initialize(): a gzip model is used as-is, an unpacked tar is re-gzipped in the browser, and anything else throws so the rnnoise fallback runs and the host is told the truth. Measured in a two-browser call (fan noise, SNR 15 dB): before, 4.9 dB of noise removed, i.e. none beyond gain differences; after, 30.9 dB. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_017PCb59uY3UKqDqt7B6hzd5
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
117467b1fb
commit
16f5f3d1a3
@@ -11,6 +11,7 @@ import { type AudioProcessorOptions } from "livekit-client";
|
||||
import {
|
||||
assertModuleExport,
|
||||
type LotusDenoiseConfig,
|
||||
normalizeDfnModelBytes,
|
||||
LotusDenoiseProcessor,
|
||||
prepareDenoiseAssets,
|
||||
releasePreparedDenoiseAssets,
|
||||
@@ -245,3 +246,37 @@ describe("assertModuleExport (#26)", () => {
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe("normalizeDfnModelBytes", () => {
|
||||
const tar = (): Uint8Array => {
|
||||
const b = new Uint8Array(1024);
|
||||
b.set(new TextEncoder().encode("ustar"), 257);
|
||||
b.set(new TextEncoder().encode("model.onnx"), 0);
|
||||
return b;
|
||||
};
|
||||
const gunzip = async (buf: ArrayBuffer): Promise<Uint8Array> =>
|
||||
new Uint8Array(
|
||||
await new Response(
|
||||
new Response(buf).body!.pipeThrough(new DecompressionStream("gzip")),
|
||||
).arrayBuffer(),
|
||||
);
|
||||
|
||||
test("passes a gzipped model through untouched", async () => {
|
||||
const gz = new Uint8Array([0x1f, 0x8b, 8, 0, 1, 2, 3]).buffer;
|
||||
expect(await normalizeDfnModelBytes(gz)).toBe(gz);
|
||||
});
|
||||
|
||||
test("re-gzips a tar that a Content-Encoding server already unpacked", async () => {
|
||||
const out = await normalizeDfnModelBytes(tar().buffer as ArrayBuffer);
|
||||
const u8 = new Uint8Array(out);
|
||||
expect([u8[0], u8[1]]).toEqual([0x1f, 0x8b]);
|
||||
expect(await gunzip(out)).toEqual(tar());
|
||||
});
|
||||
|
||||
test("rejects anything else (e.g. an HTML error page)", async () => {
|
||||
const html = new TextEncoder().encode("<!doctype html><title>404</title>");
|
||||
await expect(
|
||||
normalizeDfnModelBytes(html.buffer as ArrayBuffer),
|
||||
).rejects.toThrow(/neither gzip nor tar/);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -144,6 +144,8 @@ interface DfnCore {
|
||||
initialize: () => Promise<void>;
|
||||
createAudioWorkletNode: (ctx: AudioContext) => Promise<AudioNode>;
|
||||
destroy: () => void;
|
||||
/** Set by `initialize()`; the model bytes are handed to the worklet as-is. */
|
||||
assets?: { wasmModule: WebAssembly.Module; modelBytes: ArrayBuffer } | null;
|
||||
}
|
||||
interface DfnModule {
|
||||
DeepFilterNet3Core: new (opts: {
|
||||
@@ -167,6 +169,35 @@ export function assertModuleExport<T>(
|
||||
return mod as T;
|
||||
}
|
||||
|
||||
/**
|
||||
* Make sure the DeepFilterNet model is the gzipped tarball its wasm expects.
|
||||
*
|
||||
* A server or proxy that sends the `.tar.gz` with `Content-Encoding: gzip`
|
||||
* (vite preview does; some CDNs and nginx `gzip_static` setups do too) makes
|
||||
* `fetch()` hand back the inner tar. The library's worklet then fails inside
|
||||
* `df_create`, swallows the error and passes the raw mic straight through,
|
||||
* while we report DeepFilterNet as active. An already-decompressed tar is
|
||||
* re-gzipped; anything else throws, so the rnnoise fallback runs and the host
|
||||
* is told the truth.
|
||||
*/
|
||||
export async function normalizeDfnModelBytes(
|
||||
bytes: ArrayBuffer,
|
||||
): Promise<ArrayBuffer> {
|
||||
const u8 = new Uint8Array(bytes);
|
||||
if (u8.length > 2 && u8[0] === 0x1f && u8[1] === 0x8b) return bytes;
|
||||
const isTar =
|
||||
u8.length > 262 &&
|
||||
String.fromCharCode(...u8.subarray(257, 262)) === "ustar";
|
||||
if (!isTar)
|
||||
throw new Error(
|
||||
`denoise: DeepFilterNet model is neither gzip nor tar (${u8.length} bytes)`,
|
||||
);
|
||||
const gz = new Response(u8).body!.pipeThrough(
|
||||
new CompressionStream("gzip"),
|
||||
);
|
||||
return new Response(gz).arrayBuffer();
|
||||
}
|
||||
|
||||
async function loadDfnCore(config: LotusDenoiseConfig): Promise<DfnCore> {
|
||||
const base = config.assetBase;
|
||||
const url = `${base}deepfilternet/index.esm.js`;
|
||||
@@ -185,6 +216,10 @@ async function loadDfnCore(config: LotusDenoiseConfig): Promise<DfnCore> {
|
||||
assetConfig: { cdnUrl: dfnBase },
|
||||
});
|
||||
await core.initialize();
|
||||
if (core.assets)
|
||||
core.assets.modelBytes = await normalizeDfnModelBytes(
|
||||
core.assets.modelBytes,
|
||||
);
|
||||
return core;
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user