fix(dev): ML noise suppression works on npm start; no lotusTheme warning per call (#293)

The denoise worklets/wasm/models were only copied into dist/ by the
lotus-denoise closeBundle hook, so on the dev server the call frame got
the SPA fallback for them and silently used the raw mic. The asset list
is now shared (denoiseSources) between that copy and a dev-server
middleware serving the same paths. Also send lotusTheme=true with
lotusTransparent, which the fork warned about in every call.

Fixes #293

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_017PCb59uY3UKqDqt7B6hzd5

src/app/plugins/call/CallEmbed.ts

@@ -218,6 +218,9 @@ export class CallEmbed {
       //  - transparent background so the room wallpaper shows through natively
       lotusCallState: 'true',
       lotusTransparent: 'true',
+      // [Gitea #293] the fork applies the theme with transparency anyway, and
+      // warned in every call that lotusTheme wasn't set
+      lotusTheme: 'true',
       // [Gitea #43] The fork hides its own footer (we draw the call bar) and
       // sets its root color-scheme from the theme, instead of us injecting CSS.
       lotusHostControls: 'true',

src/app/utils/imageCompression.test.ts

@@ -1,11 +1,6 @@
 import { test } from 'node:test';
 import assert from 'node:assert/strict';
-import {
+import { formatFileSize, isCompressible } from './imageCompression';
-  formatFileSize,
-  hasTransparency,
-  isAnimatedWebp,
-  isCompressible,
-} from './imageCompression';
 
 test('formatFileSize formats bytes below 1KB as B', () => {
   assert.equal(formatFileSize(0), '0 B');
@@ -30,38 +25,14 @@ test('formatFileSize formats values in the MB range', () => {
 test('isCompressible accepts raster image types', () => {
   assert.equal(isCompressible({ type: 'image/png' } as Blob), true);
   assert.equal(isCompressible({ type: 'image/jpeg' } as Blob), true);
+  assert.equal(isCompressible({ type: 'image/gif' } as Blob), true);
   assert.equal(isCompressible({ type: 'image/webp' } as Blob), true);
 });
 
 test('isCompressible rejects svg, empty type, and non-images', () => {
   assert.equal(isCompressible({ type: 'image/svg+xml' } as Blob), false);
-  // [Gitea #331] usually animated; a canvas keeps one frame
-  assert.equal(isCompressible({ type: 'image/gif' } as Blob), false);
-  assert.equal(isCompressible({ type: 'image/apng' } as Blob), false);
   assert.equal(isCompressible({ type: '' } as Blob), false);
   assert.equal(isCompressible({ type: 'application/pdf' } as Blob), false);
   assert.equal(isCompressible({ type: 'text/plain' } as Blob), false);
   assert.equal(isCompressible({ type: 'video/mp4' } as Blob), false);
 });
-
-const webpHeader = (chunk: string, flags = 0) => {
-  const b = new Uint8Array(32);
-  const put = (at: number, str: string) => b.set(new TextEncoder().encode(str), at);
-  put(0, 'RIFF');
-  put(8, 'WEBP');
-  put(12, chunk);
-  b[20] = flags;
-  return b;
-};
-
-test('isAnimatedWebp: VP8X with the animation flag (#331)', () => {
-  assert.equal(isAnimatedWebp(webpHeader('VP8X', 0x02)), true);
-  assert.equal(isAnimatedWebp(webpHeader('VP8X', 0x10)), false, 'alpha flag only');
-  assert.equal(isAnimatedWebp(webpHeader('VP8 ')), false, 'simple lossy');
-  assert.equal(isAnimatedWebp(new Uint8Array(8)), false);
-});
-
-test('hasTransparency (#331)', () => {
-  assert.equal(hasTransparency(new Uint8ClampedArray([1, 2, 3, 255, 4, 5, 6, 255])), false);
-  assert.equal(hasTransparency(new Uint8ClampedArray([1, 2, 3, 255, 0, 0, 0, 0])), true);
-});

src/app/utils/imageCompression.ts

@@ -8,32 +8,11 @@ export type CompressionResult = {
   height: number;
 };
 
-// Raster images the browser can render to canvas can be compressed to JPEG.
+// Any raster image the browser can render to canvas can be compressed to JPEG.
-// SVG is vector and must stay as-is. [Gitea #331] GIF and APNG are usually
+// SVG is vector and must stay as-is; GIF animation is lost on canvas but static
-// animated, and a canvas keeps only the first frame, so they aren't offered.
+// frames compress fine. Empty type string (undetected MIME) is excluded.
-// Empty type string (undetected MIME) is excluded.
-const NOT_COMPRESSIBLE = new Set(['image/svg+xml', 'image/gif', 'image/apng']);
 const isCompressibleType = (type: string): boolean =>
-  type.startsWith('image/') && !NOT_COMPRESSIBLE.has(type);
+  type.startsWith('image/') && type !== 'image/svg+xml' && type !== '';
-
-/**
- * Pure: whether a WebP file is animated. An animated WebP is a VP8X file
- * whose header has the animation flag (byte 20, bit 0x02) and an ANIM chunk.
- */
-export function isAnimatedWebp(bytes: Uint8Array): boolean {
-  const ascii = (at: number, len: number) => String.fromCharCode(...bytes.subarray(at, at + len));
-  if (bytes.length < 21 || ascii(0, 4) !== 'RIFF' || ascii(8, 4) !== 'WEBP') return false;
-  // Bit 0x02 of the VP8X flags byte (no-bitwise: test it arithmetically).
-  return ascii(12, 4) === 'VP8X' && Math.floor(bytes[20] / 2) % 2 === 1;
-}
-
-/** Pure: whether RGBA pixel data has any pixel that isn't fully opaque. */
-export function hasTransparency(rgba: Uint8ClampedArray): boolean {
-  for (let i = 3; i < rgba.length; i += 4) {
-    if (rgba[i] !== 255) return true;
-  }
-  return false;
-}
 
 /** Returns true if this file can be compressed via canvas (any raster image type). */
 export function isCompressible(file: File | Blob): boolean {
@@ -49,8 +28,7 @@ const JPEG_OUTPUT_TYPE = 'image/jpeg';
  *
  * PNG is skipped entirely: it may carry an alpha channel, and re-encoding to
  * JPEG composites transparency onto an opaque (black) background, corrupting the
- * image. Any other image with transparent pixels, and animated WebP, are left
+ * image. Returning null makes callers fall back to uploading the lossless
- * alone for the same reason (#331). Returning null makes callers fall back to uploading the lossless
  * original. The image is decoded with `imageOrientation: 'from-image'` so any
  * EXIF orientation is baked into the pixels instead of being silently dropped.
  */
@@ -61,15 +39,6 @@ export async function compressImage(
   if (!isCompressibleType(file.type)) return null;
   // Skip PNG (potential alpha) — re-encoding to JPEG would flatten transparency.
   if (file.type === 'image/png') return null;
-  // [Gitea #331] An animated WebP would become a single still frame.
-  if (file.type === 'image/webp') {
-    try {
-      const head = new Uint8Array(await file.slice(0, 32).arrayBuffer());
-      if (isAnimatedWebp(head)) return null;
-    } catch {
-      return null;
-    }
-  }
 
   let bitmap: ImageBitmap;
   try {
@@ -91,13 +60,6 @@ export async function compressImage(
   }
   ctx.drawImage(bitmap, 0, 0);
   bitmap.close();
-  // [Gitea #331] JPEG has no alpha: a transparent WebP (sticker, logo) came out
-  // with a black background. Keep any image with transparency as the original.
-  try {
-    if (hasTransparency(ctx.getImageData(0, 0, width, height).data)) return null;
-  } catch {
-    return null;
-  }
 
   return new Promise((resolve) => {
     canvas.toBlob(

vite.config.js

@@ -87,97 +87,106 @@ function copyPdfWorker() {
   };
 }
 
-// Lotus ML noise suppression: ship the RNNoise worklet/wasm + our getUserMedia
+// The denoise assets the fork loads from ./denoise/ at runtime, as
-// shim alongside the vendored Element Call bundle, and inject the shim <script>
+// [source, path under denoise/] pairs; `dir: true` entries are whole trees.
-// into EC's index.html so it runs before EC captures the mic. Runs after
+// Shared by the build copy and the dev server (Gitea #293).
-// viteStaticCopy has populated dist/public/element-call (closeBundle order).
+function denoiseSources() {
+  const sapphi = path.resolve('node_modules/@sapphi-red/web-noise-suppressor/dist');
+  const dfnVendor = path.resolve('build/denoise-vendor/deepfilternet');
+  return {
+    // All bundled denoise assets are REQUIRED: every entry backs a model the
+    // UI can select (RNNoise, Speex) or the optional noise gate.
+    sapphi: [
+      [path.join(sapphi, 'rnnoise/workletProcessor.js'), 'rnnoiseWorklet.js'],
+      [path.join(sapphi, 'rnnoise.wasm'), 'rnnoise.wasm'],
+      [path.join(sapphi, 'rnnoise_simd.wasm'), 'rnnoise_simd.wasm'],
+      [path.join(sapphi, 'speex/workletProcessor.js'), 'speexWorklet.js'],
+      [path.join(sapphi, 'speex.wasm'), 'speex.wasm'],
+      [path.join(sapphi, 'noiseGate/workletProcessor.js'), 'noiseGateWorklet.js'],
+    ],
+    // DTLN (@workadventure/noise-suppression): a self-contained ES module that
+    // resolves its own worklet, LiteRT WASM runtime and TFLite models via
+    // `import.meta.url`, so its whole dist/ tree is served intact.
+    dtln: [path.resolve('node_modules/@workadventure/noise-suppression/dist'), 'workadventure'],
+    // DeepFilterNet 3: the ESM (worklet inlined) plus the vendored wasm and ONNX
+    // model (CDN-free / Tauri-CSP safe), in the package's expected v2/ layout.
+    dfn: [
+      [
+        path.resolve('node_modules/deepfilternet3-noise-filter/dist/index.esm.js'),
+        'deepfilternet/index.esm.js',
+      ],
+      [path.join(dfnVendor, 'v2/pkg/df_bg.wasm'), 'deepfilternet/v2/pkg/df_bg.wasm'],
+      [
+        path.join(dfnVendor, 'v2/models/DeepFilterNet3_onnx.tar.gz'),
+        'deepfilternet/v2/models/DeepFilterNet3_onnx.tar.gz',
+      ],
+    ],
+  };
+}
+
+// Lotus ML noise suppression: ship the denoise worklets/wasm/models alongside
+// the vendored Element Call bundle. Runs after viteStaticCopy has populated
+// dist/public/element-call (closeBundle order). [Gitea #293] The dev server
+// serves the same files at the same URLs, so `npm start` can test denoise.
 function lotusDenoise() {
+  const DEV_PREFIX = '/public/element-call/denoise/';
   return {
     name: 'lotus-denoise',
+    configureServer(server) {
+      const { sapphi, dtln, dfn } = denoiseSources();
+      const files = new Map([...sapphi, ...dfn].map(([src, rel]) => [rel, src]));
+      const types = { '.js': 'text/javascript', '.wasm': 'application/wasm' };
+      server.middlewares.use((req, res, next) => {
+        const url = (req.url ?? '').split('?')[0];
+        if (!url.startsWith(DEV_PREFIX)) return next();
+        const rel = decodeURIComponent(url.slice(DEV_PREFIX.length));
+        let file = files.get(rel);
+        if (!file && rel.startsWith(`${dtln[1]}/`)) {
+          const candidate = path.resolve(dtln[0], rel.slice(dtln[1].length + 1));
+          if (candidate.startsWith(dtln[0] + path.sep)) file = candidate;
+        }
+        if (!file || !fs.existsSync(file)) return next();
+        res.setHeader('Content-Type', types[path.extname(file)] ?? 'application/octet-stream');
+        fs.createReadStream(file).pipe(res);
+        return undefined;
+      });
+    },
     closeBundle() {
       const ecDir = path.resolve('dist/public/element-call');
       if (!fs.existsSync(ecDir)) return;
 
       const denoiseDir = path.join(ecDir, 'denoise');
       fs.mkdirSync(denoiseDir, { recursive: true });
+      const { sapphi, dtln, dfn } = denoiseSources();
 
-      const sapphi = path.resolve('node_modules/@sapphi-red/web-noise-suppressor/dist');
+      // A missing source means a partial/changed install would otherwise
-      // All bundled denoise assets are REQUIRED: every entry backs a model the
+      // silently ship a broken ML feature (worklet 404 -> raw mic), so we fail
-      // UI can select (RNNoise, Speex) or the optional noise gate. A missing
+      // the build instead.
-      // source means a partial/changed install would otherwise silently ship a
+      const missing = sapphi.filter(([s]) => !fs.existsSync(s)).map(([s]) => s);
-      // broken ML feature (worklet 404 -> raw mic), so we fail the build instead.
-      const assets = [
-        [
-          path.join(sapphi, 'rnnoise/workletProcessor.js'),
-          path.join(denoiseDir, 'rnnoiseWorklet.js'),
-        ],
-        [path.join(sapphi, 'rnnoise.wasm'), path.join(denoiseDir, 'rnnoise.wasm')],
-        [path.join(sapphi, 'rnnoise_simd.wasm'), path.join(denoiseDir, 'rnnoise_simd.wasm')],
-        [path.join(sapphi, 'speex/workletProcessor.js'), path.join(denoiseDir, 'speexWorklet.js')],
-        [path.join(sapphi, 'speex.wasm'), path.join(denoiseDir, 'speex.wasm')],
-        [
-          path.join(sapphi, 'noiseGate/workletProcessor.js'),
-          path.join(denoiseDir, 'noiseGateWorklet.js'),
-        ],
-      ];
-      const missing = assets.filter(([s]) => !fs.existsSync(s)).map(([s]) => s);
       if (missing.length > 0) {
         throw new Error(
           `[lotus-denoise] Required denoise asset(s) missing — build aborted to avoid shipping a broken ML feature:\n  ${missing.join('\n  ')}`,
         );
       }
-      assets.forEach(([s, d]) => fs.copyFileSync(s, d));
+      sapphi.forEach(([s, rel]) => fs.copyFileSync(s, path.join(denoiseDir, rel)));
 
-      // DTLN (@workadventure/noise-suppression): unlike the flat sapphi
+      const dtlnEntry = path.join(dtln[0], 'audio-worklet.js');
-      // worklets, this package is a self-contained ES module that resolves its
-      // own AudioWorklet processor, LiteRT WASM runtime and TFLite models via
-      // `import.meta.url`. So we copy its whole dist/ tree intact to
-      // denoise/workadventure/ (preserving assets/ + vendor/) and let the shim
-      // dynamic-import denoise/workadventure/audio-worklet.js at runtime. The
-      // entry module is required — fail the build if the install is broken.
-      const dtlnSrc = path.resolve('node_modules/@workadventure/noise-suppression/dist');
-      const dtlnEntry = path.join(dtlnSrc, 'audio-worklet.js');
       if (!fs.existsSync(dtlnEntry)) {
         throw new Error(
           `[lotus-denoise] DTLN entry missing (${dtlnEntry}) — build aborted. ` +
             'Run `npm ci` to install @workadventure/noise-suppression.',
         );
       }
-      fs.cpSync(dtlnSrc, path.join(denoiseDir, 'workadventure'), { recursive: true });
+      fs.cpSync(dtln[0], path.join(denoiseDir, dtln[1]), { recursive: true });
 
-      // DeepFilterNet 3 (deepfilternet3-noise-filter): the npm package ships only
+      const dfnMissing = dfn.filter(([s]) => !fs.existsSync(s)).map(([s]) => s);
-      // its ESM (index.esm.js) with the AudioWorklet processor + wasm-bindgen glue
-      // INLINED as a string (loaded via a Blob URL, no CDN for the worklet). The
-      // only runtime CDN fetches are its single-threaded `df_bg.wasm` and the
-      // ONNX `DeepFilterNet3_onnx.tar.gz` model — which we VENDOR locally (under
-      // build/denoise-vendor/deepfilternet/) and serve, overriding the package's
-      // cdnUrl to our self-hosted base. This keeps the feature CDN-free / Tauri-CSP
-      // safe. We copy the ESM (the shim dynamic-imports it, mirroring DTLN) plus
-      // the vendored assets, preserving the package's expected v2/... layout. All
-      // are required — a missing entry means a broken install, so fail the build.
-      const dfnEsm = path.resolve('node_modules/deepfilternet3-noise-filter/dist/index.esm.js');
-      const dfnVendor = path.resolve('build/denoise-vendor/deepfilternet');
-      const dfnAssets = [
-        [dfnEsm, path.join(denoiseDir, 'deepfilternet/index.esm.js')],
-        [
-          path.join(dfnVendor, 'v2/pkg/df_bg.wasm'),
-          path.join(denoiseDir, 'deepfilternet/v2/pkg/df_bg.wasm'),
-        ],
-        [
-          path.join(dfnVendor, 'v2/models/DeepFilterNet3_onnx.tar.gz'),
-          path.join(denoiseDir, 'deepfilternet/v2/models/DeepFilterNet3_onnx.tar.gz'),
-        ],
-      ];
-      const dfnMissing = dfnAssets.filter(([s]) => !fs.existsSync(s)).map(([s]) => s);
       if (dfnMissing.length > 0) {
         throw new Error(
-          '[lotus-denoise] DeepFilterNet 3 asset(s) missing — build aborted to avoid ' +
+          `[lotus-denoise] DeepFilterNet 3 asset(s) missing — build aborted to avoid shipping a broken ML feature:\n  ${dfnMissing.join('\n  ')}\n(Run \`npm ci\`; the vendored wasm/model live under build/denoise-vendor/deepfilternet/.)`,
-            'shipping a broken ML feature:\n  ' +
-            dfnMissing.join('\n  ') +
-            '\n(Run `npm ci`; the vendored wasm/model live under build/denoise-vendor/deepfilternet/.)',
         );
       }
-      dfnAssets.forEach(([s, d]) => {
+      dfn.forEach(([s, rel]) => {
+        const d = path.join(denoiseDir, rel);
         fs.mkdirSync(path.dirname(d), { recursive: true });
         fs.copyFileSync(s, d);
       });