diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index fa8c58a74..2275676a9 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -62,3 +62,35 @@ jobs: gzip_size=$(gzip -c "$f" | wc -c | awk '{printf "%.1f kB", $1/1024}') echo "| $name | $size | $gzip_size |" >> $GITHUB_STEP_SUMMARY done + + # ── Desktop build trigger ────────────────────────────────────────────── + # Gated on `build` succeeding so a broken push (e.g. failing `npm ci` or + # `npm run build`) never bumps the cinny-desktop submodule and kicks off the + # slow Tauri release builds, which would only error out downstream. Only + # runs on a real push to lotus — not on pull_request CI runs. + trigger-desktop: + name: Trigger Desktop Build + needs: build + if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/lotus' }} + runs-on: ubuntu-latest + steps: + - name: Bump cinny submodule + env: + TOKEN: ${{ secrets.RELEASE_TOKEN }} + run: | + CINNY_SHA="${{ github.sha }}" + git clone "https://x-access-token:$TOKEN@code.lotusguild.org/LotusGuild/cinny-desktop.git" desktop + cd desktop + git config user.email "ci@lotusguild.org" + git config user.name "Lotus CI" + git submodule update --init cinny + git -C cinny fetch origin + git -C cinny checkout "$CINNY_SHA" + git add cinny + if git diff --cached --quiet; then + echo "Submodule already at $CINNY_SHA, nothing to do" + else + git commit -m "chore: bump cinny submodule to ${CINNY_SHA:0:8}" + git push origin main + echo "Pushed — cinny-desktop release.yml will start via on:push trigger" + fi diff --git a/.gitea/workflows/trigger-desktop.yml b/.gitea/workflows/trigger-desktop.yml deleted file mode 100644 index afa856754..000000000 --- a/.gitea/workflows/trigger-desktop.yml +++ /dev/null @@ -1,30 +0,0 @@ -name: Trigger Desktop Build - -on: - push: - branches: [lotus] - -jobs: - trigger: - runs-on: ubuntu-latest - steps: - - name: Bump cinny submodule - env: - TOKEN: ${{ secrets.RELEASE_TOKEN }} - run: | - CINNY_SHA="${{ github.sha }}" - git clone "https://x-access-token:$TOKEN@code.lotusguild.org/LotusGuild/cinny-desktop.git" desktop - cd desktop - git config user.email "ci@lotusguild.org" - git config user.name "Lotus CI" - git submodule update --init cinny - git -C cinny fetch origin - git -C cinny checkout "$CINNY_SHA" - git add cinny - if git diff --cached --quiet; then - echo "Submodule already at $CINNY_SHA, nothing to do" - else - git commit -m "chore: bump cinny submodule to ${CINNY_SHA:0:8}" - git push origin main - echo "Pushed — cinny-desktop release.yml will start via on:push trigger" - fi diff --git a/LOTUS_BUGS.md b/LOTUS_BUGS.md index 455b3f832..6d6a65dc0 100644 --- a/LOTUS_BUGS.md +++ b/LOTUS_BUGS.md @@ -60,6 +60,7 @@ This document tracks identified bugs, edge cases, and architectural discrepancie ### 8. Seasonal Themes and Chat Backgrounds need EXTREME design improvements. - **Issue:** Basic css or random moving lines are not good artwork or design theory. Requires extensive research on css backgrounds wallpapers and app theming, these should be multi-day projects PER background and theme. As if a whole team spent a entire project sprint on a single one. + --- ## 📱 PWA & Mobile Issues diff --git a/LOTUS_DENOISE_ENGINEERING_REVIEW.md b/LOTUS_DENOISE_ENGINEERING_REVIEW.md index 4c0224aa1..1f0a591d5 100644 --- a/LOTUS_DENOISE_ENGINEERING_REVIEW.md +++ b/LOTUS_DENOISE_ENGINEERING_REVIEW.md @@ -1,43 +1,54 @@ # Engineering Review: Multi-Model ML Noise Suppression Upgrade (P5-30) ## Overview + This PR implements a robust, modular, and high-fidelity client-side audio processing pipeline for noise suppression (NS) within Lotus Chat. It addresses issues with static noise artifacts, suboptimal sample rate resampling, and the lack of transparency in the audio processing chain. ## 1. Architectural Changes + ### 1.1 Audio Processing Pipeline (`lotus-denoise.js`) -* **Decoupled Initialization:** The shim now treats the audio chain as a configurable graph: `Source` → `Noise Gate` (optional) → `ML Model` → `LiveKit`. -* **Series Processing:** We enabled the browser-native suppressor (Google NSNet2) to run in series with the ML model. The native engine handles stationary noise (fan hum) efficiently, while the ML model focuses on transient "life" noise (keyboard clicks, mouse taps). -* **Hardware Fidelity:** Removed forced `48kHz` capture constraints in `getUserMedia`. This allows high-end audio interfaces (e.g., Rode/Scarlett at 48kHz) to pass raw audio without low-quality browser-level resampling, which was previously creating "static" artifacts. -* **SIMD Optimization:** Added runtime `WebAssembly.validate` checks to detect SIMD support. The pipeline dynamically selects `rnnoise_simd.wasm` over standard WASM if supported, reducing CPU utilization. -* **Failure Resilience:** Wrapped the entire graph initialization in `Promise.all` + `try/catch`. If any component (WASM loading, AudioWorklet initialization) fails, the shim sends a `postMessage` failure report and falls back to the raw microphone stream, ensuring calls never drop due to suppression errors. + +- **Decoupled Initialization:** The shim now treats the audio chain as a configurable graph: `Source` → `Noise Gate` (optional) → `ML Model` → `LiveKit`. +- **Series Processing:** We enabled the browser-native suppressor (Google NSNet2) to run in series with the ML model. The native engine handles stationary noise (fan hum) efficiently, while the ML model focuses on transient "life" noise (keyboard clicks, mouse taps). +- **Hardware Fidelity:** Removed forced `48kHz` capture constraints in `getUserMedia`. This allows high-end audio interfaces (e.g., Rode/Scarlett at 48kHz) to pass raw audio without low-quality browser-level resampling, which was previously creating "static" artifacts. +- **SIMD Optimization:** Added runtime `WebAssembly.validate` checks to detect SIMD support. The pipeline dynamically selects `rnnoise_simd.wasm` over standard WASM if supported, reducing CPU utilization. +- **Failure Resilience:** Wrapped the entire graph initialization in `Promise.all` + `try/catch`. If any component (WASM loading, AudioWorklet initialization) fails, the shim sends a `postMessage` failure report and falls back to the raw microphone stream, ensuring calls never drop due to suppression errors. ### 1.2 Multi-Model Support + Added support for 4 distinct processing models: + 1. **RNNoise (Mozilla):** Default lightweight hybrid model. 2. **Speex (Legacy):** DSP-based fallback for extremely low-CPU requirements. 3. **DTLN (Balanced):** Deep learning model (~15% CPU). Improved transient handling. 4. **DeepFilterNet 3 (Pro):** Studio-grade Deep Learning (~25-50%+ CPU). Designed for high-fidelity noise removal. ## 2. Infrastructure & Build Integration (`vite.config.js`) -* **Automated Asset Pipeline:** Added rules to copy model assets (TFLite models, WASM runtimes) from `node_modules` into the `denoise/` directory during build. -* **CI-Friendly:** The copy logic now includes `console.warn` fallbacks for missing assets to prevent build failures in environments where `npm install` hasn't yet finished, facilitating robust CI/CD integration. -* **Self-Hosting:** All assets are explicitly served from the `/denoise/` path, ensuring full privacy and avoiding external CDN dependencies at runtime. + +- **Automated Asset Pipeline:** Added rules to copy model assets (TFLite models, WASM runtimes) from `node_modules` into the `denoise/` directory during build. +- **CI-Friendly:** The copy logic now includes `console.warn` fallbacks for missing assets to prevent build failures in environments where `npm install` hasn't yet finished, facilitating robust CI/CD integration. +- **Self-Hosting:** All assets are explicitly served from the `/denoise/` path, ensuring full privacy and avoiding external CDN dependencies at runtime. ## 3. UI & UX Improvements + ### 3.1 Settings & Tuning (`General.tsx`) -* **Capability Detection:** Created `lotusDenoiseUtils.ts` to verify support for `AudioContext` and `AudioWorklet`. The ML option is programmatically disabled in unsupported browsers (e.g., Safari/Mobile) with a clear requirement list. -* **Comparison Chart:** Added a UI table listing `Model`, `CPU Usage`, `Quality`, and `Transient Handling` to allow users to make informed decisions based on their hardware. -* **Live Tuning:** Added a `MicMeter` component using an `AnalyserNode` to provide real-time visual feedback, enabling users to calibrate the **Noise Gate Threshold** (-100dB to 0dB) precisely to their microphone's noise floor. + +- **Capability Detection:** Created `lotusDenoiseUtils.ts` to verify support for `AudioContext` and `AudioWorklet`. The ML option is programmatically disabled in unsupported browsers (e.g., Safari/Mobile) with a clear requirement list. +- **Comparison Chart:** Added a UI table listing `Model`, `CPU Usage`, `Quality`, and `Transient Handling` to allow users to make informed decisions based on their hardware. +- **Live Tuning:** Added a `MicMeter` component using an `AnalyserNode` to provide real-time visual feedback, enabling users to calibrate the **Noise Gate Threshold** (-100dB to 0dB) precisely to their microphone's noise floor. ### 3.2 Error Reporting -* **Inter-Iframe Comms:** The shim now reports status and failures to the parent `LotusChat` host via `window.parent.postMessage`. -* **System Toasts:** Added `LotusDenoiseFeature` in `ClientNonUIFeatures.tsx`. It listens for these events and triggers a non-intrusive system toast if the noise suppression falls back to raw mic, ensuring users know their microphone status. + +- **Inter-Iframe Comms:** The shim now reports status and failures to the parent `LotusChat` host via `window.parent.postMessage`. +- **System Toasts:** Added `LotusDenoiseFeature` in `ClientNonUIFeatures.tsx`. It listens for these events and triggers a non-intrusive system toast if the noise suppression falls back to raw mic, ensuring users know their microphone status. ## 4. Technical Debt & Safety -* **Settings Persistence:** Added strongly-typed settings fields for `callDenoiseModel`, `callDenoiseNativeNS`, `callDenoiseGate`, and `callDenoiseGateThreshold` to `settings.ts`. -* **Clean Teardown:** Improved `cleanup()` logic in `lotus-denoise.js` to ensure the `AudioContext` and `MediaStreamTracks` are properly released, preventing potential memory leaks or microphone "hanging" after calls. + +- **Settings Persistence:** Added strongly-typed settings fields for `callDenoiseModel`, `callDenoiseNativeNS`, `callDenoiseGate`, and `callDenoiseGateThreshold` to `settings.ts`. +- **Clean Teardown:** Improved `cleanup()` logic in `lotus-denoise.js` to ensure the `AudioContext` and `MediaStreamTracks` are properly released, preventing potential memory leaks or microphone "hanging" after calls. ## Testing Instructions for Senior Engineer + 1. **Calibration:** Go to Settings, enable ML NS, toggle on Noise Gate, and click "Test Microphone". Confirm the meter reflects real-time audio. 2. **Validation:** Test "Series Suppression ON" vs "OFF" with a fan running in the background to confirm native NS is effectively handling the stationary noise. 3. **Fallback Test:** Introduce a malformed model request (via devtools console) to verify the System Toast notification functions. diff --git a/LOTUS_FEATURES.md b/LOTUS_FEATURES.md index 95585e36f..e5ef926ce 100644 --- a/LOTUS_FEATURES.md +++ b/LOTUS_FEATURES.md @@ -416,8 +416,9 @@ A comprehensive mic noise-suppression system in **Settings → General → Calls | **ML (Advanced)** | Custom ML pipeline supporting multiple models, series suppression, and gates. | **Advanced Features & Test Options:** + - **Multiple ML Models:** Toggle between **RNNoise** (standard hybrid) and **Speex** (legacy DSP-based) to compare artifact levels and suppression strength. -- **Series Suppression (Combination):** Optional toggle to run the browser's native stationary noise filter *before* the ML model. This allows testing the individual performance of the ML model vs the combined effectiveness at removing fan hum. +- **Series Suppression (Combination):** Optional toggle to run the browser's native stationary noise filter _before_ the ML model. This allows testing the individual performance of the ML model vs the combined effectiveness at removing fan hum. - **Noise Gate:** Configurable hardware-style gate with a dB threshold. Hard-cuts all audio when input is below the threshold, ensuring absolute silence between sentences. - **Live Microphone Meter:** A real-time volume visualizer in the settings panel to help users accurately tune their Noise Gate threshold. - **High-Fidelity Capture:** Captures at hardware native rates (supporting high-end gear like **Scarlett Solo + PodMic**) and handles high-quality resampling via Web Audio to prevent the "static" artifacts caused by low-quality browser pre-resamplers. @@ -442,7 +443,6 @@ A comprehensive mic noise-suppression system in **Settings → General → Calls - `src/app/utils/lotusDenoiseUtils.ts` — support detection and model comparison metadata - `src/app/features/settings/general/General.tsx` — advanced settings UI + mic meter - ### Call Button Scoping The call button is shown only in DMs and invite-only rooms that do not have an `m.space.parent` event. It is hidden in public rooms and space channels to avoid accidental broadcast calls. diff --git a/build/lotus-denoise.js b/build/lotus-denoise.js index b82640067..c06e2a9fa 100644 --- a/build/lotus-denoise.js +++ b/build/lotus-denoise.js @@ -66,7 +66,12 @@ function checkSimd() { try { - return WebAssembly.validate(new Uint8Array([0, 97, 115, 109, 1, 0, 0, 0, 1, 5, 1, 96, 0, 1, 123, 3, 2, 1, 0, 10, 10, 1, 8, 0, 65, 0, 253, 15, 253, 98, 11])) + return WebAssembly.validate( + new Uint8Array([ + 0, 97, 115, 109, 1, 0, 0, 0, 1, 5, 1, 96, 0, 1, 123, 3, 2, 1, 0, 10, 10, 1, 8, 0, 65, 0, + 253, 15, 253, 98, 11, + ]), + ) ? Promise.resolve(true) : Promise.resolve(false); } catch (e) { @@ -79,19 +84,22 @@ var p = PROCESSORS[modelId]; if (!p || !p.wasm) return Promise.resolve(null); - wasmPromises[modelId] = (modelId === 'rnnoise' ? checkSimd() : Promise.resolve(false)).then(function (simd) { - var file = (simd && p.simdWasm) ? p.simdWasm : p.wasm; - return fetch(ASSET_BASE + file).then(function (r) { - if (!r.ok) { - if (simd && p.simdWasm) return fetch(ASSET_BASE + p.wasm).then(function(r2) { - if (!r2.ok) throw new Error(modelId + ' wasm failed'); - return r2.arrayBuffer(); - }); - throw new Error(modelId + ' wasm failed'); - } - return r.arrayBuffer(); - }); - }); + wasmPromises[modelId] = (modelId === 'rnnoise' ? checkSimd() : Promise.resolve(false)).then( + function (simd) { + var file = simd && p.simdWasm ? p.simdWasm : p.wasm; + return fetch(ASSET_BASE + file).then(function (r) { + if (!r.ok) { + if (simd && p.simdWasm) + return fetch(ASSET_BASE + p.wasm).then(function (r2) { + if (!r2.ok) throw new Error(modelId + ' wasm failed'); + return r2.arrayBuffer(); + }); + throw new Error(modelId + ' wasm failed'); + } + return r.arrayBuffer(); + }); + }, + ); return wasmPromises[modelId]; } @@ -100,20 +108,30 @@ ctxPromise = (function () { var ctx = new AudioContext({ sampleRate: SAMPLE_RATE }); if (ctx.sampleRate !== SAMPLE_RATE) { - try { ctx.close(); } catch (e) {} + try { + ctx.close(); + } catch (e) {} return Promise.reject(new Error('SampleRate mismatch: ' + ctx.sampleRate)); } // Load required modules var scripts = [PROCESSORS[MODEL].script]; if (USE_GATE) scripts.push(PROCESSORS.gate.script); - return Promise.all(scripts.map(function(s) { - return ctx.audioWorklet.addModule(ASSET_BASE + s); - })).then(function () { - return ctx.state === 'suspended' ? ctx.resume().then(function () { return ctx; }) : ctx; + return Promise.all( + scripts.map(function (s) { + return ctx.audioWorklet.addModule(ASSET_BASE + s); + }), + ).then(function () { + return ctx.state === 'suspended' + ? ctx.resume().then(function () { + return ctx; + }) + : ctx; }); })(); - ctxPromise.catch(function () { ctxPromise = null; }); + ctxPromise.catch(function () { + ctxPromise = null; + }); } return ctxPromise; } @@ -140,8 +158,8 @@ openThreshold: GATE_THRESHOLD, closeThreshold: GATE_THRESHOLD - 5, holdMs: 150, - maxChannels: 1 - } + maxChannels: 1, + }, }); head.connect(gateNode); head = gateNode; @@ -166,32 +184,49 @@ function cleanup() { if (torndown) return; torndown = true; - try { mlNode.port.postMessage('destroy'); } catch (e) {} - try { source.disconnect(); mlNode.disconnect(); } catch (e) {} - try { origTrack.stop(); } catch (e) {} + try { + mlNode.port.postMessage('destroy'); + } catch (e) {} + try { + source.disconnect(); + mlNode.disconnect(); + } catch (e) {} + try { + origTrack.stop(); + } catch (e) {} } var rawStop = processedTrack.stop.bind(processedTrack); - processedTrack.stop = function () { cleanup(); rawStop(); }; + processedTrack.stop = function () { + cleanup(); + rawStop(); + }; origTrack.addEventListener('ended', function () { - try { rawStop(); } catch (e) {} + try { + rawStop(); + } catch (e) {} cleanup(); }); if (!hasNotifiedActive) { hasNotifiedActive = true; - window.parent.postMessage({ - type: 'lotus-denoise-status', - active: true, - model: MODEL, - nativeNS: USE_NATIVE_NS, - gate: USE_GATE - }, '*'); + window.parent.postMessage( + { + type: 'lotus-denoise-status', + active: true, + model: MODEL, + nativeNS: USE_NATIVE_NS, + gate: USE_GATE, + }, + '*', + ); } var out = new MediaStream(); out.addTrack(processedTrack); - stream.getVideoTracks().forEach(function (t) { out.addTrack(t); }); + stream.getVideoTracks().forEach(function (t) { + out.addTrack(t); + }); return out; }) .catch(function (e) { @@ -206,7 +241,8 @@ var wantsAudio = !!(constraints && constraints.audio); var effective = constraints; if (wantsAudio) { - var audioC = typeof constraints.audio === 'object' ? Object.assign({}, constraints.audio) : {}; + var audioC = + typeof constraints.audio === 'object' ? Object.assign({}, constraints.audio) : {}; audioC.noiseSuppression = USE_NATIVE_NS; audioC.channelCount = 1; if (audioC.echoCancellation === undefined) audioC.echoCancellation = true; @@ -218,4 +254,3 @@ }); }; })(); - diff --git a/package-lock.json b/package-lock.json index 970749d6a..2076efdb3 100644 --- a/package-lock.json +++ b/package-lock.json @@ -26,7 +26,6 @@ "@tanstack/react-query-devtools": "5.100.13", "@tanstack/react-virtual": "3.13.25", "@types/dompurify": "3.2.0", - "@workadventure/noise-suppression": "0.0.4", "await-to-js": "3.0.0", "badwords-list": "2.0.1-4", "blurhash": "2.0.5", @@ -35,7 +34,6 @@ "classnames": "2.5.1", "dateformat": "5.0.3", "dayjs": "1.11.20", - "deepfilternet3-noise-filter": "1.2.1", "domhandler": "6.0.1", "dompurify": "3.4.5", "emojibase": "17.0.0", @@ -4858,15 +4856,6 @@ } } }, - "node_modules/@workadventure/noise-suppression": { - "version": "0.0.4", - "resolved": "https://registry.npmjs.org/@workadventure/noise-suppression/-/noise-suppression-0.0.4.tgz", - "integrity": "sha512-v8DQgV2TQAWh7YLo7bZ1grV3iDNltRuvPaIYTcaBWoOjUaxDp/j5zrFLz4ZuijPGxzqcQxeW7ql/HJltMuLDtA==", - "license": "MIT", - "dependencies": { - "fft.js": "^4.0.4" - } - }, "node_modules/@xobotyi/scrollbar-width": { "version": "1.9.5", "resolved": "https://registry.npmjs.org/@xobotyi/scrollbar-width/-/scrollbar-width-1.9.5.tgz", @@ -6400,18 +6389,6 @@ "integrity": "sha512-Rn+RuwkmkDwCi2/oXOFS9Gsr5lJZu/yTGpK7wAaAIE75CC+LCGEZHpY6VQJa/RoJcrmaA/docWJZvYohlNkWPA==", "dev": true }, - "node_modules/deepfilternet3-noise-filter": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/deepfilternet3-noise-filter/-/deepfilternet3-noise-filter-1.2.1.tgz", - "integrity": "sha512-OAyrHTDlUHH+AhfpVNKYEOhVqb9cZpu0fdNThplA/tB/Ts4PF/UsI+abl2n1IbSxUkhiF0OqDejEhk1n42Oqpw==", - "license": "(Apache-2.0 OR MIT)", - "engines": { - "node": ">=18.0.0" - }, - "peerDependencies": { - "livekit-client": "^2.0.0" - } - }, "node_modules/deepmerge": { "version": "4.3.1", "resolved": "https://registry.npmjs.org/deepmerge/-/deepmerge-4.3.1.tgz", @@ -7642,12 +7619,6 @@ } } }, - "node_modules/fft.js": { - "version": "4.0.4", - "resolved": "https://registry.npmjs.org/fft.js/-/fft.js-4.0.4.tgz", - "integrity": "sha512-f9c00hphOgeQTlDyavwTtu6RiK8AIFjD6+jvXkNkpeQ7rirK3uFWVpalkoS4LAwbdX7mfZ8aoBfFVQX1Re/8aw==", - "license": "MIT" - }, "node_modules/file-entry-cache": { "version": "8.0.0", "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-8.0.0.tgz", diff --git a/package.json b/package.json index 88cf54b84..51a104392 100644 --- a/package.json +++ b/package.json @@ -45,8 +45,6 @@ "@giphy/js-util": "5.2.0", "@giphy/react-components": "10.1.2", "@sapphi-red/web-noise-suppressor": "0.3.5", - "@workadventure/noise-suppression": "0.0.4", - "deepfilternet3-noise-filter": "1.2.1", "@sentry/react": "10.53.1", "@tanstack/react-query": "5.100.13", "@tanstack/react-query-devtools": "5.100.13", diff --git a/src/app/features/settings/general/General.tsx b/src/app/features/settings/general/General.tsx index 10cd742e6..1d221f031 100644 --- a/src/app/features/settings/general/General.tsx +++ b/src/app/features/settings/general/General.tsx @@ -45,6 +45,7 @@ import { ChatBackground, ComposerToolbarSettings, DateFormat, + DenoiseModelId, MessageLayout, MessageSpacing, NoiseSuppressionMode, @@ -70,6 +71,10 @@ import { BG_OPTIONS, getChatBg } from '../../lotus/chatBackground'; import { resetBootSequence, runLotusBootSequence } from '../../../../lotus-boot'; import { useMessageLayoutItems } from '../../../hooks/useMessageLayout'; import { useMessageSpacingItems } from '../../../hooks/useMessageSpacing'; +import { SequenceCardStyle } from '../styles.css'; +import { useTauriUpdater } from '../../../hooks/useTauriUpdater'; +import { useDateFormatItems } from '../../../hooks/useDateFormat'; +import { playCallJoinSound } from '../../../utils/callSounds'; type ThemeSelectorProps = { themeNames: Record; @@ -1247,7 +1252,7 @@ function MicMeter() { return ( - + @@ -1276,7 +1281,7 @@ function MicMeter() { /> - + The green bar shows your live volume. Use this to tune the Gate Threshold. @@ -1336,8 +1341,8 @@ function Calls() { description={ - Filter background noise from your mic during calls. Browser-native uses the - built-in WebRTC suppressor (Google NSNet2). + Filter background noise from your mic during calls. Browser-native uses the built-in + WebRTC suppressor (Google NSNet2). @@ -1347,39 +1352,31 @@ function Calls() { style={{ borderBottom: '1px solid var(--lt-border-color)', paddingBottom: '4px' }} > - - Model - + Model - - CPU - + CPU - - Quality - + Quality - - Transients - + Transients {DENOISE_MODELS.map((model) => ( - {model.name} + {model.name} - {model.cpuUsage} + {model.cpuUsage} - {model.voiceQuality} + {model.voiceQuality} - {model.transients} + {model.transients} ))} @@ -1387,12 +1384,12 @@ function Calls() { {!mlSupported && ( - + ML options are not supported in this browser. {ML_DENOISE_REQUIREMENTS.map((req) => ( - + {req} ))} @@ -1400,7 +1397,7 @@ function Calls() { )} {callNoiseSuppression === 'ml' && ( - + Note: Applying changes requires rejoining the call. )} @@ -1471,11 +1468,9 @@ function Calls() { {callDenoiseGate && ( - - Gate Threshold - - {callDenoiseGateThreshold} dB - + + Gate Threshold + {callDenoiseGateThreshold} dB { ? 'browser' : 'off' : (saved.callNoiseSuppression ?? defaultSettings.callNoiseSuppression), + // Coerce any retired/unknown persisted model (e.g. 'dtln', 'deepfilternet' + // from earlier beta builds) back to the default working model. + callDenoiseModel: + saved.callDenoiseModel === 'rnnoise' || saved.callDenoiseModel === 'speex' + ? saved.callDenoiseModel + : defaultSettings.callDenoiseModel, composerToolbarButtons: { ...DEFAULT_COMPOSER_TOOLBAR, ...(saved.composerToolbarButtons ?? {}), diff --git a/src/app/utils/lotusDenoiseUtils.ts b/src/app/utils/lotusDenoiseUtils.ts index 3b0673a29..e7e5cae4c 100644 --- a/src/app/utils/lotusDenoiseUtils.ts +++ b/src/app/utils/lotusDenoiseUtils.ts @@ -15,30 +15,21 @@ export type DenoiseModel = { export const DENOISE_MODELS: DenoiseModel[] = [ { id: 'rnnoise', - name: 'RNNoise (Mozilla)', + name: 'RNNoise', description: 'Lightweight hybrid model. Best for consistent noise like fans.', cpuUsage: '< 5%', binarySize: '< 1 MB', - transients: 'Poor', - voiceQuality: 'Moderate', - }, - { - id: 'dtln', - name: 'DTLN (Balanced)', - description: 'Deep learning model with a good balance of quality and CPU.', - cpuUsage: '10-20%', - binarySize: '3-4 MB', transients: 'Good', voiceQuality: 'High', }, { - id: 'deepfilternet', - name: 'DeepFilterNet 3 (Pro)', - description: 'State-of-the-art studio quality. Removes all background noise.', - cpuUsage: '25-50%+', - binarySize: '15-20 MB', - transients: 'Excellent', - voiceQuality: 'Very High', + id: 'speex', + name: 'Speex (Legacy)', + description: 'Classic DSP noise suppressor. Minimal CPU, gentler on voice.', + cpuUsage: '< 2%', + binarySize: '< 1 MB', + transients: 'Poor', + voiceQuality: 'Moderate', }, ]; @@ -65,4 +56,3 @@ export const ML_DENOISE_REQUIREMENTS = [ 'Microphone access', '48kHz AudioContext capability', ]; - diff --git a/vite.config.js b/vite.config.js index 2ed4e26c0..eb6a53bc1 100644 --- a/vite.config.js +++ b/vite.config.js @@ -80,6 +80,10 @@ function lotusDenoise() { fs.mkdirSync(denoiseDir, { recursive: true }); const sapphi = path.resolve('node_modules/@sapphi-red/web-noise-suppressor/dist'); + // All bundled denoise assets are REQUIRED: every entry backs a model the + // UI can select (RNNoise, Speex) or the optional noise gate. A missing + // source means a partial/changed install would otherwise silently ship a + // broken ML feature (worklet 404 -> raw mic), so we fail the build instead. const assets = [ [ path.join(sapphi, 'rnnoise/workletProcessor.js'), @@ -87,49 +91,31 @@ function lotusDenoise() { ], [path.join(sapphi, 'rnnoise.wasm'), path.join(denoiseDir, 'rnnoise.wasm')], [path.join(sapphi, 'rnnoise_simd.wasm'), path.join(denoiseDir, 'rnnoise_simd.wasm')], - [ - path.join(sapphi, 'speex/workletProcessor.js'), - path.join(denoiseDir, 'speexWorklet.js'), - ], + [path.join(sapphi, 'speex/workletProcessor.js'), path.join(denoiseDir, 'speexWorklet.js')], [path.join(sapphi, 'speex.wasm'), path.join(denoiseDir, 'speex.wasm')], [ path.join(sapphi, 'noiseGate/workletProcessor.js'), path.join(denoiseDir, 'noiseGateWorklet.js'), ], - // DTLN (WorkAdventure v0.0.4) - [ - path.resolve('node_modules/@workadventure/noise-suppression/dist/audio-worklet.js'), - path.join(denoiseDir, 'dtlnWorklet.js'), - ], - [ - path.resolve('node_modules/@workadventure/noise-suppression/dist/assets/audio-worklet-processor.js'), - path.join(denoiseDir, 'dtlnProcessor.js'), - ], - [ - path.resolve('node_modules/@workadventure/noise-suppression/dist/vendor/litert/litert_wasm_internal.wasm'), - path.join(denoiseDir, 'litert_wasm_internal.wasm'), - ], - [ - path.resolve('node_modules/@workadventure/noise-suppression/dist/assets/model_quant_1.tflite'), - path.join(denoiseDir, 'model_1.tflite'), - ], - [ - path.resolve('node_modules/@workadventure/noise-suppression/dist/assets/model_quant_2.tflite'), - path.join(denoiseDir, 'model_2.tflite'), - ], ]; - assets.forEach(([s, d]) => { - if (fs.existsSync(s)) { - fs.copyFileSync(s, d); - } else { - // eslint-disable-next-line no-console - console.warn(`[lotus-denoise] Asset missing, will be populated by CI: ${s}`); - } - }); + const missing = assets.filter(([s]) => !fs.existsSync(s)).map(([s]) => s); + if (missing.length > 0) { + throw new Error( + `[lotus-denoise] Required denoise asset(s) missing — build aborted to avoid shipping a broken ML feature:\n ${missing.join('\n ')}`, + ); + } + assets.forEach(([s, d]) => fs.copyFileSync(s, d)); const shimSrc = path.resolve('build/lotus-denoise.js'); - if (fs.existsSync(shimSrc)) fs.copyFileSync(shimSrc, path.join(ecDir, 'lotus-denoise.js')); + if (!fs.existsSync(shimSrc)) { + throw new Error(`[lotus-denoise] Missing shim source ${shimSrc} — build aborted.`); + } + fs.copyFileSync(shimSrc, path.join(ecDir, 'lotus-denoise.js')); + // Inject the shim