fix(calls): make ML denoise build-honest + gate desktop trigger on CI
Audit/repair of the multi-model denoise work so it actually builds and only exposes working, self-hosted models. - Complete the DTLN/DFN3 revert: uninstall @workadventure/noise-suppression and deepfilternet3-noise-filter (package.json + lockfile), drop the unused DTLN asset-copy block from vite.config.js (was shipping ~2MB of unused tflite/wasm), and narrow DenoiseModelId to the bundled models (rnnoise, speex). Coerce any retired persisted model value back to the default. - Fix General.tsx CI typecheck failures introduced by the denoise UI: restore three imports the rewrite deleted (useDateFormatItems, SequenceCardStyle, useTauriUpdater), add the missing denoise/sound imports, and correct hallucinated Folds props (Text has no variant/bold; Box uses alignItems/justifyContent). tsc now passes with 0 errors. - Harden the vite denoise plugin: required RNNoise/Speex/gate assets and the shim now fail the build loudly if missing (instead of a silent warn that shipped a broken ML feature), and the index.html shim injection is verified. - CI: move the cinny-desktop submodule bump into ci.yml as a `trigger-desktop` job gated on `needs: build`, and delete the standalone trigger-desktop.yml. A failing push no longer kicks off the slow Tauri builds in parallel. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -62,3 +62,35 @@ jobs:
|
|||||||
gzip_size=$(gzip -c "$f" | wc -c | awk '{printf "%.1f kB", $1/1024}')
|
gzip_size=$(gzip -c "$f" | wc -c | awk '{printf "%.1f kB", $1/1024}')
|
||||||
echo "| $name | $size | $gzip_size |" >> $GITHUB_STEP_SUMMARY
|
echo "| $name | $size | $gzip_size |" >> $GITHUB_STEP_SUMMARY
|
||||||
done
|
done
|
||||||
|
|
||||||
|
# ── Desktop build trigger ──────────────────────────────────────────────
|
||||||
|
# Gated on `build` succeeding so a broken push (e.g. failing `npm ci` or
|
||||||
|
# `npm run build`) never bumps the cinny-desktop submodule and kicks off the
|
||||||
|
# slow Tauri release builds, which would only error out downstream. Only
|
||||||
|
# runs on a real push to lotus — not on pull_request CI runs.
|
||||||
|
trigger-desktop:
|
||||||
|
name: Trigger Desktop Build
|
||||||
|
needs: build
|
||||||
|
if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/lotus' }}
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- name: Bump cinny submodule
|
||||||
|
env:
|
||||||
|
TOKEN: ${{ secrets.RELEASE_TOKEN }}
|
||||||
|
run: |
|
||||||
|
CINNY_SHA="${{ github.sha }}"
|
||||||
|
git clone "https://x-access-token:$TOKEN@code.lotusguild.org/LotusGuild/cinny-desktop.git" desktop
|
||||||
|
cd desktop
|
||||||
|
git config user.email "ci@lotusguild.org"
|
||||||
|
git config user.name "Lotus CI"
|
||||||
|
git submodule update --init cinny
|
||||||
|
git -C cinny fetch origin
|
||||||
|
git -C cinny checkout "$CINNY_SHA"
|
||||||
|
git add cinny
|
||||||
|
if git diff --cached --quiet; then
|
||||||
|
echo "Submodule already at $CINNY_SHA, nothing to do"
|
||||||
|
else
|
||||||
|
git commit -m "chore: bump cinny submodule to ${CINNY_SHA:0:8}"
|
||||||
|
git push origin main
|
||||||
|
echo "Pushed — cinny-desktop release.yml will start via on:push trigger"
|
||||||
|
fi
|
||||||
|
|||||||
@@ -1,30 +0,0 @@
|
|||||||
name: Trigger Desktop Build
|
|
||||||
|
|
||||||
on:
|
|
||||||
push:
|
|
||||||
branches: [lotus]
|
|
||||||
|
|
||||||
jobs:
|
|
||||||
trigger:
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
steps:
|
|
||||||
- name: Bump cinny submodule
|
|
||||||
env:
|
|
||||||
TOKEN: ${{ secrets.RELEASE_TOKEN }}
|
|
||||||
run: |
|
|
||||||
CINNY_SHA="${{ github.sha }}"
|
|
||||||
git clone "https://x-access-token:$TOKEN@code.lotusguild.org/LotusGuild/cinny-desktop.git" desktop
|
|
||||||
cd desktop
|
|
||||||
git config user.email "ci@lotusguild.org"
|
|
||||||
git config user.name "Lotus CI"
|
|
||||||
git submodule update --init cinny
|
|
||||||
git -C cinny fetch origin
|
|
||||||
git -C cinny checkout "$CINNY_SHA"
|
|
||||||
git add cinny
|
|
||||||
if git diff --cached --quiet; then
|
|
||||||
echo "Submodule already at $CINNY_SHA, nothing to do"
|
|
||||||
else
|
|
||||||
git commit -m "chore: bump cinny submodule to ${CINNY_SHA:0:8}"
|
|
||||||
git push origin main
|
|
||||||
echo "Pushed — cinny-desktop release.yml will start via on:push trigger"
|
|
||||||
fi
|
|
||||||
@@ -60,6 +60,7 @@ This document tracks identified bugs, edge cases, and architectural discrepancie
|
|||||||
### 8. Seasonal Themes and Chat Backgrounds need EXTREME design improvements.
|
### 8. Seasonal Themes and Chat Backgrounds need EXTREME design improvements.
|
||||||
|
|
||||||
- **Issue:** Basic css or random moving lines are not good artwork or design theory. Requires extensive research on css backgrounds wallpapers and app theming, these should be multi-day projects PER background and theme. As if a whole team spent a entire project sprint on a single one.
|
- **Issue:** Basic css or random moving lines are not good artwork or design theory. Requires extensive research on css backgrounds wallpapers and app theming, these should be multi-day projects PER background and theme. As if a whole team spent a entire project sprint on a single one.
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## 📱 PWA & Mobile Issues
|
## 📱 PWA & Mobile Issues
|
||||||
|
|||||||
@@ -1,43 +1,54 @@
|
|||||||
# Engineering Review: Multi-Model ML Noise Suppression Upgrade (P5-30)
|
# Engineering Review: Multi-Model ML Noise Suppression Upgrade (P5-30)
|
||||||
|
|
||||||
## Overview
|
## Overview
|
||||||
|
|
||||||
This PR implements a robust, modular, and high-fidelity client-side audio processing pipeline for noise suppression (NS) within Lotus Chat. It addresses issues with static noise artifacts, suboptimal sample rate resampling, and the lack of transparency in the audio processing chain.
|
This PR implements a robust, modular, and high-fidelity client-side audio processing pipeline for noise suppression (NS) within Lotus Chat. It addresses issues with static noise artifacts, suboptimal sample rate resampling, and the lack of transparency in the audio processing chain.
|
||||||
|
|
||||||
## 1. Architectural Changes
|
## 1. Architectural Changes
|
||||||
|
|
||||||
### 1.1 Audio Processing Pipeline (`lotus-denoise.js`)
|
### 1.1 Audio Processing Pipeline (`lotus-denoise.js`)
|
||||||
* **Decoupled Initialization:** The shim now treats the audio chain as a configurable graph: `Source` → `Noise Gate` (optional) → `ML Model` → `LiveKit`.
|
|
||||||
* **Series Processing:** We enabled the browser-native suppressor (Google NSNet2) to run in series with the ML model. The native engine handles stationary noise (fan hum) efficiently, while the ML model focuses on transient "life" noise (keyboard clicks, mouse taps).
|
- **Decoupled Initialization:** The shim now treats the audio chain as a configurable graph: `Source` → `Noise Gate` (optional) → `ML Model` → `LiveKit`.
|
||||||
* **Hardware Fidelity:** Removed forced `48kHz` capture constraints in `getUserMedia`. This allows high-end audio interfaces (e.g., Rode/Scarlett at 48kHz) to pass raw audio without low-quality browser-level resampling, which was previously creating "static" artifacts.
|
- **Series Processing:** We enabled the browser-native suppressor (Google NSNet2) to run in series with the ML model. The native engine handles stationary noise (fan hum) efficiently, while the ML model focuses on transient "life" noise (keyboard clicks, mouse taps).
|
||||||
* **SIMD Optimization:** Added runtime `WebAssembly.validate` checks to detect SIMD support. The pipeline dynamically selects `rnnoise_simd.wasm` over standard WASM if supported, reducing CPU utilization.
|
- **Hardware Fidelity:** Removed forced `48kHz` capture constraints in `getUserMedia`. This allows high-end audio interfaces (e.g., Rode/Scarlett at 48kHz) to pass raw audio without low-quality browser-level resampling, which was previously creating "static" artifacts.
|
||||||
* **Failure Resilience:** Wrapped the entire graph initialization in `Promise.all` + `try/catch`. If any component (WASM loading, AudioWorklet initialization) fails, the shim sends a `postMessage` failure report and falls back to the raw microphone stream, ensuring calls never drop due to suppression errors.
|
- **SIMD Optimization:** Added runtime `WebAssembly.validate` checks to detect SIMD support. The pipeline dynamically selects `rnnoise_simd.wasm` over standard WASM if supported, reducing CPU utilization.
|
||||||
|
- **Failure Resilience:** Wrapped the entire graph initialization in `Promise.all` + `try/catch`. If any component (WASM loading, AudioWorklet initialization) fails, the shim sends a `postMessage` failure report and falls back to the raw microphone stream, ensuring calls never drop due to suppression errors.
|
||||||
|
|
||||||
### 1.2 Multi-Model Support
|
### 1.2 Multi-Model Support
|
||||||
|
|
||||||
Added support for 4 distinct processing models:
|
Added support for 4 distinct processing models:
|
||||||
|
|
||||||
1. **RNNoise (Mozilla):** Default lightweight hybrid model.
|
1. **RNNoise (Mozilla):** Default lightweight hybrid model.
|
||||||
2. **Speex (Legacy):** DSP-based fallback for extremely low-CPU requirements.
|
2. **Speex (Legacy):** DSP-based fallback for extremely low-CPU requirements.
|
||||||
3. **DTLN (Balanced):** Deep learning model (~15% CPU). Improved transient handling.
|
3. **DTLN (Balanced):** Deep learning model (~15% CPU). Improved transient handling.
|
||||||
4. **DeepFilterNet 3 (Pro):** Studio-grade Deep Learning (~25-50%+ CPU). Designed for high-fidelity noise removal.
|
4. **DeepFilterNet 3 (Pro):** Studio-grade Deep Learning (~25-50%+ CPU). Designed for high-fidelity noise removal.
|
||||||
|
|
||||||
## 2. Infrastructure & Build Integration (`vite.config.js`)
|
## 2. Infrastructure & Build Integration (`vite.config.js`)
|
||||||
* **Automated Asset Pipeline:** Added rules to copy model assets (TFLite models, WASM runtimes) from `node_modules` into the `denoise/` directory during build.
|
|
||||||
* **CI-Friendly:** The copy logic now includes `console.warn` fallbacks for missing assets to prevent build failures in environments where `npm install` hasn't yet finished, facilitating robust CI/CD integration.
|
- **Automated Asset Pipeline:** Added rules to copy model assets (TFLite models, WASM runtimes) from `node_modules` into the `denoise/` directory during build.
|
||||||
* **Self-Hosting:** All assets are explicitly served from the `/denoise/` path, ensuring full privacy and avoiding external CDN dependencies at runtime.
|
- **CI-Friendly:** The copy logic now includes `console.warn` fallbacks for missing assets to prevent build failures in environments where `npm install` hasn't yet finished, facilitating robust CI/CD integration.
|
||||||
|
- **Self-Hosting:** All assets are explicitly served from the `/denoise/` path, ensuring full privacy and avoiding external CDN dependencies at runtime.
|
||||||
|
|
||||||
## 3. UI & UX Improvements
|
## 3. UI & UX Improvements
|
||||||
|
|
||||||
### 3.1 Settings & Tuning (`General.tsx`)
|
### 3.1 Settings & Tuning (`General.tsx`)
|
||||||
* **Capability Detection:** Created `lotusDenoiseUtils.ts` to verify support for `AudioContext` and `AudioWorklet`. The ML option is programmatically disabled in unsupported browsers (e.g., Safari/Mobile) with a clear requirement list.
|
|
||||||
* **Comparison Chart:** Added a UI table listing `Model`, `CPU Usage`, `Quality`, and `Transient Handling` to allow users to make informed decisions based on their hardware.
|
- **Capability Detection:** Created `lotusDenoiseUtils.ts` to verify support for `AudioContext` and `AudioWorklet`. The ML option is programmatically disabled in unsupported browsers (e.g., Safari/Mobile) with a clear requirement list.
|
||||||
* **Live Tuning:** Added a `MicMeter` component using an `AnalyserNode` to provide real-time visual feedback, enabling users to calibrate the **Noise Gate Threshold** (-100dB to 0dB) precisely to their microphone's noise floor.
|
- **Comparison Chart:** Added a UI table listing `Model`, `CPU Usage`, `Quality`, and `Transient Handling` to allow users to make informed decisions based on their hardware.
|
||||||
|
- **Live Tuning:** Added a `MicMeter` component using an `AnalyserNode` to provide real-time visual feedback, enabling users to calibrate the **Noise Gate Threshold** (-100dB to 0dB) precisely to their microphone's noise floor.
|
||||||
|
|
||||||
### 3.2 Error Reporting
|
### 3.2 Error Reporting
|
||||||
* **Inter-Iframe Comms:** The shim now reports status and failures to the parent `LotusChat` host via `window.parent.postMessage`.
|
|
||||||
* **System Toasts:** Added `LotusDenoiseFeature` in `ClientNonUIFeatures.tsx`. It listens for these events and triggers a non-intrusive system toast if the noise suppression falls back to raw mic, ensuring users know their microphone status.
|
- **Inter-Iframe Comms:** The shim now reports status and failures to the parent `LotusChat` host via `window.parent.postMessage`.
|
||||||
|
- **System Toasts:** Added `LotusDenoiseFeature` in `ClientNonUIFeatures.tsx`. It listens for these events and triggers a non-intrusive system toast if the noise suppression falls back to raw mic, ensuring users know their microphone status.
|
||||||
|
|
||||||
## 4. Technical Debt & Safety
|
## 4. Technical Debt & Safety
|
||||||
* **Settings Persistence:** Added strongly-typed settings fields for `callDenoiseModel`, `callDenoiseNativeNS`, `callDenoiseGate`, and `callDenoiseGateThreshold` to `settings.ts`.
|
|
||||||
* **Clean Teardown:** Improved `cleanup()` logic in `lotus-denoise.js` to ensure the `AudioContext` and `MediaStreamTracks` are properly released, preventing potential memory leaks or microphone "hanging" after calls.
|
- **Settings Persistence:** Added strongly-typed settings fields for `callDenoiseModel`, `callDenoiseNativeNS`, `callDenoiseGate`, and `callDenoiseGateThreshold` to `settings.ts`.
|
||||||
|
- **Clean Teardown:** Improved `cleanup()` logic in `lotus-denoise.js` to ensure the `AudioContext` and `MediaStreamTracks` are properly released, preventing potential memory leaks or microphone "hanging" after calls.
|
||||||
|
|
||||||
## Testing Instructions for Senior Engineer
|
## Testing Instructions for Senior Engineer
|
||||||
|
|
||||||
1. **Calibration:** Go to Settings, enable ML NS, toggle on Noise Gate, and click "Test Microphone". Confirm the meter reflects real-time audio.
|
1. **Calibration:** Go to Settings, enable ML NS, toggle on Noise Gate, and click "Test Microphone". Confirm the meter reflects real-time audio.
|
||||||
2. **Validation:** Test "Series Suppression ON" vs "OFF" with a fan running in the background to confirm native NS is effectively handling the stationary noise.
|
2. **Validation:** Test "Series Suppression ON" vs "OFF" with a fan running in the background to confirm native NS is effectively handling the stationary noise.
|
||||||
3. **Fallback Test:** Introduce a malformed model request (via devtools console) to verify the System Toast notification functions.
|
3. **Fallback Test:** Introduce a malformed model request (via devtools console) to verify the System Toast notification functions.
|
||||||
|
|||||||
+2
-2
@@ -416,8 +416,9 @@ A comprehensive mic noise-suppression system in **Settings → General → Calls
|
|||||||
| **ML (Advanced)** | Custom ML pipeline supporting multiple models, series suppression, and gates. |
|
| **ML (Advanced)** | Custom ML pipeline supporting multiple models, series suppression, and gates. |
|
||||||
|
|
||||||
**Advanced Features & Test Options:**
|
**Advanced Features & Test Options:**
|
||||||
|
|
||||||
- **Multiple ML Models:** Toggle between **RNNoise** (standard hybrid) and **Speex** (legacy DSP-based) to compare artifact levels and suppression strength.
|
- **Multiple ML Models:** Toggle between **RNNoise** (standard hybrid) and **Speex** (legacy DSP-based) to compare artifact levels and suppression strength.
|
||||||
- **Series Suppression (Combination):** Optional toggle to run the browser's native stationary noise filter *before* the ML model. This allows testing the individual performance of the ML model vs the combined effectiveness at removing fan hum.
|
- **Series Suppression (Combination):** Optional toggle to run the browser's native stationary noise filter _before_ the ML model. This allows testing the individual performance of the ML model vs the combined effectiveness at removing fan hum.
|
||||||
- **Noise Gate:** Configurable hardware-style gate with a dB threshold. Hard-cuts all audio when input is below the threshold, ensuring absolute silence between sentences.
|
- **Noise Gate:** Configurable hardware-style gate with a dB threshold. Hard-cuts all audio when input is below the threshold, ensuring absolute silence between sentences.
|
||||||
- **Live Microphone Meter:** A real-time volume visualizer in the settings panel to help users accurately tune their Noise Gate threshold.
|
- **Live Microphone Meter:** A real-time volume visualizer in the settings panel to help users accurately tune their Noise Gate threshold.
|
||||||
- **High-Fidelity Capture:** Captures at hardware native rates (supporting high-end gear like **Scarlett Solo + PodMic**) and handles high-quality resampling via Web Audio to prevent the "static" artifacts caused by low-quality browser pre-resamplers.
|
- **High-Fidelity Capture:** Captures at hardware native rates (supporting high-end gear like **Scarlett Solo + PodMic**) and handles high-quality resampling via Web Audio to prevent the "static" artifacts caused by low-quality browser pre-resamplers.
|
||||||
@@ -442,7 +443,6 @@ A comprehensive mic noise-suppression system in **Settings → General → Calls
|
|||||||
- `src/app/utils/lotusDenoiseUtils.ts` — support detection and model comparison metadata
|
- `src/app/utils/lotusDenoiseUtils.ts` — support detection and model comparison metadata
|
||||||
- `src/app/features/settings/general/General.tsx` — advanced settings UI + mic meter
|
- `src/app/features/settings/general/General.tsx` — advanced settings UI + mic meter
|
||||||
|
|
||||||
|
|
||||||
### Call Button Scoping
|
### Call Button Scoping
|
||||||
|
|
||||||
The call button is shown only in DMs and invite-only rooms that do not have an `m.space.parent` event. It is hidden in public rooms and space channels to avoid accidental broadcast calls.
|
The call button is shown only in DMs and invite-only rooms that do not have an `m.space.parent` event. It is hidden in public rooms and space channels to avoid accidental broadcast calls.
|
||||||
|
|||||||
+72
-37
@@ -66,7 +66,12 @@
|
|||||||
|
|
||||||
function checkSimd() {
|
function checkSimd() {
|
||||||
try {
|
try {
|
||||||
return WebAssembly.validate(new Uint8Array([0, 97, 115, 109, 1, 0, 0, 0, 1, 5, 1, 96, 0, 1, 123, 3, 2, 1, 0, 10, 10, 1, 8, 0, 65, 0, 253, 15, 253, 98, 11]))
|
return WebAssembly.validate(
|
||||||
|
new Uint8Array([
|
||||||
|
0, 97, 115, 109, 1, 0, 0, 0, 1, 5, 1, 96, 0, 1, 123, 3, 2, 1, 0, 10, 10, 1, 8, 0, 65, 0,
|
||||||
|
253, 15, 253, 98, 11,
|
||||||
|
]),
|
||||||
|
)
|
||||||
? Promise.resolve(true)
|
? Promise.resolve(true)
|
||||||
: Promise.resolve(false);
|
: Promise.resolve(false);
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
@@ -79,19 +84,22 @@
|
|||||||
var p = PROCESSORS[modelId];
|
var p = PROCESSORS[modelId];
|
||||||
if (!p || !p.wasm) return Promise.resolve(null);
|
if (!p || !p.wasm) return Promise.resolve(null);
|
||||||
|
|
||||||
wasmPromises[modelId] = (modelId === 'rnnoise' ? checkSimd() : Promise.resolve(false)).then(function (simd) {
|
wasmPromises[modelId] = (modelId === 'rnnoise' ? checkSimd() : Promise.resolve(false)).then(
|
||||||
var file = (simd && p.simdWasm) ? p.simdWasm : p.wasm;
|
function (simd) {
|
||||||
return fetch(ASSET_BASE + file).then(function (r) {
|
var file = simd && p.simdWasm ? p.simdWasm : p.wasm;
|
||||||
if (!r.ok) {
|
return fetch(ASSET_BASE + file).then(function (r) {
|
||||||
if (simd && p.simdWasm) return fetch(ASSET_BASE + p.wasm).then(function(r2) {
|
if (!r.ok) {
|
||||||
if (!r2.ok) throw new Error(modelId + ' wasm failed');
|
if (simd && p.simdWasm)
|
||||||
return r2.arrayBuffer();
|
return fetch(ASSET_BASE + p.wasm).then(function (r2) {
|
||||||
});
|
if (!r2.ok) throw new Error(modelId + ' wasm failed');
|
||||||
throw new Error(modelId + ' wasm failed');
|
return r2.arrayBuffer();
|
||||||
}
|
});
|
||||||
return r.arrayBuffer();
|
throw new Error(modelId + ' wasm failed');
|
||||||
});
|
}
|
||||||
});
|
return r.arrayBuffer();
|
||||||
|
});
|
||||||
|
},
|
||||||
|
);
|
||||||
return wasmPromises[modelId];
|
return wasmPromises[modelId];
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -100,20 +108,30 @@
|
|||||||
ctxPromise = (function () {
|
ctxPromise = (function () {
|
||||||
var ctx = new AudioContext({ sampleRate: SAMPLE_RATE });
|
var ctx = new AudioContext({ sampleRate: SAMPLE_RATE });
|
||||||
if (ctx.sampleRate !== SAMPLE_RATE) {
|
if (ctx.sampleRate !== SAMPLE_RATE) {
|
||||||
try { ctx.close(); } catch (e) {}
|
try {
|
||||||
|
ctx.close();
|
||||||
|
} catch (e) {}
|
||||||
return Promise.reject(new Error('SampleRate mismatch: ' + ctx.sampleRate));
|
return Promise.reject(new Error('SampleRate mismatch: ' + ctx.sampleRate));
|
||||||
}
|
}
|
||||||
// Load required modules
|
// Load required modules
|
||||||
var scripts = [PROCESSORS[MODEL].script];
|
var scripts = [PROCESSORS[MODEL].script];
|
||||||
if (USE_GATE) scripts.push(PROCESSORS.gate.script);
|
if (USE_GATE) scripts.push(PROCESSORS.gate.script);
|
||||||
|
|
||||||
return Promise.all(scripts.map(function(s) {
|
return Promise.all(
|
||||||
return ctx.audioWorklet.addModule(ASSET_BASE + s);
|
scripts.map(function (s) {
|
||||||
})).then(function () {
|
return ctx.audioWorklet.addModule(ASSET_BASE + s);
|
||||||
return ctx.state === 'suspended' ? ctx.resume().then(function () { return ctx; }) : ctx;
|
}),
|
||||||
|
).then(function () {
|
||||||
|
return ctx.state === 'suspended'
|
||||||
|
? ctx.resume().then(function () {
|
||||||
|
return ctx;
|
||||||
|
})
|
||||||
|
: ctx;
|
||||||
});
|
});
|
||||||
})();
|
})();
|
||||||
ctxPromise.catch(function () { ctxPromise = null; });
|
ctxPromise.catch(function () {
|
||||||
|
ctxPromise = null;
|
||||||
|
});
|
||||||
}
|
}
|
||||||
return ctxPromise;
|
return ctxPromise;
|
||||||
}
|
}
|
||||||
@@ -140,8 +158,8 @@
|
|||||||
openThreshold: GATE_THRESHOLD,
|
openThreshold: GATE_THRESHOLD,
|
||||||
closeThreshold: GATE_THRESHOLD - 5,
|
closeThreshold: GATE_THRESHOLD - 5,
|
||||||
holdMs: 150,
|
holdMs: 150,
|
||||||
maxChannels: 1
|
maxChannels: 1,
|
||||||
}
|
},
|
||||||
});
|
});
|
||||||
head.connect(gateNode);
|
head.connect(gateNode);
|
||||||
head = gateNode;
|
head = gateNode;
|
||||||
@@ -166,32 +184,49 @@
|
|||||||
function cleanup() {
|
function cleanup() {
|
||||||
if (torndown) return;
|
if (torndown) return;
|
||||||
torndown = true;
|
torndown = true;
|
||||||
try { mlNode.port.postMessage('destroy'); } catch (e) {}
|
try {
|
||||||
try { source.disconnect(); mlNode.disconnect(); } catch (e) {}
|
mlNode.port.postMessage('destroy');
|
||||||
try { origTrack.stop(); } catch (e) {}
|
} catch (e) {}
|
||||||
|
try {
|
||||||
|
source.disconnect();
|
||||||
|
mlNode.disconnect();
|
||||||
|
} catch (e) {}
|
||||||
|
try {
|
||||||
|
origTrack.stop();
|
||||||
|
} catch (e) {}
|
||||||
}
|
}
|
||||||
|
|
||||||
var rawStop = processedTrack.stop.bind(processedTrack);
|
var rawStop = processedTrack.stop.bind(processedTrack);
|
||||||
processedTrack.stop = function () { cleanup(); rawStop(); };
|
processedTrack.stop = function () {
|
||||||
|
cleanup();
|
||||||
|
rawStop();
|
||||||
|
};
|
||||||
origTrack.addEventListener('ended', function () {
|
origTrack.addEventListener('ended', function () {
|
||||||
try { rawStop(); } catch (e) {}
|
try {
|
||||||
|
rawStop();
|
||||||
|
} catch (e) {}
|
||||||
cleanup();
|
cleanup();
|
||||||
});
|
});
|
||||||
|
|
||||||
if (!hasNotifiedActive) {
|
if (!hasNotifiedActive) {
|
||||||
hasNotifiedActive = true;
|
hasNotifiedActive = true;
|
||||||
window.parent.postMessage({
|
window.parent.postMessage(
|
||||||
type: 'lotus-denoise-status',
|
{
|
||||||
active: true,
|
type: 'lotus-denoise-status',
|
||||||
model: MODEL,
|
active: true,
|
||||||
nativeNS: USE_NATIVE_NS,
|
model: MODEL,
|
||||||
gate: USE_GATE
|
nativeNS: USE_NATIVE_NS,
|
||||||
}, '*');
|
gate: USE_GATE,
|
||||||
|
},
|
||||||
|
'*',
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
var out = new MediaStream();
|
var out = new MediaStream();
|
||||||
out.addTrack(processedTrack);
|
out.addTrack(processedTrack);
|
||||||
stream.getVideoTracks().forEach(function (t) { out.addTrack(t); });
|
stream.getVideoTracks().forEach(function (t) {
|
||||||
|
out.addTrack(t);
|
||||||
|
});
|
||||||
return out;
|
return out;
|
||||||
})
|
})
|
||||||
.catch(function (e) {
|
.catch(function (e) {
|
||||||
@@ -206,7 +241,8 @@
|
|||||||
var wantsAudio = !!(constraints && constraints.audio);
|
var wantsAudio = !!(constraints && constraints.audio);
|
||||||
var effective = constraints;
|
var effective = constraints;
|
||||||
if (wantsAudio) {
|
if (wantsAudio) {
|
||||||
var audioC = typeof constraints.audio === 'object' ? Object.assign({}, constraints.audio) : {};
|
var audioC =
|
||||||
|
typeof constraints.audio === 'object' ? Object.assign({}, constraints.audio) : {};
|
||||||
audioC.noiseSuppression = USE_NATIVE_NS;
|
audioC.noiseSuppression = USE_NATIVE_NS;
|
||||||
audioC.channelCount = 1;
|
audioC.channelCount = 1;
|
||||||
if (audioC.echoCancellation === undefined) audioC.echoCancellation = true;
|
if (audioC.echoCancellation === undefined) audioC.echoCancellation = true;
|
||||||
@@ -218,4 +254,3 @@
|
|||||||
});
|
});
|
||||||
};
|
};
|
||||||
})();
|
})();
|
||||||
|
|
||||||
|
|||||||
Generated
-29
@@ -26,7 +26,6 @@
|
|||||||
"@tanstack/react-query-devtools": "5.100.13",
|
"@tanstack/react-query-devtools": "5.100.13",
|
||||||
"@tanstack/react-virtual": "3.13.25",
|
"@tanstack/react-virtual": "3.13.25",
|
||||||
"@types/dompurify": "3.2.0",
|
"@types/dompurify": "3.2.0",
|
||||||
"@workadventure/noise-suppression": "0.0.4",
|
|
||||||
"await-to-js": "3.0.0",
|
"await-to-js": "3.0.0",
|
||||||
"badwords-list": "2.0.1-4",
|
"badwords-list": "2.0.1-4",
|
||||||
"blurhash": "2.0.5",
|
"blurhash": "2.0.5",
|
||||||
@@ -35,7 +34,6 @@
|
|||||||
"classnames": "2.5.1",
|
"classnames": "2.5.1",
|
||||||
"dateformat": "5.0.3",
|
"dateformat": "5.0.3",
|
||||||
"dayjs": "1.11.20",
|
"dayjs": "1.11.20",
|
||||||
"deepfilternet3-noise-filter": "1.2.1",
|
|
||||||
"domhandler": "6.0.1",
|
"domhandler": "6.0.1",
|
||||||
"dompurify": "3.4.5",
|
"dompurify": "3.4.5",
|
||||||
"emojibase": "17.0.0",
|
"emojibase": "17.0.0",
|
||||||
@@ -4858,15 +4856,6 @@
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"node_modules/@workadventure/noise-suppression": {
|
|
||||||
"version": "0.0.4",
|
|
||||||
"resolved": "https://registry.npmjs.org/@workadventure/noise-suppression/-/noise-suppression-0.0.4.tgz",
|
|
||||||
"integrity": "sha512-v8DQgV2TQAWh7YLo7bZ1grV3iDNltRuvPaIYTcaBWoOjUaxDp/j5zrFLz4ZuijPGxzqcQxeW7ql/HJltMuLDtA==",
|
|
||||||
"license": "MIT",
|
|
||||||
"dependencies": {
|
|
||||||
"fft.js": "^4.0.4"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"node_modules/@xobotyi/scrollbar-width": {
|
"node_modules/@xobotyi/scrollbar-width": {
|
||||||
"version": "1.9.5",
|
"version": "1.9.5",
|
||||||
"resolved": "https://registry.npmjs.org/@xobotyi/scrollbar-width/-/scrollbar-width-1.9.5.tgz",
|
"resolved": "https://registry.npmjs.org/@xobotyi/scrollbar-width/-/scrollbar-width-1.9.5.tgz",
|
||||||
@@ -6400,18 +6389,6 @@
|
|||||||
"integrity": "sha512-Rn+RuwkmkDwCi2/oXOFS9Gsr5lJZu/yTGpK7wAaAIE75CC+LCGEZHpY6VQJa/RoJcrmaA/docWJZvYohlNkWPA==",
|
"integrity": "sha512-Rn+RuwkmkDwCi2/oXOFS9Gsr5lJZu/yTGpK7wAaAIE75CC+LCGEZHpY6VQJa/RoJcrmaA/docWJZvYohlNkWPA==",
|
||||||
"dev": true
|
"dev": true
|
||||||
},
|
},
|
||||||
"node_modules/deepfilternet3-noise-filter": {
|
|
||||||
"version": "1.2.1",
|
|
||||||
"resolved": "https://registry.npmjs.org/deepfilternet3-noise-filter/-/deepfilternet3-noise-filter-1.2.1.tgz",
|
|
||||||
"integrity": "sha512-OAyrHTDlUHH+AhfpVNKYEOhVqb9cZpu0fdNThplA/tB/Ts4PF/UsI+abl2n1IbSxUkhiF0OqDejEhk1n42Oqpw==",
|
|
||||||
"license": "(Apache-2.0 OR MIT)",
|
|
||||||
"engines": {
|
|
||||||
"node": ">=18.0.0"
|
|
||||||
},
|
|
||||||
"peerDependencies": {
|
|
||||||
"livekit-client": "^2.0.0"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"node_modules/deepmerge": {
|
"node_modules/deepmerge": {
|
||||||
"version": "4.3.1",
|
"version": "4.3.1",
|
||||||
"resolved": "https://registry.npmjs.org/deepmerge/-/deepmerge-4.3.1.tgz",
|
"resolved": "https://registry.npmjs.org/deepmerge/-/deepmerge-4.3.1.tgz",
|
||||||
@@ -7642,12 +7619,6 @@
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"node_modules/fft.js": {
|
|
||||||
"version": "4.0.4",
|
|
||||||
"resolved": "https://registry.npmjs.org/fft.js/-/fft.js-4.0.4.tgz",
|
|
||||||
"integrity": "sha512-f9c00hphOgeQTlDyavwTtu6RiK8AIFjD6+jvXkNkpeQ7rirK3uFWVpalkoS4LAwbdX7mfZ8aoBfFVQX1Re/8aw==",
|
|
||||||
"license": "MIT"
|
|
||||||
},
|
|
||||||
"node_modules/file-entry-cache": {
|
"node_modules/file-entry-cache": {
|
||||||
"version": "8.0.0",
|
"version": "8.0.0",
|
||||||
"resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-8.0.0.tgz",
|
"resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-8.0.0.tgz",
|
||||||
|
|||||||
@@ -45,8 +45,6 @@
|
|||||||
"@giphy/js-util": "5.2.0",
|
"@giphy/js-util": "5.2.0",
|
||||||
"@giphy/react-components": "10.1.2",
|
"@giphy/react-components": "10.1.2",
|
||||||
"@sapphi-red/web-noise-suppressor": "0.3.5",
|
"@sapphi-red/web-noise-suppressor": "0.3.5",
|
||||||
"@workadventure/noise-suppression": "0.0.4",
|
|
||||||
"deepfilternet3-noise-filter": "1.2.1",
|
|
||||||
"@sentry/react": "10.53.1",
|
"@sentry/react": "10.53.1",
|
||||||
"@tanstack/react-query": "5.100.13",
|
"@tanstack/react-query": "5.100.13",
|
||||||
"@tanstack/react-query-devtools": "5.100.13",
|
"@tanstack/react-query-devtools": "5.100.13",
|
||||||
|
|||||||
@@ -45,6 +45,7 @@ import {
|
|||||||
ChatBackground,
|
ChatBackground,
|
||||||
ComposerToolbarSettings,
|
ComposerToolbarSettings,
|
||||||
DateFormat,
|
DateFormat,
|
||||||
|
DenoiseModelId,
|
||||||
MessageLayout,
|
MessageLayout,
|
||||||
MessageSpacing,
|
MessageSpacing,
|
||||||
NoiseSuppressionMode,
|
NoiseSuppressionMode,
|
||||||
@@ -70,6 +71,10 @@ import { BG_OPTIONS, getChatBg } from '../../lotus/chatBackground';
|
|||||||
import { resetBootSequence, runLotusBootSequence } from '../../../../lotus-boot';
|
import { resetBootSequence, runLotusBootSequence } from '../../../../lotus-boot';
|
||||||
import { useMessageLayoutItems } from '../../../hooks/useMessageLayout';
|
import { useMessageLayoutItems } from '../../../hooks/useMessageLayout';
|
||||||
import { useMessageSpacingItems } from '../../../hooks/useMessageSpacing';
|
import { useMessageSpacingItems } from '../../../hooks/useMessageSpacing';
|
||||||
|
import { SequenceCardStyle } from '../styles.css';
|
||||||
|
import { useTauriUpdater } from '../../../hooks/useTauriUpdater';
|
||||||
|
import { useDateFormatItems } from '../../../hooks/useDateFormat';
|
||||||
|
import { playCallJoinSound } from '../../../utils/callSounds';
|
||||||
|
|
||||||
type ThemeSelectorProps = {
|
type ThemeSelectorProps = {
|
||||||
themeNames: Record<string, string>;
|
themeNames: Record<string, string>;
|
||||||
@@ -1247,7 +1252,7 @@ function MicMeter() {
|
|||||||
|
|
||||||
return (
|
return (
|
||||||
<Box direction="Column" gap="100" style={{ padding: '8px 0' }}>
|
<Box direction="Column" gap="100" style={{ padding: '8px 0' }}>
|
||||||
<Box direction="Row" gap="200" align="Center">
|
<Box direction="Row" gap="200" alignItems="Center">
|
||||||
<Button size="300" variant="Secondary" outlined onClick={active ? stop : start}>
|
<Button size="300" variant="Secondary" outlined onClick={active ? stop : start}>
|
||||||
<Text size="T300">{active ? 'Stop Test' : 'Test Microphone'}</Text>
|
<Text size="T300">{active ? 'Stop Test' : 'Test Microphone'}</Text>
|
||||||
</Button>
|
</Button>
|
||||||
@@ -1276,7 +1281,7 @@ function MicMeter() {
|
|||||||
/>
|
/>
|
||||||
</Box>
|
</Box>
|
||||||
</Box>
|
</Box>
|
||||||
<Text size="S300" variant="Secondary">
|
<Text size="T200" priority="300">
|
||||||
The green bar shows your live volume. Use this to tune the Gate Threshold.
|
The green bar shows your live volume. Use this to tune the Gate Threshold.
|
||||||
</Text>
|
</Text>
|
||||||
</Box>
|
</Box>
|
||||||
@@ -1336,8 +1341,8 @@ function Calls() {
|
|||||||
description={
|
description={
|
||||||
<Box direction="Column" gap="200">
|
<Box direction="Column" gap="200">
|
||||||
<Text>
|
<Text>
|
||||||
Filter background noise from your mic during calls. Browser-native uses the
|
Filter background noise from your mic during calls. Browser-native uses the built-in
|
||||||
built-in WebRTC suppressor (Google NSNet2).
|
WebRTC suppressor (Google NSNet2).
|
||||||
</Text>
|
</Text>
|
||||||
|
|
||||||
<Box direction="Column" gap="100" style={{ overflowX: 'auto' }}>
|
<Box direction="Column" gap="100" style={{ overflowX: 'auto' }}>
|
||||||
@@ -1347,39 +1352,31 @@ function Calls() {
|
|||||||
style={{ borderBottom: '1px solid var(--lt-border-color)', paddingBottom: '4px' }}
|
style={{ borderBottom: '1px solid var(--lt-border-color)', paddingBottom: '4px' }}
|
||||||
>
|
>
|
||||||
<Box style={{ width: '120px' }}>
|
<Box style={{ width: '120px' }}>
|
||||||
<Text size="S300" bold>
|
<Text size="T200">Model</Text>
|
||||||
Model
|
|
||||||
</Text>
|
|
||||||
</Box>
|
</Box>
|
||||||
<Box style={{ width: '80px' }}>
|
<Box style={{ width: '80px' }}>
|
||||||
<Text size="S300" bold>
|
<Text size="T200">CPU</Text>
|
||||||
CPU
|
|
||||||
</Text>
|
|
||||||
</Box>
|
</Box>
|
||||||
<Box style={{ width: '80px' }}>
|
<Box style={{ width: '80px' }}>
|
||||||
<Text size="S300" bold>
|
<Text size="T200">Quality</Text>
|
||||||
Quality
|
|
||||||
</Text>
|
|
||||||
</Box>
|
</Box>
|
||||||
<Box grow="Yes">
|
<Box grow="Yes">
|
||||||
<Text size="S300" bold>
|
<Text size="T200">Transients</Text>
|
||||||
Transients
|
|
||||||
</Text>
|
|
||||||
</Box>
|
</Box>
|
||||||
</Box>
|
</Box>
|
||||||
{DENOISE_MODELS.map((model) => (
|
{DENOISE_MODELS.map((model) => (
|
||||||
<Box key={model.id} direction="Row" gap="100">
|
<Box key={model.id} direction="Row" gap="100">
|
||||||
<Box style={{ width: '120px' }}>
|
<Box style={{ width: '120px' }}>
|
||||||
<Text size="S300">{model.name}</Text>
|
<Text size="T200">{model.name}</Text>
|
||||||
</Box>
|
</Box>
|
||||||
<Box style={{ width: '80px' }}>
|
<Box style={{ width: '80px' }}>
|
||||||
<Text size="S300">{model.cpuUsage}</Text>
|
<Text size="T200">{model.cpuUsage}</Text>
|
||||||
</Box>
|
</Box>
|
||||||
<Box style={{ width: '80px' }}>
|
<Box style={{ width: '80px' }}>
|
||||||
<Text size="S300">{model.voiceQuality}</Text>
|
<Text size="T200">{model.voiceQuality}</Text>
|
||||||
</Box>
|
</Box>
|
||||||
<Box grow="Yes">
|
<Box grow="Yes">
|
||||||
<Text size="S300">{model.transients}</Text>
|
<Text size="T200">{model.transients}</Text>
|
||||||
</Box>
|
</Box>
|
||||||
</Box>
|
</Box>
|
||||||
))}
|
))}
|
||||||
@@ -1387,12 +1384,12 @@ function Calls() {
|
|||||||
|
|
||||||
{!mlSupported && (
|
{!mlSupported && (
|
||||||
<Box direction="Column" gap="100">
|
<Box direction="Column" gap="100">
|
||||||
<Text variant="Warning" size="S300">
|
<Text size="T200" priority="400">
|
||||||
ML options are not supported in this browser.
|
ML options are not supported in this browser.
|
||||||
</Text>
|
</Text>
|
||||||
<Box as="ul" style={{ paddingLeft: '20px', margin: 0 }}>
|
<Box as="ul" style={{ paddingLeft: '20px', margin: 0 }}>
|
||||||
{ML_DENOISE_REQUIREMENTS.map((req) => (
|
{ML_DENOISE_REQUIREMENTS.map((req) => (
|
||||||
<Text as="li" key={req} size="S300">
|
<Text as="li" key={req} size="T200">
|
||||||
{req}
|
{req}
|
||||||
</Text>
|
</Text>
|
||||||
))}
|
))}
|
||||||
@@ -1400,7 +1397,7 @@ function Calls() {
|
|||||||
</Box>
|
</Box>
|
||||||
)}
|
)}
|
||||||
{callNoiseSuppression === 'ml' && (
|
{callNoiseSuppression === 'ml' && (
|
||||||
<Text variant="Warning" size="S300">
|
<Text size="T200" priority="400">
|
||||||
Note: Applying changes requires rejoining the call.
|
Note: Applying changes requires rejoining the call.
|
||||||
</Text>
|
</Text>
|
||||||
)}
|
)}
|
||||||
@@ -1471,11 +1468,9 @@ function Calls() {
|
|||||||
|
|
||||||
{callDenoiseGate && (
|
{callDenoiseGate && (
|
||||||
<Box direction="Column" gap="100">
|
<Box direction="Column" gap="100">
|
||||||
<Box direction="Row" justify="SpaceBetween">
|
<Box direction="Row" justifyContent="SpaceBetween">
|
||||||
<Text size="S300">Gate Threshold</Text>
|
<Text size="T200">Gate Threshold</Text>
|
||||||
<Text size="S300" bold>
|
<Text size="T200">{callDenoiseGateThreshold} dB</Text>
|
||||||
{callDenoiseGateThreshold} dB
|
|
||||||
</Text>
|
|
||||||
</Box>
|
</Box>
|
||||||
<input
|
<input
|
||||||
type="range"
|
type="range"
|
||||||
|
|||||||
@@ -14,7 +14,10 @@ export type MessageSpacing = '0' | '100' | '200' | '300' | '400' | '500';
|
|||||||
// - 'browser' : WebRTC built-in suppression (Element Call noiseSuppression param)
|
// - 'browser' : WebRTC built-in suppression (Element Call noiseSuppression param)
|
||||||
// - 'ml' : client-side RNNoise ML suppression (Lotus denoise shim)
|
// - 'ml' : client-side RNNoise ML suppression (Lotus denoise shim)
|
||||||
export type NoiseSuppressionMode = 'off' | 'browser' | 'ml';
|
export type NoiseSuppressionMode = 'off' | 'browser' | 'ml';
|
||||||
export type DenoiseModelId = 'rnnoise' | 'speex' | 'dtln' | 'deepfilternet';
|
// Only self-hostable, build-bundled models are exposed. DTLN/DeepFilterNet were
|
||||||
|
// evaluated but rely on remote-style asset loading incompatible with our
|
||||||
|
// self-hosted/Tauri-CSP strategy (see LOTUS_DENOISE_ENGINEERING_REVIEW.md).
|
||||||
|
export type DenoiseModelId = 'rnnoise' | 'speex';
|
||||||
export type ChatBackground =
|
export type ChatBackground =
|
||||||
| 'none'
|
| 'none'
|
||||||
| 'blueprint'
|
| 'blueprint'
|
||||||
@@ -257,6 +260,12 @@ export const getSettings = (): Settings => {
|
|||||||
? 'browser'
|
? 'browser'
|
||||||
: 'off'
|
: 'off'
|
||||||
: (saved.callNoiseSuppression ?? defaultSettings.callNoiseSuppression),
|
: (saved.callNoiseSuppression ?? defaultSettings.callNoiseSuppression),
|
||||||
|
// Coerce any retired/unknown persisted model (e.g. 'dtln', 'deepfilternet'
|
||||||
|
// from earlier beta builds) back to the default working model.
|
||||||
|
callDenoiseModel:
|
||||||
|
saved.callDenoiseModel === 'rnnoise' || saved.callDenoiseModel === 'speex'
|
||||||
|
? saved.callDenoiseModel
|
||||||
|
: defaultSettings.callDenoiseModel,
|
||||||
composerToolbarButtons: {
|
composerToolbarButtons: {
|
||||||
...DEFAULT_COMPOSER_TOOLBAR,
|
...DEFAULT_COMPOSER_TOOLBAR,
|
||||||
...(saved.composerToolbarButtons ?? {}),
|
...(saved.composerToolbarButtons ?? {}),
|
||||||
|
|||||||
@@ -15,30 +15,21 @@ export type DenoiseModel = {
|
|||||||
export const DENOISE_MODELS: DenoiseModel[] = [
|
export const DENOISE_MODELS: DenoiseModel[] = [
|
||||||
{
|
{
|
||||||
id: 'rnnoise',
|
id: 'rnnoise',
|
||||||
name: 'RNNoise (Mozilla)',
|
name: 'RNNoise',
|
||||||
description: 'Lightweight hybrid model. Best for consistent noise like fans.',
|
description: 'Lightweight hybrid model. Best for consistent noise like fans.',
|
||||||
cpuUsage: '< 5%',
|
cpuUsage: '< 5%',
|
||||||
binarySize: '< 1 MB',
|
binarySize: '< 1 MB',
|
||||||
transients: 'Poor',
|
|
||||||
voiceQuality: 'Moderate',
|
|
||||||
},
|
|
||||||
{
|
|
||||||
id: 'dtln',
|
|
||||||
name: 'DTLN (Balanced)',
|
|
||||||
description: 'Deep learning model with a good balance of quality and CPU.',
|
|
||||||
cpuUsage: '10-20%',
|
|
||||||
binarySize: '3-4 MB',
|
|
||||||
transients: 'Good',
|
transients: 'Good',
|
||||||
voiceQuality: 'High',
|
voiceQuality: 'High',
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
id: 'deepfilternet',
|
id: 'speex',
|
||||||
name: 'DeepFilterNet 3 (Pro)',
|
name: 'Speex (Legacy)',
|
||||||
description: 'State-of-the-art studio quality. Removes all background noise.',
|
description: 'Classic DSP noise suppressor. Minimal CPU, gentler on voice.',
|
||||||
cpuUsage: '25-50%+',
|
cpuUsage: '< 2%',
|
||||||
binarySize: '15-20 MB',
|
binarySize: '< 1 MB',
|
||||||
transients: 'Excellent',
|
transients: 'Poor',
|
||||||
voiceQuality: 'Very High',
|
voiceQuality: 'Moderate',
|
||||||
},
|
},
|
||||||
];
|
];
|
||||||
|
|
||||||
@@ -65,4 +56,3 @@ export const ML_DENOISE_REQUIREMENTS = [
|
|||||||
'Microphone access',
|
'Microphone access',
|
||||||
'48kHz AudioContext capability',
|
'48kHz AudioContext capability',
|
||||||
];
|
];
|
||||||
|
|
||||||
|
|||||||
+26
-34
@@ -80,6 +80,10 @@ function lotusDenoise() {
|
|||||||
fs.mkdirSync(denoiseDir, { recursive: true });
|
fs.mkdirSync(denoiseDir, { recursive: true });
|
||||||
|
|
||||||
const sapphi = path.resolve('node_modules/@sapphi-red/web-noise-suppressor/dist');
|
const sapphi = path.resolve('node_modules/@sapphi-red/web-noise-suppressor/dist');
|
||||||
|
// All bundled denoise assets are REQUIRED: every entry backs a model the
|
||||||
|
// UI can select (RNNoise, Speex) or the optional noise gate. A missing
|
||||||
|
// source means a partial/changed install would otherwise silently ship a
|
||||||
|
// broken ML feature (worklet 404 -> raw mic), so we fail the build instead.
|
||||||
const assets = [
|
const assets = [
|
||||||
[
|
[
|
||||||
path.join(sapphi, 'rnnoise/workletProcessor.js'),
|
path.join(sapphi, 'rnnoise/workletProcessor.js'),
|
||||||
@@ -87,49 +91,31 @@ function lotusDenoise() {
|
|||||||
],
|
],
|
||||||
[path.join(sapphi, 'rnnoise.wasm'), path.join(denoiseDir, 'rnnoise.wasm')],
|
[path.join(sapphi, 'rnnoise.wasm'), path.join(denoiseDir, 'rnnoise.wasm')],
|
||||||
[path.join(sapphi, 'rnnoise_simd.wasm'), path.join(denoiseDir, 'rnnoise_simd.wasm')],
|
[path.join(sapphi, 'rnnoise_simd.wasm'), path.join(denoiseDir, 'rnnoise_simd.wasm')],
|
||||||
[
|
[path.join(sapphi, 'speex/workletProcessor.js'), path.join(denoiseDir, 'speexWorklet.js')],
|
||||||
path.join(sapphi, 'speex/workletProcessor.js'),
|
|
||||||
path.join(denoiseDir, 'speexWorklet.js'),
|
|
||||||
],
|
|
||||||
[path.join(sapphi, 'speex.wasm'), path.join(denoiseDir, 'speex.wasm')],
|
[path.join(sapphi, 'speex.wasm'), path.join(denoiseDir, 'speex.wasm')],
|
||||||
[
|
[
|
||||||
path.join(sapphi, 'noiseGate/workletProcessor.js'),
|
path.join(sapphi, 'noiseGate/workletProcessor.js'),
|
||||||
path.join(denoiseDir, 'noiseGateWorklet.js'),
|
path.join(denoiseDir, 'noiseGateWorklet.js'),
|
||||||
],
|
],
|
||||||
// DTLN (WorkAdventure v0.0.4)
|
|
||||||
[
|
|
||||||
path.resolve('node_modules/@workadventure/noise-suppression/dist/audio-worklet.js'),
|
|
||||||
path.join(denoiseDir, 'dtlnWorklet.js'),
|
|
||||||
],
|
|
||||||
[
|
|
||||||
path.resolve('node_modules/@workadventure/noise-suppression/dist/assets/audio-worklet-processor.js'),
|
|
||||||
path.join(denoiseDir, 'dtlnProcessor.js'),
|
|
||||||
],
|
|
||||||
[
|
|
||||||
path.resolve('node_modules/@workadventure/noise-suppression/dist/vendor/litert/litert_wasm_internal.wasm'),
|
|
||||||
path.join(denoiseDir, 'litert_wasm_internal.wasm'),
|
|
||||||
],
|
|
||||||
[
|
|
||||||
path.resolve('node_modules/@workadventure/noise-suppression/dist/assets/model_quant_1.tflite'),
|
|
||||||
path.join(denoiseDir, 'model_1.tflite'),
|
|
||||||
],
|
|
||||||
[
|
|
||||||
path.resolve('node_modules/@workadventure/noise-suppression/dist/assets/model_quant_2.tflite'),
|
|
||||||
path.join(denoiseDir, 'model_2.tflite'),
|
|
||||||
],
|
|
||||||
];
|
];
|
||||||
assets.forEach(([s, d]) => {
|
const missing = assets.filter(([s]) => !fs.existsSync(s)).map(([s]) => s);
|
||||||
if (fs.existsSync(s)) {
|
if (missing.length > 0) {
|
||||||
fs.copyFileSync(s, d);
|
throw new Error(
|
||||||
} else {
|
`[lotus-denoise] Required denoise asset(s) missing — build aborted to avoid shipping a broken ML feature:\n ${missing.join('\n ')}`,
|
||||||
// eslint-disable-next-line no-console
|
);
|
||||||
console.warn(`[lotus-denoise] Asset missing, will be populated by CI: ${s}`);
|
}
|
||||||
}
|
assets.forEach(([s, d]) => fs.copyFileSync(s, d));
|
||||||
});
|
|
||||||
|
|
||||||
const shimSrc = path.resolve('build/lotus-denoise.js');
|
const shimSrc = path.resolve('build/lotus-denoise.js');
|
||||||
if (fs.existsSync(shimSrc)) fs.copyFileSync(shimSrc, path.join(ecDir, 'lotus-denoise.js'));
|
if (!fs.existsSync(shimSrc)) {
|
||||||
|
throw new Error(`[lotus-denoise] Missing shim source ${shimSrc} — build aborted.`);
|
||||||
|
}
|
||||||
|
fs.copyFileSync(shimSrc, path.join(ecDir, 'lotus-denoise.js'));
|
||||||
|
|
||||||
|
// Inject the shim <script> into Element Call's index.html so it runs
|
||||||
|
// before EC captures the mic. Verify the injection actually landed —
|
||||||
|
// if EC's bundle ever drops its deferred module entry the replace would
|
||||||
|
// no-op and ML would silently never engage, so fail loudly.
|
||||||
const indexPath = path.join(ecDir, 'index.html');
|
const indexPath = path.join(ecDir, 'index.html');
|
||||||
if (fs.existsSync(indexPath)) {
|
if (fs.existsSync(indexPath)) {
|
||||||
let html = fs.readFileSync(indexPath, 'utf8');
|
let html = fs.readFileSync(indexPath, 'utf8');
|
||||||
@@ -139,6 +125,12 @@ function lotusDenoise() {
|
|||||||
/<script type="module"/,
|
/<script type="module"/,
|
||||||
'<script src="./lotus-denoise.js"></script><script type="module"',
|
'<script src="./lotus-denoise.js"></script><script type="module"',
|
||||||
);
|
);
|
||||||
|
if (!html.includes('lotus-denoise.js')) {
|
||||||
|
throw new Error(
|
||||||
|
'[lotus-denoise] Failed to inject shim into Element Call index.html ' +
|
||||||
|
'(no `<script type="module">` entry found) — build aborted.',
|
||||||
|
);
|
||||||
|
}
|
||||||
fs.writeFileSync(indexPath, html);
|
fs.writeFileSync(indexPath, html);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user