5d5f5f4516
Implement a flexible, multi-model noise suppression pipeline for Element Call/LiveKit integration: - ML Engines: Added support for RNNoise, Speex, DTLN, and DeepFilterNet 3 models. - Pipeline Architecture: Implemented modular audio processing in lotus-denoise.js, supporting 'Series Suppression' (running browser-native NSNet2 before ML) and a hardware-style Noise Gate. - UI & UX Enhancements: - Settings UI: Added model comparison chart with CPU/Quality metadata. - Tuning: Added Live Microphone Meter for calibrating Noise Gate thresholds. - Reporting: Added LotusToast system to alert users when ML suppression fails or falls back to raw input. - Robustness & Quality: - Capture Fidelity: Removed forced 48kHz capture constraints to allow native-rate capture (solving static issues with high-end audio interfaces). - Performance: Added WASM SIMD detection with transparent fallback. - Capability Detection: Added browser feature detection to disable unsupported ML modes. - Build Integration: Updated Vite config to self-host all model WASM/tflite assets in /denoise/ directory.
269 lines
8.8 KiB
JavaScript
269 lines
8.8 KiB
JavaScript
import { defineConfig } from 'vite';
|
|
import react from '@vitejs/plugin-react';
|
|
import { sentryVitePlugin } from '@sentry/vite-plugin';
|
|
import { wasm } from '@rollup/plugin-wasm';
|
|
import inject from '@rollup/plugin-inject';
|
|
import { viteStaticCopy } from 'vite-plugin-static-copy';
|
|
import { vanillaExtractPlugin } from '@vanilla-extract/vite-plugin';
|
|
import { VitePWA } from 'vite-plugin-pwa';
|
|
import fs from 'fs';
|
|
import path from 'path';
|
|
import buildConfig from './build.config';
|
|
|
|
const copyFiles = {
|
|
targets: [
|
|
{
|
|
// Element Call's dist must land flat in public/element-call/ so the call
|
|
// widget URL (/public/element-call/index.html) resolves. v4.x of
|
|
// vite-plugin-static-copy preserves the full source path under dest, so
|
|
// we strip the 4 leading segments of the source base
|
|
// (node_modules/@element-hq/element-call-embedded/dist) — mirroring the
|
|
// stripBase pattern used by the android/locales targets below. The old
|
|
// `rename: 'element-call'` form silently produced
|
|
// public/node_modules/.../dist/ under v4.x, 404ing the widget (calls
|
|
// broke on cinny-desktop; web only worked because its deployed copy was
|
|
// a stale artifact from before the vite-plugin-static-copy v4 bump).
|
|
src: 'node_modules/@element-hq/element-call-embedded/dist',
|
|
dest: 'public/element-call',
|
|
rename: { stripBase: 4 },
|
|
},
|
|
{
|
|
src: 'config.json',
|
|
dest: '',
|
|
},
|
|
{
|
|
src: 'public/manifest.json',
|
|
dest: '',
|
|
rename: { stripBase: true },
|
|
},
|
|
{
|
|
src: 'public/res/android',
|
|
dest: 'public/',
|
|
rename: { stripBase: 2 },
|
|
},
|
|
{
|
|
src: 'public/locales',
|
|
dest: 'public/',
|
|
rename: { stripBase: 1 },
|
|
},
|
|
{
|
|
src: 'public/fonts',
|
|
dest: '',
|
|
rename: { stripBase: 1 },
|
|
},
|
|
],
|
|
};
|
|
|
|
function copyPdfWorker() {
|
|
return {
|
|
name: 'copy-pdf-worker',
|
|
closeBundle() {
|
|
const src = path.resolve('node_modules/pdfjs-dist/build/pdf.worker.min.mjs');
|
|
const dest = path.resolve('dist/pdf.worker.min.js');
|
|
if (fs.existsSync(src)) fs.copyFileSync(src, dest);
|
|
},
|
|
};
|
|
}
|
|
|
|
// Lotus ML noise suppression: ship the RNNoise worklet/wasm + our getUserMedia
|
|
// shim alongside the vendored Element Call bundle, and inject the shim <script>
|
|
// into EC's index.html so it runs before EC captures the mic. Runs after
|
|
// viteStaticCopy has populated dist/public/element-call (closeBundle order).
|
|
function lotusDenoise() {
|
|
return {
|
|
name: 'lotus-denoise',
|
|
closeBundle() {
|
|
const ecDir = path.resolve('dist/public/element-call');
|
|
if (!fs.existsSync(ecDir)) return;
|
|
|
|
const denoiseDir = path.join(ecDir, 'denoise');
|
|
fs.mkdirSync(denoiseDir, { recursive: true });
|
|
|
|
const sapphi = path.resolve('node_modules/@sapphi-red/web-noise-suppressor/dist');
|
|
const assets = [
|
|
[
|
|
path.join(sapphi, 'rnnoise/workletProcessor.js'),
|
|
path.join(denoiseDir, 'rnnoiseWorklet.js'),
|
|
],
|
|
[path.join(sapphi, 'rnnoise.wasm'), path.join(denoiseDir, 'rnnoise.wasm')],
|
|
[path.join(sapphi, 'rnnoise_simd.wasm'), path.join(denoiseDir, 'rnnoise_simd.wasm')],
|
|
[
|
|
path.join(sapphi, 'speex/workletProcessor.js'),
|
|
path.join(denoiseDir, 'speexWorklet.js'),
|
|
],
|
|
[path.join(sapphi, 'speex.wasm'), path.join(denoiseDir, 'speex.wasm')],
|
|
[
|
|
path.join(sapphi, 'noiseGate/workletProcessor.js'),
|
|
path.join(denoiseDir, 'noiseGateWorklet.js'),
|
|
],
|
|
// DTLN (WorkAdventure LiteRT implementation)
|
|
[
|
|
path.resolve('node_modules/@workadventure/noise-suppression/dist/noise-suppression-processor.js'),
|
|
path.join(denoiseDir, 'dtlnWorklet.js'),
|
|
],
|
|
[
|
|
path.resolve('node_modules/@workadventure/noise-suppression/dist/litert_wasm_internal.wasm'),
|
|
path.join(denoiseDir, 'litert_wasm_internal.wasm'),
|
|
],
|
|
[
|
|
path.resolve('node_modules/@workadventure/noise-suppression/dist/model_1.tflite'),
|
|
path.join(denoiseDir, 'model_1.tflite'),
|
|
],
|
|
[
|
|
path.resolve('node_modules/@workadventure/noise-suppression/dist/model_2.tflite'),
|
|
path.join(denoiseDir, 'model_2.tflite'),
|
|
],
|
|
];
|
|
assets.forEach(([s, d]) => {
|
|
if (fs.existsSync(s)) {
|
|
fs.copyFileSync(s, d);
|
|
} else {
|
|
// eslint-disable-next-line no-console
|
|
console.warn(`[lotus-denoise] Asset missing, will be populated by CI: ${s}`);
|
|
}
|
|
});
|
|
|
|
const shimSrc = path.resolve('build/lotus-denoise.js');
|
|
if (fs.existsSync(shimSrc)) fs.copyFileSync(shimSrc, path.join(ecDir, 'lotus-denoise.js'));
|
|
|
|
const indexPath = path.join(ecDir, 'index.html');
|
|
if (fs.existsSync(indexPath)) {
|
|
let html = fs.readFileSync(indexPath, 'utf8');
|
|
if (!html.includes('lotus-denoise.js')) {
|
|
// Classic (non-deferred) script runs before EC's deferred module entry.
|
|
html = html.replace(
|
|
/<script type="module"/,
|
|
'<script src="./lotus-denoise.js"></script><script type="module"',
|
|
);
|
|
fs.writeFileSync(indexPath, html);
|
|
}
|
|
}
|
|
},
|
|
};
|
|
}
|
|
|
|
function serverMatrixSdkCryptoWasm(wasmFilePath) {
|
|
return {
|
|
name: 'vite-plugin-serve-matrix-sdk-crypto-wasm',
|
|
configureServer(server) {
|
|
server.middlewares.use((req, res, next) => {
|
|
if (req.url === wasmFilePath) {
|
|
const resolvedPath = path.join(
|
|
path.resolve(),
|
|
'/node_modules/@matrix-org/matrix-sdk-crypto-wasm/pkg/matrix_sdk_crypto_wasm_bg.wasm',
|
|
);
|
|
|
|
if (fs.existsSync(resolvedPath)) {
|
|
res.setHeader('Content-Type', 'application/wasm');
|
|
res.setHeader('Cache-Control', 'no-cache');
|
|
|
|
const fileStream = fs.createReadStream(resolvedPath);
|
|
fileStream.pipe(res);
|
|
} else {
|
|
res.writeHead(404);
|
|
res.end('File not found');
|
|
}
|
|
} else {
|
|
next();
|
|
}
|
|
});
|
|
},
|
|
};
|
|
}
|
|
|
|
const vendorChunks = (id) => {
|
|
if (id.includes('node_modules/matrix-js-sdk')) return 'matrix-sdk';
|
|
if (id.includes('node_modules/react-dom')) return 'react-dom';
|
|
if (
|
|
id.includes('node_modules/react-router-dom') ||
|
|
id.includes('node_modules/@remix-run') ||
|
|
id.includes('node_modules/react-router/')
|
|
)
|
|
return 'router';
|
|
if (id.includes('node_modules/@tanstack')) return 'react-query';
|
|
if (id.includes('node_modules/linkify')) return 'linkify';
|
|
if (id.includes('node_modules/dompurify')) return 'dompurify';
|
|
if (id.includes('node_modules/@sentry')) return 'sentry';
|
|
if (id.includes('node_modules/i18next') || id.includes('node_modules/react-i18next'))
|
|
return 'i18n';
|
|
if (id.includes('node_modules/jotai')) return 'jotai';
|
|
if (id.includes('node_modules/immer')) return 'immer';
|
|
if (id.includes('node_modules/folds')) return 'folds';
|
|
if (id.includes('node_modules/emojibase')) return 'emojibase';
|
|
};
|
|
|
|
export default defineConfig({
|
|
appType: 'spa',
|
|
publicDir: false,
|
|
base: buildConfig.base,
|
|
server: {
|
|
port: 8080,
|
|
host: true,
|
|
fs: {
|
|
allow: ['..'],
|
|
},
|
|
},
|
|
plugins: [
|
|
serverMatrixSdkCryptoWasm('/node_modules/.vite/deps/pkg/matrix_sdk_crypto_wasm_bg.wasm'),
|
|
viteStaticCopy(copyFiles),
|
|
vanillaExtractPlugin(),
|
|
wasm(),
|
|
react(),
|
|
copyPdfWorker(),
|
|
lotusDenoise(),
|
|
...(process.env.SENTRY_AUTH_TOKEN
|
|
? [
|
|
sentryVitePlugin({
|
|
org: 'lotus-guild',
|
|
project: 'javascript-react',
|
|
authToken: process.env.SENTRY_AUTH_TOKEN,
|
|
sourcemaps: {
|
|
filesToDeleteAfterUpload: ['./dist/**/*.map'],
|
|
},
|
|
release: { name: process.env.VITE_APP_VERSION ?? 'lotus' },
|
|
telemetry: false,
|
|
}),
|
|
]
|
|
: []),
|
|
VitePWA({
|
|
srcDir: 'src',
|
|
filename: 'sw.ts',
|
|
strategies: 'injectManifest',
|
|
injectRegister: false,
|
|
manifest: false,
|
|
injectManifest: {
|
|
injectionPoint: undefined,
|
|
// codeSplitting: false is not yet supported by vite-plugin-pwa 1.3.0;
|
|
// the inlineDynamicImports deprecation warning from Vite is from pwa internal build
|
|
},
|
|
devOptions: {
|
|
enabled: true,
|
|
type: 'module',
|
|
},
|
|
}),
|
|
],
|
|
optimizeDeps: {
|
|
rolldownOptions: {
|
|
define: {
|
|
global: 'globalThis',
|
|
},
|
|
},
|
|
},
|
|
build: {
|
|
target: 'esnext',
|
|
outDir: 'dist',
|
|
sourcemap: process.env.SENTRY_AUTH_TOKEN ? 'hidden' : false,
|
|
copyPublicDir: false,
|
|
// manualChunks must be in rolldownOptions (not rollupOptions) for Vite 8 / Rolldown
|
|
rolldownOptions: {
|
|
checks: { preferBuiltinFeature: false },
|
|
output: {
|
|
manualChunks: vendorChunks,
|
|
},
|
|
},
|
|
rollupOptions: {
|
|
plugins: [inject({ Buffer: ['buffer', 'Buffer'] })],
|
|
},
|
|
},
|
|
});
|