feat(calls): 3-tier mic noise suppression with on-device ML (P5-30)
Replace the boolean call noise-suppression setting with a 3-way control (Off / Browser-native / ML beta) in Settings -> General -> Calls. - Off: noiseSuppression=false to Element Call - Browser-native: EC's built-in WebRTC suppressor (prior default) - ML (beta): on-device RNNoise (@sapphi-red/web-noise-suppressor) Element Call captures the mic inside its iframe and publishes to LiveKit, so the host can't reach that track; LiveKit's Krisp filter is Cloud-only (we self-host the SFU) and EC's own RNNoise PR #3892 is unmerged. The ML tier instead injects a same-origin pre-init shim into the vendored EC index.html (build/lotus-denoise.js, wired by the lotusDenoise vite plugin) that patches getUserMedia and routes the captured mic through an RNNoise AudioWorklet before LiveKit sees it -- the same post-capture pipeline as #3892, with no EC fork/AGPL/rebase burden. Falls back to the raw mic if setup fails; keeps echoCancellation/AGC on the raw capture. - settings.ts: callNoiseSuppression -> 'off'|'browser'|'ml' + legacy boolean migration (true->browser, false->off) - CallEmbed/useCallEmbed: tier maps to noiseSuppression param and appends lotusDenoise=ml (native suppressor off in ML mode) - vite.config.js: copy RNNoise worklet/wasm + shim into the EC bundle and inject the shim <script> before EC's module entry - docs: LOTUS_FEATURES.md, LOTUS_TODO.md (P5-30 done) Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -42,13 +42,11 @@ import {
|
||||
DateFormat,
|
||||
MessageLayout,
|
||||
MessageSpacing,
|
||||
NoiseSuppressionMode,
|
||||
Settings,
|
||||
settingsAtom,
|
||||
} from '../../../state/settings';
|
||||
import {
|
||||
SeasonalPreview,
|
||||
SeasonTheme,
|
||||
} from '../../../components/seasonal/SeasonalEffect';
|
||||
import { SeasonalPreview, SeasonTheme } from '../../../components/seasonal/SeasonalEffect';
|
||||
import { SettingTile } from '../../../components/setting-tile';
|
||||
import { KeySymbol } from '../../../utils/key-symbol';
|
||||
import { isMacOS } from '../../../utils/user-agent';
|
||||
@@ -1235,12 +1233,16 @@ function Calls() {
|
||||
<SequenceCard className={SequenceCardStyle} variant="SurfaceVariant" direction="Column">
|
||||
<SettingTile
|
||||
title="Noise Suppression"
|
||||
description="Apply AI noise suppression to filter background noise during calls (powered by Element Call)."
|
||||
description="Filter background noise from your mic during calls. Browser-native uses the built-in WebRTC suppressor; ML runs on-device RNNoise for stronger, Krisp-style removal (higher CPU)."
|
||||
after={
|
||||
<Switch
|
||||
variant="Primary"
|
||||
<SettingsSelect<NoiseSuppressionMode>
|
||||
value={callNoiseSuppression}
|
||||
onChange={setCallNoiseSuppression}
|
||||
options={[
|
||||
{ value: 'off', label: 'Off' },
|
||||
{ value: 'browser', label: 'Browser-native' },
|
||||
{ value: 'ml', label: 'ML (beta)' },
|
||||
]}
|
||||
/>
|
||||
}
|
||||
/>
|
||||
@@ -1346,22 +1348,25 @@ function Calls() {
|
||||
);
|
||||
}
|
||||
|
||||
const SEASONAL_OPTIONS: { value: Settings['seasonalThemeOverride']; label: string; emoji: string }[] =
|
||||
[
|
||||
{ value: 'auto', label: 'Auto', emoji: '🗓' },
|
||||
{ value: 'off', label: 'Off', emoji: '×' },
|
||||
{ value: 'newyear', label: 'New Year', emoji: '🎆' },
|
||||
{ value: 'lunar', label: 'Lunar New Year', emoji: '🏮' },
|
||||
{ value: 'valentines', label: "Valentine's", emoji: '💖' },
|
||||
{ value: 'stpatricks', label: "St. Patrick's", emoji: '🍀' },
|
||||
{ value: 'aprilfools', label: 'April Fools', emoji: '?' },
|
||||
{ value: 'earthday', label: 'Earth Day', emoji: '🌱' },
|
||||
{ value: 'autumn', label: 'Autumn', emoji: '🍂' },
|
||||
{ value: 'halloween', label: 'Halloween', emoji: '🎃' },
|
||||
{ value: 'christmas', label: 'Christmas', emoji: '❄️' },
|
||||
{ value: 'arcade', label: 'Arcade Day', emoji: '👾' },
|
||||
{ value: 'deepspace', label: 'Deep Space', emoji: '🚀' },
|
||||
];
|
||||
const SEASONAL_OPTIONS: {
|
||||
value: Settings['seasonalThemeOverride'];
|
||||
label: string;
|
||||
emoji: string;
|
||||
}[] = [
|
||||
{ value: 'auto', label: 'Auto', emoji: '🗓' },
|
||||
{ value: 'off', label: 'Off', emoji: '×' },
|
||||
{ value: 'newyear', label: 'New Year', emoji: '🎆' },
|
||||
{ value: 'lunar', label: 'Lunar New Year', emoji: '🏮' },
|
||||
{ value: 'valentines', label: "Valentine's", emoji: '💖' },
|
||||
{ value: 'stpatricks', label: "St. Patrick's", emoji: '🍀' },
|
||||
{ value: 'aprilfools', label: 'April Fools', emoji: '?' },
|
||||
{ value: 'earthday', label: 'Earth Day', emoji: '🌱' },
|
||||
{ value: 'autumn', label: 'Autumn', emoji: '🍂' },
|
||||
{ value: 'halloween', label: 'Halloween', emoji: '🎃' },
|
||||
{ value: 'christmas', label: 'Christmas', emoji: '❄️' },
|
||||
{ value: 'arcade', label: 'Arcade Day', emoji: '👾' },
|
||||
{ value: 'deepspace', label: 'Deep Space', emoji: '🚀' },
|
||||
];
|
||||
|
||||
function SeasonalBgGrid({
|
||||
value,
|
||||
|
||||
@@ -15,7 +15,7 @@ import { CallControlState } from '../plugins/call/CallControlState';
|
||||
import { useCallMembersChange, useCallSession } from './useCall';
|
||||
import { CallPreferences } from '../state/callPreferences';
|
||||
import { useSetting } from '../state/hooks/settings';
|
||||
import { settingsAtom } from '../state/settings';
|
||||
import { NoiseSuppressionMode, settingsAtom } from '../state/settings';
|
||||
import { unlockCallSounds } from '../utils/callSounds';
|
||||
|
||||
const CallEmbedContext = createContext<CallEmbed | undefined>(undefined);
|
||||
@@ -45,7 +45,7 @@ export const createCallEmbed = (
|
||||
themeKind: ElementCallThemeKind,
|
||||
container: HTMLElement,
|
||||
pref?: CallPreferences,
|
||||
noiseSuppression = true,
|
||||
denoiseMode: NoiseSuppressionMode = 'browser',
|
||||
forceAudioOff = false,
|
||||
): CallEmbed => {
|
||||
const rtcSession = mx.matrixRTC.getRoomSession(room);
|
||||
@@ -59,7 +59,7 @@ export const createCallEmbed = (
|
||||
room,
|
||||
intent,
|
||||
themeKind,
|
||||
noiseSuppression,
|
||||
denoiseMode,
|
||||
initialAudio,
|
||||
initialVideo,
|
||||
);
|
||||
@@ -96,7 +96,7 @@ export const useCallStart = (dm = false) => {
|
||||
theme.kind,
|
||||
container,
|
||||
pref,
|
||||
callNoiseSuppression ?? true,
|
||||
callNoiseSuppression ?? 'browser',
|
||||
!!pttMode,
|
||||
);
|
||||
|
||||
|
||||
@@ -18,6 +18,7 @@ import {
|
||||
} from 'matrix-widget-api';
|
||||
import { CallWidgetDriver } from './CallWidgetDriver';
|
||||
import { trimTrailingSlash } from '../../utils/common';
|
||||
import { NoiseSuppressionMode } from '../../state/settings';
|
||||
import {
|
||||
ElementCallIntent,
|
||||
ElementCallThemeKind,
|
||||
@@ -100,7 +101,7 @@ export class CallEmbed {
|
||||
room: Room,
|
||||
intent: ElementCallIntent,
|
||||
themeKind: ElementCallThemeKind,
|
||||
noiseSuppression = true,
|
||||
denoiseMode: NoiseSuppressionMode = 'browser',
|
||||
initialAudio = true,
|
||||
initialVideo = false,
|
||||
): Widget {
|
||||
@@ -124,12 +125,21 @@ export class CallEmbed {
|
||||
perParticipantE2EE: room.hasEncryptionStateEvent().toString(),
|
||||
lang: 'en-EN',
|
||||
theme: themeKind,
|
||||
noiseSuppression: noiseSuppression.toString(),
|
||||
// EC's built-in WebRTC suppressor: on only for 'browser' tier. For 'ml' we
|
||||
// disable it here so RNNoise (the Lotus denoise shim) owns suppression and
|
||||
// the two don't fight each other.
|
||||
noiseSuppression: (denoiseMode === 'browser').toString(),
|
||||
audio: initialAudio.toString(),
|
||||
video: initialVideo.toString(),
|
||||
header: 'none',
|
||||
});
|
||||
|
||||
if (denoiseMode === 'ml') {
|
||||
// Signal the Lotus denoise shim (injected into the EC index.html) to route
|
||||
// the mic through the RNNoise worklet before LiveKit publishes the track.
|
||||
params.append('lotusDenoise', 'ml');
|
||||
}
|
||||
|
||||
if (CallEmbed.startingCall(intent)) {
|
||||
params.append('sendNotificationType', CallEmbed.dmCall(intent) ? 'ring' : 'notification');
|
||||
}
|
||||
|
||||
@@ -9,6 +9,11 @@ export type DateFormat =
|
||||
| 'YYYY-MM-DD'
|
||||
| '';
|
||||
export type MessageSpacing = '0' | '100' | '200' | '300' | '400' | '500';
|
||||
// Call mic noise suppression tier:
|
||||
// - 'off' : no suppression
|
||||
// - 'browser' : WebRTC built-in suppression (Element Call noiseSuppression param)
|
||||
// - 'ml' : client-side RNNoise ML suppression (Lotus denoise shim)
|
||||
export type NoiseSuppressionMode = 'off' | 'browser' | 'ml';
|
||||
export type ChatBackground =
|
||||
| 'none'
|
||||
| 'blueprint'
|
||||
@@ -109,7 +114,7 @@ export interface Settings {
|
||||
perMessageProfiles: boolean;
|
||||
|
||||
cameraOnJoin: boolean;
|
||||
callNoiseSuppression: boolean;
|
||||
callNoiseSuppression: NoiseSuppressionMode;
|
||||
pttMode: boolean;
|
||||
pttKey: string;
|
||||
|
||||
@@ -199,7 +204,7 @@ const defaultSettings: Settings = {
|
||||
perMessageProfiles: false,
|
||||
|
||||
cameraOnJoin: false,
|
||||
callNoiseSuppression: true,
|
||||
callNoiseSuppression: 'browser',
|
||||
pttMode: false,
|
||||
pttKey: 'Space',
|
||||
|
||||
@@ -235,6 +240,14 @@ export const getSettings = (): Settings => {
|
||||
return {
|
||||
...defaultSettings,
|
||||
...saved,
|
||||
// Migrate legacy boolean callNoiseSuppression -> 3-way mode:
|
||||
// true => browser-native, false => off. New string values pass through.
|
||||
callNoiseSuppression:
|
||||
typeof saved.callNoiseSuppression === 'boolean'
|
||||
? saved.callNoiseSuppression
|
||||
? 'browser'
|
||||
: 'off'
|
||||
: (saved.callNoiseSuppression ?? defaultSettings.callNoiseSuppression),
|
||||
composerToolbarButtons: {
|
||||
...DEFAULT_COMPOSER_TOOLBAR,
|
||||
...(saved.composerToolbarButtons ?? {}),
|
||||
|
||||
Reference in New Issue
Block a user