Compare commits
10
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
f793523e03 | ||
|
|
576c80a989 | ||
|
|
949103d649 | ||
|
|
9e6fe75f14 | ||
|
|
f180f76196 | ||
|
|
4ca3b0695c | ||
|
|
283b231c7a | ||
|
|
16f5f3d1a3 | ||
|
|
117467b1fb | ||
|
|
43cbd8409d |
+8
-8
@@ -64,9 +64,9 @@
|
||||
"@rolldown/plugin-babel": "^0.2.3",
|
||||
"@sentry/react": "^8.0.0",
|
||||
"@sentry/vite-plugin": "^3.0.0",
|
||||
"@storybook/addon-docs": "^10.3.6",
|
||||
"@storybook/addon-vitest": "^10.3.6",
|
||||
"@storybook/react-vite": "^10.3.6",
|
||||
"@storybook/addon-docs": "^10.6.3",
|
||||
"@storybook/addon-vitest": "^10.6.3",
|
||||
"@storybook/react-vite": "^10.6.3",
|
||||
"@testing-library/dom": "^10.1.0",
|
||||
"@testing-library/jest-dom": "^6.6.3",
|
||||
"@testing-library/react": "^16.0.0",
|
||||
@@ -86,9 +86,9 @@
|
||||
"@vector-im/compound-design-tokens": "^11.0.0",
|
||||
"@vector-im/compound-web": "^10.0.0",
|
||||
"@vitejs/plugin-react": "^6.0.2",
|
||||
"@vitest/browser-playwright": "^4.1.5",
|
||||
"@vitest/browser-playwright": "^4.1.12",
|
||||
"@vitest/coverage-v8": "^4.0.18",
|
||||
"@vitest/ui": "4.1.11",
|
||||
"@vitest/ui": "^4.1.12",
|
||||
"babel-plugin-react-compiler": "^1.0.0",
|
||||
"classnames": "^2.3.1",
|
||||
"copy-to-clipboard": "^3.3.3",
|
||||
@@ -123,18 +123,18 @@
|
||||
"react-use-measure": "^2.1.1",
|
||||
"rxjs": "^7.8.1",
|
||||
"sass": "^1.42.1",
|
||||
"storybook": "^10.3.6",
|
||||
"storybook": "^10.6.3",
|
||||
"typescript": "^5.8.3",
|
||||
"unique-names-generator": "^4.6.0",
|
||||
"uuid": "^14.0.0",
|
||||
"vaul": "^1.0.0",
|
||||
"vite": "^8.0.0",
|
||||
"vite": "^8.3.5",
|
||||
"vite-plugin-html": "^3.2.2",
|
||||
"vite-plugin-node-polyfills": "^0.28.0",
|
||||
"vite-plugin-node-stdlib-browser": "^0.2.1",
|
||||
"vite-plugin-svgr": "^4.0.0",
|
||||
"vite-plugin-wasm": "^3.6.0",
|
||||
"vitest": "^4.1.5",
|
||||
"vitest": "^4.1.12",
|
||||
"vitest-axe": "^1.0.0-pre.3"
|
||||
},
|
||||
"packageManager": "pnpm@11.21.0+sha512.521705bce689924eac72f5a3587122f362689ef6571e55ba80076fd637c11132ecffada26fad4ea79c485bfddbfd3d5a2a5b05805a77e893de71ec8a6cca3bb1"
|
||||
|
||||
@@ -71,8 +71,9 @@ widgetTest("Sharing screen in group call", async ({ addUser, browserName }) => {
|
||||
.locator('iframe[title="Element Call"]')
|
||||
.contentFrame();
|
||||
|
||||
// Expect 4 (3 + screen share) video tiles
|
||||
await expect(frame.locator("video")).toHaveCount(4, {
|
||||
// Expect 6 video tiles: 3 cameras in the grid, plus the spotlight
|
||||
// carousel's screen share and the 2 remote cameras (lotus #67)
|
||||
await expect(frame.locator("video")).toHaveCount(6, {
|
||||
timeout: 5000,
|
||||
});
|
||||
|
||||
@@ -112,8 +113,9 @@ widgetTest("Sharing screen in group call", async ({ addUser, browserName }) => {
|
||||
.locator('iframe[title="Element Call"]')
|
||||
.contentFrame();
|
||||
|
||||
// Expect 5 (2 + screen share) video tiles
|
||||
await expect(frame.locator("video")).toHaveCount(5, {
|
||||
// Expect 7 video tiles: 3 cameras in the grid, plus the spotlight
|
||||
// carousel's 2 screen shares and 2 remote cameras (lotus #67)
|
||||
await expect(frame.locator("video")).toHaveCount(7, {
|
||||
timeout: 5000,
|
||||
});
|
||||
|
||||
@@ -121,8 +123,12 @@ widgetTest("Sharing screen in group call", async ({ addUser, browserName }) => {
|
||||
frame.locator('video[data-lk-source="screen_share"]'),
|
||||
).toHaveCount(2);
|
||||
|
||||
// Expect 2 indicators at the bottom
|
||||
await expect(frame.getByTestId("screenshare-indicator")).toHaveCount(2);
|
||||
// Expect 4 indicators at the bottom: the 2 shares first, then the 2
|
||||
// remote cameras (lotus #67)
|
||||
await expect(frame.getByTestId("screenshare-indicator")).toHaveCount(4);
|
||||
await expect(
|
||||
frame.locator('[data-testid="screenshare-indicator"][data-kind="user"]'),
|
||||
).toHaveCount(2);
|
||||
|
||||
// Check the first indicator is visible
|
||||
await expect(
|
||||
|
||||
Generated
+418
-348
File diff suppressed because it is too large
Load Diff
+18
-4
@@ -28,24 +28,38 @@ overrides:
|
||||
# - https://github.com/advisories/GHSA-7r86-cg39-jmmj
|
||||
# - https://github.com/advisories/GHSA-23c5-xmqv-rm74
|
||||
"minimatch": "^10.2.3"
|
||||
# Security fixes for transitive build/test tooling (none of these ship in the
|
||||
# embedded call bundle; checked against the npm advisory database 2026-10-25).
|
||||
"tinypool": "^2.1.2"
|
||||
"@vitest/mocker": "^4.1.12"
|
||||
"brace-expansion@>=4": "^5.0.12"
|
||||
"browserslist": "^4.28.7"
|
||||
"smol-toml": "^1.8.1"
|
||||
"source-map-js": "^1.2.2"
|
||||
"postcss-selector-parser@>=7": "^7.1.6"
|
||||
"baseline-browser-mapping": "^2.11.0"
|
||||
"fflate@<0.5": "^0.4.9"
|
||||
"fflate@>=0.8": "^0.8.3"
|
||||
"pbkdf2": "^3.1.7"
|
||||
"zod@>=4": "^4.6.6"
|
||||
# Security fix: https://github.com/element-hq/element-call/security/dependabot/109
|
||||
"glob": "^10.5.0"
|
||||
# Security fixes:
|
||||
# - https://github.com/element-hq/element-call/security/dependabot/110
|
||||
# - https://github.com/element-hq/element-call/security/dependabot/122
|
||||
# - https://github.com/element-hq/element-call/security/dependabot/179
|
||||
"qs": "^6.15.2"
|
||||
"qs": "^6.16.0"
|
||||
# Security fixes:
|
||||
# - https://github.com/element-hq/element-call/security/dependabot/106
|
||||
# - https://github.com/element-hq/element-call/security/dependabot/203
|
||||
"js-yaml": "^4.2.0"
|
||||
"js-yaml": "^4.3.2"
|
||||
# Storybook declares support for 0.27.0 only but empirically works fine with 0.28.0.
|
||||
"esbuild": "^0.28.0"
|
||||
# Security fixes:
|
||||
# - https://github.com/nodejs/undici/releases/tag/v6.24.0
|
||||
# - https://github.com/element-hq/element-call/security/dependabot/201
|
||||
# - https://github.com/element-hq/element-call/security/dependabot/199
|
||||
"undici": "^6.27.0"
|
||||
"undici": "^6.29.1"
|
||||
# Security fix: https://github.com/advisories/GHSA-rf6f-7fwh-wjgh
|
||||
"flatted": "^3.4.2"
|
||||
# Security fix: https://github.com/element-hq/element-call/security/dependabot/184
|
||||
@@ -59,4 +73,4 @@ overrides:
|
||||
# - https://github.com/element-hq/element-call/security/dependabot/196
|
||||
# - https://github.com/element-hq/element-call/security/dependabot/195
|
||||
# - https://github.com/element-hq/element-call/security/dependabot/193
|
||||
"dompurify": "^3.4.11"
|
||||
"dompurify": "^3.4.17"
|
||||
|
||||
@@ -36,6 +36,7 @@ describe("LotusWidgetActions", () => {
|
||||
LotusWidgetActions.PromptScreenshare,
|
||||
LotusWidgetActions.Reconnect,
|
||||
LotusWidgetActions.DeviceDecision,
|
||||
LotusWidgetActions.SetDenoise,
|
||||
];
|
||||
|
||||
expect(new Set(LOTUS_TO_WIDGET_ACTIONS)).toEqual(new Set(expectedToWidget));
|
||||
|
||||
@@ -41,11 +41,19 @@ export enum LotusWidgetActions {
|
||||
RequestState = "io.lotus.request_state",
|
||||
/**
|
||||
* fromWidget: real state of the in-source denoise engine —
|
||||
* `{ active: boolean, model: string, error?: string }` — sent once the
|
||||
* `{ active: boolean, model: string, mode: "ml", error?: string }` (or
|
||||
* `{ active: false, mode }` after a switch away from ML) — sent once the
|
||||
* processor attaches (or after the rnnoise fallback also fails), so the host
|
||||
* toggle can reflect reality rather than the requested state.
|
||||
*/
|
||||
DenoiseState = "io.lotus.denoise_state",
|
||||
/**
|
||||
* toWidget: switch noise suppression mid-call —
|
||||
* `{ mode: "off" | "browser" | "ml", model?, nativeNS?, gate?, gateThreshold? }`.
|
||||
* `model`/`gate`/`gateThreshold` only matter for "ml"; `nativeNS` stacks the
|
||||
* browser suppressor in front of the model. Omitted fields keep their value.
|
||||
*/
|
||||
SetDenoise = "io.lotus.set_denoise",
|
||||
/**
|
||||
* fromWidget: one-shot end-of-call readout for the local participant —
|
||||
* `{ durationMs, reconnects, poorMs, verdict }` (#143) — sent on SFU
|
||||
@@ -163,4 +171,5 @@ export const LOTUS_TO_WIDGET_ACTIONS: LotusWidgetActions[] = [
|
||||
LotusWidgetActions.PromptScreenshare,
|
||||
LotusWidgetActions.Reconnect,
|
||||
LotusWidgetActions.DeviceDecision,
|
||||
LotusWidgetActions.SetDenoise,
|
||||
];
|
||||
|
||||
@@ -25,6 +25,7 @@ import {
|
||||
echoCancellationSetting,
|
||||
noiseSuppressionSetting,
|
||||
} from "../settings/settings.ts";
|
||||
import { setLotusNoiseSuppression } from "./lotusCaptureOverride.ts";
|
||||
|
||||
// [lotus] Upstream v0.25.0 moved the audio-capture constraints to Settings.
|
||||
// The Lotus host still drives them per-call via URL params (it turns browser
|
||||
@@ -83,6 +84,7 @@ beforeEach(() => {
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
setLotusNoiseSuppression(null);
|
||||
testScope.end();
|
||||
vi.mocked(LivekitRoom).mockClear();
|
||||
});
|
||||
@@ -123,6 +125,7 @@ describe("[lotus] audio-capture URL param overrides", () => {
|
||||
expect(capturedAudioDefaults()).toMatchObject({
|
||||
echoCancellation: true,
|
||||
noiseSuppression: false,
|
||||
voiceIsolation: false,
|
||||
autoGainControl: false,
|
||||
});
|
||||
});
|
||||
@@ -137,4 +140,16 @@ describe("[lotus] audio-capture URL param overrides", () => {
|
||||
createRoom();
|
||||
expect(capturedAudioDefaults()).toMatchObject({ echoCancellation: false });
|
||||
});
|
||||
|
||||
test("a mid-call set_denoise choice beats the noiseSuppression the call joined with", () => {
|
||||
getUrlParams.mockReturnValue({ noiseSuppression: false });
|
||||
setLotusNoiseSuppression(true);
|
||||
createRoom();
|
||||
expect(capturedAudioDefaults()).toMatchObject({ noiseSuppression: true });
|
||||
|
||||
getUrlParams.mockReturnValue({ noiseSuppression: true });
|
||||
setLotusNoiseSuppression(false);
|
||||
createRoom();
|
||||
expect(capturedAudioDefaults()).toMatchObject({ noiseSuppression: false });
|
||||
});
|
||||
});
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
/*
|
||||
Copyright 2026 Lotus Guild
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
/**
|
||||
* Browser noise suppression chosen live by the host (`io.lotus.set_denoise`),
|
||||
* overriding the `noiseSuppression` URL param the call was opened with. Read by
|
||||
* ConnectionFactory, so a connection created after a mid-call switch (e.g. an
|
||||
* SFU reconnect) captures the mic the way the user last picked, not the way
|
||||
* they joined. Dependency-free so ConnectionFactory can import it cheaply.
|
||||
*/
|
||||
let noiseSuppressionOverride: boolean | null = null;
|
||||
|
||||
export function setLotusNoiseSuppression(on: boolean | null): void {
|
||||
noiseSuppressionOverride = on;
|
||||
}
|
||||
|
||||
/** The override if the host set one, else the URL param (default on). */
|
||||
export function lotusNoiseSuppressionAllowed(
|
||||
urlParam: boolean | undefined,
|
||||
): boolean {
|
||||
return noiseSuppressionOverride ?? urlParam !== false;
|
||||
}
|
||||
@@ -9,7 +9,12 @@ import { afterEach, beforeEach, describe, expect, test, vi } from "vitest";
|
||||
import { ParticipantEvent, Track } from "livekit-client";
|
||||
|
||||
import { type CallViewModel } from "../state/CallViewModel/CallViewModel";
|
||||
import { startLotusDenoise } from "./lotusDenoise";
|
||||
import {
|
||||
applySetDenoise,
|
||||
browserNsFor,
|
||||
type DenoiseTarget,
|
||||
startLotusDenoise,
|
||||
} from "./lotusDenoise";
|
||||
import { LotusWidgetActions } from "./lotusActions";
|
||||
|
||||
// Track constructed instances so tests can assert exactly one processor was
|
||||
@@ -42,9 +47,25 @@ vi.mock("./lotusDenoiseProcessor", () => ({
|
||||
releasePreparedDenoiseAssets: vi.fn().mockResolvedValue(undefined),
|
||||
}));
|
||||
const hostSend = vi.hoisted(() => vi.fn().mockResolvedValue({}));
|
||||
// Host → widget actions: tests call `sendAction()` to deliver one.
|
||||
const actionHandlers = vi.hoisted(
|
||||
() => new Map<string, (ev: CustomEvent) => void>(),
|
||||
);
|
||||
vi.mock("../widget", () => ({
|
||||
widget: { api: { transport: { send: hostSend } } },
|
||||
widget: {
|
||||
api: { transport: { send: hostSend, reply: vi.fn() } },
|
||||
lazyActions: {
|
||||
on: (name: string, cb: (ev: CustomEvent) => void): void => {
|
||||
actionHandlers.set(name, cb);
|
||||
},
|
||||
off: (name: string): void => {
|
||||
actionHandlers.delete(name);
|
||||
},
|
||||
},
|
||||
},
|
||||
}));
|
||||
const sendAction = (name: string, data: unknown): void =>
|
||||
actionHandlers.get(name)?.(new CustomEvent(name, { detail: { data } }));
|
||||
|
||||
/** A promise plus externally-callable resolve, for controlling ordering. */
|
||||
function deferred<T>(): {
|
||||
@@ -110,6 +131,7 @@ beforeEach(() => {
|
||||
instances.length = 0;
|
||||
calls.length = 0;
|
||||
hostSend.mockClear();
|
||||
actionHandlers.clear();
|
||||
// `lotusParam`/`lotusFlag` cache the URL params at first read; seed the hash
|
||||
// before startLotusDenoise() so the dedicated `lotusDenoiseSource` flag reads on.
|
||||
window.location.hash = "#/room?lotusDenoiseSource=1";
|
||||
@@ -188,6 +210,7 @@ describe("startLotusDenoise", () => {
|
||||
expect(hostSend).toHaveBeenCalledWith(LotusWidgetActions.DenoiseState, {
|
||||
active: true,
|
||||
model: "rnnoise",
|
||||
mode: "ml",
|
||||
});
|
||||
});
|
||||
|
||||
@@ -217,6 +240,7 @@ describe("startLotusDenoise", () => {
|
||||
expect(hostSend).toHaveBeenCalledWith(LotusWidgetActions.DenoiseState, {
|
||||
active: true,
|
||||
model: "rnnoise",
|
||||
mode: "ml",
|
||||
});
|
||||
});
|
||||
|
||||
@@ -239,6 +263,7 @@ describe("startLotusDenoise", () => {
|
||||
active: false,
|
||||
error: "no worklet for you",
|
||||
model: "rnnoise",
|
||||
mode: "ml",
|
||||
});
|
||||
});
|
||||
|
||||
@@ -272,3 +297,157 @@ describe("startLotusDenoise", () => {
|
||||
expect(instances[0].setMicMuted).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
/** A mic whose processor and capture constraints behave like LiveKit's. */
|
||||
function liveMic(): {
|
||||
getProcessor: () => { config: { model: string } } | undefined;
|
||||
setProcessor: ReturnType<typeof vi.fn>;
|
||||
stopProcessor: ReturnType<typeof vi.fn>;
|
||||
restartTrack: ReturnType<typeof vi.fn>;
|
||||
constraints: MediaTrackConstraints;
|
||||
mediaStreamTrack: { getSettings: () => MediaTrackSettings };
|
||||
isMuted: boolean;
|
||||
} {
|
||||
let attached: { config: { model: string } } | undefined;
|
||||
return {
|
||||
isMuted: false,
|
||||
getProcessor: () => attached,
|
||||
setProcessor: vi.fn(async (p: { config: { model: string } }) => {
|
||||
await Promise.resolve();
|
||||
attached = p;
|
||||
}),
|
||||
stopProcessor: vi.fn(async () => {
|
||||
await Promise.resolve();
|
||||
attached = undefined;
|
||||
}),
|
||||
restartTrack: vi.fn().mockResolvedValue(undefined),
|
||||
constraints: { echoCancellation: true, autoGainControl: true },
|
||||
mediaStreamTrack: { getSettings: () => ({ deviceId: "mic-1" }) },
|
||||
};
|
||||
}
|
||||
|
||||
describe("io.lotus.set_denoise (switch mid-call)", () => {
|
||||
test("swaps the ML model on the live mic without re-capturing it", async () => {
|
||||
window.location.hash =
|
||||
"#/room?lotusDenoiseSource=1&lotusModel=dtln&noiseSuppression=false";
|
||||
const mic = liveMic();
|
||||
const { vm } = makeRoomAndVm(mic);
|
||||
startLotusDenoise(vm);
|
||||
await flush();
|
||||
expect(mic.getProcessor()?.config.model).toBe("dtln");
|
||||
|
||||
sendAction(LotusWidgetActions.SetDenoise, {
|
||||
mode: "ml",
|
||||
model: "deepfilternet",
|
||||
});
|
||||
await flush();
|
||||
await flush();
|
||||
|
||||
expect(mic.stopProcessor).toHaveBeenCalledTimes(1);
|
||||
expect(mic.getProcessor()?.config.model).toBe("deepfilternet");
|
||||
expect(mic.restartTrack).not.toHaveBeenCalled();
|
||||
expect(hostSend).toHaveBeenLastCalledWith(LotusWidgetActions.DenoiseState, {
|
||||
active: true,
|
||||
model: "deepfilternet",
|
||||
mode: "ml",
|
||||
});
|
||||
});
|
||||
|
||||
test("ML → browser-native drops the model and re-captures with browser NS on", async () => {
|
||||
window.location.hash =
|
||||
"#/room?lotusDenoiseSource=1&lotusModel=rnnoise&noiseSuppression=false";
|
||||
const mic = liveMic();
|
||||
const { vm } = makeRoomAndVm(mic);
|
||||
startLotusDenoise(vm);
|
||||
await flush();
|
||||
|
||||
sendAction(LotusWidgetActions.SetDenoise, { mode: "browser" });
|
||||
await flush();
|
||||
await flush();
|
||||
|
||||
expect(mic.stopProcessor).toHaveBeenCalledTimes(1);
|
||||
expect(mic.getProcessor()).toBeUndefined();
|
||||
expect(mic.restartTrack).toHaveBeenCalledWith({
|
||||
deviceId: "mic-1",
|
||||
channelCount: 1,
|
||||
echoCancellation: true,
|
||||
autoGainControl: true,
|
||||
noiseSuppression: true,
|
||||
voiceIsolation: true,
|
||||
});
|
||||
expect(hostSend).toHaveBeenLastCalledWith(LotusWidgetActions.DenoiseState, {
|
||||
active: false,
|
||||
mode: "browser",
|
||||
});
|
||||
});
|
||||
|
||||
test("browser-native → ML works even when the call joined without ML", async () => {
|
||||
window.location.hash = "#/room?noiseSuppression=true";
|
||||
const mic = liveMic();
|
||||
const { vm } = makeRoomAndVm(mic);
|
||||
startLotusDenoise(vm);
|
||||
await flush();
|
||||
expect(mic.setProcessor).not.toHaveBeenCalled();
|
||||
|
||||
sendAction(LotusWidgetActions.SetDenoise, { mode: "ml", model: "speex" });
|
||||
await flush();
|
||||
await flush();
|
||||
|
||||
expect(mic.restartTrack).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ noiseSuppression: false }),
|
||||
);
|
||||
expect(mic.getProcessor()?.config.model).toBe("speex");
|
||||
});
|
||||
|
||||
test("ignores a payload without a valid mode", async () => {
|
||||
window.location.hash = "#/room?lotusDenoiseSource=1";
|
||||
const mic = liveMic();
|
||||
const { vm } = makeRoomAndVm(mic);
|
||||
startLotusDenoise(vm);
|
||||
await flush();
|
||||
sendAction(LotusWidgetActions.SetDenoise, { mode: "loud" });
|
||||
await flush();
|
||||
expect(mic.stopProcessor).not.toHaveBeenCalled();
|
||||
expect(mic.restartTrack).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe("applySetDenoise", () => {
|
||||
const base: DenoiseTarget = {
|
||||
mode: "ml",
|
||||
nativeNS: false,
|
||||
config: {
|
||||
model: "rnnoise",
|
||||
assetBase: "https://x/denoise/",
|
||||
gate: false,
|
||||
gateThreshold: -45,
|
||||
floor: 0.15,
|
||||
},
|
||||
};
|
||||
|
||||
test("keeps fields the payload doesn't name or gets wrong", () => {
|
||||
expect(
|
||||
applySetDenoise(base, { mode: "ml", model: "nope", gate: "yes" }),
|
||||
).toEqual(base);
|
||||
});
|
||||
|
||||
test("moves only the named fields", () => {
|
||||
const next = applySetDenoise(base, {
|
||||
mode: "ml",
|
||||
model: "dtln",
|
||||
gateThreshold: -50,
|
||||
});
|
||||
expect(next?.config).toEqual({
|
||||
...base.config,
|
||||
model: "dtln",
|
||||
gateThreshold: -50,
|
||||
});
|
||||
});
|
||||
|
||||
test("browser NS is on for browser mode and for ML with nativeNS", () => {
|
||||
expect(browserNsFor({ ...base, mode: "browser" })).toBe(true);
|
||||
expect(browserNsFor({ ...base, mode: "off" })).toBe(false);
|
||||
expect(browserNsFor(base)).toBe(false);
|
||||
expect(browserNsFor({ ...base, nativeNS: true })).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
+184
-29
@@ -13,8 +13,12 @@ import {
|
||||
type TrackPublication,
|
||||
} from "livekit-client";
|
||||
import { logger } from "matrix-js-sdk/lib/logger";
|
||||
import { type IWidgetApiRequest } from "matrix-widget-api";
|
||||
|
||||
import { type CallViewModel } from "../state/CallViewModel/CallViewModel";
|
||||
import { widget } from "../widget";
|
||||
import { setLotusNoiseSuppression } from "./lotusCaptureOverride";
|
||||
import { withMicTapsReleased } from "./lotusMicLevel";
|
||||
import {
|
||||
LotusWidgetActions,
|
||||
lotusFlag,
|
||||
@@ -64,49 +68,135 @@ function safeAssetBase(raw: string | null): string {
|
||||
}
|
||||
}
|
||||
|
||||
export type LotusNoiseMode = "off" | "browser" | "ml";
|
||||
|
||||
/** What the host wants: browser suppression and/or an ML model. */
|
||||
export interface DenoiseTarget {
|
||||
mode: LotusNoiseMode;
|
||||
/** Stack the browser's suppressor in front of the model (ML only). */
|
||||
nativeNS: boolean;
|
||||
config: LotusDenoiseConfig;
|
||||
}
|
||||
|
||||
const MODELS: ReadonlySet<string> = new Set<LotusDenoiseModel>([
|
||||
"rnnoise",
|
||||
"speex",
|
||||
"dtln",
|
||||
"deepfilternet",
|
||||
]);
|
||||
const isModel = (v: unknown): v is LotusDenoiseModel =>
|
||||
typeof v === "string" && MODELS.has(v);
|
||||
|
||||
/** Whether the mic should be captured with the browser's noise suppression. */
|
||||
export const browserNsFor = (t: DenoiseTarget): boolean =>
|
||||
t.mode === "browser" || (t.mode === "ml" && t.nativeNS);
|
||||
|
||||
/**
|
||||
* Apply an `io.lotus.set_denoise` payload to the current target. Unknown or
|
||||
* malformed fields keep their current value, so a partial payload only moves
|
||||
* what it names; a payload with no valid `mode` is ignored (returns null).
|
||||
*/
|
||||
export function applySetDenoise(
|
||||
current: DenoiseTarget,
|
||||
data: unknown,
|
||||
): DenoiseTarget | null {
|
||||
const d = (data ?? {}) as Record<string, unknown>;
|
||||
if (d.mode !== "off" && d.mode !== "browser" && d.mode !== "ml") return null;
|
||||
const model = isModel(d.model) ? d.model : current.config.model;
|
||||
return {
|
||||
mode: d.mode,
|
||||
nativeNS: typeof d.nativeNS === "boolean" ? d.nativeNS : current.nativeNS,
|
||||
config: {
|
||||
...current.config,
|
||||
model,
|
||||
gate: typeof d.gate === "boolean" ? d.gate : current.config.gate,
|
||||
gateThreshold:
|
||||
typeof d.gateThreshold === "number" && Number.isFinite(d.gateThreshold)
|
||||
? d.gateThreshold
|
||||
: current.config.gateThreshold,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
const sameMlConfig = (a: LotusDenoiseConfig, b: LotusDenoiseConfig): boolean =>
|
||||
a.model === b.model &&
|
||||
a.gate === b.gate &&
|
||||
a.gateThreshold === b.gateThreshold &&
|
||||
a.floor === b.floor;
|
||||
|
||||
/**
|
||||
* Re-acquire the mic with browser noise suppression on or off, keeping the
|
||||
* rest of its constraints. LiveKit re-attaches any processor to the new track;
|
||||
* the level-meter clone is dropped first so the new constraint actually takes.
|
||||
* The deviceId must be passed: without one LiveKit requests `audio: true` and
|
||||
* silently drops every other constraint.
|
||||
*/
|
||||
async function restartCapture(
|
||||
mic: LocalAudioTrack,
|
||||
noiseSuppression: boolean,
|
||||
): Promise<void> {
|
||||
const c = mic.constraints;
|
||||
await withMicTapsReleased(async () => {
|
||||
await mic.restartTrack({
|
||||
deviceId: c.deviceId ?? mic.mediaStreamTrack.getSettings().deviceId,
|
||||
channelCount: c.channelCount ?? 1,
|
||||
echoCancellation: c.echoCancellation,
|
||||
autoGainControl: c.autoGainControl,
|
||||
noiseSuppression,
|
||||
voiceIsolation: noiseSuppression,
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
export function startLotusDenoise(vm: CallViewModel): () => void {
|
||||
// Gate on a DEDICATED flag — NOT the existing `lotusDenoise=ml` that the
|
||||
// host's build-time getUserMedia shim already uses. Otherwise simply shipping
|
||||
// this fork (while the host still injects its shim and still sets
|
||||
// lotusDenoise=ml) would denoise twice. The host opts into the in-source
|
||||
// engine with `lotusDenoiseSource=1` AND stops injecting the shim at the same
|
||||
// time. Default off ⇒ the fork is inert and behaviour is unchanged.
|
||||
if (!lotusFlag("lotusDenoiseSource")) return () => undefined;
|
||||
// time. Without the flag the engine stays idle unless the host switches to ML
|
||||
// mid-call (io.lotus.set_denoise), which needs the widget API.
|
||||
const w = widget;
|
||||
const mlAtJoin = lotusFlag("lotusDenoiseSource");
|
||||
if (!mlAtJoin && !w) return () => undefined;
|
||||
|
||||
const requested = lotusParam("lotusModel");
|
||||
const model: LotusDenoiseConfig["model"] =
|
||||
requested === "speex" ||
|
||||
requested === "dtln" ||
|
||||
requested === "deepfilternet"
|
||||
? requested
|
||||
: "rnnoise";
|
||||
const model: LotusDenoiseConfig["model"] = isModel(requested)
|
||||
? requested
|
||||
: "rnnoise";
|
||||
|
||||
const rawThreshold = lotusParam("lotusGateThreshold");
|
||||
const rawFloor = lotusParam("lotusDenoiseFloor");
|
||||
const config: LotusDenoiseConfig = {
|
||||
model,
|
||||
assetBase: safeAssetBase(lotusParam("lotusDenoiseBase")),
|
||||
gate: lotusFlag("lotusGate"),
|
||||
// Default -45 (matches the reference shim); accept an explicit 0 (don't
|
||||
// coerce it away via `|| default`).
|
||||
gateThreshold:
|
||||
rawThreshold !== null && Number.isFinite(Number(rawThreshold))
|
||||
? Number(rawThreshold)
|
||||
: -45,
|
||||
// Dry/wet attenuation floor. Default 0.15 (~-16 dB) tames the
|
||||
// over-suppression "underwater"/pumping artifact; host can tune via
|
||||
// `lotusDenoiseFloor` (0 = full suppression, no floor).
|
||||
floor:
|
||||
rawFloor !== null && Number.isFinite(Number(rawFloor))
|
||||
? Math.min(0.5, Math.max(0, Number(rawFloor)))
|
||||
: 0.15,
|
||||
const browserNsAtJoin = lotusParam("noiseSuppression") !== "false";
|
||||
// `let`: io.lotus.set_denoise replaces it mid-call.
|
||||
let target: DenoiseTarget = {
|
||||
mode: mlAtJoin ? "ml" : browserNsAtJoin ? "browser" : "off",
|
||||
nativeNS: mlAtJoin && browserNsAtJoin,
|
||||
config: {
|
||||
model,
|
||||
assetBase: safeAssetBase(lotusParam("lotusDenoiseBase")),
|
||||
gate: lotusFlag("lotusGate"),
|
||||
// Default -45 (matches the reference shim); accept an explicit 0 (don't
|
||||
// coerce it away via `|| default`).
|
||||
gateThreshold:
|
||||
rawThreshold !== null && Number.isFinite(Number(rawThreshold))
|
||||
? Number(rawThreshold)
|
||||
: -45,
|
||||
// Dry/wet attenuation floor. Default 0.15 (~-16 dB) tames the
|
||||
// over-suppression "underwater"/pumping artifact; host can tune via
|
||||
// `lotusDenoiseFloor` (0 = full suppression, no floor).
|
||||
floor:
|
||||
rawFloor !== null && Number.isFinite(Number(rawFloor))
|
||||
? Math.min(0.5, Math.max(0, Number(rawFloor)))
|
||||
: 0.15,
|
||||
},
|
||||
};
|
||||
|
||||
// [lotus #7] Start fetching wasm / creating the AudioContext / addModule-ing
|
||||
// the worklet (and loading the DFN model) NOW, before any mic track exists.
|
||||
// `setProcessor()` holds LiveKit's trackChangeLock while awaiting `init()`,
|
||||
// so anything still loading there freezes mute/unmute/device-switch.
|
||||
void prepareDenoiseAssets(config);
|
||||
if (target.mode === "ml") void prepareDenoiseAssets(target.config);
|
||||
|
||||
const micOf = (room: LivekitRoom): LocalAudioTrack | undefined =>
|
||||
room.localParticipant.getTrackPublication(Track.Source.Microphone)
|
||||
@@ -127,7 +217,7 @@ export function startLotusDenoise(vm: CallViewModel): () => void {
|
||||
mic: LocalAudioTrack,
|
||||
model: LotusDenoiseModel,
|
||||
): Promise<void> => {
|
||||
const processor = new LotusDenoiseProcessor({ ...config, model });
|
||||
const processor = new LotusDenoiseProcessor({ ...target.config, model });
|
||||
pendingProcessors.set(room, processor);
|
||||
// [lotus #9] Seed the mute state so a processor attached while already
|
||||
// muted starts suspended rather than burning inference on silence.
|
||||
@@ -142,11 +232,15 @@ export function startLotusDenoise(vm: CallViewModel): () => void {
|
||||
}
|
||||
};
|
||||
|
||||
// In-flight apply() per room, so a mid-call switch waits for it to settle.
|
||||
const inflight = new Map<LivekitRoom, Promise<void>>();
|
||||
|
||||
const apply = (room: LivekitRoom): void => {
|
||||
if (target.mode !== "ml") return;
|
||||
const mic = micOf(room);
|
||||
if (!mic || mic.getProcessor() || pendingProcessors.has(room)) return;
|
||||
void (async () => {
|
||||
let model = config.model;
|
||||
const run = (async (): Promise<void> => {
|
||||
let model = target.config.model;
|
||||
try {
|
||||
try {
|
||||
await attach(room, mic, model);
|
||||
@@ -164,6 +258,7 @@ export function startLotusDenoise(vm: CallViewModel): () => void {
|
||||
lotusSendToHost(LotusWidgetActions.DenoiseState, {
|
||||
active: true,
|
||||
model,
|
||||
mode: "ml",
|
||||
});
|
||||
} catch (e) {
|
||||
logger.warn("[lotus] denoise unavailable; publishing raw mic", e);
|
||||
@@ -171,11 +266,69 @@ export function startLotusDenoise(vm: CallViewModel): () => void {
|
||||
active: false,
|
||||
error: e instanceof Error ? e.message : String(e),
|
||||
model,
|
||||
mode: "ml",
|
||||
});
|
||||
}
|
||||
})();
|
||||
inflight.set(room, run);
|
||||
void run.finally(() => {
|
||||
if (inflight.get(room) === run) inflight.delete(room);
|
||||
});
|
||||
};
|
||||
|
||||
// ── Mid-call switching (io.lotus.set_denoise) ───────────────────────────
|
||||
// `applied` is what the mic is actually set up for. Switches run one at a
|
||||
// time; each works from `applied` to the latest `target`, so a burst of
|
||||
// changes collapses into the last one.
|
||||
let applied: DenoiseTarget = target;
|
||||
let queue: Promise<void> = Promise.resolve();
|
||||
|
||||
const reconfigure = async (): Promise<void> => {
|
||||
const prev = applied;
|
||||
const next = target;
|
||||
if (prev === next || stopped) return;
|
||||
const nsChanged = browserNsFor(prev) !== browserNsFor(next);
|
||||
const swapProcessor =
|
||||
prev.mode === "ml" &&
|
||||
(next.mode !== "ml" || !sameMlConfig(prev.config, next.config));
|
||||
setLotusNoiseSuppression(browserNsFor(next));
|
||||
// Load the new model while the old one keeps running, so the switch
|
||||
// itself only re-wires the graph instead of sending raw audio meanwhile.
|
||||
if (next.mode === "ml") await prepareDenoiseAssets(next.config);
|
||||
for (const room of rooms) {
|
||||
await inflight.get(room)?.catch(() => undefined);
|
||||
const mic = micOf(room);
|
||||
if (!mic) continue;
|
||||
try {
|
||||
if (swapProcessor && mic.getProcessor()) await mic.stopProcessor();
|
||||
if (nsChanged) await restartCapture(mic, browserNsFor(next));
|
||||
} catch (e) {
|
||||
logger.warn("[lotus] set_denoise: re-capturing the mic failed", e);
|
||||
}
|
||||
apply(room);
|
||||
}
|
||||
applied = next;
|
||||
if (next.mode !== "ml")
|
||||
lotusSendToHost(LotusWidgetActions.DenoiseState, {
|
||||
active: false,
|
||||
mode: next.mode,
|
||||
});
|
||||
};
|
||||
|
||||
const onSetDenoise = (ev: CustomEvent<IWidgetApiRequest>): void => {
|
||||
w?.api.transport.reply(ev.detail, {});
|
||||
const next = applySetDenoise(target, ev.detail.data);
|
||||
if (!next) return;
|
||||
logger.info(
|
||||
`[lotus] set_denoise: ${next.mode}${next.mode === "ml" ? ` (${next.config.model})` : ""}`,
|
||||
);
|
||||
target = next;
|
||||
queue = queue.then(reconfigure).catch((e) => {
|
||||
logger.warn("[lotus] set_denoise failed", e);
|
||||
});
|
||||
};
|
||||
w?.lazyActions.on(LotusWidgetActions.SetDenoise, onSetDenoise);
|
||||
|
||||
// [lotus #9] Suspend/resume the processor's own AudioContext with the mic's
|
||||
// mute state (LiveKit keeps the muted track flowing silent frames, so the
|
||||
// worklet would otherwise run inference the whole time the user is muted).
|
||||
@@ -231,6 +384,8 @@ export function startLotusDenoise(vm: CallViewModel): () => void {
|
||||
|
||||
return () => {
|
||||
stopped = true;
|
||||
w?.lazyActions.off(LotusWidgetActions.SetDenoise, onSetDenoise);
|
||||
setLotusNoiseSuppression(null);
|
||||
sub.unsubscribe();
|
||||
for (const off of roomListeners.values()) off();
|
||||
roomListeners.clear();
|
||||
|
||||
@@ -11,6 +11,7 @@ import { type AudioProcessorOptions } from "livekit-client";
|
||||
import {
|
||||
assertModuleExport,
|
||||
type LotusDenoiseConfig,
|
||||
normalizeDfnModelBytes,
|
||||
LotusDenoiseProcessor,
|
||||
prepareDenoiseAssets,
|
||||
releasePreparedDenoiseAssets,
|
||||
@@ -245,3 +246,37 @@ describe("assertModuleExport (#26)", () => {
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe("normalizeDfnModelBytes", () => {
|
||||
const tar = (): Uint8Array => {
|
||||
const b = new Uint8Array(1024);
|
||||
b.set(new TextEncoder().encode("ustar"), 257);
|
||||
b.set(new TextEncoder().encode("model.onnx"), 0);
|
||||
return b;
|
||||
};
|
||||
const gunzip = async (buf: ArrayBuffer): Promise<Uint8Array> =>
|
||||
new Uint8Array(
|
||||
await new Response(
|
||||
new Response(buf).body!.pipeThrough(new DecompressionStream("gzip")),
|
||||
).arrayBuffer(),
|
||||
);
|
||||
|
||||
test("passes a gzipped model through untouched", async () => {
|
||||
const gz = new Uint8Array([0x1f, 0x8b, 8, 0, 1, 2, 3]).buffer;
|
||||
expect(await normalizeDfnModelBytes(gz)).toBe(gz);
|
||||
});
|
||||
|
||||
test("re-gzips a tar that a Content-Encoding server already unpacked", async () => {
|
||||
const out = await normalizeDfnModelBytes(tar().buffer as ArrayBuffer);
|
||||
const u8 = new Uint8Array(out);
|
||||
expect([u8[0], u8[1]]).toEqual([0x1f, 0x8b]);
|
||||
expect(await gunzip(out)).toEqual(tar());
|
||||
});
|
||||
|
||||
test("rejects anything else (e.g. an HTML error page)", async () => {
|
||||
const html = new TextEncoder().encode("<!doctype html><title>404</title>");
|
||||
await expect(
|
||||
normalizeDfnModelBytes(html.buffer as ArrayBuffer),
|
||||
).rejects.toThrow(/neither gzip nor tar/);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -144,6 +144,8 @@ interface DfnCore {
|
||||
initialize: () => Promise<void>;
|
||||
createAudioWorkletNode: (ctx: AudioContext) => Promise<AudioNode>;
|
||||
destroy: () => void;
|
||||
/** Set by `initialize()`; the model bytes are handed to the worklet as-is. */
|
||||
assets?: { wasmModule: WebAssembly.Module; modelBytes: ArrayBuffer } | null;
|
||||
}
|
||||
interface DfnModule {
|
||||
DeepFilterNet3Core: new (opts: {
|
||||
@@ -167,6 +169,33 @@ export function assertModuleExport<T>(
|
||||
return mod as T;
|
||||
}
|
||||
|
||||
/**
|
||||
* Make sure the DeepFilterNet model is the gzipped tarball its wasm expects.
|
||||
*
|
||||
* A server or proxy that sends the `.tar.gz` with `Content-Encoding: gzip`
|
||||
* (vite preview does; some CDNs and nginx `gzip_static` setups do too) makes
|
||||
* `fetch()` hand back the inner tar. The library's worklet then fails inside
|
||||
* `df_create`, swallows the error and passes the raw mic straight through,
|
||||
* while we report DeepFilterNet as active. An already-decompressed tar is
|
||||
* re-gzipped; anything else throws, so the rnnoise fallback runs and the host
|
||||
* is told the truth.
|
||||
*/
|
||||
export async function normalizeDfnModelBytes(
|
||||
bytes: ArrayBuffer,
|
||||
): Promise<ArrayBuffer> {
|
||||
const u8 = new Uint8Array(bytes);
|
||||
if (u8.length > 2 && u8[0] === 0x1f && u8[1] === 0x8b) return bytes;
|
||||
const isTar =
|
||||
u8.length > 262 &&
|
||||
String.fromCharCode(...u8.subarray(257, 262)) === "ustar";
|
||||
if (!isTar)
|
||||
throw new Error(
|
||||
`denoise: DeepFilterNet model is neither gzip nor tar (${u8.length} bytes)`,
|
||||
);
|
||||
const gz = new Response(u8).body!.pipeThrough(new CompressionStream("gzip"));
|
||||
return new Response(gz).arrayBuffer();
|
||||
}
|
||||
|
||||
async function loadDfnCore(config: LotusDenoiseConfig): Promise<DfnCore> {
|
||||
const base = config.assetBase;
|
||||
const url = `${base}deepfilternet/index.esm.js`;
|
||||
@@ -178,13 +207,19 @@ async function loadDfnCore(config: LotusDenoiseConfig): Promise<DfnCore> {
|
||||
);
|
||||
const core = new mod.DeepFilterNet3Core({
|
||||
sampleRate: 48_000,
|
||||
// 60, not 80: full-strength suppression is the main source of the
|
||||
// "over-processed" character; a lower level keeps voice natural while
|
||||
// the dry/wet floor handles the noise tail.
|
||||
noiseReductionLevel: 60,
|
||||
// Attenuation limit in dB. At 60 the background is muted outright
|
||||
// (measured -61 to -64 dB), so every word drops into dead silence and
|
||||
// listeners hear the mic "cutting out" at word ends. 25 keeps a quiet
|
||||
// background bed and scored best on room-echo speech with fan and
|
||||
// keyboard noise (PESQ 2.72 / 2.38 vs 2.33 / 2.04 at 60).
|
||||
noiseReductionLevel: 25,
|
||||
assetConfig: { cdnUrl: dfnBase },
|
||||
});
|
||||
await core.initialize();
|
||||
if (core.assets)
|
||||
core.assets.modelBytes = await normalizeDfnModelBytes(
|
||||
core.assets.modelBytes,
|
||||
);
|
||||
return core;
|
||||
}
|
||||
|
||||
|
||||
@@ -6,10 +6,12 @@ Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import {
|
||||
type LocalTrack,
|
||||
type LocalTrackPublication,
|
||||
type Room as LivekitRoom,
|
||||
RoomEvent,
|
||||
Track,
|
||||
TrackEvent,
|
||||
} from "livekit-client";
|
||||
import { Observable, type Subscription, share, switchMap } from "rxjs";
|
||||
import { distinctUntilChanged, map } from "rxjs/operators";
|
||||
@@ -37,15 +39,35 @@ export interface LocalMicSample {
|
||||
|
||||
const micPublication = (
|
||||
room: LivekitRoom,
|
||||
): { track: MediaStreamTrack; muted: boolean } | null => {
|
||||
): { track: MediaStreamTrack; local: LocalTrack; muted: boolean } | null => {
|
||||
const pub: LocalTrackPublication | undefined =
|
||||
room.localParticipant.getTrackPublication(Track.Source.Microphone);
|
||||
const track = pub?.track?.mediaStreamTrack;
|
||||
return track && track.readyState === "live"
|
||||
? { track, muted: pub?.isMuted ?? false }
|
||||
const local = pub?.track;
|
||||
const track = local?.mediaStreamTrack;
|
||||
return local && track && track.readyState === "live"
|
||||
? { track, local, muted: pub?.isMuted ?? false }
|
||||
: null;
|
||||
};
|
||||
|
||||
// Every live tap, so a mic re-capture can drop the clones first.
|
||||
const liveTaps = new Set<{ release: () => void; reconcile: () => void }>();
|
||||
|
||||
/**
|
||||
* Run a mic re-capture (`restartTrack`) with every level-meter clone stopped,
|
||||
* then re-tap the new track. Chrome makes all open tracks of a device share
|
||||
* one set of audio processing, so a clone still holding the old track makes
|
||||
* the new capture ignore its `noiseSuppression` (or AGC/EC) constraint.
|
||||
*/
|
||||
export async function withMicTapsReleased<T>(fn: () => Promise<T>): Promise<T> {
|
||||
const taps = [...liveTaps];
|
||||
taps.forEach((t) => t.release());
|
||||
try {
|
||||
return await fn();
|
||||
} finally {
|
||||
taps.forEach((t) => t.reconcile());
|
||||
}
|
||||
}
|
||||
|
||||
/** Pure: the same LiveKit rooms, in the same order (by identity). */
|
||||
export const sameRooms = <T>(a: readonly T[], b: readonly T[]): boolean =>
|
||||
a.length === b.length && a.every((r, i) => r === b[i]);
|
||||
@@ -79,11 +101,17 @@ export function observeLocalMicSample$(
|
||||
let clone: MediaStreamTrack | null = null;
|
||||
let timer: ReturnType<typeof setInterval> | undefined;
|
||||
let tapped: MediaStreamTrack | null = null;
|
||||
// LiveKit swaps the MediaStreamTrack inside the same LocalTrack on
|
||||
// restartTrack (device switch, re-capture) without any room event,
|
||||
// so follow its Restarted event or the clone keeps the old one open.
|
||||
let tappedLocal: LocalTrack | null = null;
|
||||
let muted = false;
|
||||
|
||||
const stopTap = (): void => {
|
||||
if (timer !== undefined) clearInterval(timer);
|
||||
timer = undefined;
|
||||
tappedLocal?.off(TrackEvent.Restarted, reconcile);
|
||||
tappedLocal = null;
|
||||
clone?.stop();
|
||||
clone = null;
|
||||
void ctx?.close().catch(() => undefined);
|
||||
@@ -92,7 +120,12 @@ export function observeLocalMicSample$(
|
||||
subscriber.next(null);
|
||||
};
|
||||
|
||||
const startTap = (source: MediaStreamTrack): void => {
|
||||
const startTap = (
|
||||
source: MediaStreamTrack,
|
||||
local: LocalTrack,
|
||||
): void => {
|
||||
tappedLocal = local;
|
||||
local.on(TrackEvent.Restarted, reconcile);
|
||||
try {
|
||||
clone = source.clone();
|
||||
// Muting disables the published track; the clone must still hear.
|
||||
@@ -116,15 +149,22 @@ export function observeLocalMicSample$(
|
||||
}
|
||||
};
|
||||
|
||||
const reconcile = (): void => {
|
||||
function reconcile(): void {
|
||||
const pub =
|
||||
rooms.map(micPublication).find((p) => p !== null) ?? null;
|
||||
muted = pub?.muted ?? false;
|
||||
const track = pub?.track ?? null;
|
||||
if (track === tapped) return;
|
||||
if (tapped) stopTap();
|
||||
if (track) startTap(track);
|
||||
if (track && pub) startTap(track, pub.local);
|
||||
}
|
||||
const control = {
|
||||
release: (): void => {
|
||||
if (tapped) stopTap();
|
||||
},
|
||||
reconcile,
|
||||
};
|
||||
liveTaps.add(control);
|
||||
|
||||
const events = [
|
||||
RoomEvent.TrackMuted,
|
||||
@@ -143,6 +183,7 @@ export function observeLocalMicSample$(
|
||||
rooms.forEach((room) =>
|
||||
events.forEach((ev) => room.off(ev, reconcile)),
|
||||
);
|
||||
liveTaps.delete(control);
|
||||
if (tapped) stopTap();
|
||||
};
|
||||
}),
|
||||
|
||||
@@ -18,6 +18,7 @@ import {
|
||||
overrideSpotlight$,
|
||||
pinnedMedia$,
|
||||
resolveManualSpotlight,
|
||||
spotlightCameras,
|
||||
} from "./lotusSpotlight";
|
||||
import { parseFocusPayload } from "./lotusFocus";
|
||||
|
||||
@@ -27,6 +28,7 @@ interface FakeMedia {
|
||||
type: "user";
|
||||
local: boolean;
|
||||
speaking$: BehaviorSubject<boolean>;
|
||||
videoEnabled$: BehaviorSubject<boolean>;
|
||||
}
|
||||
|
||||
function media(
|
||||
@@ -34,6 +36,7 @@ function media(
|
||||
deviceId: string,
|
||||
speaking = false,
|
||||
local = false,
|
||||
video = false,
|
||||
): FakeMedia {
|
||||
return {
|
||||
id: `${userId}:${deviceId}`,
|
||||
@@ -41,6 +44,7 @@ function media(
|
||||
type: "user",
|
||||
local,
|
||||
speaking$: new BehaviorSubject(speaking),
|
||||
videoEnabled$: new BehaviorSubject(video),
|
||||
};
|
||||
}
|
||||
const asUser = (m: FakeMedia): UserMediaViewModel =>
|
||||
@@ -186,6 +190,81 @@ describe("overrideSpotlight$", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("spotlightCameras (#67)", () => {
|
||||
const me = media("@me:x", "d", false, true, true);
|
||||
const carol = media("@carol:x", "d", false, false, true);
|
||||
const dave = media("@dave:x", "d", false, false, true);
|
||||
const users = [me, carol, bob, dave];
|
||||
|
||||
test("shares first, then remote cameras that are on, in order", () => {
|
||||
expect(
|
||||
spotlightCameras([screen], users, new Set([me, carol, dave]), undefined),
|
||||
).toEqual([screen, carol, dave]);
|
||||
});
|
||||
|
||||
test("the pin sits right after the shares and appears once", () => {
|
||||
expect(
|
||||
spotlightCameras([screen], users, new Set([carol, dave]), dave),
|
||||
).toEqual([screen, dave, carol]);
|
||||
});
|
||||
|
||||
test("the pinned camera is kept even with its video off", () => {
|
||||
expect(spotlightCameras([screen], users, new Set([carol]), bob)).toEqual([
|
||||
screen,
|
||||
bob,
|
||||
carol,
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("overrideSpotlight$ cameras during a share (#67)", () => {
|
||||
test("cameras join and leave the carousel live", () => {
|
||||
const carol = media("@carol:x", "d", false, false, true);
|
||||
const me = media("@me:x", "d", false, true, true);
|
||||
// Own fixture: other tests leave subscriptions on the shared `bob`.
|
||||
const erin = media("@erin:x", "d");
|
||||
const users$ = new BehaviorSubject([
|
||||
asUser(me),
|
||||
asUser(carol),
|
||||
asUser(erin),
|
||||
]);
|
||||
const seen: unknown[][] = [];
|
||||
const sub = overrideSpotlight$(
|
||||
of(asUser(carol)),
|
||||
of(null),
|
||||
of([screen as unknown as ScreenShareViewModel]),
|
||||
of(undefined),
|
||||
users$,
|
||||
).subscribe((r) => seen.push(r.spotlight));
|
||||
|
||||
erin.videoEnabled$.next(true); // erin turns their camera on
|
||||
carol.videoEnabled$.next(false); // carol turns hers off
|
||||
users$.next([asUser(me), asUser(carol)]); // erin leaves
|
||||
erin.videoEnabled$.next(false);
|
||||
sub.unsubscribe();
|
||||
|
||||
expect(seen).toEqual([
|
||||
[screen, carol],
|
||||
[screen, carol, erin],
|
||||
[screen, erin],
|
||||
[screen],
|
||||
]);
|
||||
});
|
||||
|
||||
test("no share: cameras are not added (upstream speaker-follows)", () => {
|
||||
const carol = media("@carol:x", "d", false, false, true);
|
||||
let spotlight;
|
||||
overrideSpotlight$(
|
||||
of(asUser(bob)),
|
||||
of(null),
|
||||
of([]),
|
||||
of(undefined),
|
||||
of([asUser(carol), asUser(bob)]),
|
||||
).subscribe((r) => (spotlight = r.spotlight));
|
||||
expect(spotlight).toEqual([bob]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("createManualSpotlight (#3 / #16)", () => {
|
||||
function setup(initialLayout: LayoutMode = "grid") {
|
||||
const pin$ = new BehaviorSubject<ManualSpotlight | null>(null);
|
||||
|
||||
@@ -26,6 +26,7 @@ import { type MediaViewModel } from "../state/media/MediaViewModel";
|
||||
import { type LocalUserMediaViewModel } from "../state/media/LocalUserMediaViewModel";
|
||||
import { type LayoutMode } from "../state/LayoutSwitchViewModel";
|
||||
import { type Behavior } from "../state/Behavior";
|
||||
import { shallowEquals } from "../utils/array";
|
||||
|
||||
interface SpotlightAndPip {
|
||||
spotlight: MediaViewModel[];
|
||||
@@ -102,6 +103,44 @@ export function pinnedMedia$(
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* [lotus #67] The spotlight carousel during a screenshare: every share first
|
||||
* (upstream order), then the host-pinned camera if any (included even with
|
||||
* its video off, as before #67), then every other REMOTE camera whose video
|
||||
* is on, in `userMedia` order so the dots don't reshuffle. Each camera
|
||||
* appears once. The local camera is left out: the PiP and grid show it.
|
||||
*/
|
||||
export function spotlightCameras<S, U extends { local: boolean }>(
|
||||
screenShares: S[],
|
||||
userMedia: U[],
|
||||
videoOn: ReadonlySet<U>,
|
||||
pinned: U | undefined,
|
||||
): (S | U)[] {
|
||||
const cameras = userMedia.filter(
|
||||
(m) => m !== pinned && !m.local && videoOn.has(m),
|
||||
);
|
||||
return [...screenShares, ...(pinned ? [pinned] : []), ...cameras];
|
||||
}
|
||||
|
||||
/**
|
||||
* The remote user media whose camera is currently on, as a live set. Only
|
||||
* subscribes to `videoEnabled$` of remote items (the local camera never goes
|
||||
* in the carousel).
|
||||
*/
|
||||
function remoteVideoOn$(
|
||||
userMedia$: Observable<UserMediaViewModel[]>,
|
||||
): Observable<Set<UserMediaViewModel>> {
|
||||
return userMedia$.pipe(
|
||||
switchMap((items) => {
|
||||
const remote = items.filter((m) => !m.local);
|
||||
if (remote.length === 0) return of(new Set<UserMediaViewModel>());
|
||||
return combineLatest(remote.map((m) => m.videoEnabled$)).pipe(
|
||||
map((flags) => new Set(remote.filter((_, i) => flags[i]))),
|
||||
);
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* [lotus #4] Manual spotlight override.
|
||||
*
|
||||
@@ -121,6 +160,12 @@ export function pinnedMedia$(
|
||||
* screenshare"); the PiP keeps showing the auto-selected speaker and is
|
||||
* only blanked when that speaker IS the pinned participant (#29).
|
||||
*
|
||||
* [lotus #67] During a screenshare the spotlight also holds every remote
|
||||
* camera that is on (after the shares and the pin, see
|
||||
* {@link spotlightCameras}), so viewers can flip to a camera with the same
|
||||
* arrows/dots they use between shares. The share stays the default view;
|
||||
* SpotlightTile handles that via `useLotusSpotlightFallback`.
|
||||
*
|
||||
* @param autoSpotlightSpeaker$ upstream's speaker-follows auto selection
|
||||
* @param manualSpotlight$ host pin, or `null` for auto (default)
|
||||
* @param screenShares$ current screen-share view models
|
||||
@@ -146,23 +191,38 @@ export function overrideSpotlight$(
|
||||
return screenShares$.pipe(
|
||||
switchMap((screenShares) => {
|
||||
if (screenShares.length > 0)
|
||||
// During a screenshare, if the host has explicitly pinned a
|
||||
// participant, surface that camera in the spotlight alongside the
|
||||
// shared screen (the whole point of "focus camera during screenshare").
|
||||
// With no manual pin this is unchanged: the screenshare alone is
|
||||
// spotlighted.
|
||||
return pinned$.pipe(
|
||||
map((pinned) =>
|
||||
// During a screenshare the spotlight holds the shares, then the
|
||||
// host-pinned camera (#4, "focus camera during screenshare"), then the
|
||||
// other remote cameras that are on (#67).
|
||||
return combineLatest([
|
||||
pinned$,
|
||||
userMedia$,
|
||||
remoteVideoOn$(userMedia$),
|
||||
]).pipe(
|
||||
map(([pinned, userMedia, videoOn]) => ({
|
||||
pinned,
|
||||
spotlight: spotlightCameras(
|
||||
screenShares,
|
||||
userMedia,
|
||||
videoOn,
|
||||
pinned,
|
||||
),
|
||||
})),
|
||||
distinctUntilChanged(
|
||||
(a, b) =>
|
||||
a.pinned === b.pinned && shallowEquals(a.spotlight, b.spotlight),
|
||||
),
|
||||
map(({ pinned, spotlight }) =>
|
||||
pinned
|
||||
? {
|
||||
spotlight: [...screenShares, pinned],
|
||||
spotlight,
|
||||
// [lotus #29] Keep the active-speaker PiP; it is only
|
||||
// redundant when the speaker is the pinned participant.
|
||||
pip$: autoSpotlightSpeaker$.pipe(
|
||||
map((auto) => (auto === pinned ? undefined : auto)),
|
||||
),
|
||||
}
|
||||
: { spotlight: screenShares, pip$: spotlightSpeaker$ },
|
||||
: { spotlight, pip$: spotlightSpeaker$ },
|
||||
),
|
||||
);
|
||||
|
||||
|
||||
@@ -0,0 +1,42 @@
|
||||
/*
|
||||
Copyright 2026 Lotus Guild
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { describe, expect, test } from "vitest";
|
||||
|
||||
import { spotlightFallbackTarget } from "./lotusSpotlightView";
|
||||
|
||||
const share = { id: "s", type: "screen share" };
|
||||
const share2 = { id: "s2", type: "screen share" };
|
||||
const cam = { id: "c", type: "user" };
|
||||
const cam2 = { id: "c2", type: "user" };
|
||||
|
||||
describe("spotlightFallbackTarget (#67)", () => {
|
||||
test("nothing to do while the visible item is still there", () => {
|
||||
expect(spotlightFallbackTarget([share, cam], "c", true)).toBeNull();
|
||||
expect(spotlightFallbackTarget([share, cam], "s", true)).toBeNull();
|
||||
});
|
||||
|
||||
test("visible camera gone: fall back to the first share", () => {
|
||||
expect(spotlightFallbackTarget([share, cam2], "c", true)).toBe("s");
|
||||
});
|
||||
|
||||
test("visible share ended: fall back to the remaining first item", () => {
|
||||
expect(spotlightFallbackTarget([share2, cam], "s", true)).toBe("s2");
|
||||
});
|
||||
|
||||
test("share started while a camera was spotlighted: jump to the share", () => {
|
||||
expect(spotlightFallbackTarget([share, cam], "c", false)).toBe("s");
|
||||
});
|
||||
|
||||
test("no share and no change: speaker-follows is left alone", () => {
|
||||
expect(spotlightFallbackTarget([cam2], "c2", false)).toBeNull();
|
||||
});
|
||||
|
||||
test("empty media: nothing to do", () => {
|
||||
expect(spotlightFallbackTarget([], "c", true)).toBeNull();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,72 @@
|
||||
/*
|
||||
Copyright 2026 Lotus Guild
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { useEffect, useRef } from "react";
|
||||
|
||||
/**
|
||||
* What the spotlight carousel should jump to after `media` changes, or `null`
|
||||
* to leave it alone. Pure; see {@link useLotusSpotlightFallback}.
|
||||
*
|
||||
* - The visible item is gone (its camera turned off, its owner left, or a
|
||||
* share ended): show the first item rather than whichever neighbour the
|
||||
* scroll happens to land on.
|
||||
* - A share has just started (the first item became a screen share): show
|
||||
* it, even if the previously spotlighted camera is still in the list.
|
||||
*/
|
||||
export function spotlightFallbackTarget(
|
||||
media: readonly { id: string; type: string }[],
|
||||
visibleId: string | undefined,
|
||||
firstWasShare: boolean,
|
||||
): string | null {
|
||||
const first = media[0];
|
||||
if (first === undefined) return null;
|
||||
const shareStarted = first.type === "screen share" && !firstWasShare;
|
||||
const visibleGone =
|
||||
visibleId !== undefined && !media.some((m) => m.id === visibleId);
|
||||
if (!shareStarted && !visibleGone) return null;
|
||||
return first.id === visibleId ? null : first.id;
|
||||
}
|
||||
|
||||
/**
|
||||
* [lotus #67] Keeps the spotlight carousel on a sensible item now that it can
|
||||
* hold cameras as well as screenshares during a share (see
|
||||
* `spotlightCameras` in lotusSpotlight.ts).
|
||||
*
|
||||
* Scrolls the item into view directly and also sets `visibleId`: when the
|
||||
* target is already at the scroll position (e.g. the visible item was the
|
||||
* last one), the tile's IntersectionObserver never fires, so the arrows and
|
||||
* dots would otherwise stay pointed at an item that no longer exists.
|
||||
*/
|
||||
export function useLotusSpotlightFallback(
|
||||
root: HTMLElement | null,
|
||||
media: readonly { id: string; type: string }[],
|
||||
visibleId: string | undefined,
|
||||
setVisibleId: (id: string) => void,
|
||||
): void {
|
||||
const firstIsShare = media[0]?.type === "screen share";
|
||||
const firstWasShare = useRef(firstIsShare);
|
||||
|
||||
useEffect(() => {
|
||||
const target = spotlightFallbackTarget(
|
||||
media,
|
||||
visibleId,
|
||||
firstWasShare.current,
|
||||
);
|
||||
firstWasShare.current = firstIsShare;
|
||||
if (target === null) return;
|
||||
|
||||
const item = root?.querySelector<HTMLElement>(
|
||||
`[data-id="${CSS.escape(target)}"]`,
|
||||
);
|
||||
const scroller = item?.parentElement;
|
||||
if (item && scroller)
|
||||
scroller.scrollLeft +=
|
||||
item.getBoundingClientRect().left -
|
||||
scroller.getBoundingClientRect().left;
|
||||
setVisibleId(target);
|
||||
}, [root, media, visibleId, firstIsShare, setVisibleId]);
|
||||
}
|
||||
@@ -47,6 +47,7 @@ import {
|
||||
// default to true, so with no params this is byte-for-byte upstream behaviour;
|
||||
// `?autoGainControl=false` forces AGC off regardless of the stored Setting.
|
||||
import { getUrlParams } from "../../../UrlParams.ts";
|
||||
import { lotusNoiseSuppressionAllowed } from "../../../lotus/lotusCaptureOverride.ts";
|
||||
|
||||
// TODO evaluate if this should be done like the Publisher Factory
|
||||
export interface ConnectionFactory {
|
||||
@@ -193,9 +194,16 @@ function generateRoomOption({
|
||||
echoCancellation:
|
||||
echoCancellationSetting.getValue() &&
|
||||
getUrlParams().echoCancellation !== false,
|
||||
// [lotus] The host can switch this mid-call (io.lotus.set_denoise).
|
||||
noiseSuppression:
|
||||
noiseSuppressionSetting.getValue() &&
|
||||
getUrlParams().noiseSuppression !== false,
|
||||
lotusNoiseSuppressionAllowed(getUrlParams().noiseSuppression),
|
||||
// [lotus] LiveKit defaults voiceIsolation ON, which (where the platform
|
||||
// implements it) is one more voice filter in front of the ML model or
|
||||
// in "off" mode. Tie it to browser noise suppression.
|
||||
voiceIsolation:
|
||||
noiseSuppressionSetting.getValue() &&
|
||||
lotusNoiseSuppressionAllowed(getUrlParams().noiseSuppression),
|
||||
autoGainControl:
|
||||
autoGainControlSetting.getValue() &&
|
||||
getUrlParams().autoGainControl !== false,
|
||||
|
||||
@@ -214,3 +214,90 @@ test("SpotlightTile displays ringing media", async () => {
|
||||
act(() => pickupState$.next("timeout"));
|
||||
screen.getByText("Call ended");
|
||||
});
|
||||
|
||||
test("[lotus #67] falls back to the share when the visible camera goes away", async () => {
|
||||
// A controllable IntersectionObserver so the tile's visibleId follows
|
||||
// navigation the way it does in a browser.
|
||||
let report: ((entries: IntersectionObserverEntry[]) => void) | undefined;
|
||||
const RealIO = global.IntersectionObserver;
|
||||
global.IntersectionObserver = class {
|
||||
public constructor(
|
||||
cb: (entries: IntersectionObserverEntry[]) => void,
|
||||
options?: IntersectionObserverInit,
|
||||
) {
|
||||
// Only the carousel's own observer (rooted on the tile); LiveKit's
|
||||
// video elements create observers of their own.
|
||||
if (options?.root) report = cb;
|
||||
}
|
||||
public observe(): void {}
|
||||
public unobserve(): void {}
|
||||
public disconnect(): void {}
|
||||
} as unknown as typeof IntersectionObserver;
|
||||
|
||||
try {
|
||||
const share = mockRemoteScreenShare(
|
||||
mockRtcMembership("@alice:example.org", "AAAA"),
|
||||
{},
|
||||
mockRemoteParticipant({}),
|
||||
);
|
||||
const camera = mockRemoteMedia(
|
||||
mockRtcMembership("@carol:example.org", "CCCC"),
|
||||
{ rawDisplayName: "Carol", getMxcAvatarUrl: () => "mxc://carol" },
|
||||
mockRemoteParticipant({}),
|
||||
);
|
||||
// The mocks all use the id "remote"; the carousel keys on ids.
|
||||
Object.assign(camera, { id: "remote-camera" });
|
||||
const media$ = new BehaviorSubject([share, camera]);
|
||||
|
||||
const user = userEvent.setup();
|
||||
const { container } = render(
|
||||
<TooltipProvider>
|
||||
<SpotlightTile
|
||||
vm={
|
||||
new SpotlightTileViewModel(
|
||||
media$,
|
||||
constant(false),
|
||||
constant("solid"),
|
||||
)
|
||||
}
|
||||
targetWidth={300}
|
||||
targetHeight={200}
|
||||
expanded={false}
|
||||
onToggleExpanded={vi.fn()}
|
||||
showIndicators
|
||||
showNameTags
|
||||
showRingingStatus
|
||||
focusable
|
||||
/>
|
||||
</TooltipProvider>,
|
||||
);
|
||||
const dots = (): HTMLElement[] =>
|
||||
screen.getAllByTestId("screenshare-indicator");
|
||||
expect(dots().map((d) => d.dataset.kind)).toEqual(["screen share", "user"]);
|
||||
expect(dots()[0].dataset.visible).toBe("true");
|
||||
|
||||
// Flip to Carol's camera; the browser would report it as intersecting.
|
||||
await user.click(screen.getByRole("button", { name: "Next" }));
|
||||
const carolItem = container.querySelector(`[data-id="${camera.id}"]`)!;
|
||||
act(() =>
|
||||
report!([
|
||||
{
|
||||
isIntersecting: true,
|
||||
target: carolItem,
|
||||
} as unknown as IntersectionObserverEntry,
|
||||
]),
|
||||
);
|
||||
expect(dots()[1].dataset.visible).toBe("true");
|
||||
expect(isInaccessible(screen.getByText("Carol"))).toBe(false);
|
||||
|
||||
// Carol turns her camera off: she drops out and the share is shown again.
|
||||
act(() => media$.next([share]));
|
||||
expect(dots()).toHaveLength(1);
|
||||
expect(dots()[0].dataset.visible).toBe("true");
|
||||
expect(
|
||||
isInaccessible(container.querySelector(`[data-id="${share.id}"]`)!),
|
||||
).toBe(false);
|
||||
} finally {
|
||||
global.IntersectionObserver = RealIO;
|
||||
}
|
||||
});
|
||||
|
||||
@@ -43,6 +43,7 @@ import { useLatest } from "../useLatest";
|
||||
import { type SpotlightTileViewModel } from "../state/TileViewModel";
|
||||
import { useBehavior } from "../useBehavior";
|
||||
import { muteScreenshareAudio$ } from "../lotus/lotusScreenshareAudio";
|
||||
import { useLotusSpotlightFallback } from "../lotus/lotusSpotlightView";
|
||||
import { type MemberMediaViewModel } from "../state/media/MemberMediaViewModel";
|
||||
import { type LocalUserMediaViewModel } from "../state/media/LocalUserMediaViewModel";
|
||||
import { type RemoteUserMediaViewModel } from "../state/media/RemoteUserMediaViewModel";
|
||||
@@ -487,6 +488,10 @@ export const SpotlightTile: FC<Props> = ({
|
||||
[visibleId],
|
||||
);
|
||||
|
||||
// [lotus #67] The carousel can hold cameras during a share: keep the share
|
||||
// as the default view and fall back to it when the visible camera goes away.
|
||||
useLotusSpotlightFallback(useBehavior(root$), media, visibleId, setVisibleId);
|
||||
|
||||
const onBackClick = useCallback(() => {
|
||||
const media = latestMedia.current;
|
||||
const visibleIndex = media.findIndex(
|
||||
@@ -593,6 +598,7 @@ export const SpotlightTile: FC<Props> = ({
|
||||
{media.map((vm) => (
|
||||
<div
|
||||
data-testid="screenshare-indicator"
|
||||
data-kind={vm.type}
|
||||
key={vm.id}
|
||||
className={styles.item}
|
||||
data-visible={vm.id === visibleId}
|
||||
|
||||
Reference in New Issue
Block a user