Compare commits
15
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
1b609d997b | ||
|
|
d9ac9a0fa4 | ||
|
|
66bfead7f1 | ||
|
|
ea579cb998 | ||
|
|
bc0e5ed432 | ||
|
|
778712409e | ||
|
|
e9a59336c7 | ||
|
|
c7494e68ca | ||
|
|
68eafcb9a8 | ||
|
|
dcba5b6b7e | ||
|
|
1c1394b6ef | ||
|
|
501e3fb5ac | ||
|
|
bb639bb92d | ||
|
|
e5d5f13923 | ||
|
|
e504a31efd |
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@lotusguild/element-call-embedded",
|
||||
"version": "0.25.0-lotus.1",
|
||||
"version": "0.25.0-lotus.5",
|
||||
"files": [
|
||||
"README.md",
|
||||
"LICENSE-AGPL-3.0",
|
||||
|
||||
@@ -116,6 +116,7 @@
|
||||
"peer_connection_timeout_description": "Connection to the media server timed out. Try switching to a different network or disabling your VPN. If the problem persists, see our <0>troubleshooting guide</0> or contact your server administrator.",
|
||||
"room_creation_restricted": "Failed to create call",
|
||||
"room_creation_restricted_description": "Call creation might be restricted to authorized users only. Try again later, or contact your server admin if the problem persists.",
|
||||
"sfu_token_refused": "Can't join this call",
|
||||
"sticky_events_required": "Homeserver does not support Matrix 2.0 calls",
|
||||
"sticky_events_required_description": "This deployment is configured to use Matrix 2.0 call mode, but the homeserver does not advertise support for sticky events (MSC4354). Ask your server admin to upgrade, or switch the deployment to a compatible mode.",
|
||||
"unexpected_ec_error": "An unexpected error occurred (<0>Error Code:</0> <1>{{ errorCode }}</1>). Please contact your server admin."
|
||||
|
||||
@@ -46,8 +46,12 @@ unconditionally select the container so we can use cq units */
|
||||
|
||||
/* On a landscape phone the fixed 180px filmstrip rail squeezes the spotlight to
|
||||
a sliver; shrink the rail (keeping the 4:3 tile ratio) to give the spotlight
|
||||
room. Desktop landscape (height > 400px) is unaffected. */
|
||||
@media (max-height: 400px) {
|
||||
room. Desktop landscape (height > 400px) is unaffected.
|
||||
[lotus #32] `max-height: 400px` alone also matches a short-but-wide desktop
|
||||
or PiP window that has nothing to do with a phone, so require a coarse
|
||||
(touch) pointer too — a resized desktop/PiP window kept its mouse pointer,
|
||||
so it won't match this and keeps the full-size rail. */
|
||||
@media (max-height: 400px) and (pointer: coarse) {
|
||||
.layer {
|
||||
--grid-slot-width: 132px;
|
||||
}
|
||||
|
||||
@@ -100,6 +100,26 @@ body.lotus-theme {
|
||||
--video-tile-background: var(--cpd-color-bg-subtle-secondary);
|
||||
}
|
||||
|
||||
/* [lotus #21] Subtle contrast guard for elements that sit directly on the
|
||||
transparent canvas (no opaque tile background behind them) when
|
||||
lotusTransparent is set: the host's real wallpaper is unknown to us, so a
|
||||
light text-shadow/backdrop keeps name tags and header/footer controls legible
|
||||
regardless of what's behind them. Kept behind `.lotus-transparent` so upstream
|
||||
(non-Lotus) layouts are completely untouched. Hooked off stable data-testid/
|
||||
element selectors rather than CSS-module class names, since those are hashed
|
||||
per-build and owned by their own component stylesheets. */
|
||||
body.lotus-transparent [data-testid="name_tag"],
|
||||
body.lotus-transparent header,
|
||||
body.lotus-transparent [data-testid="footer-container"] {
|
||||
text-shadow: 0 1px 3px rgba(0, 0, 0, 0.6);
|
||||
}
|
||||
|
||||
body.lotus-transparent header,
|
||||
body.lotus-transparent [data-testid="footer-container"] {
|
||||
backdrop-filter: blur(8px);
|
||||
-webkit-backdrop-filter: blur(8px);
|
||||
}
|
||||
|
||||
@media (min-height: 330px) {
|
||||
body[data-background="gradient"]::before {
|
||||
content: "";
|
||||
|
||||
@@ -52,9 +52,16 @@ afterEach(() => {
|
||||
vi.mock("@livekit/components-react", async (importOriginal) => {
|
||||
return {
|
||||
...(await importOriginal()),
|
||||
AudioTrack: (props: { trackRef: TrackReference }): ReactNode => {
|
||||
AudioTrack: (props: {
|
||||
trackRef: TrackReference;
|
||||
muted?: boolean;
|
||||
}): ReactNode => {
|
||||
return (
|
||||
<audio data-testid={"audio"}>
|
||||
<audio
|
||||
data-testid={"audio"}
|
||||
data-source={props.trackRef.publication.source}
|
||||
data-muted={String(!!props.muted)}
|
||||
>
|
||||
{getTrackReferenceId(props.trackRef)}
|
||||
</audio>
|
||||
);
|
||||
@@ -83,6 +90,7 @@ function renderTestComponent(
|
||||
kind: Track.Kind;
|
||||
source: Track.Source;
|
||||
}[],
|
||||
props: { muted?: boolean; screenshareAudioMuted?: boolean } = {},
|
||||
): RenderResult {
|
||||
const liveKitParticipants = livekitParticipantIdentities.map((identity) =>
|
||||
mockRemoteParticipant({ identity }),
|
||||
@@ -117,6 +125,7 @@ function renderTestComponent(
|
||||
validIdentities={participants.map((p) => p.identity)}
|
||||
livekitRoom={livekitRoom}
|
||||
url={""}
|
||||
{...props}
|
||||
/>
|
||||
</MediaDevicesProvider>,
|
||||
);
|
||||
@@ -286,3 +295,55 @@ it("should setup audioContext gain and pan", () => {
|
||||
expect(testAudioContext.gain.gain.value).toEqual(0.1);
|
||||
expect(testAudioContext.pan.pan.value).toEqual(1);
|
||||
});
|
||||
|
||||
// [lotus] The host's "Mute Screenshare Audio" mutes ONLY screenshare-audio
|
||||
// elements, via the same `muted` prop path as deafen (so it survives re-renders
|
||||
// and later-published shares); deafen still mutes everything.
|
||||
it("screenshareAudioMuted mutes only ScreenShareAudio tracks", () => {
|
||||
const explicitTracks = [
|
||||
{
|
||||
participantId: "@alice:DEV0",
|
||||
kind: Track.Kind.Audio,
|
||||
source: Track.Source.Microphone,
|
||||
},
|
||||
{
|
||||
participantId: "@alice:DEV0",
|
||||
kind: Track.Kind.Audio,
|
||||
source: Track.Source.ScreenShareAudio,
|
||||
},
|
||||
];
|
||||
const mutedBySource = (r: RenderResult): Record<string, string> =>
|
||||
Object.fromEntries(
|
||||
r
|
||||
.queryAllByTestId("audio")
|
||||
.map((el) => [el.dataset.source, el.dataset.muted]),
|
||||
);
|
||||
|
||||
expect(
|
||||
mutedBySource(
|
||||
renderTestComponent(
|
||||
[{ userId: "@alice", deviceId: "DEV0" }],
|
||||
["@alice:DEV0"],
|
||||
explicitTracks,
|
||||
{ screenshareAudioMuted: true },
|
||||
),
|
||||
),
|
||||
).toEqual({
|
||||
[Track.Source.Microphone]: "false",
|
||||
[Track.Source.ScreenShareAudio]: "true",
|
||||
});
|
||||
|
||||
expect(
|
||||
mutedBySource(
|
||||
renderTestComponent(
|
||||
[{ userId: "@alice", deviceId: "DEV0" }],
|
||||
["@alice:DEV0"],
|
||||
explicitTracks,
|
||||
{ muted: true },
|
||||
),
|
||||
),
|
||||
).toEqual({
|
||||
[Track.Source.Microphone]: "true",
|
||||
[Track.Source.ScreenShareAudio]: "true",
|
||||
});
|
||||
});
|
||||
|
||||
@@ -38,6 +38,12 @@ export interface MatrixAudioRendererProps {
|
||||
* If set to `true`, the server will stop sending audio track data to the client.
|
||||
*/
|
||||
muted?: boolean;
|
||||
/**
|
||||
* [lotus] If set to `true`, mutes only the `Track.Source.ScreenShareAudio`
|
||||
* tracks (the host's "Mute Screenshare Audio" control) — same mechanism as
|
||||
* `muted`, so it holds across re-renders and later-published shares.
|
||||
*/
|
||||
screenshareAudioMuted?: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -58,6 +64,7 @@ export function LivekitRoomAudioRenderer({
|
||||
livekitRoom,
|
||||
validIdentities,
|
||||
muted,
|
||||
screenshareAudioMuted,
|
||||
}: MatrixAudioRendererProps): ReactNode {
|
||||
const logger = rootLogger.getChild("[MatrixAudioRenderer]");
|
||||
const tracks = useTracks(
|
||||
@@ -143,7 +150,11 @@ export function LivekitRoomAudioRenderer({
|
||||
<AudioTrackWithAudioNodes
|
||||
key={getTrackReferenceId(trackRef)}
|
||||
trackRef={trackRef}
|
||||
muted={muted}
|
||||
muted={
|
||||
muted ||
|
||||
(screenshareAudioMuted &&
|
||||
trackRef.publication.source === Track.Source.ScreenShareAudio)
|
||||
}
|
||||
audioContext={shouldUseAudioContext ? audioContext : undefined}
|
||||
audioNodes={audioNodes}
|
||||
/>
|
||||
|
||||
@@ -20,7 +20,7 @@ import { MatrixError } from "matrix-js-sdk";
|
||||
import { getSFUConfigWithOpenID, type OpenIDClientParts } from "./openIDSFU";
|
||||
import { testJWTToken } from "../utils/test-fixtures";
|
||||
import { ownMemberMock } from "../utils/test";
|
||||
import { FailToGetOpenIdToken } from "../utils/errors";
|
||||
import { FailToGetOpenIdToken, SFUTokenRefusedError } from "../utils/errors";
|
||||
|
||||
const sfuUrl = "https://sfu.example.org";
|
||||
|
||||
@@ -91,6 +91,31 @@ describe("getSFUConfigWithOpenID", () => {
|
||||
expect.fail("Expected test to throw;");
|
||||
});
|
||||
|
||||
it("[lotus] surfaces a 403 refusal's reason instead of the generic error", async () => {
|
||||
fetchMock.post("https://sfu.example.org/sfu/get", () => {
|
||||
return {
|
||||
status: 403,
|
||||
body: { errcode: "M_FORBIDDEN", error: "This voice channel is full." },
|
||||
};
|
||||
});
|
||||
try {
|
||||
await getSFUConfigWithOpenID(
|
||||
matrixClient,
|
||||
ownMemberMock,
|
||||
"https://sfu.example.org",
|
||||
"!example_room_id",
|
||||
);
|
||||
} catch (ex: unknown) {
|
||||
expect(ex).toBeInstanceOf(SFUTokenRefusedError);
|
||||
expect((ex as SFUTokenRefusedError).localisedMessage).toEqual(
|
||||
"This voice channel is full.",
|
||||
);
|
||||
void (await fetchMock.flush());
|
||||
return;
|
||||
}
|
||||
expect.fail("Expected test to throw;");
|
||||
});
|
||||
|
||||
it("should retry without delay params if the JWT service legacy endpoint returns M_BAD_JSON 400", async () => {
|
||||
let callCount = 0;
|
||||
|
||||
|
||||
@@ -8,6 +8,7 @@ Please see LICENSE in the repository root for full details.
|
||||
import {
|
||||
type IOpenIDToken,
|
||||
type MatrixClient,
|
||||
MatrixError,
|
||||
parseErrorResponse,
|
||||
} from "matrix-js-sdk";
|
||||
import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager";
|
||||
@@ -16,6 +17,7 @@ import { type Logger } from "matrix-js-sdk/lib/logger";
|
||||
import {
|
||||
FailToGetOpenIdToken,
|
||||
NoMatrix2AuthorizationService,
|
||||
SFUTokenRefusedError,
|
||||
} from "../utils/errors";
|
||||
import { doNetworkOperationWithRetry } from "../utils/matrix";
|
||||
import { Config } from "../config/Config";
|
||||
@@ -165,6 +167,14 @@ export async function getSFUConfigWithOpenID(
|
||||
logger?.info(`Got JWT from call's active focus URL.`);
|
||||
return extractFullConfigFromToken(sfuConfig);
|
||||
} catch (ex) {
|
||||
// [lotus] A 403 from the token service carries the reason the user needs
|
||||
// ("This voice channel is full.") — surface it instead of the generic error.
|
||||
if (ex instanceof MatrixError && ex.httpStatus === 403) {
|
||||
const reason = (ex.data as { error?: unknown } | undefined)?.error;
|
||||
if (typeof reason === "string" && reason.trim()) {
|
||||
throw new SFUTokenRefusedError(reason, ex);
|
||||
}
|
||||
}
|
||||
throw new FailToGetOpenIdToken(
|
||||
ex instanceof Error ? ex : new Error(`Unknown error ${ex}`),
|
||||
);
|
||||
|
||||
@@ -0,0 +1,47 @@
|
||||
/*
|
||||
Copyright 2026 Lotus Guild
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { describe, expect, test } from "vitest";
|
||||
|
||||
import { LOTUS_TO_WIDGET_ACTIONS, LotusWidgetActions } from "./lotusActions";
|
||||
|
||||
describe("LotusWidgetActions", () => {
|
||||
test("every action value is namespaced under io.lotus.*", () => {
|
||||
for (const value of Object.values(LotusWidgetActions)) {
|
||||
expect(value).toMatch(/^io\.lotus\./);
|
||||
}
|
||||
});
|
||||
|
||||
test("LOTUS_TO_WIDGET_ACTIONS contains exactly the toWidget actions", () => {
|
||||
// CallState is the only fromWidget action (host <- widget); everything
|
||||
// else is toWidget (host -> widget) and must be allow-listed so
|
||||
// `initializeWidget` accepts it.
|
||||
const expectedToWidget = [
|
||||
LotusWidgetActions.FocusParticipant,
|
||||
LotusWidgetActions.InjectAudio,
|
||||
LotusWidgetActions.SetQuality,
|
||||
LotusWidgetActions.Decorations,
|
||||
LotusWidgetActions.SetDeafen,
|
||||
];
|
||||
|
||||
expect(new Set(LOTUS_TO_WIDGET_ACTIONS)).toEqual(new Set(expectedToWidget));
|
||||
expect(LOTUS_TO_WIDGET_ACTIONS).toHaveLength(expectedToWidget.length);
|
||||
});
|
||||
|
||||
test("LOTUS_TO_WIDGET_ACTIONS excludes the fromWidget actions", () => {
|
||||
// CallState, RequestState and DenoiseState are all fromWidget (widget ->
|
||||
// host) per their doc comments in lotusActions.ts, so none of them should
|
||||
// ever be allow-listed as a toWidget action.
|
||||
expect(LOTUS_TO_WIDGET_ACTIONS).not.toContain(LotusWidgetActions.CallState);
|
||||
expect(LOTUS_TO_WIDGET_ACTIONS).not.toContain(
|
||||
LotusWidgetActions.RequestState,
|
||||
);
|
||||
expect(LOTUS_TO_WIDGET_ACTIONS).not.toContain(
|
||||
LotusWidgetActions.DenoiseState,
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -31,6 +31,21 @@ export enum LotusWidgetActions {
|
||||
Decorations = "io.lotus.decorations",
|
||||
/** toWidget: deafen remote audio (and optionally mute screenshare audio). */
|
||||
SetDeafen = "io.lotus.set_deafen",
|
||||
/**
|
||||
* fromWidget: sent on (re)registration of a lotus toWidget handler (e.g.
|
||||
* decorations) so the host can re-push state that would otherwise only be
|
||||
* sent on change (decorations roster, the current focus pin) after an
|
||||
* EC-side reconnect/remount. cinny's `resendForkState()` should respond to
|
||||
* this the same way it responds to a fresh join.
|
||||
*/
|
||||
RequestState = "io.lotus.request_state",
|
||||
/**
|
||||
* fromWidget: real state of the in-source denoise engine —
|
||||
* `{ active: boolean, model: string, error?: string }` — sent once the
|
||||
* processor attaches (or after the rnnoise fallback also fails), so the host
|
||||
* toggle can reflect reality rather than the requested state.
|
||||
*/
|
||||
DenoiseState = "io.lotus.denoise_state",
|
||||
}
|
||||
|
||||
/** toWidget Lotus actions that must be allow-listed in `initializeWidget`. */
|
||||
|
||||
@@ -88,6 +88,16 @@ afterEach(() => {
|
||||
});
|
||||
|
||||
describe("[lotus] audio-capture URL param overrides", () => {
|
||||
test("mic capture is always requested mono (stereo interfaces on Firefox published L-only)", () => {
|
||||
getUrlParams.mockReturnValue({
|
||||
echoCancellation: false,
|
||||
noiseSuppression: false,
|
||||
autoGainControl: false,
|
||||
});
|
||||
createRoom();
|
||||
expect(capturedAudioDefaults()).toMatchObject({ channelCount: 1 });
|
||||
});
|
||||
|
||||
test("with params defaulted to true, the Settings govern (upstream behaviour)", () => {
|
||||
getUrlParams.mockReturnValue({
|
||||
echoCancellation: true,
|
||||
|
||||
@@ -0,0 +1,225 @@
|
||||
/*
|
||||
Copyright 2026 Lotus Guild
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { EventEmitter } from "events";
|
||||
import { afterEach, beforeEach, expect, test, vi } from "vitest";
|
||||
import { of } from "rxjs";
|
||||
|
||||
import { type CallViewModel } from "../state/CallViewModel/CallViewModel";
|
||||
import { startLotusAudioInject } from "./lotusAudioInject";
|
||||
import { LotusWidgetActions } from "./lotusActions";
|
||||
|
||||
const lazyActions = new EventEmitter();
|
||||
const reply = vi.fn();
|
||||
|
||||
vi.mock("../widget", () => ({
|
||||
widget: {
|
||||
api: { transport: { reply: (...args: unknown[]) => reply(...args) } },
|
||||
// Getter: `vi.mock` factories run at import time, before the const above.
|
||||
get lazyActions(): EventEmitter {
|
||||
return lazyActions;
|
||||
},
|
||||
},
|
||||
}));
|
||||
|
||||
function send(data: unknown): void {
|
||||
lazyActions.emit(LotusWidgetActions.InjectAudio, { detail: { data } });
|
||||
}
|
||||
|
||||
/** Flush the microtask queue enough times to drain the async awaits in
|
||||
* `playInjectedClip` (fetch -> arrayBuffer -> resume -> decodeAudioData ->
|
||||
* publishTrack...), none of which use real timers in this test. */
|
||||
async function flush(times = 30): Promise<void> {
|
||||
for (let i = 0; i < times; i++) await Promise.resolve();
|
||||
}
|
||||
|
||||
function makeTrack(): MediaStreamTrack {
|
||||
return {
|
||||
clone: vi.fn(() => makeTrack()),
|
||||
stop: vi.fn(),
|
||||
} as unknown as MediaStreamTrack;
|
||||
}
|
||||
|
||||
function makeRoom(isMicrophoneEnabled: boolean): {
|
||||
localParticipant: {
|
||||
isMicrophoneEnabled: boolean;
|
||||
publishTrack: ReturnType<typeof vi.fn>;
|
||||
unpublishTrack: ReturnType<typeof vi.fn>;
|
||||
};
|
||||
} {
|
||||
return {
|
||||
localParticipant: {
|
||||
isMicrophoneEnabled,
|
||||
publishTrack: vi.fn(async (clone: unknown) => {
|
||||
await Promise.resolve();
|
||||
return { track: clone };
|
||||
}),
|
||||
unpublishTrack: vi.fn().mockResolvedValue(undefined),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function mockVm(rooms: unknown[]): CallViewModel {
|
||||
return {
|
||||
allConnections$: of({
|
||||
getConnections: () => rooms.map((livekitRoom) => ({ livekitRoom })),
|
||||
}),
|
||||
} as unknown as CallViewModel;
|
||||
}
|
||||
|
||||
/** Fake AudioContext/nodes good enough to drive playInjectedClip end to end,
|
||||
* tracking how many contexts and destinations get constructed (#14). */
|
||||
let contextInstances: FakeAudioContext[];
|
||||
class FakeAudioContext {
|
||||
public state = "running";
|
||||
public resume = vi.fn().mockResolvedValue(undefined);
|
||||
public close = vi.fn().mockResolvedValue(undefined);
|
||||
public decodeAudioData = vi
|
||||
.fn()
|
||||
.mockResolvedValue({ duration: 0.01 } as AudioBuffer);
|
||||
public createMediaStreamDestination = vi.fn(() => ({
|
||||
stream: { getAudioTracks: (): MediaStreamTrack[] => [makeTrack()] },
|
||||
}));
|
||||
public createGain = vi.fn(() => ({
|
||||
gain: { value: 0 },
|
||||
connect: vi.fn((n: unknown) => n),
|
||||
disconnect: vi.fn(),
|
||||
}));
|
||||
public createBufferSource = vi.fn(() => ({
|
||||
buffer: undefined as AudioBuffer | undefined,
|
||||
connect: vi.fn((n: unknown) => n),
|
||||
disconnect: vi.fn(),
|
||||
start: vi.fn(),
|
||||
stop: vi.fn(),
|
||||
onended: null as (() => void) | null,
|
||||
addEventListener: vi.fn(),
|
||||
}));
|
||||
public constructor() {
|
||||
contextInstances.push(this);
|
||||
}
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
contextInstances = [];
|
||||
reply.mockClear();
|
||||
// The module sets a real `setTimeout` guard per clip (MAX_CLIP_MS safety
|
||||
// net); use fake timers so a test ending before that guard fires doesn't
|
||||
// leave a real timer pending.
|
||||
vi.useFakeTimers();
|
||||
vi.stubGlobal("AudioContext", FakeAudioContext);
|
||||
vi.stubGlobal(
|
||||
"fetch",
|
||||
vi.fn().mockResolvedValue({
|
||||
ok: true,
|
||||
arrayBuffer: async () => {
|
||||
await Promise.resolve();
|
||||
return new ArrayBuffer(8);
|
||||
},
|
||||
}),
|
||||
);
|
||||
// `lotusParam`/`lotusFlag` re-parse `window.location` on every call, so
|
||||
// just set the flag before each test.
|
||||
window.location.hash = "#/room?lotusAudioInject=1";
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
lazyActions.removeAllListeners();
|
||||
vi.unstubAllGlobals();
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
test("#13: a muted local mic blocks the clip and replies with a machine-readable reason", async () => {
|
||||
const room = makeRoom(false);
|
||||
const vm = mockVm([room]);
|
||||
const stop = startLotusAudioInject(vm);
|
||||
|
||||
send({ url: "https://example.com/clip.mp3" });
|
||||
await flush();
|
||||
|
||||
expect(reply).toHaveBeenCalledWith(expect.anything(), {
|
||||
played: false,
|
||||
reason: "muted",
|
||||
});
|
||||
expect(room.localParticipant.publishTrack).not.toHaveBeenCalled();
|
||||
// No context should even be created for a clip that never plays.
|
||||
expect(contextInstances).toHaveLength(0);
|
||||
|
||||
stop();
|
||||
});
|
||||
|
||||
test("#13: an unmuted local mic plays the clip normally", async () => {
|
||||
const room = makeRoom(true);
|
||||
const vm = mockVm([room]);
|
||||
const stop = startLotusAudioInject(vm);
|
||||
|
||||
send({ url: "https://example.com/clip.mp3" });
|
||||
await flush();
|
||||
|
||||
expect(reply).toHaveBeenCalledWith(expect.anything(), {});
|
||||
expect(room.localParticipant.publishTrack).toHaveBeenCalledTimes(1);
|
||||
|
||||
stop();
|
||||
});
|
||||
|
||||
test("#14: one shared AudioContext/destination is reused across clips, and closed only on the last teardown", async () => {
|
||||
const room = makeRoom(true);
|
||||
const vm = mockVm([room]);
|
||||
const stop = startLotusAudioInject(vm);
|
||||
|
||||
send({ url: "https://example.com/a.mp3" });
|
||||
await flush();
|
||||
expect(contextInstances).toHaveLength(1);
|
||||
const ctx = contextInstances[0];
|
||||
expect(ctx.createMediaStreamDestination).toHaveBeenCalledTimes(1);
|
||||
|
||||
// Finish the first clip (as if it played to completion) before starting a
|
||||
// second one, matching the module's own "one clip at a time" contract.
|
||||
const firstSource = ctx.createBufferSource.mock.results[0]!.value as {
|
||||
onended: (() => void) | null;
|
||||
};
|
||||
firstSource.onended?.();
|
||||
await flush();
|
||||
expect(room.localParticipant.unpublishTrack).toHaveBeenCalledTimes(1);
|
||||
|
||||
// A second clip must reuse the SAME context/destination rather than
|
||||
// creating a new one.
|
||||
send({ url: "https://example.com/b.mp3" });
|
||||
await flush();
|
||||
expect(contextInstances).toHaveLength(1);
|
||||
expect(ctx.createMediaStreamDestination).toHaveBeenCalledTimes(1);
|
||||
expect(ctx.close).not.toHaveBeenCalled();
|
||||
|
||||
const secondSource = ctx.createBufferSource.mock.results[1]!.value as {
|
||||
onended: (() => void) | null;
|
||||
};
|
||||
secondSource.onended?.();
|
||||
await flush();
|
||||
|
||||
// Tearing down the (only) active instance closes the shared context.
|
||||
stop();
|
||||
expect(ctx.close).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
test("#14: the shared context stays open while another instance is still active", async () => {
|
||||
const roomA = makeRoom(true);
|
||||
const roomB = makeRoom(true);
|
||||
const stopA = startLotusAudioInject(mockVm([roomA]));
|
||||
const stopB = startLotusAudioInject(mockVm([roomB]));
|
||||
|
||||
send({ url: "https://example.com/a.mp3" });
|
||||
await flush();
|
||||
expect(contextInstances).toHaveLength(1);
|
||||
const ctx = contextInstances[0];
|
||||
|
||||
// Tearing down the first (of two) active instances must not close the
|
||||
// context out from under the other one.
|
||||
stopA();
|
||||
expect(ctx.close).not.toHaveBeenCalled();
|
||||
|
||||
stopB();
|
||||
expect(ctx.close).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
@@ -17,6 +17,30 @@ import { lotusFlag } from "./lotusWidget";
|
||||
/** Hard cap so a malformed/huge clip can't hold a published track open forever. */
|
||||
const MAX_CLIP_MS = 30_000;
|
||||
|
||||
// [lotus] One shared module-level AudioContext + MediaStreamAudioDestinationNode
|
||||
// for ALL injected clips (#14), instead of `new AudioContext()` per clip. An
|
||||
// in-call page already sits close to Chrome's per-document AudioContext limit
|
||||
// (~6) between useAudioContext, MatrixAudioRenderer, LiveKit's own Room
|
||||
// context and LotusDenoiseProcessor; rapid clip replacement (replace-mode
|
||||
// closing the previous clip's context in the background) could transiently
|
||||
// exceed the cap and make `new AudioContext()` throw. Lazily created on first
|
||||
// use, ref-counted by the number of active `startLotusAudioInject` instances,
|
||||
// and closed only when the last one tears down.
|
||||
let sharedCtx: AudioContext | undefined;
|
||||
let sharedDest: MediaStreamAudioDestinationNode | undefined;
|
||||
let handlerCount = 0;
|
||||
|
||||
function acquireSharedAudio(): {
|
||||
ctx: AudioContext;
|
||||
dest: MediaStreamAudioDestinationNode;
|
||||
} {
|
||||
if (!sharedCtx || sharedCtx.state === "closed") {
|
||||
sharedCtx = new AudioContext();
|
||||
sharedDest = sharedCtx.createMediaStreamDestination();
|
||||
}
|
||||
return { ctx: sharedCtx, dest: sharedDest! };
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle the host's `io.lotus.inject_audio` toWidget action (#3): mix a
|
||||
* soundboard clip into the call so other participants hear it.
|
||||
@@ -38,6 +62,10 @@ export function startLotusAudioInject(vm: CallViewModel): () => void {
|
||||
const w = widget;
|
||||
if (!w) return () => undefined;
|
||||
|
||||
// [lotus] Count this instance toward the shared AudioContext's lifetime
|
||||
// (#14) — closed only once the last active instance tears down.
|
||||
handlerCount++;
|
||||
|
||||
// Track the set of connected LiveKit rooms to publish into. Drive off the
|
||||
// LOCAL participant's connection(s), not `livekitRoomItems$` — that stream
|
||||
// omits rooms with no remote members, so inject would no-op while you're
|
||||
@@ -52,16 +80,19 @@ export function startLotusAudioInject(vm: CallViewModel): () => void {
|
||||
const activeClips = new Set<() => void>();
|
||||
|
||||
const handler = (ev: CustomEvent<IWidgetApiRequest>): void => {
|
||||
// Always ack so the transport doesn't hang, but only act when the host has
|
||||
// explicitly opted in: audio-inject publishes under the local user's
|
||||
// identity, so it must not be silently armed for every call.
|
||||
w.api.transport.reply(ev.detail, {});
|
||||
if (!lotusFlag("lotusAudioInject")) return;
|
||||
if (!lotusFlag("lotusAudioInject")) {
|
||||
// Always ack so the transport doesn't hang, but only act when the host
|
||||
// has explicitly opted in: audio-inject publishes under the local
|
||||
// user's identity, so it must not be silently armed for every call.
|
||||
w.api.transport.reply(ev.detail, {});
|
||||
return;
|
||||
}
|
||||
const data = ev.detail.data as
|
||||
| { url?: unknown; volume?: unknown }
|
||||
| undefined;
|
||||
const url = typeof data?.url === "string" ? safeMediaUrl(data.url) : null;
|
||||
if (!url) {
|
||||
w.api.transport.reply(ev.detail, {});
|
||||
logger.warn("[lotus] inject_audio: missing/invalid url");
|
||||
return;
|
||||
}
|
||||
@@ -69,6 +100,21 @@ export function startLotusAudioInject(vm: CallViewModel): () => void {
|
||||
typeof data?.volume === "number" && data.volume >= 0 && data.volume <= 1
|
||||
? data.volume
|
||||
: 1;
|
||||
|
||||
// [lotus] Gate on the local mic being enabled (#13): the clip is
|
||||
// published as an independent track, fully decoupled from the mic
|
||||
// publication's mute state, so without this a muted (or push-to-talk
|
||||
// idle) user could still transmit soundboard audio under their own
|
||||
// identity — breaking the "I am muted, nothing I do makes noise" mental
|
||||
// model. Reply with a machine-readable reason so cinny's soundboard UI
|
||||
// can surface a hint instead of the click silently doing nothing.
|
||||
const micEnabled = rooms[0]?.localParticipant.isMicrophoneEnabled ?? true;
|
||||
if (!micEnabled) {
|
||||
w.api.transport.reply(ev.detail, { played: false, reason: "muted" });
|
||||
return;
|
||||
}
|
||||
|
||||
w.api.transport.reply(ev.detail, {});
|
||||
void playInjectedClip(url, volume, rooms, activeClips).catch((e) =>
|
||||
logger.warn("[lotus] inject_audio failed", e),
|
||||
);
|
||||
@@ -86,6 +132,15 @@ export function startLotusAudioInject(vm: CallViewModel): () => void {
|
||||
// activeClips while we iterate it.
|
||||
// eslint-disable-next-line unicorn/no-useless-spread
|
||||
for (const abort of [...activeClips]) abort();
|
||||
// [lotus] Close the shared AudioContext only when the last active
|
||||
// instance tears down (#14).
|
||||
handlerCount--;
|
||||
if (handlerCount === 0 && sharedCtx) {
|
||||
const ctx = sharedCtx;
|
||||
sharedCtx = undefined;
|
||||
sharedDest = undefined;
|
||||
void ctx.close().catch(() => undefined);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
@@ -146,23 +201,25 @@ async function playInjectedClip(
|
||||
throw e;
|
||||
}
|
||||
if (aborted) return;
|
||||
if (!resp.ok) throw new Error(`fetch ${url} -> ${resp.status}`);
|
||||
if (!resp.ok) {
|
||||
activeClips.delete(placeholder);
|
||||
throw new Error(`fetch ${url} -> ${resp.status}`);
|
||||
}
|
||||
const arrayBuffer = await resp.arrayBuffer();
|
||||
if (aborted) return;
|
||||
|
||||
const ctx = new AudioContext();
|
||||
// [lotus] Reuse the shared module-level context/destination (#14) rather
|
||||
// than `new AudioContext()` per clip — see the declaration above.
|
||||
const { ctx, dest } = acquireSharedAudio();
|
||||
// The action arrives via host postMessage, not a gesture in this iframe, so
|
||||
// the context may start suspended — resume it or the clip is silent and
|
||||
// `onended` never fires.
|
||||
// `onended` never fires. A no-op if an earlier clip already resumed it.
|
||||
try {
|
||||
await ctx.resume();
|
||||
} catch {
|
||||
/* best effort */
|
||||
}
|
||||
if (aborted) {
|
||||
void ctx.close();
|
||||
return;
|
||||
}
|
||||
if (aborted) return;
|
||||
if (ctx.state !== "running")
|
||||
logger.warn(`[lotus] inject_audio: AudioContext is ${ctx.state}`);
|
||||
|
||||
@@ -170,15 +227,13 @@ async function playInjectedClip(
|
||||
try {
|
||||
buffer = await ctx.decodeAudioData(arrayBuffer);
|
||||
} catch (e) {
|
||||
void ctx.close();
|
||||
activeClips.delete(placeholder);
|
||||
throw e;
|
||||
}
|
||||
if (aborted) {
|
||||
void ctx.close();
|
||||
return;
|
||||
}
|
||||
if (aborted) return;
|
||||
|
||||
const dest = ctx.createMediaStreamDestination();
|
||||
// Per clip, only the BufferSource/GainNode are created (#14) — the shared
|
||||
// context/destination are reused across every clip.
|
||||
const gain = ctx.createGain();
|
||||
gain.gain.value = volume;
|
||||
const source = ctx.createBufferSource();
|
||||
@@ -187,7 +242,9 @@ async function playInjectedClip(
|
||||
|
||||
const mst = dest.stream.getAudioTracks()[0];
|
||||
if (!mst) {
|
||||
void ctx.close();
|
||||
source.disconnect();
|
||||
gain.disconnect();
|
||||
activeClips.delete(placeholder);
|
||||
throw new Error("no audio track from destination");
|
||||
}
|
||||
|
||||
@@ -221,13 +278,17 @@ async function playInjectedClip(
|
||||
} catch {
|
||||
/* already stopped */
|
||||
}
|
||||
// [lotus] Dispose only this clip's own nodes (#14) — the shared
|
||||
// AudioContext/destination outlive it and are closed separately, only
|
||||
// when the last startLotusAudioInject instance tears down.
|
||||
source.disconnect();
|
||||
gain.disconnect();
|
||||
for (const entry of publications) {
|
||||
if (entry?.pub.track)
|
||||
void entry.room.localParticipant
|
||||
.unpublishTrack(entry.pub.track, true)
|
||||
.catch(() => undefined);
|
||||
}
|
||||
void ctx.close().catch(() => undefined);
|
||||
};
|
||||
// Swap the synchronous placeholder for the real cleanup: from here an abort
|
||||
// (teardown or a newer clip) must unpublish the LIVE track, not just cancel a
|
||||
|
||||
@@ -0,0 +1,128 @@
|
||||
/*
|
||||
Copyright 2026 Lotus Guild
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { afterEach, beforeEach, expect, test, vi } from "vitest";
|
||||
import { BehaviorSubject, of } from "rxjs";
|
||||
|
||||
import { type CallViewModel } from "../state/CallViewModel/CallViewModel";
|
||||
import { startLotusCallState } from "./lotusCallState";
|
||||
|
||||
const send = vi.fn().mockResolvedValue({});
|
||||
|
||||
vi.mock("../widget", () => ({
|
||||
widget: { api: { transport: { send: (...a: unknown[]) => send(...a) } } },
|
||||
}));
|
||||
|
||||
// `lotusFlag`/`lotusParam` (lotusWidget.ts) read `window.location` directly
|
||||
// and memoize per module load, so drive the flag through the URL once, at
|
||||
// import time (NOT per-test/in a hook: re-navigating with
|
||||
// `window.history.pushState` between tests was observed to corrupt rxjs's
|
||||
// shared `asyncScheduler` under `vi.useFakeTimers()`, silently starving a
|
||||
// later test's `throttleTime` of any emission).
|
||||
window.history.pushState({}, "", "/?lotusCallState=1");
|
||||
|
||||
beforeEach(() => {
|
||||
send.mockClear();
|
||||
vi.useFakeTimers();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
interface Member {
|
||||
id: string;
|
||||
userId: string;
|
||||
speaking$: BehaviorSubject<boolean>;
|
||||
audioEnabled$: BehaviorSubject<boolean>;
|
||||
videoEnabled$: BehaviorSubject<boolean>;
|
||||
}
|
||||
|
||||
function mockMember(id: string, userId: string): Member {
|
||||
return {
|
||||
id,
|
||||
userId,
|
||||
speaking$: new BehaviorSubject(false),
|
||||
audioEnabled$: new BehaviorSubject(true),
|
||||
videoEnabled$: new BehaviorSubject(true),
|
||||
};
|
||||
}
|
||||
|
||||
function mockVm(members: Member[]): CallViewModel {
|
||||
return { userMedia$: of(members) } as unknown as CallViewModel;
|
||||
}
|
||||
|
||||
function participantsOf(call: number): unknown[] {
|
||||
return (send.mock.calls[call][1] as { participants: unknown[] }).participants;
|
||||
}
|
||||
|
||||
test("[lotus #20] rapid speaking toggles in a busy call are capped well under the old 8/sec rate", () => {
|
||||
const members = Array.from({ length: 15 }, (_, i) =>
|
||||
mockMember(`@u${i}:example.org:DEV`, `@u${i}:example.org`),
|
||||
);
|
||||
const stop = startLotusCallState(mockVm(members));
|
||||
|
||||
// Simulate a noisy multi-person conversation: flip `speaking` on every
|
||||
// member every 20ms (50Hz of raw churn) for 2 seconds.
|
||||
for (let t = 0; t < 2000; t += 20) {
|
||||
for (const m of members) m.speaking$.next(!m.speaking$.value);
|
||||
vi.advanceTimersByTime(20);
|
||||
}
|
||||
|
||||
// The old leading+trailing 250ms throttle allowed ~8 sends/sec => up to 16
|
||||
// over 2s. Trailing-only 500ms must cap this to at most 4 (one per window).
|
||||
expect(send.mock.calls.length).toBeLessThanOrEqual(5);
|
||||
expect(send.mock.calls.length).toBeGreaterThan(0);
|
||||
|
||||
// Drain any still-pending trailing-edge throttle action before tearing
|
||||
// down: `rxjs`'s default `asyncScheduler` is a process-wide singleton, and
|
||||
// leaving a scheduled action dangling across a test/timer-implementation
|
||||
// boundary can wedge its queue for every later test in this file.
|
||||
vi.advanceTimersByTime(500);
|
||||
stop();
|
||||
});
|
||||
|
||||
test("[lotus #20] a mute/camera change and a same-tick no-op speaking flip are deduped field-wise", () => {
|
||||
const alice = mockMember("@alice:example.org:DEV", "@alice:example.org");
|
||||
const stop = startLotusCallState(mockVm([alice]));
|
||||
|
||||
vi.advanceTimersByTime(500);
|
||||
send.mockClear();
|
||||
|
||||
// No actual change: re-emitting the same speaking value must not count as
|
||||
// a change (distinctUntilChanged happens before the throttle).
|
||||
alice.speaking$.next(false);
|
||||
vi.advanceTimersByTime(500);
|
||||
expect(send).not.toHaveBeenCalled();
|
||||
|
||||
alice.audioEnabled$.next(false);
|
||||
vi.advanceTimersByTime(500);
|
||||
expect(send).toHaveBeenCalledTimes(1);
|
||||
expect(participantsOf(0)).toEqual([
|
||||
{
|
||||
id: "@alice:example.org:DEV",
|
||||
userId: "@alice:example.org",
|
||||
speaking: false,
|
||||
audioEnabled: false,
|
||||
videoEnabled: true,
|
||||
},
|
||||
]);
|
||||
|
||||
stop();
|
||||
});
|
||||
|
||||
test("does nothing (and sends nothing) once torn down", () => {
|
||||
const alice = mockMember("@alice:example.org:DEV", "@alice:example.org");
|
||||
const stop = startLotusCallState(mockVm([alice]));
|
||||
vi.advanceTimersByTime(500);
|
||||
send.mockClear();
|
||||
|
||||
stop();
|
||||
alice.speaking$.next(true);
|
||||
vi.advanceTimersByTime(1000);
|
||||
expect(send).not.toHaveBeenCalled();
|
||||
});
|
||||
@@ -27,6 +27,29 @@ interface ParticipantState {
|
||||
videoEnabled: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* [lotus #20] Field-wise equality for `ParticipantState[]`, used in place of
|
||||
* `JSON.stringify` comparison: cheaper (no serialisation of every
|
||||
* participant on every emission) and just as correct, since array order here
|
||||
* is stable (it mirrors `members` from `userMedia$`).
|
||||
*/
|
||||
function participantsEqual(
|
||||
a: ParticipantState[],
|
||||
b: ParticipantState[],
|
||||
): boolean {
|
||||
return (
|
||||
a.length === b.length &&
|
||||
a.every(
|
||||
(p, i) =>
|
||||
p.id === b[i].id &&
|
||||
p.userId === b[i].userId &&
|
||||
p.speaking === b[i].speaking &&
|
||||
p.audioEnabled === b[i].audioEnabled &&
|
||||
p.videoEnabled === b[i].videoEnabled,
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Stream per-participant speaking / mute / camera state to the Lotus host
|
||||
* (cinny) over the widget API, so the host can drive speaking rings, mute
|
||||
@@ -73,11 +96,19 @@ export function startLotusCallState(vm: CallViewModel): () => void {
|
||||
),
|
||||
),
|
||||
// `speaking` flips rapidly; drop no-op repeats BEFORE throttling so
|
||||
// the throttle window isn't spent re-emitting an unchanged value, then
|
||||
// cap the send rate. 250ms is plenty for speaking rings / mute badges
|
||||
// and keeps the request/response widget traffic modest.
|
||||
distinctUntilChanged((a, b) => JSON.stringify(a) === JSON.stringify(b)),
|
||||
throttleTime(250, undefined, { leading: true, trailing: true }),
|
||||
// the throttle window isn't spent re-emitting an unchanged value.
|
||||
// Field-wise (cheaper than JSON.stringify, and correct: array order is
|
||||
// stable since it mirrors `members` from userMedia$).
|
||||
distinctUntilChanged(participantsEqual),
|
||||
// [lotus #20] `speaking` is the field that flips constantly in an active
|
||||
// conversation; mute/camera toggles are rare and user-intentional and
|
||||
// would ideally stay prompt, but a single combined stream is far
|
||||
// simpler than splitting it, and the leading+trailing 250ms window
|
||||
// previously allowed ~8 sends/sec (each re-serialising every
|
||||
// participant) in a busy call. Trailing-only + a longer window caps
|
||||
// that to 2/sec while still reflecting mute/camera changes within
|
||||
// 500ms.
|
||||
throttleTime(500, undefined, { leading: false, trailing: true }),
|
||||
)
|
||||
.subscribe((participants) => {
|
||||
lotusSendToHost(LotusWidgetActions.CallState, { participants });
|
||||
|
||||
@@ -7,13 +7,11 @@ Please see LICENSE in the repository root for full details.
|
||||
|
||||
import { EventEmitter } from "events";
|
||||
import { afterEach, beforeEach, expect, test, vi } from "vitest";
|
||||
import { of } from "rxjs";
|
||||
import { Track } from "livekit-client";
|
||||
|
||||
import { type CallViewModel } from "../state/CallViewModel/CallViewModel";
|
||||
import { setAudioEnabled$ } from "../controls";
|
||||
import { startLotusDeafen } from "./lotusDeafen";
|
||||
import { LotusWidgetActions } from "./lotusActions";
|
||||
import { setScreenshareAudioMuted$ } from "./lotusScreenshareAudio";
|
||||
|
||||
const lazyActions = new EventEmitter();
|
||||
|
||||
@@ -27,18 +25,6 @@ vi.mock("../widget", () => ({
|
||||
},
|
||||
}));
|
||||
|
||||
/** Minimal CallViewModel stub: no connections, so no livekit rooms. */
|
||||
function mockVm(participants: unknown[] = []): CallViewModel {
|
||||
const livekitRoom = {
|
||||
remoteParticipants: new Map(participants.map((p, i) => [String(i), p])),
|
||||
on: vi.fn(),
|
||||
off: vi.fn(),
|
||||
};
|
||||
return {
|
||||
allConnections$: of({ getConnections: () => [{ livekitRoom }] }),
|
||||
} as unknown as CallViewModel;
|
||||
}
|
||||
|
||||
function send(data: unknown): void {
|
||||
lazyActions.emit(LotusWidgetActions.SetDeafen, {
|
||||
detail: { data },
|
||||
@@ -60,7 +46,7 @@ afterEach(() => {
|
||||
});
|
||||
|
||||
test("deafen mutes, and undeafen unmutes, EC's global audio output", () => {
|
||||
const stop = startLotusDeafen(mockVm());
|
||||
const stop = startLotusDeafen();
|
||||
|
||||
send({ deafened: true, screenshareAudioMuted: false });
|
||||
expect(emissions).toEqual([false]);
|
||||
@@ -73,7 +59,7 @@ test("deafen mutes, and undeafen unmutes, EC's global audio output", () => {
|
||||
});
|
||||
|
||||
test("re-sending the same state is idempotent (host resendForkState)", () => {
|
||||
const stop = startLotusDeafen(mockVm());
|
||||
const stop = startLotusDeafen();
|
||||
|
||||
send({ deafened: true, screenshareAudioMuted: false });
|
||||
send({ deafened: true, screenshareAudioMuted: false });
|
||||
@@ -86,7 +72,7 @@ test("re-sending the same state is idempotent (host resendForkState)", () => {
|
||||
});
|
||||
|
||||
test("undeafen does not re-enable audio the user had muted themselves", () => {
|
||||
const stop = startLotusDeafen(mockVm());
|
||||
const stop = startLotusDeafen();
|
||||
|
||||
// The user mutes all audio through EC's own control first.
|
||||
setAudioEnabled$.next(false);
|
||||
@@ -101,48 +87,44 @@ test("undeafen does not re-enable audio the user had muted themselves", () => {
|
||||
expect(emissions).toEqual([false]);
|
||||
});
|
||||
|
||||
test("deafen never touches per-participant volume", () => {
|
||||
const participant = { setVolume: vi.fn() };
|
||||
const stop = startLotusDeafen(mockVm([participant]));
|
||||
test("deafen never touches the screenshare-audio flag", () => {
|
||||
const stop = startLotusDeafen();
|
||||
|
||||
send({ deafened: true, screenshareAudioMuted: false });
|
||||
send({ deafened: false, screenshareAudioMuted: false });
|
||||
expect(participant.setVolume).not.toHaveBeenCalled();
|
||||
expect(setScreenshareAudioMuted$.value).toBe(false);
|
||||
|
||||
stop();
|
||||
});
|
||||
|
||||
test("screenshare-audio mute is applied per source and only undone for participants we muted", () => {
|
||||
const participant = { setVolume: vi.fn() };
|
||||
const stop = startLotusDeafen(mockVm([participant]));
|
||||
test("screenshare-audio mute is a global flag the renderer reads, reset on teardown", () => {
|
||||
const seen: boolean[] = [];
|
||||
const flagSub = setScreenshareAudioMuted$.subscribe((v) => seen.push(v));
|
||||
const stop = startLotusDeafen();
|
||||
|
||||
// Not muted yet: no volume writes at all.
|
||||
// Not muted yet: nothing pushed beyond the BehaviorSubject's initial value.
|
||||
send({ deafened: false, screenshareAudioMuted: false });
|
||||
expect(participant.setVolume).not.toHaveBeenCalled();
|
||||
expect(seen).toEqual([false]);
|
||||
|
||||
send({ deafened: false, screenshareAudioMuted: true });
|
||||
expect(participant.setVolume).toHaveBeenCalledWith(
|
||||
0,
|
||||
Track.Source.ScreenShareAudio,
|
||||
);
|
||||
expect(setScreenshareAudioMuted$.value).toBe(true);
|
||||
// Re-applying the same state (host resend after reconnect) is a no-op.
|
||||
send({ deafened: false, screenshareAudioMuted: true });
|
||||
expect(seen).toEqual([false, true]);
|
||||
|
||||
participant.setVolume.mockClear();
|
||||
send({ deafened: false, screenshareAudioMuted: false });
|
||||
expect(participant.setVolume).toHaveBeenCalledWith(
|
||||
1,
|
||||
Track.Source.ScreenShareAudio,
|
||||
);
|
||||
|
||||
// Releasing again writes nothing: we no longer own that participant.
|
||||
participant.setVolume.mockClear();
|
||||
send({ deafened: false, screenshareAudioMuted: false });
|
||||
expect(participant.setVolume).not.toHaveBeenCalled();
|
||||
// Deafen + undeafen while screenshare audio is muted leaves it muted.
|
||||
send({ deafened: true, screenshareAudioMuted: true });
|
||||
send({ deafened: false, screenshareAudioMuted: true });
|
||||
expect(setScreenshareAudioMuted$.value).toBe(true);
|
||||
|
||||
// Leaving the call clears it so the next call starts clean.
|
||||
stop();
|
||||
expect(setScreenshareAudioMuted$.value).toBe(false);
|
||||
flagSub.unsubscribe();
|
||||
});
|
||||
|
||||
test("a partial payload only moves the flag it names", () => {
|
||||
const stop = startLotusDeafen(mockVm());
|
||||
const stop = startLotusDeafen();
|
||||
|
||||
send({ deafened: true });
|
||||
expect(emissions).toEqual([false]);
|
||||
|
||||
+17
-71
@@ -5,19 +5,13 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import {
|
||||
type RemoteParticipant,
|
||||
type Room as LivekitRoom,
|
||||
RoomEvent,
|
||||
Track,
|
||||
} from "livekit-client";
|
||||
import { logger } from "matrix-js-sdk/lib/logger";
|
||||
import { type IWidgetApiRequest } from "matrix-widget-api";
|
||||
|
||||
import { type CallViewModel } from "../state/CallViewModel/CallViewModel";
|
||||
import { setAudioEnabled$ } from "../controls";
|
||||
import { widget } from "../widget";
|
||||
import { LotusWidgetActions } from "./lotusActions";
|
||||
import { setScreenshareAudioMuted$ } from "./lotusScreenshareAudio";
|
||||
|
||||
/**
|
||||
* Handle the host's `io.lotus.set_deafen` toWidget action, replacing cinny's
|
||||
@@ -41,22 +35,23 @@ import { LotusWidgetActions } from "./lotusActions";
|
||||
* volume/mute state.
|
||||
*
|
||||
* `screenshareAudioMuted` is a NARROWER, independent host control (drop shared
|
||||
* tab/game audio while still hearing voices). EC has no global per-source mute,
|
||||
* so that one still has to go through
|
||||
* `RemoteParticipant.setVolume(volume, Track.Source.ScreenShareAudio)` — whose
|
||||
* verified signature in livekit-client ^2.18.1 is
|
||||
* `setVolume(volume, source?: Track.Source.Microphone | Track.Source.ScreenShareAudio)`.
|
||||
* It is re-applied to late joiners via `RoomEvent.ParticipantConnected`, and we
|
||||
* only ever restore participants we muted ourselves. Known limitation: EC's own
|
||||
* screenshare volume slider writes the same `volumeMap`, so a user who moves
|
||||
* that slider while screenshare audio is host-muted wins; the mic path (the
|
||||
* actual deafen) is no longer affected by that race at all.
|
||||
* tab/game audio while still hearing voices). It goes through the same
|
||||
* mechanism as deafen — a global behavior (`muteScreenshareAudio$`) that
|
||||
* `LivekitRoomAudioRenderer` turns into the `muted` prop of every
|
||||
* `Track.Source.ScreenShareAudio` element — rather than
|
||||
* `RemoteParticipant.setVolume(0, ScreenShareAudio)`: EC's own
|
||||
* `createVolumeControls` writes volume 1 through that very setter whenever a
|
||||
* new screenshare media item resolves, so a sharer who stopped and re-shared
|
||||
* (or a late joiner's share) came back at full volume while the host's button
|
||||
* still said "Unmute Screenshare Audio".
|
||||
*
|
||||
* Undeafen restores the user's OWN output-enabled state as it was before the
|
||||
* deafen (and never touches the `mute-all-audio` setting), so a user who had
|
||||
* already muted all audio themselves stays muted.
|
||||
*
|
||||
* State is closure-scoped (per invocation, matching the sibling lotus modules).
|
||||
* State is closure-scoped (per invocation, matching the sibling lotus modules);
|
||||
* the screenshare-audio flag additionally lives in `setScreenshareAudioMuted$`
|
||||
* so the renderer can read it, and is reset on teardown.
|
||||
* Applying the same state twice is a no-op, so the host's
|
||||
* `CallControl.resendForkState()` after a reconnect is safe. The host re-sends
|
||||
* the current state on every call join (CallControl.forceState), so a fresh
|
||||
@@ -64,7 +59,7 @@ import { LotusWidgetActions } from "./lotusActions";
|
||||
*
|
||||
* No effect unless the host sends the action. Returns a teardown function.
|
||||
*/
|
||||
export function startLotusDeafen(vm: CallViewModel): () => void {
|
||||
export function startLotusDeafen(): () => void {
|
||||
const w = widget;
|
||||
if (!w) return () => undefined;
|
||||
|
||||
@@ -107,53 +102,6 @@ export function startLotusDeafen(vm: CallViewModel): () => void {
|
||||
}
|
||||
});
|
||||
|
||||
// Participants whose screenshare audio WE turned down, so undoing the host's
|
||||
// screenshare mute never writes a volume to anyone else.
|
||||
const screenshareMuted = new Set<RemoteParticipant>();
|
||||
|
||||
const applyToParticipant = (p: RemoteParticipant): void => {
|
||||
if (screenshareAudioMuted) {
|
||||
p.setVolume(0, Track.Source.ScreenShareAudio);
|
||||
screenshareMuted.add(p);
|
||||
} else if (screenshareMuted.delete(p)) {
|
||||
p.setVolume(1, Track.Source.ScreenShareAudio);
|
||||
}
|
||||
};
|
||||
|
||||
const applyToRoom = (room: LivekitRoom): void =>
|
||||
room.remoteParticipants.forEach(applyToParticipant);
|
||||
|
||||
// Per-room ParticipantConnected listeners, so LATE JOINERS pick up the
|
||||
// current screenshare-audio mute the moment they connect. Drive off the
|
||||
// local participant's connection(s), not `livekitRoomItems$` — that stream is
|
||||
// empty until a matrix-validated REMOTE member resolves, so listeners would
|
||||
// be attached too late for the first joiner (the sibling lotus modules all
|
||||
// use `allConnections$` for the same reason).
|
||||
const roomListeners = new Map<LivekitRoom, () => void>();
|
||||
let rooms: LivekitRoom[] = [];
|
||||
|
||||
const sub = vm.allConnections$.subscribe((data) => {
|
||||
const next = data.getConnections().map((c) => c.livekitRoom);
|
||||
rooms = next;
|
||||
// Detach listeners for rooms that went away.
|
||||
for (const [room, off] of roomListeners) {
|
||||
if (!next.includes(room)) {
|
||||
off();
|
||||
roomListeners.delete(room);
|
||||
}
|
||||
}
|
||||
// Attach to new rooms + apply the current state to their participants.
|
||||
for (const room of next) {
|
||||
if (!roomListeners.has(room)) {
|
||||
room.on(RoomEvent.ParticipantConnected, applyToParticipant);
|
||||
roomListeners.set(room, () =>
|
||||
room.off(RoomEvent.ParticipantConnected, applyToParticipant),
|
||||
);
|
||||
applyToRoom(room);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
const handler = (ev: CustomEvent<IWidgetApiRequest>): void => {
|
||||
w.api.transport.reply(ev.detail, {});
|
||||
const data = ev.detail.data as
|
||||
@@ -168,20 +116,18 @@ export function startLotusDeafen(vm: CallViewModel): () => void {
|
||||
`[lotus] set_deafen: deafened=${deafened} screenshareAudioMuted=${screenshareAudioMuted}`,
|
||||
);
|
||||
applyGlobalMute();
|
||||
rooms.forEach(applyToRoom);
|
||||
if (setScreenshareAudioMuted$.value !== screenshareAudioMuted)
|
||||
setScreenshareAudioMuted$.next(screenshareAudioMuted);
|
||||
};
|
||||
|
||||
w.lazyActions.on(LotusWidgetActions.SetDeafen, handler);
|
||||
return () => {
|
||||
sub.unsubscribe();
|
||||
for (const off of roomListeners.values()) off();
|
||||
roomListeners.clear();
|
||||
// Leave the user's own output state as they had it before deafen.
|
||||
if (deafened) {
|
||||
deafened = false;
|
||||
applyGlobalMute();
|
||||
}
|
||||
screenshareMuted.clear();
|
||||
if (setScreenshareAudioMuted$.value) setScreenshareAudioMuted$.next(false);
|
||||
audioSub.unsubscribe();
|
||||
w.lazyActions.off(LotusWidgetActions.SetDeafen, handler);
|
||||
};
|
||||
|
||||
@@ -0,0 +1,127 @@
|
||||
/*
|
||||
Copyright 2026 Lotus Guild
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { EventEmitter } from "events";
|
||||
import { afterEach, beforeEach, expect, test, vi } from "vitest";
|
||||
import { act, renderHook } from "@testing-library/react";
|
||||
|
||||
import {
|
||||
ALLOWED_DECORATION_ORIGINS,
|
||||
startLotusDecorations,
|
||||
useLotusDecoration,
|
||||
} from "./lotusDecorations";
|
||||
import { LotusWidgetActions } from "./lotusActions";
|
||||
|
||||
const lazyActions = new EventEmitter();
|
||||
const reply = vi.fn();
|
||||
const send = vi.fn().mockResolvedValue({});
|
||||
|
||||
vi.mock("../widget", () => ({
|
||||
widget: {
|
||||
api: {
|
||||
transport: {
|
||||
reply: (...a: unknown[]) => reply(...a),
|
||||
send: (...a: unknown[]) => send(...a),
|
||||
},
|
||||
},
|
||||
// Getter: `vi.mock` factories run at import time, before the const above.
|
||||
get lazyActions(): EventEmitter {
|
||||
return lazyActions;
|
||||
},
|
||||
},
|
||||
}));
|
||||
|
||||
function pushDecorations(decorations: Record<string, unknown>): void {
|
||||
// `useSyncExternalStore`'s re-render from the module-level `emit()` needs to
|
||||
// be flushed inside `act()`, since the emitter fires outside of React's own
|
||||
// event handling.
|
||||
act(() => {
|
||||
lazyActions.emit(LotusWidgetActions.Decorations, {
|
||||
detail: { data: { decorations } },
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
reply.mockClear();
|
||||
send.mockClear();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
lazyActions.removeAllListeners();
|
||||
});
|
||||
|
||||
test("[lotus #28] safeImageUrl only accepts the pinned decoration CDN origin (or blob:)", () => {
|
||||
const stop = startLotusDecorations();
|
||||
const { result } = renderHook(() => useLotusDecoration("@alice:example.org"));
|
||||
|
||||
pushDecorations({
|
||||
"@alice:example.org": `${ALLOWED_DECORATION_ORIGINS[0]}/fox_hat.png`,
|
||||
});
|
||||
expect(result.current).toBe(`${ALLOWED_DECORATION_ORIGINS[0]}/fox_hat.png`);
|
||||
|
||||
// A different https host is rejected outright, even though it was
|
||||
// previously allowed by the "any https" check.
|
||||
pushDecorations({ "@alice:example.org": "https://evil.example/x.png" });
|
||||
expect(result.current).toBeUndefined();
|
||||
|
||||
// Path traversal off the allowed origin is still on-origin, so the origin
|
||||
// check alone doesn't stop it (validation of the slug itself is the host's
|
||||
// job) — but a completely different scheme/host must never get through.
|
||||
pushDecorations({
|
||||
"@alice:example.org": "javascript:alert(1)",
|
||||
});
|
||||
expect(result.current).toBeUndefined();
|
||||
|
||||
pushDecorations({ "@alice:example.org": "blob:https://example.org/abc" });
|
||||
expect(result.current).toBe("blob:https://example.org/abc");
|
||||
|
||||
stop();
|
||||
});
|
||||
|
||||
test("[lotus #17] the roster survives a handler remount within the same page session", () => {
|
||||
const stop1 = startLotusDecorations();
|
||||
pushDecorations({
|
||||
"@alice:example.org": `${ALLOWED_DECORATION_ORIGINS[0]}/fox_hat.png`,
|
||||
});
|
||||
|
||||
const { result } = renderHook(() => useLotusDecoration("@alice:example.org"));
|
||||
expect(result.current).toBe(`${ALLOWED_DECORATION_ORIGINS[0]}/fox_hat.png`);
|
||||
|
||||
// Tear the handler down (as InCallView would on an EC-side remount) and
|
||||
// bring it back up, WITHOUT the host re-pushing anything.
|
||||
stop1();
|
||||
const stop2 = startLotusDecorations();
|
||||
|
||||
// The roster must still be there — it must not have been wiped to {} by
|
||||
// the teardown.
|
||||
expect(result.current).toBe(`${ALLOWED_DECORATION_ORIGINS[0]}/fox_hat.png`);
|
||||
|
||||
stop2();
|
||||
});
|
||||
|
||||
test("[lotus #17] (re)registering the handler asks the host to re-push state", () => {
|
||||
const stop1 = startLotusDecorations();
|
||||
expect(send).toHaveBeenCalledWith(LotusWidgetActions.RequestState, {});
|
||||
send.mockClear();
|
||||
|
||||
stop1();
|
||||
const stop2 = startLotusDecorations();
|
||||
expect(send).toHaveBeenCalledWith(LotusWidgetActions.RequestState, {});
|
||||
|
||||
stop2();
|
||||
});
|
||||
|
||||
test("a second concurrent registration does not re-request state or double-register", () => {
|
||||
const stop1 = startLotusDecorations();
|
||||
send.mockClear();
|
||||
const stop2 = startLotusDecorations();
|
||||
expect(send).not.toHaveBeenCalled();
|
||||
|
||||
stop2();
|
||||
stop1();
|
||||
});
|
||||
@@ -9,7 +9,7 @@ import { useSyncExternalStore } from "react";
|
||||
import { type IWidgetApiRequest } from "matrix-widget-api";
|
||||
|
||||
import { widget } from "../widget";
|
||||
import { LotusWidgetActions } from "./lotusActions";
|
||||
import { LotusWidgetActions, lotusSendToHost } from "./lotusWidget";
|
||||
|
||||
/**
|
||||
* Avatar decorations (#6 / A6). The Lotus host (cinny) owns the decoration
|
||||
@@ -38,11 +38,26 @@ export function useLotusDecoration(userId: string): string | undefined {
|
||||
return useSyncExternalStore(subscribe, () => decorations[userId]);
|
||||
}
|
||||
|
||||
// [lotus #28] The decoration URL comes from the host, which builds it from an
|
||||
// arbitrary (unvalidated on the wire) profile field. `safeImageUrl` used to
|
||||
// accept ANY https origin, so the only thing stopping a third-party image
|
||||
// beacon on every tile was the host happening to build the URL itself. Pin it
|
||||
// to the actual decoration CDN origin(s) — cinny's `DECORATION_CDN`
|
||||
// (`avatarDecorations.ts`) — plus `blob:`, which EC itself may use for local
|
||||
// previews. Exported so a future CDN move is a one-line edit here.
|
||||
export const ALLOWED_DECORATION_ORIGINS: readonly string[] = [
|
||||
"https://drive.lotusguild.org",
|
||||
];
|
||||
|
||||
function safeImageUrl(raw: unknown): string | null {
|
||||
if (typeof raw !== "string") return null;
|
||||
try {
|
||||
const u = new URL(raw, window.location.href);
|
||||
return u.protocol === "https:" || u.protocol === "blob:" ? u.href : null;
|
||||
if (u.protocol === "blob:") return u.href;
|
||||
return u.protocol === "https:" &&
|
||||
ALLOWED_DECORATION_ORIGINS.includes(u.origin)
|
||||
? u.href
|
||||
: null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
@@ -87,6 +102,13 @@ export function startLotusDecorations(): () => void {
|
||||
unregister = (): void => {
|
||||
w.lazyActions.off(LotusWidgetActions.Decorations, handler);
|
||||
};
|
||||
// [lotus #17] An EC-side remount of InCallView/ActiveCall tears this
|
||||
// handler down and back up while cinny stays joined, and the host only
|
||||
// re-pushes decorations on a CHANGE to its roster — with an unchanged
|
||||
// member list it never re-sends, so the tiles would otherwise lose their
|
||||
// decorations for the rest of the call. Ask the host to re-push whatever
|
||||
// it currently has every time the handler (re)registers.
|
||||
lotusSendToHost(LotusWidgetActions.RequestState, {});
|
||||
}
|
||||
registrations += 1;
|
||||
|
||||
@@ -96,11 +118,15 @@ export function startLotusDecorations(): () => void {
|
||||
registrations = 0;
|
||||
unregister?.();
|
||||
unregister = null;
|
||||
// Reset the roster once the last registration goes away, so a decoration
|
||||
// pushed in call A can't leak onto a shared user in call B before the
|
||||
// host re-pushes. Notify listeners so any still-mounted tile drops the
|
||||
// now-stale overlay via the render path.
|
||||
decorations = {};
|
||||
// [lotus #17] Do NOT wipe the roster here. It used to reset to `{}` so a
|
||||
// decoration from call A couldn't leak onto a same-named user in call B,
|
||||
// but `decorations` is a module-scope singleton for the lifetime of the
|
||||
// PAGE (a fresh call is a fresh page/iframe load, which resets this
|
||||
// module anyway), while an EC-side handler remount within the SAME call
|
||||
// was wiping live decorations that the host has no reason to re-send
|
||||
// (see the request_state ask above, which covers hosts that don't
|
||||
// proactively resend). Listeners are still notified so a torn-down
|
||||
// period doesn't itself change anything visible.
|
||||
emit();
|
||||
}
|
||||
};
|
||||
|
||||
@@ -10,21 +10,40 @@ import { ParticipantEvent, Track } from "livekit-client";
|
||||
|
||||
import { type CallViewModel } from "../state/CallViewModel/CallViewModel";
|
||||
import { startLotusDenoise } from "./lotusDenoise";
|
||||
import { LotusWidgetActions } from "./lotusActions";
|
||||
|
||||
// Track constructed instances so tests can assert exactly one processor was
|
||||
// built across racing apply() calls, and can assert the pending one is
|
||||
// destroyed on early teardown. `vi.hoisted` is required because `vi.mock`
|
||||
// factories are hoisted above this file's other top-level statements.
|
||||
const instances = vi.hoisted(
|
||||
() => [] as { destroy: ReturnType<typeof vi.fn> }[],
|
||||
() =>
|
||||
[] as {
|
||||
config: { model: string };
|
||||
destroy: ReturnType<typeof vi.fn>;
|
||||
setMicMuted: ReturnType<typeof vi.fn>;
|
||||
}[],
|
||||
);
|
||||
// Ordered log of the heavy-lifting calls, to assert prepare runs before
|
||||
// setProcessor (#7).
|
||||
const calls = vi.hoisted(() => [] as string[]);
|
||||
vi.mock("./lotusDenoiseProcessor", () => ({
|
||||
LotusDenoiseProcessor: class {
|
||||
public destroy = vi.fn().mockResolvedValue(undefined);
|
||||
public constructor() {
|
||||
public setMicMuted = vi.fn();
|
||||
public constructor(public config: { model: string }) {
|
||||
instances.push(this);
|
||||
}
|
||||
},
|
||||
prepareDenoiseAssets: vi.fn(async () => {
|
||||
calls.push("prepare");
|
||||
await Promise.resolve();
|
||||
}),
|
||||
releasePreparedDenoiseAssets: vi.fn().mockResolvedValue(undefined),
|
||||
}));
|
||||
const hostSend = vi.hoisted(() => vi.fn().mockResolvedValue({}));
|
||||
vi.mock("../widget", () => ({
|
||||
widget: { api: { transport: { send: hostSend } } },
|
||||
}));
|
||||
|
||||
/** A promise plus externally-callable resolve, for controlling ordering. */
|
||||
@@ -42,19 +61,23 @@ function deferred<T>(): {
|
||||
function makeRoomAndVm(mic: {
|
||||
getProcessor: () => unknown;
|
||||
setProcessor: ReturnType<typeof vi.fn>;
|
||||
isMuted?: boolean;
|
||||
}): {
|
||||
vm: CallViewModel;
|
||||
room: { localParticipant: Record<string, unknown> };
|
||||
firePublished: () => void;
|
||||
fire: (event: string, arg?: unknown) => void;
|
||||
} {
|
||||
const handlers = new Map<string, () => void>();
|
||||
const handlers = new Map<string, (arg?: unknown) => void>();
|
||||
const localParticipant = {
|
||||
getTrackPublication: (
|
||||
source: Track.Source,
|
||||
): { track: typeof mic } | undefined =>
|
||||
source === Track.Source.Microphone ? { track: mic } : undefined,
|
||||
on: (event: string, cb: () => void): Map<string, () => void> =>
|
||||
handlers.set(event, cb),
|
||||
on: (
|
||||
event: string,
|
||||
cb: (arg?: unknown) => void,
|
||||
): Map<string, (arg?: unknown) => void> => handlers.set(event, cb),
|
||||
off: (event: string): boolean => handlers.delete(event),
|
||||
};
|
||||
const room = { localParticipant };
|
||||
@@ -74,11 +97,19 @@ function makeRoomAndVm(mic: {
|
||||
vm,
|
||||
room,
|
||||
firePublished: () => handlers.get(ParticipantEvent.LocalTrackPublished)?.(),
|
||||
fire: (event, arg) => handlers.get(event)?.(arg),
|
||||
};
|
||||
}
|
||||
|
||||
/** Let the async apply() chain (attach → retry → host notify) settle. */
|
||||
async function flush(): Promise<void> {
|
||||
for (let i = 0; i < 8; i++) await Promise.resolve();
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
instances.length = 0;
|
||||
calls.length = 0;
|
||||
hostSend.mockClear();
|
||||
// `lotusParam`/`lotusFlag` cache the URL params at first read; seed the hash
|
||||
// before startLotusDenoise() so the dedicated `lotusDenoiseSource` flag reads on.
|
||||
window.location.hash = "#/room?lotusDenoiseSource=1";
|
||||
@@ -95,6 +126,7 @@ describe("startLotusDenoise", () => {
|
||||
const mic = {
|
||||
getProcessor: (): unknown => attached,
|
||||
setProcessor: vi.fn(async (p: unknown) => {
|
||||
calls.push("setProcessor");
|
||||
await setProcessorDeferred.promise;
|
||||
attached = p;
|
||||
}),
|
||||
@@ -102,6 +134,8 @@ describe("startLotusDenoise", () => {
|
||||
const { vm, firePublished } = makeRoomAndVm(mic);
|
||||
|
||||
startLotusDenoise(vm);
|
||||
// [#7] The heavy assets were kicked off before setProcessor() was called.
|
||||
expect(calls).toEqual(["prepare", "setProcessor"]);
|
||||
// Simulate a second LocalTrackPublished (e.g. camera) firing before the
|
||||
// first setProcessor() has resolved.
|
||||
firePublished();
|
||||
@@ -138,4 +172,103 @@ describe("startLotusDenoise", () => {
|
||||
setProcessorDeferred.resolve();
|
||||
await Promise.resolve();
|
||||
});
|
||||
|
||||
test("reports the active model to the host once attached (#8)", async () => {
|
||||
let attached: unknown;
|
||||
const mic = {
|
||||
getProcessor: (): unknown => attached,
|
||||
setProcessor: vi.fn(async (p: unknown) => {
|
||||
await Promise.resolve();
|
||||
attached = p;
|
||||
}),
|
||||
};
|
||||
const { vm } = makeRoomAndVm(mic);
|
||||
startLotusDenoise(vm);
|
||||
await flush();
|
||||
expect(hostSend).toHaveBeenCalledWith(LotusWidgetActions.DenoiseState, {
|
||||
active: true,
|
||||
model: "rnnoise",
|
||||
});
|
||||
});
|
||||
|
||||
test("retries once with rnnoise when the requested model fails to init (#8)", async () => {
|
||||
window.location.hash =
|
||||
"#/room?lotusDenoiseSource=1&lotusModel=deepfilternet";
|
||||
let attached: unknown;
|
||||
const mic = {
|
||||
getProcessor: (): unknown => attached,
|
||||
setProcessor: vi.fn(async (p: { config: { model: string } }) => {
|
||||
await Promise.resolve();
|
||||
if (p.config.model === "deepfilternet")
|
||||
throw new Error("dfn assets missing");
|
||||
attached = p;
|
||||
}),
|
||||
};
|
||||
const { vm } = makeRoomAndVm(mic);
|
||||
startLotusDenoise(vm);
|
||||
await flush();
|
||||
|
||||
expect(instances.map((i) => i.config.model)).toEqual([
|
||||
"deepfilternet",
|
||||
"rnnoise",
|
||||
]);
|
||||
expect(mic.setProcessor).toHaveBeenCalledTimes(2);
|
||||
expect(hostSend).toHaveBeenCalledTimes(1);
|
||||
expect(hostSend).toHaveBeenCalledWith(LotusWidgetActions.DenoiseState, {
|
||||
active: true,
|
||||
model: "rnnoise",
|
||||
});
|
||||
});
|
||||
|
||||
test("notifies the host with the error when the rnnoise fallback also fails (#8)", async () => {
|
||||
window.location.hash = "#/room?lotusDenoiseSource=1&lotusModel=dtln";
|
||||
const mic = {
|
||||
getProcessor: (): unknown => undefined,
|
||||
setProcessor: vi.fn(async () => {
|
||||
await Promise.resolve();
|
||||
throw new Error("no worklet for you");
|
||||
}),
|
||||
};
|
||||
const { vm } = makeRoomAndVm(mic);
|
||||
startLotusDenoise(vm);
|
||||
await flush();
|
||||
|
||||
expect(instances.map((i) => i.config.model)).toEqual(["dtln", "rnnoise"]);
|
||||
expect(hostSend).toHaveBeenCalledTimes(1);
|
||||
expect(hostSend).toHaveBeenCalledWith(LotusWidgetActions.DenoiseState, {
|
||||
active: false,
|
||||
error: "no worklet for you",
|
||||
model: "rnnoise",
|
||||
});
|
||||
});
|
||||
|
||||
test("mirrors mic TrackMuted/TrackUnmuted onto the processor (#9)", async () => {
|
||||
let attached: unknown;
|
||||
const mic = {
|
||||
isMuted: true,
|
||||
getProcessor: (): unknown => attached,
|
||||
setProcessor: vi.fn(async (p: unknown) => {
|
||||
await Promise.resolve();
|
||||
attached = p;
|
||||
}),
|
||||
};
|
||||
const { vm, fire } = makeRoomAndVm(mic);
|
||||
startLotusDenoise(vm);
|
||||
// Seeded from the mic's current state before setProcessor().
|
||||
expect(instances[0].setMicMuted).toHaveBeenLastCalledWith(true);
|
||||
await flush();
|
||||
|
||||
const micPub = { source: Track.Source.Microphone };
|
||||
const camPub = { source: Track.Source.Camera };
|
||||
fire(ParticipantEvent.TrackUnmuted, micPub);
|
||||
expect(instances[0].setMicMuted).toHaveBeenLastCalledWith(false);
|
||||
fire(ParticipantEvent.TrackMuted, micPub);
|
||||
expect(instances[0].setMicMuted).toHaveBeenLastCalledWith(true);
|
||||
|
||||
// Camera mute must not touch the audio context.
|
||||
instances[0].setMicMuted.mockClear();
|
||||
fire(ParticipantEvent.TrackMuted, camPub);
|
||||
fire(ParticipantEvent.TrackUnmuted, camPub);
|
||||
expect(instances[0].setMicMuted).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
+89
-15
@@ -10,14 +10,23 @@ import {
|
||||
ParticipantEvent,
|
||||
type Room as LivekitRoom,
|
||||
Track,
|
||||
type TrackPublication,
|
||||
} from "livekit-client";
|
||||
import { logger } from "matrix-js-sdk/lib/logger";
|
||||
|
||||
import { type CallViewModel } from "../state/CallViewModel/CallViewModel";
|
||||
import { lotusFlag, lotusParam } from "./lotusWidget";
|
||||
import {
|
||||
LotusWidgetActions,
|
||||
lotusFlag,
|
||||
lotusParam,
|
||||
lotusSendToHost,
|
||||
} from "./lotusWidget";
|
||||
import {
|
||||
type LotusDenoiseConfig,
|
||||
type LotusDenoiseModel,
|
||||
LotusDenoiseProcessor,
|
||||
prepareDenoiseAssets,
|
||||
releasePreparedDenoiseAssets,
|
||||
} from "./lotusDenoiseProcessor";
|
||||
|
||||
/**
|
||||
@@ -93,6 +102,12 @@ export function startLotusDenoise(vm: CallViewModel): () => void {
|
||||
: 0.15,
|
||||
};
|
||||
|
||||
// [lotus #7] Start fetching wasm / creating the AudioContext / addModule-ing
|
||||
// the worklet (and loading the DFN model) NOW, before any mic track exists.
|
||||
// `setProcessor()` holds LiveKit's trackChangeLock while awaiting `init()`,
|
||||
// so anything still loading there freezes mute/unmute/device-switch.
|
||||
void prepareDenoiseAssets(config);
|
||||
|
||||
const micOf = (room: LivekitRoom): LocalAudioTrack | undefined =>
|
||||
room.localParticipant.getTrackPublication(Track.Source.Microphone)
|
||||
?.track as LocalAudioTrack | undefined;
|
||||
@@ -105,23 +120,73 @@ export function startLotusDenoise(vm: CallViewModel): () => void {
|
||||
// AudioContext + model load) before the first has attached. Track an
|
||||
// in-flight setProcessor per room and skip `apply()` while one is pending.
|
||||
const pendingProcessors = new Map<LivekitRoom, LotusDenoiseProcessor>();
|
||||
let stopped = false;
|
||||
|
||||
const attach = async (
|
||||
room: LivekitRoom,
|
||||
mic: LocalAudioTrack,
|
||||
model: LotusDenoiseModel,
|
||||
): Promise<void> => {
|
||||
const processor = new LotusDenoiseProcessor({ ...config, model });
|
||||
pendingProcessors.set(room, processor);
|
||||
// [lotus #9] Seed the mute state so a processor attached while already
|
||||
// muted starts suspended rather than burning inference on silence.
|
||||
processor.setMicMuted(mic.isMuted);
|
||||
try {
|
||||
await mic.setProcessor(processor);
|
||||
} finally {
|
||||
// Only clear if we're still the pending entry (a teardown that ran
|
||||
// while this was in flight may have already replaced/removed it).
|
||||
if (pendingProcessors.get(room) === processor)
|
||||
pendingProcessors.delete(room);
|
||||
}
|
||||
};
|
||||
|
||||
const apply = (room: LivekitRoom): void => {
|
||||
const mic = micOf(room);
|
||||
if (!mic || mic.getProcessor() || pendingProcessors.has(room)) return;
|
||||
const processor = new LotusDenoiseProcessor(config);
|
||||
pendingProcessors.set(room, processor);
|
||||
void mic
|
||||
.setProcessor(processor)
|
||||
.catch((e) => logger.warn("[lotus] denoise setProcessor failed", e))
|
||||
.finally(() => {
|
||||
// Only clear if we're still the pending entry (a teardown that ran
|
||||
// while this was in flight may have already replaced/removed it).
|
||||
if (pendingProcessors.get(room) === processor)
|
||||
pendingProcessors.delete(room);
|
||||
});
|
||||
void (async () => {
|
||||
let model = config.model;
|
||||
try {
|
||||
try {
|
||||
await attach(room, mic, model);
|
||||
} catch (e) {
|
||||
logger.warn("[lotus] denoise setProcessor failed", e);
|
||||
// [lotus #8] A failed init leaves LiveKit with no processor (it only
|
||||
// assigns after init resolves), so retry once with the smallest,
|
||||
// most portable tier before giving up.
|
||||
if (model === "rnnoise" || stopped) throw e;
|
||||
model = "rnnoise";
|
||||
await attach(room, mic, model);
|
||||
}
|
||||
// [lotus #8] Tell the host what is ACTUALLY running so its toggle
|
||||
// reflects reality (including the fallback model).
|
||||
lotusSendToHost(LotusWidgetActions.DenoiseState, {
|
||||
active: true,
|
||||
model,
|
||||
});
|
||||
} catch (e) {
|
||||
logger.warn("[lotus] denoise unavailable; publishing raw mic", e);
|
||||
lotusSendToHost(LotusWidgetActions.DenoiseState, {
|
||||
active: false,
|
||||
error: e instanceof Error ? e.message : String(e),
|
||||
model,
|
||||
});
|
||||
}
|
||||
})();
|
||||
};
|
||||
|
||||
// [lotus #9] Suspend/resume the processor's own AudioContext with the mic's
|
||||
// mute state (LiveKit keeps the muted track flowing silent frames, so the
|
||||
// worklet would otherwise run inference the whole time the user is muted).
|
||||
const onMuteChange =
|
||||
(room: LivekitRoom, muted: boolean) =>
|
||||
(pub: TrackPublication): void => {
|
||||
if (pub.source !== Track.Source.Microphone) return;
|
||||
const p = pendingProcessors.get(room) ?? micOf(room)?.getProcessor();
|
||||
if (p instanceof LotusDenoiseProcessor) p.setMicMuted(muted);
|
||||
};
|
||||
|
||||
const roomListeners = new Map<LivekitRoom, () => void>();
|
||||
let rooms: LivekitRoom[] = [];
|
||||
|
||||
@@ -143,22 +208,29 @@ export function startLotusDenoise(vm: CallViewModel): () => void {
|
||||
// Re-attach on every (re)publish — this is what makes denoise survive
|
||||
// reconnects (A7), unlike the old getUserMedia patch.
|
||||
const onPublished = (): void => apply(room);
|
||||
const onMuted = onMuteChange(room, true);
|
||||
const onUnmuted = onMuteChange(room, false);
|
||||
room.localParticipant.on(
|
||||
ParticipantEvent.LocalTrackPublished,
|
||||
onPublished,
|
||||
);
|
||||
roomListeners.set(room, () =>
|
||||
room.localParticipant.on(ParticipantEvent.TrackMuted, onMuted);
|
||||
room.localParticipant.on(ParticipantEvent.TrackUnmuted, onUnmuted);
|
||||
roomListeners.set(room, () => {
|
||||
room.localParticipant.off(
|
||||
ParticipantEvent.LocalTrackPublished,
|
||||
onPublished,
|
||||
),
|
||||
);
|
||||
);
|
||||
room.localParticipant.off(ParticipantEvent.TrackMuted, onMuted);
|
||||
room.localParticipant.off(ParticipantEvent.TrackUnmuted, onUnmuted);
|
||||
});
|
||||
apply(room);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
return () => {
|
||||
stopped = true;
|
||||
sub.unsubscribe();
|
||||
for (const off of roomListeners.values()) off();
|
||||
roomListeners.clear();
|
||||
@@ -175,5 +247,7 @@ export function startLotusDenoise(vm: CallViewModel): () => void {
|
||||
}
|
||||
}
|
||||
pendingProcessors.clear();
|
||||
// [lotus #7] Close any prepared-but-never-claimed context (e.g. no mic).
|
||||
void releasePreparedDenoiseAssets();
|
||||
};
|
||||
}
|
||||
|
||||
@@ -5,19 +5,97 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { describe, expect, test } from "vitest";
|
||||
import { afterEach, beforeEach, describe, expect, test, vi } from "vitest";
|
||||
import { type AudioProcessorOptions } from "livekit-client";
|
||||
|
||||
import { LotusDenoiseProcessor } from "./lotusDenoiseProcessor";
|
||||
import {
|
||||
assertModuleExport,
|
||||
type LotusDenoiseConfig,
|
||||
LotusDenoiseProcessor,
|
||||
prepareDenoiseAssets,
|
||||
releasePreparedDenoiseAssets,
|
||||
} from "./lotusDenoiseProcessor";
|
||||
|
||||
function makeProcessor(): LotusDenoiseProcessor {
|
||||
return new LotusDenoiseProcessor({
|
||||
model: "rnnoise",
|
||||
assetBase: "https://example.invalid/denoise/",
|
||||
gate: false,
|
||||
gateThreshold: -45,
|
||||
floor: 0.15,
|
||||
const baseConfig: LotusDenoiseConfig = {
|
||||
model: "rnnoise",
|
||||
assetBase: "https://example.invalid/denoise/",
|
||||
gate: false,
|
||||
gateThreshold: -45,
|
||||
floor: 0.15,
|
||||
};
|
||||
|
||||
function makeProcessor(
|
||||
overrides: Partial<LotusDenoiseConfig> = {},
|
||||
): LotusDenoiseProcessor {
|
||||
return new LotusDenoiseProcessor({ ...baseConfig, ...overrides });
|
||||
}
|
||||
|
||||
/**
|
||||
* Minimal fake AudioContext: tracks state, records suspend/resume, and lets a
|
||||
* test fire `statechange` like the browser would after an OS interruption.
|
||||
*/
|
||||
class FakeAudioContext extends EventTarget {
|
||||
public static created: FakeAudioContext[] = [];
|
||||
public state: AudioContextState = "running";
|
||||
public readonly sampleRate: number;
|
||||
public readonly audioWorklet = {
|
||||
addModule: vi.fn().mockResolvedValue(undefined),
|
||||
};
|
||||
public suspend = vi.fn(async (): Promise<void> => {
|
||||
await Promise.resolve();
|
||||
this.state = "suspended";
|
||||
});
|
||||
public resume = vi.fn(async (): Promise<void> => {
|
||||
await Promise.resolve();
|
||||
this.state = "running";
|
||||
});
|
||||
public close = vi.fn(async (): Promise<void> => {
|
||||
await Promise.resolve();
|
||||
this.state = "closed";
|
||||
});
|
||||
public constructor(opts?: { sampleRate?: number }) {
|
||||
super();
|
||||
this.sampleRate = opts?.sampleRate ?? 48_000;
|
||||
FakeAudioContext.created.push(this);
|
||||
}
|
||||
/** Simulate an external (OS/browser) suspension. */
|
||||
public externallySuspend(): void {
|
||||
this.state = "suspended";
|
||||
this.dispatchEvent(new Event("statechange"));
|
||||
}
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
FakeAudioContext.created = [];
|
||||
vi.stubGlobal("AudioContext", FakeAudioContext);
|
||||
// The flat wasm prefetch just warms a cache; give it a 200 so prepare()
|
||||
// succeeds without a network.
|
||||
vi.stubGlobal(
|
||||
"fetch",
|
||||
vi.fn().mockResolvedValue({
|
||||
ok: true,
|
||||
arrayBuffer: (): ArrayBuffer => new ArrayBuffer(8),
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await releasePreparedDenoiseAssets();
|
||||
vi.unstubAllGlobals();
|
||||
});
|
||||
|
||||
/** Skip the real graph (needs worklets) — init() only needs a track back. */
|
||||
function stubGraph(processor: LotusDenoiseProcessor): void {
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
(processor as any).buildGraph = async (): Promise<unknown> => {
|
||||
await Promise.resolve();
|
||||
return {
|
||||
source: { disconnect: vi.fn() },
|
||||
nodes: [],
|
||||
disposes: [],
|
||||
track: { stop: vi.fn() },
|
||||
};
|
||||
};
|
||||
}
|
||||
|
||||
describe("LotusDenoiseProcessor.restart", () => {
|
||||
@@ -53,3 +131,117 @@ describe("LotusDenoiseProcessor.restart", () => {
|
||||
expect(processor.processedTrack).not.toBe(rawTrack);
|
||||
});
|
||||
});
|
||||
|
||||
describe("prepareDenoiseAssets (#7)", () => {
|
||||
test("init() adopts the context/worklet prepared before setProcessor instead of loading inline", async () => {
|
||||
await prepareDenoiseAssets(baseConfig);
|
||||
// The heavy parts ran up front: one context, worklet module added, wasm fetched.
|
||||
expect(FakeAudioContext.created).toHaveLength(1);
|
||||
const prepared = FakeAudioContext.created[0];
|
||||
expect(prepared.audioWorklet.addModule).toHaveBeenCalledWith(
|
||||
`${baseConfig.assetBase}rnnoiseWorklet.js`,
|
||||
);
|
||||
expect(fetch).toHaveBeenCalled();
|
||||
|
||||
const processor = makeProcessor();
|
||||
stubGraph(processor);
|
||||
await processor.init({
|
||||
track: {} as MediaStreamTrack,
|
||||
} as unknown as AudioProcessorOptions);
|
||||
|
||||
// No second AudioContext / addModule inside init (i.e. under the lock).
|
||||
expect(FakeAudioContext.created).toHaveLength(1);
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
expect((processor as any).ctx).toBe(prepared);
|
||||
|
||||
// The claim is one-shot: a later processor prepares its own.
|
||||
const second = makeProcessor();
|
||||
stubGraph(second);
|
||||
await second.init({
|
||||
track: {} as MediaStreamTrack,
|
||||
} as unknown as AudioProcessorOptions);
|
||||
expect(FakeAudioContext.created).toHaveLength(2);
|
||||
|
||||
await processor.destroy();
|
||||
await second.destroy();
|
||||
});
|
||||
|
||||
test("releasePreparedDenoiseAssets closes an unclaimed prepared context", async () => {
|
||||
await prepareDenoiseAssets(baseConfig);
|
||||
const prepared = FakeAudioContext.created[0];
|
||||
await releasePreparedDenoiseAssets();
|
||||
expect(prepared.close).toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe("LotusDenoiseProcessor.setMicMuted (#9)", () => {
|
||||
test("suspends the owned context on mute, resumes on unmute, and the statechange watcher leaves the mute suspension alone", async () => {
|
||||
const processor = makeProcessor();
|
||||
stubGraph(processor);
|
||||
await processor.init({
|
||||
track: {} as MediaStreamTrack,
|
||||
} as unknown as AudioProcessorOptions);
|
||||
const ctx = FakeAudioContext.created[0];
|
||||
expect(ctx.state).toBe("running");
|
||||
|
||||
processor.setMicMuted(true);
|
||||
await Promise.resolve();
|
||||
expect(ctx.suspend).toHaveBeenCalledTimes(1);
|
||||
expect(ctx.state).toBe("suspended");
|
||||
|
||||
// The browser fires statechange for our own suspend too; the watcher must
|
||||
// NOT undo it while muted.
|
||||
ctx.resume.mockClear();
|
||||
ctx.dispatchEvent(new Event("statechange"));
|
||||
await Promise.resolve();
|
||||
expect(ctx.resume).not.toHaveBeenCalled();
|
||||
expect(ctx.state).toBe("suspended");
|
||||
|
||||
processor.setMicMuted(false);
|
||||
await Promise.resolve();
|
||||
expect(ctx.resume).toHaveBeenCalledTimes(1);
|
||||
expect(ctx.state).toBe("running");
|
||||
|
||||
// An EXTERNAL suspension while unmuted is still healed by the watcher.
|
||||
ctx.resume.mockClear();
|
||||
ctx.externallySuspend();
|
||||
await Promise.resolve();
|
||||
expect(ctx.resume).toHaveBeenCalledTimes(1);
|
||||
|
||||
await processor.destroy();
|
||||
});
|
||||
|
||||
test("a processor attached while already muted starts suspended", async () => {
|
||||
const processor = makeProcessor();
|
||||
stubGraph(processor);
|
||||
processor.setMicMuted(true);
|
||||
await processor.init({
|
||||
track: {} as MediaStreamTrack,
|
||||
} as unknown as AudioProcessorOptions);
|
||||
const ctx = FakeAudioContext.created[0];
|
||||
expect(ctx.state).toBe("suspended");
|
||||
await processor.destroy();
|
||||
});
|
||||
});
|
||||
|
||||
describe("assertModuleExport (#26)", () => {
|
||||
test("returns the module when the export is a function", () => {
|
||||
const mod = { createNoiseSuppressionAudioWorklet: (): void => undefined };
|
||||
expect(
|
||||
assertModuleExport(mod, "createNoiseSuppressionAudioWorklet", "x.js"),
|
||||
).toBe(mod);
|
||||
});
|
||||
|
||||
test("throws a clear error naming the module and export when missing", () => {
|
||||
expect(() =>
|
||||
assertModuleExport(
|
||||
{ other: 1 },
|
||||
"DeepFilterNet3Core",
|
||||
"dfn/index.esm.js",
|
||||
),
|
||||
).toThrow(/dfn\/index\.esm\.js does not export DeepFilterNet3Core/);
|
||||
expect(() => assertModuleExport(undefined, "X", "y.js")).toThrow(
|
||||
/does not export X/,
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -95,7 +95,9 @@ async function fetchWasm(url: string): Promise<ArrayBuffer> {
|
||||
* lands, and a still-suspended context degrades to (temporary) silence that the
|
||||
* watcher heals, not a hang.
|
||||
*/
|
||||
async function resumeCtx(ctx: AudioContext, timeoutMs = 3_000): Promise<void> {
|
||||
// [lotus] 500 ms, not 3 s: this still runs under LiveKit's trackChangeLock (#7),
|
||||
// and the statechange watcher heals a still-suspended context later anyway.
|
||||
async function resumeCtx(ctx: AudioContext, timeoutMs = 500): Promise<void> {
|
||||
await Promise.race([
|
||||
ctx.resume().catch(() => undefined),
|
||||
new Promise<void>((resolve) => setTimeout(resolve, timeoutMs)),
|
||||
@@ -127,6 +129,226 @@ interface Graph {
|
||||
track: MediaStreamTrack;
|
||||
}
|
||||
|
||||
// [lotus #26] Minimal local contracts for the two dynamically-imported ESM
|
||||
// helpers (not bundled here — see the CONTRACT note above). Their exports are
|
||||
// asserted at runtime so an asset bump that renames/removes one fails loudly
|
||||
// (and flows into the rnnoise fallback in lotusDenoise.ts) instead of as a
|
||||
// vague TypeError deep inside `init()`.
|
||||
interface DtlnModule {
|
||||
createNoiseSuppressionAudioWorklet: (
|
||||
ctx: AudioContext,
|
||||
opts: { bypassUntilReady: boolean },
|
||||
) => Promise<MlNode>;
|
||||
}
|
||||
interface DfnCore {
|
||||
initialize: () => Promise<void>;
|
||||
createAudioWorkletNode: (ctx: AudioContext) => Promise<AudioNode>;
|
||||
destroy: () => void;
|
||||
}
|
||||
interface DfnModule {
|
||||
DeepFilterNet3Core: new (opts: {
|
||||
sampleRate: number;
|
||||
noiseReductionLevel: number;
|
||||
assetConfig: { cdnUrl: string };
|
||||
}) => DfnCore;
|
||||
}
|
||||
|
||||
/** Throw a clear error if a dynamic-import module lacks an expected export. */
|
||||
export function assertModuleExport<T>(
|
||||
mod: unknown,
|
||||
name: string,
|
||||
url: string,
|
||||
): T {
|
||||
const exp = (mod as Record<string, unknown> | null | undefined)?.[name];
|
||||
if (typeof exp !== "function")
|
||||
throw new Error(
|
||||
`denoise: ${url} does not export ${name} (got ${typeof exp}) — asset/version mismatch`,
|
||||
);
|
||||
return mod as T;
|
||||
}
|
||||
|
||||
async function loadDfnCore(config: LotusDenoiseConfig): Promise<DfnCore> {
|
||||
const base = config.assetBase;
|
||||
const url = `${base}deepfilternet/index.esm.js`;
|
||||
const dfnBase = new URL(`${base}deepfilternet`, window.location.href).href;
|
||||
const mod = assertModuleExport<DfnModule>(
|
||||
await import(/* @vite-ignore */ url),
|
||||
"DeepFilterNet3Core",
|
||||
url,
|
||||
);
|
||||
const core = new mod.DeepFilterNet3Core({
|
||||
sampleRate: 48_000,
|
||||
// 60, not 80: full-strength suppression is the main source of the
|
||||
// "over-processed" character; a lower level keeps voice natural while
|
||||
// the dry/wet floor handles the noise tail.
|
||||
noiseReductionLevel: 60,
|
||||
assetConfig: { cdnUrl: dfnBase },
|
||||
});
|
||||
await core.initialize();
|
||||
return core;
|
||||
}
|
||||
|
||||
async function loadDtlnModule(config: LotusDenoiseConfig): Promise<DtlnModule> {
|
||||
const url = `${config.assetBase}workadventure/audio-worklet.js`;
|
||||
return assertModuleExport<DtlnModule>(
|
||||
await import(/* @vite-ignore */ url),
|
||||
"createNoiseSuppressionAudioWorklet",
|
||||
url,
|
||||
);
|
||||
}
|
||||
|
||||
/** Which wasm file a flat model uses (SIMD build when supported). */
|
||||
function flatWasmFiles(model: "rnnoise" | "speex"): {
|
||||
primary: string;
|
||||
fallback?: string;
|
||||
} {
|
||||
const flat = FLAT[model];
|
||||
const useSimd = model === "rnnoise" && !!flat.simdWasm && supportsSimd();
|
||||
return useSimd
|
||||
? { primary: flat.simdWasm!, fallback: flat.wasm }
|
||||
: { primary: flat.wasm };
|
||||
}
|
||||
|
||||
// [lotus #24] Force every node in the graph to a single, explicitly-downmixed
|
||||
// channel. Without `channelCountMode: "explicit"` the default ("max") IGNORES
|
||||
// `channelCount`, so a stereo capture device would feed 2 channels into a
|
||||
// worklet configured with `maxChannels: 1` and sum a stereo dry copy against a
|
||||
// mono wet one at the destination.
|
||||
const MONO: AudioNodeOptions = {
|
||||
channelCount: 1,
|
||||
channelCountMode: "explicit",
|
||||
channelInterpretation: "speakers",
|
||||
};
|
||||
|
||||
// [lotus #25] Algorithmic latency of each model in samples at its native rate,
|
||||
// used to delay the DRY copy of the floor mix so it lines up with the wet path
|
||||
// (otherwise the sum comb-filters — a hollow/phasey colouration on voice).
|
||||
// - rnnoise: 480-sample (10 ms @ 48 kHz) frames; the sapphi worklet buffers
|
||||
// 128-sample quanta up to one frame, so the wet path lags by one frame.
|
||||
// - speex: the sapphi speex worklet uses the same 480-sample framing.
|
||||
// - dtln: 512-sample block / 128 hop @ 16 kHz (~32 ms) per the DTLN paper —
|
||||
// best-known, unmeasured (the floor is not mixed for dtln, see buildGraph).
|
||||
// - deepfilternet: 480-sample hop + 2-frame lookahead @ 48 kHz (~30 ms) per
|
||||
// DeepFilterNet3 — best-known, unmeasured (floor not mixed for dfn either).
|
||||
const DRY_DELAY_SAMPLES: Record<LotusDenoiseModel, number> = {
|
||||
rnnoise: 480,
|
||||
speex: 480,
|
||||
dtln: 512,
|
||||
deepfilternet: 1440,
|
||||
};
|
||||
|
||||
/**
|
||||
* Create the model-rate context and register the flat/gate worklet modules.
|
||||
* Closes the context (and rethrows) on any failure so nothing half-built leaks.
|
||||
*/
|
||||
async function createModelContext(
|
||||
config: LotusDenoiseConfig,
|
||||
): Promise<AudioContext> {
|
||||
const rate = sampleRateFor(config.model);
|
||||
const ctx = new AudioContext({ sampleRate: rate });
|
||||
try {
|
||||
if (ctx.sampleRate !== rate)
|
||||
throw new Error(`denoise: got ${ctx.sampleRate}Hz, need ${rate}Hz`);
|
||||
// Flat models register via addModule here; DTLN/DeepFilterNet bring their
|
||||
// own processor via the dynamic-imported helper (see buildMlNode).
|
||||
if (config.model === "rnnoise" || config.model === "speex")
|
||||
await ctx.audioWorklet.addModule(
|
||||
config.assetBase + FLAT[config.model].script,
|
||||
);
|
||||
if (config.gate)
|
||||
await ctx.audioWorklet.addModule(config.assetBase + GATE.script);
|
||||
return ctx;
|
||||
} catch (e) {
|
||||
await ctx.close().catch(() => undefined);
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
// [lotus #7] Everything heavy that `init()` needs but that does NOT depend on
|
||||
// the mic track: the AudioContext + worklet modules, the flat wasm binary, and
|
||||
// (DFN) the fully-initialised model core. `LocalAudioTrack.setProcessor()`
|
||||
// holds LiveKit's `trackChangeLock` while awaiting `init()`, so every
|
||||
// mute/unmute/device-switch queues behind it — prepare these as soon as the
|
||||
// flag is seen (before any track exists) so `init()` only wires them up.
|
||||
interface PreparedAssets {
|
||||
ctx: AudioContext;
|
||||
dfnCore?: DfnCore;
|
||||
}
|
||||
const preparedAssets = new Map<string, Promise<PreparedAssets>>();
|
||||
const preparedKey = (c: LotusDenoiseConfig): string =>
|
||||
`${c.model}|${c.gate ? 1 : 0}|${c.assetBase}`;
|
||||
|
||||
async function prepareUncached(
|
||||
config: LotusDenoiseConfig,
|
||||
): Promise<PreparedAssets> {
|
||||
const ctx = await createModelContext(config);
|
||||
try {
|
||||
let dfnCore: DfnCore | undefined;
|
||||
if (config.model === "rnnoise" || config.model === "speex") {
|
||||
const { primary, fallback } = flatWasmFiles(config.model);
|
||||
// Warm the wasm cache; a SIMD miss is fine — buildMlNode falls back.
|
||||
await fetchWasm(config.assetBase + primary).catch(async () =>
|
||||
fallback ? fetchWasm(config.assetBase + fallback) : undefined,
|
||||
);
|
||||
} else if (config.model === "dtln") {
|
||||
await loadDtlnModule(config); // warms the browser's module map
|
||||
} else {
|
||||
dfnCore = await loadDfnCore(config);
|
||||
}
|
||||
return { ctx, dfnCore };
|
||||
} catch (e) {
|
||||
await ctx.close().catch(() => undefined);
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Prefetch/prepare the assets for `config` (idempotent per model). Never
|
||||
* rejects: a failed prepare is evicted so `init()` simply loads inline and
|
||||
* surfaces the real error there.
|
||||
*/
|
||||
export async function prepareDenoiseAssets(
|
||||
config: LotusDenoiseConfig,
|
||||
): Promise<void> {
|
||||
const key = preparedKey(config);
|
||||
let p = preparedAssets.get(key);
|
||||
if (!p) {
|
||||
p = prepareUncached(config);
|
||||
void p.catch((e) => {
|
||||
if (preparedAssets.get(key) === p) preparedAssets.delete(key);
|
||||
logger.warn(`[lotus] denoise prepare failed (${config.model})`, e);
|
||||
});
|
||||
preparedAssets.set(key, p);
|
||||
}
|
||||
await p.catch(() => undefined);
|
||||
}
|
||||
|
||||
/** Take (one-shot) the prepared assets for `config`, if any were prepared. */
|
||||
function claimPreparedAssets(
|
||||
config: LotusDenoiseConfig,
|
||||
): Promise<PreparedAssets> | undefined {
|
||||
const key = preparedKey(config);
|
||||
const p = preparedAssets.get(key);
|
||||
if (p) preparedAssets.delete(key);
|
||||
return p;
|
||||
}
|
||||
|
||||
/** Close any prepared-but-unclaimed contexts (call on feature teardown). */
|
||||
export async function releasePreparedDenoiseAssets(): Promise<void> {
|
||||
const all = [...preparedAssets.values()];
|
||||
preparedAssets.clear();
|
||||
await Promise.all(
|
||||
all.map(async (p) =>
|
||||
p
|
||||
.then(async (a) => {
|
||||
safeCall(() => a.dfnCore?.destroy());
|
||||
if (a.ctx.state !== "closed") await a.ctx.close();
|
||||
})
|
||||
.catch(() => undefined),
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* A LiveKit audio TrackProcessor that runs Lotus ML noise suppression
|
||||
* (RNNoise / Speex / DTLN / DeepFilterNet) on the local microphone track, as a
|
||||
@@ -151,9 +373,31 @@ export class LotusDenoiseProcessor implements TrackProcessor<
|
||||
private ctx?: AudioContext;
|
||||
private graph?: Graph;
|
||||
private ctxStateHandler?: () => void;
|
||||
private preparedDfnCore?: DfnCore;
|
||||
// [lotus #9] True while the mic is muted: we suspend our own context so the
|
||||
// worklet stops running inference on silence, and the statechange watcher
|
||||
// must not "heal" that intentional suspension.
|
||||
private micMuted = false;
|
||||
|
||||
public constructor(private readonly config: LotusDenoiseConfig) {}
|
||||
|
||||
/**
|
||||
* [lotus #9] Mirror the mic's mute state onto the owned context. EC uses
|
||||
* `stopMicTrackOnMute: false`, so a muted mic keeps producing (silent) frames
|
||||
* and the ML worklet would otherwise keep running full inference for the
|
||||
* whole time the user is muted.
|
||||
*/
|
||||
public setMicMuted(muted: boolean): void {
|
||||
this.micMuted = muted;
|
||||
const ctx = this.ctx;
|
||||
if (!ctx || ctx.state === "closed") return;
|
||||
if (muted) {
|
||||
if (ctx.state === "running") void ctx.suspend().catch(() => undefined);
|
||||
} else if (ctx.state === "suspended" && this.graph) {
|
||||
void ctx.resume().catch(() => undefined);
|
||||
}
|
||||
}
|
||||
|
||||
public async init(_opts: AudioProcessorOptions): Promise<void> {
|
||||
try {
|
||||
await this.ensureContext();
|
||||
@@ -163,6 +407,9 @@ export class LotusDenoiseProcessor implements TrackProcessor<
|
||||
// Don't orphan the owned context if graph construction fails (browsers
|
||||
// cap live AudioContexts, so repeated failed inits could exhaust them).
|
||||
// The caller degrades to the raw mic; we just release our resources.
|
||||
const core = this.preparedDfnCore;
|
||||
this.preparedDfnCore = undefined;
|
||||
if (core) safeCall(() => core.destroy());
|
||||
await this.closeContext();
|
||||
throw e;
|
||||
}
|
||||
@@ -194,6 +441,9 @@ export class LotusDenoiseProcessor implements TrackProcessor<
|
||||
this.disposeGraph(this.graph);
|
||||
this.graph = undefined;
|
||||
this.processedTrack = undefined;
|
||||
const core = this.preparedDfnCore;
|
||||
this.preparedDfnCore = undefined;
|
||||
if (core) safeCall(() => core.destroy());
|
||||
await this.closeContext();
|
||||
}
|
||||
|
||||
@@ -209,7 +459,7 @@ export class LotusDenoiseProcessor implements TrackProcessor<
|
||||
if (ctx.state !== "closed") await ctx.close().catch(() => undefined);
|
||||
}
|
||||
|
||||
/** Create (once) the model-rate context + register the flat worklet modules. */
|
||||
/** Adopt the prepared context (or create one) + install the state watcher. */
|
||||
private async ensureContext(): Promise<void> {
|
||||
const rate = sampleRateFor(this.config.model);
|
||||
if (
|
||||
@@ -217,36 +467,43 @@ export class LotusDenoiseProcessor implements TrackProcessor<
|
||||
this.ctx.state !== "closed" &&
|
||||
this.ctx.sampleRate === rate
|
||||
) {
|
||||
if (this.ctx.state === "suspended") await resumeCtx(this.ctx);
|
||||
if (this.ctx.state === "suspended" && !this.micMuted)
|
||||
await resumeCtx(this.ctx);
|
||||
return;
|
||||
}
|
||||
await this.closeContext();
|
||||
|
||||
const ctx = new AudioContext({ sampleRate: rate });
|
||||
// [lotus #7] Prefer the context/modules/model prepared before
|
||||
// setProcessor() was called; only load inline if nothing was prepared
|
||||
// (e.g. the rnnoise fallback path, or a second processor after a
|
||||
// republish).
|
||||
const claimed = await claimPreparedAssets(this.config)?.catch(
|
||||
() => undefined,
|
||||
);
|
||||
let ctx: AudioContext;
|
||||
if (claimed && claimed.ctx.state !== "closed") {
|
||||
ctx = claimed.ctx;
|
||||
this.preparedDfnCore = claimed.dfnCore;
|
||||
} else {
|
||||
ctx = await createModelContext(this.config);
|
||||
}
|
||||
try {
|
||||
if (ctx.sampleRate !== rate)
|
||||
throw new Error(`denoise: got ${ctx.sampleRate}Hz, need ${rate}Hz`);
|
||||
|
||||
// Auto-resume if the OS/browser suspends the context mid-call (mobile
|
||||
// backgrounding, audio interruption): the dest node otherwise emits
|
||||
// silence with no recovery. Only resume while a graph is live.
|
||||
// silence with no recovery. Only resume while a graph is live and the
|
||||
// suspension isn't our own mute suspension (#9).
|
||||
const onStateChange = (): void => {
|
||||
if (ctx.state === "suspended" && this.graph)
|
||||
if (ctx.state === "suspended" && this.graph && !this.micMuted)
|
||||
void ctx.resume().catch(() => undefined);
|
||||
};
|
||||
ctx.addEventListener("statechange", onStateChange);
|
||||
|
||||
// Flat models register via addModule here; DTLN/DeepFilterNet bring their
|
||||
// own processor via the dynamic-imported helper (see buildMlNode).
|
||||
if (this.config.model === "rnnoise" || this.config.model === "speex")
|
||||
await ctx.audioWorklet.addModule(
|
||||
this.config.assetBase + FLAT[this.config.model].script,
|
||||
);
|
||||
if (this.config.gate)
|
||||
await ctx.audioWorklet.addModule(this.config.assetBase + GATE.script);
|
||||
// The action can arrive via host postMessage, not a gesture in this
|
||||
// iframe, so the context can start suspended — resume without hanging.
|
||||
if (ctx.state === "suspended") await resumeCtx(ctx);
|
||||
if (ctx.state === "suspended" && !this.micMuted) await resumeCtx(ctx);
|
||||
// Attached while already muted (#9): don't let a prepared, running
|
||||
// context burn inference until the first unmute.
|
||||
else if (ctx.state === "running" && this.micMuted)
|
||||
await ctx.suspend().catch(() => undefined);
|
||||
|
||||
this.ctx = ctx;
|
||||
this.ctxStateHandler = onStateChange;
|
||||
@@ -260,7 +517,7 @@ export class LotusDenoiseProcessor implements TrackProcessor<
|
||||
private async buildGraph(track: MediaStreamTrack): Promise<Graph> {
|
||||
const ctx = this.ctx!;
|
||||
const source = ctx.createMediaStreamSource(new MediaStream([track]));
|
||||
const dest = ctx.createMediaStreamDestination();
|
||||
const dest = new MediaStreamAudioDestinationNode(ctx, MONO);
|
||||
const nodes: AudioNode[] = [];
|
||||
const disposes: (() => void)[] = [];
|
||||
|
||||
@@ -277,6 +534,7 @@ export class LotusDenoiseProcessor implements TrackProcessor<
|
||||
// made the threshold operate on pre-denoise levels. Gate the residual.
|
||||
if (this.config.gate) {
|
||||
const gate = new AudioWorkletNode(ctx, GATE.name, {
|
||||
...MONO,
|
||||
processorOptions: {
|
||||
openThreshold: this.config.gateThreshold,
|
||||
closeThreshold: this.config.gateThreshold - 5,
|
||||
@@ -289,11 +547,11 @@ export class LotusDenoiseProcessor implements TrackProcessor<
|
||||
nodes.push(gate);
|
||||
}
|
||||
|
||||
// Only mix a dry floor for the LOW-LATENCY flat models (RNNoise/Speex).
|
||||
// DTLN/DeepFilterNet add tens of ms of algorithmic latency, so summing an
|
||||
// undelayed dry copy would comb-filter the voice — for those we rely on
|
||||
// the model's own level (e.g. DFN noiseReductionLevel) instead. RNNoise is
|
||||
// also where the "robotic/underwater" reports come from, so this targets it.
|
||||
// Only mix a dry floor for the flat models (RNNoise/Speex), whose
|
||||
// framing latency is known exactly (DRY_DELAY_SAMPLES); the DTLN/DFN
|
||||
// figures are best-known estimates, so for those we rely on the model's
|
||||
// own level (e.g. DFN noiseReductionLevel) instead. RNNoise is also where
|
||||
// the "robotic/underwater" reports come from, so this targets it.
|
||||
const lowLatency =
|
||||
this.config.model === "rnnoise" || this.config.model === "speex";
|
||||
const floor = lowLatency
|
||||
@@ -305,17 +563,25 @@ export class LotusDenoiseProcessor implements TrackProcessor<
|
||||
// (kills the "underwater"/pumping artifact). During speech (denoised ≈
|
||||
// original) the two sum back to ~unity; in noise-only gaps the output
|
||||
// floors at `floor` × original instead of digital silence.
|
||||
const wetGain = ctx.createGain();
|
||||
wetGain.gain.value = 1 - floor;
|
||||
const wetGain = new GainNode(ctx, { ...MONO, gain: 1 - floor });
|
||||
wetHead.connect(wetGain);
|
||||
wetGain.connect(dest);
|
||||
nodes.push(wetGain);
|
||||
|
||||
const dryGain = ctx.createGain();
|
||||
dryGain.gain.value = floor;
|
||||
source.connect(dryGain);
|
||||
// [lotus #25] Delay the dry copy by the model's algorithmic latency so
|
||||
// it sums in phase with the (framed, hence delayed) wet path instead
|
||||
// of comb-filtering against it.
|
||||
const delaySec = DRY_DELAY_SAMPLES[this.config.model] / ctx.sampleRate;
|
||||
const dryDelay = new DelayNode(ctx, {
|
||||
...MONO,
|
||||
maxDelayTime: Math.max(delaySec, 1 / ctx.sampleRate),
|
||||
delayTime: delaySec,
|
||||
});
|
||||
const dryGain = new GainNode(ctx, { ...MONO, gain: floor });
|
||||
source.connect(dryDelay);
|
||||
dryDelay.connect(dryGain);
|
||||
dryGain.connect(dest);
|
||||
nodes.push(dryGain);
|
||||
nodes.push(dryDelay, dryGain);
|
||||
} else {
|
||||
wetHead.connect(dest);
|
||||
}
|
||||
@@ -350,48 +616,36 @@ export class LotusDenoiseProcessor implements TrackProcessor<
|
||||
if (model === "dtln") {
|
||||
// Self-contained ESM that resolves its own processor + LiteRT wasm +
|
||||
// TFLite models. bypassUntilReady passes raw audio until the model loads.
|
||||
const mod = await import(
|
||||
/* @vite-ignore */ `${base}workadventure/audio-worklet.js`
|
||||
);
|
||||
return (await mod.createNoiseSuppressionAudioWorklet(ctx, {
|
||||
const mod = await loadDtlnModule(this.config);
|
||||
return await mod.createNoiseSuppressionAudioWorklet(ctx, {
|
||||
bypassUntilReady: true,
|
||||
})) as MlNode;
|
||||
});
|
||||
}
|
||||
|
||||
if (model === "deepfilternet") {
|
||||
const dfnBase = new URL(`${base}deepfilternet`, window.location.href)
|
||||
.href;
|
||||
const mod = await import(
|
||||
/* @vite-ignore */ `${base}deepfilternet/index.esm.js`
|
||||
);
|
||||
const core = new mod.DeepFilterNet3Core({
|
||||
sampleRate: 48_000,
|
||||
// 60, not 80: full-strength suppression is the main source of the
|
||||
// "over-processed" character; a lower level keeps voice natural while
|
||||
// the dry/wet floor handles the noise tail.
|
||||
noiseReductionLevel: 60,
|
||||
assetConfig: { cdnUrl: dfnBase },
|
||||
});
|
||||
await core.initialize();
|
||||
const node = (await core.createAudioWorkletNode(ctx)) as AudioNode;
|
||||
// [lotus #7] Use the core initialised by prepareDenoiseAssets() if we
|
||||
// have one (first graph); later rebuilds (restart) load a fresh core.
|
||||
const prepared = this.preparedDfnCore;
|
||||
this.preparedDfnCore = undefined;
|
||||
const core = prepared ?? (await loadDfnCore(this.config));
|
||||
const node = await core.createAudioWorkletNode(ctx);
|
||||
return { node, dispose: () => safeCall(() => core.destroy()) };
|
||||
}
|
||||
|
||||
// Flat sapphi worklet (rnnoise/speex).
|
||||
const flat = FLAT[model];
|
||||
const useSimd = model === "rnnoise" && !!flat.simdWasm && supportsSimd();
|
||||
const wasmFile = useSimd ? flat.simdWasm! : flat.wasm;
|
||||
const { primary, fallback } = flatWasmFiles(model);
|
||||
let wasmBinary: ArrayBuffer;
|
||||
try {
|
||||
wasmBinary = await fetchWasm(base + wasmFile);
|
||||
wasmBinary = await fetchWasm(base + primary);
|
||||
} catch (e) {
|
||||
if (useSimd) {
|
||||
wasmCache.delete(base + wasmFile);
|
||||
wasmBinary = await fetchWasm(base + flat.wasm); // fall back to non-SIMD
|
||||
if (fallback) {
|
||||
wasmCache.delete(base + primary);
|
||||
wasmBinary = await fetchWasm(base + fallback); // fall back to non-SIMD
|
||||
} else throw e;
|
||||
}
|
||||
const node = new AudioWorkletNode(ctx, flat.name, {
|
||||
channelCount: 1,
|
||||
...MONO,
|
||||
numberOfInputs: 1,
|
||||
numberOfOutputs: 1,
|
||||
processorOptions: { maxChannels: 1, wasmBinary },
|
||||
|
||||
+26
-11
@@ -10,12 +10,33 @@ import { type IWidgetApiRequest } from "matrix-widget-api";
|
||||
import { type CallViewModel } from "../state/CallViewModel/CallViewModel";
|
||||
import { widget } from "../widget";
|
||||
import { LotusWidgetActions } from "./lotusActions";
|
||||
import { type ManualSpotlight } from "./lotusSpotlight";
|
||||
|
||||
/**
|
||||
* Parse a `focus_participant` payload into a pin, `null` to clear, or
|
||||
* `undefined` to leave the current pin alone (#30). Exported for tests.
|
||||
*
|
||||
* Mirror deafen's partial-payload semantics: a payload that OMITS `userId`
|
||||
* must keep the current spotlight, not clear it. Only act when the key is
|
||||
* actually present — an explicit `null` clears, a string pins that user. An
|
||||
* optional `id` (EC media id `${userId}:${deviceId}`, as sent to the host in
|
||||
* `io.lotus.call_state`) selects a specific device of that user.
|
||||
*/
|
||||
export function parseFocusPayload(
|
||||
data: unknown,
|
||||
): ManualSpotlight | null | undefined {
|
||||
if (typeof data !== "object" || data === null || !("userId" in data))
|
||||
return undefined;
|
||||
const { userId, id } = data as { userId?: unknown; id?: unknown };
|
||||
if (typeof userId !== "string") return null;
|
||||
return { userId, id: typeof id === "string" ? id : null };
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle the host's `io.lotus.focus_participant` toWidget action (#4): pin a
|
||||
* participant to the spotlight by Matrix user id, or clear it with
|
||||
* `{ userId: null }`. This replaces cinny's old DOM `.click()` tile-selector
|
||||
* hack with a real, layout-aware spotlight override.
|
||||
* participant to the spotlight by Matrix user id (and optionally media id), or
|
||||
* clear it with `{ userId: null }`. This replaces cinny's old DOM `.click()`
|
||||
* tile-selector hack with a real, layout-aware spotlight override.
|
||||
*
|
||||
* No effect unless the host actually sends the action, so registering the
|
||||
* handler whenever we're a widget is safe. Returns a teardown function.
|
||||
@@ -27,14 +48,8 @@ export function startLotusFocus(vm: CallViewModel): () => void {
|
||||
const handler = (ev: CustomEvent<IWidgetApiRequest>): void => {
|
||||
// Always reply so the host transport doesn't time out.
|
||||
w.api.transport.reply(ev.detail, {});
|
||||
const data = ev.detail.data as { userId?: unknown } | undefined;
|
||||
// Mirror deafen's partial-payload semantics: a payload that OMITS `userId`
|
||||
// must keep the current spotlight, not clear it. Only act when the key is
|
||||
// actually present — an explicit `null` clears, a string pins that user.
|
||||
if (data && "userId" in data) {
|
||||
const userId = typeof data.userId === "string" ? data.userId : null;
|
||||
vm.setManualSpotlight(userId);
|
||||
}
|
||||
const target = parseFocusPayload(ev.detail.data);
|
||||
if (target !== undefined) vm.setManualSpotlight(target);
|
||||
};
|
||||
|
||||
w.lazyActions.on(LotusWidgetActions.FocusParticipant, handler);
|
||||
|
||||
@@ -7,7 +7,11 @@ Please see LICENSE in the repository root for full details.
|
||||
|
||||
import { describe, expect, test, vi } from "vitest";
|
||||
|
||||
import { buildPatch, patchSender } from "./lotusQuality";
|
||||
import {
|
||||
applyProportionalMaxBitrate,
|
||||
buildPatch,
|
||||
patchSender,
|
||||
} from "./lotusQuality";
|
||||
|
||||
function makeSender(
|
||||
initialEncodings: RTCRtpEncodingParameters[] = [{}],
|
||||
@@ -91,3 +95,65 @@ describe("lotusQuality set_quality -> clear (#11)", () => {
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe("lotusQuality screenshare simulcast cap (#12)", () => {
|
||||
test("caps the aggregate across a 3-layer simulcast publication, not per-layer", async () => {
|
||||
// A typical VP8 simulcast screenshare publication: low/medium/high
|
||||
// layers with existing presets, ordered low-to-high like LiveKit
|
||||
// publishes them.
|
||||
const sender = makeSender([
|
||||
{ maxBitrate: 150_000 },
|
||||
{ maxBitrate: 500_000 },
|
||||
{ maxBitrate: 1_500_000 },
|
||||
]);
|
||||
const writtenKeys = new WeakMap<
|
||||
RTCRtpSender,
|
||||
Set<keyof RTCRtpEncodingParameters>
|
||||
>();
|
||||
|
||||
const cap = 750_000;
|
||||
const patch = buildPatch(sender, { maxBitrate: cap }, writtenKeys);
|
||||
await patchSender(sender, patch, writtenKeys);
|
||||
|
||||
const encodings = sender.getParameters().encodings;
|
||||
expect(encodings).toHaveLength(3);
|
||||
// Each layer keeps its old proportion of the total, but the aggregate
|
||||
// across all layers must not exceed the requested cap — that's the bug:
|
||||
// writing `cap` into every layer let the aggregate reach ~3x the cap.
|
||||
const oldTotal = 150_000 + 500_000 + 1_500_000;
|
||||
expect(encodings[0].maxBitrate).toBe(
|
||||
Math.floor((150_000 / oldTotal) * cap),
|
||||
);
|
||||
expect(encodings[1].maxBitrate).toBe(
|
||||
Math.floor((500_000 / oldTotal) * cap),
|
||||
);
|
||||
expect(encodings[2].maxBitrate).toBe(
|
||||
Math.floor((1_500_000 / oldTotal) * cap),
|
||||
);
|
||||
// The highest layer still ends up with the largest share.
|
||||
expect(encodings[2].maxBitrate).toBeGreaterThan(encodings[1].maxBitrate!);
|
||||
expect(encodings[1].maxBitrate).toBeGreaterThan(encodings[0].maxBitrate!);
|
||||
|
||||
const aggregate = encodings.reduce(
|
||||
(sum, enc) => sum + (enc.maxBitrate ?? 0),
|
||||
0,
|
||||
);
|
||||
expect(aggregate).toBeLessThanOrEqual(cap);
|
||||
});
|
||||
|
||||
test("a single encoding keeps the previous (non-scaled) behaviour", () => {
|
||||
const encodings: RTCRtpEncodingParameters[] = [{ maxBitrate: 2_000_000 }];
|
||||
applyProportionalMaxBitrate(encodings, 500_000);
|
||||
expect(encodings).toEqual([{ maxBitrate: 500_000 }]);
|
||||
});
|
||||
|
||||
test("falls back to capping only the highest layer when there are no prior ratios", () => {
|
||||
const encodings: RTCRtpEncodingParameters[] = [
|
||||
{}, // no prior cap on any layer to derive a ratio from
|
||||
{},
|
||||
];
|
||||
applyProportionalMaxBitrate(encodings, 500_000);
|
||||
expect(encodings[0].maxBitrate).toBeUndefined();
|
||||
expect(encodings[1].maxBitrate).toBe(500_000);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -198,6 +198,43 @@ export function buildPatch(
|
||||
return patch;
|
||||
}
|
||||
|
||||
// [lotus] Treat `cap` as a budget for the WHOLE simulcast publication rather
|
||||
// than a per-encoding value, and distribute it across encodings in
|
||||
// proportion to their existing (pre-patch) ratios — instead of writing `cap`
|
||||
// into every encoding (#12), which let the aggregate reach up to N× the
|
||||
// requested cap (and also raised the low layers far above their presets).
|
||||
// With a single encoding this reduces to the previous behaviour: that
|
||||
// encoding simply gets `cap`.
|
||||
// Exported for unit testing only (see lotusQuality.test.ts) — not part of
|
||||
// the module's public surface used by callers.
|
||||
export function applyProportionalMaxBitrate(
|
||||
encodings: RTCRtpEncodingParameters[],
|
||||
cap: number,
|
||||
): void {
|
||||
if (encodings.length <= 1) {
|
||||
for (const enc of encodings) enc.maxBitrate = cap;
|
||||
return;
|
||||
}
|
||||
const total = encodings.reduce(
|
||||
(sum, enc) =>
|
||||
sum + (typeof enc.maxBitrate === "number" ? enc.maxBitrate : 0),
|
||||
0,
|
||||
);
|
||||
if (total <= 0) {
|
||||
// No existing ratios to scale from: fall back to capping only the
|
||||
// highest layer (LiveKit orders simulcast encodings low-to-high, so
|
||||
// that's the last one) and leave the others alone rather than guess.
|
||||
encodings[encodings.length - 1].maxBitrate = cap;
|
||||
return;
|
||||
}
|
||||
// Math.floor (not round) so rounding error can only push the aggregate
|
||||
// under the cap, never over it.
|
||||
for (const enc of encodings) {
|
||||
const prev = typeof enc.maxBitrate === "number" ? enc.maxBitrate : 0;
|
||||
enc.maxBitrate = Math.floor((prev / total) * cap);
|
||||
}
|
||||
}
|
||||
|
||||
// Exported for unit testing only (see lotusQuality.test.ts) — not part of
|
||||
// the module's public surface used by callers.
|
||||
export async function patchSender(
|
||||
@@ -210,10 +247,26 @@ export async function patchSender(
|
||||
const params = sender.getParameters();
|
||||
if (!params.encodings || params.encodings.length === 0)
|
||||
params.encodings = [{}];
|
||||
// Apply to EVERY encoding, not just encodings[0]: screenshare publishes
|
||||
// with simulcast (VP8), so encodings[0] is the small layer and the
|
||||
// full-resolution layer — the real bandwidth hog — is a later encoding.
|
||||
for (const enc of params.encodings) Object.assign(enc, patch);
|
||||
// Screenshare publishes with simulcast (VP8): encodings[0] is the small
|
||||
// layer and the full-resolution layer — the real bandwidth hog — is a
|
||||
// later encoding. maxFramerate (and an `undefined` clear) still apply to
|
||||
// every encoding, but maxBitrate is special-cased (#12): writing the same
|
||||
// value into every encoding let the aggregate reach N× the requested cap
|
||||
// and also raised the low layers far above their presets, so instead we
|
||||
// treat the requested value as a budget for the whole publication and
|
||||
// distribute it across encodings proportionally to their existing
|
||||
// ratios.
|
||||
const { maxBitrate, ...rest } = patch;
|
||||
for (const enc of params.encodings) Object.assign(enc, rest);
|
||||
if ("maxBitrate" in patch) {
|
||||
if (maxBitrate === undefined) {
|
||||
// Clearing the cap (#11): drop it from every encoding rather than
|
||||
// scaling — there's no budget to distribute.
|
||||
for (const enc of params.encodings) enc.maxBitrate = undefined;
|
||||
} else {
|
||||
applyProportionalMaxBitrate(params.encodings, maxBitrate);
|
||||
}
|
||||
}
|
||||
await sender.setParameters(params);
|
||||
// Remember only the caps that are still active (defined) after this
|
||||
// write, so a later clear knows exactly which keys to unset.
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
/*
|
||||
Copyright 2026 Lotus Guild
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { BehaviorSubject } from "rxjs";
|
||||
|
||||
import { globalScope } from "../state/ObservableScope";
|
||||
|
||||
/**
|
||||
* Whether the host has muted remote screenshare AUDIO (`io.lotus.set_deafen`
|
||||
* `screenshareAudioMuted`). Consumed by `LivekitRoomAudioRenderer` exactly the
|
||||
* way `muteAllAudio$` is: it becomes the `muted` prop of every
|
||||
* `Track.Source.ScreenShareAudio` element, so it survives re-renders, applies
|
||||
* to tracks that are published LATER (a sharer stopping and re-sharing, a late
|
||||
* joiner) and never fights EC's per-tile volume controls — which was the
|
||||
* failure mode of the old `RemoteParticipant.setVolume(0, ScreenShareAudio)`
|
||||
* approach: `createVolumeControls` writes its own volume (1) through the same
|
||||
* sink the moment a new screenshare media item appears, un-muting it.
|
||||
*/
|
||||
export const setScreenshareAudioMuted$ = new BehaviorSubject(false);
|
||||
|
||||
export const muteScreenshareAudio$ = globalScope.behavior(
|
||||
setScreenshareAudioMuted$,
|
||||
);
|
||||
@@ -0,0 +1,295 @@
|
||||
/*
|
||||
Copyright 2026 Lotus Guild
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { describe, expect, test, vi } from "vitest";
|
||||
import { BehaviorSubject, of, Subject } from "rxjs";
|
||||
|
||||
import { withTestScheduler } from "../utils/test";
|
||||
import { type UserMediaViewModel } from "../state/media/UserMediaViewModel";
|
||||
import { type ScreenShareViewModel } from "../state/media/ScreenShareViewModel";
|
||||
import { type LayoutMode } from "../state/LayoutSwitchViewModel";
|
||||
import {
|
||||
createManualSpotlight,
|
||||
type ManualSpotlight,
|
||||
overrideSpotlight$,
|
||||
pinnedMedia$,
|
||||
resolveManualSpotlight,
|
||||
} from "./lotusSpotlight";
|
||||
import { parseFocusPayload } from "./lotusFocus";
|
||||
|
||||
interface FakeMedia {
|
||||
id: string;
|
||||
userId: string;
|
||||
type: "user";
|
||||
local: boolean;
|
||||
speaking$: BehaviorSubject<boolean>;
|
||||
}
|
||||
|
||||
function media(
|
||||
userId: string,
|
||||
deviceId: string,
|
||||
speaking = false,
|
||||
local = false,
|
||||
): FakeMedia {
|
||||
return {
|
||||
id: `${userId}:${deviceId}`,
|
||||
userId,
|
||||
type: "user",
|
||||
local,
|
||||
speaking$: new BehaviorSubject(speaking),
|
||||
};
|
||||
}
|
||||
const asUser = (m: FakeMedia): UserMediaViewModel =>
|
||||
m as unknown as UserMediaViewModel;
|
||||
const screen = { id: "@a:x:d1:screen", userId: "@a:x", type: "screen share" };
|
||||
|
||||
const aliceDesk = media("@alice:x", "desk");
|
||||
const alicePhone = media("@alice:x", "phone", true);
|
||||
const bob = media("@bob:x", "d");
|
||||
const all = [aliceDesk, alicePhone, bob];
|
||||
|
||||
describe("resolveManualSpotlight (#30)", () => {
|
||||
const speaking = (m: FakeMedia): boolean => m.speaking$.value;
|
||||
|
||||
test("null pin resolves to nothing", () => {
|
||||
expect(resolveManualSpotlight(null, all, speaking)).toBeUndefined();
|
||||
});
|
||||
|
||||
test("prefers the media id when given", () => {
|
||||
expect(
|
||||
resolveManualSpotlight(
|
||||
{ userId: "@alice:x", id: "@alice:x:desk" },
|
||||
all,
|
||||
speaking,
|
||||
),
|
||||
).toBe(aliceDesk);
|
||||
});
|
||||
|
||||
test("falls back to the userId when the id is not present", () => {
|
||||
expect(
|
||||
resolveManualSpotlight(
|
||||
{ userId: "@alice:x", id: "@alice:x:tablet" },
|
||||
all,
|
||||
speaking,
|
||||
),
|
||||
).toBe(alicePhone);
|
||||
});
|
||||
|
||||
test("userId only: prefers the speaking device, else the first", () => {
|
||||
expect(
|
||||
resolveManualSpotlight({ userId: "@alice:x", id: null }, all, speaking),
|
||||
).toBe(alicePhone);
|
||||
expect(
|
||||
resolveManualSpotlight(
|
||||
{ userId: "@alice:x", id: null },
|
||||
all,
|
||||
() => false,
|
||||
),
|
||||
).toBe(aliceDesk);
|
||||
});
|
||||
|
||||
test("absent user resolves to nothing", () => {
|
||||
expect(
|
||||
resolveManualSpotlight({ userId: "@carol:x", id: null }, all, speaking),
|
||||
).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe("pinnedMedia$", () => {
|
||||
test("follows the speaking device live for a userId-only pin", () => {
|
||||
withTestScheduler(({ expectObservable, schedule }) => {
|
||||
const desk = media("@alice:x", "desk", true);
|
||||
const phone = media("@alice:x", "phone", false);
|
||||
const pin$ = new BehaviorSubject<ManualSpotlight | null>({
|
||||
userId: "@alice:x",
|
||||
id: null,
|
||||
});
|
||||
schedule("-a", {
|
||||
a: () => {
|
||||
desk.speaking$.next(false);
|
||||
phone.speaking$.next(true);
|
||||
},
|
||||
});
|
||||
expectObservable(
|
||||
pinnedMedia$(pin$, of([asUser(desk), asUser(phone)])),
|
||||
).toBe("dp", { d: desk, p: phone });
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("overrideSpotlight$", () => {
|
||||
test("no pin: identical to upstream", () => {
|
||||
withTestScheduler(({ expectObservable }) => {
|
||||
const result$ = overrideSpotlight$(
|
||||
of(asUser(bob)),
|
||||
of(null),
|
||||
of([]),
|
||||
of(undefined),
|
||||
of(all.map(asUser)),
|
||||
);
|
||||
expectObservable(result$.pipe()).toBe("(a|)", {
|
||||
a: expect.objectContaining({ spotlight: [bob] }),
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
test("pin replaces the auto speaker without a screenshare", () => {
|
||||
withTestScheduler(({ expectObservable }) => {
|
||||
const result$ = overrideSpotlight$(
|
||||
of(asUser(bob)),
|
||||
of({ userId: "@alice:x", id: "@alice:x:desk" }),
|
||||
of([]),
|
||||
of(undefined),
|
||||
of(all.map(asUser)),
|
||||
);
|
||||
expectObservable(result$).toBe("(a|)", {
|
||||
a: expect.objectContaining({ spotlight: [aliceDesk] }),
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
test("#29: PiP keeps the auto speaker when a third party is pinned", () => {
|
||||
withTestScheduler(({ expectObservable }) => {
|
||||
let pip$;
|
||||
overrideSpotlight$(
|
||||
of(asUser(bob)),
|
||||
of({ userId: "@alice:x", id: null }),
|
||||
of([screen as unknown as ScreenShareViewModel]),
|
||||
of(undefined),
|
||||
of(all.map(asUser)),
|
||||
).subscribe((r) => {
|
||||
expect(r.spotlight).toEqual([screen, alicePhone]);
|
||||
pip$ = r.pip$;
|
||||
});
|
||||
expectObservable(pip$!).toBe("(b|)", { b: bob });
|
||||
});
|
||||
});
|
||||
|
||||
test("#29: PiP is blanked only when the auto speaker is the pinned one", () => {
|
||||
withTestScheduler(({ expectObservable }) => {
|
||||
let pip$;
|
||||
overrideSpotlight$(
|
||||
of(asUser(alicePhone)),
|
||||
of({ userId: "@alice:x", id: null }),
|
||||
of([screen as unknown as ScreenShareViewModel]),
|
||||
of(undefined),
|
||||
of(all.map(asUser)),
|
||||
).subscribe((r) => {
|
||||
pip$ = r.pip$;
|
||||
});
|
||||
expectObservable(pip$!).toBe("(u|)", { u: undefined });
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("createManualSpotlight (#3 / #16)", () => {
|
||||
function setup(initialLayout: LayoutMode = "grid") {
|
||||
const pin$ = new BehaviorSubject<ManualSpotlight | null>(null);
|
||||
const layout$ = new BehaviorSubject<LayoutMode>(initialLayout);
|
||||
const setLayout = vi.fn((m: LayoutMode) => layout$.next(m));
|
||||
const userMedia$ = new BehaviorSubject<{ userId: string }[]>([
|
||||
{ userId: "@alice:x" },
|
||||
{ userId: "@bob:x" },
|
||||
]);
|
||||
const leave$ = new Subject<void>();
|
||||
const ctl = createManualSpotlight(
|
||||
pin$,
|
||||
userMedia$,
|
||||
leave$,
|
||||
{ layout$, setLayout },
|
||||
5000,
|
||||
);
|
||||
const sub = ctl.effects$.subscribe();
|
||||
return { pin$, layout$, setLayout, userMedia$, leave$, ctl, sub };
|
||||
}
|
||||
|
||||
test("pinning forces spotlight and clearing restores the displaced mode", () => {
|
||||
const { ctl, layout$, setLayout, pin$ } = setup("grid");
|
||||
ctl.setManualSpotlight({ userId: "@alice:x", id: null });
|
||||
expect(pin$.value).toEqual({ userId: "@alice:x", id: null });
|
||||
expect(setLayout).toHaveBeenLastCalledWith("spotlight");
|
||||
// Re-pinning someone else does not touch the layout again.
|
||||
ctl.setManualSpotlight({ userId: "@bob:x", id: null });
|
||||
expect(setLayout).toHaveBeenCalledTimes(1);
|
||||
ctl.setManualSpotlight(null);
|
||||
expect(layout$.value).toBe("grid");
|
||||
expect(setLayout).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
test("already in spotlight: nothing is forced or restored", () => {
|
||||
const { ctl, setLayout } = setup("spotlight");
|
||||
ctl.setManualSpotlight({ userId: "@alice:x", id: null });
|
||||
ctl.setManualSpotlight(null);
|
||||
expect(setLayout).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
test("does not clobber a layout the user chose while pinned", () => {
|
||||
const { ctl, layout$, setLayout } = setup("grid");
|
||||
ctl.setManualSpotlight({ userId: "@alice:x", id: null });
|
||||
layout$.next("grid"); // user switched back manually
|
||||
ctl.setManualSpotlight(null);
|
||||
expect(setLayout).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
test("#16: pin is cleared when the user is gone for 5 s, not on a blip", () => {
|
||||
vi.useFakeTimers();
|
||||
try {
|
||||
const { ctl, pin$, userMedia$, layout$ } = setup("grid");
|
||||
ctl.setManualSpotlight({ userId: "@alice:x", id: null });
|
||||
// Transient absence: back within the window.
|
||||
userMedia$.next([{ userId: "@bob:x" }]);
|
||||
vi.advanceTimersByTime(2000);
|
||||
userMedia$.next([{ userId: "@alice:x" }, { userId: "@bob:x" }]);
|
||||
vi.advanceTimersByTime(6000);
|
||||
expect(pin$.value).not.toBeNull();
|
||||
// Real leave.
|
||||
userMedia$.next([{ userId: "@bob:x" }]);
|
||||
vi.advanceTimersByTime(4999);
|
||||
expect(pin$.value).not.toBeNull();
|
||||
vi.advanceTimersByTime(1);
|
||||
expect(pin$.value).toBeNull();
|
||||
expect(layout$.value).toBe("grid");
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
test("#16: pin is cleared on leave$", () => {
|
||||
const { ctl, pin$, leave$, layout$ } = setup("grid");
|
||||
ctl.setManualSpotlight({ userId: "@alice:x", id: null });
|
||||
expect(layout$.value).toBe("spotlight");
|
||||
leave$.next();
|
||||
expect(pin$.value).toBeNull();
|
||||
expect(layout$.value).toBe("grid");
|
||||
});
|
||||
});
|
||||
|
||||
describe("parseFocusPayload (#30)", () => {
|
||||
test("omitted userId leaves the pin alone", () => {
|
||||
expect(parseFocusPayload(undefined)).toBeUndefined();
|
||||
expect(parseFocusPayload({})).toBeUndefined();
|
||||
expect(parseFocusPayload({ id: "@a:x:d" })).toBeUndefined();
|
||||
});
|
||||
test("null / non-string userId clears", () => {
|
||||
expect(parseFocusPayload({ userId: null })).toBeNull();
|
||||
expect(parseFocusPayload({ userId: 42 })).toBeNull();
|
||||
});
|
||||
test("userId with and without id", () => {
|
||||
expect(parseFocusPayload({ userId: "@a:x" })).toEqual({
|
||||
userId: "@a:x",
|
||||
id: null,
|
||||
});
|
||||
expect(parseFocusPayload({ userId: "@a:x", id: "@a:x:d" })).toEqual({
|
||||
userId: "@a:x",
|
||||
id: "@a:x:d",
|
||||
});
|
||||
expect(parseFocusPayload({ userId: "@a:x", id: 7 })).toEqual({
|
||||
userId: "@a:x",
|
||||
id: null,
|
||||
});
|
||||
});
|
||||
});
|
||||
+191
-28
@@ -5,18 +5,103 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { combineLatest, map, type Observable, of, switchMap } from "rxjs";
|
||||
import {
|
||||
type BehaviorSubject,
|
||||
combineLatest,
|
||||
debounceTime,
|
||||
distinctUntilChanged,
|
||||
filter,
|
||||
ignoreElements,
|
||||
map,
|
||||
merge,
|
||||
type Observable,
|
||||
of,
|
||||
switchMap,
|
||||
tap,
|
||||
} from "rxjs";
|
||||
|
||||
import { type UserMediaViewModel } from "../state/media/UserMediaViewModel";
|
||||
import { type ScreenShareViewModel } from "../state/media/ScreenShareViewModel";
|
||||
import { type MediaViewModel } from "../state/media/MediaViewModel";
|
||||
import { type LocalUserMediaViewModel } from "../state/media/LocalUserMediaViewModel";
|
||||
import { type LayoutMode } from "../state/LayoutSwitchViewModel";
|
||||
import { type Behavior } from "../state/Behavior";
|
||||
|
||||
interface SpotlightAndPip {
|
||||
spotlight: MediaViewModel[];
|
||||
pip$: Observable<UserMediaViewModel | undefined>;
|
||||
}
|
||||
|
||||
/**
|
||||
* [lotus #4/#30] A host-requested spotlight pin. `id` is the EC media id
|
||||
* (`${userId}:${deviceId}`, the same value `io.lotus.call_state` sends the
|
||||
* host) and is preferred when present; `userId` alone resolves to the
|
||||
* currently-speaking device of that user, else their first device.
|
||||
*/
|
||||
export interface ManualSpotlight {
|
||||
userId: string;
|
||||
id: string | null;
|
||||
}
|
||||
|
||||
/**
|
||||
* How long (ms) a pinned participant may be absent from `userMedia$` before
|
||||
* the pin is dropped (#16). Long enough to ride out a transient LiveKit
|
||||
* reconnect, short enough that a real leave restores speaker-follows promptly.
|
||||
*/
|
||||
export const MANUAL_SPOTLIGHT_GONE_MS = 5000;
|
||||
|
||||
/**
|
||||
* Resolve the pin against the current media list (#30). Pure; the observable
|
||||
* wrapper below feeds it the live `speaking` state.
|
||||
*/
|
||||
export function resolveManualSpotlight<
|
||||
M extends { id: string; userId: string },
|
||||
>(
|
||||
target: ManualSpotlight | null,
|
||||
mediaItems: M[],
|
||||
speaking: (m: M) => boolean,
|
||||
): M | undefined {
|
||||
if (target === null) return undefined;
|
||||
if (target.id !== null) {
|
||||
const byId = mediaItems.find((m) => m.id === target.id);
|
||||
if (byId) return byId;
|
||||
// The pinned device is gone but the user may still be here on another
|
||||
// one — fall through to the userId match rather than dropping the pin.
|
||||
}
|
||||
const devices = mediaItems.filter((m) => m.userId === target.userId);
|
||||
if (devices.length <= 1) return devices[0];
|
||||
return devices.find(speaking) ?? devices[0];
|
||||
}
|
||||
|
||||
/**
|
||||
* The pinned media item, or `undefined` when there is no pin or the pinned
|
||||
* participant is not (currently) in the call.
|
||||
*/
|
||||
export function pinnedMedia$(
|
||||
manualSpotlight$: Observable<ManualSpotlight | null>,
|
||||
userMedia$: Observable<UserMediaViewModel[]>,
|
||||
): Observable<UserMediaViewModel | undefined> {
|
||||
return combineLatest([manualSpotlight$, userMedia$]).pipe(
|
||||
switchMap(([target, mediaItems]) => {
|
||||
if (target === null) return of(undefined);
|
||||
const devices = mediaItems.filter((m) => m.userId === target.userId);
|
||||
// Only subscribe to speaking$ when there is actually a choice to make.
|
||||
const speaking$: Observable<Set<UserMediaViewModel>> =
|
||||
devices.length > 1 && !devices.some((m) => m.id === target.id)
|
||||
? combineLatest(devices.map((m) => m.speaking$)).pipe(
|
||||
map((flags) => new Set(devices.filter((_, i) => flags[i]))),
|
||||
)
|
||||
: of(new Set<UserMediaViewModel>());
|
||||
return speaking$.pipe(
|
||||
map((speaking) =>
|
||||
resolveManualSpotlight(target, mediaItems, (m) => speaking.has(m)),
|
||||
),
|
||||
);
|
||||
}),
|
||||
distinctUntilChanged(),
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* [lotus #4] Manual spotlight override.
|
||||
*
|
||||
@@ -27,41 +112,35 @@ interface SpotlightAndPip {
|
||||
* `spotlightSpeaker$` auto-selection unchanged and just routes the
|
||||
* screenshare/spotlight computation through this wrapper at one call point.
|
||||
*
|
||||
* Behaviour is IDENTICAL to upstream whenever `manualSpotlightUserId$` stays
|
||||
* Behaviour is IDENTICAL to upstream whenever `manualSpotlight$` stays
|
||||
* `null` (the default). When it names a participant that is still present:
|
||||
* - with no screenshare, that participant is spotlighted instead of the
|
||||
* auto-selected active speaker;
|
||||
* - during a screenshare, that participant's camera is surfaced in the
|
||||
* spotlight ALONGSIDE the shared screen (#4 / A5 "focus camera during
|
||||
* screenshare"), and the redundant PiP is hidden.
|
||||
* screenshare"); the PiP keeps showing the auto-selected speaker and is
|
||||
* only blanked when that speaker IS the pinned participant (#29).
|
||||
*
|
||||
* @param autoSpotlightSpeaker$ upstream's speaker-follows auto selection
|
||||
* @param manualSpotlightUserId$ host-pinned userId, or `null` for auto (default)
|
||||
* @param manualSpotlight$ host pin, or `null` for auto (default)
|
||||
* @param screenShares$ current screen-share view models
|
||||
* @param localUserMediaForPip$ local media suitable for the PiP
|
||||
* @param userMedia$ all user media in the call (to resolve the pinned userId)
|
||||
* @param userMedia$ all user media in the call (to resolve the pin)
|
||||
*/
|
||||
export function overrideSpotlight$(
|
||||
autoSpotlightSpeaker$: Observable<UserMediaViewModel | undefined>,
|
||||
manualSpotlightUserId$: Observable<string | null>,
|
||||
manualSpotlight$: Observable<ManualSpotlight | null>,
|
||||
screenShares$: Observable<ScreenShareViewModel[]>,
|
||||
localUserMediaForPip$: Observable<LocalUserMediaViewModel | undefined>,
|
||||
userMedia$: Observable<UserMediaViewModel[]>,
|
||||
): Observable<SpotlightAndPip> {
|
||||
const pinned$ = pinnedMedia$(manualSpotlight$, userMedia$);
|
||||
|
||||
// The effective spotlight speaker: the host-pinned participant when set and
|
||||
// still present, otherwise upstream's auto-selected speaker.
|
||||
const spotlightSpeaker$ = combineLatest([
|
||||
autoSpotlightSpeaker$,
|
||||
manualSpotlightUserId$,
|
||||
userMedia$,
|
||||
]).pipe(
|
||||
map(([auto, manualUserId, mediaItems]) => {
|
||||
if (manualUserId !== null) {
|
||||
const pinned = mediaItems.find((m) => m.userId === manualUserId);
|
||||
if (pinned) return pinned;
|
||||
}
|
||||
return auto;
|
||||
}),
|
||||
const spotlightSpeaker$ = combineLatest(
|
||||
[autoSpotlightSpeaker$, pinned$],
|
||||
(auto, pinned) => pinned ?? auto,
|
||||
);
|
||||
|
||||
return screenShares$.pipe(
|
||||
@@ -72,16 +151,19 @@ export function overrideSpotlight$(
|
||||
// shared screen (the whole point of "focus camera during screenshare").
|
||||
// With no manual pin this is unchanged: the screenshare alone is
|
||||
// spotlighted.
|
||||
return combineLatest([manualSpotlightUserId$, userMedia$]).pipe(
|
||||
map(([manualUserId, mediaItems]) => {
|
||||
const pinned =
|
||||
manualUserId !== null
|
||||
? mediaItems.find((m) => m.userId === manualUserId)
|
||||
: undefined;
|
||||
return pinned
|
||||
? { spotlight: [...screenShares, pinned], pip$: of(undefined) }
|
||||
: { spotlight: screenShares, pip$: spotlightSpeaker$ };
|
||||
}),
|
||||
return pinned$.pipe(
|
||||
map((pinned) =>
|
||||
pinned
|
||||
? {
|
||||
spotlight: [...screenShares, pinned],
|
||||
// [lotus #29] Keep the active-speaker PiP; it is only
|
||||
// redundant when the speaker is the pinned participant.
|
||||
pip$: autoSpotlightSpeaker$.pipe(
|
||||
map((auto) => (auto === pinned ? undefined : auto)),
|
||||
),
|
||||
}
|
||||
: { spotlight: screenShares, pip$: spotlightSpeaker$ },
|
||||
),
|
||||
);
|
||||
|
||||
return spotlightSpeaker$.pipe(
|
||||
@@ -96,3 +178,84 @@ export function overrideSpotlight$(
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
export interface ManualSpotlightController {
|
||||
/** Pin `target`, or clear with `null`. */
|
||||
setManualSpotlight: (target: ManualSpotlight | null) => void;
|
||||
/**
|
||||
* Side effects that must run for the lifetime of the call. Emits nothing;
|
||||
* the caller subscribes it through its scope.
|
||||
*/
|
||||
effects$: Observable<never>;
|
||||
}
|
||||
|
||||
/**
|
||||
* [lotus #3/#16] State + side effects behind `CallViewModel.setManualSpotlight`.
|
||||
*
|
||||
* - #3: a pin only feeds `spotlight$`, which the default grid layout ignores
|
||||
* (and 1:1 layouts bypass entirely), so pinning forces the layout switch to
|
||||
* "spotlight" and remembers the mode it displaced; clearing the pin restores
|
||||
* that mode if the user has not switched layouts themselves meanwhile.
|
||||
* - #16: the pin is dropped when the pinned participant has been absent from
|
||||
* `userMedia$` for {@link MANUAL_SPOTLIGHT_GONE_MS} (a transient reconnect
|
||||
* is shorter than that), and on `leave$`.
|
||||
*
|
||||
* Pure with respect to its inputs so it can be unit-tested with fake
|
||||
* observables; CallViewModel only wires it up.
|
||||
*
|
||||
* @param manualSpotlight$ the pin state, owned by the caller (it has to exist
|
||||
* before the layout switch does, because `spotlightAndPip$` feeds into it)
|
||||
*/
|
||||
export function createManualSpotlight(
|
||||
manualSpotlight$: BehaviorSubject<ManualSpotlight | null>,
|
||||
userMedia$: Observable<{ userId: string }[]>,
|
||||
leave$: Observable<unknown>,
|
||||
layout: {
|
||||
layout$: Behavior<LayoutMode>;
|
||||
setLayout: (mode: LayoutMode) => void;
|
||||
},
|
||||
goneMs = MANUAL_SPOTLIGHT_GONE_MS,
|
||||
): ManualSpotlightController {
|
||||
// The layout mode we displaced by forcing spotlight, if any.
|
||||
let displacedLayout: LayoutMode | null = null;
|
||||
|
||||
const setManualSpotlight = (target: ManualSpotlight | null): void => {
|
||||
const hadPin = manualSpotlight$.value !== null;
|
||||
manualSpotlight$.next(target);
|
||||
if (target !== null) {
|
||||
if (!hadPin && layout.layout$.value !== "spotlight") {
|
||||
displacedLayout = layout.layout$.value;
|
||||
layout.setLayout("spotlight");
|
||||
}
|
||||
} else if (displacedLayout !== null) {
|
||||
// Only restore if the user hasn't picked a layout themselves since.
|
||||
if (layout.layout$.value === "spotlight")
|
||||
layout.setLayout(displacedLayout);
|
||||
displacedLayout = null;
|
||||
}
|
||||
};
|
||||
|
||||
// #16: pinned user has left for good (absent for `goneMs`), or we hung up.
|
||||
const pinnedGone$ = manualSpotlight$.pipe(
|
||||
switchMap((target) =>
|
||||
target === null
|
||||
? of(false)
|
||||
: userMedia$.pipe(
|
||||
map((items) => !items.some((m) => m.userId === target.userId)),
|
||||
distinctUntilChanged(),
|
||||
// A transient absence shorter than goneMs collapses back to
|
||||
// `false` before the timer fires and is ignored.
|
||||
debounceTime(goneMs),
|
||||
filter((gone) => gone),
|
||||
),
|
||||
),
|
||||
filter((gone) => gone),
|
||||
);
|
||||
const effects$ = merge(pinnedGone$, leave$).pipe(
|
||||
filter(() => manualSpotlight$.value !== null),
|
||||
tap(() => setManualSpotlight(null)),
|
||||
ignoreElements(),
|
||||
);
|
||||
|
||||
return { setManualSpotlight, effects$ };
|
||||
}
|
||||
|
||||
@@ -0,0 +1,85 @@
|
||||
/*
|
||||
Copyright 2026 Lotus Guild
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { afterEach, describe, expect, test, vi } from "vitest";
|
||||
|
||||
vi.mock("../widget", () => ({ widget: null }));
|
||||
|
||||
import { lotusFlag, lotusParam, lotusSendToHost } from "./lotusWidget";
|
||||
import { LotusWidgetActions } from "./lotusActions";
|
||||
|
||||
/**
|
||||
* Point `window.location` at a URL for the duration of one assertion. Uses a
|
||||
* path relative to the current origin — `pushState` throws a `SecurityError`
|
||||
* on a cross-origin URL, and jsdom's default test origin doesn't match a
|
||||
* hard-coded `http://localhost/`.
|
||||
*/
|
||||
function setLocation(path: string): void {
|
||||
window.history.pushState({}, "", path);
|
||||
}
|
||||
|
||||
describe("lotusParam / lotusFlag", () => {
|
||||
afterEach(() => {
|
||||
setLocation("/");
|
||||
});
|
||||
|
||||
test("reads a param from the query string", () => {
|
||||
setLocation("/?lotusTheme=1");
|
||||
expect(lotusParam("lotusTheme")).toBe("1");
|
||||
});
|
||||
|
||||
test("reads a param from the hash fragment's query portion", () => {
|
||||
setLocation("/#/room?lotusTransparent=1");
|
||||
expect(lotusParam("lotusTransparent")).toBe("1");
|
||||
});
|
||||
|
||||
test("the hash fragment wins over the query string for the same key", () => {
|
||||
setLocation("/?lotusTheme=fromQuery#/room?lotusTheme=fromHash");
|
||||
expect(lotusParam("lotusTheme")).toBe("fromHash");
|
||||
});
|
||||
|
||||
test("falls back to the query string for keys absent from the hash", () => {
|
||||
setLocation("/?onlyInQuery=1#/room?lotusTheme=1");
|
||||
expect(lotusParam("onlyInQuery")).toBe("1");
|
||||
});
|
||||
|
||||
test("returns null when the param is present nowhere", () => {
|
||||
setLocation("/#/room");
|
||||
expect(lotusParam("missing")).toBeNull();
|
||||
});
|
||||
|
||||
test.each([
|
||||
["1", true],
|
||||
["true", true],
|
||||
["0", false],
|
||||
["false", false],
|
||||
[null, false],
|
||||
])("lotusFlag treats %s as %s", (value, expected) => {
|
||||
setLocation(value === null ? "/" : `/?f=${value}`);
|
||||
expect(lotusFlag("f")).toBe(expected);
|
||||
});
|
||||
|
||||
// [lotus #33] lotusParam must re-derive from window.location on every call
|
||||
// rather than caching the first read, so a later navigation (e.g. lobby ->
|
||||
// in-call) that rewrites the URL is picked up.
|
||||
test("re-reads window.location on every call instead of caching the first result", () => {
|
||||
setLocation("/?lotusTheme=1");
|
||||
expect(lotusParam("lotusTheme")).toBe("1");
|
||||
|
||||
setLocation("/?lotusTheme=0");
|
||||
expect(lotusParam("lotusTheme")).toBe("0");
|
||||
|
||||
setLocation("/");
|
||||
expect(lotusParam("lotusTheme")).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe("lotusSendToHost", () => {
|
||||
test("returns false when no widget transport is available", () => {
|
||||
expect(lotusSendToHost(LotusWidgetActions.CallState, {})).toBe(false);
|
||||
});
|
||||
});
|
||||
+13
-14
@@ -21,27 +21,26 @@ import type { LotusWidgetActions } from "./lotusActions";
|
||||
|
||||
export { LotusWidgetActions } from "./lotusActions";
|
||||
|
||||
let cachedParams: URLSearchParams | undefined;
|
||||
|
||||
/**
|
||||
* Read a URL param from either the query string or the hash fragment (Element
|
||||
* Call passes widget params via both depending on host), without depending on
|
||||
* EC's own `getUrlParams` parser (keeps the rebase surface small).
|
||||
*
|
||||
* [lotus #33] Re-parsed from `window.location` on every call instead of being
|
||||
* cached at first use: parsing is cheap, and caching risked returning a
|
||||
* stale value if `location.hash`/`search` are ever rewritten after first
|
||||
* read (e.g. during in-iframe navigation).
|
||||
*/
|
||||
export function lotusParam(name: string): string | null {
|
||||
if (!cachedParams) {
|
||||
// Match EC's own ParamParser precedence: the hash fragment wins over the
|
||||
// query string. So seed from the fragment first, then fill gaps from query.
|
||||
const hash = window.location.hash.replace(/^#\/?/, "");
|
||||
const hashQuery = hash.includes("?")
|
||||
? hash.slice(hash.indexOf("?") + 1)
|
||||
: "";
|
||||
cachedParams = new URLSearchParams(hashQuery);
|
||||
for (const [k, v] of new URLSearchParams(window.location.search)) {
|
||||
if (!cachedParams.has(k)) cachedParams.append(k, v);
|
||||
}
|
||||
// Match EC's own ParamParser precedence: the hash fragment wins over the
|
||||
// query string. So seed from the fragment first, then fill gaps from query.
|
||||
const hash = window.location.hash.replace(/^#\/?/, "");
|
||||
const hashQuery = hash.includes("?") ? hash.slice(hash.indexOf("?") + 1) : "";
|
||||
const params = new URLSearchParams(hashQuery);
|
||||
for (const [k, v] of new URLSearchParams(window.location.search)) {
|
||||
if (!params.has(k)) params.append(k, v);
|
||||
}
|
||||
return cachedParams.get(name);
|
||||
return params.get(name);
|
||||
}
|
||||
|
||||
/** Whether a boolean-ish Lotus feature flag is enabled. */
|
||||
|
||||
@@ -73,6 +73,7 @@ import { matrixRTCMode as matrixRTCModeSetting } from "../settings/settings";
|
||||
import { ReactionsReader } from "../reactions/ReactionsReader";
|
||||
import { LivekitRoomAudioRenderer } from "../livekit/MatrixAudioRenderer.tsx";
|
||||
import { muteAllAudio$ } from "../state/MuteAllAudioModel.ts";
|
||||
import { muteScreenshareAudio$ } from "../lotus/lotusScreenshareAudio";
|
||||
import { useMediaDevices } from "../MediaDevicesContext.ts";
|
||||
import { EarpieceOverlay } from "./EarpieceOverlay.tsx";
|
||||
import {
|
||||
@@ -261,6 +262,8 @@ export const InCallView: FC<InCallViewProps> = ({
|
||||
const { showControls, header: headerStyle } = useUrlParams();
|
||||
|
||||
const muteAllAudio = useBehavior(muteAllAudio$);
|
||||
// [lotus] host-driven "Mute Screenshare Audio" (io.lotus.set_deafen).
|
||||
const muteScreenshareAudio = useBehavior(muteScreenshareAudio$);
|
||||
const toggleAudio = useBehavior(muteStates.audio.toggle$);
|
||||
const toggleVideo = useBehavior(muteStates.video.toggle$);
|
||||
const setAudioEnabled = useBehavior(muteStates.audio.setEnabled$);
|
||||
@@ -309,7 +312,7 @@ export const InCallView: FC<InCallViewProps> = ({
|
||||
// [lotus] Handle the host's io.lotus.set_deafen action to silence remote
|
||||
// audio (and optionally screenshare audio) at the LiveKit source. No-op
|
||||
// unless the host sends the action.
|
||||
useEffect(() => startLotusDeafen(vm), [vm]);
|
||||
useEffect(() => startLotusDeafen(), []);
|
||||
|
||||
const fatalCallError = useBehavior(vm.fatalError$);
|
||||
// Stop the rendering and throw for the error boundary
|
||||
@@ -657,6 +660,7 @@ export const InCallView: FC<InCallViewProps> = ({
|
||||
livekitRoom={livekitRoom}
|
||||
validIdentities={participants}
|
||||
muted={muteAllAudio}
|
||||
screenshareAudioMuted={muteScreenshareAudio}
|
||||
/>
|
||||
))}
|
||||
{renderContent()}
|
||||
|
||||
@@ -158,7 +158,11 @@ import { type LocalUserMediaViewModel } from "../media/LocalUserMediaViewModel.t
|
||||
import { type RemoteUserMediaViewModel } from "../media/RemoteUserMediaViewModel.ts";
|
||||
// [lotus #4] Manual spotlight override, extracted so this file stays byte-close
|
||||
// to upstream (see the file for the behaviour contract).
|
||||
import { overrideSpotlight$ } from "../../lotus/lotusSpotlight";
|
||||
import {
|
||||
createManualSpotlight,
|
||||
type ManualSpotlight,
|
||||
overrideSpotlight$,
|
||||
} from "../../lotus/lotusSpotlight";
|
||||
import {
|
||||
createRingingMedia,
|
||||
type RingingMediaViewModel,
|
||||
@@ -267,10 +271,11 @@ export interface CallViewModel {
|
||||
*/
|
||||
toggleScreenSharing: (() => void) | null;
|
||||
/**
|
||||
* [lotus] Pin a participant to the spotlight by Matrix user id (#4
|
||||
* focus-participant). Pass null to clear and restore speaker-follows.
|
||||
* [lotus] Pin a participant to the spotlight (#4 focus-participant) by
|
||||
* Matrix user id and, optionally, EC media id (#30). Pass null to clear and
|
||||
* restore speaker-follows.
|
||||
*/
|
||||
setManualSpotlight: (userId: string | null) => void;
|
||||
setManualSpotlight: (target: ManualSpotlight | null) => void;
|
||||
/**
|
||||
* Whether we are sharing our screen.
|
||||
*/
|
||||
@@ -955,11 +960,6 @@ export function createCallViewModel$(
|
||||
),
|
||||
);
|
||||
|
||||
// [lotus #4] Host-pinned spotlight target (io.lotus.focus_participant). null =
|
||||
// follow the active speaker (upstream default), so this is inert unless the
|
||||
// host pins someone. Consumed by overrideSpotlight$ in spotlightAndPip$.
|
||||
const manualSpotlightUserId$ = new BehaviorSubject<string | null>(null);
|
||||
|
||||
const grid$ = scope.behavior<UserMediaViewModel[]>(
|
||||
userMedia$.pipe(
|
||||
switchMap((mediaItems) => {
|
||||
@@ -1000,6 +1000,12 @@ export function createCallViewModel$(
|
||||
),
|
||||
);
|
||||
|
||||
// [lotus #4] Host-pinned spotlight target (io.lotus.focus_participant). null =
|
||||
// follow the active speaker (upstream default), so this is inert unless the
|
||||
// host pins someone. Consumed by overrideSpotlight$ in spotlightAndPip$;
|
||||
// driven by createManualSpotlight below (after the layout switch exists).
|
||||
const manualSpotlight$ = new BehaviorSubject<ManualSpotlight | null>(null);
|
||||
|
||||
const spotlightAndPip$ = scope.behavior<{
|
||||
spotlight: MediaViewModel[];
|
||||
pip$: Observable<UserMediaViewModel | undefined>;
|
||||
@@ -1016,7 +1022,7 @@ export function createCallViewModel$(
|
||||
// stays close to upstream and rebases cleanly.
|
||||
return overrideSpotlight$(
|
||||
spotlightSpeaker$,
|
||||
manualSpotlightUserId$,
|
||||
manualSpotlight$,
|
||||
screenShares$,
|
||||
localUserMediaForPip$,
|
||||
userMedia$,
|
||||
@@ -1088,12 +1094,31 @@ export function createCallViewModel$(
|
||||
hasRemoteScreenShares$,
|
||||
);
|
||||
|
||||
// [lotus #3/#16] Pinning forces the layout switch to "spotlight" (grid and
|
||||
// 1:1 layouts would otherwise never render the pin) and the pin is dropped
|
||||
// when the participant leaves for good or we hang up; see
|
||||
// src/lotus/lotusSpotlight.ts.
|
||||
const manualSpotlight = createManualSpotlight(
|
||||
manualSpotlight$,
|
||||
userMedia$,
|
||||
leave$,
|
||||
layoutSwitchVm,
|
||||
);
|
||||
manualSpotlight.effects$.pipe(scope.bind()).subscribe();
|
||||
|
||||
const gridLayoutMedia$: Observable<GridLayoutMedia> = combineLatest(
|
||||
[grid$, spotlight$],
|
||||
(grid, spotlight) => ({
|
||||
[grid$, spotlight$, manualSpotlight$],
|
||||
(grid, spotlight, manual) => ({
|
||||
type: "grid",
|
||||
edgeToEdge: false,
|
||||
spotlight: spotlight.some((vm) => vm.type === "screen share")
|
||||
// [lotus #3] Also surface the spotlight when it holds a host-pinned
|
||||
// participant, so the pin is visible in window modes that ignore the
|
||||
// layout switch (narrow). Unchanged when manual is null.
|
||||
spotlight: spotlight.some(
|
||||
(vm) =>
|
||||
vm.type === "screen share" ||
|
||||
(manual !== null && vm.userId === manual.userId),
|
||||
)
|
||||
? spotlight
|
||||
: undefined,
|
||||
grid,
|
||||
@@ -1796,8 +1821,7 @@ export function createCallViewModel$(
|
||||
join: localMembership.requestJoinAndPublish,
|
||||
leave: localMembership.requestDisconnect,
|
||||
toggleScreenSharing: toggleScreenSharing,
|
||||
setManualSpotlight: (userId: string | null): void =>
|
||||
manualSpotlightUserId$.next(userId),
|
||||
setManualSpotlight: manualSpotlight.setManualSpotlight,
|
||||
sharingScreen$: sharingScreen$,
|
||||
|
||||
tapScreen: (): void => screenTap$.next(),
|
||||
|
||||
@@ -183,6 +183,13 @@ function generateRoomOption({
|
||||
audioCaptureDefaults: {
|
||||
...liveKitOptions.audioCaptureDefaults,
|
||||
deviceId: devices.audioInput.selected$.value?.id,
|
||||
// [lotus] Voice is mono. Without this, Firefox captures a 2-channel
|
||||
// audio interface (e.g. a Scarlett Solo with one XLR mic on input 1) as
|
||||
// stereo and, with browser audio processing off, publishes it that way —
|
||||
// peers hear the speaker in the left ear only. Chrome downmixes such
|
||||
// captures itself, which is why it only showed on Firefox. Screenshare
|
||||
// audio is captured separately and is unaffected.
|
||||
channelCount: 1,
|
||||
echoCancellation:
|
||||
echoCancellationSetting.getValue() &&
|
||||
getUrlParams().echoCancellation !== false,
|
||||
|
||||
@@ -120,8 +120,11 @@ Please see LICENSE in the repository root for full details.
|
||||
opacity: 50%;
|
||||
}
|
||||
|
||||
/* [lotus #6] Profile decoration overlaid on the tile avatar. Shares the
|
||||
avatar's centred box and size so frame-style decorations sit around it. */
|
||||
/* [lotus #4] Profile decoration overlaid on the tile avatar. Sized larger than
|
||||
the avatar's box (62cqmin vs. the avatar's 50cqmin — matching cinny's own
|
||||
~50px avatar + 8px outward inset ratio, `AvatarDecoration.tsx`'s
|
||||
DEFAULT_INSET) so frame-style decoration artwork bleeds outside the avatar
|
||||
circle and surrounds it, instead of overlapping/clipping it 1:1. */
|
||||
.lotusDecoration {
|
||||
position: absolute;
|
||||
top: 50%;
|
||||
@@ -131,10 +134,19 @@ avatar's centred box and size so frame-style decorations sit around it. */
|
||||
object-fit: contain;
|
||||
}
|
||||
|
||||
/* [lotus #19] Decorations are animated APNGs with no static asset to freeze
|
||||
to; hide them under prefers-reduced-motion, matching cinny's own
|
||||
AvatarDecoration guard and the host's behaviour of rendering just the avatar. */
|
||||
@media (prefers-reduced-motion: reduce) {
|
||||
.lotusDecoration {
|
||||
display: none;
|
||||
}
|
||||
}
|
||||
|
||||
@container mediaView (width > 0) {
|
||||
.lotusDecoration {
|
||||
inline-size: 50cqmin;
|
||||
block-size: 50cqmin;
|
||||
inline-size: 62cqmin;
|
||||
block-size: 62cqmin;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -113,6 +113,15 @@ export const MediaView: FC<Props> = ({
|
||||
}) => {
|
||||
const { t } = useTranslation();
|
||||
const decoration = useLotusDecoration(userId);
|
||||
// [lotus #4] Track the URL of a decoration that failed to load (e.g. a 404
|
||||
// from the CDN, reachable given the unvalidated slug the host builds it
|
||||
// from) so we can hide the broken-image box instead of leaving it over the
|
||||
// avatar. Comparing against the current `decoration` (rather than a bare
|
||||
// boolean) means a new/changed decoration URL automatically gets a fresh
|
||||
// chance to load.
|
||||
const [erroredDecoration, setErroredDecoration] = useState<
|
||||
string | undefined
|
||||
>(undefined);
|
||||
const [handRaiseTimerVisible] = useSetting(showHandRaisedTimer);
|
||||
const [showConnectionStats] = useSetting(showConnectionStatsSetting);
|
||||
|
||||
@@ -188,15 +197,19 @@ export const MediaView: FC<Props> = ({
|
||||
style={{ display: video && videoEnabled ? "none" : "initial" }}
|
||||
/>
|
||||
{decoration &&
|
||||
decoration !== erroredDecoration &&
|
||||
!(video && videoEnabled) && (
|
||||
// [lotus #6] Profile decoration overlay, shown only when the avatar
|
||||
// is visible (i.e. not when live video is showing). Pushed by the
|
||||
// host via io.lotus.decorations; undefined unless opted in.
|
||||
// [lotus #4] Hidden entirely under prefers-reduced-motion (CSS) and
|
||||
// on a load error (onError), matching cinny's own guards.
|
||||
<img
|
||||
className={styles.lotusDecoration}
|
||||
src={decoration}
|
||||
alt=""
|
||||
aria-hidden
|
||||
onError={() => setErroredDecoration(decoration)}
|
||||
/>
|
||||
)}
|
||||
{video?.publication !== undefined && (
|
||||
|
||||
+22
-4
@@ -6,7 +6,7 @@ Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { logger } from "matrix-js-sdk/lib/logger";
|
||||
import { useState, useEffect } from "react";
|
||||
import { useState, useEffect, useRef } from "react";
|
||||
import { useObservableEagerState } from "observable-hooks";
|
||||
|
||||
import {
|
||||
@@ -172,13 +172,22 @@ export function useAudioContext<S extends string>(
|
||||
useMediaDevices().audioOutput.selected$,
|
||||
)?.id;
|
||||
const { controlledAudioDevices } = useUrlParams();
|
||||
// [lotus] Tracks whether audioOutputId has ever resolved to a real device,
|
||||
// so we can tell "MediaDevices hasn't resolved yet" (skip) apart from "the
|
||||
// previously selected output device just disappeared" (revert to default,
|
||||
// #27) — both present as audioOutputId === undefined.
|
||||
const outputHasResolved = useRef(false);
|
||||
|
||||
// Update the sink ID whenever we change devices.
|
||||
useEffect(() => {
|
||||
if (typeof audioOutputId === "string") outputHasResolved.current = true;
|
||||
if (
|
||||
!audioContext ||
|
||||
!("setSinkId" in audioContext) ||
|
||||
controlledAudioDevices
|
||||
)
|
||||
return;
|
||||
if (
|
||||
audioContext &&
|
||||
"setSinkId" in audioContext &&
|
||||
!controlledAudioDevices &&
|
||||
// Skip until a device is actually selected. audioOutputId is undefined
|
||||
// before MediaDevices resolves (e.g. on the Tauri desktop webview, where
|
||||
// the selected$ observable emits undefined first); setSinkId(undefined)
|
||||
@@ -191,6 +200,15 @@ export function useAudioContext<S extends string>(
|
||||
audioContext.setSinkId(audioOutputId).catch((ex) => {
|
||||
logger.warn("Unable to change sink for audio context", ex);
|
||||
});
|
||||
} else if (outputHasResolved.current) {
|
||||
// [lotus] The previously selected output device disappeared (selected$
|
||||
// re-emits undefined once devices have already resolved), not the
|
||||
// pre-resolution case above. Revert to the default sink instead of
|
||||
// leaving the context pinned to a now-nonexistent device (#27).
|
||||
// @ts-expect-error - setSinkId doesn't exist yet in types, maybe because it's not supported everywhere.
|
||||
audioContext.setSinkId("").catch((ex) => {
|
||||
logger.warn("Unable to revert sink for audio context", ex);
|
||||
});
|
||||
}
|
||||
}, [audioContext, audioOutputId, controlledAudioDevices]);
|
||||
const { pan: earpiecePan, volume: earpieceVolume } = useEarpieceAudioConfig();
|
||||
|
||||
+18
-3
@@ -8,6 +8,7 @@ Please see LICENSE in the repository root for full details.
|
||||
import { useEffect, useLayoutEffect, useRef, useState } from "react";
|
||||
import { WidgetApiToWidgetAction } from "matrix-widget-api";
|
||||
import { type IThemeChangeActionRequest } from "matrix-widget-api";
|
||||
import { logger } from "matrix-js-sdk/lib/logger";
|
||||
|
||||
import { getUrlParams } from "./UrlParams";
|
||||
import { widget } from "./widget";
|
||||
@@ -60,8 +61,22 @@ export const useTheme = (): void => {
|
||||
document.body.classList.remove("no-theme");
|
||||
// [lotus #5] Native theming hooks, opted in by the host via URL flags, so
|
||||
// it no longer has to inject CSS into the iframe after load.
|
||||
if (lotusFlag("lotusTransparent"))
|
||||
document.body.classList.add("lotus-transparent");
|
||||
if (lotusFlag("lotusTheme")) document.body.classList.add("lotus-theme");
|
||||
const lotusTransparent = lotusFlag("lotusTransparent");
|
||||
const lotusTheme = lotusFlag("lotusTheme");
|
||||
if (lotusTransparent) document.body.classList.add("lotus-transparent");
|
||||
// [lotus #21] The two flags are only meaningful together: a transparent
|
||||
// canvas with no matching theme leaves text/icon colours computed against
|
||||
// the default dark canvas token while the real pixels behind them are
|
||||
// whatever the host painted. If a host sets lotusTransparent without
|
||||
// lotusTheme, warn and apply lotusTheme anyway rather than silently
|
||||
// risking unreadable UI.
|
||||
if (lotusTransparent && !lotusTheme) {
|
||||
logger.warn(
|
||||
"[lotus] lotusTransparent is set without lotusTheme; applying " +
|
||||
"lotusTheme anyway since the two flags are only meaningful together.",
|
||||
);
|
||||
}
|
||||
if (lotusTheme || lotusTransparent)
|
||||
document.body.classList.add("lotus-theme");
|
||||
}, [previousTheme, requestedTheme]);
|
||||
};
|
||||
|
||||
@@ -23,6 +23,8 @@ export enum ErrorCode {
|
||||
E2EE_NOT_SUPPORTED = "E2EE_NOT_SUPPORTED",
|
||||
STICKY_EVENTS_NOT_SUPPORTED = "STICKY_EVENTS_NOT_SUPPORTED",
|
||||
OPEN_ID_ERROR = "OPEN_ID_ERROR",
|
||||
/** [lotus] The SFU token service refused us with a human-readable reason (e.g. the voice-limit guard: channel full / no permission). */
|
||||
SFU_TOKEN_REFUSED = "SFU_TOKEN_REFUSED",
|
||||
NO_MATRIX_2_AUTHORIZATION_SERVICE = "NO_MATRIX_2_0_AUTHORIZATION_SERVICE",
|
||||
SFU_ERROR = "SFU_ERROR",
|
||||
UNKNOWN_ERROR = "UNKNOWN_ERROR",
|
||||
@@ -234,6 +236,24 @@ export class FailToStartLivekitConnection extends ElementCallError {
|
||||
/**
|
||||
* Error indicating that a LiveKit's server has hit its track limits.
|
||||
*/
|
||||
/**
|
||||
* [lotus] The SFU token service answered 403 with a reason we can show verbatim
|
||||
* — the voice-limit guard says things like "This voice channel is full." or
|
||||
* "You don't have permission to share your screen here." Without this the user
|
||||
* only ever saw "Something went wrong (OPEN_ID_ERROR)".
|
||||
*/
|
||||
export class SFUTokenRefusedError extends ElementCallError {
|
||||
public constructor(reason: string, cause?: Error) {
|
||||
super(
|
||||
t("error.sfu_token_refused"),
|
||||
ErrorCode.SFU_TOKEN_REFUSED,
|
||||
ErrorCategory.CONFIGURATION_ISSUE,
|
||||
reason,
|
||||
cause,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
export class InsufficientCapacityError extends ElementCallError {
|
||||
public constructor() {
|
||||
super(
|
||||
|
||||
+18
-2
@@ -115,14 +115,30 @@ export const initializeWidget = (
|
||||
ElementWidgetActions.JoinCall,
|
||||
ElementWidgetActions.HangupCall,
|
||||
ElementWidgetActions.DeviceMute,
|
||||
// [lotus] custom toWidget actions handled by the fork (focus, audio-inject)
|
||||
...LOTUS_TO_WIDGET_ACTIONS,
|
||||
].forEach((action) => {
|
||||
api.on(`action:${action}`, (ev: CustomEvent<IWidgetApiRequest>) => {
|
||||
ev.preventDefault();
|
||||
lazyActions.emit(action, ev);
|
||||
});
|
||||
});
|
||||
// [lotus] custom toWidget actions handled by the fork (focus,
|
||||
// audio-inject, decorations, ...). Unlike the upstream actions above
|
||||
// (whose handlers are process-lifetime), lotus handlers are
|
||||
// registered/torn down with their owning React effect, so a request can
|
||||
// arrive while none is mounted. `LazyEventEmitter.emit` would otherwise
|
||||
// backlog it forever (never replied to, replayed stale on the next
|
||||
// registration — see LazyEventEmitter), and the host's
|
||||
// `transport.send` would hang until its own timeout. Reply immediately
|
||||
// with `{}` when there's no handler instead, so a torn-down lotus
|
||||
// action behaves like a no-op rather than a silent stall.
|
||||
LOTUS_TO_WIDGET_ACTIONS.forEach((action) => {
|
||||
api.on(`action:${action}`, (ev: CustomEvent<IWidgetApiRequest>) => {
|
||||
ev.preventDefault();
|
||||
if (!lazyActions.emit(action, ev)) {
|
||||
api.transport.reply(ev.detail, {});
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
// Now, initialize the matryoshka MatrixClient (so named because it routes
|
||||
// all requests through the host client via the widget API)
|
||||
|
||||
Reference in New Issue
Block a user