Compare commits
11
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
bc0e5ed432 | ||
|
|
778712409e | ||
|
|
e9a59336c7 | ||
|
|
c7494e68ca | ||
|
|
68eafcb9a8 | ||
|
|
dcba5b6b7e | ||
|
|
1c1394b6ef | ||
|
|
501e3fb5ac | ||
|
|
bb639bb92d | ||
|
|
e5d5f13923 | ||
|
|
e504a31efd |
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@lotusguild/element-call-embedded",
|
||||
"version": "0.25.0-lotus.1",
|
||||
"version": "0.25.0-lotus.3",
|
||||
"files": [
|
||||
"README.md",
|
||||
"LICENSE-AGPL-3.0",
|
||||
|
||||
@@ -46,8 +46,12 @@ unconditionally select the container so we can use cq units */
|
||||
|
||||
/* On a landscape phone the fixed 180px filmstrip rail squeezes the spotlight to
|
||||
a sliver; shrink the rail (keeping the 4:3 tile ratio) to give the spotlight
|
||||
room. Desktop landscape (height > 400px) is unaffected. */
|
||||
@media (max-height: 400px) {
|
||||
room. Desktop landscape (height > 400px) is unaffected.
|
||||
[lotus #32] `max-height: 400px` alone also matches a short-but-wide desktop
|
||||
or PiP window that has nothing to do with a phone, so require a coarse
|
||||
(touch) pointer too — a resized desktop/PiP window kept its mouse pointer,
|
||||
so it won't match this and keeps the full-size rail. */
|
||||
@media (max-height: 400px) and (pointer: coarse) {
|
||||
.layer {
|
||||
--grid-slot-width: 132px;
|
||||
}
|
||||
|
||||
@@ -100,6 +100,26 @@ body.lotus-theme {
|
||||
--video-tile-background: var(--cpd-color-bg-subtle-secondary);
|
||||
}
|
||||
|
||||
/* [lotus #21] Subtle contrast guard for elements that sit directly on the
|
||||
transparent canvas (no opaque tile background behind them) when
|
||||
lotusTransparent is set: the host's real wallpaper is unknown to us, so a
|
||||
light text-shadow/backdrop keeps name tags and header/footer controls legible
|
||||
regardless of what's behind them. Kept behind `.lotus-transparent` so upstream
|
||||
(non-Lotus) layouts are completely untouched. Hooked off stable data-testid/
|
||||
element selectors rather than CSS-module class names, since those are hashed
|
||||
per-build and owned by their own component stylesheets. */
|
||||
body.lotus-transparent [data-testid="name_tag"],
|
||||
body.lotus-transparent header,
|
||||
body.lotus-transparent [data-testid="footer-container"] {
|
||||
text-shadow: 0 1px 3px rgba(0, 0, 0, 0.6);
|
||||
}
|
||||
|
||||
body.lotus-transparent header,
|
||||
body.lotus-transparent [data-testid="footer-container"] {
|
||||
backdrop-filter: blur(8px);
|
||||
-webkit-backdrop-filter: blur(8px);
|
||||
}
|
||||
|
||||
@media (min-height: 330px) {
|
||||
body[data-background="gradient"]::before {
|
||||
content: "";
|
||||
|
||||
@@ -0,0 +1,47 @@
|
||||
/*
|
||||
Copyright 2026 Lotus Guild
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { describe, expect, test } from "vitest";
|
||||
|
||||
import { LOTUS_TO_WIDGET_ACTIONS, LotusWidgetActions } from "./lotusActions";
|
||||
|
||||
describe("LotusWidgetActions", () => {
|
||||
test("every action value is namespaced under io.lotus.*", () => {
|
||||
for (const value of Object.values(LotusWidgetActions)) {
|
||||
expect(value).toMatch(/^io\.lotus\./);
|
||||
}
|
||||
});
|
||||
|
||||
test("LOTUS_TO_WIDGET_ACTIONS contains exactly the toWidget actions", () => {
|
||||
// CallState is the only fromWidget action (host <- widget); everything
|
||||
// else is toWidget (host -> widget) and must be allow-listed so
|
||||
// `initializeWidget` accepts it.
|
||||
const expectedToWidget = [
|
||||
LotusWidgetActions.FocusParticipant,
|
||||
LotusWidgetActions.InjectAudio,
|
||||
LotusWidgetActions.SetQuality,
|
||||
LotusWidgetActions.Decorations,
|
||||
LotusWidgetActions.SetDeafen,
|
||||
];
|
||||
|
||||
expect(new Set(LOTUS_TO_WIDGET_ACTIONS)).toEqual(new Set(expectedToWidget));
|
||||
expect(LOTUS_TO_WIDGET_ACTIONS).toHaveLength(expectedToWidget.length);
|
||||
});
|
||||
|
||||
test("LOTUS_TO_WIDGET_ACTIONS excludes the fromWidget actions", () => {
|
||||
// CallState, RequestState and DenoiseState are all fromWidget (widget ->
|
||||
// host) per their doc comments in lotusActions.ts, so none of them should
|
||||
// ever be allow-listed as a toWidget action.
|
||||
expect(LOTUS_TO_WIDGET_ACTIONS).not.toContain(LotusWidgetActions.CallState);
|
||||
expect(LOTUS_TO_WIDGET_ACTIONS).not.toContain(
|
||||
LotusWidgetActions.RequestState,
|
||||
);
|
||||
expect(LOTUS_TO_WIDGET_ACTIONS).not.toContain(
|
||||
LotusWidgetActions.DenoiseState,
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -31,6 +31,21 @@ export enum LotusWidgetActions {
|
||||
Decorations = "io.lotus.decorations",
|
||||
/** toWidget: deafen remote audio (and optionally mute screenshare audio). */
|
||||
SetDeafen = "io.lotus.set_deafen",
|
||||
/**
|
||||
* fromWidget: sent on (re)registration of a lotus toWidget handler (e.g.
|
||||
* decorations) so the host can re-push state that would otherwise only be
|
||||
* sent on change (decorations roster, the current focus pin) after an
|
||||
* EC-side reconnect/remount. cinny's `resendForkState()` should respond to
|
||||
* this the same way it responds to a fresh join.
|
||||
*/
|
||||
RequestState = "io.lotus.request_state",
|
||||
/**
|
||||
* fromWidget: real state of the in-source denoise engine —
|
||||
* `{ active: boolean, model: string, error?: string }` — sent once the
|
||||
* processor attaches (or after the rnnoise fallback also fails), so the host
|
||||
* toggle can reflect reality rather than the requested state.
|
||||
*/
|
||||
DenoiseState = "io.lotus.denoise_state",
|
||||
}
|
||||
|
||||
/** toWidget Lotus actions that must be allow-listed in `initializeWidget`. */
|
||||
|
||||
@@ -88,6 +88,16 @@ afterEach(() => {
|
||||
});
|
||||
|
||||
describe("[lotus] audio-capture URL param overrides", () => {
|
||||
test("mic capture is always requested mono (stereo interfaces on Firefox published L-only)", () => {
|
||||
getUrlParams.mockReturnValue({
|
||||
echoCancellation: false,
|
||||
noiseSuppression: false,
|
||||
autoGainControl: false,
|
||||
});
|
||||
createRoom();
|
||||
expect(capturedAudioDefaults()).toMatchObject({ channelCount: 1 });
|
||||
});
|
||||
|
||||
test("with params defaulted to true, the Settings govern (upstream behaviour)", () => {
|
||||
getUrlParams.mockReturnValue({
|
||||
echoCancellation: true,
|
||||
|
||||
@@ -0,0 +1,225 @@
|
||||
/*
|
||||
Copyright 2026 Lotus Guild
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { EventEmitter } from "events";
|
||||
import { afterEach, beforeEach, expect, test, vi } from "vitest";
|
||||
import { of } from "rxjs";
|
||||
|
||||
import { type CallViewModel } from "../state/CallViewModel/CallViewModel";
|
||||
import { startLotusAudioInject } from "./lotusAudioInject";
|
||||
import { LotusWidgetActions } from "./lotusActions";
|
||||
|
||||
const lazyActions = new EventEmitter();
|
||||
const reply = vi.fn();
|
||||
|
||||
vi.mock("../widget", () => ({
|
||||
widget: {
|
||||
api: { transport: { reply: (...args: unknown[]) => reply(...args) } },
|
||||
// Getter: `vi.mock` factories run at import time, before the const above.
|
||||
get lazyActions(): EventEmitter {
|
||||
return lazyActions;
|
||||
},
|
||||
},
|
||||
}));
|
||||
|
||||
function send(data: unknown): void {
|
||||
lazyActions.emit(LotusWidgetActions.InjectAudio, { detail: { data } });
|
||||
}
|
||||
|
||||
/** Flush the microtask queue enough times to drain the async awaits in
|
||||
* `playInjectedClip` (fetch -> arrayBuffer -> resume -> decodeAudioData ->
|
||||
* publishTrack...), none of which use real timers in this test. */
|
||||
async function flush(times = 30): Promise<void> {
|
||||
for (let i = 0; i < times; i++) await Promise.resolve();
|
||||
}
|
||||
|
||||
function makeTrack(): MediaStreamTrack {
|
||||
return {
|
||||
clone: vi.fn(() => makeTrack()),
|
||||
stop: vi.fn(),
|
||||
} as unknown as MediaStreamTrack;
|
||||
}
|
||||
|
||||
function makeRoom(isMicrophoneEnabled: boolean): {
|
||||
localParticipant: {
|
||||
isMicrophoneEnabled: boolean;
|
||||
publishTrack: ReturnType<typeof vi.fn>;
|
||||
unpublishTrack: ReturnType<typeof vi.fn>;
|
||||
};
|
||||
} {
|
||||
return {
|
||||
localParticipant: {
|
||||
isMicrophoneEnabled,
|
||||
publishTrack: vi.fn(async (clone: unknown) => {
|
||||
await Promise.resolve();
|
||||
return { track: clone };
|
||||
}),
|
||||
unpublishTrack: vi.fn().mockResolvedValue(undefined),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function mockVm(rooms: unknown[]): CallViewModel {
|
||||
return {
|
||||
allConnections$: of({
|
||||
getConnections: () => rooms.map((livekitRoom) => ({ livekitRoom })),
|
||||
}),
|
||||
} as unknown as CallViewModel;
|
||||
}
|
||||
|
||||
/** Fake AudioContext/nodes good enough to drive playInjectedClip end to end,
|
||||
* tracking how many contexts and destinations get constructed (#14). */
|
||||
let contextInstances: FakeAudioContext[];
|
||||
class FakeAudioContext {
|
||||
public state = "running";
|
||||
public resume = vi.fn().mockResolvedValue(undefined);
|
||||
public close = vi.fn().mockResolvedValue(undefined);
|
||||
public decodeAudioData = vi
|
||||
.fn()
|
||||
.mockResolvedValue({ duration: 0.01 } as AudioBuffer);
|
||||
public createMediaStreamDestination = vi.fn(() => ({
|
||||
stream: { getAudioTracks: (): MediaStreamTrack[] => [makeTrack()] },
|
||||
}));
|
||||
public createGain = vi.fn(() => ({
|
||||
gain: { value: 0 },
|
||||
connect: vi.fn((n: unknown) => n),
|
||||
disconnect: vi.fn(),
|
||||
}));
|
||||
public createBufferSource = vi.fn(() => ({
|
||||
buffer: undefined as AudioBuffer | undefined,
|
||||
connect: vi.fn((n: unknown) => n),
|
||||
disconnect: vi.fn(),
|
||||
start: vi.fn(),
|
||||
stop: vi.fn(),
|
||||
onended: null as (() => void) | null,
|
||||
addEventListener: vi.fn(),
|
||||
}));
|
||||
public constructor() {
|
||||
contextInstances.push(this);
|
||||
}
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
contextInstances = [];
|
||||
reply.mockClear();
|
||||
// The module sets a real `setTimeout` guard per clip (MAX_CLIP_MS safety
|
||||
// net); use fake timers so a test ending before that guard fires doesn't
|
||||
// leave a real timer pending.
|
||||
vi.useFakeTimers();
|
||||
vi.stubGlobal("AudioContext", FakeAudioContext);
|
||||
vi.stubGlobal(
|
||||
"fetch",
|
||||
vi.fn().mockResolvedValue({
|
||||
ok: true,
|
||||
arrayBuffer: async () => {
|
||||
await Promise.resolve();
|
||||
return new ArrayBuffer(8);
|
||||
},
|
||||
}),
|
||||
);
|
||||
// `lotusParam`/`lotusFlag` re-parse `window.location` on every call, so
|
||||
// just set the flag before each test.
|
||||
window.location.hash = "#/room?lotusAudioInject=1";
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
lazyActions.removeAllListeners();
|
||||
vi.unstubAllGlobals();
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
test("#13: a muted local mic blocks the clip and replies with a machine-readable reason", async () => {
|
||||
const room = makeRoom(false);
|
||||
const vm = mockVm([room]);
|
||||
const stop = startLotusAudioInject(vm);
|
||||
|
||||
send({ url: "https://example.com/clip.mp3" });
|
||||
await flush();
|
||||
|
||||
expect(reply).toHaveBeenCalledWith(expect.anything(), {
|
||||
played: false,
|
||||
reason: "muted",
|
||||
});
|
||||
expect(room.localParticipant.publishTrack).not.toHaveBeenCalled();
|
||||
// No context should even be created for a clip that never plays.
|
||||
expect(contextInstances).toHaveLength(0);
|
||||
|
||||
stop();
|
||||
});
|
||||
|
||||
test("#13: an unmuted local mic plays the clip normally", async () => {
|
||||
const room = makeRoom(true);
|
||||
const vm = mockVm([room]);
|
||||
const stop = startLotusAudioInject(vm);
|
||||
|
||||
send({ url: "https://example.com/clip.mp3" });
|
||||
await flush();
|
||||
|
||||
expect(reply).toHaveBeenCalledWith(expect.anything(), {});
|
||||
expect(room.localParticipant.publishTrack).toHaveBeenCalledTimes(1);
|
||||
|
||||
stop();
|
||||
});
|
||||
|
||||
test("#14: one shared AudioContext/destination is reused across clips, and closed only on the last teardown", async () => {
|
||||
const room = makeRoom(true);
|
||||
const vm = mockVm([room]);
|
||||
const stop = startLotusAudioInject(vm);
|
||||
|
||||
send({ url: "https://example.com/a.mp3" });
|
||||
await flush();
|
||||
expect(contextInstances).toHaveLength(1);
|
||||
const ctx = contextInstances[0];
|
||||
expect(ctx.createMediaStreamDestination).toHaveBeenCalledTimes(1);
|
||||
|
||||
// Finish the first clip (as if it played to completion) before starting a
|
||||
// second one, matching the module's own "one clip at a time" contract.
|
||||
const firstSource = ctx.createBufferSource.mock.results[0]!.value as {
|
||||
onended: (() => void) | null;
|
||||
};
|
||||
firstSource.onended?.();
|
||||
await flush();
|
||||
expect(room.localParticipant.unpublishTrack).toHaveBeenCalledTimes(1);
|
||||
|
||||
// A second clip must reuse the SAME context/destination rather than
|
||||
// creating a new one.
|
||||
send({ url: "https://example.com/b.mp3" });
|
||||
await flush();
|
||||
expect(contextInstances).toHaveLength(1);
|
||||
expect(ctx.createMediaStreamDestination).toHaveBeenCalledTimes(1);
|
||||
expect(ctx.close).not.toHaveBeenCalled();
|
||||
|
||||
const secondSource = ctx.createBufferSource.mock.results[1]!.value as {
|
||||
onended: (() => void) | null;
|
||||
};
|
||||
secondSource.onended?.();
|
||||
await flush();
|
||||
|
||||
// Tearing down the (only) active instance closes the shared context.
|
||||
stop();
|
||||
expect(ctx.close).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
test("#14: the shared context stays open while another instance is still active", async () => {
|
||||
const roomA = makeRoom(true);
|
||||
const roomB = makeRoom(true);
|
||||
const stopA = startLotusAudioInject(mockVm([roomA]));
|
||||
const stopB = startLotusAudioInject(mockVm([roomB]));
|
||||
|
||||
send({ url: "https://example.com/a.mp3" });
|
||||
await flush();
|
||||
expect(contextInstances).toHaveLength(1);
|
||||
const ctx = contextInstances[0];
|
||||
|
||||
// Tearing down the first (of two) active instances must not close the
|
||||
// context out from under the other one.
|
||||
stopA();
|
||||
expect(ctx.close).not.toHaveBeenCalled();
|
||||
|
||||
stopB();
|
||||
expect(ctx.close).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
@@ -17,6 +17,30 @@ import { lotusFlag } from "./lotusWidget";
|
||||
/** Hard cap so a malformed/huge clip can't hold a published track open forever. */
|
||||
const MAX_CLIP_MS = 30_000;
|
||||
|
||||
// [lotus] One shared module-level AudioContext + MediaStreamAudioDestinationNode
|
||||
// for ALL injected clips (#14), instead of `new AudioContext()` per clip. An
|
||||
// in-call page already sits close to Chrome's per-document AudioContext limit
|
||||
// (~6) between useAudioContext, MatrixAudioRenderer, LiveKit's own Room
|
||||
// context and LotusDenoiseProcessor; rapid clip replacement (replace-mode
|
||||
// closing the previous clip's context in the background) could transiently
|
||||
// exceed the cap and make `new AudioContext()` throw. Lazily created on first
|
||||
// use, ref-counted by the number of active `startLotusAudioInject` instances,
|
||||
// and closed only when the last one tears down.
|
||||
let sharedCtx: AudioContext | undefined;
|
||||
let sharedDest: MediaStreamAudioDestinationNode | undefined;
|
||||
let handlerCount = 0;
|
||||
|
||||
function acquireSharedAudio(): {
|
||||
ctx: AudioContext;
|
||||
dest: MediaStreamAudioDestinationNode;
|
||||
} {
|
||||
if (!sharedCtx || sharedCtx.state === "closed") {
|
||||
sharedCtx = new AudioContext();
|
||||
sharedDest = sharedCtx.createMediaStreamDestination();
|
||||
}
|
||||
return { ctx: sharedCtx, dest: sharedDest! };
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle the host's `io.lotus.inject_audio` toWidget action (#3): mix a
|
||||
* soundboard clip into the call so other participants hear it.
|
||||
@@ -38,6 +62,10 @@ export function startLotusAudioInject(vm: CallViewModel): () => void {
|
||||
const w = widget;
|
||||
if (!w) return () => undefined;
|
||||
|
||||
// [lotus] Count this instance toward the shared AudioContext's lifetime
|
||||
// (#14) — closed only once the last active instance tears down.
|
||||
handlerCount++;
|
||||
|
||||
// Track the set of connected LiveKit rooms to publish into. Drive off the
|
||||
// LOCAL participant's connection(s), not `livekitRoomItems$` — that stream
|
||||
// omits rooms with no remote members, so inject would no-op while you're
|
||||
@@ -52,16 +80,19 @@ export function startLotusAudioInject(vm: CallViewModel): () => void {
|
||||
const activeClips = new Set<() => void>();
|
||||
|
||||
const handler = (ev: CustomEvent<IWidgetApiRequest>): void => {
|
||||
// Always ack so the transport doesn't hang, but only act when the host has
|
||||
// explicitly opted in: audio-inject publishes under the local user's
|
||||
// identity, so it must not be silently armed for every call.
|
||||
w.api.transport.reply(ev.detail, {});
|
||||
if (!lotusFlag("lotusAudioInject")) return;
|
||||
if (!lotusFlag("lotusAudioInject")) {
|
||||
// Always ack so the transport doesn't hang, but only act when the host
|
||||
// has explicitly opted in: audio-inject publishes under the local
|
||||
// user's identity, so it must not be silently armed for every call.
|
||||
w.api.transport.reply(ev.detail, {});
|
||||
return;
|
||||
}
|
||||
const data = ev.detail.data as
|
||||
| { url?: unknown; volume?: unknown }
|
||||
| undefined;
|
||||
const url = typeof data?.url === "string" ? safeMediaUrl(data.url) : null;
|
||||
if (!url) {
|
||||
w.api.transport.reply(ev.detail, {});
|
||||
logger.warn("[lotus] inject_audio: missing/invalid url");
|
||||
return;
|
||||
}
|
||||
@@ -69,6 +100,21 @@ export function startLotusAudioInject(vm: CallViewModel): () => void {
|
||||
typeof data?.volume === "number" && data.volume >= 0 && data.volume <= 1
|
||||
? data.volume
|
||||
: 1;
|
||||
|
||||
// [lotus] Gate on the local mic being enabled (#13): the clip is
|
||||
// published as an independent track, fully decoupled from the mic
|
||||
// publication's mute state, so without this a muted (or push-to-talk
|
||||
// idle) user could still transmit soundboard audio under their own
|
||||
// identity — breaking the "I am muted, nothing I do makes noise" mental
|
||||
// model. Reply with a machine-readable reason so cinny's soundboard UI
|
||||
// can surface a hint instead of the click silently doing nothing.
|
||||
const micEnabled = rooms[0]?.localParticipant.isMicrophoneEnabled ?? true;
|
||||
if (!micEnabled) {
|
||||
w.api.transport.reply(ev.detail, { played: false, reason: "muted" });
|
||||
return;
|
||||
}
|
||||
|
||||
w.api.transport.reply(ev.detail, {});
|
||||
void playInjectedClip(url, volume, rooms, activeClips).catch((e) =>
|
||||
logger.warn("[lotus] inject_audio failed", e),
|
||||
);
|
||||
@@ -86,6 +132,15 @@ export function startLotusAudioInject(vm: CallViewModel): () => void {
|
||||
// activeClips while we iterate it.
|
||||
// eslint-disable-next-line unicorn/no-useless-spread
|
||||
for (const abort of [...activeClips]) abort();
|
||||
// [lotus] Close the shared AudioContext only when the last active
|
||||
// instance tears down (#14).
|
||||
handlerCount--;
|
||||
if (handlerCount === 0 && sharedCtx) {
|
||||
const ctx = sharedCtx;
|
||||
sharedCtx = undefined;
|
||||
sharedDest = undefined;
|
||||
void ctx.close().catch(() => undefined);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
@@ -146,23 +201,25 @@ async function playInjectedClip(
|
||||
throw e;
|
||||
}
|
||||
if (aborted) return;
|
||||
if (!resp.ok) throw new Error(`fetch ${url} -> ${resp.status}`);
|
||||
if (!resp.ok) {
|
||||
activeClips.delete(placeholder);
|
||||
throw new Error(`fetch ${url} -> ${resp.status}`);
|
||||
}
|
||||
const arrayBuffer = await resp.arrayBuffer();
|
||||
if (aborted) return;
|
||||
|
||||
const ctx = new AudioContext();
|
||||
// [lotus] Reuse the shared module-level context/destination (#14) rather
|
||||
// than `new AudioContext()` per clip — see the declaration above.
|
||||
const { ctx, dest } = acquireSharedAudio();
|
||||
// The action arrives via host postMessage, not a gesture in this iframe, so
|
||||
// the context may start suspended — resume it or the clip is silent and
|
||||
// `onended` never fires.
|
||||
// `onended` never fires. A no-op if an earlier clip already resumed it.
|
||||
try {
|
||||
await ctx.resume();
|
||||
} catch {
|
||||
/* best effort */
|
||||
}
|
||||
if (aborted) {
|
||||
void ctx.close();
|
||||
return;
|
||||
}
|
||||
if (aborted) return;
|
||||
if (ctx.state !== "running")
|
||||
logger.warn(`[lotus] inject_audio: AudioContext is ${ctx.state}`);
|
||||
|
||||
@@ -170,15 +227,13 @@ async function playInjectedClip(
|
||||
try {
|
||||
buffer = await ctx.decodeAudioData(arrayBuffer);
|
||||
} catch (e) {
|
||||
void ctx.close();
|
||||
activeClips.delete(placeholder);
|
||||
throw e;
|
||||
}
|
||||
if (aborted) {
|
||||
void ctx.close();
|
||||
return;
|
||||
}
|
||||
if (aborted) return;
|
||||
|
||||
const dest = ctx.createMediaStreamDestination();
|
||||
// Per clip, only the BufferSource/GainNode are created (#14) — the shared
|
||||
// context/destination are reused across every clip.
|
||||
const gain = ctx.createGain();
|
||||
gain.gain.value = volume;
|
||||
const source = ctx.createBufferSource();
|
||||
@@ -187,7 +242,9 @@ async function playInjectedClip(
|
||||
|
||||
const mst = dest.stream.getAudioTracks()[0];
|
||||
if (!mst) {
|
||||
void ctx.close();
|
||||
source.disconnect();
|
||||
gain.disconnect();
|
||||
activeClips.delete(placeholder);
|
||||
throw new Error("no audio track from destination");
|
||||
}
|
||||
|
||||
@@ -221,13 +278,17 @@ async function playInjectedClip(
|
||||
} catch {
|
||||
/* already stopped */
|
||||
}
|
||||
// [lotus] Dispose only this clip's own nodes (#14) — the shared
|
||||
// AudioContext/destination outlive it and are closed separately, only
|
||||
// when the last startLotusAudioInject instance tears down.
|
||||
source.disconnect();
|
||||
gain.disconnect();
|
||||
for (const entry of publications) {
|
||||
if (entry?.pub.track)
|
||||
void entry.room.localParticipant
|
||||
.unpublishTrack(entry.pub.track, true)
|
||||
.catch(() => undefined);
|
||||
}
|
||||
void ctx.close().catch(() => undefined);
|
||||
};
|
||||
// Swap the synchronous placeholder for the real cleanup: from here an abort
|
||||
// (teardown or a newer clip) must unpublish the LIVE track, not just cancel a
|
||||
|
||||
@@ -0,0 +1,128 @@
|
||||
/*
|
||||
Copyright 2026 Lotus Guild
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { afterEach, beforeEach, expect, test, vi } from "vitest";
|
||||
import { BehaviorSubject, of } from "rxjs";
|
||||
|
||||
import { type CallViewModel } from "../state/CallViewModel/CallViewModel";
|
||||
import { startLotusCallState } from "./lotusCallState";
|
||||
|
||||
const send = vi.fn().mockResolvedValue({});
|
||||
|
||||
vi.mock("../widget", () => ({
|
||||
widget: { api: { transport: { send: (...a: unknown[]) => send(...a) } } },
|
||||
}));
|
||||
|
||||
// `lotusFlag`/`lotusParam` (lotusWidget.ts) read `window.location` directly
|
||||
// and memoize per module load, so drive the flag through the URL once, at
|
||||
// import time (NOT per-test/in a hook: re-navigating with
|
||||
// `window.history.pushState` between tests was observed to corrupt rxjs's
|
||||
// shared `asyncScheduler` under `vi.useFakeTimers()`, silently starving a
|
||||
// later test's `throttleTime` of any emission).
|
||||
window.history.pushState({}, "", "/?lotusCallState=1");
|
||||
|
||||
beforeEach(() => {
|
||||
send.mockClear();
|
||||
vi.useFakeTimers();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
interface Member {
|
||||
id: string;
|
||||
userId: string;
|
||||
speaking$: BehaviorSubject<boolean>;
|
||||
audioEnabled$: BehaviorSubject<boolean>;
|
||||
videoEnabled$: BehaviorSubject<boolean>;
|
||||
}
|
||||
|
||||
function mockMember(id: string, userId: string): Member {
|
||||
return {
|
||||
id,
|
||||
userId,
|
||||
speaking$: new BehaviorSubject(false),
|
||||
audioEnabled$: new BehaviorSubject(true),
|
||||
videoEnabled$: new BehaviorSubject(true),
|
||||
};
|
||||
}
|
||||
|
||||
function mockVm(members: Member[]): CallViewModel {
|
||||
return { userMedia$: of(members) } as unknown as CallViewModel;
|
||||
}
|
||||
|
||||
function participantsOf(call: number): unknown[] {
|
||||
return (send.mock.calls[call][1] as { participants: unknown[] }).participants;
|
||||
}
|
||||
|
||||
test("[lotus #20] rapid speaking toggles in a busy call are capped well under the old 8/sec rate", () => {
|
||||
const members = Array.from({ length: 15 }, (_, i) =>
|
||||
mockMember(`@u${i}:example.org:DEV`, `@u${i}:example.org`),
|
||||
);
|
||||
const stop = startLotusCallState(mockVm(members));
|
||||
|
||||
// Simulate a noisy multi-person conversation: flip `speaking` on every
|
||||
// member every 20ms (50Hz of raw churn) for 2 seconds.
|
||||
for (let t = 0; t < 2000; t += 20) {
|
||||
for (const m of members) m.speaking$.next(!m.speaking$.value);
|
||||
vi.advanceTimersByTime(20);
|
||||
}
|
||||
|
||||
// The old leading+trailing 250ms throttle allowed ~8 sends/sec => up to 16
|
||||
// over 2s. Trailing-only 500ms must cap this to at most 4 (one per window).
|
||||
expect(send.mock.calls.length).toBeLessThanOrEqual(5);
|
||||
expect(send.mock.calls.length).toBeGreaterThan(0);
|
||||
|
||||
// Drain any still-pending trailing-edge throttle action before tearing
|
||||
// down: `rxjs`'s default `asyncScheduler` is a process-wide singleton, and
|
||||
// leaving a scheduled action dangling across a test/timer-implementation
|
||||
// boundary can wedge its queue for every later test in this file.
|
||||
vi.advanceTimersByTime(500);
|
||||
stop();
|
||||
});
|
||||
|
||||
test("[lotus #20] a mute/camera change and a same-tick no-op speaking flip are deduped field-wise", () => {
|
||||
const alice = mockMember("@alice:example.org:DEV", "@alice:example.org");
|
||||
const stop = startLotusCallState(mockVm([alice]));
|
||||
|
||||
vi.advanceTimersByTime(500);
|
||||
send.mockClear();
|
||||
|
||||
// No actual change: re-emitting the same speaking value must not count as
|
||||
// a change (distinctUntilChanged happens before the throttle).
|
||||
alice.speaking$.next(false);
|
||||
vi.advanceTimersByTime(500);
|
||||
expect(send).not.toHaveBeenCalled();
|
||||
|
||||
alice.audioEnabled$.next(false);
|
||||
vi.advanceTimersByTime(500);
|
||||
expect(send).toHaveBeenCalledTimes(1);
|
||||
expect(participantsOf(0)).toEqual([
|
||||
{
|
||||
id: "@alice:example.org:DEV",
|
||||
userId: "@alice:example.org",
|
||||
speaking: false,
|
||||
audioEnabled: false,
|
||||
videoEnabled: true,
|
||||
},
|
||||
]);
|
||||
|
||||
stop();
|
||||
});
|
||||
|
||||
test("does nothing (and sends nothing) once torn down", () => {
|
||||
const alice = mockMember("@alice:example.org:DEV", "@alice:example.org");
|
||||
const stop = startLotusCallState(mockVm([alice]));
|
||||
vi.advanceTimersByTime(500);
|
||||
send.mockClear();
|
||||
|
||||
stop();
|
||||
alice.speaking$.next(true);
|
||||
vi.advanceTimersByTime(1000);
|
||||
expect(send).not.toHaveBeenCalled();
|
||||
});
|
||||
@@ -27,6 +27,29 @@ interface ParticipantState {
|
||||
videoEnabled: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* [lotus #20] Field-wise equality for `ParticipantState[]`, used in place of
|
||||
* `JSON.stringify` comparison: cheaper (no serialisation of every
|
||||
* participant on every emission) and just as correct, since array order here
|
||||
* is stable (it mirrors `members` from `userMedia$`).
|
||||
*/
|
||||
function participantsEqual(
|
||||
a: ParticipantState[],
|
||||
b: ParticipantState[],
|
||||
): boolean {
|
||||
return (
|
||||
a.length === b.length &&
|
||||
a.every(
|
||||
(p, i) =>
|
||||
p.id === b[i].id &&
|
||||
p.userId === b[i].userId &&
|
||||
p.speaking === b[i].speaking &&
|
||||
p.audioEnabled === b[i].audioEnabled &&
|
||||
p.videoEnabled === b[i].videoEnabled,
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Stream per-participant speaking / mute / camera state to the Lotus host
|
||||
* (cinny) over the widget API, so the host can drive speaking rings, mute
|
||||
@@ -73,11 +96,19 @@ export function startLotusCallState(vm: CallViewModel): () => void {
|
||||
),
|
||||
),
|
||||
// `speaking` flips rapidly; drop no-op repeats BEFORE throttling so
|
||||
// the throttle window isn't spent re-emitting an unchanged value, then
|
||||
// cap the send rate. 250ms is plenty for speaking rings / mute badges
|
||||
// and keeps the request/response widget traffic modest.
|
||||
distinctUntilChanged((a, b) => JSON.stringify(a) === JSON.stringify(b)),
|
||||
throttleTime(250, undefined, { leading: true, trailing: true }),
|
||||
// the throttle window isn't spent re-emitting an unchanged value.
|
||||
// Field-wise (cheaper than JSON.stringify, and correct: array order is
|
||||
// stable since it mirrors `members` from userMedia$).
|
||||
distinctUntilChanged(participantsEqual),
|
||||
// [lotus #20] `speaking` is the field that flips constantly in an active
|
||||
// conversation; mute/camera toggles are rare and user-intentional and
|
||||
// would ideally stay prompt, but a single combined stream is far
|
||||
// simpler than splitting it, and the leading+trailing 250ms window
|
||||
// previously allowed ~8 sends/sec (each re-serialising every
|
||||
// participant) in a busy call. Trailing-only + a longer window caps
|
||||
// that to 2/sec while still reflecting mute/camera changes within
|
||||
// 500ms.
|
||||
throttleTime(500, undefined, { leading: false, trailing: true }),
|
||||
)
|
||||
.subscribe((participants) => {
|
||||
lotusSendToHost(LotusWidgetActions.CallState, { participants });
|
||||
|
||||
@@ -0,0 +1,127 @@
|
||||
/*
|
||||
Copyright 2026 Lotus Guild
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { EventEmitter } from "events";
|
||||
import { afterEach, beforeEach, expect, test, vi } from "vitest";
|
||||
import { act, renderHook } from "@testing-library/react";
|
||||
|
||||
import {
|
||||
ALLOWED_DECORATION_ORIGINS,
|
||||
startLotusDecorations,
|
||||
useLotusDecoration,
|
||||
} from "./lotusDecorations";
|
||||
import { LotusWidgetActions } from "./lotusActions";
|
||||
|
||||
const lazyActions = new EventEmitter();
|
||||
const reply = vi.fn();
|
||||
const send = vi.fn().mockResolvedValue({});
|
||||
|
||||
vi.mock("../widget", () => ({
|
||||
widget: {
|
||||
api: {
|
||||
transport: {
|
||||
reply: (...a: unknown[]) => reply(...a),
|
||||
send: (...a: unknown[]) => send(...a),
|
||||
},
|
||||
},
|
||||
// Getter: `vi.mock` factories run at import time, before the const above.
|
||||
get lazyActions(): EventEmitter {
|
||||
return lazyActions;
|
||||
},
|
||||
},
|
||||
}));
|
||||
|
||||
function pushDecorations(decorations: Record<string, unknown>): void {
|
||||
// `useSyncExternalStore`'s re-render from the module-level `emit()` needs to
|
||||
// be flushed inside `act()`, since the emitter fires outside of React's own
|
||||
// event handling.
|
||||
act(() => {
|
||||
lazyActions.emit(LotusWidgetActions.Decorations, {
|
||||
detail: { data: { decorations } },
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
reply.mockClear();
|
||||
send.mockClear();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
lazyActions.removeAllListeners();
|
||||
});
|
||||
|
||||
test("[lotus #28] safeImageUrl only accepts the pinned decoration CDN origin (or blob:)", () => {
|
||||
const stop = startLotusDecorations();
|
||||
const { result } = renderHook(() => useLotusDecoration("@alice:example.org"));
|
||||
|
||||
pushDecorations({
|
||||
"@alice:example.org": `${ALLOWED_DECORATION_ORIGINS[0]}/fox_hat.png`,
|
||||
});
|
||||
expect(result.current).toBe(`${ALLOWED_DECORATION_ORIGINS[0]}/fox_hat.png`);
|
||||
|
||||
// A different https host is rejected outright, even though it was
|
||||
// previously allowed by the "any https" check.
|
||||
pushDecorations({ "@alice:example.org": "https://evil.example/x.png" });
|
||||
expect(result.current).toBeUndefined();
|
||||
|
||||
// Path traversal off the allowed origin is still on-origin, so the origin
|
||||
// check alone doesn't stop it (validation of the slug itself is the host's
|
||||
// job) — but a completely different scheme/host must never get through.
|
||||
pushDecorations({
|
||||
"@alice:example.org": "javascript:alert(1)",
|
||||
});
|
||||
expect(result.current).toBeUndefined();
|
||||
|
||||
pushDecorations({ "@alice:example.org": "blob:https://example.org/abc" });
|
||||
expect(result.current).toBe("blob:https://example.org/abc");
|
||||
|
||||
stop();
|
||||
});
|
||||
|
||||
test("[lotus #17] the roster survives a handler remount within the same page session", () => {
|
||||
const stop1 = startLotusDecorations();
|
||||
pushDecorations({
|
||||
"@alice:example.org": `${ALLOWED_DECORATION_ORIGINS[0]}/fox_hat.png`,
|
||||
});
|
||||
|
||||
const { result } = renderHook(() => useLotusDecoration("@alice:example.org"));
|
||||
expect(result.current).toBe(`${ALLOWED_DECORATION_ORIGINS[0]}/fox_hat.png`);
|
||||
|
||||
// Tear the handler down (as InCallView would on an EC-side remount) and
|
||||
// bring it back up, WITHOUT the host re-pushing anything.
|
||||
stop1();
|
||||
const stop2 = startLotusDecorations();
|
||||
|
||||
// The roster must still be there — it must not have been wiped to {} by
|
||||
// the teardown.
|
||||
expect(result.current).toBe(`${ALLOWED_DECORATION_ORIGINS[0]}/fox_hat.png`);
|
||||
|
||||
stop2();
|
||||
});
|
||||
|
||||
test("[lotus #17] (re)registering the handler asks the host to re-push state", () => {
|
||||
const stop1 = startLotusDecorations();
|
||||
expect(send).toHaveBeenCalledWith(LotusWidgetActions.RequestState, {});
|
||||
send.mockClear();
|
||||
|
||||
stop1();
|
||||
const stop2 = startLotusDecorations();
|
||||
expect(send).toHaveBeenCalledWith(LotusWidgetActions.RequestState, {});
|
||||
|
||||
stop2();
|
||||
});
|
||||
|
||||
test("a second concurrent registration does not re-request state or double-register", () => {
|
||||
const stop1 = startLotusDecorations();
|
||||
send.mockClear();
|
||||
const stop2 = startLotusDecorations();
|
||||
expect(send).not.toHaveBeenCalled();
|
||||
|
||||
stop2();
|
||||
stop1();
|
||||
});
|
||||
@@ -9,7 +9,7 @@ import { useSyncExternalStore } from "react";
|
||||
import { type IWidgetApiRequest } from "matrix-widget-api";
|
||||
|
||||
import { widget } from "../widget";
|
||||
import { LotusWidgetActions } from "./lotusActions";
|
||||
import { LotusWidgetActions, lotusSendToHost } from "./lotusWidget";
|
||||
|
||||
/**
|
||||
* Avatar decorations (#6 / A6). The Lotus host (cinny) owns the decoration
|
||||
@@ -38,11 +38,26 @@ export function useLotusDecoration(userId: string): string | undefined {
|
||||
return useSyncExternalStore(subscribe, () => decorations[userId]);
|
||||
}
|
||||
|
||||
// [lotus #28] The decoration URL comes from the host, which builds it from an
|
||||
// arbitrary (unvalidated on the wire) profile field. `safeImageUrl` used to
|
||||
// accept ANY https origin, so the only thing stopping a third-party image
|
||||
// beacon on every tile was the host happening to build the URL itself. Pin it
|
||||
// to the actual decoration CDN origin(s) — cinny's `DECORATION_CDN`
|
||||
// (`avatarDecorations.ts`) — plus `blob:`, which EC itself may use for local
|
||||
// previews. Exported so a future CDN move is a one-line edit here.
|
||||
export const ALLOWED_DECORATION_ORIGINS: readonly string[] = [
|
||||
"https://drive.lotusguild.org",
|
||||
];
|
||||
|
||||
function safeImageUrl(raw: unknown): string | null {
|
||||
if (typeof raw !== "string") return null;
|
||||
try {
|
||||
const u = new URL(raw, window.location.href);
|
||||
return u.protocol === "https:" || u.protocol === "blob:" ? u.href : null;
|
||||
if (u.protocol === "blob:") return u.href;
|
||||
return u.protocol === "https:" &&
|
||||
ALLOWED_DECORATION_ORIGINS.includes(u.origin)
|
||||
? u.href
|
||||
: null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
@@ -87,6 +102,13 @@ export function startLotusDecorations(): () => void {
|
||||
unregister = (): void => {
|
||||
w.lazyActions.off(LotusWidgetActions.Decorations, handler);
|
||||
};
|
||||
// [lotus #17] An EC-side remount of InCallView/ActiveCall tears this
|
||||
// handler down and back up while cinny stays joined, and the host only
|
||||
// re-pushes decorations on a CHANGE to its roster — with an unchanged
|
||||
// member list it never re-sends, so the tiles would otherwise lose their
|
||||
// decorations for the rest of the call. Ask the host to re-push whatever
|
||||
// it currently has every time the handler (re)registers.
|
||||
lotusSendToHost(LotusWidgetActions.RequestState, {});
|
||||
}
|
||||
registrations += 1;
|
||||
|
||||
@@ -96,11 +118,15 @@ export function startLotusDecorations(): () => void {
|
||||
registrations = 0;
|
||||
unregister?.();
|
||||
unregister = null;
|
||||
// Reset the roster once the last registration goes away, so a decoration
|
||||
// pushed in call A can't leak onto a shared user in call B before the
|
||||
// host re-pushes. Notify listeners so any still-mounted tile drops the
|
||||
// now-stale overlay via the render path.
|
||||
decorations = {};
|
||||
// [lotus #17] Do NOT wipe the roster here. It used to reset to `{}` so a
|
||||
// decoration from call A couldn't leak onto a same-named user in call B,
|
||||
// but `decorations` is a module-scope singleton for the lifetime of the
|
||||
// PAGE (a fresh call is a fresh page/iframe load, which resets this
|
||||
// module anyway), while an EC-side handler remount within the SAME call
|
||||
// was wiping live decorations that the host has no reason to re-send
|
||||
// (see the request_state ask above, which covers hosts that don't
|
||||
// proactively resend). Listeners are still notified so a torn-down
|
||||
// period doesn't itself change anything visible.
|
||||
emit();
|
||||
}
|
||||
};
|
||||
|
||||
@@ -10,21 +10,40 @@ import { ParticipantEvent, Track } from "livekit-client";
|
||||
|
||||
import { type CallViewModel } from "../state/CallViewModel/CallViewModel";
|
||||
import { startLotusDenoise } from "./lotusDenoise";
|
||||
import { LotusWidgetActions } from "./lotusActions";
|
||||
|
||||
// Track constructed instances so tests can assert exactly one processor was
|
||||
// built across racing apply() calls, and can assert the pending one is
|
||||
// destroyed on early teardown. `vi.hoisted` is required because `vi.mock`
|
||||
// factories are hoisted above this file's other top-level statements.
|
||||
const instances = vi.hoisted(
|
||||
() => [] as { destroy: ReturnType<typeof vi.fn> }[],
|
||||
() =>
|
||||
[] as {
|
||||
config: { model: string };
|
||||
destroy: ReturnType<typeof vi.fn>;
|
||||
setMicMuted: ReturnType<typeof vi.fn>;
|
||||
}[],
|
||||
);
|
||||
// Ordered log of the heavy-lifting calls, to assert prepare runs before
|
||||
// setProcessor (#7).
|
||||
const calls = vi.hoisted(() => [] as string[]);
|
||||
vi.mock("./lotusDenoiseProcessor", () => ({
|
||||
LotusDenoiseProcessor: class {
|
||||
public destroy = vi.fn().mockResolvedValue(undefined);
|
||||
public constructor() {
|
||||
public setMicMuted = vi.fn();
|
||||
public constructor(public config: { model: string }) {
|
||||
instances.push(this);
|
||||
}
|
||||
},
|
||||
prepareDenoiseAssets: vi.fn(async () => {
|
||||
calls.push("prepare");
|
||||
await Promise.resolve();
|
||||
}),
|
||||
releasePreparedDenoiseAssets: vi.fn().mockResolvedValue(undefined),
|
||||
}));
|
||||
const hostSend = vi.hoisted(() => vi.fn().mockResolvedValue({}));
|
||||
vi.mock("../widget", () => ({
|
||||
widget: { api: { transport: { send: hostSend } } },
|
||||
}));
|
||||
|
||||
/** A promise plus externally-callable resolve, for controlling ordering. */
|
||||
@@ -42,19 +61,23 @@ function deferred<T>(): {
|
||||
function makeRoomAndVm(mic: {
|
||||
getProcessor: () => unknown;
|
||||
setProcessor: ReturnType<typeof vi.fn>;
|
||||
isMuted?: boolean;
|
||||
}): {
|
||||
vm: CallViewModel;
|
||||
room: { localParticipant: Record<string, unknown> };
|
||||
firePublished: () => void;
|
||||
fire: (event: string, arg?: unknown) => void;
|
||||
} {
|
||||
const handlers = new Map<string, () => void>();
|
||||
const handlers = new Map<string, (arg?: unknown) => void>();
|
||||
const localParticipant = {
|
||||
getTrackPublication: (
|
||||
source: Track.Source,
|
||||
): { track: typeof mic } | undefined =>
|
||||
source === Track.Source.Microphone ? { track: mic } : undefined,
|
||||
on: (event: string, cb: () => void): Map<string, () => void> =>
|
||||
handlers.set(event, cb),
|
||||
on: (
|
||||
event: string,
|
||||
cb: (arg?: unknown) => void,
|
||||
): Map<string, (arg?: unknown) => void> => handlers.set(event, cb),
|
||||
off: (event: string): boolean => handlers.delete(event),
|
||||
};
|
||||
const room = { localParticipant };
|
||||
@@ -74,11 +97,19 @@ function makeRoomAndVm(mic: {
|
||||
vm,
|
||||
room,
|
||||
firePublished: () => handlers.get(ParticipantEvent.LocalTrackPublished)?.(),
|
||||
fire: (event, arg) => handlers.get(event)?.(arg),
|
||||
};
|
||||
}
|
||||
|
||||
/** Let the async apply() chain (attach → retry → host notify) settle. */
|
||||
async function flush(): Promise<void> {
|
||||
for (let i = 0; i < 8; i++) await Promise.resolve();
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
instances.length = 0;
|
||||
calls.length = 0;
|
||||
hostSend.mockClear();
|
||||
// `lotusParam`/`lotusFlag` cache the URL params at first read; seed the hash
|
||||
// before startLotusDenoise() so the dedicated `lotusDenoiseSource` flag reads on.
|
||||
window.location.hash = "#/room?lotusDenoiseSource=1";
|
||||
@@ -95,6 +126,7 @@ describe("startLotusDenoise", () => {
|
||||
const mic = {
|
||||
getProcessor: (): unknown => attached,
|
||||
setProcessor: vi.fn(async (p: unknown) => {
|
||||
calls.push("setProcessor");
|
||||
await setProcessorDeferred.promise;
|
||||
attached = p;
|
||||
}),
|
||||
@@ -102,6 +134,8 @@ describe("startLotusDenoise", () => {
|
||||
const { vm, firePublished } = makeRoomAndVm(mic);
|
||||
|
||||
startLotusDenoise(vm);
|
||||
// [#7] The heavy assets were kicked off before setProcessor() was called.
|
||||
expect(calls).toEqual(["prepare", "setProcessor"]);
|
||||
// Simulate a second LocalTrackPublished (e.g. camera) firing before the
|
||||
// first setProcessor() has resolved.
|
||||
firePublished();
|
||||
@@ -138,4 +172,103 @@ describe("startLotusDenoise", () => {
|
||||
setProcessorDeferred.resolve();
|
||||
await Promise.resolve();
|
||||
});
|
||||
|
||||
test("reports the active model to the host once attached (#8)", async () => {
|
||||
let attached: unknown;
|
||||
const mic = {
|
||||
getProcessor: (): unknown => attached,
|
||||
setProcessor: vi.fn(async (p: unknown) => {
|
||||
await Promise.resolve();
|
||||
attached = p;
|
||||
}),
|
||||
};
|
||||
const { vm } = makeRoomAndVm(mic);
|
||||
startLotusDenoise(vm);
|
||||
await flush();
|
||||
expect(hostSend).toHaveBeenCalledWith(LotusWidgetActions.DenoiseState, {
|
||||
active: true,
|
||||
model: "rnnoise",
|
||||
});
|
||||
});
|
||||
|
||||
test("retries once with rnnoise when the requested model fails to init (#8)", async () => {
|
||||
window.location.hash =
|
||||
"#/room?lotusDenoiseSource=1&lotusModel=deepfilternet";
|
||||
let attached: unknown;
|
||||
const mic = {
|
||||
getProcessor: (): unknown => attached,
|
||||
setProcessor: vi.fn(async (p: { config: { model: string } }) => {
|
||||
await Promise.resolve();
|
||||
if (p.config.model === "deepfilternet")
|
||||
throw new Error("dfn assets missing");
|
||||
attached = p;
|
||||
}),
|
||||
};
|
||||
const { vm } = makeRoomAndVm(mic);
|
||||
startLotusDenoise(vm);
|
||||
await flush();
|
||||
|
||||
expect(instances.map((i) => i.config.model)).toEqual([
|
||||
"deepfilternet",
|
||||
"rnnoise",
|
||||
]);
|
||||
expect(mic.setProcessor).toHaveBeenCalledTimes(2);
|
||||
expect(hostSend).toHaveBeenCalledTimes(1);
|
||||
expect(hostSend).toHaveBeenCalledWith(LotusWidgetActions.DenoiseState, {
|
||||
active: true,
|
||||
model: "rnnoise",
|
||||
});
|
||||
});
|
||||
|
||||
test("notifies the host with the error when the rnnoise fallback also fails (#8)", async () => {
|
||||
window.location.hash = "#/room?lotusDenoiseSource=1&lotusModel=dtln";
|
||||
const mic = {
|
||||
getProcessor: (): unknown => undefined,
|
||||
setProcessor: vi.fn(async () => {
|
||||
await Promise.resolve();
|
||||
throw new Error("no worklet for you");
|
||||
}),
|
||||
};
|
||||
const { vm } = makeRoomAndVm(mic);
|
||||
startLotusDenoise(vm);
|
||||
await flush();
|
||||
|
||||
expect(instances.map((i) => i.config.model)).toEqual(["dtln", "rnnoise"]);
|
||||
expect(hostSend).toHaveBeenCalledTimes(1);
|
||||
expect(hostSend).toHaveBeenCalledWith(LotusWidgetActions.DenoiseState, {
|
||||
active: false,
|
||||
error: "no worklet for you",
|
||||
model: "rnnoise",
|
||||
});
|
||||
});
|
||||
|
||||
test("mirrors mic TrackMuted/TrackUnmuted onto the processor (#9)", async () => {
|
||||
let attached: unknown;
|
||||
const mic = {
|
||||
isMuted: true,
|
||||
getProcessor: (): unknown => attached,
|
||||
setProcessor: vi.fn(async (p: unknown) => {
|
||||
await Promise.resolve();
|
||||
attached = p;
|
||||
}),
|
||||
};
|
||||
const { vm, fire } = makeRoomAndVm(mic);
|
||||
startLotusDenoise(vm);
|
||||
// Seeded from the mic's current state before setProcessor().
|
||||
expect(instances[0].setMicMuted).toHaveBeenLastCalledWith(true);
|
||||
await flush();
|
||||
|
||||
const micPub = { source: Track.Source.Microphone };
|
||||
const camPub = { source: Track.Source.Camera };
|
||||
fire(ParticipantEvent.TrackUnmuted, micPub);
|
||||
expect(instances[0].setMicMuted).toHaveBeenLastCalledWith(false);
|
||||
fire(ParticipantEvent.TrackMuted, micPub);
|
||||
expect(instances[0].setMicMuted).toHaveBeenLastCalledWith(true);
|
||||
|
||||
// Camera mute must not touch the audio context.
|
||||
instances[0].setMicMuted.mockClear();
|
||||
fire(ParticipantEvent.TrackMuted, camPub);
|
||||
fire(ParticipantEvent.TrackUnmuted, camPub);
|
||||
expect(instances[0].setMicMuted).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
+89
-15
@@ -10,14 +10,23 @@ import {
|
||||
ParticipantEvent,
|
||||
type Room as LivekitRoom,
|
||||
Track,
|
||||
type TrackPublication,
|
||||
} from "livekit-client";
|
||||
import { logger } from "matrix-js-sdk/lib/logger";
|
||||
|
||||
import { type CallViewModel } from "../state/CallViewModel/CallViewModel";
|
||||
import { lotusFlag, lotusParam } from "./lotusWidget";
|
||||
import {
|
||||
LotusWidgetActions,
|
||||
lotusFlag,
|
||||
lotusParam,
|
||||
lotusSendToHost,
|
||||
} from "./lotusWidget";
|
||||
import {
|
||||
type LotusDenoiseConfig,
|
||||
type LotusDenoiseModel,
|
||||
LotusDenoiseProcessor,
|
||||
prepareDenoiseAssets,
|
||||
releasePreparedDenoiseAssets,
|
||||
} from "./lotusDenoiseProcessor";
|
||||
|
||||
/**
|
||||
@@ -93,6 +102,12 @@ export function startLotusDenoise(vm: CallViewModel): () => void {
|
||||
: 0.15,
|
||||
};
|
||||
|
||||
// [lotus #7] Start fetching wasm / creating the AudioContext / addModule-ing
|
||||
// the worklet (and loading the DFN model) NOW, before any mic track exists.
|
||||
// `setProcessor()` holds LiveKit's trackChangeLock while awaiting `init()`,
|
||||
// so anything still loading there freezes mute/unmute/device-switch.
|
||||
void prepareDenoiseAssets(config);
|
||||
|
||||
const micOf = (room: LivekitRoom): LocalAudioTrack | undefined =>
|
||||
room.localParticipant.getTrackPublication(Track.Source.Microphone)
|
||||
?.track as LocalAudioTrack | undefined;
|
||||
@@ -105,23 +120,73 @@ export function startLotusDenoise(vm: CallViewModel): () => void {
|
||||
// AudioContext + model load) before the first has attached. Track an
|
||||
// in-flight setProcessor per room and skip `apply()` while one is pending.
|
||||
const pendingProcessors = new Map<LivekitRoom, LotusDenoiseProcessor>();
|
||||
let stopped = false;
|
||||
|
||||
const attach = async (
|
||||
room: LivekitRoom,
|
||||
mic: LocalAudioTrack,
|
||||
model: LotusDenoiseModel,
|
||||
): Promise<void> => {
|
||||
const processor = new LotusDenoiseProcessor({ ...config, model });
|
||||
pendingProcessors.set(room, processor);
|
||||
// [lotus #9] Seed the mute state so a processor attached while already
|
||||
// muted starts suspended rather than burning inference on silence.
|
||||
processor.setMicMuted(mic.isMuted);
|
||||
try {
|
||||
await mic.setProcessor(processor);
|
||||
} finally {
|
||||
// Only clear if we're still the pending entry (a teardown that ran
|
||||
// while this was in flight may have already replaced/removed it).
|
||||
if (pendingProcessors.get(room) === processor)
|
||||
pendingProcessors.delete(room);
|
||||
}
|
||||
};
|
||||
|
||||
const apply = (room: LivekitRoom): void => {
|
||||
const mic = micOf(room);
|
||||
if (!mic || mic.getProcessor() || pendingProcessors.has(room)) return;
|
||||
const processor = new LotusDenoiseProcessor(config);
|
||||
pendingProcessors.set(room, processor);
|
||||
void mic
|
||||
.setProcessor(processor)
|
||||
.catch((e) => logger.warn("[lotus] denoise setProcessor failed", e))
|
||||
.finally(() => {
|
||||
// Only clear if we're still the pending entry (a teardown that ran
|
||||
// while this was in flight may have already replaced/removed it).
|
||||
if (pendingProcessors.get(room) === processor)
|
||||
pendingProcessors.delete(room);
|
||||
});
|
||||
void (async () => {
|
||||
let model = config.model;
|
||||
try {
|
||||
try {
|
||||
await attach(room, mic, model);
|
||||
} catch (e) {
|
||||
logger.warn("[lotus] denoise setProcessor failed", e);
|
||||
// [lotus #8] A failed init leaves LiveKit with no processor (it only
|
||||
// assigns after init resolves), so retry once with the smallest,
|
||||
// most portable tier before giving up.
|
||||
if (model === "rnnoise" || stopped) throw e;
|
||||
model = "rnnoise";
|
||||
await attach(room, mic, model);
|
||||
}
|
||||
// [lotus #8] Tell the host what is ACTUALLY running so its toggle
|
||||
// reflects reality (including the fallback model).
|
||||
lotusSendToHost(LotusWidgetActions.DenoiseState, {
|
||||
active: true,
|
||||
model,
|
||||
});
|
||||
} catch (e) {
|
||||
logger.warn("[lotus] denoise unavailable; publishing raw mic", e);
|
||||
lotusSendToHost(LotusWidgetActions.DenoiseState, {
|
||||
active: false,
|
||||
error: e instanceof Error ? e.message : String(e),
|
||||
model,
|
||||
});
|
||||
}
|
||||
})();
|
||||
};
|
||||
|
||||
// [lotus #9] Suspend/resume the processor's own AudioContext with the mic's
|
||||
// mute state (LiveKit keeps the muted track flowing silent frames, so the
|
||||
// worklet would otherwise run inference the whole time the user is muted).
|
||||
const onMuteChange =
|
||||
(room: LivekitRoom, muted: boolean) =>
|
||||
(pub: TrackPublication): void => {
|
||||
if (pub.source !== Track.Source.Microphone) return;
|
||||
const p = pendingProcessors.get(room) ?? micOf(room)?.getProcessor();
|
||||
if (p instanceof LotusDenoiseProcessor) p.setMicMuted(muted);
|
||||
};
|
||||
|
||||
const roomListeners = new Map<LivekitRoom, () => void>();
|
||||
let rooms: LivekitRoom[] = [];
|
||||
|
||||
@@ -143,22 +208,29 @@ export function startLotusDenoise(vm: CallViewModel): () => void {
|
||||
// Re-attach on every (re)publish — this is what makes denoise survive
|
||||
// reconnects (A7), unlike the old getUserMedia patch.
|
||||
const onPublished = (): void => apply(room);
|
||||
const onMuted = onMuteChange(room, true);
|
||||
const onUnmuted = onMuteChange(room, false);
|
||||
room.localParticipant.on(
|
||||
ParticipantEvent.LocalTrackPublished,
|
||||
onPublished,
|
||||
);
|
||||
roomListeners.set(room, () =>
|
||||
room.localParticipant.on(ParticipantEvent.TrackMuted, onMuted);
|
||||
room.localParticipant.on(ParticipantEvent.TrackUnmuted, onUnmuted);
|
||||
roomListeners.set(room, () => {
|
||||
room.localParticipant.off(
|
||||
ParticipantEvent.LocalTrackPublished,
|
||||
onPublished,
|
||||
),
|
||||
);
|
||||
);
|
||||
room.localParticipant.off(ParticipantEvent.TrackMuted, onMuted);
|
||||
room.localParticipant.off(ParticipantEvent.TrackUnmuted, onUnmuted);
|
||||
});
|
||||
apply(room);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
return () => {
|
||||
stopped = true;
|
||||
sub.unsubscribe();
|
||||
for (const off of roomListeners.values()) off();
|
||||
roomListeners.clear();
|
||||
@@ -175,5 +247,7 @@ export function startLotusDenoise(vm: CallViewModel): () => void {
|
||||
}
|
||||
}
|
||||
pendingProcessors.clear();
|
||||
// [lotus #7] Close any prepared-but-never-claimed context (e.g. no mic).
|
||||
void releasePreparedDenoiseAssets();
|
||||
};
|
||||
}
|
||||
|
||||
@@ -5,19 +5,97 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { describe, expect, test } from "vitest";
|
||||
import { afterEach, beforeEach, describe, expect, test, vi } from "vitest";
|
||||
import { type AudioProcessorOptions } from "livekit-client";
|
||||
|
||||
import { LotusDenoiseProcessor } from "./lotusDenoiseProcessor";
|
||||
import {
|
||||
assertModuleExport,
|
||||
type LotusDenoiseConfig,
|
||||
LotusDenoiseProcessor,
|
||||
prepareDenoiseAssets,
|
||||
releasePreparedDenoiseAssets,
|
||||
} from "./lotusDenoiseProcessor";
|
||||
|
||||
function makeProcessor(): LotusDenoiseProcessor {
|
||||
return new LotusDenoiseProcessor({
|
||||
model: "rnnoise",
|
||||
assetBase: "https://example.invalid/denoise/",
|
||||
gate: false,
|
||||
gateThreshold: -45,
|
||||
floor: 0.15,
|
||||
const baseConfig: LotusDenoiseConfig = {
|
||||
model: "rnnoise",
|
||||
assetBase: "https://example.invalid/denoise/",
|
||||
gate: false,
|
||||
gateThreshold: -45,
|
||||
floor: 0.15,
|
||||
};
|
||||
|
||||
function makeProcessor(
|
||||
overrides: Partial<LotusDenoiseConfig> = {},
|
||||
): LotusDenoiseProcessor {
|
||||
return new LotusDenoiseProcessor({ ...baseConfig, ...overrides });
|
||||
}
|
||||
|
||||
/**
|
||||
* Minimal fake AudioContext: tracks state, records suspend/resume, and lets a
|
||||
* test fire `statechange` like the browser would after an OS interruption.
|
||||
*/
|
||||
class FakeAudioContext extends EventTarget {
|
||||
public static created: FakeAudioContext[] = [];
|
||||
public state: AudioContextState = "running";
|
||||
public readonly sampleRate: number;
|
||||
public readonly audioWorklet = {
|
||||
addModule: vi.fn().mockResolvedValue(undefined),
|
||||
};
|
||||
public suspend = vi.fn(async (): Promise<void> => {
|
||||
await Promise.resolve();
|
||||
this.state = "suspended";
|
||||
});
|
||||
public resume = vi.fn(async (): Promise<void> => {
|
||||
await Promise.resolve();
|
||||
this.state = "running";
|
||||
});
|
||||
public close = vi.fn(async (): Promise<void> => {
|
||||
await Promise.resolve();
|
||||
this.state = "closed";
|
||||
});
|
||||
public constructor(opts?: { sampleRate?: number }) {
|
||||
super();
|
||||
this.sampleRate = opts?.sampleRate ?? 48_000;
|
||||
FakeAudioContext.created.push(this);
|
||||
}
|
||||
/** Simulate an external (OS/browser) suspension. */
|
||||
public externallySuspend(): void {
|
||||
this.state = "suspended";
|
||||
this.dispatchEvent(new Event("statechange"));
|
||||
}
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
FakeAudioContext.created = [];
|
||||
vi.stubGlobal("AudioContext", FakeAudioContext);
|
||||
// The flat wasm prefetch just warms a cache; give it a 200 so prepare()
|
||||
// succeeds without a network.
|
||||
vi.stubGlobal(
|
||||
"fetch",
|
||||
vi.fn().mockResolvedValue({
|
||||
ok: true,
|
||||
arrayBuffer: (): ArrayBuffer => new ArrayBuffer(8),
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await releasePreparedDenoiseAssets();
|
||||
vi.unstubAllGlobals();
|
||||
});
|
||||
|
||||
/** Skip the real graph (needs worklets) — init() only needs a track back. */
|
||||
function stubGraph(processor: LotusDenoiseProcessor): void {
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
(processor as any).buildGraph = async (): Promise<unknown> => {
|
||||
await Promise.resolve();
|
||||
return {
|
||||
source: { disconnect: vi.fn() },
|
||||
nodes: [],
|
||||
disposes: [],
|
||||
track: { stop: vi.fn() },
|
||||
};
|
||||
};
|
||||
}
|
||||
|
||||
describe("LotusDenoiseProcessor.restart", () => {
|
||||
@@ -53,3 +131,117 @@ describe("LotusDenoiseProcessor.restart", () => {
|
||||
expect(processor.processedTrack).not.toBe(rawTrack);
|
||||
});
|
||||
});
|
||||
|
||||
describe("prepareDenoiseAssets (#7)", () => {
|
||||
test("init() adopts the context/worklet prepared before setProcessor instead of loading inline", async () => {
|
||||
await prepareDenoiseAssets(baseConfig);
|
||||
// The heavy parts ran up front: one context, worklet module added, wasm fetched.
|
||||
expect(FakeAudioContext.created).toHaveLength(1);
|
||||
const prepared = FakeAudioContext.created[0];
|
||||
expect(prepared.audioWorklet.addModule).toHaveBeenCalledWith(
|
||||
`${baseConfig.assetBase}rnnoiseWorklet.js`,
|
||||
);
|
||||
expect(fetch).toHaveBeenCalled();
|
||||
|
||||
const processor = makeProcessor();
|
||||
stubGraph(processor);
|
||||
await processor.init({
|
||||
track: {} as MediaStreamTrack,
|
||||
} as unknown as AudioProcessorOptions);
|
||||
|
||||
// No second AudioContext / addModule inside init (i.e. under the lock).
|
||||
expect(FakeAudioContext.created).toHaveLength(1);
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
expect((processor as any).ctx).toBe(prepared);
|
||||
|
||||
// The claim is one-shot: a later processor prepares its own.
|
||||
const second = makeProcessor();
|
||||
stubGraph(second);
|
||||
await second.init({
|
||||
track: {} as MediaStreamTrack,
|
||||
} as unknown as AudioProcessorOptions);
|
||||
expect(FakeAudioContext.created).toHaveLength(2);
|
||||
|
||||
await processor.destroy();
|
||||
await second.destroy();
|
||||
});
|
||||
|
||||
test("releasePreparedDenoiseAssets closes an unclaimed prepared context", async () => {
|
||||
await prepareDenoiseAssets(baseConfig);
|
||||
const prepared = FakeAudioContext.created[0];
|
||||
await releasePreparedDenoiseAssets();
|
||||
expect(prepared.close).toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe("LotusDenoiseProcessor.setMicMuted (#9)", () => {
|
||||
test("suspends the owned context on mute, resumes on unmute, and the statechange watcher leaves the mute suspension alone", async () => {
|
||||
const processor = makeProcessor();
|
||||
stubGraph(processor);
|
||||
await processor.init({
|
||||
track: {} as MediaStreamTrack,
|
||||
} as unknown as AudioProcessorOptions);
|
||||
const ctx = FakeAudioContext.created[0];
|
||||
expect(ctx.state).toBe("running");
|
||||
|
||||
processor.setMicMuted(true);
|
||||
await Promise.resolve();
|
||||
expect(ctx.suspend).toHaveBeenCalledTimes(1);
|
||||
expect(ctx.state).toBe("suspended");
|
||||
|
||||
// The browser fires statechange for our own suspend too; the watcher must
|
||||
// NOT undo it while muted.
|
||||
ctx.resume.mockClear();
|
||||
ctx.dispatchEvent(new Event("statechange"));
|
||||
await Promise.resolve();
|
||||
expect(ctx.resume).not.toHaveBeenCalled();
|
||||
expect(ctx.state).toBe("suspended");
|
||||
|
||||
processor.setMicMuted(false);
|
||||
await Promise.resolve();
|
||||
expect(ctx.resume).toHaveBeenCalledTimes(1);
|
||||
expect(ctx.state).toBe("running");
|
||||
|
||||
// An EXTERNAL suspension while unmuted is still healed by the watcher.
|
||||
ctx.resume.mockClear();
|
||||
ctx.externallySuspend();
|
||||
await Promise.resolve();
|
||||
expect(ctx.resume).toHaveBeenCalledTimes(1);
|
||||
|
||||
await processor.destroy();
|
||||
});
|
||||
|
||||
test("a processor attached while already muted starts suspended", async () => {
|
||||
const processor = makeProcessor();
|
||||
stubGraph(processor);
|
||||
processor.setMicMuted(true);
|
||||
await processor.init({
|
||||
track: {} as MediaStreamTrack,
|
||||
} as unknown as AudioProcessorOptions);
|
||||
const ctx = FakeAudioContext.created[0];
|
||||
expect(ctx.state).toBe("suspended");
|
||||
await processor.destroy();
|
||||
});
|
||||
});
|
||||
|
||||
describe("assertModuleExport (#26)", () => {
|
||||
test("returns the module when the export is a function", () => {
|
||||
const mod = { createNoiseSuppressionAudioWorklet: (): void => undefined };
|
||||
expect(
|
||||
assertModuleExport(mod, "createNoiseSuppressionAudioWorklet", "x.js"),
|
||||
).toBe(mod);
|
||||
});
|
||||
|
||||
test("throws a clear error naming the module and export when missing", () => {
|
||||
expect(() =>
|
||||
assertModuleExport(
|
||||
{ other: 1 },
|
||||
"DeepFilterNet3Core",
|
||||
"dfn/index.esm.js",
|
||||
),
|
||||
).toThrow(/dfn\/index\.esm\.js does not export DeepFilterNet3Core/);
|
||||
expect(() => assertModuleExport(undefined, "X", "y.js")).toThrow(
|
||||
/does not export X/,
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -95,7 +95,9 @@ async function fetchWasm(url: string): Promise<ArrayBuffer> {
|
||||
* lands, and a still-suspended context degrades to (temporary) silence that the
|
||||
* watcher heals, not a hang.
|
||||
*/
|
||||
async function resumeCtx(ctx: AudioContext, timeoutMs = 3_000): Promise<void> {
|
||||
// [lotus] 500 ms, not 3 s: this still runs under LiveKit's trackChangeLock (#7),
|
||||
// and the statechange watcher heals a still-suspended context later anyway.
|
||||
async function resumeCtx(ctx: AudioContext, timeoutMs = 500): Promise<void> {
|
||||
await Promise.race([
|
||||
ctx.resume().catch(() => undefined),
|
||||
new Promise<void>((resolve) => setTimeout(resolve, timeoutMs)),
|
||||
@@ -127,6 +129,226 @@ interface Graph {
|
||||
track: MediaStreamTrack;
|
||||
}
|
||||
|
||||
// [lotus #26] Minimal local contracts for the two dynamically-imported ESM
|
||||
// helpers (not bundled here — see the CONTRACT note above). Their exports are
|
||||
// asserted at runtime so an asset bump that renames/removes one fails loudly
|
||||
// (and flows into the rnnoise fallback in lotusDenoise.ts) instead of as a
|
||||
// vague TypeError deep inside `init()`.
|
||||
interface DtlnModule {
|
||||
createNoiseSuppressionAudioWorklet: (
|
||||
ctx: AudioContext,
|
||||
opts: { bypassUntilReady: boolean },
|
||||
) => Promise<MlNode>;
|
||||
}
|
||||
interface DfnCore {
|
||||
initialize: () => Promise<void>;
|
||||
createAudioWorkletNode: (ctx: AudioContext) => Promise<AudioNode>;
|
||||
destroy: () => void;
|
||||
}
|
||||
interface DfnModule {
|
||||
DeepFilterNet3Core: new (opts: {
|
||||
sampleRate: number;
|
||||
noiseReductionLevel: number;
|
||||
assetConfig: { cdnUrl: string };
|
||||
}) => DfnCore;
|
||||
}
|
||||
|
||||
/** Throw a clear error if a dynamic-import module lacks an expected export. */
|
||||
export function assertModuleExport<T>(
|
||||
mod: unknown,
|
||||
name: string,
|
||||
url: string,
|
||||
): T {
|
||||
const exp = (mod as Record<string, unknown> | null | undefined)?.[name];
|
||||
if (typeof exp !== "function")
|
||||
throw new Error(
|
||||
`denoise: ${url} does not export ${name} (got ${typeof exp}) — asset/version mismatch`,
|
||||
);
|
||||
return mod as T;
|
||||
}
|
||||
|
||||
async function loadDfnCore(config: LotusDenoiseConfig): Promise<DfnCore> {
|
||||
const base = config.assetBase;
|
||||
const url = `${base}deepfilternet/index.esm.js`;
|
||||
const dfnBase = new URL(`${base}deepfilternet`, window.location.href).href;
|
||||
const mod = assertModuleExport<DfnModule>(
|
||||
await import(/* @vite-ignore */ url),
|
||||
"DeepFilterNet3Core",
|
||||
url,
|
||||
);
|
||||
const core = new mod.DeepFilterNet3Core({
|
||||
sampleRate: 48_000,
|
||||
// 60, not 80: full-strength suppression is the main source of the
|
||||
// "over-processed" character; a lower level keeps voice natural while
|
||||
// the dry/wet floor handles the noise tail.
|
||||
noiseReductionLevel: 60,
|
||||
assetConfig: { cdnUrl: dfnBase },
|
||||
});
|
||||
await core.initialize();
|
||||
return core;
|
||||
}
|
||||
|
||||
async function loadDtlnModule(config: LotusDenoiseConfig): Promise<DtlnModule> {
|
||||
const url = `${config.assetBase}workadventure/audio-worklet.js`;
|
||||
return assertModuleExport<DtlnModule>(
|
||||
await import(/* @vite-ignore */ url),
|
||||
"createNoiseSuppressionAudioWorklet",
|
||||
url,
|
||||
);
|
||||
}
|
||||
|
||||
/** Which wasm file a flat model uses (SIMD build when supported). */
|
||||
function flatWasmFiles(model: "rnnoise" | "speex"): {
|
||||
primary: string;
|
||||
fallback?: string;
|
||||
} {
|
||||
const flat = FLAT[model];
|
||||
const useSimd = model === "rnnoise" && !!flat.simdWasm && supportsSimd();
|
||||
return useSimd
|
||||
? { primary: flat.simdWasm!, fallback: flat.wasm }
|
||||
: { primary: flat.wasm };
|
||||
}
|
||||
|
||||
// [lotus #24] Force every node in the graph to a single, explicitly-downmixed
|
||||
// channel. Without `channelCountMode: "explicit"` the default ("max") IGNORES
|
||||
// `channelCount`, so a stereo capture device would feed 2 channels into a
|
||||
// worklet configured with `maxChannels: 1` and sum a stereo dry copy against a
|
||||
// mono wet one at the destination.
|
||||
const MONO: AudioNodeOptions = {
|
||||
channelCount: 1,
|
||||
channelCountMode: "explicit",
|
||||
channelInterpretation: "speakers",
|
||||
};
|
||||
|
||||
// [lotus #25] Algorithmic latency of each model in samples at its native rate,
|
||||
// used to delay the DRY copy of the floor mix so it lines up with the wet path
|
||||
// (otherwise the sum comb-filters — a hollow/phasey colouration on voice).
|
||||
// - rnnoise: 480-sample (10 ms @ 48 kHz) frames; the sapphi worklet buffers
|
||||
// 128-sample quanta up to one frame, so the wet path lags by one frame.
|
||||
// - speex: the sapphi speex worklet uses the same 480-sample framing.
|
||||
// - dtln: 512-sample block / 128 hop @ 16 kHz (~32 ms) per the DTLN paper —
|
||||
// best-known, unmeasured (the floor is not mixed for dtln, see buildGraph).
|
||||
// - deepfilternet: 480-sample hop + 2-frame lookahead @ 48 kHz (~30 ms) per
|
||||
// DeepFilterNet3 — best-known, unmeasured (floor not mixed for dfn either).
|
||||
const DRY_DELAY_SAMPLES: Record<LotusDenoiseModel, number> = {
|
||||
rnnoise: 480,
|
||||
speex: 480,
|
||||
dtln: 512,
|
||||
deepfilternet: 1440,
|
||||
};
|
||||
|
||||
/**
|
||||
* Create the model-rate context and register the flat/gate worklet modules.
|
||||
* Closes the context (and rethrows) on any failure so nothing half-built leaks.
|
||||
*/
|
||||
async function createModelContext(
|
||||
config: LotusDenoiseConfig,
|
||||
): Promise<AudioContext> {
|
||||
const rate = sampleRateFor(config.model);
|
||||
const ctx = new AudioContext({ sampleRate: rate });
|
||||
try {
|
||||
if (ctx.sampleRate !== rate)
|
||||
throw new Error(`denoise: got ${ctx.sampleRate}Hz, need ${rate}Hz`);
|
||||
// Flat models register via addModule here; DTLN/DeepFilterNet bring their
|
||||
// own processor via the dynamic-imported helper (see buildMlNode).
|
||||
if (config.model === "rnnoise" || config.model === "speex")
|
||||
await ctx.audioWorklet.addModule(
|
||||
config.assetBase + FLAT[config.model].script,
|
||||
);
|
||||
if (config.gate)
|
||||
await ctx.audioWorklet.addModule(config.assetBase + GATE.script);
|
||||
return ctx;
|
||||
} catch (e) {
|
||||
await ctx.close().catch(() => undefined);
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
// [lotus #7] Everything heavy that `init()` needs but that does NOT depend on
|
||||
// the mic track: the AudioContext + worklet modules, the flat wasm binary, and
|
||||
// (DFN) the fully-initialised model core. `LocalAudioTrack.setProcessor()`
|
||||
// holds LiveKit's `trackChangeLock` while awaiting `init()`, so every
|
||||
// mute/unmute/device-switch queues behind it — prepare these as soon as the
|
||||
// flag is seen (before any track exists) so `init()` only wires them up.
|
||||
interface PreparedAssets {
|
||||
ctx: AudioContext;
|
||||
dfnCore?: DfnCore;
|
||||
}
|
||||
const preparedAssets = new Map<string, Promise<PreparedAssets>>();
|
||||
const preparedKey = (c: LotusDenoiseConfig): string =>
|
||||
`${c.model}|${c.gate ? 1 : 0}|${c.assetBase}`;
|
||||
|
||||
async function prepareUncached(
|
||||
config: LotusDenoiseConfig,
|
||||
): Promise<PreparedAssets> {
|
||||
const ctx = await createModelContext(config);
|
||||
try {
|
||||
let dfnCore: DfnCore | undefined;
|
||||
if (config.model === "rnnoise" || config.model === "speex") {
|
||||
const { primary, fallback } = flatWasmFiles(config.model);
|
||||
// Warm the wasm cache; a SIMD miss is fine — buildMlNode falls back.
|
||||
await fetchWasm(config.assetBase + primary).catch(async () =>
|
||||
fallback ? fetchWasm(config.assetBase + fallback) : undefined,
|
||||
);
|
||||
} else if (config.model === "dtln") {
|
||||
await loadDtlnModule(config); // warms the browser's module map
|
||||
} else {
|
||||
dfnCore = await loadDfnCore(config);
|
||||
}
|
||||
return { ctx, dfnCore };
|
||||
} catch (e) {
|
||||
await ctx.close().catch(() => undefined);
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Prefetch/prepare the assets for `config` (idempotent per model). Never
|
||||
* rejects: a failed prepare is evicted so `init()` simply loads inline and
|
||||
* surfaces the real error there.
|
||||
*/
|
||||
export async function prepareDenoiseAssets(
|
||||
config: LotusDenoiseConfig,
|
||||
): Promise<void> {
|
||||
const key = preparedKey(config);
|
||||
let p = preparedAssets.get(key);
|
||||
if (!p) {
|
||||
p = prepareUncached(config);
|
||||
void p.catch((e) => {
|
||||
if (preparedAssets.get(key) === p) preparedAssets.delete(key);
|
||||
logger.warn(`[lotus] denoise prepare failed (${config.model})`, e);
|
||||
});
|
||||
preparedAssets.set(key, p);
|
||||
}
|
||||
await p.catch(() => undefined);
|
||||
}
|
||||
|
||||
/** Take (one-shot) the prepared assets for `config`, if any were prepared. */
|
||||
function claimPreparedAssets(
|
||||
config: LotusDenoiseConfig,
|
||||
): Promise<PreparedAssets> | undefined {
|
||||
const key = preparedKey(config);
|
||||
const p = preparedAssets.get(key);
|
||||
if (p) preparedAssets.delete(key);
|
||||
return p;
|
||||
}
|
||||
|
||||
/** Close any prepared-but-unclaimed contexts (call on feature teardown). */
|
||||
export async function releasePreparedDenoiseAssets(): Promise<void> {
|
||||
const all = [...preparedAssets.values()];
|
||||
preparedAssets.clear();
|
||||
await Promise.all(
|
||||
all.map(async (p) =>
|
||||
p
|
||||
.then(async (a) => {
|
||||
safeCall(() => a.dfnCore?.destroy());
|
||||
if (a.ctx.state !== "closed") await a.ctx.close();
|
||||
})
|
||||
.catch(() => undefined),
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* A LiveKit audio TrackProcessor that runs Lotus ML noise suppression
|
||||
* (RNNoise / Speex / DTLN / DeepFilterNet) on the local microphone track, as a
|
||||
@@ -151,9 +373,31 @@ export class LotusDenoiseProcessor implements TrackProcessor<
|
||||
private ctx?: AudioContext;
|
||||
private graph?: Graph;
|
||||
private ctxStateHandler?: () => void;
|
||||
private preparedDfnCore?: DfnCore;
|
||||
// [lotus #9] True while the mic is muted: we suspend our own context so the
|
||||
// worklet stops running inference on silence, and the statechange watcher
|
||||
// must not "heal" that intentional suspension.
|
||||
private micMuted = false;
|
||||
|
||||
public constructor(private readonly config: LotusDenoiseConfig) {}
|
||||
|
||||
/**
|
||||
* [lotus #9] Mirror the mic's mute state onto the owned context. EC uses
|
||||
* `stopMicTrackOnMute: false`, so a muted mic keeps producing (silent) frames
|
||||
* and the ML worklet would otherwise keep running full inference for the
|
||||
* whole time the user is muted.
|
||||
*/
|
||||
public setMicMuted(muted: boolean): void {
|
||||
this.micMuted = muted;
|
||||
const ctx = this.ctx;
|
||||
if (!ctx || ctx.state === "closed") return;
|
||||
if (muted) {
|
||||
if (ctx.state === "running") void ctx.suspend().catch(() => undefined);
|
||||
} else if (ctx.state === "suspended" && this.graph) {
|
||||
void ctx.resume().catch(() => undefined);
|
||||
}
|
||||
}
|
||||
|
||||
public async init(_opts: AudioProcessorOptions): Promise<void> {
|
||||
try {
|
||||
await this.ensureContext();
|
||||
@@ -163,6 +407,9 @@ export class LotusDenoiseProcessor implements TrackProcessor<
|
||||
// Don't orphan the owned context if graph construction fails (browsers
|
||||
// cap live AudioContexts, so repeated failed inits could exhaust them).
|
||||
// The caller degrades to the raw mic; we just release our resources.
|
||||
const core = this.preparedDfnCore;
|
||||
this.preparedDfnCore = undefined;
|
||||
if (core) safeCall(() => core.destroy());
|
||||
await this.closeContext();
|
||||
throw e;
|
||||
}
|
||||
@@ -194,6 +441,9 @@ export class LotusDenoiseProcessor implements TrackProcessor<
|
||||
this.disposeGraph(this.graph);
|
||||
this.graph = undefined;
|
||||
this.processedTrack = undefined;
|
||||
const core = this.preparedDfnCore;
|
||||
this.preparedDfnCore = undefined;
|
||||
if (core) safeCall(() => core.destroy());
|
||||
await this.closeContext();
|
||||
}
|
||||
|
||||
@@ -209,7 +459,7 @@ export class LotusDenoiseProcessor implements TrackProcessor<
|
||||
if (ctx.state !== "closed") await ctx.close().catch(() => undefined);
|
||||
}
|
||||
|
||||
/** Create (once) the model-rate context + register the flat worklet modules. */
|
||||
/** Adopt the prepared context (or create one) + install the state watcher. */
|
||||
private async ensureContext(): Promise<void> {
|
||||
const rate = sampleRateFor(this.config.model);
|
||||
if (
|
||||
@@ -217,36 +467,43 @@ export class LotusDenoiseProcessor implements TrackProcessor<
|
||||
this.ctx.state !== "closed" &&
|
||||
this.ctx.sampleRate === rate
|
||||
) {
|
||||
if (this.ctx.state === "suspended") await resumeCtx(this.ctx);
|
||||
if (this.ctx.state === "suspended" && !this.micMuted)
|
||||
await resumeCtx(this.ctx);
|
||||
return;
|
||||
}
|
||||
await this.closeContext();
|
||||
|
||||
const ctx = new AudioContext({ sampleRate: rate });
|
||||
// [lotus #7] Prefer the context/modules/model prepared before
|
||||
// setProcessor() was called; only load inline if nothing was prepared
|
||||
// (e.g. the rnnoise fallback path, or a second processor after a
|
||||
// republish).
|
||||
const claimed = await claimPreparedAssets(this.config)?.catch(
|
||||
() => undefined,
|
||||
);
|
||||
let ctx: AudioContext;
|
||||
if (claimed && claimed.ctx.state !== "closed") {
|
||||
ctx = claimed.ctx;
|
||||
this.preparedDfnCore = claimed.dfnCore;
|
||||
} else {
|
||||
ctx = await createModelContext(this.config);
|
||||
}
|
||||
try {
|
||||
if (ctx.sampleRate !== rate)
|
||||
throw new Error(`denoise: got ${ctx.sampleRate}Hz, need ${rate}Hz`);
|
||||
|
||||
// Auto-resume if the OS/browser suspends the context mid-call (mobile
|
||||
// backgrounding, audio interruption): the dest node otherwise emits
|
||||
// silence with no recovery. Only resume while a graph is live.
|
||||
// silence with no recovery. Only resume while a graph is live and the
|
||||
// suspension isn't our own mute suspension (#9).
|
||||
const onStateChange = (): void => {
|
||||
if (ctx.state === "suspended" && this.graph)
|
||||
if (ctx.state === "suspended" && this.graph && !this.micMuted)
|
||||
void ctx.resume().catch(() => undefined);
|
||||
};
|
||||
ctx.addEventListener("statechange", onStateChange);
|
||||
|
||||
// Flat models register via addModule here; DTLN/DeepFilterNet bring their
|
||||
// own processor via the dynamic-imported helper (see buildMlNode).
|
||||
if (this.config.model === "rnnoise" || this.config.model === "speex")
|
||||
await ctx.audioWorklet.addModule(
|
||||
this.config.assetBase + FLAT[this.config.model].script,
|
||||
);
|
||||
if (this.config.gate)
|
||||
await ctx.audioWorklet.addModule(this.config.assetBase + GATE.script);
|
||||
// The action can arrive via host postMessage, not a gesture in this
|
||||
// iframe, so the context can start suspended — resume without hanging.
|
||||
if (ctx.state === "suspended") await resumeCtx(ctx);
|
||||
if (ctx.state === "suspended" && !this.micMuted) await resumeCtx(ctx);
|
||||
// Attached while already muted (#9): don't let a prepared, running
|
||||
// context burn inference until the first unmute.
|
||||
else if (ctx.state === "running" && this.micMuted)
|
||||
await ctx.suspend().catch(() => undefined);
|
||||
|
||||
this.ctx = ctx;
|
||||
this.ctxStateHandler = onStateChange;
|
||||
@@ -260,7 +517,7 @@ export class LotusDenoiseProcessor implements TrackProcessor<
|
||||
private async buildGraph(track: MediaStreamTrack): Promise<Graph> {
|
||||
const ctx = this.ctx!;
|
||||
const source = ctx.createMediaStreamSource(new MediaStream([track]));
|
||||
const dest = ctx.createMediaStreamDestination();
|
||||
const dest = new MediaStreamAudioDestinationNode(ctx, MONO);
|
||||
const nodes: AudioNode[] = [];
|
||||
const disposes: (() => void)[] = [];
|
||||
|
||||
@@ -277,6 +534,7 @@ export class LotusDenoiseProcessor implements TrackProcessor<
|
||||
// made the threshold operate on pre-denoise levels. Gate the residual.
|
||||
if (this.config.gate) {
|
||||
const gate = new AudioWorkletNode(ctx, GATE.name, {
|
||||
...MONO,
|
||||
processorOptions: {
|
||||
openThreshold: this.config.gateThreshold,
|
||||
closeThreshold: this.config.gateThreshold - 5,
|
||||
@@ -289,11 +547,11 @@ export class LotusDenoiseProcessor implements TrackProcessor<
|
||||
nodes.push(gate);
|
||||
}
|
||||
|
||||
// Only mix a dry floor for the LOW-LATENCY flat models (RNNoise/Speex).
|
||||
// DTLN/DeepFilterNet add tens of ms of algorithmic latency, so summing an
|
||||
// undelayed dry copy would comb-filter the voice — for those we rely on
|
||||
// the model's own level (e.g. DFN noiseReductionLevel) instead. RNNoise is
|
||||
// also where the "robotic/underwater" reports come from, so this targets it.
|
||||
// Only mix a dry floor for the flat models (RNNoise/Speex), whose
|
||||
// framing latency is known exactly (DRY_DELAY_SAMPLES); the DTLN/DFN
|
||||
// figures are best-known estimates, so for those we rely on the model's
|
||||
// own level (e.g. DFN noiseReductionLevel) instead. RNNoise is also where
|
||||
// the "robotic/underwater" reports come from, so this targets it.
|
||||
const lowLatency =
|
||||
this.config.model === "rnnoise" || this.config.model === "speex";
|
||||
const floor = lowLatency
|
||||
@@ -305,17 +563,25 @@ export class LotusDenoiseProcessor implements TrackProcessor<
|
||||
// (kills the "underwater"/pumping artifact). During speech (denoised ≈
|
||||
// original) the two sum back to ~unity; in noise-only gaps the output
|
||||
// floors at `floor` × original instead of digital silence.
|
||||
const wetGain = ctx.createGain();
|
||||
wetGain.gain.value = 1 - floor;
|
||||
const wetGain = new GainNode(ctx, { ...MONO, gain: 1 - floor });
|
||||
wetHead.connect(wetGain);
|
||||
wetGain.connect(dest);
|
||||
nodes.push(wetGain);
|
||||
|
||||
const dryGain = ctx.createGain();
|
||||
dryGain.gain.value = floor;
|
||||
source.connect(dryGain);
|
||||
// [lotus #25] Delay the dry copy by the model's algorithmic latency so
|
||||
// it sums in phase with the (framed, hence delayed) wet path instead
|
||||
// of comb-filtering against it.
|
||||
const delaySec = DRY_DELAY_SAMPLES[this.config.model] / ctx.sampleRate;
|
||||
const dryDelay = new DelayNode(ctx, {
|
||||
...MONO,
|
||||
maxDelayTime: Math.max(delaySec, 1 / ctx.sampleRate),
|
||||
delayTime: delaySec,
|
||||
});
|
||||
const dryGain = new GainNode(ctx, { ...MONO, gain: floor });
|
||||
source.connect(dryDelay);
|
||||
dryDelay.connect(dryGain);
|
||||
dryGain.connect(dest);
|
||||
nodes.push(dryGain);
|
||||
nodes.push(dryDelay, dryGain);
|
||||
} else {
|
||||
wetHead.connect(dest);
|
||||
}
|
||||
@@ -350,48 +616,36 @@ export class LotusDenoiseProcessor implements TrackProcessor<
|
||||
if (model === "dtln") {
|
||||
// Self-contained ESM that resolves its own processor + LiteRT wasm +
|
||||
// TFLite models. bypassUntilReady passes raw audio until the model loads.
|
||||
const mod = await import(
|
||||
/* @vite-ignore */ `${base}workadventure/audio-worklet.js`
|
||||
);
|
||||
return (await mod.createNoiseSuppressionAudioWorklet(ctx, {
|
||||
const mod = await loadDtlnModule(this.config);
|
||||
return await mod.createNoiseSuppressionAudioWorklet(ctx, {
|
||||
bypassUntilReady: true,
|
||||
})) as MlNode;
|
||||
});
|
||||
}
|
||||
|
||||
if (model === "deepfilternet") {
|
||||
const dfnBase = new URL(`${base}deepfilternet`, window.location.href)
|
||||
.href;
|
||||
const mod = await import(
|
||||
/* @vite-ignore */ `${base}deepfilternet/index.esm.js`
|
||||
);
|
||||
const core = new mod.DeepFilterNet3Core({
|
||||
sampleRate: 48_000,
|
||||
// 60, not 80: full-strength suppression is the main source of the
|
||||
// "over-processed" character; a lower level keeps voice natural while
|
||||
// the dry/wet floor handles the noise tail.
|
||||
noiseReductionLevel: 60,
|
||||
assetConfig: { cdnUrl: dfnBase },
|
||||
});
|
||||
await core.initialize();
|
||||
const node = (await core.createAudioWorkletNode(ctx)) as AudioNode;
|
||||
// [lotus #7] Use the core initialised by prepareDenoiseAssets() if we
|
||||
// have one (first graph); later rebuilds (restart) load a fresh core.
|
||||
const prepared = this.preparedDfnCore;
|
||||
this.preparedDfnCore = undefined;
|
||||
const core = prepared ?? (await loadDfnCore(this.config));
|
||||
const node = await core.createAudioWorkletNode(ctx);
|
||||
return { node, dispose: () => safeCall(() => core.destroy()) };
|
||||
}
|
||||
|
||||
// Flat sapphi worklet (rnnoise/speex).
|
||||
const flat = FLAT[model];
|
||||
const useSimd = model === "rnnoise" && !!flat.simdWasm && supportsSimd();
|
||||
const wasmFile = useSimd ? flat.simdWasm! : flat.wasm;
|
||||
const { primary, fallback } = flatWasmFiles(model);
|
||||
let wasmBinary: ArrayBuffer;
|
||||
try {
|
||||
wasmBinary = await fetchWasm(base + wasmFile);
|
||||
wasmBinary = await fetchWasm(base + primary);
|
||||
} catch (e) {
|
||||
if (useSimd) {
|
||||
wasmCache.delete(base + wasmFile);
|
||||
wasmBinary = await fetchWasm(base + flat.wasm); // fall back to non-SIMD
|
||||
if (fallback) {
|
||||
wasmCache.delete(base + primary);
|
||||
wasmBinary = await fetchWasm(base + fallback); // fall back to non-SIMD
|
||||
} else throw e;
|
||||
}
|
||||
const node = new AudioWorkletNode(ctx, flat.name, {
|
||||
channelCount: 1,
|
||||
...MONO,
|
||||
numberOfInputs: 1,
|
||||
numberOfOutputs: 1,
|
||||
processorOptions: { maxChannels: 1, wasmBinary },
|
||||
|
||||
+26
-11
@@ -10,12 +10,33 @@ import { type IWidgetApiRequest } from "matrix-widget-api";
|
||||
import { type CallViewModel } from "../state/CallViewModel/CallViewModel";
|
||||
import { widget } from "../widget";
|
||||
import { LotusWidgetActions } from "./lotusActions";
|
||||
import { type ManualSpotlight } from "./lotusSpotlight";
|
||||
|
||||
/**
|
||||
* Parse a `focus_participant` payload into a pin, `null` to clear, or
|
||||
* `undefined` to leave the current pin alone (#30). Exported for tests.
|
||||
*
|
||||
* Mirror deafen's partial-payload semantics: a payload that OMITS `userId`
|
||||
* must keep the current spotlight, not clear it. Only act when the key is
|
||||
* actually present — an explicit `null` clears, a string pins that user. An
|
||||
* optional `id` (EC media id `${userId}:${deviceId}`, as sent to the host in
|
||||
* `io.lotus.call_state`) selects a specific device of that user.
|
||||
*/
|
||||
export function parseFocusPayload(
|
||||
data: unknown,
|
||||
): ManualSpotlight | null | undefined {
|
||||
if (typeof data !== "object" || data === null || !("userId" in data))
|
||||
return undefined;
|
||||
const { userId, id } = data as { userId?: unknown; id?: unknown };
|
||||
if (typeof userId !== "string") return null;
|
||||
return { userId, id: typeof id === "string" ? id : null };
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle the host's `io.lotus.focus_participant` toWidget action (#4): pin a
|
||||
* participant to the spotlight by Matrix user id, or clear it with
|
||||
* `{ userId: null }`. This replaces cinny's old DOM `.click()` tile-selector
|
||||
* hack with a real, layout-aware spotlight override.
|
||||
* participant to the spotlight by Matrix user id (and optionally media id), or
|
||||
* clear it with `{ userId: null }`. This replaces cinny's old DOM `.click()`
|
||||
* tile-selector hack with a real, layout-aware spotlight override.
|
||||
*
|
||||
* No effect unless the host actually sends the action, so registering the
|
||||
* handler whenever we're a widget is safe. Returns a teardown function.
|
||||
@@ -27,14 +48,8 @@ export function startLotusFocus(vm: CallViewModel): () => void {
|
||||
const handler = (ev: CustomEvent<IWidgetApiRequest>): void => {
|
||||
// Always reply so the host transport doesn't time out.
|
||||
w.api.transport.reply(ev.detail, {});
|
||||
const data = ev.detail.data as { userId?: unknown } | undefined;
|
||||
// Mirror deafen's partial-payload semantics: a payload that OMITS `userId`
|
||||
// must keep the current spotlight, not clear it. Only act when the key is
|
||||
// actually present — an explicit `null` clears, a string pins that user.
|
||||
if (data && "userId" in data) {
|
||||
const userId = typeof data.userId === "string" ? data.userId : null;
|
||||
vm.setManualSpotlight(userId);
|
||||
}
|
||||
const target = parseFocusPayload(ev.detail.data);
|
||||
if (target !== undefined) vm.setManualSpotlight(target);
|
||||
};
|
||||
|
||||
w.lazyActions.on(LotusWidgetActions.FocusParticipant, handler);
|
||||
|
||||
@@ -7,7 +7,11 @@ Please see LICENSE in the repository root for full details.
|
||||
|
||||
import { describe, expect, test, vi } from "vitest";
|
||||
|
||||
import { buildPatch, patchSender } from "./lotusQuality";
|
||||
import {
|
||||
applyProportionalMaxBitrate,
|
||||
buildPatch,
|
||||
patchSender,
|
||||
} from "./lotusQuality";
|
||||
|
||||
function makeSender(
|
||||
initialEncodings: RTCRtpEncodingParameters[] = [{}],
|
||||
@@ -91,3 +95,65 @@ describe("lotusQuality set_quality -> clear (#11)", () => {
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe("lotusQuality screenshare simulcast cap (#12)", () => {
|
||||
test("caps the aggregate across a 3-layer simulcast publication, not per-layer", async () => {
|
||||
// A typical VP8 simulcast screenshare publication: low/medium/high
|
||||
// layers with existing presets, ordered low-to-high like LiveKit
|
||||
// publishes them.
|
||||
const sender = makeSender([
|
||||
{ maxBitrate: 150_000 },
|
||||
{ maxBitrate: 500_000 },
|
||||
{ maxBitrate: 1_500_000 },
|
||||
]);
|
||||
const writtenKeys = new WeakMap<
|
||||
RTCRtpSender,
|
||||
Set<keyof RTCRtpEncodingParameters>
|
||||
>();
|
||||
|
||||
const cap = 750_000;
|
||||
const patch = buildPatch(sender, { maxBitrate: cap }, writtenKeys);
|
||||
await patchSender(sender, patch, writtenKeys);
|
||||
|
||||
const encodings = sender.getParameters().encodings;
|
||||
expect(encodings).toHaveLength(3);
|
||||
// Each layer keeps its old proportion of the total, but the aggregate
|
||||
// across all layers must not exceed the requested cap — that's the bug:
|
||||
// writing `cap` into every layer let the aggregate reach ~3x the cap.
|
||||
const oldTotal = 150_000 + 500_000 + 1_500_000;
|
||||
expect(encodings[0].maxBitrate).toBe(
|
||||
Math.floor((150_000 / oldTotal) * cap),
|
||||
);
|
||||
expect(encodings[1].maxBitrate).toBe(
|
||||
Math.floor((500_000 / oldTotal) * cap),
|
||||
);
|
||||
expect(encodings[2].maxBitrate).toBe(
|
||||
Math.floor((1_500_000 / oldTotal) * cap),
|
||||
);
|
||||
// The highest layer still ends up with the largest share.
|
||||
expect(encodings[2].maxBitrate).toBeGreaterThan(encodings[1].maxBitrate!);
|
||||
expect(encodings[1].maxBitrate).toBeGreaterThan(encodings[0].maxBitrate!);
|
||||
|
||||
const aggregate = encodings.reduce(
|
||||
(sum, enc) => sum + (enc.maxBitrate ?? 0),
|
||||
0,
|
||||
);
|
||||
expect(aggregate).toBeLessThanOrEqual(cap);
|
||||
});
|
||||
|
||||
test("a single encoding keeps the previous (non-scaled) behaviour", () => {
|
||||
const encodings: RTCRtpEncodingParameters[] = [{ maxBitrate: 2_000_000 }];
|
||||
applyProportionalMaxBitrate(encodings, 500_000);
|
||||
expect(encodings).toEqual([{ maxBitrate: 500_000 }]);
|
||||
});
|
||||
|
||||
test("falls back to capping only the highest layer when there are no prior ratios", () => {
|
||||
const encodings: RTCRtpEncodingParameters[] = [
|
||||
{}, // no prior cap on any layer to derive a ratio from
|
||||
{},
|
||||
];
|
||||
applyProportionalMaxBitrate(encodings, 500_000);
|
||||
expect(encodings[0].maxBitrate).toBeUndefined();
|
||||
expect(encodings[1].maxBitrate).toBe(500_000);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -198,6 +198,43 @@ export function buildPatch(
|
||||
return patch;
|
||||
}
|
||||
|
||||
// [lotus] Treat `cap` as a budget for the WHOLE simulcast publication rather
|
||||
// than a per-encoding value, and distribute it across encodings in
|
||||
// proportion to their existing (pre-patch) ratios — instead of writing `cap`
|
||||
// into every encoding (#12), which let the aggregate reach up to N× the
|
||||
// requested cap (and also raised the low layers far above their presets).
|
||||
// With a single encoding this reduces to the previous behaviour: that
|
||||
// encoding simply gets `cap`.
|
||||
// Exported for unit testing only (see lotusQuality.test.ts) — not part of
|
||||
// the module's public surface used by callers.
|
||||
export function applyProportionalMaxBitrate(
|
||||
encodings: RTCRtpEncodingParameters[],
|
||||
cap: number,
|
||||
): void {
|
||||
if (encodings.length <= 1) {
|
||||
for (const enc of encodings) enc.maxBitrate = cap;
|
||||
return;
|
||||
}
|
||||
const total = encodings.reduce(
|
||||
(sum, enc) =>
|
||||
sum + (typeof enc.maxBitrate === "number" ? enc.maxBitrate : 0),
|
||||
0,
|
||||
);
|
||||
if (total <= 0) {
|
||||
// No existing ratios to scale from: fall back to capping only the
|
||||
// highest layer (LiveKit orders simulcast encodings low-to-high, so
|
||||
// that's the last one) and leave the others alone rather than guess.
|
||||
encodings[encodings.length - 1].maxBitrate = cap;
|
||||
return;
|
||||
}
|
||||
// Math.floor (not round) so rounding error can only push the aggregate
|
||||
// under the cap, never over it.
|
||||
for (const enc of encodings) {
|
||||
const prev = typeof enc.maxBitrate === "number" ? enc.maxBitrate : 0;
|
||||
enc.maxBitrate = Math.floor((prev / total) * cap);
|
||||
}
|
||||
}
|
||||
|
||||
// Exported for unit testing only (see lotusQuality.test.ts) — not part of
|
||||
// the module's public surface used by callers.
|
||||
export async function patchSender(
|
||||
@@ -210,10 +247,26 @@ export async function patchSender(
|
||||
const params = sender.getParameters();
|
||||
if (!params.encodings || params.encodings.length === 0)
|
||||
params.encodings = [{}];
|
||||
// Apply to EVERY encoding, not just encodings[0]: screenshare publishes
|
||||
// with simulcast (VP8), so encodings[0] is the small layer and the
|
||||
// full-resolution layer — the real bandwidth hog — is a later encoding.
|
||||
for (const enc of params.encodings) Object.assign(enc, patch);
|
||||
// Screenshare publishes with simulcast (VP8): encodings[0] is the small
|
||||
// layer and the full-resolution layer — the real bandwidth hog — is a
|
||||
// later encoding. maxFramerate (and an `undefined` clear) still apply to
|
||||
// every encoding, but maxBitrate is special-cased (#12): writing the same
|
||||
// value into every encoding let the aggregate reach N× the requested cap
|
||||
// and also raised the low layers far above their presets, so instead we
|
||||
// treat the requested value as a budget for the whole publication and
|
||||
// distribute it across encodings proportionally to their existing
|
||||
// ratios.
|
||||
const { maxBitrate, ...rest } = patch;
|
||||
for (const enc of params.encodings) Object.assign(enc, rest);
|
||||
if ("maxBitrate" in patch) {
|
||||
if (maxBitrate === undefined) {
|
||||
// Clearing the cap (#11): drop it from every encoding rather than
|
||||
// scaling — there's no budget to distribute.
|
||||
for (const enc of params.encodings) enc.maxBitrate = undefined;
|
||||
} else {
|
||||
applyProportionalMaxBitrate(params.encodings, maxBitrate);
|
||||
}
|
||||
}
|
||||
await sender.setParameters(params);
|
||||
// Remember only the caps that are still active (defined) after this
|
||||
// write, so a later clear knows exactly which keys to unset.
|
||||
|
||||
@@ -0,0 +1,295 @@
|
||||
/*
|
||||
Copyright 2026 Lotus Guild
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { describe, expect, test, vi } from "vitest";
|
||||
import { BehaviorSubject, of, Subject } from "rxjs";
|
||||
|
||||
import { withTestScheduler } from "../utils/test";
|
||||
import { type UserMediaViewModel } from "../state/media/UserMediaViewModel";
|
||||
import { type ScreenShareViewModel } from "../state/media/ScreenShareViewModel";
|
||||
import { type LayoutMode } from "../state/LayoutSwitchViewModel";
|
||||
import {
|
||||
createManualSpotlight,
|
||||
type ManualSpotlight,
|
||||
overrideSpotlight$,
|
||||
pinnedMedia$,
|
||||
resolveManualSpotlight,
|
||||
} from "./lotusSpotlight";
|
||||
import { parseFocusPayload } from "./lotusFocus";
|
||||
|
||||
interface FakeMedia {
|
||||
id: string;
|
||||
userId: string;
|
||||
type: "user";
|
||||
local: boolean;
|
||||
speaking$: BehaviorSubject<boolean>;
|
||||
}
|
||||
|
||||
function media(
|
||||
userId: string,
|
||||
deviceId: string,
|
||||
speaking = false,
|
||||
local = false,
|
||||
): FakeMedia {
|
||||
return {
|
||||
id: `${userId}:${deviceId}`,
|
||||
userId,
|
||||
type: "user",
|
||||
local,
|
||||
speaking$: new BehaviorSubject(speaking),
|
||||
};
|
||||
}
|
||||
const asUser = (m: FakeMedia): UserMediaViewModel =>
|
||||
m as unknown as UserMediaViewModel;
|
||||
const screen = { id: "@a:x:d1:screen", userId: "@a:x", type: "screen share" };
|
||||
|
||||
const aliceDesk = media("@alice:x", "desk");
|
||||
const alicePhone = media("@alice:x", "phone", true);
|
||||
const bob = media("@bob:x", "d");
|
||||
const all = [aliceDesk, alicePhone, bob];
|
||||
|
||||
describe("resolveManualSpotlight (#30)", () => {
|
||||
const speaking = (m: FakeMedia): boolean => m.speaking$.value;
|
||||
|
||||
test("null pin resolves to nothing", () => {
|
||||
expect(resolveManualSpotlight(null, all, speaking)).toBeUndefined();
|
||||
});
|
||||
|
||||
test("prefers the media id when given", () => {
|
||||
expect(
|
||||
resolveManualSpotlight(
|
||||
{ userId: "@alice:x", id: "@alice:x:desk" },
|
||||
all,
|
||||
speaking,
|
||||
),
|
||||
).toBe(aliceDesk);
|
||||
});
|
||||
|
||||
test("falls back to the userId when the id is not present", () => {
|
||||
expect(
|
||||
resolveManualSpotlight(
|
||||
{ userId: "@alice:x", id: "@alice:x:tablet" },
|
||||
all,
|
||||
speaking,
|
||||
),
|
||||
).toBe(alicePhone);
|
||||
});
|
||||
|
||||
test("userId only: prefers the speaking device, else the first", () => {
|
||||
expect(
|
||||
resolveManualSpotlight({ userId: "@alice:x", id: null }, all, speaking),
|
||||
).toBe(alicePhone);
|
||||
expect(
|
||||
resolveManualSpotlight(
|
||||
{ userId: "@alice:x", id: null },
|
||||
all,
|
||||
() => false,
|
||||
),
|
||||
).toBe(aliceDesk);
|
||||
});
|
||||
|
||||
test("absent user resolves to nothing", () => {
|
||||
expect(
|
||||
resolveManualSpotlight({ userId: "@carol:x", id: null }, all, speaking),
|
||||
).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe("pinnedMedia$", () => {
|
||||
test("follows the speaking device live for a userId-only pin", () => {
|
||||
withTestScheduler(({ expectObservable, schedule }) => {
|
||||
const desk = media("@alice:x", "desk", true);
|
||||
const phone = media("@alice:x", "phone", false);
|
||||
const pin$ = new BehaviorSubject<ManualSpotlight | null>({
|
||||
userId: "@alice:x",
|
||||
id: null,
|
||||
});
|
||||
schedule("-a", {
|
||||
a: () => {
|
||||
desk.speaking$.next(false);
|
||||
phone.speaking$.next(true);
|
||||
},
|
||||
});
|
||||
expectObservable(
|
||||
pinnedMedia$(pin$, of([asUser(desk), asUser(phone)])),
|
||||
).toBe("dp", { d: desk, p: phone });
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("overrideSpotlight$", () => {
|
||||
test("no pin: identical to upstream", () => {
|
||||
withTestScheduler(({ expectObservable }) => {
|
||||
const result$ = overrideSpotlight$(
|
||||
of(asUser(bob)),
|
||||
of(null),
|
||||
of([]),
|
||||
of(undefined),
|
||||
of(all.map(asUser)),
|
||||
);
|
||||
expectObservable(result$.pipe()).toBe("(a|)", {
|
||||
a: expect.objectContaining({ spotlight: [bob] }),
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
test("pin replaces the auto speaker without a screenshare", () => {
|
||||
withTestScheduler(({ expectObservable }) => {
|
||||
const result$ = overrideSpotlight$(
|
||||
of(asUser(bob)),
|
||||
of({ userId: "@alice:x", id: "@alice:x:desk" }),
|
||||
of([]),
|
||||
of(undefined),
|
||||
of(all.map(asUser)),
|
||||
);
|
||||
expectObservable(result$).toBe("(a|)", {
|
||||
a: expect.objectContaining({ spotlight: [aliceDesk] }),
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
test("#29: PiP keeps the auto speaker when a third party is pinned", () => {
|
||||
withTestScheduler(({ expectObservable }) => {
|
||||
let pip$;
|
||||
overrideSpotlight$(
|
||||
of(asUser(bob)),
|
||||
of({ userId: "@alice:x", id: null }),
|
||||
of([screen as unknown as ScreenShareViewModel]),
|
||||
of(undefined),
|
||||
of(all.map(asUser)),
|
||||
).subscribe((r) => {
|
||||
expect(r.spotlight).toEqual([screen, alicePhone]);
|
||||
pip$ = r.pip$;
|
||||
});
|
||||
expectObservable(pip$!).toBe("(b|)", { b: bob });
|
||||
});
|
||||
});
|
||||
|
||||
test("#29: PiP is blanked only when the auto speaker is the pinned one", () => {
|
||||
withTestScheduler(({ expectObservable }) => {
|
||||
let pip$;
|
||||
overrideSpotlight$(
|
||||
of(asUser(alicePhone)),
|
||||
of({ userId: "@alice:x", id: null }),
|
||||
of([screen as unknown as ScreenShareViewModel]),
|
||||
of(undefined),
|
||||
of(all.map(asUser)),
|
||||
).subscribe((r) => {
|
||||
pip$ = r.pip$;
|
||||
});
|
||||
expectObservable(pip$!).toBe("(u|)", { u: undefined });
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("createManualSpotlight (#3 / #16)", () => {
|
||||
function setup(initialLayout: LayoutMode = "grid") {
|
||||
const pin$ = new BehaviorSubject<ManualSpotlight | null>(null);
|
||||
const layout$ = new BehaviorSubject<LayoutMode>(initialLayout);
|
||||
const setLayout = vi.fn((m: LayoutMode) => layout$.next(m));
|
||||
const userMedia$ = new BehaviorSubject<{ userId: string }[]>([
|
||||
{ userId: "@alice:x" },
|
||||
{ userId: "@bob:x" },
|
||||
]);
|
||||
const leave$ = new Subject<void>();
|
||||
const ctl = createManualSpotlight(
|
||||
pin$,
|
||||
userMedia$,
|
||||
leave$,
|
||||
{ layout$, setLayout },
|
||||
5000,
|
||||
);
|
||||
const sub = ctl.effects$.subscribe();
|
||||
return { pin$, layout$, setLayout, userMedia$, leave$, ctl, sub };
|
||||
}
|
||||
|
||||
test("pinning forces spotlight and clearing restores the displaced mode", () => {
|
||||
const { ctl, layout$, setLayout, pin$ } = setup("grid");
|
||||
ctl.setManualSpotlight({ userId: "@alice:x", id: null });
|
||||
expect(pin$.value).toEqual({ userId: "@alice:x", id: null });
|
||||
expect(setLayout).toHaveBeenLastCalledWith("spotlight");
|
||||
// Re-pinning someone else does not touch the layout again.
|
||||
ctl.setManualSpotlight({ userId: "@bob:x", id: null });
|
||||
expect(setLayout).toHaveBeenCalledTimes(1);
|
||||
ctl.setManualSpotlight(null);
|
||||
expect(layout$.value).toBe("grid");
|
||||
expect(setLayout).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
test("already in spotlight: nothing is forced or restored", () => {
|
||||
const { ctl, setLayout } = setup("spotlight");
|
||||
ctl.setManualSpotlight({ userId: "@alice:x", id: null });
|
||||
ctl.setManualSpotlight(null);
|
||||
expect(setLayout).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
test("does not clobber a layout the user chose while pinned", () => {
|
||||
const { ctl, layout$, setLayout } = setup("grid");
|
||||
ctl.setManualSpotlight({ userId: "@alice:x", id: null });
|
||||
layout$.next("grid"); // user switched back manually
|
||||
ctl.setManualSpotlight(null);
|
||||
expect(setLayout).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
test("#16: pin is cleared when the user is gone for 5 s, not on a blip", () => {
|
||||
vi.useFakeTimers();
|
||||
try {
|
||||
const { ctl, pin$, userMedia$, layout$ } = setup("grid");
|
||||
ctl.setManualSpotlight({ userId: "@alice:x", id: null });
|
||||
// Transient absence: back within the window.
|
||||
userMedia$.next([{ userId: "@bob:x" }]);
|
||||
vi.advanceTimersByTime(2000);
|
||||
userMedia$.next([{ userId: "@alice:x" }, { userId: "@bob:x" }]);
|
||||
vi.advanceTimersByTime(6000);
|
||||
expect(pin$.value).not.toBeNull();
|
||||
// Real leave.
|
||||
userMedia$.next([{ userId: "@bob:x" }]);
|
||||
vi.advanceTimersByTime(4999);
|
||||
expect(pin$.value).not.toBeNull();
|
||||
vi.advanceTimersByTime(1);
|
||||
expect(pin$.value).toBeNull();
|
||||
expect(layout$.value).toBe("grid");
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
test("#16: pin is cleared on leave$", () => {
|
||||
const { ctl, pin$, leave$, layout$ } = setup("grid");
|
||||
ctl.setManualSpotlight({ userId: "@alice:x", id: null });
|
||||
expect(layout$.value).toBe("spotlight");
|
||||
leave$.next();
|
||||
expect(pin$.value).toBeNull();
|
||||
expect(layout$.value).toBe("grid");
|
||||
});
|
||||
});
|
||||
|
||||
describe("parseFocusPayload (#30)", () => {
|
||||
test("omitted userId leaves the pin alone", () => {
|
||||
expect(parseFocusPayload(undefined)).toBeUndefined();
|
||||
expect(parseFocusPayload({})).toBeUndefined();
|
||||
expect(parseFocusPayload({ id: "@a:x:d" })).toBeUndefined();
|
||||
});
|
||||
test("null / non-string userId clears", () => {
|
||||
expect(parseFocusPayload({ userId: null })).toBeNull();
|
||||
expect(parseFocusPayload({ userId: 42 })).toBeNull();
|
||||
});
|
||||
test("userId with and without id", () => {
|
||||
expect(parseFocusPayload({ userId: "@a:x" })).toEqual({
|
||||
userId: "@a:x",
|
||||
id: null,
|
||||
});
|
||||
expect(parseFocusPayload({ userId: "@a:x", id: "@a:x:d" })).toEqual({
|
||||
userId: "@a:x",
|
||||
id: "@a:x:d",
|
||||
});
|
||||
expect(parseFocusPayload({ userId: "@a:x", id: 7 })).toEqual({
|
||||
userId: "@a:x",
|
||||
id: null,
|
||||
});
|
||||
});
|
||||
});
|
||||
+191
-28
@@ -5,18 +5,103 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { combineLatest, map, type Observable, of, switchMap } from "rxjs";
|
||||
import {
|
||||
type BehaviorSubject,
|
||||
combineLatest,
|
||||
debounceTime,
|
||||
distinctUntilChanged,
|
||||
filter,
|
||||
ignoreElements,
|
||||
map,
|
||||
merge,
|
||||
type Observable,
|
||||
of,
|
||||
switchMap,
|
||||
tap,
|
||||
} from "rxjs";
|
||||
|
||||
import { type UserMediaViewModel } from "../state/media/UserMediaViewModel";
|
||||
import { type ScreenShareViewModel } from "../state/media/ScreenShareViewModel";
|
||||
import { type MediaViewModel } from "../state/media/MediaViewModel";
|
||||
import { type LocalUserMediaViewModel } from "../state/media/LocalUserMediaViewModel";
|
||||
import { type LayoutMode } from "../state/LayoutSwitchViewModel";
|
||||
import { type Behavior } from "../state/Behavior";
|
||||
|
||||
interface SpotlightAndPip {
|
||||
spotlight: MediaViewModel[];
|
||||
pip$: Observable<UserMediaViewModel | undefined>;
|
||||
}
|
||||
|
||||
/**
|
||||
* [lotus #4/#30] A host-requested spotlight pin. `id` is the EC media id
|
||||
* (`${userId}:${deviceId}`, the same value `io.lotus.call_state` sends the
|
||||
* host) and is preferred when present; `userId` alone resolves to the
|
||||
* currently-speaking device of that user, else their first device.
|
||||
*/
|
||||
export interface ManualSpotlight {
|
||||
userId: string;
|
||||
id: string | null;
|
||||
}
|
||||
|
||||
/**
|
||||
* How long (ms) a pinned participant may be absent from `userMedia$` before
|
||||
* the pin is dropped (#16). Long enough to ride out a transient LiveKit
|
||||
* reconnect, short enough that a real leave restores speaker-follows promptly.
|
||||
*/
|
||||
export const MANUAL_SPOTLIGHT_GONE_MS = 5000;
|
||||
|
||||
/**
|
||||
* Resolve the pin against the current media list (#30). Pure; the observable
|
||||
* wrapper below feeds it the live `speaking` state.
|
||||
*/
|
||||
export function resolveManualSpotlight<
|
||||
M extends { id: string; userId: string },
|
||||
>(
|
||||
target: ManualSpotlight | null,
|
||||
mediaItems: M[],
|
||||
speaking: (m: M) => boolean,
|
||||
): M | undefined {
|
||||
if (target === null) return undefined;
|
||||
if (target.id !== null) {
|
||||
const byId = mediaItems.find((m) => m.id === target.id);
|
||||
if (byId) return byId;
|
||||
// The pinned device is gone but the user may still be here on another
|
||||
// one — fall through to the userId match rather than dropping the pin.
|
||||
}
|
||||
const devices = mediaItems.filter((m) => m.userId === target.userId);
|
||||
if (devices.length <= 1) return devices[0];
|
||||
return devices.find(speaking) ?? devices[0];
|
||||
}
|
||||
|
||||
/**
|
||||
* The pinned media item, or `undefined` when there is no pin or the pinned
|
||||
* participant is not (currently) in the call.
|
||||
*/
|
||||
export function pinnedMedia$(
|
||||
manualSpotlight$: Observable<ManualSpotlight | null>,
|
||||
userMedia$: Observable<UserMediaViewModel[]>,
|
||||
): Observable<UserMediaViewModel | undefined> {
|
||||
return combineLatest([manualSpotlight$, userMedia$]).pipe(
|
||||
switchMap(([target, mediaItems]) => {
|
||||
if (target === null) return of(undefined);
|
||||
const devices = mediaItems.filter((m) => m.userId === target.userId);
|
||||
// Only subscribe to speaking$ when there is actually a choice to make.
|
||||
const speaking$: Observable<Set<UserMediaViewModel>> =
|
||||
devices.length > 1 && !devices.some((m) => m.id === target.id)
|
||||
? combineLatest(devices.map((m) => m.speaking$)).pipe(
|
||||
map((flags) => new Set(devices.filter((_, i) => flags[i]))),
|
||||
)
|
||||
: of(new Set<UserMediaViewModel>());
|
||||
return speaking$.pipe(
|
||||
map((speaking) =>
|
||||
resolveManualSpotlight(target, mediaItems, (m) => speaking.has(m)),
|
||||
),
|
||||
);
|
||||
}),
|
||||
distinctUntilChanged(),
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* [lotus #4] Manual spotlight override.
|
||||
*
|
||||
@@ -27,41 +112,35 @@ interface SpotlightAndPip {
|
||||
* `spotlightSpeaker$` auto-selection unchanged and just routes the
|
||||
* screenshare/spotlight computation through this wrapper at one call point.
|
||||
*
|
||||
* Behaviour is IDENTICAL to upstream whenever `manualSpotlightUserId$` stays
|
||||
* Behaviour is IDENTICAL to upstream whenever `manualSpotlight$` stays
|
||||
* `null` (the default). When it names a participant that is still present:
|
||||
* - with no screenshare, that participant is spotlighted instead of the
|
||||
* auto-selected active speaker;
|
||||
* - during a screenshare, that participant's camera is surfaced in the
|
||||
* spotlight ALONGSIDE the shared screen (#4 / A5 "focus camera during
|
||||
* screenshare"), and the redundant PiP is hidden.
|
||||
* screenshare"); the PiP keeps showing the auto-selected speaker and is
|
||||
* only blanked when that speaker IS the pinned participant (#29).
|
||||
*
|
||||
* @param autoSpotlightSpeaker$ upstream's speaker-follows auto selection
|
||||
* @param manualSpotlightUserId$ host-pinned userId, or `null` for auto (default)
|
||||
* @param manualSpotlight$ host pin, or `null` for auto (default)
|
||||
* @param screenShares$ current screen-share view models
|
||||
* @param localUserMediaForPip$ local media suitable for the PiP
|
||||
* @param userMedia$ all user media in the call (to resolve the pinned userId)
|
||||
* @param userMedia$ all user media in the call (to resolve the pin)
|
||||
*/
|
||||
export function overrideSpotlight$(
|
||||
autoSpotlightSpeaker$: Observable<UserMediaViewModel | undefined>,
|
||||
manualSpotlightUserId$: Observable<string | null>,
|
||||
manualSpotlight$: Observable<ManualSpotlight | null>,
|
||||
screenShares$: Observable<ScreenShareViewModel[]>,
|
||||
localUserMediaForPip$: Observable<LocalUserMediaViewModel | undefined>,
|
||||
userMedia$: Observable<UserMediaViewModel[]>,
|
||||
): Observable<SpotlightAndPip> {
|
||||
const pinned$ = pinnedMedia$(manualSpotlight$, userMedia$);
|
||||
|
||||
// The effective spotlight speaker: the host-pinned participant when set and
|
||||
// still present, otherwise upstream's auto-selected speaker.
|
||||
const spotlightSpeaker$ = combineLatest([
|
||||
autoSpotlightSpeaker$,
|
||||
manualSpotlightUserId$,
|
||||
userMedia$,
|
||||
]).pipe(
|
||||
map(([auto, manualUserId, mediaItems]) => {
|
||||
if (manualUserId !== null) {
|
||||
const pinned = mediaItems.find((m) => m.userId === manualUserId);
|
||||
if (pinned) return pinned;
|
||||
}
|
||||
return auto;
|
||||
}),
|
||||
const spotlightSpeaker$ = combineLatest(
|
||||
[autoSpotlightSpeaker$, pinned$],
|
||||
(auto, pinned) => pinned ?? auto,
|
||||
);
|
||||
|
||||
return screenShares$.pipe(
|
||||
@@ -72,16 +151,19 @@ export function overrideSpotlight$(
|
||||
// shared screen (the whole point of "focus camera during screenshare").
|
||||
// With no manual pin this is unchanged: the screenshare alone is
|
||||
// spotlighted.
|
||||
return combineLatest([manualSpotlightUserId$, userMedia$]).pipe(
|
||||
map(([manualUserId, mediaItems]) => {
|
||||
const pinned =
|
||||
manualUserId !== null
|
||||
? mediaItems.find((m) => m.userId === manualUserId)
|
||||
: undefined;
|
||||
return pinned
|
||||
? { spotlight: [...screenShares, pinned], pip$: of(undefined) }
|
||||
: { spotlight: screenShares, pip$: spotlightSpeaker$ };
|
||||
}),
|
||||
return pinned$.pipe(
|
||||
map((pinned) =>
|
||||
pinned
|
||||
? {
|
||||
spotlight: [...screenShares, pinned],
|
||||
// [lotus #29] Keep the active-speaker PiP; it is only
|
||||
// redundant when the speaker is the pinned participant.
|
||||
pip$: autoSpotlightSpeaker$.pipe(
|
||||
map((auto) => (auto === pinned ? undefined : auto)),
|
||||
),
|
||||
}
|
||||
: { spotlight: screenShares, pip$: spotlightSpeaker$ },
|
||||
),
|
||||
);
|
||||
|
||||
return spotlightSpeaker$.pipe(
|
||||
@@ -96,3 +178,84 @@ export function overrideSpotlight$(
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
export interface ManualSpotlightController {
|
||||
/** Pin `target`, or clear with `null`. */
|
||||
setManualSpotlight: (target: ManualSpotlight | null) => void;
|
||||
/**
|
||||
* Side effects that must run for the lifetime of the call. Emits nothing;
|
||||
* the caller subscribes it through its scope.
|
||||
*/
|
||||
effects$: Observable<never>;
|
||||
}
|
||||
|
||||
/**
|
||||
* [lotus #3/#16] State + side effects behind `CallViewModel.setManualSpotlight`.
|
||||
*
|
||||
* - #3: a pin only feeds `spotlight$`, which the default grid layout ignores
|
||||
* (and 1:1 layouts bypass entirely), so pinning forces the layout switch to
|
||||
* "spotlight" and remembers the mode it displaced; clearing the pin restores
|
||||
* that mode if the user has not switched layouts themselves meanwhile.
|
||||
* - #16: the pin is dropped when the pinned participant has been absent from
|
||||
* `userMedia$` for {@link MANUAL_SPOTLIGHT_GONE_MS} (a transient reconnect
|
||||
* is shorter than that), and on `leave$`.
|
||||
*
|
||||
* Pure with respect to its inputs so it can be unit-tested with fake
|
||||
* observables; CallViewModel only wires it up.
|
||||
*
|
||||
* @param manualSpotlight$ the pin state, owned by the caller (it has to exist
|
||||
* before the layout switch does, because `spotlightAndPip$` feeds into it)
|
||||
*/
|
||||
export function createManualSpotlight(
|
||||
manualSpotlight$: BehaviorSubject<ManualSpotlight | null>,
|
||||
userMedia$: Observable<{ userId: string }[]>,
|
||||
leave$: Observable<unknown>,
|
||||
layout: {
|
||||
layout$: Behavior<LayoutMode>;
|
||||
setLayout: (mode: LayoutMode) => void;
|
||||
},
|
||||
goneMs = MANUAL_SPOTLIGHT_GONE_MS,
|
||||
): ManualSpotlightController {
|
||||
// The layout mode we displaced by forcing spotlight, if any.
|
||||
let displacedLayout: LayoutMode | null = null;
|
||||
|
||||
const setManualSpotlight = (target: ManualSpotlight | null): void => {
|
||||
const hadPin = manualSpotlight$.value !== null;
|
||||
manualSpotlight$.next(target);
|
||||
if (target !== null) {
|
||||
if (!hadPin && layout.layout$.value !== "spotlight") {
|
||||
displacedLayout = layout.layout$.value;
|
||||
layout.setLayout("spotlight");
|
||||
}
|
||||
} else if (displacedLayout !== null) {
|
||||
// Only restore if the user hasn't picked a layout themselves since.
|
||||
if (layout.layout$.value === "spotlight")
|
||||
layout.setLayout(displacedLayout);
|
||||
displacedLayout = null;
|
||||
}
|
||||
};
|
||||
|
||||
// #16: pinned user has left for good (absent for `goneMs`), or we hung up.
|
||||
const pinnedGone$ = manualSpotlight$.pipe(
|
||||
switchMap((target) =>
|
||||
target === null
|
||||
? of(false)
|
||||
: userMedia$.pipe(
|
||||
map((items) => !items.some((m) => m.userId === target.userId)),
|
||||
distinctUntilChanged(),
|
||||
// A transient absence shorter than goneMs collapses back to
|
||||
// `false` before the timer fires and is ignored.
|
||||
debounceTime(goneMs),
|
||||
filter((gone) => gone),
|
||||
),
|
||||
),
|
||||
filter((gone) => gone),
|
||||
);
|
||||
const effects$ = merge(pinnedGone$, leave$).pipe(
|
||||
filter(() => manualSpotlight$.value !== null),
|
||||
tap(() => setManualSpotlight(null)),
|
||||
ignoreElements(),
|
||||
);
|
||||
|
||||
return { setManualSpotlight, effects$ };
|
||||
}
|
||||
|
||||
@@ -0,0 +1,85 @@
|
||||
/*
|
||||
Copyright 2026 Lotus Guild
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { afterEach, describe, expect, test, vi } from "vitest";
|
||||
|
||||
vi.mock("../widget", () => ({ widget: null }));
|
||||
|
||||
import { lotusFlag, lotusParam, lotusSendToHost } from "./lotusWidget";
|
||||
import { LotusWidgetActions } from "./lotusActions";
|
||||
|
||||
/**
|
||||
* Point `window.location` at a URL for the duration of one assertion. Uses a
|
||||
* path relative to the current origin — `pushState` throws a `SecurityError`
|
||||
* on a cross-origin URL, and jsdom's default test origin doesn't match a
|
||||
* hard-coded `http://localhost/`.
|
||||
*/
|
||||
function setLocation(path: string): void {
|
||||
window.history.pushState({}, "", path);
|
||||
}
|
||||
|
||||
describe("lotusParam / lotusFlag", () => {
|
||||
afterEach(() => {
|
||||
setLocation("/");
|
||||
});
|
||||
|
||||
test("reads a param from the query string", () => {
|
||||
setLocation("/?lotusTheme=1");
|
||||
expect(lotusParam("lotusTheme")).toBe("1");
|
||||
});
|
||||
|
||||
test("reads a param from the hash fragment's query portion", () => {
|
||||
setLocation("/#/room?lotusTransparent=1");
|
||||
expect(lotusParam("lotusTransparent")).toBe("1");
|
||||
});
|
||||
|
||||
test("the hash fragment wins over the query string for the same key", () => {
|
||||
setLocation("/?lotusTheme=fromQuery#/room?lotusTheme=fromHash");
|
||||
expect(lotusParam("lotusTheme")).toBe("fromHash");
|
||||
});
|
||||
|
||||
test("falls back to the query string for keys absent from the hash", () => {
|
||||
setLocation("/?onlyInQuery=1#/room?lotusTheme=1");
|
||||
expect(lotusParam("onlyInQuery")).toBe("1");
|
||||
});
|
||||
|
||||
test("returns null when the param is present nowhere", () => {
|
||||
setLocation("/#/room");
|
||||
expect(lotusParam("missing")).toBeNull();
|
||||
});
|
||||
|
||||
test.each([
|
||||
["1", true],
|
||||
["true", true],
|
||||
["0", false],
|
||||
["false", false],
|
||||
[null, false],
|
||||
])("lotusFlag treats %s as %s", (value, expected) => {
|
||||
setLocation(value === null ? "/" : `/?f=${value}`);
|
||||
expect(lotusFlag("f")).toBe(expected);
|
||||
});
|
||||
|
||||
// [lotus #33] lotusParam must re-derive from window.location on every call
|
||||
// rather than caching the first read, so a later navigation (e.g. lobby ->
|
||||
// in-call) that rewrites the URL is picked up.
|
||||
test("re-reads window.location on every call instead of caching the first result", () => {
|
||||
setLocation("/?lotusTheme=1");
|
||||
expect(lotusParam("lotusTheme")).toBe("1");
|
||||
|
||||
setLocation("/?lotusTheme=0");
|
||||
expect(lotusParam("lotusTheme")).toBe("0");
|
||||
|
||||
setLocation("/");
|
||||
expect(lotusParam("lotusTheme")).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe("lotusSendToHost", () => {
|
||||
test("returns false when no widget transport is available", () => {
|
||||
expect(lotusSendToHost(LotusWidgetActions.CallState, {})).toBe(false);
|
||||
});
|
||||
});
|
||||
+13
-14
@@ -21,27 +21,26 @@ import type { LotusWidgetActions } from "./lotusActions";
|
||||
|
||||
export { LotusWidgetActions } from "./lotusActions";
|
||||
|
||||
let cachedParams: URLSearchParams | undefined;
|
||||
|
||||
/**
|
||||
* Read a URL param from either the query string or the hash fragment (Element
|
||||
* Call passes widget params via both depending on host), without depending on
|
||||
* EC's own `getUrlParams` parser (keeps the rebase surface small).
|
||||
*
|
||||
* [lotus #33] Re-parsed from `window.location` on every call instead of being
|
||||
* cached at first use: parsing is cheap, and caching risked returning a
|
||||
* stale value if `location.hash`/`search` are ever rewritten after first
|
||||
* read (e.g. during in-iframe navigation).
|
||||
*/
|
||||
export function lotusParam(name: string): string | null {
|
||||
if (!cachedParams) {
|
||||
// Match EC's own ParamParser precedence: the hash fragment wins over the
|
||||
// query string. So seed from the fragment first, then fill gaps from query.
|
||||
const hash = window.location.hash.replace(/^#\/?/, "");
|
||||
const hashQuery = hash.includes("?")
|
||||
? hash.slice(hash.indexOf("?") + 1)
|
||||
: "";
|
||||
cachedParams = new URLSearchParams(hashQuery);
|
||||
for (const [k, v] of new URLSearchParams(window.location.search)) {
|
||||
if (!cachedParams.has(k)) cachedParams.append(k, v);
|
||||
}
|
||||
// Match EC's own ParamParser precedence: the hash fragment wins over the
|
||||
// query string. So seed from the fragment first, then fill gaps from query.
|
||||
const hash = window.location.hash.replace(/^#\/?/, "");
|
||||
const hashQuery = hash.includes("?") ? hash.slice(hash.indexOf("?") + 1) : "";
|
||||
const params = new URLSearchParams(hashQuery);
|
||||
for (const [k, v] of new URLSearchParams(window.location.search)) {
|
||||
if (!params.has(k)) params.append(k, v);
|
||||
}
|
||||
return cachedParams.get(name);
|
||||
return params.get(name);
|
||||
}
|
||||
|
||||
/** Whether a boolean-ish Lotus feature flag is enabled. */
|
||||
|
||||
@@ -158,7 +158,11 @@ import { type LocalUserMediaViewModel } from "../media/LocalUserMediaViewModel.t
|
||||
import { type RemoteUserMediaViewModel } from "../media/RemoteUserMediaViewModel.ts";
|
||||
// [lotus #4] Manual spotlight override, extracted so this file stays byte-close
|
||||
// to upstream (see the file for the behaviour contract).
|
||||
import { overrideSpotlight$ } from "../../lotus/lotusSpotlight";
|
||||
import {
|
||||
createManualSpotlight,
|
||||
type ManualSpotlight,
|
||||
overrideSpotlight$,
|
||||
} from "../../lotus/lotusSpotlight";
|
||||
import {
|
||||
createRingingMedia,
|
||||
type RingingMediaViewModel,
|
||||
@@ -267,10 +271,11 @@ export interface CallViewModel {
|
||||
*/
|
||||
toggleScreenSharing: (() => void) | null;
|
||||
/**
|
||||
* [lotus] Pin a participant to the spotlight by Matrix user id (#4
|
||||
* focus-participant). Pass null to clear and restore speaker-follows.
|
||||
* [lotus] Pin a participant to the spotlight (#4 focus-participant) by
|
||||
* Matrix user id and, optionally, EC media id (#30). Pass null to clear and
|
||||
* restore speaker-follows.
|
||||
*/
|
||||
setManualSpotlight: (userId: string | null) => void;
|
||||
setManualSpotlight: (target: ManualSpotlight | null) => void;
|
||||
/**
|
||||
* Whether we are sharing our screen.
|
||||
*/
|
||||
@@ -955,11 +960,6 @@ export function createCallViewModel$(
|
||||
),
|
||||
);
|
||||
|
||||
// [lotus #4] Host-pinned spotlight target (io.lotus.focus_participant). null =
|
||||
// follow the active speaker (upstream default), so this is inert unless the
|
||||
// host pins someone. Consumed by overrideSpotlight$ in spotlightAndPip$.
|
||||
const manualSpotlightUserId$ = new BehaviorSubject<string | null>(null);
|
||||
|
||||
const grid$ = scope.behavior<UserMediaViewModel[]>(
|
||||
userMedia$.pipe(
|
||||
switchMap((mediaItems) => {
|
||||
@@ -1000,6 +1000,12 @@ export function createCallViewModel$(
|
||||
),
|
||||
);
|
||||
|
||||
// [lotus #4] Host-pinned spotlight target (io.lotus.focus_participant). null =
|
||||
// follow the active speaker (upstream default), so this is inert unless the
|
||||
// host pins someone. Consumed by overrideSpotlight$ in spotlightAndPip$;
|
||||
// driven by createManualSpotlight below (after the layout switch exists).
|
||||
const manualSpotlight$ = new BehaviorSubject<ManualSpotlight | null>(null);
|
||||
|
||||
const spotlightAndPip$ = scope.behavior<{
|
||||
spotlight: MediaViewModel[];
|
||||
pip$: Observable<UserMediaViewModel | undefined>;
|
||||
@@ -1016,7 +1022,7 @@ export function createCallViewModel$(
|
||||
// stays close to upstream and rebases cleanly.
|
||||
return overrideSpotlight$(
|
||||
spotlightSpeaker$,
|
||||
manualSpotlightUserId$,
|
||||
manualSpotlight$,
|
||||
screenShares$,
|
||||
localUserMediaForPip$,
|
||||
userMedia$,
|
||||
@@ -1088,12 +1094,31 @@ export function createCallViewModel$(
|
||||
hasRemoteScreenShares$,
|
||||
);
|
||||
|
||||
// [lotus #3/#16] Pinning forces the layout switch to "spotlight" (grid and
|
||||
// 1:1 layouts would otherwise never render the pin) and the pin is dropped
|
||||
// when the participant leaves for good or we hang up; see
|
||||
// src/lotus/lotusSpotlight.ts.
|
||||
const manualSpotlight = createManualSpotlight(
|
||||
manualSpotlight$,
|
||||
userMedia$,
|
||||
leave$,
|
||||
layoutSwitchVm,
|
||||
);
|
||||
manualSpotlight.effects$.pipe(scope.bind()).subscribe();
|
||||
|
||||
const gridLayoutMedia$: Observable<GridLayoutMedia> = combineLatest(
|
||||
[grid$, spotlight$],
|
||||
(grid, spotlight) => ({
|
||||
[grid$, spotlight$, manualSpotlight$],
|
||||
(grid, spotlight, manual) => ({
|
||||
type: "grid",
|
||||
edgeToEdge: false,
|
||||
spotlight: spotlight.some((vm) => vm.type === "screen share")
|
||||
// [lotus #3] Also surface the spotlight when it holds a host-pinned
|
||||
// participant, so the pin is visible in window modes that ignore the
|
||||
// layout switch (narrow). Unchanged when manual is null.
|
||||
spotlight: spotlight.some(
|
||||
(vm) =>
|
||||
vm.type === "screen share" ||
|
||||
(manual !== null && vm.userId === manual.userId),
|
||||
)
|
||||
? spotlight
|
||||
: undefined,
|
||||
grid,
|
||||
@@ -1796,8 +1821,7 @@ export function createCallViewModel$(
|
||||
join: localMembership.requestJoinAndPublish,
|
||||
leave: localMembership.requestDisconnect,
|
||||
toggleScreenSharing: toggleScreenSharing,
|
||||
setManualSpotlight: (userId: string | null): void =>
|
||||
manualSpotlightUserId$.next(userId),
|
||||
setManualSpotlight: manualSpotlight.setManualSpotlight,
|
||||
sharingScreen$: sharingScreen$,
|
||||
|
||||
tapScreen: (): void => screenTap$.next(),
|
||||
|
||||
@@ -183,6 +183,13 @@ function generateRoomOption({
|
||||
audioCaptureDefaults: {
|
||||
...liveKitOptions.audioCaptureDefaults,
|
||||
deviceId: devices.audioInput.selected$.value?.id,
|
||||
// [lotus] Voice is mono. Without this, Firefox captures a 2-channel
|
||||
// audio interface (e.g. a Scarlett Solo with one XLR mic on input 1) as
|
||||
// stereo and, with browser audio processing off, publishes it that way —
|
||||
// peers hear the speaker in the left ear only. Chrome downmixes such
|
||||
// captures itself, which is why it only showed on Firefox. Screenshare
|
||||
// audio is captured separately and is unaffected.
|
||||
channelCount: 1,
|
||||
echoCancellation:
|
||||
echoCancellationSetting.getValue() &&
|
||||
getUrlParams().echoCancellation !== false,
|
||||
|
||||
@@ -120,8 +120,11 @@ Please see LICENSE in the repository root for full details.
|
||||
opacity: 50%;
|
||||
}
|
||||
|
||||
/* [lotus #6] Profile decoration overlaid on the tile avatar. Shares the
|
||||
avatar's centred box and size so frame-style decorations sit around it. */
|
||||
/* [lotus #4] Profile decoration overlaid on the tile avatar. Sized larger than
|
||||
the avatar's box (62cqmin vs. the avatar's 50cqmin — matching cinny's own
|
||||
~50px avatar + 8px outward inset ratio, `AvatarDecoration.tsx`'s
|
||||
DEFAULT_INSET) so frame-style decoration artwork bleeds outside the avatar
|
||||
circle and surrounds it, instead of overlapping/clipping it 1:1. */
|
||||
.lotusDecoration {
|
||||
position: absolute;
|
||||
top: 50%;
|
||||
@@ -131,10 +134,19 @@ avatar's centred box and size so frame-style decorations sit around it. */
|
||||
object-fit: contain;
|
||||
}
|
||||
|
||||
/* [lotus #19] Decorations are animated APNGs with no static asset to freeze
|
||||
to; hide them under prefers-reduced-motion, matching cinny's own
|
||||
AvatarDecoration guard and the host's behaviour of rendering just the avatar. */
|
||||
@media (prefers-reduced-motion: reduce) {
|
||||
.lotusDecoration {
|
||||
display: none;
|
||||
}
|
||||
}
|
||||
|
||||
@container mediaView (width > 0) {
|
||||
.lotusDecoration {
|
||||
inline-size: 50cqmin;
|
||||
block-size: 50cqmin;
|
||||
inline-size: 62cqmin;
|
||||
block-size: 62cqmin;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -113,6 +113,15 @@ export const MediaView: FC<Props> = ({
|
||||
}) => {
|
||||
const { t } = useTranslation();
|
||||
const decoration = useLotusDecoration(userId);
|
||||
// [lotus #4] Track the URL of a decoration that failed to load (e.g. a 404
|
||||
// from the CDN, reachable given the unvalidated slug the host builds it
|
||||
// from) so we can hide the broken-image box instead of leaving it over the
|
||||
// avatar. Comparing against the current `decoration` (rather than a bare
|
||||
// boolean) means a new/changed decoration URL automatically gets a fresh
|
||||
// chance to load.
|
||||
const [erroredDecoration, setErroredDecoration] = useState<
|
||||
string | undefined
|
||||
>(undefined);
|
||||
const [handRaiseTimerVisible] = useSetting(showHandRaisedTimer);
|
||||
const [showConnectionStats] = useSetting(showConnectionStatsSetting);
|
||||
|
||||
@@ -188,15 +197,19 @@ export const MediaView: FC<Props> = ({
|
||||
style={{ display: video && videoEnabled ? "none" : "initial" }}
|
||||
/>
|
||||
{decoration &&
|
||||
decoration !== erroredDecoration &&
|
||||
!(video && videoEnabled) && (
|
||||
// [lotus #6] Profile decoration overlay, shown only when the avatar
|
||||
// is visible (i.e. not when live video is showing). Pushed by the
|
||||
// host via io.lotus.decorations; undefined unless opted in.
|
||||
// [lotus #4] Hidden entirely under prefers-reduced-motion (CSS) and
|
||||
// on a load error (onError), matching cinny's own guards.
|
||||
<img
|
||||
className={styles.lotusDecoration}
|
||||
src={decoration}
|
||||
alt=""
|
||||
aria-hidden
|
||||
onError={() => setErroredDecoration(decoration)}
|
||||
/>
|
||||
)}
|
||||
{video?.publication !== undefined && (
|
||||
|
||||
+22
-4
@@ -6,7 +6,7 @@ Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { logger } from "matrix-js-sdk/lib/logger";
|
||||
import { useState, useEffect } from "react";
|
||||
import { useState, useEffect, useRef } from "react";
|
||||
import { useObservableEagerState } from "observable-hooks";
|
||||
|
||||
import {
|
||||
@@ -172,13 +172,22 @@ export function useAudioContext<S extends string>(
|
||||
useMediaDevices().audioOutput.selected$,
|
||||
)?.id;
|
||||
const { controlledAudioDevices } = useUrlParams();
|
||||
// [lotus] Tracks whether audioOutputId has ever resolved to a real device,
|
||||
// so we can tell "MediaDevices hasn't resolved yet" (skip) apart from "the
|
||||
// previously selected output device just disappeared" (revert to default,
|
||||
// #27) — both present as audioOutputId === undefined.
|
||||
const outputHasResolved = useRef(false);
|
||||
|
||||
// Update the sink ID whenever we change devices.
|
||||
useEffect(() => {
|
||||
if (typeof audioOutputId === "string") outputHasResolved.current = true;
|
||||
if (
|
||||
!audioContext ||
|
||||
!("setSinkId" in audioContext) ||
|
||||
controlledAudioDevices
|
||||
)
|
||||
return;
|
||||
if (
|
||||
audioContext &&
|
||||
"setSinkId" in audioContext &&
|
||||
!controlledAudioDevices &&
|
||||
// Skip until a device is actually selected. audioOutputId is undefined
|
||||
// before MediaDevices resolves (e.g. on the Tauri desktop webview, where
|
||||
// the selected$ observable emits undefined first); setSinkId(undefined)
|
||||
@@ -191,6 +200,15 @@ export function useAudioContext<S extends string>(
|
||||
audioContext.setSinkId(audioOutputId).catch((ex) => {
|
||||
logger.warn("Unable to change sink for audio context", ex);
|
||||
});
|
||||
} else if (outputHasResolved.current) {
|
||||
// [lotus] The previously selected output device disappeared (selected$
|
||||
// re-emits undefined once devices have already resolved), not the
|
||||
// pre-resolution case above. Revert to the default sink instead of
|
||||
// leaving the context pinned to a now-nonexistent device (#27).
|
||||
// @ts-expect-error - setSinkId doesn't exist yet in types, maybe because it's not supported everywhere.
|
||||
audioContext.setSinkId("").catch((ex) => {
|
||||
logger.warn("Unable to revert sink for audio context", ex);
|
||||
});
|
||||
}
|
||||
}, [audioContext, audioOutputId, controlledAudioDevices]);
|
||||
const { pan: earpiecePan, volume: earpieceVolume } = useEarpieceAudioConfig();
|
||||
|
||||
+18
-3
@@ -8,6 +8,7 @@ Please see LICENSE in the repository root for full details.
|
||||
import { useEffect, useLayoutEffect, useRef, useState } from "react";
|
||||
import { WidgetApiToWidgetAction } from "matrix-widget-api";
|
||||
import { type IThemeChangeActionRequest } from "matrix-widget-api";
|
||||
import { logger } from "matrix-js-sdk/lib/logger";
|
||||
|
||||
import { getUrlParams } from "./UrlParams";
|
||||
import { widget } from "./widget";
|
||||
@@ -60,8 +61,22 @@ export const useTheme = (): void => {
|
||||
document.body.classList.remove("no-theme");
|
||||
// [lotus #5] Native theming hooks, opted in by the host via URL flags, so
|
||||
// it no longer has to inject CSS into the iframe after load.
|
||||
if (lotusFlag("lotusTransparent"))
|
||||
document.body.classList.add("lotus-transparent");
|
||||
if (lotusFlag("lotusTheme")) document.body.classList.add("lotus-theme");
|
||||
const lotusTransparent = lotusFlag("lotusTransparent");
|
||||
const lotusTheme = lotusFlag("lotusTheme");
|
||||
if (lotusTransparent) document.body.classList.add("lotus-transparent");
|
||||
// [lotus #21] The two flags are only meaningful together: a transparent
|
||||
// canvas with no matching theme leaves text/icon colours computed against
|
||||
// the default dark canvas token while the real pixels behind them are
|
||||
// whatever the host painted. If a host sets lotusTransparent without
|
||||
// lotusTheme, warn and apply lotusTheme anyway rather than silently
|
||||
// risking unreadable UI.
|
||||
if (lotusTransparent && !lotusTheme) {
|
||||
logger.warn(
|
||||
"[lotus] lotusTransparent is set without lotusTheme; applying " +
|
||||
"lotusTheme anyway since the two flags are only meaningful together.",
|
||||
);
|
||||
}
|
||||
if (lotusTheme || lotusTransparent)
|
||||
document.body.classList.add("lotus-theme");
|
||||
}, [previousTheme, requestedTheme]);
|
||||
};
|
||||
|
||||
+18
-2
@@ -115,14 +115,30 @@ export const initializeWidget = (
|
||||
ElementWidgetActions.JoinCall,
|
||||
ElementWidgetActions.HangupCall,
|
||||
ElementWidgetActions.DeviceMute,
|
||||
// [lotus] custom toWidget actions handled by the fork (focus, audio-inject)
|
||||
...LOTUS_TO_WIDGET_ACTIONS,
|
||||
].forEach((action) => {
|
||||
api.on(`action:${action}`, (ev: CustomEvent<IWidgetApiRequest>) => {
|
||||
ev.preventDefault();
|
||||
lazyActions.emit(action, ev);
|
||||
});
|
||||
});
|
||||
// [lotus] custom toWidget actions handled by the fork (focus,
|
||||
// audio-inject, decorations, ...). Unlike the upstream actions above
|
||||
// (whose handlers are process-lifetime), lotus handlers are
|
||||
// registered/torn down with their owning React effect, so a request can
|
||||
// arrive while none is mounted. `LazyEventEmitter.emit` would otherwise
|
||||
// backlog it forever (never replied to, replayed stale on the next
|
||||
// registration — see LazyEventEmitter), and the host's
|
||||
// `transport.send` would hang until its own timeout. Reply immediately
|
||||
// with `{}` when there's no handler instead, so a torn-down lotus
|
||||
// action behaves like a no-op rather than a silent stall.
|
||||
LOTUS_TO_WIDGET_ACTIONS.forEach((action) => {
|
||||
api.on(`action:${action}`, (ev: CustomEvent<IWidgetApiRequest>) => {
|
||||
ev.preventDefault();
|
||||
if (!lazyActions.emit(action, ev)) {
|
||||
api.transport.reply(ev.detail, {});
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
// Now, initialize the matryoshka MatrixClient (so named because it routes
|
||||
// all requests through the host client via the widget API)
|
||||
|
||||
Reference in New Issue
Block a user