From f793523e03bfd696e174748ef26392818a9a16fa Mon Sep 17 00:00:00 2001 From: Jared Vititoe Date: Mon, 9 Nov 2026 01:10:33 -0500 Subject: [PATCH] feat(spotlight): flip between cameras as well as screenshares during a share (#67) During a screenshare the spotlight carousel holds the shares first, then the host-pinned camera, then every other remote camera that is on (join order). The share stays the default view; the view falls back to the first share when the visible camera turns off or leaves. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PPmy3tPq869XDW4njjVaKA --- playwright/widget/screen-share.test.ts | 18 ++++-- src/lotus/lotusSpotlight.test.ts | 79 +++++++++++++++++++++++ src/lotus/lotusSpotlight.ts | 78 ++++++++++++++++++++--- src/lotus/lotusSpotlightView.test.ts | 42 +++++++++++++ src/lotus/lotusSpotlightView.ts | 72 +++++++++++++++++++++ src/tile/SpotlightTile.test.tsx | 87 ++++++++++++++++++++++++++ src/tile/SpotlightTile.tsx | 6 ++ 7 files changed, 367 insertions(+), 15 deletions(-) create mode 100644 src/lotus/lotusSpotlightView.test.ts create mode 100644 src/lotus/lotusSpotlightView.ts diff --git a/playwright/widget/screen-share.test.ts b/playwright/widget/screen-share.test.ts index c6b03c3f..8b99587f 100644 --- a/playwright/widget/screen-share.test.ts +++ b/playwright/widget/screen-share.test.ts @@ -71,8 +71,9 @@ widgetTest("Sharing screen in group call", async ({ addUser, browserName }) => { .locator('iframe[title="Element Call"]') .contentFrame(); - // Expect 4 (3 + screen share) video tiles - await expect(frame.locator("video")).toHaveCount(4, { + // Expect 6 video tiles: 3 cameras in the grid, plus the spotlight + // carousel's screen share and the 2 remote cameras (lotus #67) + await expect(frame.locator("video")).toHaveCount(6, { timeout: 5000, }); @@ -112,8 +113,9 @@ widgetTest("Sharing screen in group call", async ({ addUser, browserName }) => { .locator('iframe[title="Element Call"]') .contentFrame(); - // Expect 5 (2 + screen share) video tiles - await expect(frame.locator("video")).toHaveCount(5, { + // Expect 7 video tiles: 3 cameras in the grid, plus the spotlight + // carousel's 2 screen shares and 2 remote cameras (lotus #67) + await expect(frame.locator("video")).toHaveCount(7, { timeout: 5000, }); @@ -121,8 +123,12 @@ widgetTest("Sharing screen in group call", async ({ addUser, browserName }) => { frame.locator('video[data-lk-source="screen_share"]'), ).toHaveCount(2); - // Expect 2 indicators at the bottom - await expect(frame.getByTestId("screenshare-indicator")).toHaveCount(2); + // Expect 4 indicators at the bottom: the 2 shares first, then the 2 + // remote cameras (lotus #67) + await expect(frame.getByTestId("screenshare-indicator")).toHaveCount(4); + await expect( + frame.locator('[data-testid="screenshare-indicator"][data-kind="user"]'), + ).toHaveCount(2); // Check the first indicator is visible await expect( diff --git a/src/lotus/lotusSpotlight.test.ts b/src/lotus/lotusSpotlight.test.ts index 0f9897aa..ee6ab956 100644 --- a/src/lotus/lotusSpotlight.test.ts +++ b/src/lotus/lotusSpotlight.test.ts @@ -18,6 +18,7 @@ import { overrideSpotlight$, pinnedMedia$, resolveManualSpotlight, + spotlightCameras, } from "./lotusSpotlight"; import { parseFocusPayload } from "./lotusFocus"; @@ -27,6 +28,7 @@ interface FakeMedia { type: "user"; local: boolean; speaking$: BehaviorSubject; + videoEnabled$: BehaviorSubject; } function media( @@ -34,6 +36,7 @@ function media( deviceId: string, speaking = false, local = false, + video = false, ): FakeMedia { return { id: `${userId}:${deviceId}`, @@ -41,6 +44,7 @@ function media( type: "user", local, speaking$: new BehaviorSubject(speaking), + videoEnabled$: new BehaviorSubject(video), }; } const asUser = (m: FakeMedia): UserMediaViewModel => @@ -186,6 +190,81 @@ describe("overrideSpotlight$", () => { }); }); +describe("spotlightCameras (#67)", () => { + const me = media("@me:x", "d", false, true, true); + const carol = media("@carol:x", "d", false, false, true); + const dave = media("@dave:x", "d", false, false, true); + const users = [me, carol, bob, dave]; + + test("shares first, then remote cameras that are on, in order", () => { + expect( + spotlightCameras([screen], users, new Set([me, carol, dave]), undefined), + ).toEqual([screen, carol, dave]); + }); + + test("the pin sits right after the shares and appears once", () => { + expect( + spotlightCameras([screen], users, new Set([carol, dave]), dave), + ).toEqual([screen, dave, carol]); + }); + + test("the pinned camera is kept even with its video off", () => { + expect(spotlightCameras([screen], users, new Set([carol]), bob)).toEqual([ + screen, + bob, + carol, + ]); + }); +}); + +describe("overrideSpotlight$ cameras during a share (#67)", () => { + test("cameras join and leave the carousel live", () => { + const carol = media("@carol:x", "d", false, false, true); + const me = media("@me:x", "d", false, true, true); + // Own fixture: other tests leave subscriptions on the shared `bob`. + const erin = media("@erin:x", "d"); + const users$ = new BehaviorSubject([ + asUser(me), + asUser(carol), + asUser(erin), + ]); + const seen: unknown[][] = []; + const sub = overrideSpotlight$( + of(asUser(carol)), + of(null), + of([screen as unknown as ScreenShareViewModel]), + of(undefined), + users$, + ).subscribe((r) => seen.push(r.spotlight)); + + erin.videoEnabled$.next(true); // erin turns their camera on + carol.videoEnabled$.next(false); // carol turns hers off + users$.next([asUser(me), asUser(carol)]); // erin leaves + erin.videoEnabled$.next(false); + sub.unsubscribe(); + + expect(seen).toEqual([ + [screen, carol], + [screen, carol, erin], + [screen, erin], + [screen], + ]); + }); + + test("no share: cameras are not added (upstream speaker-follows)", () => { + const carol = media("@carol:x", "d", false, false, true); + let spotlight; + overrideSpotlight$( + of(asUser(bob)), + of(null), + of([]), + of(undefined), + of([asUser(carol), asUser(bob)]), + ).subscribe((r) => (spotlight = r.spotlight)); + expect(spotlight).toEqual([bob]); + }); +}); + describe("createManualSpotlight (#3 / #16)", () => { function setup(initialLayout: LayoutMode = "grid") { const pin$ = new BehaviorSubject(null); diff --git a/src/lotus/lotusSpotlight.ts b/src/lotus/lotusSpotlight.ts index 7827df6f..a8511dc3 100644 --- a/src/lotus/lotusSpotlight.ts +++ b/src/lotus/lotusSpotlight.ts @@ -26,6 +26,7 @@ import { type MediaViewModel } from "../state/media/MediaViewModel"; import { type LocalUserMediaViewModel } from "../state/media/LocalUserMediaViewModel"; import { type LayoutMode } from "../state/LayoutSwitchViewModel"; import { type Behavior } from "../state/Behavior"; +import { shallowEquals } from "../utils/array"; interface SpotlightAndPip { spotlight: MediaViewModel[]; @@ -102,6 +103,44 @@ export function pinnedMedia$( ); } +/** + * [lotus #67] The spotlight carousel during a screenshare: every share first + * (upstream order), then the host-pinned camera if any (included even with + * its video off, as before #67), then every other REMOTE camera whose video + * is on, in `userMedia` order so the dots don't reshuffle. Each camera + * appears once. The local camera is left out: the PiP and grid show it. + */ +export function spotlightCameras( + screenShares: S[], + userMedia: U[], + videoOn: ReadonlySet, + pinned: U | undefined, +): (S | U)[] { + const cameras = userMedia.filter( + (m) => m !== pinned && !m.local && videoOn.has(m), + ); + return [...screenShares, ...(pinned ? [pinned] : []), ...cameras]; +} + +/** + * The remote user media whose camera is currently on, as a live set. Only + * subscribes to `videoEnabled$` of remote items (the local camera never goes + * in the carousel). + */ +function remoteVideoOn$( + userMedia$: Observable, +): Observable> { + return userMedia$.pipe( + switchMap((items) => { + const remote = items.filter((m) => !m.local); + if (remote.length === 0) return of(new Set()); + return combineLatest(remote.map((m) => m.videoEnabled$)).pipe( + map((flags) => new Set(remote.filter((_, i) => flags[i]))), + ); + }), + ); +} + /** * [lotus #4] Manual spotlight override. * @@ -121,6 +160,12 @@ export function pinnedMedia$( * screenshare"); the PiP keeps showing the auto-selected speaker and is * only blanked when that speaker IS the pinned participant (#29). * + * [lotus #67] During a screenshare the spotlight also holds every remote + * camera that is on (after the shares and the pin, see + * {@link spotlightCameras}), so viewers can flip to a camera with the same + * arrows/dots they use between shares. The share stays the default view; + * SpotlightTile handles that via `useLotusSpotlightFallback`. + * * @param autoSpotlightSpeaker$ upstream's speaker-follows auto selection * @param manualSpotlight$ host pin, or `null` for auto (default) * @param screenShares$ current screen-share view models @@ -146,23 +191,38 @@ export function overrideSpotlight$( return screenShares$.pipe( switchMap((screenShares) => { if (screenShares.length > 0) - // During a screenshare, if the host has explicitly pinned a - // participant, surface that camera in the spotlight alongside the - // shared screen (the whole point of "focus camera during screenshare"). - // With no manual pin this is unchanged: the screenshare alone is - // spotlighted. - return pinned$.pipe( - map((pinned) => + // During a screenshare the spotlight holds the shares, then the + // host-pinned camera (#4, "focus camera during screenshare"), then the + // other remote cameras that are on (#67). + return combineLatest([ + pinned$, + userMedia$, + remoteVideoOn$(userMedia$), + ]).pipe( + map(([pinned, userMedia, videoOn]) => ({ + pinned, + spotlight: spotlightCameras( + screenShares, + userMedia, + videoOn, + pinned, + ), + })), + distinctUntilChanged( + (a, b) => + a.pinned === b.pinned && shallowEquals(a.spotlight, b.spotlight), + ), + map(({ pinned, spotlight }) => pinned ? { - spotlight: [...screenShares, pinned], + spotlight, // [lotus #29] Keep the active-speaker PiP; it is only // redundant when the speaker is the pinned participant. pip$: autoSpotlightSpeaker$.pipe( map((auto) => (auto === pinned ? undefined : auto)), ), } - : { spotlight: screenShares, pip$: spotlightSpeaker$ }, + : { spotlight, pip$: spotlightSpeaker$ }, ), ); diff --git a/src/lotus/lotusSpotlightView.test.ts b/src/lotus/lotusSpotlightView.test.ts new file mode 100644 index 00000000..7c4c270d --- /dev/null +++ b/src/lotus/lotusSpotlightView.test.ts @@ -0,0 +1,42 @@ +/* +Copyright 2026 Lotus Guild + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { describe, expect, test } from "vitest"; + +import { spotlightFallbackTarget } from "./lotusSpotlightView"; + +const share = { id: "s", type: "screen share" }; +const share2 = { id: "s2", type: "screen share" }; +const cam = { id: "c", type: "user" }; +const cam2 = { id: "c2", type: "user" }; + +describe("spotlightFallbackTarget (#67)", () => { + test("nothing to do while the visible item is still there", () => { + expect(spotlightFallbackTarget([share, cam], "c", true)).toBeNull(); + expect(spotlightFallbackTarget([share, cam], "s", true)).toBeNull(); + }); + + test("visible camera gone: fall back to the first share", () => { + expect(spotlightFallbackTarget([share, cam2], "c", true)).toBe("s"); + }); + + test("visible share ended: fall back to the remaining first item", () => { + expect(spotlightFallbackTarget([share2, cam], "s", true)).toBe("s2"); + }); + + test("share started while a camera was spotlighted: jump to the share", () => { + expect(spotlightFallbackTarget([share, cam], "c", false)).toBe("s"); + }); + + test("no share and no change: speaker-follows is left alone", () => { + expect(spotlightFallbackTarget([cam2], "c2", false)).toBeNull(); + }); + + test("empty media: nothing to do", () => { + expect(spotlightFallbackTarget([], "c", true)).toBeNull(); + }); +}); diff --git a/src/lotus/lotusSpotlightView.ts b/src/lotus/lotusSpotlightView.ts new file mode 100644 index 00000000..0e93060e --- /dev/null +++ b/src/lotus/lotusSpotlightView.ts @@ -0,0 +1,72 @@ +/* +Copyright 2026 Lotus Guild + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { useEffect, useRef } from "react"; + +/** + * What the spotlight carousel should jump to after `media` changes, or `null` + * to leave it alone. Pure; see {@link useLotusSpotlightFallback}. + * + * - The visible item is gone (its camera turned off, its owner left, or a + * share ended): show the first item rather than whichever neighbour the + * scroll happens to land on. + * - A share has just started (the first item became a screen share): show + * it, even if the previously spotlighted camera is still in the list. + */ +export function spotlightFallbackTarget( + media: readonly { id: string; type: string }[], + visibleId: string | undefined, + firstWasShare: boolean, +): string | null { + const first = media[0]; + if (first === undefined) return null; + const shareStarted = first.type === "screen share" && !firstWasShare; + const visibleGone = + visibleId !== undefined && !media.some((m) => m.id === visibleId); + if (!shareStarted && !visibleGone) return null; + return first.id === visibleId ? null : first.id; +} + +/** + * [lotus #67] Keeps the spotlight carousel on a sensible item now that it can + * hold cameras as well as screenshares during a share (see + * `spotlightCameras` in lotusSpotlight.ts). + * + * Scrolls the item into view directly and also sets `visibleId`: when the + * target is already at the scroll position (e.g. the visible item was the + * last one), the tile's IntersectionObserver never fires, so the arrows and + * dots would otherwise stay pointed at an item that no longer exists. + */ +export function useLotusSpotlightFallback( + root: HTMLElement | null, + media: readonly { id: string; type: string }[], + visibleId: string | undefined, + setVisibleId: (id: string) => void, +): void { + const firstIsShare = media[0]?.type === "screen share"; + const firstWasShare = useRef(firstIsShare); + + useEffect(() => { + const target = spotlightFallbackTarget( + media, + visibleId, + firstWasShare.current, + ); + firstWasShare.current = firstIsShare; + if (target === null) return; + + const item = root?.querySelector( + `[data-id="${CSS.escape(target)}"]`, + ); + const scroller = item?.parentElement; + if (item && scroller) + scroller.scrollLeft += + item.getBoundingClientRect().left - + scroller.getBoundingClientRect().left; + setVisibleId(target); + }, [root, media, visibleId, firstIsShare, setVisibleId]); +} diff --git a/src/tile/SpotlightTile.test.tsx b/src/tile/SpotlightTile.test.tsx index 854d853d..b810276b 100644 --- a/src/tile/SpotlightTile.test.tsx +++ b/src/tile/SpotlightTile.test.tsx @@ -214,3 +214,90 @@ test("SpotlightTile displays ringing media", async () => { act(() => pickupState$.next("timeout")); screen.getByText("Call ended"); }); + +test("[lotus #67] falls back to the share when the visible camera goes away", async () => { + // A controllable IntersectionObserver so the tile's visibleId follows + // navigation the way it does in a browser. + let report: ((entries: IntersectionObserverEntry[]) => void) | undefined; + const RealIO = global.IntersectionObserver; + global.IntersectionObserver = class { + public constructor( + cb: (entries: IntersectionObserverEntry[]) => void, + options?: IntersectionObserverInit, + ) { + // Only the carousel's own observer (rooted on the tile); LiveKit's + // video elements create observers of their own. + if (options?.root) report = cb; + } + public observe(): void {} + public unobserve(): void {} + public disconnect(): void {} + } as unknown as typeof IntersectionObserver; + + try { + const share = mockRemoteScreenShare( + mockRtcMembership("@alice:example.org", "AAAA"), + {}, + mockRemoteParticipant({}), + ); + const camera = mockRemoteMedia( + mockRtcMembership("@carol:example.org", "CCCC"), + { rawDisplayName: "Carol", getMxcAvatarUrl: () => "mxc://carol" }, + mockRemoteParticipant({}), + ); + // The mocks all use the id "remote"; the carousel keys on ids. + Object.assign(camera, { id: "remote-camera" }); + const media$ = new BehaviorSubject([share, camera]); + + const user = userEvent.setup(); + const { container } = render( + + + , + ); + const dots = (): HTMLElement[] => + screen.getAllByTestId("screenshare-indicator"); + expect(dots().map((d) => d.dataset.kind)).toEqual(["screen share", "user"]); + expect(dots()[0].dataset.visible).toBe("true"); + + // Flip to Carol's camera; the browser would report it as intersecting. + await user.click(screen.getByRole("button", { name: "Next" })); + const carolItem = container.querySelector(`[data-id="${camera.id}"]`)!; + act(() => + report!([ + { + isIntersecting: true, + target: carolItem, + } as unknown as IntersectionObserverEntry, + ]), + ); + expect(dots()[1].dataset.visible).toBe("true"); + expect(isInaccessible(screen.getByText("Carol"))).toBe(false); + + // Carol turns her camera off: she drops out and the share is shown again. + act(() => media$.next([share])); + expect(dots()).toHaveLength(1); + expect(dots()[0].dataset.visible).toBe("true"); + expect( + isInaccessible(container.querySelector(`[data-id="${share.id}"]`)!), + ).toBe(false); + } finally { + global.IntersectionObserver = RealIO; + } +}); diff --git a/src/tile/SpotlightTile.tsx b/src/tile/SpotlightTile.tsx index 50a476b3..b1597262 100644 --- a/src/tile/SpotlightTile.tsx +++ b/src/tile/SpotlightTile.tsx @@ -43,6 +43,7 @@ import { useLatest } from "../useLatest"; import { type SpotlightTileViewModel } from "../state/TileViewModel"; import { useBehavior } from "../useBehavior"; import { muteScreenshareAudio$ } from "../lotus/lotusScreenshareAudio"; +import { useLotusSpotlightFallback } from "../lotus/lotusSpotlightView"; import { type MemberMediaViewModel } from "../state/media/MemberMediaViewModel"; import { type LocalUserMediaViewModel } from "../state/media/LocalUserMediaViewModel"; import { type RemoteUserMediaViewModel } from "../state/media/RemoteUserMediaViewModel"; @@ -487,6 +488,10 @@ export const SpotlightTile: FC = ({ [visibleId], ); + // [lotus #67] The carousel can hold cameras during a share: keep the share + // as the default view and fall back to it when the visible camera goes away. + useLotusSpotlightFallback(useBehavior(root$), media, visibleId, setVisibleId); + const onBackClick = useCallback(() => { const media = latestMedia.current; const visibleIndex = media.findIndex( @@ -593,6 +598,7 @@ export const SpotlightTile: FC = ({ {media.map((vm) => (