diff --git a/LOTUS.md b/LOTUS.md index e4779ed5..21b8f353 100644 --- a/LOTUS.md +++ b/LOTUS.md @@ -33,32 +33,33 @@ The host sets these on the widget URL (cinny: `CallEmbed.getWidget`). The table is generated from `src/lotus/lotusActions.ts`; the last column was checked against cinny's `lotus` branch. -| Action | Direction | Payload / meaning | Handled by cinny (`lotus`) | -| -------------------------------- | ----------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------- | -| `io.lotus.call_state` | call → host | in-call per-participant speaking / mute state. NOTE: matrix-widget-api `transport.send` is request/response — the host MUST register a handler that replies/acks each one (cinny's `listenAction` does, replying `{}`). If the host has no handler, ClientWidgetApi immediately error-replies "unsupported from-widget action", so the data is silently dropped and each throttled send rejects (caught) — functional miss + log churn, not a hang. | yes | -| `io.lotus.focus_participant` | host → call | pin/spotlight (or clear, with userId=null) a participant. | yes | -| `io.lotus.inject_audio` | host → call | play an audio clip into the call as a separate published track. | yes | -| `io.lotus.set_quality` | host → call | set audio/screenshare encoding quality (bitrate/framerate). | yes | -| `io.lotus.decorations` | host → call | per-user avatar-decoration image URLs for in-call tiles. | yes | -| `io.lotus.set_deafen` | host → call | deafen remote audio (and optionally mute screenshare audio). | yes | -| `io.lotus.request_state` | call → host | sent on (re)registration of a lotus toWidget handler (e.g. decorations) so the host can re-push state that would otherwise only be sent on change (decorations roster, the current focus pin) after an EC-side reconnect/remount. cinny's `resendForkState()` should respond to this the same way it responds to a fresh join. | yes | -| `io.lotus.denoise_state` | call → host | real state of the in-source denoise engine — `{ active: boolean, model: string, error?: string }` — sent once the processor attaches (or after the rnnoise fallback also fails), so the host toggle can reflect reality rather than the requested state. | yes | -| `io.lotus.call_summary` | call → host | one-shot end-of-call readout for the local participant — `{ durationMs, reconnects, poorMs, verdict }` (#143) — sent on SFU disconnect or in-call teardown, whichever comes first. | yes | -| `io.lotus.set_audio_output` | host → call | select the audio output device `{ deviceId }` (#119). | yes | -| `io.lotus.audio_output_state` | call → host | the currently selected output `{ deviceId }` (#119). | cinny #417 | -| `io.lotus.screenshare_notice` | call → host | local screenshare reminder `{ kind: "ended" \| "no-frames" \| "alone" }` — window closed, no frames for 15 s, or 30 min of sharing with nobody else in the call (#39). Each fires at most once per share. | yes | -| `io.lotus.set_layout` | host → call | switch the call layout `{ layout: "grid" \| "spotlight" }` (cinny #43 — replaces the host clicking EC's hidden layout radio). | yes | -| `io.lotus.open_settings` | host → call | open EC's settings modal, or close it with `{ open: false }`; omit `open` to toggle (cinny #43). | yes | -| `io.lotus.toggle_reactions` | host → call | toggle the reactions / raise-hand menu (cinny #43). | yes | -| `io.lotus.set_screenshare` | host → call | start/stop sharing `{ on?: boolean }` (omit to toggle; cinny #43). Starting calls getDisplayMedia, which needs the user's click: the host sends this with Capability Delegation (`delegate: "display-capture"`), and without it the browser rejects the share. | yes | -| `io.lotus.set_hotkeys` | host → call | `{ codes: string[] }` — KeyboardEvent.code values (PTT, deafen) to report while focus is inside this frame (cinny #43). `[]` stops. | yes | -| `io.lotus.set_frame_screenshare` | host → call | `{ visible: boolean }` — whether the in-frame screenshare button (`lotusFrameScreenshare`) may show, e.g. false while the room's call policy forbids screensharing (cinny #43). Defaults to visible. | yes | -| `io.lotus.prompt_screenshare` | host → call | `{}` — show the "Share your screen?" prompt inside this frame (cinny #43). For hosts that can't delegate the user's click: the prompt's Share button is clicked here, so getDisplayMedia gets its activation. | yes | -| `io.lotus.hotkey` | call → host | a watched key went down/up here, or this window's focus changed; see `LotusHotkeyReport` (cinny #43). | yes | -| `io.lotus.controls_state` | call → host | `{ screensharing: boolean, layout: "grid" \| "spotlight" \| null, screenshareAction: true, hotkeys: true, frameScreenshare: false, screensharePrompt: boolean, screensharePromptOpen: boolean }` whenever either changes (cinny #43), so the host stops reading EC's DOM for them. Its arrival also tells the host this fork supports the actions above. | yes | -| `io.lotus.mic_level` | call → host | local mic level `{ bars: 0 \| 1 \| 2 \| 3 }` (cinny #146), sent only when it changes (≤ 10 Hz); 0 while muted or with no mic. | yes | -| `io.lotus.call_error` | call → host | the call's error page is up (`{ code, recoverable }`) or cleared (`{ code: null }`) (cinny #318), so the host stops showing the call as joined after "Connection lost". | cinny #418 | -| `io.lotus.reconnect` | host → call | press the error page's Reconnect / Try again (cinny #318). | cinny #418 | +| Action | Direction | Payload / meaning | Handled by cinny (`lotus`) | +| ---------------------------------- | ----------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------- | +| `io.lotus.call_state` | call → host | in-call per-participant speaking / mute state. NOTE: matrix-widget-api `transport.send` is request/response — the host MUST register a handler that replies/acks each one (cinny's `listenAction` does, replying `{}`). If the host has no handler, ClientWidgetApi immediately error-replies "unsupported from-widget action", so the data is silently dropped and each throttled send rejects (caught) — functional miss + log churn, not a hang. | yes | +| `io.lotus.focus_participant` | host → call | pin/spotlight (or clear, with userId=null) a participant. | yes | +| `io.lotus.inject_audio` | host → call | play an audio clip into the call as a separate published track. | yes | +| `io.lotus.set_quality` | host → call | set audio/screenshare encoding quality (bitrate/framerate). | yes | +| `io.lotus.decorations` | host → call | per-user avatar-decoration image URLs for in-call tiles. | yes | +| `io.lotus.set_deafen` | host → call | deafen remote audio (and optionally mute screenshare audio). | yes | +| `io.lotus.request_state` | call → host | sent on (re)registration of a lotus toWidget handler (e.g. decorations) so the host can re-push state that would otherwise only be sent on change (decorations roster, the current focus pin) after an EC-side reconnect/remount. cinny's `resendForkState()` should respond to this the same way it responds to a fresh join. | yes | +| `io.lotus.denoise_state` | call → host | real state of the in-source denoise engine — `{ active: boolean, model: string, error?: string }` — sent once the processor attaches (or after the rnnoise fallback also fails), so the host toggle can reflect reality rather than the requested state. | yes | +| `io.lotus.call_summary` | call → host | one-shot end-of-call readout for the local participant — `{ durationMs, reconnects, poorMs, verdict }` (#143) — sent on SFU disconnect or in-call teardown, whichever comes first. | yes | +| `io.lotus.set_audio_output` | host → call | select the audio output device `{ deviceId }` (#119). | yes | +| `io.lotus.audio_output_state` | call → host | the currently selected output `{ deviceId }` (#119). | cinny #417 | +| `io.lotus.screenshare_audio_state` | call → host | `{ present }`: whether any remote screenshare audio is being received, sent on change (cinny #277). | cinny #277 | +| `io.lotus.screenshare_notice` | call → host | local screenshare reminder `{ kind: "ended" \| "no-frames" \| "alone" }` — window closed, no frames for 15 s, or 30 min of sharing with nobody else in the call (#39). Each fires at most once per share. | yes | +| `io.lotus.set_layout` | host → call | switch the call layout `{ layout: "grid" \| "spotlight" }` (cinny #43 — replaces the host clicking EC's hidden layout radio). | yes | +| `io.lotus.open_settings` | host → call | open EC's settings modal, or close it with `{ open: false }`; omit `open` to toggle (cinny #43). | yes | +| `io.lotus.toggle_reactions` | host → call | toggle the reactions / raise-hand menu (cinny #43). | yes | +| `io.lotus.set_screenshare` | host → call | start/stop sharing `{ on?: boolean }` (omit to toggle; cinny #43). Starting calls getDisplayMedia, which needs the user's click: the host sends this with Capability Delegation (`delegate: "display-capture"`), and without it the browser rejects the share. | yes | +| `io.lotus.set_hotkeys` | host → call | `{ codes: string[] }` — KeyboardEvent.code values (PTT, deafen) to report while focus is inside this frame (cinny #43). `[]` stops. | yes | +| `io.lotus.set_frame_screenshare` | host → call | `{ visible: boolean }` — whether the in-frame screenshare button (`lotusFrameScreenshare`) may show, e.g. false while the room's call policy forbids screensharing (cinny #43). Defaults to visible. | yes | +| `io.lotus.prompt_screenshare` | host → call | `{}` — show the "Share your screen?" prompt inside this frame (cinny #43). For hosts that can't delegate the user's click: the prompt's Share button is clicked here, so getDisplayMedia gets its activation. | yes | +| `io.lotus.hotkey` | call → host | a watched key went down/up here, or this window's focus changed; see `LotusHotkeyReport` (cinny #43). | yes | +| `io.lotus.controls_state` | call → host | `{ screensharing: boolean, layout: "grid" \| "spotlight" \| null, screenshareAction: true, hotkeys: true, frameScreenshare: false, screensharePrompt: boolean, screensharePromptOpen: boolean }` whenever either changes (cinny #43), so the host stops reading EC's DOM for them. Its arrival also tells the host this fork supports the actions above. | yes | +| `io.lotus.mic_level` | call → host | local mic level `{ bars: 0 \| 1 \| 2 \| 3 }` (cinny #146), sent only when it changes (≤ 10 Hz); 0 while muted or with no mic. | yes | +| `io.lotus.call_error` | call → host | the call's error page is up (`{ code, recoverable }`) or cleared (`{ code: null }`) (cinny #318), so the host stops showing the call as joined after "Connection lost". | cinny #418 | +| `io.lotus.reconnect` | host → call | press the error page's Reconnect / Try again (cinny #318). | cinny #418 | Entries naming a cinny PR are handled once that PR merges. diff --git a/src/livekit/MatrixAudioRenderer.tsx b/src/livekit/MatrixAudioRenderer.tsx index cf92f361..103d6434 100644 --- a/src/livekit/MatrixAudioRenderer.tsx +++ b/src/livekit/MatrixAudioRenderer.tsx @@ -19,6 +19,10 @@ import { logger as rootLogger } from "matrix-js-sdk/lib/logger"; import { useEarpieceAudioConfig } from "../MediaDevicesContext"; import { useReactiveState } from "../useReactiveState"; import * as controls from "../controls"; +import { + clearScreenshareAudio, + reportScreenshareAudio, +} from "../lotus/lotusScreenshareAudio"; export interface MatrixAudioRendererProps { /** @@ -97,6 +101,18 @@ export function LivekitRoomAudioRenderer({ return true; }); + // [lotus / cinny #277] Tell the host whether remote screenshare audio is + // playing, so its "Mute Screenshare Audio" control only shows when useful. + const hasScreenshareAudio = tracks.some( + (ref) => + !ref.participant.isLocal && + ref.publication.source === Track.Source.ScreenShareAudio, + ); + useEffect(() => { + reportScreenshareAudio(url, hasScreenshareAudio); + }, [url, hasScreenshareAudio]); + useEffect(() => (): void => clearScreenshareAudio(url), [url]); + // This component is also (in addition to the "only play audio for connected members" logic above) // responsible for mimicking earpiece audio on iPhones. // The Safari audio devices enumeration does not expose an earpiece audio device. diff --git a/src/lotus/lotusActions.ts b/src/lotus/lotusActions.ts index 1bce5b4c..65b6875d 100644 --- a/src/lotus/lotusActions.ts +++ b/src/lotus/lotusActions.ts @@ -56,6 +56,12 @@ export enum LotusWidgetActions { SetAudioOutput = "io.lotus.set_audio_output", /** fromWidget: the currently selected output `{ deviceId }` (#119). */ AudioOutputState = "io.lotus.audio_output_state", + /** + * fromWidget: `{ present: boolean }`, whether any remote screenshare audio + * is being received, so the host's "Mute Screenshare Audio" control only + * shows when there is something to mute (cinny #277). Sent on change. + */ + ScreenshareAudioState = "io.lotus.screenshare_audio_state", /** * fromWidget: local screenshare reminder `{ kind: "ended" | "no-frames" | "alone" }` * — window closed, no frames for 15 s, or 30 min of sharing with nobody diff --git a/src/lotus/lotusScreenshareAudio.test.ts b/src/lotus/lotusScreenshareAudio.test.ts new file mode 100644 index 00000000..72359822 --- /dev/null +++ b/src/lotus/lotusScreenshareAudio.test.ts @@ -0,0 +1,59 @@ +/* +Copyright 2026 Lotus Guild + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { beforeEach, expect, test, vi } from "vitest"; + +import { LotusWidgetActions } from "./lotusActions"; +import { + clearScreenshareAudio, + reportScreenshareAudio, + resetScreenshareAudioPresence, +} from "./lotusScreenshareAudio"; + +const send = vi.fn().mockResolvedValue({}); + +vi.mock("../widget", () => ({ + widget: { + api: { transport: { send: (...a: unknown[]): unknown => send(...a) } }, + }, +})); + +const sent = (): unknown[] => + send.mock.calls + .filter(([action]) => action === LotusWidgetActions.ScreenshareAudioState) + .map(([, data]) => data); + +beforeEach(() => { + resetScreenshareAudioPresence(); + send.mockClear(); +}); + +test("[cinny #277] reports presence only when the combined answer changes", () => { + reportScreenshareAudio("sfu-a", false); + reportScreenshareAudio("sfu-a", false); + reportScreenshareAudio("sfu-a", true); + reportScreenshareAudio("sfu-a", true); + reportScreenshareAudio("sfu-a", false); + expect(sent()).toEqual([ + { present: false }, + { present: true }, + { present: false }, + ]); +}); + +test("[cinny #277] any room with screenshare audio counts as present", () => { + reportScreenshareAudio("sfu-a", true); + reportScreenshareAudio("sfu-b", false); + expect(sent()).toEqual([{ present: true }]); + // A share is still playing in sfu-b after sfu-a's stops. + reportScreenshareAudio("sfu-b", true); + reportScreenshareAudio("sfu-a", false); + expect(sent()).toEqual([{ present: true }]); + // The renderer for sfu-b unmounts (call left): nothing is playing. + clearScreenshareAudio("sfu-b"); + expect(sent()).toEqual([{ present: true }, { present: false }]); +}); diff --git a/src/lotus/lotusScreenshareAudio.ts b/src/lotus/lotusScreenshareAudio.ts index ab9f404d..254618d9 100644 --- a/src/lotus/lotusScreenshareAudio.ts +++ b/src/lotus/lotusScreenshareAudio.ts @@ -8,6 +8,8 @@ Please see LICENSE in the repository root for full details. import { BehaviorSubject } from "rxjs"; import { globalScope } from "../state/ObservableScope"; +import { LotusWidgetActions } from "./lotusActions"; +import { lotusSendToHost } from "./lotusWidget"; /** * Whether the host has muted remote screenshare AUDIO (`io.lotus.set_deafen` @@ -25,3 +27,40 @@ export const setScreenshareAudioMuted$ = new BehaviorSubject(false); export const muteScreenshareAudio$ = globalScope.behavior( setScreenshareAudioMuted$, ); + +/** + * [cinny #277] Whether any remote participant's screenshare AUDIO is being + * received, so the host shows its "Mute Screenshare Audio" control only when + * there is something to mute. Each `LivekitRoomAudioRenderer` (one per SFU) + * reports for its own room; the host gets `io.lotus.screenshare_audio_state + * { present }` whenever the combined answer changes. + */ +const presentByRoom = new Map(); +let lastSent: boolean | undefined; + +function sendIfChanged(): void { + const present = [...presentByRoom.values()].some(Boolean); + if (present === lastSent) return; + lastSent = present; + lotusSendToHost(LotusWidgetActions.ScreenshareAudioState, { present }); +} + +export function reportScreenshareAudio( + roomKey: string, + present: boolean, +): void { + presentByRoom.set(roomKey, present); + sendIfChanged(); +} + +/** The renderer for `roomKey` went away (call left, SFU switched). */ +export function clearScreenshareAudio(roomKey: string): void { + presentByRoom.delete(roomKey); + sendIfChanged(); +} + +/** Test-only: forget all reports. */ +export function resetScreenshareAudioPresence(): void { + presentByRoom.clear(); + lastSent = undefined; +}