From 6d739db8b87f561c9b2a8267f9f7fdf50b3ff5e6 Mon Sep 17 00:00:00 2001 From: Lotus CI Date: Mon, 29 Jun 2026 23:24:57 -0400 Subject: [PATCH] lotus(#7): audio/screenshare quality controls via widget action Adds io.lotus.set_quality (toWidget): caps mic audio bitrate and screenshare bitrate/framerate via RTCRtpSender.setParameters (no republish). Settings are sticky and re-applied on LocalTrackPublished so they survive mute/unmute and reconnects. These encoding controls lived in EC's module scope, unreachable from the host against the prebuilt bundle. Additive: no-op unless the host sends the action. Co-Authored-By: Claude Opus 4.8 --- src/lotus/lotusActions.ts | 3 + src/lotus/lotusQuality.ts | 147 ++++++++++++++++++++++++++++++++++++++ src/room/InCallView.tsx | 4 ++ 3 files changed, 154 insertions(+) create mode 100644 src/lotus/lotusQuality.ts diff --git a/src/lotus/lotusActions.ts b/src/lotus/lotusActions.ts index 44cb4339..83e58d33 100644 --- a/src/lotus/lotusActions.ts +++ b/src/lotus/lotusActions.ts @@ -23,10 +23,13 @@ export enum LotusWidgetActions { FocusParticipant = "io.lotus.focus_participant", /** toWidget: mix an audio clip into the local published mic track. */ InjectAudio = "io.lotus.inject_audio", + /** toWidget: set audio/screenshare encoding quality (bitrate/framerate). */ + SetQuality = "io.lotus.set_quality", } /** toWidget Lotus actions that must be allow-listed in `initializeWidget`. */ export const LOTUS_TO_WIDGET_ACTIONS: LotusWidgetActions[] = [ LotusWidgetActions.FocusParticipant, LotusWidgetActions.InjectAudio, + LotusWidgetActions.SetQuality, ]; diff --git a/src/lotus/lotusQuality.ts b/src/lotus/lotusQuality.ts new file mode 100644 index 00000000..d35d604c --- /dev/null +++ b/src/lotus/lotusQuality.ts @@ -0,0 +1,147 @@ +/* +Copyright 2026 Lotus Guild + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { + type LocalTrack, + ParticipantEvent, + type Room as LivekitRoom, + Track, +} from "livekit-client"; +import { logger } from "matrix-js-sdk/lib/logger"; +import { type IWidgetApiRequest } from "matrix-widget-api"; + +import { type CallViewModel } from "../state/CallViewModel/CallViewModel"; +import { widget } from "../widget"; +import { LotusWidgetActions } from "./lotusActions"; + +interface QualitySettings { + /** Max audio (mic) bitrate in bits/sec. */ + audioMaxBitrate?: number; + /** Max screenshare video bitrate in bits/sec. */ + screenshareMaxBitrate?: number; + /** Max screenshare framerate in fps. */ + screenshareMaxFramerate?: number; +} + +/** + * Handle the host's `io.lotus.set_quality` toWidget action (#7): apply + * audio/screenshare encoding limits (bitrate, framerate) to the local + * published tracks via `RTCRtpSender.setParameters` — no republish needed. + * These controls live in EC's module scope and were unreachable from the host + * against the prebuilt bundle. + * + * Settings are sticky and re-applied whenever a matching local track is + * (re)published, so they survive mute/unmute and reconnects. The server-side + * voice-limit-guard remains the enforcement backstop. + * + * No effect unless the host sends the action. Returns a teardown function. + */ +export function startLotusQuality(vm: CallViewModel): () => void { + const w = widget; + if (!w) return () => undefined; + + const settings: QualitySettings = {}; + // Per-room LocalTrackPublished listeners, so sticky settings re-apply on + // every (re)publish. + const roomListeners = new Map void>(); + let rooms: LivekitRoom[] = []; + + const applyToRoom = (room: LivekitRoom): void => { + const lp = room.localParticipant; + + if (settings.audioMaxBitrate !== undefined) { + const mic = lp.getTrackPublication(Track.Source.Microphone)?.track as + | LocalTrack + | undefined; + void patchSender(mic?.sender, { maxBitrate: settings.audioMaxBitrate }); + } + + const ssPatch: Partial = {}; + if (settings.screenshareMaxBitrate !== undefined) + ssPatch.maxBitrate = settings.screenshareMaxBitrate; + if (settings.screenshareMaxFramerate !== undefined) + ssPatch.maxFramerate = settings.screenshareMaxFramerate; + if (Object.keys(ssPatch).length > 0) { + const ss = lp.getTrackPublication(Track.Source.ScreenShare)?.track as + | LocalTrack + | undefined; + void patchSender(ss?.sender, ssPatch); + } + }; + + const applyToAll = (): void => rooms.forEach(applyToRoom); + + // Keep the LocalTrackPublished listeners in sync with the connected rooms. + const sub = vm.livekitRoomItems$.subscribe((items) => { + const next = items.map((i) => i.livekitRoom); + rooms = next; + // Remove listeners for rooms that went away. + for (const [room, off] of roomListeners) { + if (!next.includes(room)) { + off(); + roomListeners.delete(room); + } + } + // Add listeners for new rooms + apply current settings to them. + for (const room of next) { + if (!roomListeners.has(room)) { + const onPublished = (): void => applyToRoom(room); + room.localParticipant.on( + ParticipantEvent.LocalTrackPublished, + onPublished, + ); + roomListeners.set(room, () => + room.localParticipant.off( + ParticipantEvent.LocalTrackPublished, + onPublished, + ), + ); + applyToRoom(room); + } + } + }); + + const handler = (ev: CustomEvent): void => { + void w.api.transport.reply(ev.detail, {}); + const data = ev.detail.data as Record | undefined; + if (!data) return; + for (const key of [ + "audioMaxBitrate", + "screenshareMaxBitrate", + "screenshareMaxFramerate", + ] as const) { + const v = data[key]; + if (v === null) settings[key] = undefined; + else if (typeof v === "number" && v > 0) settings[key] = v; + } + applyToAll(); + }; + + w.lazyActions.on(LotusWidgetActions.SetQuality, handler); + return () => { + sub.unsubscribe(); + for (const off of roomListeners.values()) off(); + roomListeners.clear(); + w.lazyActions.off(LotusWidgetActions.SetQuality, handler); + }; +} + +async function patchSender( + sender: RTCRtpSender | undefined, + patch: Partial, +): Promise { + if (!sender) return; + try { + const params = sender.getParameters(); + if (!params.encodings || params.encodings.length === 0) + params.encodings = [{}]; + Object.assign(params.encodings[0], patch); + await sender.setParameters(params); + } catch (e) { + logger.warn("[lotus] set_quality: setParameters failed", e); + } +} diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index 5f29ea1f..d902433d 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -32,6 +32,7 @@ import { widget } from "../widget"; import { startLotusCallState } from "../lotus/lotusCallState"; import { startLotusFocus } from "../lotus/lotusFocus"; import { startLotusAudioInject } from "../lotus/lotusAudioInject"; +import { startLotusQuality } from "../lotus/lotusQuality"; import styles from "./InCallView.module.css"; import { GridTile } from "../tile/GridTile"; import { SettingsModal, defaultSettingsTab } from "../settings/SettingsModal"; @@ -291,6 +292,9 @@ export const InCallView: FC = ({ // [lotus] Handle the host's io.lotus.inject_audio action to mix a soundboard // clip into the call as a separate track (#3). No-op unless the host sends it. useEffect(() => startLotusAudioInject(vm), [vm]); + // [lotus] Handle the host's io.lotus.set_quality action to cap audio/ + // screenshare encoding bitrate/framerate (#7). No-op unless the host sends it. + useEffect(() => startLotusQuality(vm), [vm]); const fatalCallError = useBehavior(vm.fatalError$); // Stop the rendering and throw for the error boundary