lotus(#7): audio/screenshare quality controls via widget action
Adds io.lotus.set_quality (toWidget): caps mic audio bitrate and screenshare bitrate/framerate via RTCRtpSender.setParameters (no republish). Settings are sticky and re-applied on LocalTrackPublished so they survive mute/unmute and reconnects. These encoding controls lived in EC's module scope, unreachable from the host against the prebuilt bundle. Additive: no-op unless the host sends the action. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
parent
2ab9427c09
commit
6d739db8b8
@@ -23,10 +23,13 @@ export enum LotusWidgetActions {
|
||||
FocusParticipant = "io.lotus.focus_participant",
|
||||
/** toWidget: mix an audio clip into the local published mic track. */
|
||||
InjectAudio = "io.lotus.inject_audio",
|
||||
/** toWidget: set audio/screenshare encoding quality (bitrate/framerate). */
|
||||
SetQuality = "io.lotus.set_quality",
|
||||
}
|
||||
|
||||
/** toWidget Lotus actions that must be allow-listed in `initializeWidget`. */
|
||||
export const LOTUS_TO_WIDGET_ACTIONS: LotusWidgetActions[] = [
|
||||
LotusWidgetActions.FocusParticipant,
|
||||
LotusWidgetActions.InjectAudio,
|
||||
LotusWidgetActions.SetQuality,
|
||||
];
|
||||
|
||||
@@ -0,0 +1,147 @@
|
||||
/*
|
||||
Copyright 2026 Lotus Guild
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import {
|
||||
type LocalTrack,
|
||||
ParticipantEvent,
|
||||
type Room as LivekitRoom,
|
||||
Track,
|
||||
} from "livekit-client";
|
||||
import { logger } from "matrix-js-sdk/lib/logger";
|
||||
import { type IWidgetApiRequest } from "matrix-widget-api";
|
||||
|
||||
import { type CallViewModel } from "../state/CallViewModel/CallViewModel";
|
||||
import { widget } from "../widget";
|
||||
import { LotusWidgetActions } from "./lotusActions";
|
||||
|
||||
interface QualitySettings {
|
||||
/** Max audio (mic) bitrate in bits/sec. */
|
||||
audioMaxBitrate?: number;
|
||||
/** Max screenshare video bitrate in bits/sec. */
|
||||
screenshareMaxBitrate?: number;
|
||||
/** Max screenshare framerate in fps. */
|
||||
screenshareMaxFramerate?: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle the host's `io.lotus.set_quality` toWidget action (#7): apply
|
||||
* audio/screenshare encoding limits (bitrate, framerate) to the local
|
||||
* published tracks via `RTCRtpSender.setParameters` — no republish needed.
|
||||
* These controls live in EC's module scope and were unreachable from the host
|
||||
* against the prebuilt bundle.
|
||||
*
|
||||
* Settings are sticky and re-applied whenever a matching local track is
|
||||
* (re)published, so they survive mute/unmute and reconnects. The server-side
|
||||
* voice-limit-guard remains the enforcement backstop.
|
||||
*
|
||||
* No effect unless the host sends the action. Returns a teardown function.
|
||||
*/
|
||||
export function startLotusQuality(vm: CallViewModel): () => void {
|
||||
const w = widget;
|
||||
if (!w) return () => undefined;
|
||||
|
||||
const settings: QualitySettings = {};
|
||||
// Per-room LocalTrackPublished listeners, so sticky settings re-apply on
|
||||
// every (re)publish.
|
||||
const roomListeners = new Map<LivekitRoom, () => void>();
|
||||
let rooms: LivekitRoom[] = [];
|
||||
|
||||
const applyToRoom = (room: LivekitRoom): void => {
|
||||
const lp = room.localParticipant;
|
||||
|
||||
if (settings.audioMaxBitrate !== undefined) {
|
||||
const mic = lp.getTrackPublication(Track.Source.Microphone)?.track as
|
||||
| LocalTrack
|
||||
| undefined;
|
||||
void patchSender(mic?.sender, { maxBitrate: settings.audioMaxBitrate });
|
||||
}
|
||||
|
||||
const ssPatch: Partial<RTCRtpEncodingParameters> = {};
|
||||
if (settings.screenshareMaxBitrate !== undefined)
|
||||
ssPatch.maxBitrate = settings.screenshareMaxBitrate;
|
||||
if (settings.screenshareMaxFramerate !== undefined)
|
||||
ssPatch.maxFramerate = settings.screenshareMaxFramerate;
|
||||
if (Object.keys(ssPatch).length > 0) {
|
||||
const ss = lp.getTrackPublication(Track.Source.ScreenShare)?.track as
|
||||
| LocalTrack
|
||||
| undefined;
|
||||
void patchSender(ss?.sender, ssPatch);
|
||||
}
|
||||
};
|
||||
|
||||
const applyToAll = (): void => rooms.forEach(applyToRoom);
|
||||
|
||||
// Keep the LocalTrackPublished listeners in sync with the connected rooms.
|
||||
const sub = vm.livekitRoomItems$.subscribe((items) => {
|
||||
const next = items.map((i) => i.livekitRoom);
|
||||
rooms = next;
|
||||
// Remove listeners for rooms that went away.
|
||||
for (const [room, off] of roomListeners) {
|
||||
if (!next.includes(room)) {
|
||||
off();
|
||||
roomListeners.delete(room);
|
||||
}
|
||||
}
|
||||
// Add listeners for new rooms + apply current settings to them.
|
||||
for (const room of next) {
|
||||
if (!roomListeners.has(room)) {
|
||||
const onPublished = (): void => applyToRoom(room);
|
||||
room.localParticipant.on(
|
||||
ParticipantEvent.LocalTrackPublished,
|
||||
onPublished,
|
||||
);
|
||||
roomListeners.set(room, () =>
|
||||
room.localParticipant.off(
|
||||
ParticipantEvent.LocalTrackPublished,
|
||||
onPublished,
|
||||
),
|
||||
);
|
||||
applyToRoom(room);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
const handler = (ev: CustomEvent<IWidgetApiRequest>): void => {
|
||||
void w.api.transport.reply(ev.detail, {});
|
||||
const data = ev.detail.data as Record<string, unknown> | undefined;
|
||||
if (!data) return;
|
||||
for (const key of [
|
||||
"audioMaxBitrate",
|
||||
"screenshareMaxBitrate",
|
||||
"screenshareMaxFramerate",
|
||||
] as const) {
|
||||
const v = data[key];
|
||||
if (v === null) settings[key] = undefined;
|
||||
else if (typeof v === "number" && v > 0) settings[key] = v;
|
||||
}
|
||||
applyToAll();
|
||||
};
|
||||
|
||||
w.lazyActions.on(LotusWidgetActions.SetQuality, handler);
|
||||
return () => {
|
||||
sub.unsubscribe();
|
||||
for (const off of roomListeners.values()) off();
|
||||
roomListeners.clear();
|
||||
w.lazyActions.off(LotusWidgetActions.SetQuality, handler);
|
||||
};
|
||||
}
|
||||
|
||||
async function patchSender(
|
||||
sender: RTCRtpSender | undefined,
|
||||
patch: Partial<RTCRtpEncodingParameters>,
|
||||
): Promise<void> {
|
||||
if (!sender) return;
|
||||
try {
|
||||
const params = sender.getParameters();
|
||||
if (!params.encodings || params.encodings.length === 0)
|
||||
params.encodings = [{}];
|
||||
Object.assign(params.encodings[0], patch);
|
||||
await sender.setParameters(params);
|
||||
} catch (e) {
|
||||
logger.warn("[lotus] set_quality: setParameters failed", e);
|
||||
}
|
||||
}
|
||||
@@ -32,6 +32,7 @@ import { widget } from "../widget";
|
||||
import { startLotusCallState } from "../lotus/lotusCallState";
|
||||
import { startLotusFocus } from "../lotus/lotusFocus";
|
||||
import { startLotusAudioInject } from "../lotus/lotusAudioInject";
|
||||
import { startLotusQuality } from "../lotus/lotusQuality";
|
||||
import styles from "./InCallView.module.css";
|
||||
import { GridTile } from "../tile/GridTile";
|
||||
import { SettingsModal, defaultSettingsTab } from "../settings/SettingsModal";
|
||||
@@ -291,6 +292,9 @@ export const InCallView: FC<InCallViewProps> = ({
|
||||
// [lotus] Handle the host's io.lotus.inject_audio action to mix a soundboard
|
||||
// clip into the call as a separate track (#3). No-op unless the host sends it.
|
||||
useEffect(() => startLotusAudioInject(vm), [vm]);
|
||||
// [lotus] Handle the host's io.lotus.set_quality action to cap audio/
|
||||
// screenshare encoding bitrate/framerate (#7). No-op unless the host sends it.
|
||||
useEffect(() => startLotusQuality(vm), [vm]);
|
||||
|
||||
const fatalCallError = useBehavior(vm.fatalError$);
|
||||
// Stop the rendering and throw for the error boundary
|
||||
|
||||
Reference in New Issue
Block a user