lotus(#7): audio/screenshare quality controls via widget action
Adds io.lotus.set_quality (toWidget): caps mic audio bitrate and screenshare bitrate/framerate via RTCRtpSender.setParameters (no republish). Settings are sticky and re-applied on LocalTrackPublished so they survive mute/unmute and reconnects. These encoding controls lived in EC's module scope, unreachable from the host against the prebuilt bundle. Additive: no-op unless the host sends the action. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
parent
2ab9427c09
commit
6d739db8b8
@@ -23,10 +23,13 @@ export enum LotusWidgetActions {
|
|||||||
FocusParticipant = "io.lotus.focus_participant",
|
FocusParticipant = "io.lotus.focus_participant",
|
||||||
/** toWidget: mix an audio clip into the local published mic track. */
|
/** toWidget: mix an audio clip into the local published mic track. */
|
||||||
InjectAudio = "io.lotus.inject_audio",
|
InjectAudio = "io.lotus.inject_audio",
|
||||||
|
/** toWidget: set audio/screenshare encoding quality (bitrate/framerate). */
|
||||||
|
SetQuality = "io.lotus.set_quality",
|
||||||
}
|
}
|
||||||
|
|
||||||
/** toWidget Lotus actions that must be allow-listed in `initializeWidget`. */
|
/** toWidget Lotus actions that must be allow-listed in `initializeWidget`. */
|
||||||
export const LOTUS_TO_WIDGET_ACTIONS: LotusWidgetActions[] = [
|
export const LOTUS_TO_WIDGET_ACTIONS: LotusWidgetActions[] = [
|
||||||
LotusWidgetActions.FocusParticipant,
|
LotusWidgetActions.FocusParticipant,
|
||||||
LotusWidgetActions.InjectAudio,
|
LotusWidgetActions.InjectAudio,
|
||||||
|
LotusWidgetActions.SetQuality,
|
||||||
];
|
];
|
||||||
|
|||||||
@@ -0,0 +1,147 @@
|
|||||||
|
/*
|
||||||
|
Copyright 2026 Lotus Guild
|
||||||
|
|
||||||
|
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||||
|
Please see LICENSE in the repository root for full details.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import {
|
||||||
|
type LocalTrack,
|
||||||
|
ParticipantEvent,
|
||||||
|
type Room as LivekitRoom,
|
||||||
|
Track,
|
||||||
|
} from "livekit-client";
|
||||||
|
import { logger } from "matrix-js-sdk/lib/logger";
|
||||||
|
import { type IWidgetApiRequest } from "matrix-widget-api";
|
||||||
|
|
||||||
|
import { type CallViewModel } from "../state/CallViewModel/CallViewModel";
|
||||||
|
import { widget } from "../widget";
|
||||||
|
import { LotusWidgetActions } from "./lotusActions";
|
||||||
|
|
||||||
|
interface QualitySettings {
|
||||||
|
/** Max audio (mic) bitrate in bits/sec. */
|
||||||
|
audioMaxBitrate?: number;
|
||||||
|
/** Max screenshare video bitrate in bits/sec. */
|
||||||
|
screenshareMaxBitrate?: number;
|
||||||
|
/** Max screenshare framerate in fps. */
|
||||||
|
screenshareMaxFramerate?: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Handle the host's `io.lotus.set_quality` toWidget action (#7): apply
|
||||||
|
* audio/screenshare encoding limits (bitrate, framerate) to the local
|
||||||
|
* published tracks via `RTCRtpSender.setParameters` — no republish needed.
|
||||||
|
* These controls live in EC's module scope and were unreachable from the host
|
||||||
|
* against the prebuilt bundle.
|
||||||
|
*
|
||||||
|
* Settings are sticky and re-applied whenever a matching local track is
|
||||||
|
* (re)published, so they survive mute/unmute and reconnects. The server-side
|
||||||
|
* voice-limit-guard remains the enforcement backstop.
|
||||||
|
*
|
||||||
|
* No effect unless the host sends the action. Returns a teardown function.
|
||||||
|
*/
|
||||||
|
export function startLotusQuality(vm: CallViewModel): () => void {
|
||||||
|
const w = widget;
|
||||||
|
if (!w) return () => undefined;
|
||||||
|
|
||||||
|
const settings: QualitySettings = {};
|
||||||
|
// Per-room LocalTrackPublished listeners, so sticky settings re-apply on
|
||||||
|
// every (re)publish.
|
||||||
|
const roomListeners = new Map<LivekitRoom, () => void>();
|
||||||
|
let rooms: LivekitRoom[] = [];
|
||||||
|
|
||||||
|
const applyToRoom = (room: LivekitRoom): void => {
|
||||||
|
const lp = room.localParticipant;
|
||||||
|
|
||||||
|
if (settings.audioMaxBitrate !== undefined) {
|
||||||
|
const mic = lp.getTrackPublication(Track.Source.Microphone)?.track as
|
||||||
|
| LocalTrack
|
||||||
|
| undefined;
|
||||||
|
void patchSender(mic?.sender, { maxBitrate: settings.audioMaxBitrate });
|
||||||
|
}
|
||||||
|
|
||||||
|
const ssPatch: Partial<RTCRtpEncodingParameters> = {};
|
||||||
|
if (settings.screenshareMaxBitrate !== undefined)
|
||||||
|
ssPatch.maxBitrate = settings.screenshareMaxBitrate;
|
||||||
|
if (settings.screenshareMaxFramerate !== undefined)
|
||||||
|
ssPatch.maxFramerate = settings.screenshareMaxFramerate;
|
||||||
|
if (Object.keys(ssPatch).length > 0) {
|
||||||
|
const ss = lp.getTrackPublication(Track.Source.ScreenShare)?.track as
|
||||||
|
| LocalTrack
|
||||||
|
| undefined;
|
||||||
|
void patchSender(ss?.sender, ssPatch);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const applyToAll = (): void => rooms.forEach(applyToRoom);
|
||||||
|
|
||||||
|
// Keep the LocalTrackPublished listeners in sync with the connected rooms.
|
||||||
|
const sub = vm.livekitRoomItems$.subscribe((items) => {
|
||||||
|
const next = items.map((i) => i.livekitRoom);
|
||||||
|
rooms = next;
|
||||||
|
// Remove listeners for rooms that went away.
|
||||||
|
for (const [room, off] of roomListeners) {
|
||||||
|
if (!next.includes(room)) {
|
||||||
|
off();
|
||||||
|
roomListeners.delete(room);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Add listeners for new rooms + apply current settings to them.
|
||||||
|
for (const room of next) {
|
||||||
|
if (!roomListeners.has(room)) {
|
||||||
|
const onPublished = (): void => applyToRoom(room);
|
||||||
|
room.localParticipant.on(
|
||||||
|
ParticipantEvent.LocalTrackPublished,
|
||||||
|
onPublished,
|
||||||
|
);
|
||||||
|
roomListeners.set(room, () =>
|
||||||
|
room.localParticipant.off(
|
||||||
|
ParticipantEvent.LocalTrackPublished,
|
||||||
|
onPublished,
|
||||||
|
),
|
||||||
|
);
|
||||||
|
applyToRoom(room);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
const handler = (ev: CustomEvent<IWidgetApiRequest>): void => {
|
||||||
|
void w.api.transport.reply(ev.detail, {});
|
||||||
|
const data = ev.detail.data as Record<string, unknown> | undefined;
|
||||||
|
if (!data) return;
|
||||||
|
for (const key of [
|
||||||
|
"audioMaxBitrate",
|
||||||
|
"screenshareMaxBitrate",
|
||||||
|
"screenshareMaxFramerate",
|
||||||
|
] as const) {
|
||||||
|
const v = data[key];
|
||||||
|
if (v === null) settings[key] = undefined;
|
||||||
|
else if (typeof v === "number" && v > 0) settings[key] = v;
|
||||||
|
}
|
||||||
|
applyToAll();
|
||||||
|
};
|
||||||
|
|
||||||
|
w.lazyActions.on(LotusWidgetActions.SetQuality, handler);
|
||||||
|
return () => {
|
||||||
|
sub.unsubscribe();
|
||||||
|
for (const off of roomListeners.values()) off();
|
||||||
|
roomListeners.clear();
|
||||||
|
w.lazyActions.off(LotusWidgetActions.SetQuality, handler);
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
async function patchSender(
|
||||||
|
sender: RTCRtpSender | undefined,
|
||||||
|
patch: Partial<RTCRtpEncodingParameters>,
|
||||||
|
): Promise<void> {
|
||||||
|
if (!sender) return;
|
||||||
|
try {
|
||||||
|
const params = sender.getParameters();
|
||||||
|
if (!params.encodings || params.encodings.length === 0)
|
||||||
|
params.encodings = [{}];
|
||||||
|
Object.assign(params.encodings[0], patch);
|
||||||
|
await sender.setParameters(params);
|
||||||
|
} catch (e) {
|
||||||
|
logger.warn("[lotus] set_quality: setParameters failed", e);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -32,6 +32,7 @@ import { widget } from "../widget";
|
|||||||
import { startLotusCallState } from "../lotus/lotusCallState";
|
import { startLotusCallState } from "../lotus/lotusCallState";
|
||||||
import { startLotusFocus } from "../lotus/lotusFocus";
|
import { startLotusFocus } from "../lotus/lotusFocus";
|
||||||
import { startLotusAudioInject } from "../lotus/lotusAudioInject";
|
import { startLotusAudioInject } from "../lotus/lotusAudioInject";
|
||||||
|
import { startLotusQuality } from "../lotus/lotusQuality";
|
||||||
import styles from "./InCallView.module.css";
|
import styles from "./InCallView.module.css";
|
||||||
import { GridTile } from "../tile/GridTile";
|
import { GridTile } from "../tile/GridTile";
|
||||||
import { SettingsModal, defaultSettingsTab } from "../settings/SettingsModal";
|
import { SettingsModal, defaultSettingsTab } from "../settings/SettingsModal";
|
||||||
@@ -291,6 +292,9 @@ export const InCallView: FC<InCallViewProps> = ({
|
|||||||
// [lotus] Handle the host's io.lotus.inject_audio action to mix a soundboard
|
// [lotus] Handle the host's io.lotus.inject_audio action to mix a soundboard
|
||||||
// clip into the call as a separate track (#3). No-op unless the host sends it.
|
// clip into the call as a separate track (#3). No-op unless the host sends it.
|
||||||
useEffect(() => startLotusAudioInject(vm), [vm]);
|
useEffect(() => startLotusAudioInject(vm), [vm]);
|
||||||
|
// [lotus] Handle the host's io.lotus.set_quality action to cap audio/
|
||||||
|
// screenshare encoding bitrate/framerate (#7). No-op unless the host sends it.
|
||||||
|
useEffect(() => startLotusQuality(vm), [vm]);
|
||||||
|
|
||||||
const fatalCallError = useBehavior(vm.fatalError$);
|
const fatalCallError = useBehavior(vm.fatalError$);
|
||||||
// Stop the rendering and throw for the error boundary
|
// Stop the rendering and throw for the error boundary
|
||||||
|
|||||||
Reference in New Issue
Block a user