Files
cinny/src/app/plugins/call/CallControl.ts
T

604 lines
21 KiB
TypeScript
Raw Normal View History

+4
2026-03-07 18:03:32 +11:00
import { ClientWidgetApi } from 'matrix-widget-api';
import { EventEmitter } from 'events';
+4
2026-03-07 18:03:32 +11:00
import { CallControlState } from './CallControlState';
import { ElementMediaStateDetail, ElementMediaStatePayload, ElementWidgetActions } from './types';
import { getCallDocument } from './utils';
+4
2026-03-07 18:03:32 +11:00
export enum CallControlEvent {
StateUpdate = 'state_update',
}
/**
* [lotus #7 / P5-31] Payload for the fork's `io.lotus.set_quality` action.
* All fields optional; `null` clears that cap. Bits/sec for bitrates, fps for
* framerate.
*/
export type LotusQualityPayload = {
audioMaxBitrate?: number | null;
screenshareMaxBitrate?: number | null;
screenshareMaxFramerate?: number | null;
};
+4
2026-03-07 18:03:32 +11:00
export class CallControl extends EventEmitter implements CallControlState {
private state: CallControlState;
private call: ClientWidgetApi;
private iframe: HTMLIFrameElement;
private bodyMutationObserver: MutationObserver;
private controlMutationObserver: MutationObserver;
// C-H3: coalesces bursts of body-subtree mutations into a single debounced
// re-observe pass so a busy EC re-render doesn't thrash the control observer.
private bodyMutationTimer?: ReturnType<typeof setTimeout>;
// [Gitea #56] Tracks the participant currently pinned via focusCameraParticipant(),
// so callers (MemberGlance) can render a "Focus camera" / "Unfocus camera" toggle
// instead of a one-way pin with no way back. null == no manual pin (speaker-follows).
private _focusedUserId: string | null = null;
private _focusedMediaId: string | null = null;
// C-M3: last quality payload requested via setQuality(). Held so we can (re)send
// it once joined (io.lotus.set_quality must not be sent before call-join — a
// pre-join send pends to a 10s widget timeout, mirroring the deafen gate).
private lastQuality: LotusQualityPayload | null = null;
// C-M5: set true by CallControls while a push-to-talk key is held. A PTT hold
// unmutes the mic transiently, and onMediaState() must NOT treat that as a
// user-initiated unmute that auto-undeafens the user.
public pttActive = false;
// Deafen mutes the mic; undeafen should give it back ONLY if it was on
// before (Discord semantics — verified against the real client, Gitea #173:
// undeafen used to leave you muted). Cleared by any manual mic toggle.
private micOnBeforeDeafen = false;
// P6-2: mirrors CallEmbed.joined. Set true from forceState(), which CallEmbed
// invokes only from onCallJoined(). Gates io.lotus.set_deafen so we never send
// before the fork's widget handler mounts (pre-join sends pend to a 10s
// timeout — io.lotus toWidget actions must only be sent after call-join).
private joined = false;
private get document(): Document | undefined {
return getCallDocument(this.iframe);
}
private get screenshareButton(): HTMLElement | undefined {
const screenshareBtn = this.document?.querySelector(
'[data-testid="incall_screenshare"]',
) as HTMLElement | null;
return screenshareBtn ?? undefined;
}
private get leaveButton(): Element | undefined {
const leaveBtn = this.document?.querySelector('[data-testid="incall_leave"]');
return leaveBtn ?? undefined;
}
private get settingsButton(): HTMLElement | undefined {
// EC 0.20.1: settings button moved to bottom-left; fall back to bottom-center.
const settingsButtonLeft = this.document?.querySelector(
'[data-testid="settings-bottom-left"]',
) as HTMLButtonElement | undefined;
const settingsButtonCenter = this.document?.querySelector(
'[data-testid="settings-bottom-center"]',
) as HTMLButtonElement | undefined;
return settingsButtonLeft ?? settingsButtonCenter ?? undefined;
}
private get reactionsButton(): HTMLElement | undefined {
// EC 0.20.1: reactions/raise-hand button sits just before the leave button.
const reactionsButton = this.leaveButton?.previousElementSibling as HTMLElement | null;
return reactionsButton ?? undefined;
}
private get spotlightButton(): HTMLInputElement | undefined {
const spotlightButton = this.document?.querySelector(
'input[value="spotlight"]',
) as HTMLInputElement | null;
return spotlightButton ?? undefined;
}
private get gridButton(): HTMLInputElement | undefined {
const gridButton = this.document?.querySelector(
'input[value="grid"]',
) as HTMLInputElement | null;
return gridButton ?? undefined;
}
+4
2026-03-07 18:03:32 +11:00
constructor(state: CallControlState, call: ClientWidgetApi, iframe: HTMLIFrameElement) {
super();
this.state = state;
this.call = call;
this.iframe = iframe;
this.bodyMutationObserver = new MutationObserver(this.onBodyMutation.bind(this));
this.controlMutationObserver = new MutationObserver(this.onControlMutation.bind(this));
+4
2026-03-07 18:03:32 +11:00
}
public getState(): CallControlState {
return this.state;
}
public get microphone(): boolean {
return this.state.microphone;
}
public get video(): boolean {
return this.state.video;
}
public get sound(): boolean {
return this.state.sound;
}
public get screenshare(): boolean {
return this.state.screenshare;
}
public get spotlight(): boolean {
return this.state.spotlight;
}
public get screenshareAudioMuted(): boolean {
return this.state.screenshareAudioMuted;
}
+4
2026-03-07 18:03:32 +11:00
public async applyState() {
await this.setMediaState({
audio_enabled: this.microphone,
video_enabled: this.video,
});
this.setSound();
+4
2026-03-07 18:03:32 +11:00
this.emitStateUpdate();
}
public async forceState(desired: CallControlState) {
this.state = new CallControlState(
desired.microphone,
desired.video,
desired.sound,
this.screenshare,
this.spotlight,
this.screenshareAudioMuted,
);
await this.applyState();
// P6-2: CallEmbed calls forceState() only from onCallJoined(), so this is
// the join transition. Flip the gate open, then push the current deafen
// state to the fork's freshly-mounted handler. (setSound() above ran while
// this.joined was still false, so it was gated — this is the first send.)
this.joined = true;
this.sendDeafenState();
this.sendQuality();
}
/**
* C-H1 / C-M3: re-push the sticky fork-side state (deafen + quality) after an
* EC reconnect. Unlike forceState() this does NOT touch mic/video, so a
* reconnect can't clobber the user's live media state — it only re-arms the
* fork handlers that remount on reconnect.
*/
public resendForkState(): void {
this.sendDeafenState();
this.sendQuality();
if (this._audioOutputId !== undefined) this.sendAudioOutput(this._audioOutputId);
// [Gitea #17] The pin lives fork-side and is dropped on a handler remount.
if (this._focusedUserId !== null) this.sendFocus(this._focusedUserId, this._focusedMediaId);
}
public startObserving() {
if (!this.document) return;
// C-H3: watch the whole body subtree (not just direct children) so we
// re-bind the control observer when EC re-renders its controls deeper in the
// tree. Debounced via onBodyMutation() to avoid thrashing on busy renders.
this.bodyMutationObserver.observe(this.document.body, {
childList: true,
subtree: true,
});
this.applyBodyMutation();
}
private onBodyMutation() {
// C-H3: coalesce a burst of subtree mutations into one debounced pass.
if (this.bodyMutationTimer !== undefined) return;
this.bodyMutationTimer = setTimeout(() => {
this.bodyMutationTimer = undefined;
this.applyBodyMutation();
}, 100);
}
private applyBodyMutation() {
if (!this.document) return;
this.document.body.style.setProperty('background', 'none', 'important');
const controls = this.leaveButton?.parentElement?.parentElement;
if (controls) {
controls.style.setProperty('position', 'absolute');
controls.style.setProperty('visibility', 'hidden');
}
this.observeControls();
}
private observeControls() {
this.controlMutationObserver.disconnect();
const screenshareBtn = this.screenshareButton;
if (screenshareBtn) {
this.controlMutationObserver.observe(screenshareBtn, {
attributes: true,
attributeFilter: ['data-kind'],
});
}
const spotlightBtn = this.spotlightButton;
if (spotlightBtn) {
this.controlMutationObserver.observe(spotlightBtn, {
attributes: true,
});
}
this.onControlMutation();
}
private async setMediaState(state: ElementMediaStatePayload) {
// transport.send resolves once EC has ACK'd the command, which is enough to
// consider the mute applied. We deliberately do NOT gate completion on a
// follow-up DeviceMute state-echo: EC may elide it (e.g. when the requested
// state already matches its current state) or skip it during teardown,
// which would strand this promise forever and block applyState(). The echo,
// when it does arrive, is still handled authoritatively by onMediaState().
return this.call.transport.send(ElementWidgetActions.DeviceMute, state);
+4
2026-03-07 18:03:32 +11:00
}
// [Gitea #209] Deafen is applied by the fork (io.lotus.set_deafen). The
// iframe-DOM `.muted` fallback that used to live here (plus the per-member
// re-apply in useCallMemberSoundSync) fought EC's audio renderer and is gone
// now that the pin is ≥ 0.25.0-lotus.2 — late joiners are the fork's job.
private setSound(): void {
this.sendDeafenState();
+4
2026-03-07 18:03:32 +11:00
}
private applyScreenshareAudioMuted(): void {
if (!this.sound) return;
this.sendDeafenState();
}
// P6-2: send deafen state to the fork (io.lotus.set_deafen). Join-gated: the
// fork's handler only exists once joined; onCallJoined() re-sends the current
// state so a pre-join deafen is not lost.
// [Gitea #119] Output device (headset ↔ speakers) chosen from the host's
// call bar; the fork applies it with mediaDevices.audioOutput.select().
private _audioOutputId: string | undefined;
public get audioOutputId(): string | undefined {
return this._audioOutputId;
}
public setAudioOutput(deviceId: string): void {
this._audioOutputId = deviceId;
this.sendAudioOutput(deviceId);
}
private sendAudioOutput(deviceId: string): void {
if (!this.joined) return;
this.call.transport.send('io.lotus.set_audio_output', { deviceId }).catch(() => undefined);
}
private sendDeafenState(): void {
if (!this.joined) return;
this.call.transport
.send('io.lotus.set_deafen', {
deafened: !this.sound,
screenshareAudioMuted: this.screenshareAudioMuted,
})
.catch(() => undefined);
}
+4
2026-03-07 18:03:32 +11:00
public onMediaState(evt: CustomEvent<ElementMediaStateDetail>) {
const { data } = evt.detail;
if (!data) return;
const state = new CallControlState(
data.audio_enabled ?? this.microphone,
data.video_enabled ?? this.video,
this.sound,
this.screenshare,
this.spotlight,
this.screenshareAudioMuted,
+4
2026-03-07 18:03:32 +11:00
);
this.state = state;
this.emitStateUpdate();
// C-M5: auto-undeafen when the mic turns on, but NOT for a transient
// push-to-talk unmute — a PTT tap while deafened must not silently
// un-deafen the user.
if (this.microphone && !this.sound && !this.pttActive) {
+4
2026-03-07 18:03:32 +11:00
this.toggleSound();
}
}
// [Gitea #43] Set once the fork reports io.lotus.controls_state: from then on
// layout / settings / reactions go over the widget API and screenshare +
// layout state come from that report, not from EC's DOM. Older forks never
// send it and keep the DOM path below.
private forkControls = false;
/** [Gitea #43] The fork's `io.lotus.controls_state` report. */
public onControlsState(data: unknown) {
if (typeof data !== 'object' || data === null) return;
const { screensharing, layout } = data as { screensharing?: unknown; layout?: unknown };
this.forkControls = true;
this.applyControls(
typeof screensharing === 'boolean' ? screensharing : this.screenshare,
layout === 'spotlight' || layout === 'grid' ? layout === 'spotlight' : this.spotlight,
);
}
private onControlMutation() {
if (this.forkControls) return;
const screenshare: boolean = this.screenshareButton?.getAttribute('data-kind') === 'primary';
const spotlight: boolean = this.spotlightButton?.checked ?? false;
this.applyControls(screenshare, spotlight);
}
private applyControls(screenshare: boolean, spotlight: boolean) {
const wasScreensharing = this.screenshare;
// C-M6: when a screenshare stops, clear the screenshare-audio mute so a
// later screenshare doesn't start pre-muted.
const screenshareAudioMuted =
wasScreensharing && !screenshare ? false : this.screenshareAudioMuted;
// C-H3: the body observer now watches subtree:true, so this fires on any DOM
// churn in EC's controls. Only re-emit (→ re-render every consumer) when one
// of the values this method derives actually changed — microphone/video/sound
// are copied unchanged from the current state here.
if (
this.state.screenshare === screenshare &&
this.state.spotlight === spotlight &&
this.state.screenshareAudioMuted === screenshareAudioMuted
) {
return;
}
this.state = new CallControlState(
this.microphone,
this.video,
this.sound,
screenshare,
spotlight,
screenshareAudioMuted,
);
this.emitStateUpdate();
}
public setMicrophone(enabled: boolean) {
const payload: ElementMediaStatePayload = {
audio_enabled: enabled,
video_enabled: this.video,
};
return this.setMediaState(payload);
}
+4
2026-03-07 18:03:32 +11:00
public toggleMicrophone() {
const payload: ElementMediaStatePayload = {
audio_enabled: !this.microphone,
video_enabled: this.video,
};
return this.setMediaState(payload);
}
public toggleVideo() {
const payload: ElementMediaStatePayload = {
audio_enabled: this.microphone,
video_enabled: !this.video,
};
return this.setMediaState(payload);
}
public toggleSound() {
const sound = !this.sound;
// P6-2: commit state before setSound()/applyScreenshareAudioMuted() so
// sendDeafenState() (which reads this.sound) reports the new value.
const state = new CallControlState(
this.microphone,
this.video,
sound,
this.screenshare,
this.spotlight,
this.screenshareAudioMuted,
);
+4
2026-03-07 18:03:32 +11:00
this.state = state;
this.setSound();
// After un-deafening, re-apply screenshare audio mute if active
if (sound) this.applyScreenshareAudioMuted();
+4
2026-03-07 18:03:32 +11:00
this.emitStateUpdate();
if (!sound) {
this.micOnBeforeDeafen = this.microphone;
if (this.microphone) this.toggleMicrophone();
} else if (this.micOnBeforeDeafen && !this.microphone) {
this.micOnBeforeDeafen = false;
+4
2026-03-07 18:03:32 +11:00
this.toggleMicrophone();
}
}
public toggleScreenshareAudio() {
const screenshareAudioMuted = !this.screenshareAudioMuted;
this.state = new CallControlState(
this.microphone,
this.video,
this.sound,
this.screenshare,
this.spotlight,
screenshareAudioMuted,
);
this.emitStateUpdate();
this.applyScreenshareAudioMuted();
}
public toggleScreenshare() {
this.screenshareButton?.click();
}
public toggleSpotlight() {
if (this.forkControls) {
this.sendForkAction('io.lotus.set_layout', {
layout: this.spotlight ? 'grid' : 'spotlight',
});
return;
}
if (this.spotlight) {
this.gridButton?.click();
return;
}
this.spotlightButton?.click();
}
public toggleReactions() {
if (this.forkControls) {
this.sendForkAction('io.lotus.toggle_reactions', {});
return;
}
this.reactionsButton?.click();
}
public toggleSettings() {
if (this.forkControls) {
// Same as clicking EC's settings button: opens the modal.
this.sendForkAction('io.lotus.open_settings', { open: true });
return;
}
this.settingsButton?.click();
}
private sendForkAction(action: string, data: Record<string, unknown>): void {
this.call.transport.send(action, data).catch(() => undefined);
}
/**
* Focus a specific participant's camera tile in Element Call.
*
* EC renders video tiles as `[data-testid="videoTile"]`. Each tile wraps a
* mute-status indicator with `aria-label` set to the participant's Matrix
* user ID. We find the tile containing that user, switch to spotlight mode
* if needed, then click the tile so EC's internal focus handler runs.
*
* Falls back to a plain spotlight toggle if the tile is not found (e.g. the
* participant has their camera off and EC didn't render a video tile for
* them yet).
*/
public get focusedUserId(): string | null {
return this._focusedUserId;
}
private sendFocus(userId: string, id: string | null): void {
// [EC#30] `id` is the fork's per-device media id (userId:deviceId) taken
// from io.lotus.call_state, so a multi-device user pins the right device;
// the fork falls back to userId (preferring the speaking device) when absent.
this.call.transport
.send('io.lotus.focus_participant', id ? { userId, id } : { userId })
.catch(() => undefined);
}
public focusCameraParticipant(userId: string, id: string | null = null): void {
// [lotus #4] Pin the participant via the fork's widget action instead of
// DOM-poking tiles. EC's layout honors it — including surfacing the camera
// alongside a screenshare (A5) — and it's version-stable. The fork always
// acks, so the promise resolves regardless.
this._focusedUserId = userId;
this._focusedMediaId = id;
this.sendFocus(userId, id);
// [Gitea #56] Notify state-update listeners so the menu can flip to "Unfocus camera".
this.emitStateUpdate();
}
/** [lotus #4] Clear any manual spotlight pin and return to speaker-follows. */
public clearFocusParticipant(): void {
// [Gitea #56] No-op (and no redundant widget send) if nothing is pinned —
// dispose() calls this unconditionally on every call teardown.
if (this._focusedUserId === null) return;
this._focusedUserId = null;
this._focusedMediaId = null;
this.call.transport.send('io.lotus.focus_participant', { userId: null }).catch(() => undefined);
this.emitStateUpdate();
}
/**
* [lotus #3 / P5-15] Inject a soundboard clip into the call so other
* participants hear it. The fork publishes it as a separate LiveKit audio
* track (`io.lotus.inject_audio`) rather than splicing the mic. `url` must be
* an https/blob URL the widget can fetch WITHOUT credentials — the host
* resolves an mxc clip to a `blob:` object URL first (authenticated media
* can't be fetched cross-realm by the widget). `volume` is 0–1.
*
* The local user does not hear their own published track, so callers should
* also play the clip locally for feedback.
*/
public injectAudio(url: string, volume = 1): Promise<{ played: boolean; reason?: string }> {
// [EC#13] The fork now refuses while the local mic is muted and replies
// { played:false, reason:"muted" }; older forks reply {} (treated as played).
return this.call.transport
.send<{ url: string; volume: number }, { played?: boolean; reason?: string }>(
'io.lotus.inject_audio',
{ url, volume },
)
.then((r) => ({ played: r?.played !== false, reason: r?.reason }))
.catch(() => ({ played: true }));
}
/**
* [lotus #7 / P5-31] Apply audio/screenshare encoding limits to the local
* published tracks (the fork's `io.lotus.set_quality` action, via
* `RTCRtpSender.setParameters` — no republish). Bitrates are bits/sec,
* framerate is fps. A field set to `null` clears that cap. Settings are
* sticky fork-side (re-applied on every re-publish / reconnect). Values are
* clamped fork-side, so out-of-range input can't brick the encoder.
*/
public setQuality(settings: LotusQualityPayload): void {
// C-M3: remember the request and only send once joined; sendQuality() gates
// on this.joined so a pre-join call is a no-op that we replay on join.
this.lastQuality = settings;
this.sendQuality();
}
// C-M3: push the last-requested quality to the fork. Gated on this.joined so
// we never send io.lotus.set_quality before the fork's handler mounts (a
// pre-join send would pend to a 10s widget timeout).
private sendQuality(): void {
if (!this.joined || !this.lastQuality) return;
this.call.transport.send('io.lotus.set_quality', this.lastQuality).catch(() => undefined);
}
public dispose() {
if (this.bodyMutationTimer !== undefined) {
clearTimeout(this.bodyMutationTimer);
this.bodyMutationTimer = undefined;
}
// [Gitea #56] Don't let a manual focus pin outlive the call.
this.clearFocusParticipant();
this.bodyMutationObserver.disconnect();
this.controlMutationObserver.disconnect();
}
+4
2026-03-07 18:03:32 +11:00
private emitStateUpdate() {
this.emit(CallControlEvent.StateUpdate);
}
}