import { ClientWidgetApi } from 'matrix-widget-api'; import { EventEmitter } from 'events'; import { CallControlState } from './CallControlState'; import { ElementMediaStateDetail, ElementMediaStatePayload, ElementWidgetActions } from './types'; import { canDelegateCapability } from './utils'; export enum CallControlEvent { StateUpdate = 'state_update', } /** * [lotus #7 / P5-31] Payload for the fork's `io.lotus.set_quality` action. * All fields optional; `null` clears that cap. Bits/sec for bitrates, fps for * framerate. */ export type LotusQualityPayload = { audioMaxBitrate?: number | null; screenshareMaxBitrate?: number | null; screenshareMaxFramerate?: number | null; }; /** fromWidget `io.lotus.hotkey` (Gitea #43): a watched key in the call frame. */ export type ForkHotkeyReport = | { type: 'keydown' | 'keyup'; code: string; repeat: boolean; ctrlKey: boolean; altKey: boolean; metaKey: boolean; shiftKey: boolean; /** Went to a text field in the frame. */ editable: boolean; /** Went to a button/link in the frame. */ interactive: boolean; } | { type: 'focus' }; export function parseForkHotkeyReport(data: unknown): ForkHotkeyReport | null { if (typeof data !== 'object' || data === null) return null; const d = data as Record; if (d.type === 'focus') return { type: 'focus' }; if ((d.type !== 'keydown' && d.type !== 'keyup') || typeof d.code !== 'string') return null; return { type: d.type, code: d.code, repeat: d.repeat === true, ctrlKey: d.ctrlKey === true, altKey: d.altKey === true, metaKey: d.metaKey === true, shiftKey: d.shiftKey === true, editable: d.editable === true, interactive: d.interactive === true, }; } export class CallControl extends EventEmitter implements CallControlState { private state: CallControlState; private call: ClientWidgetApi; private iframe: HTMLIFrameElement; // [Gitea #56] Tracks the participant currently pinned via focusCameraParticipant(), // so callers (MemberGlance) can render a "Focus camera" / "Unfocus camera" toggle // instead of a one-way pin with no way back. null == no manual pin (speaker-follows). private _focusedUserId: string | null = null; private _focusedMediaId: string | null = null; // C-M3: last quality payload requested via setQuality(). Held so we can (re)send // it once joined (io.lotus.set_quality must not be sent before call-join — a // pre-join send pends to a 10s widget timeout, mirroring the deafen gate). private lastQuality: LotusQualityPayload | null = null; // C-M5: set true by CallControls while a push-to-talk key is held. A PTT hold // unmutes the mic transiently, and onMediaState() must NOT treat that as a // user-initiated unmute that auto-undeafens the user. public pttActive = false; // Deafen mutes the mic; undeafen should give it back ONLY if it was on // before (Discord semantics — verified against the real client, Gitea #173: // undeafen used to leave you muted). Cleared by any manual mic toggle. private micOnBeforeDeafen = false; // P6-2: mirrors CallEmbed.joined. Set true from forceState(), which CallEmbed // invokes only from onCallJoined(). Gates io.lotus.set_deafen so we never send // before the fork's widget handler mounts (pre-join sends pend to a 10s // timeout — io.lotus toWidget actions must only be sent after call-join). private joined = false; constructor(state: CallControlState, call: ClientWidgetApi, iframe: HTMLIFrameElement) { super(); this.state = state; this.call = call; this.iframe = iframe; } public getState(): CallControlState { return this.state; } public get microphone(): boolean { return this.state.microphone; } public get video(): boolean { return this.state.video; } public get sound(): boolean { return this.state.sound; } public get screenshare(): boolean { return this.state.screenshare; } public get spotlight(): boolean { return this.state.spotlight; } public get screenshareAudioMuted(): boolean { return this.state.screenshareAudioMuted; } public async applyState() { await this.setMediaState({ audio_enabled: this.microphone, video_enabled: this.video, }); this.setSound(); this.emitStateUpdate(); } public async forceState(desired: CallControlState) { this.state = new CallControlState( desired.microphone, desired.video, desired.sound, this.screenshare, this.spotlight, this.screenshareAudioMuted, ); await this.applyState(); // P6-2: CallEmbed calls forceState() only from onCallJoined(), so this is // the join transition. Flip the gate open, then push the current deafen // state to the fork's freshly-mounted handler. (setSound() above ran while // this.joined was still false, so it was gated — this is the first send.) this.joined = true; this.sendDeafenState(); this.sendQuality(); this.sendHotkeyCodes(); this.sendFrameScreenshareAllowed(); } /** * C-H1 / C-M3: re-push the sticky fork-side state (deafen + quality) after an * EC reconnect. Unlike forceState() this does NOT touch mic/video, so a * reconnect can't clobber the user's live media state — it only re-arms the * fork handlers that remount on reconnect. */ public resendForkState(): void { this.sendDeafenState(); this.sendQuality(); if (this._audioOutputId !== undefined) this.sendAudioOutput(this._audioOutputId); // [Gitea #17] The pin lives fork-side and is dropped on a handler remount. if (this._focusedUserId !== null) this.sendFocus(this._focusedUserId, this._focusedMediaId); this.sendHotkeyCodes(); this.sendFrameScreenshareAllowed(); } private async setMediaState(state: ElementMediaStatePayload) { // transport.send resolves once EC has ACK'd the command, which is enough to // consider the mute applied. We deliberately do NOT gate completion on a // follow-up DeviceMute state-echo: EC may elide it (e.g. when the requested // state already matches its current state) or skip it during teardown, // which would strand this promise forever and block applyState(). The echo, // when it does arrive, is still handled authoritatively by onMediaState(). return this.call.transport.send(ElementWidgetActions.DeviceMute, state); } // [Gitea #209] Deafen is applied by the fork (io.lotus.set_deafen). The // iframe-DOM `.muted` fallback that used to live here (plus the per-member // re-apply in useCallMemberSoundSync) fought EC's audio renderer and is gone // now that the pin is ≥ 0.25.0-lotus.2 — late joiners are the fork's job. private setSound(): void { this.sendDeafenState(); } private applyScreenshareAudioMuted(): void { if (!this.sound) return; this.sendDeafenState(); } // P6-2: send deafen state to the fork (io.lotus.set_deafen). Join-gated: the // fork's handler only exists once joined; onCallJoined() re-sends the current // state so a pre-join deafen is not lost. // [Gitea #119] Output device (headset ↔ speakers) chosen from the host's // call bar; the fork applies it with mediaDevices.audioOutput.select(). private _audioOutputId: string | undefined; public get audioOutputId(): string | undefined { return this._audioOutputId; } public setAudioOutput(deviceId: string): void { this._audioOutputId = deviceId; this.sendAudioOutput(deviceId); } private sendAudioOutput(deviceId: string): void { if (!this.joined) return; this.call.transport.send('io.lotus.set_audio_output', { deviceId }).catch(() => undefined); } private sendDeafenState(): void { if (!this.joined) return; this.call.transport .send('io.lotus.set_deafen', { deafened: !this.sound, screenshareAudioMuted: this.screenshareAudioMuted, }) .catch(() => undefined); } public onMediaState(evt: CustomEvent) { const { data } = evt.detail; if (!data) return; const state = new CallControlState( data.audio_enabled ?? this.microphone, data.video_enabled ?? this.video, this.sound, this.screenshare, this.spotlight, this.screenshareAudioMuted, ); this.state = state; this.emitStateUpdate(); // C-M5: auto-undeafen when the mic turns on, but NOT for a transient // push-to-talk unmute — a PTT tap while deafened must not silently // un-deafen the user. if (this.microphone && !this.sound && !this.pttActive) { this.toggleSound(); } } // [Gitea #43] The fork has reported io.lotus.controls_state, so its Lotus // action handlers are mounted. Screenshare and layout state come from that // report; the host never reads or clicks EC's DOM. private forkReady = false; // [Gitea #43] The fork shows EC's own screenshare button inside the frame // (lotusFrameScreenshare: no Capability Delegation here), so the host bar // hides its own. private _frameScreenshare = false; private frameScreenshareListeners = new Set<() => void>(); // [Gitea #43] The fork draws the "Share your screen?" prompt inside the frame // on request (io.lotus.prompt_screenshare), so where the click can't be // delegated our call-bar button asks for that instead of starting the share. private _screensharePrompt = false; // [Gitea #43] The fork's in-frame prompt is showing: anything the host lays // over the frame (the picture-in-picture overlay) must let clicks through. private _screensharePromptOpen = false; // [Gitea #43] Whether the room's call policy allows screensharing; the fork // hides its in-frame button when it doesn't. private frameScreenshareAllowed = true; private hotkeyCodes = new Map(); private hotkeyListeners = new Set<(report: ForkHotkeyReport) => void>(); /** EC's own screenshare button is shown in the frame instead of ours. */ public get frameScreenshare(): boolean { return this._frameScreenshare; } /** * Starting a share must go through the fork's in-frame prompt: this engine * can't delegate the click (Firefox, Safari, WebKitGTK) and the fork has * the prompt. Stopping never needs it. */ public get screenshareNeedsPrompt(): boolean { return this._screensharePrompt && !canDelegateCapability(); } /** The fork's in-frame "Share your screen?" prompt is showing. */ public get screensharePromptOpen(): boolean { return this._screensharePromptOpen; } /** * Ask the fork to show "Share your screen?" inside the call frame; its Share * button starts the share with the frame's own click. */ public promptScreenshare(): void { this.sendForkAction('io.lotus.prompt_screenshare', {}); } /** * Subscribe to `frameScreenshare` / `screenshareNeedsPrompt` / * `screensharePromptOpen` changes. Returns an unsubscribe. */ public onFrameScreenshareChange(cb: () => void): () => void { this.frameScreenshareListeners.add(cb); return () => { this.frameScreenshareListeners.delete(cb); }; } /** Room call policy for the in-frame screenshare button. */ public setFrameScreenshareAllowed(allowed: boolean): void { this.frameScreenshareAllowed = allowed; this.sendFrameScreenshareAllowed(); } private sendFrameScreenshareAllowed(): void { if (!this.joined || !this.forkReady) return; if (!this._frameScreenshare && !this._screensharePrompt) return; this.sendForkAction('io.lotus.set_frame_screenshare', { visible: this.frameScreenshareAllowed, }); } /** Key codes `source` (e.g. 'ptt', 'deafen') wants reported from the frame. */ public setHotkeyCodes(source: string, codes: string[]): void { this.hotkeyCodes.set(source, codes); this.sendHotkeyCodes(); } private sendHotkeyCodes(): void { if (!this.joined || !this.forkReady) return; const codes = [...new Set([...this.hotkeyCodes.values()].flat())]; this.call.transport.send('io.lotus.set_hotkeys', { codes }).catch(() => undefined); } /** Subscribe to the fork's `io.lotus.hotkey` reports. Returns an unsubscribe. */ public onHotkey(cb: (report: ForkHotkeyReport) => void): () => void { this.hotkeyListeners.add(cb); return () => { this.hotkeyListeners.delete(cb); }; } /** [Gitea #43] The fork's `io.lotus.hotkey` report. */ public onHotkeyReport(data: unknown): void { const report = parseForkHotkeyReport(data); if (report) this.hotkeyListeners.forEach((l) => l(report)); } /** [Gitea #43] The fork's `io.lotus.controls_state` report. */ public onControlsState(data: unknown) { if (typeof data !== 'object' || data === null) return; const { screensharing, layout, frameScreenshare, screensharePrompt, screensharePromptOpen } = data as { screensharing?: unknown; layout?: unknown; frameScreenshare?: unknown; screensharePrompt?: unknown; screensharePromptOpen?: unknown; }; const firstReport = !this.forkReady; this.forkReady = true; const frame = frameScreenshare === true; const prompt = screensharePrompt === true; const promptOpen = screensharePromptOpen === true; if (promptOpen !== this._screensharePromptOpen) { this._screensharePromptOpen = promptOpen; if (frame === this._frameScreenshare && prompt === this._screensharePrompt) { this.frameScreenshareListeners.forEach((l) => l()); } } if (frame !== this._frameScreenshare || prompt !== this._screensharePrompt) { const promptTurnedOn = prompt && !this._screensharePrompt; this._frameScreenshare = frame; this._screensharePrompt = prompt; this.frameScreenshareListeners.forEach((l) => l()); if (promptTurnedOn && !firstReport) this.sendFrameScreenshareAllowed(); } if (firstReport) { this.sendHotkeyCodes(); this.sendFrameScreenshareAllowed(); } this.applyControls( typeof screensharing === 'boolean' ? screensharing : this.screenshare, layout === 'spotlight' || layout === 'grid' ? layout === 'spotlight' : this.spotlight, ); } private applyControls(screenshare: boolean, spotlight: boolean) { const wasScreensharing = this.screenshare; // C-M6: when a screenshare stops, clear the screenshare-audio mute so a // later screenshare doesn't start pre-muted. const screenshareAudioMuted = wasScreensharing && !screenshare ? false : this.screenshareAudioMuted; // C-H3: the body observer now watches subtree:true, so this fires on any DOM // churn in EC's controls. Only re-emit (→ re-render every consumer) when one // of the values this method derives actually changed — microphone/video/sound // are copied unchanged from the current state here. if ( this.state.screenshare === screenshare && this.state.spotlight === spotlight && this.state.screenshareAudioMuted === screenshareAudioMuted ) { return; } this.state = new CallControlState( this.microphone, this.video, this.sound, screenshare, spotlight, screenshareAudioMuted, ); this.emitStateUpdate(); } public setMicrophone(enabled: boolean) { const payload: ElementMediaStatePayload = { audio_enabled: enabled, video_enabled: this.video, }; return this.setMediaState(payload); } public toggleMicrophone() { const payload: ElementMediaStatePayload = { audio_enabled: !this.microphone, video_enabled: this.video, }; return this.setMediaState(payload); } public toggleVideo() { const payload: ElementMediaStatePayload = { audio_enabled: this.microphone, video_enabled: !this.video, }; return this.setMediaState(payload); } public toggleSound() { const sound = !this.sound; // P6-2: commit state before setSound()/applyScreenshareAudioMuted() so // sendDeafenState() (which reads this.sound) reports the new value. const state = new CallControlState( this.microphone, this.video, sound, this.screenshare, this.spotlight, this.screenshareAudioMuted, ); this.state = state; this.setSound(); // After un-deafening, re-apply screenshare audio mute if active if (sound) this.applyScreenshareAudioMuted(); this.emitStateUpdate(); if (!sound) { this.micOnBeforeDeafen = this.microphone; if (this.microphone) this.toggleMicrophone(); } else if (this.micOnBeforeDeafen && !this.microphone) { this.micOnBeforeDeafen = false; this.toggleMicrophone(); } } public toggleScreenshareAudio() { const screenshareAudioMuted = !this.screenshareAudioMuted; this.state = new CallControlState( this.microphone, this.video, this.sound, this.screenshare, this.spotlight, screenshareAudioMuted, ); this.emitStateUpdate(); this.applyScreenshareAudioMuted(); } /** * Must be called synchronously inside the user's click: starting a share * calls getDisplayMedia in the frame, which needs that click. Where the * browser can hand the click over (Capability Delegation, Chromium incl. * WebView2) it is sent with delegation. Elsewhere the host bar hides its * button and EC's own shows in the frame (`frameScreenshare`); a plain send * still stops a share, which needs no click. */ public toggleScreenshare() { const data = { on: !this.screenshare }; if (canDelegateCapability()) { this.sendDelegated('io.lotus.set_screenshare', data, 'display-capture'); return; } this.sendForkAction('io.lotus.set_screenshare', data); } /** * Send a widget action with `postMessage(…, { delegate })` so the frame * receives the user's activation for `capability`. matrix-widget-api has no * postMessage options, so swap its `sendInternal` for this one synchronous * send (the request is posted before `send()` returns). Delegation refuses a * `*` target origin, so the frame's real origin is used. */ private sendDelegated(action: string, data: Record, capability: string): void { const target = this.iframe.contentWindow; if (!target) return; const targetOrigin = new URL(this.iframe.src, window.location.href).origin; const transport = this.call.transport as unknown as { sendInternal: (message: unknown) => void; }; const hadOwn = Object.prototype.hasOwnProperty.call(transport, 'sendInternal'); const original = transport.sendInternal; transport.sendInternal = (message: unknown) => target.postMessage(message, { targetOrigin, delegate: capability, } as WindowPostMessageOptions); try { this.call.transport.send(action, data).catch(() => undefined); } finally { if (hadOwn) transport.sendInternal = original; else delete (transport as { sendInternal?: unknown }).sendInternal; } } public toggleSpotlight() { this.sendForkAction('io.lotus.set_layout', { layout: this.spotlight ? 'grid' : 'spotlight', }); } public toggleReactions() { this.sendForkAction('io.lotus.toggle_reactions', {}); } public toggleSettings() { // Same as clicking EC's settings button: opens the modal. this.sendForkAction('io.lotus.open_settings', { open: true }); } private sendForkAction(action: string, data: Record): void { this.call.transport.send(action, data).catch(() => undefined); } /** * Focus a specific participant's camera tile in Element Call. * * EC renders video tiles as `[data-testid="videoTile"]`. Each tile wraps a * mute-status indicator with `aria-label` set to the participant's Matrix * user ID. We find the tile containing that user, switch to spotlight mode * if needed, then click the tile so EC's internal focus handler runs. * * Falls back to a plain spotlight toggle if the tile is not found (e.g. the * participant has their camera off and EC didn't render a video tile for * them yet). */ public get focusedUserId(): string | null { return this._focusedUserId; } private sendFocus(userId: string, id: string | null): void { // [EC#30] `id` is the fork's per-device media id (userId:deviceId) taken // from io.lotus.call_state, so a multi-device user pins the right device; // the fork falls back to userId (preferring the speaking device) when absent. this.call.transport .send('io.lotus.focus_participant', id ? { userId, id } : { userId }) .catch(() => undefined); } public focusCameraParticipant(userId: string, id: string | null = null): void { // [lotus #4] Pin the participant via the fork's widget action instead of // DOM-poking tiles. EC's layout honors it — including surfacing the camera // alongside a screenshare (A5) — and it's version-stable. The fork always // acks, so the promise resolves regardless. this._focusedUserId = userId; this._focusedMediaId = id; this.sendFocus(userId, id); // [Gitea #56] Notify state-update listeners so the menu can flip to "Unfocus camera". this.emitStateUpdate(); } /** [lotus #4] Clear any manual spotlight pin and return to speaker-follows. */ public clearFocusParticipant(): void { // [Gitea #56] No-op (and no redundant widget send) if nothing is pinned — // dispose() calls this unconditionally on every call teardown. if (this._focusedUserId === null) return; this._focusedUserId = null; this._focusedMediaId = null; this.call.transport.send('io.lotus.focus_participant', { userId: null }).catch(() => undefined); this.emitStateUpdate(); } /** * [lotus #3 / P5-15] Inject a soundboard clip into the call so other * participants hear it. The fork publishes it as a separate LiveKit audio * track (`io.lotus.inject_audio`) rather than splicing the mic. `url` must be * an https/blob URL the widget can fetch WITHOUT credentials — the host * resolves an mxc clip to a `blob:` object URL first (authenticated media * can't be fetched cross-realm by the widget) — and `audio` the same clip's * bytes, which the fork prefers. `volume` is 0–1. * * The local user does not hear their own published track, so callers should * also play the clip locally for feedback. */ public injectAudio( url: string, volume = 1, audio?: ArrayBuffer, ): Promise<{ played: boolean; reason?: string }> { // [EC#13] The fork now refuses while the local mic is muted and replies // { played:false, reason:"muted" }; older forks reply {} (treated as played). // [Gitea #43] `audio` carries the clip's bytes: a `blob:` URL only works on // this origin, so a call page on its own origin can't fetch it. Forks that // predate `audio` ignore it and use `url`. return this.call.transport .send< { url: string; volume: number; audio?: ArrayBuffer }, { played?: boolean; reason?: string } >('io.lotus.inject_audio', audio ? { url, volume, audio } : { url, volume }) .then((r) => ({ played: r?.played !== false, reason: r?.reason })) .catch(() => ({ played: true })); } /** * [lotus #7 / P5-31] Apply audio/screenshare encoding limits to the local * published tracks (the fork's `io.lotus.set_quality` action, via * `RTCRtpSender.setParameters` — no republish). Bitrates are bits/sec, * framerate is fps. A field set to `null` clears that cap. Settings are * sticky fork-side (re-applied on every re-publish / reconnect). Values are * clamped fork-side, so out-of-range input can't brick the encoder. */ public setQuality(settings: LotusQualityPayload): void { // C-M3: remember the request and only send once joined; sendQuality() gates // on this.joined so a pre-join call is a no-op that we replay on join. this.lastQuality = settings; this.sendQuality(); } // C-M3: push the last-requested quality to the fork. Gated on this.joined so // we never send io.lotus.set_quality before the fork's handler mounts (a // pre-join send would pend to a 10s widget timeout). private sendQuality(): void { if (!this.joined || !this.lastQuality) return; this.call.transport.send('io.lotus.set_quality', this.lastQuality).catch(() => undefined); } public dispose() { // [Gitea #56] Don't let a manual focus pin outlive the call. this.clearFocusParticipant(); } private emitStateUpdate() { this.emit(CallControlEvent.StateUpdate); } }