diff --git a/src/lotus/lotusAudioInject.ts b/src/lotus/lotusAudioInject.ts new file mode 100644 index 00000000..79c2d972 --- /dev/null +++ b/src/lotus/lotusAudioInject.ts @@ -0,0 +1,139 @@ +/* +Copyright 2026 Lotus Guild + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { type Room as LivekitRoom, Track } from "livekit-client"; +import { logger } from "matrix-js-sdk/lib/logger"; +import { type IWidgetApiRequest } from "matrix-widget-api"; + +import { type CallViewModel } from "../state/CallViewModel/CallViewModel"; +import { widget } from "../widget"; +import { LotusWidgetActions } from "./lotusActions"; + +/** Hard cap so a malformed/huge clip can't hold a published track open forever. */ +const MAX_CLIP_MS = 30_000; + +/** + * Handle the host's `io.lotus.inject_audio` toWidget action (#3): mix a + * soundboard clip into the call so other participants hear it. + * + * Rather than splice into the local mic track (which would fight the denoise + * pipeline), we publish the clip as a separate `Unknown`-source LiveKit audio + * track — which `MatrixAudioRenderer` already renders for valid call members — + * and unpublish it when the clip ends. This is the real call-audio injection + * that was impossible against the prebuilt EC bundle (LiveKit's + * LocalParticipant lived in EC's module scope). + * + * Action data: `{ url: string, volume?: number }`. `url` must already be a + * fetchable http(s)/blob URL (the host resolves mxc → media URL). + * + * No effect unless the host sends the action. Returns a teardown function. + */ +export function startLotusAudioInject(vm: CallViewModel): () => void { + const w = widget; + if (!w) return () => undefined; + + // Track the set of connected LiveKit rooms to publish into. + let rooms: LivekitRoom[] = []; + const sub = vm.livekitRoomItems$.subscribe((items) => { + rooms = items.map((i) => i.livekitRoom); + }); + + const handler = (ev: CustomEvent): void => { + void w.api.transport.reply(ev.detail, {}); + const data = ev.detail.data as + | { url?: unknown; volume?: unknown } + | undefined; + const url = typeof data?.url === "string" ? data.url : null; + if (!url) { + logger.warn("[lotus] inject_audio: missing url"); + return; + } + const volume = + typeof data?.volume === "number" && + data.volume >= 0 && + data.volume <= 1 + ? data.volume + : 1; + void playInjectedClip(url, volume, rooms).catch((e) => + logger.warn("[lotus] inject_audio failed", e), + ); + }; + + w.lazyActions.on(LotusWidgetActions.InjectAudio, handler); + return () => { + sub.unsubscribe(); + w.lazyActions.off(LotusWidgetActions.InjectAudio, handler); + }; +} + +async function playInjectedClip( + url: string, + volume: number, + rooms: LivekitRoom[], +): Promise { + if (rooms.length === 0) { + logger.warn("[lotus] inject_audio: no connected rooms"); + return; + } + + const resp = await fetch(url); + if (!resp.ok) throw new Error(`fetch ${url} -> ${resp.status}`); + const arrayBuffer = await resp.arrayBuffer(); + + const ctx = new AudioContext(); + const buffer = await ctx.decodeAudioData(arrayBuffer); + const dest = ctx.createMediaStreamDestination(); + const gain = ctx.createGain(); + gain.gain.value = volume; + const source = ctx.createBufferSource(); + source.buffer = buffer; + source.connect(gain).connect(dest); + + const mst = dest.stream.getAudioTracks()[0]; + if (!mst) { + void ctx.close(); + throw new Error("no audio track from destination"); + } + + // Publish (a clone of) the clip track to every connected room. + const publications = await Promise.all( + rooms.map(async (room) => { + try { + const pub = await room.localParticipant.publishTrack(mst.clone(), { + source: Track.Source.Unknown, + name: "lotus-soundboard", + dtx: false, + red: false, + }); + return { room, pub }; + } catch (e) { + logger.warn("[lotus] inject_audio: publish failed", e); + return null; + } + }), + ); + + let cleanedUp = false; + const cleanup = (): void => { + if (cleanedUp) return; + cleanedUp = true; + for (const entry of publications) { + if (entry?.pub.track) + void entry.room.localParticipant + .unpublishTrack(entry.pub.track, true) + .catch(() => undefined); + } + void ctx.close().catch(() => undefined); + }; + + source.onended = cleanup; + // Safety net: clip metadata can lie, so force teardown after the cap. + const guard = setTimeout(cleanup, Math.min(MAX_CLIP_MS, buffer.duration * 1000 + 500)); + source.addEventListener("ended", () => clearTimeout(guard)); + + source.start(); +} diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index 5bc2b115..5f29ea1f 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -31,6 +31,7 @@ import { useCallViewKeyboardShortcuts } from "../useCallViewKeyboardShortcuts"; import { widget } from "../widget"; import { startLotusCallState } from "../lotus/lotusCallState"; import { startLotusFocus } from "../lotus/lotusFocus"; +import { startLotusAudioInject } from "../lotus/lotusAudioInject"; import styles from "./InCallView.module.css"; import { GridTile } from "../tile/GridTile"; import { SettingsModal, defaultSettingsTab } from "../settings/SettingsModal"; @@ -287,6 +288,9 @@ export const InCallView: FC = ({ // [lotus] Handle the host's io.lotus.focus_participant action to pin a // participant to the spotlight (#4). No-op unless the host sends it. useEffect(() => startLotusFocus(vm), [vm]); + // [lotus] Handle the host's io.lotus.inject_audio action to mix a soundboard + // clip into the call as a separate track (#3). No-op unless the host sends it. + useEffect(() => startLotusAudioInject(vm), [vm]); const fatalCallError = useBehavior(vm.fatalError$); // Stop the rendering and throw for the error boundary