element-call/src/lotus/lotusAudioInject.ts

/*
Copyright 2026 Lotus Guild

SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/

import {
  type LocalTrack,
  type Room as LivekitRoom,
  Track,
  TrackEvent,
} from "livekit-client";
import { logger } from "matrix-js-sdk/lib/logger";
import { type IWidgetApiRequest } from "matrix-widget-api";

import { type CallViewModel } from "../state/CallViewModel/CallViewModel";
import { widget } from "../widget";
import { LotusWidgetActions } from "./lotusActions";
import { lotusFlag } from "./lotusWidget";

/** Hard cap so a malformed/huge clip can't hold a published track open forever. */
const MAX_CLIP_MS = 30_000;

// [lotus] One shared module-level AudioContext + MediaStreamAudioDestinationNode
// for ALL injected clips (#14), instead of `new AudioContext()` per clip. An
// in-call page already sits close to Chrome's per-document AudioContext limit
// (~6) between useAudioContext, MatrixAudioRenderer, LiveKit's own Room
// context and LotusDenoiseProcessor; rapid clip replacement (replace-mode
// closing the previous clip's context in the background) could transiently
// exceed the cap and make `new AudioContext()` throw. Lazily created on first
// use, ref-counted by the number of active `startLotusAudioInject` instances,
// and closed only when the last one tears down.
let sharedCtx: AudioContext | undefined;
let sharedDest: MediaStreamAudioDestinationNode | undefined;
let handlerCount = 0;

function acquireSharedAudio(): {
  ctx: AudioContext;
  dest: MediaStreamAudioDestinationNode;
} {
  if (!sharedCtx || sharedCtx.state === "closed") {
    sharedCtx = new AudioContext();
    sharedDest = sharedCtx.createMediaStreamDestination();
  }
  return { ctx: sharedCtx, dest: sharedDest! };
}

/**
 * Handle the host's `io.lotus.inject_audio` toWidget action (#3): mix a
 * soundboard clip into the call so other participants hear it.
 *
 * Rather than splice into the local mic track (which would fight the denoise
 * pipeline), we publish the clip as a separate `Unknown`-source LiveKit audio
 * track — which `MatrixAudioRenderer` already renders for valid call members —
 * and unpublish it when the clip ends. This is the real call-audio injection
 * that was impossible against the prebuilt EC bundle (LiveKit's
 * LocalParticipant lived in EC's module scope).
 *
 * [EC #47] Exception: rooms with a call policy issue tokens that only allow
 * listed sources, which refuse that separate track. There the clip is mixed
 * into the published microphone for its duration (`playIntoMicrophone`).
 *
 * Action data: `{ url?: string, audio?: ArrayBuffer, volume?: number }`.
 * `audio` is the clip's bytes (cinny #43: a host `blob:` URL can't be fetched
 * once EC is served from its own origin, so the host sends the bytes and they
 * win); otherwise `url` must be an https/blob URL (older hosts).
 *
 * No effect unless the host sends the action. Returns a teardown function that
 * also aborts any clip still playing.
 */
/**
 * Pure: whether these LiveKit permissions allow publishing the soundboard's
 * Unknown-source track (EC #47). No permissions or an empty source list
 * means unrestricted; otherwise UNKNOWN (TrackSource 0) must be allowed.
 */
export function canPublishSoundboard(
  perm:
    | { canPublish?: boolean; canPublishSources?: readonly number[] }
    | undefined,
): boolean {
  if (!perm) return true;
  if (perm.canPublish === false) return false;
  const sources = perm.canPublishSources ?? [];
  return sources.length === 0 || sources.includes(0);
}

/**
 * Pure: how a clip can reach the call under these permissions (EC #47).
 * - `"track"`: as its own Unknown-source track (unrestricted rooms).
 * - `"mic"`: mixed into the published microphone track. Rooms with a call
 *   policy grant only listed sources (microphone, maybe screenshare), which
 *   refuse the separate track, but the mic is allowed.
 * - `null`: neither (no publishing at all).
 */
export function soundboardMode(
  perm:
    | { canPublish?: boolean; canPublishSources?: readonly number[] }
    | undefined,
): "track" | "mic" | null {
  if (canPublishSoundboard(perm)) return "track";
  if (perm?.canPublish === false) return null;
  // TrackSource.MICROPHONE = 2
  return perm?.canPublishSources?.includes(2) ? "mic" : null;
}

export function startLotusAudioInject(vm: CallViewModel): () => void {
  const w = widget;
  if (!w) return () => undefined;

  // [lotus] Count this instance toward the shared AudioContext's lifetime
  // (#14) — closed only once the last active instance tears down.
  handlerCount++;

  // Track the set of connected LiveKit rooms to publish into. Drive off the
  // LOCAL participant's connection(s), not `livekitRoomItems$` — that stream
  // omits rooms with no remote members, so inject would no-op while you're
  // alone. Map the connections to their livekit rooms like `lotusDenoise.ts`.
  let rooms: LivekitRoom[] = [];
  const sub = vm.allConnections$.subscribe((data) => {
    rooms = data.getConnections().map((c) => c.livekitRoom);
  });

  // In-flight clips, so we can abort them on teardown (unmount / vm change /
  // call leave) instead of leaving audio blasting to peers.
  const activeClips = new Set<() => void>();

  const handler = (ev: CustomEvent<IWidgetApiRequest>): void => {
    if (!lotusFlag("lotusAudioInject")) {
      // Always ack so the transport doesn't hang, but only act when the host
      // has explicitly opted in: audio-inject publishes under the local
      // user's identity, so it must not be silently armed for every call.
      w.api.transport.reply(ev.detail, {});
      return;
    }
    const data = ev.detail.data as
      | { url?: unknown; audio?: unknown; volume?: unknown }
      | undefined;
    const clip = parseInjectSource(data);
    if (!clip) {
      w.api.transport.reply(ev.detail, {});
      logger.warn("[lotus] inject_audio: missing/invalid url or audio");
      return;
    }
    const volume =
      typeof data?.volume === "number" && data.volume >= 0 && data.volume <= 1
        ? data.volume
        : 1;

    // [lotus] Gate on the local mic being enabled (#13): the clip is
    // published as an independent track, fully decoupled from the mic
    // publication's mute state, so without this a muted (or push-to-talk
    // idle) user could still transmit soundboard audio under their own
    // identity — breaking the "I am muted, nothing I do makes noise" mental
    // model. Reply with a machine-readable reason so cinny's soundboard UI
    // can surface a hint instead of the click silently doing nothing.
    const micEnabled = rooms[0]?.localParticipant.isMicrophoneEnabled ?? true;
    if (!micEnabled) {
      w.api.transport.reply(ev.detail, { played: false, reason: "muted" });
      return;
    }

    // [EC #47] Clips publish as an Unknown-source track, which rooms with a
    // call policy refuse (their tokens list allowed sources). There the clip
    // is mixed into the published microphone instead; only when neither is
    // possible say so up front instead of failing silently.
    const modes = rooms.map((r) =>
      soundboardMode(r.localParticipant.permissions),
    );
    if (modes.some((m) => m === null)) {
      w.api.transport.reply(ev.detail, {
        played: false,
        reason: "not_permitted",
      });
      return;
    }
    const mode = modes.includes("mic") ? "mic" : "track";

    w.api.transport.reply(ev.detail, {});
    void playInjectedClip(clip, volume, rooms, activeClips, mode).catch((e) =>
      logger.warn("[lotus] inject_audio failed", e),
    );
  };

  w.lazyActions.on(LotusWidgetActions.InjectAudio, handler);
  return () => {
    sub.unsubscribe();
    w.lazyActions.off(LotusWidgetActions.InjectAudio, handler);
    // Abort anything still playing.
    // oxlint-disable-next-line unicorn/no-useless-spread -- the spread is a
    // required defensive copy: abort() deletes from activeClips while we
    // iterate it.
    // The spread is a required defensive copy: abort() deletes from
    // activeClips while we iterate it.
    // eslint-disable-next-line unicorn/no-useless-spread
    for (const abort of [...activeClips]) abort();
    // [lotus] Close the shared AudioContext only when the last active
    // instance tears down (#14).
    handlerCount--;
    if (handlerCount === 0 && sharedCtx) {
      const ctx = sharedCtx;
      sharedCtx = undefined;
      sharedDest = undefined;
      void ctx.close().catch(() => undefined);
    }
  };
}

/** Largest clip accepted as bytes (the host's soundboard clips are small). */
export const MAX_INJECT_BYTES = 8 * 1024 * 1024;

/**
 * The clip to play: the bytes the host sent (`audio`), or else a fetchable
 * `url`. Exported for tests.
 */
export function parseInjectSource(
  data: { url?: unknown; audio?: unknown } | undefined,
): string | ArrayBuffer | null {
  const audio = data?.audio;
  if (
    audio instanceof ArrayBuffer &&
    audio.byteLength > 0 &&
    audio.byteLength <= MAX_INJECT_BYTES
  ) {
    return audio;
  }
  return typeof data?.url === "string" ? safeMediaUrl(data.url) : null;
}

/** Only allow fetchable media URLs; never same-origin credentialed GETs etc. */
function safeMediaUrl(raw: string): string | null {
  try {
    const u = new URL(raw, window.location.href);
    return u.protocol === "https:" || u.protocol === "blob:" ? u.href : null;
  } catch {
    return null;
  }
}

async function playInjectedClip(
  clip: string | ArrayBuffer,
  volume: number,
  rooms: LivekitRoom[],
  activeClips: Set<() => void>,
  mode: "track" | "mic" = "track",
): Promise<void> {
  if (rooms.length === 0) {
    logger.warn("[lotus] inject_audio: no connected rooms");
    return;
  }

  // Max ONE clip at a time (replace mode): stop any in-flight or playing clip
  // before starting a new one, so clips can't overlap or be spammed.
  // The spread is a required defensive copy: abort() deletes from
  // activeClips while we iterate it.
  // eslint-disable-next-line unicorn/no-useless-spread
  for (const abort of [...activeClips]) abort();

  // A second inject action can arrive while THIS one is still awaiting its
  // fetch/decode/publish — before its real cleanup() exists. Register a
  // synchronous placeholder abort NOW, BEFORE the first await, so the
  // replace-mode loop above (run by that later action) cancels this one;
  // otherwise both clips would sail past their awaits and DOUBLE-PUBLISH. The
  // placeholder aborts the in-flight fetch and flips `aborted`, which we check
  // after every await; the real cleanup() replaces it once the track is live.
  let aborted = false;
  const controller = new AbortController();
  const placeholder = (): void => {
    aborted = true;
    controller.abort();
    activeClips.delete(placeholder);
  };
  activeClips.add(placeholder);

  let arrayBuffer: ArrayBuffer;
  if (typeof clip !== "string") {
    arrayBuffer = clip;
  } else {
    let resp: Response;
    try {
      resp = await fetch(clip, {
        credentials: "omit",
        mode: "cors",
        signal: controller.signal,
      });
    } catch (e) {
      // Superseded by a newer clip mid-fetch — expected, not a failure.
      if (aborted) return;
      throw e;
    }
    if (aborted) return;
    if (!resp.ok) {
      activeClips.delete(placeholder);
      throw new Error(`fetch ${clip} -> ${resp.status}`);
    }
    arrayBuffer = await resp.arrayBuffer();
    if (aborted) return;
  }

  // [lotus] Reuse the shared module-level context/destination (#14) rather
  // than `new AudioContext()` per clip — see the declaration above.
  const { ctx, dest } = acquireSharedAudio();
  // The action arrives via host postMessage, not a gesture in this iframe, so
  // the context may start suspended — resume it or the clip is silent and
  // `onended` never fires. A no-op if an earlier clip already resumed it.
  try {
    await ctx.resume();
  } catch {
    /* best effort */
  }
  if (aborted) return;
  if (ctx.state !== "running")
    logger.warn(`[lotus] inject_audio: AudioContext is ${ctx.state}`);

  let buffer: AudioBuffer;
  try {
    buffer = await ctx.decodeAudioData(arrayBuffer);
  } catch (e) {
    activeClips.delete(placeholder);
    throw e;
  }
  if (aborted) return;

  // Per clip, only the BufferSource/GainNode are created (#14) — the shared
  // context/destination are reused across every clip.
  const gain = ctx.createGain();
  gain.gain.value = volume;
  const source = ctx.createBufferSource();
  source.buffer = buffer;
  source.connect(gain);

  if (mode === "mic") {
    playIntoMicrophone(
      ctx,
      source,
      gain,
      buffer,
      rooms,
      activeClips,
      placeholder,
      () => aborted,
    );
    return;
  }

  gain.connect(dest);
  const mst = dest.stream.getAudioTracks()[0];
  if (!mst) {
    source.disconnect();
    gain.disconnect();
    activeClips.delete(placeholder);
    throw new Error("no audio track from destination");
  }

  // Publish (a clone of) the clip track to every connected room.
  const publications = await Promise.all(
    rooms.map(async (room) => {
      const clone = mst.clone();
      try {
        const pub = await room.localParticipant.publishTrack(clone, {
          source: Track.Source.Unknown,
          name: "lotus-soundboard",
          dtx: false,
          red: false,
        });
        return { room, pub };
      } catch (e) {
        clone.stop(); // don't leak the clone if publish failed
        logger.warn("[lotus] inject_audio: publish failed", e);
        return null;
      }
    }),
  );

  let cleanedUp = false;
  const cleanup = (): void => {
    if (cleanedUp) return;
    cleanedUp = true;
    activeClips.delete(cleanup);
    try {
      source.stop();
    } catch {
      /* already stopped */
    }
    // [lotus] Dispose only this clip's own nodes (#14) — the shared
    // AudioContext/destination outlive it and are closed separately, only
    // when the last startLotusAudioInject instance tears down.
    source.disconnect();
    gain.disconnect();
    for (const entry of publications) {
      if (entry?.pub.track)
        void entry.room.localParticipant
          .unpublishTrack(entry.pub.track, true)
          .catch(() => undefined);
    }
  };
  // Swap the synchronous placeholder for the real cleanup: from here an abort
  // (teardown or a newer clip) must unpublish the LIVE track, not just cancel a
  // fetch. This delete+add is synchronous (no await), so a newer clip's
  // replace-mode loop always sees exactly one of {placeholder, cleanup}.
  activeClips.delete(placeholder);
  activeClips.add(cleanup);

  // If a newer clip aborted us WHILE we were publishing, tear down now so we
  // don't leave an orphan track published after it ran its replace-mode loop.
  if (aborted) {
    cleanup();
    return;
  }

  source.onended = cleanup;
  // Safety net: clip metadata can lie (NaN/huge duration), so force teardown
  // after a sane, capped delay.
  const durationMs = Number.isFinite(buffer.duration)
    ? buffer.duration * 1000 + 500
    : MAX_CLIP_MS;
  const guard = setTimeout(
    cleanup,
    Math.min(MAX_CLIP_MS, Math.max(0, durationMs)),
  );
  source.addEventListener("ended", () => clearTimeout(guard));

  source.start();
}

/**
 * [EC #47] Mix the clip into the published microphone for its duration. For
 * every room, the track currently going out on the mic's RTCRtpSender (the
 * denoised one, when noise suppression is on) and the clip are mixed in a
 * MediaStreamAudioDestinationNode, and the sender switches to the mix with
 * `replaceTrack`. The server still sees one ordinary microphone track, which
 * rooms with a call policy allow. When the clip ends the original track goes
 * back. Muting mid-clip stops the clip (#13: muted means silent), and if
 * LiveKit swapped its own track in meanwhile (device change, denoise toggle)
 * that one is left alone.
 */
function playIntoMicrophone(
  ctx: AudioContext,
  source: AudioBufferSourceNode,
  gain: GainNode,
  buffer: AudioBuffer,
  rooms: LivekitRoom[],
  activeClips: Set<() => void>,
  placeholder: () => void,
  isAborted: () => boolean,
): void {
  type Mix = {
    sender: RTCRtpSender;
    original: MediaStreamTrack;
    mixed: MediaStreamTrack;
    mic: MediaStreamAudioSourceNode;
    dest: MediaStreamAudioDestinationNode;
    track: LocalTrack;
  };
  const mixes: Mix[] = [];
  let cleanedUp = false;
  const cleanup = (): void => {
    if (cleanedUp) return;
    cleanedUp = true;
    activeClips.delete(cleanup);
    activeClips.delete(placeholder);
    try {
      source.stop();
    } catch {
      /* not started or already stopped */
    }
    source.disconnect();
    gain.disconnect();
    for (const m of mixes) {
      m.track.off(TrackEvent.Muted, cleanup);
      m.mic.disconnect();
      // Only undo our own swap: LiveKit may have replaced the track since.
      if (m.sender.track === m.mixed)
        void m.sender.replaceTrack(m.original).catch(() => undefined);
      m.mixed.stop();
    }
  };

  for (const room of rooms) {
    const track = room.localParticipant.getTrackPublication(
      Track.Source.Microphone,
    )?.track;
    const sender = track?.sender;
    const original = sender?.track;
    if (!track || !sender || !original) continue;
    const dest = ctx.createMediaStreamDestination();
    const mic = ctx.createMediaStreamSource(new MediaStream([original]));
    mic.connect(dest);
    gain.connect(dest);
    const mixed = dest.stream.getAudioTracks()[0];
    if (!mixed) {
      mic.disconnect();
      continue;
    }
    mixes.push({ sender, original, mixed, mic, dest, track });
    track.on(TrackEvent.Muted, cleanup);
  }

  activeClips.delete(placeholder);
  activeClips.add(cleanup);
  if (mixes.length === 0) {
    logger.warn("[lotus] inject_audio: no published microphone to mix into");
    cleanup();
    return;
  }
  if (isAborted()) {
    cleanup();
    return;
  }

  // Switch every sender to its mix, then start the clip once they're live so
  // its start isn't clipped.
  void Promise.all(
    mixes.map(async (m) =>
      m.sender.replaceTrack(m.mixed).catch((e) => {
        logger.warn("[lotus] inject_audio: replaceTrack failed", e);
      }),
    ),
  ).then(() => {
    if (cleanedUp || isAborted()) {
      cleanup();
      return;
    }
    source.onended = cleanup;
    const durationMs = Number.isFinite(buffer.duration)
      ? buffer.duration * 1000 + 500
      : MAX_CLIP_MS;
    const guard = setTimeout(
      cleanup,
      Math.min(MAX_CLIP_MS, Math.max(0, durationMs)),
    );
    source.addEventListener("ended", () => clearTimeout(guard));
    source.start();
  });
}