feat(lotus): soundboard works in voice channels with a call policy (#47)

Clips publish as a separate Unknown-source track, which tokens limited to
listed sources (any room with io.lotus.room_quality) refuse, so #52 could
only answer not_permitted. Where the token allows the microphone, the
clip is now mixed into the published mic for its duration: the track on
the mic's RTCRtpSender (post-denoise) and the clip feed a
MediaStreamAudioDestinationNode, the sender switches to the mix with
replaceTrack, and the original goes back when the clip ends. Muting
mid-clip stops it; a track LiveKit swapped in meanwhile is left alone.
not_permitted remains for tokens without the microphone.

Fixes #47

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_017PCb59uY3UKqDqt7B6hzd5

src/lotus/lotusAudioInject.ts

@@ -5,7 +5,12 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
 Please see LICENSE in the repository root for full details.
 */
 
-import { type Room as LivekitRoom, Track } from "livekit-client";
+import {
+  type LocalTrack,
+  type Room as LivekitRoom,
+  Track,
+  TrackEvent,
+} from "livekit-client";
 import { logger } from "matrix-js-sdk/lib/logger";
 import { type IWidgetApiRequest } from "matrix-widget-api";
 
@@ -52,6 +57,10 @@ function acquireSharedAudio(): {
  * that was impossible against the prebuilt EC bundle (LiveKit's
  * LocalParticipant lived in EC's module scope).
  *
+ * [EC #47] Exception: rooms with a call policy issue tokens that only allow
+ * listed sources, which refuse that separate track. There the clip is mixed
+ * into the published microphone for its duration (`playIntoMicrophone`).
+ *
  * Action data: `{ url?: string, audio?: ArrayBuffer, volume?: number }`.
  * `audio` is the clip's bytes (cinny #43: a host `blob:` URL can't be fetched
  * once EC is served from its own origin, so the host sends the bytes and they
@@ -76,6 +85,25 @@ export function canPublishSoundboard(
   return sources.length === 0 || sources.includes(0);
 }
 
+/**
+ * Pure: how a clip can reach the call under these permissions (EC #47).
+ * - `"track"`: as its own Unknown-source track (unrestricted rooms).
+ * - `"mic"`: mixed into the published microphone track. Rooms with a call
+ *   policy grant only listed sources (microphone, maybe screenshare), which
+ *   refuse the separate track, but the mic is allowed.
+ * - `null`: neither (no publishing at all).
+ */
+export function soundboardMode(
+  perm:
+    | { canPublish?: boolean; canPublishSources?: readonly number[] }
+    | undefined,
+): "track" | "mic" | null {
+  if (canPublishSoundboard(perm)) return "track";
+  if (perm?.canPublish === false) return null;
+  // TrackSource.MICROPHONE = 2
+  return perm?.canPublishSources?.includes(2) ? "mic" : null;
+}
+
 export function startLotusAudioInject(vm: CallViewModel): () => void {
   const w = widget;
   if (!w) return () => undefined;
@@ -132,21 +160,24 @@ export function startLotusAudioInject(vm: CallViewModel): () => void {
       return;
     }
 
-    // [EC #47] Clips publish as an Unknown-source track. Audio-only rooms get
-    // a token limited to the microphone source, where LiveKit refuses that
-    // publish *after* we replied; say so up front instead of failing silently.
-    if (
-      rooms.some((r) => !canPublishSoundboard(r.localParticipant.permissions))
-    ) {
+    // [EC #47] Clips publish as an Unknown-source track, which rooms with a
+    // call policy refuse (their tokens list allowed sources). There the clip
+    // is mixed into the published microphone instead; only when neither is
+    // possible say so up front instead of failing silently.
+    const modes = rooms.map((r) =>
+      soundboardMode(r.localParticipant.permissions),
+    );
+    if (modes.some((m) => m === null)) {
       w.api.transport.reply(ev.detail, {
         played: false,
         reason: "not_permitted",
       });
       return;
     }
+    const mode = modes.includes("mic") ? "mic" : "track";
 
     w.api.transport.reply(ev.detail, {});
-    void playInjectedClip(clip, volume, rooms, activeClips).catch((e) =>
+    void playInjectedClip(clip, volume, rooms, activeClips, mode).catch((e) =>
       logger.warn("[lotus] inject_audio failed", e),
     );
   };
@@ -211,6 +242,7 @@ async function playInjectedClip(
   volume: number,
   rooms: LivekitRoom[],
   activeClips: Set<() => void>,
+  mode: "track" | "mic" = "track",
 ): Promise<void> {
   if (rooms.length === 0) {
     logger.warn("[lotus] inject_audio: no connected rooms");
@@ -295,8 +327,23 @@ async function playInjectedClip(
   gain.gain.value = volume;
   const source = ctx.createBufferSource();
   source.buffer = buffer;
-  source.connect(gain).connect(dest);
+  source.connect(gain);
 
+  if (mode === "mic") {
+    playIntoMicrophone(
+      ctx,
+      source,
+      gain,
+      buffer,
+      rooms,
+      activeClips,
+      placeholder,
+      () => aborted,
+    );
+    return;
+  }
+
+  gain.connect(dest);
   const mst = dest.stream.getAudioTracks()[0];
   if (!mst) {
     source.disconnect();
@@ -375,3 +422,114 @@ async function playInjectedClip(
 
   source.start();
 }
+
+/**
+ * [EC #47] Mix the clip into the published microphone for its duration. For
+ * every room, the track currently going out on the mic's RTCRtpSender (the
+ * denoised one, when noise suppression is on) and the clip are mixed in a
+ * MediaStreamAudioDestinationNode, and the sender switches to the mix with
+ * `replaceTrack`. The server still sees one ordinary microphone track, which
+ * rooms with a call policy allow. When the clip ends the original track goes
+ * back. Muting mid-clip stops the clip (#13: muted means silent), and if
+ * LiveKit swapped its own track in meanwhile (device change, denoise toggle)
+ * that one is left alone.
+ */
+function playIntoMicrophone(
+  ctx: AudioContext,
+  source: AudioBufferSourceNode,
+  gain: GainNode,
+  buffer: AudioBuffer,
+  rooms: LivekitRoom[],
+  activeClips: Set<() => void>,
+  placeholder: () => void,
+  isAborted: () => boolean,
+): void {
+  type Mix = {
+    sender: RTCRtpSender;
+    original: MediaStreamTrack;
+    mixed: MediaStreamTrack;
+    mic: MediaStreamAudioSourceNode;
+    dest: MediaStreamAudioDestinationNode;
+    track: LocalTrack;
+  };
+  const mixes: Mix[] = [];
+  let cleanedUp = false;
+  const cleanup = (): void => {
+    if (cleanedUp) return;
+    cleanedUp = true;
+    activeClips.delete(cleanup);
+    activeClips.delete(placeholder);
+    try {
+      source.stop();
+    } catch {
+      /* not started or already stopped */
+    }
+    source.disconnect();
+    gain.disconnect();
+    for (const m of mixes) {
+      m.track.off(TrackEvent.Muted, cleanup);
+      m.mic.disconnect();
+      // Only undo our own swap: LiveKit may have replaced the track since.
+      if (m.sender.track === m.mixed)
+        void m.sender.replaceTrack(m.original).catch(() => undefined);
+      m.mixed.stop();
+    }
+  };
+
+  for (const room of rooms) {
+    const track = room.localParticipant.getTrackPublication(
+      Track.Source.Microphone,
+    )?.track;
+    const sender = track?.sender;
+    const original = sender?.track;
+    if (!track || !sender || !original) continue;
+    const dest = ctx.createMediaStreamDestination();
+    const mic = ctx.createMediaStreamSource(new MediaStream([original]));
+    mic.connect(dest);
+    gain.connect(dest);
+    const mixed = dest.stream.getAudioTracks()[0];
+    if (!mixed) {
+      mic.disconnect();
+      continue;
+    }
+    mixes.push({ sender, original, mixed, mic, dest, track });
+    track.on(TrackEvent.Muted, cleanup);
+  }
+
+  activeClips.delete(placeholder);
+  activeClips.add(cleanup);
+  if (mixes.length === 0) {
+    logger.warn("[lotus] inject_audio: no published microphone to mix into");
+    cleanup();
+    return;
+  }
+  if (isAborted()) {
+    cleanup();
+    return;
+  }
+
+  // Switch every sender to its mix, then start the clip once they're live so
+  // its start isn't clipped.
+  void Promise.all(
+    mixes.map(async (m) =>
+      m.sender.replaceTrack(m.mixed).catch((e) => {
+        logger.warn("[lotus] inject_audio: replaceTrack failed", e);
+      }),
+    ),
+  ).then(() => {
+    if (cleanedUp || isAborted()) {
+      cleanup();
+      return;
+    }
+    source.onended = cleanup;
+    const durationMs = Number.isFinite(buffer.duration)
+      ? buffer.duration * 1000 + 500
+      : MAX_CLIP_MS;
+    const guard = setTimeout(
+      cleanup,
+      Math.min(MAX_CLIP_MS, Math.max(0, durationMs)),
+    );
+    source.addEventListener("ended", () => clearTimeout(guard));
+    source.start();
+  });
+}