chore(lotus): 0.25.0-lotus.28

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_017PCb59uY3UKqDqt7B6hzd5

LOTUS.md

@@ -37,7 +37,7 @@ checked against cinny's `lotus` branch.
 | ---------------------------------- | ----------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------- |
 | `io.lotus.call_state`              | call → host | in-call per-participant speaking / mute state. NOTE: matrix-widget-api `transport.send` is request/response — the host MUST register a handler that replies/acks each one (cinny's `listenAction` does, replying `{}`). If the host has no handler, ClientWidgetApi immediately error-replies "unsupported from-widget action", so the data is silently dropped and each throttled send rejects (caught) — functional miss + log churn, not a hang. | yes                        |
 | `io.lotus.focus_participant`       | host → call | pin/spotlight (or clear, with userId=null) a participant.                                                                                                                                                                                                                                                                                                                                                                                           | yes                        |
-| `io.lotus.inject_audio`            | host → call | play an audio clip into the call as a separate published track.                                                                                                                                                                                                                                                                                                                                                                                     | yes                        |
+| `io.lotus.inject_audio`            | host → call | play an audio clip into the call: as a separate published track, or mixed into the microphone where the token only allows listed sources (rooms with a call policy, #47). Replies `{ played: false, reason: "muted" \| "not_permitted" }` when it can't.                                                                                                                                                                                            | yes                        |
 | `io.lotus.set_quality`             | host → call | set audio/screenshare encoding quality (bitrate/framerate).                                                                                                                                                                                                                                                                                                                                                                                         | yes                        |
 | `io.lotus.decorations`             | host → call | per-user avatar-decoration image URLs for in-call tiles.                                                                                                                                                                                                                                                                                                                                                                                            | yes                        |
 | `io.lotus.set_deafen`              | host → call | deafen remote audio (and optionally mute screenshare audio).                                                                                                                                                                                                                                                                                                                                                                                        | yes                        |

embedded/web/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lotusguild/element-call-embedded",
-  "version": "0.25.0-lotus.26",
+  "version": "0.25.0-lotus.28",
   "files": [
     "README.md",
     "LICENSE-AGPL-3.0",

src/lotus/lotusAudioInject.test.ts

@@ -13,6 +13,7 @@ import { type CallViewModel } from "../state/CallViewModel/CallViewModel";
 import {
   canPublishSoundboard,
   MAX_INJECT_BYTES,
+  soundboardMode,
   parseInjectSource,
   startLotusAudioInject,
 } from "./lotusAudioInject";
@@ -94,6 +95,10 @@ class FakeAudioContext {
     connect: vi.fn((n: unknown) => n),
     disconnect: vi.fn(),
   }));
+  public createMediaStreamSource = vi.fn(() => ({
+    connect: vi.fn((n: unknown) => n),
+    disconnect: vi.fn(),
+  }));
   public createBufferSource = vi.fn(() => ({
     buffer: undefined as AudioBuffer | undefined,
     connect: vi.fn((n: unknown) => n),
@@ -116,6 +121,12 @@ beforeEach(() => {
   // leave a real timer pending.
   vi.useFakeTimers();
   vi.stubGlobal("AudioContext", FakeAudioContext);
+  vi.stubGlobal(
+    "MediaStream",
+    class {
+      public constructor(public tracks: unknown[]) {}
+    },
+  );
   vi.stubGlobal(
     "fetch",
     vi.fn().mockResolvedValue({
@@ -273,3 +284,127 @@ test("canPublishSoundboard: refused by a microphone-only grant (EC #47)", () =>
   ).toBe(true);
   expect(canPublishSoundboard({ canPublish: false })).toBe(false);
 });
+
+test("soundboardMode: own track when unrestricted, the mic under a call policy (EC #47)", () => {
+  expect(soundboardMode(undefined)).toBe("track");
+  expect(soundboardMode({ canPublish: true, canPublishSources: [] })).toBe(
+    "track",
+  );
+  expect(soundboardMode({ canPublish: true, canPublishSources: [2, 0] })).toBe(
+    "track",
+  );
+  expect(soundboardMode({ canPublish: true, canPublishSources: [2] })).toBe(
+    "mic",
+  );
+  expect(
+    soundboardMode({ canPublish: true, canPublishSources: [2, 3, 4] }),
+  ).toBe("mic");
+  expect(
+    soundboardMode({ canPublish: true, canPublishSources: [3] }),
+  ).toBeNull();
+  expect(soundboardMode({ canPublish: false })).toBeNull();
+});
+
+/** A room whose token only allows the microphone, with a published mic track. */
+function makePolicyRoom(sources: number[]): {
+  room: unknown;
+  sender: { track: MediaStreamTrack; replaceTrack: ReturnType<typeof vi.fn> };
+  original: MediaStreamTrack;
+  micTrack: EventEmitter;
+  publishTrack: ReturnType<typeof vi.fn>;
+} {
+  const original = makeTrack();
+  const sender = {
+    track: original,
+    replaceTrack: vi.fn(async (t: MediaStreamTrack) => {
+      await Promise.resolve();
+      sender.track = t;
+    }),
+  };
+  const micTrack = Object.assign(new EventEmitter(), { sender });
+  const publishTrack = vi.fn();
+  const room = {
+    localParticipant: {
+      isMicrophoneEnabled: true,
+      permissions: { canPublish: true, canPublishSources: sources },
+      publishTrack,
+      unpublishTrack: vi.fn(),
+      getTrackPublication: (): unknown => ({ track: micTrack }),
+    },
+  };
+  return { room, sender, original, micTrack, publishTrack };
+}
+
+test("[EC #47] under a mic-only grant the clip is mixed into the microphone, then the mic is restored", async () => {
+  const { room, sender, original, publishTrack } = makePolicyRoom([2]);
+  const stop = startLotusAudioInject(mockVm([room]));
+
+  send({ url: "https://example.com/clip.mp3" });
+  await flush();
+
+  expect(reply).toHaveBeenCalledWith(expect.anything(), {});
+  expect(publishTrack).not.toHaveBeenCalled();
+  expect(sender.replaceTrack).toHaveBeenCalledTimes(1);
+  const mixed = sender.track;
+  expect(mixed).not.toBe(original);
+  const source = contextInstances[0].createBufferSource.mock.results[0].value;
+  expect(source.start).toHaveBeenCalled();
+
+  source.onended();
+  await flush();
+  expect(sender.replaceTrack).toHaveBeenLastCalledWith(original);
+  expect(sender.track).toBe(original);
+  expect(mixed.stop).toHaveBeenCalled();
+
+  stop();
+});
+
+test("[EC #47] muting mid-clip stops the clip and restores the microphone", async () => {
+  const { room, sender, original, micTrack } = makePolicyRoom([2]);
+  const stop = startLotusAudioInject(mockVm([room]));
+
+  send({ url: "https://example.com/clip.mp3" });
+  await flush();
+  const source = contextInstances[0].createBufferSource.mock.results[0].value;
+
+  micTrack.emit("muted");
+  await flush();
+  expect(source.stop).toHaveBeenCalled();
+  expect(sender.track).toBe(original);
+
+  stop();
+});
+
+test("[EC #47] a track LiveKit swapped in meanwhile is not overwritten on restore", async () => {
+  const { room, sender, original } = makePolicyRoom([2]);
+  const stop = startLotusAudioInject(mockVm([room]));
+
+  send({ url: "https://example.com/clip.mp3" });
+  await flush();
+  const newer = makeTrack();
+  sender.track = newer; // e.g. a device switch while the clip plays
+  const source = contextInstances[0].createBufferSource.mock.results[0].value;
+  source.onended();
+  await flush();
+  expect(sender.track).toBe(newer);
+  expect(sender.replaceTrack).not.toHaveBeenCalledWith(original);
+
+  stop();
+});
+
+test("[EC #47] no microphone grant at all is still refused up front", async () => {
+  const { room, publishTrack, sender } = makePolicyRoom([3]);
+  const stop = startLotusAudioInject(mockVm([room]));
+
+  send({ url: "https://example.com/clip.mp3" });
+  await flush();
+
+  expect(reply).toHaveBeenCalledWith(expect.anything(), {
+    played: false,
+    reason: "not_permitted",
+  });
+  expect(publishTrack).not.toHaveBeenCalled();
+  expect(sender.replaceTrack).not.toHaveBeenCalled();
+
+  stop();
+});

src/lotus/lotusAudioInject.ts

@@ -5,7 +5,12 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
 Please see LICENSE in the repository root for full details.
 */
 
-import { type Room as LivekitRoom, Track } from "livekit-client";
+import {
+  type LocalTrack,
+  type Room as LivekitRoom,
+  Track,
+  TrackEvent,
+} from "livekit-client";
 import { logger } from "matrix-js-sdk/lib/logger";
 import { type IWidgetApiRequest } from "matrix-widget-api";
 
@@ -52,6 +57,10 @@ function acquireSharedAudio(): {
  * that was impossible against the prebuilt EC bundle (LiveKit's
  * LocalParticipant lived in EC's module scope).
  *
+ * [EC #47] Exception: rooms with a call policy issue tokens that only allow
+ * listed sources, which refuse that separate track. There the clip is mixed
+ * into the published microphone for its duration (`playIntoMicrophone`).
+ *
  * Action data: `{ url?: string, audio?: ArrayBuffer, volume?: number }`.
  * `audio` is the clip's bytes (cinny #43: a host `blob:` URL can't be fetched
  * once EC is served from its own origin, so the host sends the bytes and they
@@ -76,6 +85,25 @@ export function canPublishSoundboard(
   return sources.length === 0 || sources.includes(0);
 }
 
+/**
+ * Pure: how a clip can reach the call under these permissions (EC #47).
+ * - `"track"`: as its own Unknown-source track (unrestricted rooms).
+ * - `"mic"`: mixed into the published microphone track. Rooms with a call
+ *   policy grant only listed sources (microphone, maybe screenshare), which
+ *   refuse the separate track, but the mic is allowed.
+ * - `null`: neither (no publishing at all).
+ */
+export function soundboardMode(
+  perm:
+    | { canPublish?: boolean; canPublishSources?: readonly number[] }
+    | undefined,
+): "track" | "mic" | null {
+  if (canPublishSoundboard(perm)) return "track";
+  if (perm?.canPublish === false) return null;
+  // TrackSource.MICROPHONE = 2
+  return perm?.canPublishSources?.includes(2) ? "mic" : null;
+}
+
 export function startLotusAudioInject(vm: CallViewModel): () => void {
   const w = widget;
   if (!w) return () => undefined;
@@ -132,21 +160,24 @@ export function startLotusAudioInject(vm: CallViewModel): () => void {
       return;
     }
 
-    // [EC #47] Clips publish as an Unknown-source track. Audio-only rooms get
-    // a token limited to the microphone source, where LiveKit refuses that
-    // publish *after* we replied; say so up front instead of failing silently.
-    if (
-      rooms.some((r) => !canPublishSoundboard(r.localParticipant.permissions))
-    ) {
+    // [EC #47] Clips publish as an Unknown-source track, which rooms with a
+    // call policy refuse (their tokens list allowed sources). There the clip
+    // is mixed into the published microphone instead; only when neither is
+    // possible say so up front instead of failing silently.
+    const modes = rooms.map((r) =>
+      soundboardMode(r.localParticipant.permissions),
+    );
+    if (modes.some((m) => m === null)) {
       w.api.transport.reply(ev.detail, {
         played: false,
         reason: "not_permitted",
       });
       return;
     }
+    const mode = modes.includes("mic") ? "mic" : "track";
 
     w.api.transport.reply(ev.detail, {});
-    void playInjectedClip(clip, volume, rooms, activeClips).catch((e) =>
+    void playInjectedClip(clip, volume, rooms, activeClips, mode).catch((e) =>
       logger.warn("[lotus] inject_audio failed", e),
     );
   };
@@ -211,6 +242,7 @@ async function playInjectedClip(
   volume: number,
   rooms: LivekitRoom[],
   activeClips: Set<() => void>,
+  mode: "track" | "mic" = "track",
 ): Promise<void> {
   if (rooms.length === 0) {
     logger.warn("[lotus] inject_audio: no connected rooms");
@@ -295,8 +327,23 @@ async function playInjectedClip(
   gain.gain.value = volume;
   const source = ctx.createBufferSource();
   source.buffer = buffer;
-  source.connect(gain).connect(dest);
+  source.connect(gain);
 
+  if (mode === "mic") {
+    playIntoMicrophone(
+      ctx,
+      source,
+      gain,
+      buffer,
+      rooms,
+      activeClips,
+      placeholder,
+      () => aborted,
+    );
+    return;
+  }
+
+  gain.connect(dest);
   const mst = dest.stream.getAudioTracks()[0];
   if (!mst) {
     source.disconnect();
@@ -375,3 +422,114 @@ async function playInjectedClip(
 
   source.start();
 }
+
+/**
+ * [EC #47] Mix the clip into the published microphone for its duration. For
+ * every room, the track currently going out on the mic's RTCRtpSender (the
+ * denoised one, when noise suppression is on) and the clip are mixed in a
+ * MediaStreamAudioDestinationNode, and the sender switches to the mix with
+ * `replaceTrack`. The server still sees one ordinary microphone track, which
+ * rooms with a call policy allow. When the clip ends the original track goes
+ * back. Muting mid-clip stops the clip (#13: muted means silent), and if
+ * LiveKit swapped its own track in meanwhile (device change, denoise toggle)
+ * that one is left alone.
+ */
+function playIntoMicrophone(
+  ctx: AudioContext,
+  source: AudioBufferSourceNode,
+  gain: GainNode,
+  buffer: AudioBuffer,
+  rooms: LivekitRoom[],
+  activeClips: Set<() => void>,
+  placeholder: () => void,
+  isAborted: () => boolean,
+): void {
+  type Mix = {
+    sender: RTCRtpSender;
+    original: MediaStreamTrack;
+    mixed: MediaStreamTrack;
+    mic: MediaStreamAudioSourceNode;
+    dest: MediaStreamAudioDestinationNode;
+    track: LocalTrack;
+  };
+  const mixes: Mix[] = [];
+  let cleanedUp = false;
+  const cleanup = (): void => {
+    if (cleanedUp) return;
+    cleanedUp = true;
+    activeClips.delete(cleanup);
+    activeClips.delete(placeholder);
+    try {
+      source.stop();
+    } catch {
+      /* not started or already stopped */
+    }
+    source.disconnect();
+    gain.disconnect();
+    for (const m of mixes) {
+      m.track.off(TrackEvent.Muted, cleanup);
+      m.mic.disconnect();
+      // Only undo our own swap: LiveKit may have replaced the track since.
+      if (m.sender.track === m.mixed)
+        void m.sender.replaceTrack(m.original).catch(() => undefined);
+      m.mixed.stop();
+    }
+  };
+
+  for (const room of rooms) {
+    const track = room.localParticipant.getTrackPublication(
+      Track.Source.Microphone,
+    )?.track;
+    const sender = track?.sender;
+    const original = sender?.track;
+    if (!track || !sender || !original) continue;
+    const dest = ctx.createMediaStreamDestination();
+    const mic = ctx.createMediaStreamSource(new MediaStream([original]));
+    mic.connect(dest);
+    gain.connect(dest);
+    const mixed = dest.stream.getAudioTracks()[0];
+    if (!mixed) {
+      mic.disconnect();
+      continue;
+    }
+    mixes.push({ sender, original, mixed, mic, dest, track });
+    track.on(TrackEvent.Muted, cleanup);
+  }
+
+  activeClips.delete(placeholder);
+  activeClips.add(cleanup);
+  if (mixes.length === 0) {
+    logger.warn("[lotus] inject_audio: no published microphone to mix into");
+    cleanup();
+    return;
+  }
+  if (isAborted()) {
+    cleanup();
+    return;
+  }
+
+  // Switch every sender to its mix, then start the clip once they're live so
+  // its start isn't clipped.
+  void Promise.all(
+    mixes.map(async (m) =>
+      m.sender.replaceTrack(m.mixed).catch((e) => {
+        logger.warn("[lotus] inject_audio: replaceTrack failed", e);
+      }),
+    ),
+  ).then(() => {
+    if (cleanedUp || isAborted()) {
+      cleanup();
+      return;
+    }
+    source.onended = cleanup;
+    const durationMs = Number.isFinite(buffer.duration)
+      ? buffer.duration * 1000 + 500
+      : MAX_CLIP_MS;
+    const guard = setTimeout(
+      cleanup,
+      Math.min(MAX_CLIP_MS, Math.max(0, durationMs)),
+    );
+    source.addEventListener("ended", () => clearTimeout(guard));
+    source.start();
+  });
+}