From 657271163f00c39182b6cbd061bacf7d71e4ee36 Mon Sep 17 00:00:00 2001 From: Lotus CI Date: Mon, 19 Oct 2026 23:23:06 -0400 Subject: [PATCH 1/2] feat(lotus): soundboard works in voice channels with a call policy (#47) Clips publish as a separate Unknown-source track, which tokens limited to listed sources (any room with io.lotus.room_quality) refuse, so #52 could only answer not_permitted. Where the token allows the microphone, the clip is now mixed into the published mic for its duration: the track on the mic's RTCRtpSender (post-denoise) and the clip feed a MediaStreamAudioDestinationNode, the sender switches to the mix with replaceTrack, and the original goes back when the clip ends. Muting mid-clip stops it; a track LiveKit swapped in meanwhile is left alone. not_permitted remains for tokens without the microphone. Fixes #47 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_017PCb59uY3UKqDqt7B6hzd5 --- LOTUS.md | 2 +- src/lotus/lotusAudioInject.test.ts | 135 ++++++++++++++++++++++ src/lotus/lotusAudioInject.ts | 176 +++++++++++++++++++++++++++-- 3 files changed, 303 insertions(+), 10 deletions(-) diff --git a/LOTUS.md b/LOTUS.md index 21b8f353..c7703705 100644 --- a/LOTUS.md +++ b/LOTUS.md @@ -37,7 +37,7 @@ checked against cinny's `lotus` branch. | ---------------------------------- | ----------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------- | | `io.lotus.call_state` | call → host | in-call per-participant speaking / mute state. NOTE: matrix-widget-api `transport.send` is request/response — the host MUST register a handler that replies/acks each one (cinny's `listenAction` does, replying `{}`). If the host has no handler, ClientWidgetApi immediately error-replies "unsupported from-widget action", so the data is silently dropped and each throttled send rejects (caught) — functional miss + log churn, not a hang. | yes | | `io.lotus.focus_participant` | host → call | pin/spotlight (or clear, with userId=null) a participant. | yes | -| `io.lotus.inject_audio` | host → call | play an audio clip into the call as a separate published track. | yes | +| `io.lotus.inject_audio` | host → call | play an audio clip into the call: as a separate published track, or mixed into the microphone where the token only allows listed sources (rooms with a call policy, #47). Replies `{ played: false, reason: "muted" \| "not_permitted" }` when it can't. | yes | | `io.lotus.set_quality` | host → call | set audio/screenshare encoding quality (bitrate/framerate). | yes | | `io.lotus.decorations` | host → call | per-user avatar-decoration image URLs for in-call tiles. | yes | | `io.lotus.set_deafen` | host → call | deafen remote audio (and optionally mute screenshare audio). | yes | diff --git a/src/lotus/lotusAudioInject.test.ts b/src/lotus/lotusAudioInject.test.ts index 9570df11..78cd7c65 100644 --- a/src/lotus/lotusAudioInject.test.ts +++ b/src/lotus/lotusAudioInject.test.ts @@ -13,6 +13,7 @@ import { type CallViewModel } from "../state/CallViewModel/CallViewModel"; import { canPublishSoundboard, MAX_INJECT_BYTES, + soundboardMode, parseInjectSource, startLotusAudioInject, } from "./lotusAudioInject"; @@ -94,6 +95,10 @@ class FakeAudioContext { connect: vi.fn((n: unknown) => n), disconnect: vi.fn(), })); + public createMediaStreamSource = vi.fn(() => ({ + connect: vi.fn((n: unknown) => n), + disconnect: vi.fn(), + })); public createBufferSource = vi.fn(() => ({ buffer: undefined as AudioBuffer | undefined, connect: vi.fn((n: unknown) => n), @@ -116,6 +121,12 @@ beforeEach(() => { // leave a real timer pending. vi.useFakeTimers(); vi.stubGlobal("AudioContext", FakeAudioContext); + vi.stubGlobal( + "MediaStream", + class { + public constructor(public tracks: unknown[]) {} + }, + ); vi.stubGlobal( "fetch", vi.fn().mockResolvedValue({ @@ -273,3 +284,127 @@ test("canPublishSoundboard: refused by a microphone-only grant (EC #47)", () => ).toBe(true); expect(canPublishSoundboard({ canPublish: false })).toBe(false); }); + +test("soundboardMode: own track when unrestricted, the mic under a call policy (EC #47)", () => { + expect(soundboardMode(undefined)).toBe("track"); + expect(soundboardMode({ canPublish: true, canPublishSources: [] })).toBe( + "track", + ); + expect(soundboardMode({ canPublish: true, canPublishSources: [2, 0] })).toBe( + "track", + ); + expect(soundboardMode({ canPublish: true, canPublishSources: [2] })).toBe( + "mic", + ); + expect( + soundboardMode({ canPublish: true, canPublishSources: [2, 3, 4] }), + ).toBe("mic"); + expect( + soundboardMode({ canPublish: true, canPublishSources: [3] }), + ).toBeNull(); + expect(soundboardMode({ canPublish: false })).toBeNull(); +}); + +/** A room whose token only allows the microphone, with a published mic track. */ +function makePolicyRoom(sources: number[]): { + room: unknown; + sender: { track: MediaStreamTrack; replaceTrack: ReturnType }; + original: MediaStreamTrack; + micTrack: EventEmitter; + publishTrack: ReturnType; +} { + const original = makeTrack(); + const sender = { + track: original, + replaceTrack: vi.fn(async (t: MediaStreamTrack) => { + await Promise.resolve(); + sender.track = t; + }), + }; + const micTrack = Object.assign(new EventEmitter(), { sender }); + const publishTrack = vi.fn(); + const room = { + localParticipant: { + isMicrophoneEnabled: true, + permissions: { canPublish: true, canPublishSources: sources }, + publishTrack, + unpublishTrack: vi.fn(), + getTrackPublication: (): unknown => ({ track: micTrack }), + }, + }; + return { room, sender, original, micTrack, publishTrack }; +} + +test("[EC #47] under a mic-only grant the clip is mixed into the microphone, then the mic is restored", async () => { + const { room, sender, original, publishTrack } = makePolicyRoom([2]); + const stop = startLotusAudioInject(mockVm([room])); + + send({ url: "https://example.com/clip.mp3" }); + await flush(); + + expect(reply).toHaveBeenCalledWith(expect.anything(), {}); + expect(publishTrack).not.toHaveBeenCalled(); + expect(sender.replaceTrack).toHaveBeenCalledTimes(1); + const mixed = sender.track; + expect(mixed).not.toBe(original); + const source = contextInstances[0].createBufferSource.mock.results[0].value; + expect(source.start).toHaveBeenCalled(); + + source.onended(); + await flush(); + expect(sender.replaceTrack).toHaveBeenLastCalledWith(original); + expect(sender.track).toBe(original); + expect(mixed.stop).toHaveBeenCalled(); + + stop(); +}); + +test("[EC #47] muting mid-clip stops the clip and restores the microphone", async () => { + const { room, sender, original, micTrack } = makePolicyRoom([2]); + const stop = startLotusAudioInject(mockVm([room])); + + send({ url: "https://example.com/clip.mp3" }); + await flush(); + const source = contextInstances[0].createBufferSource.mock.results[0].value; + + micTrack.emit("muted"); + await flush(); + expect(source.stop).toHaveBeenCalled(); + expect(sender.track).toBe(original); + + stop(); +}); + +test("[EC #47] a track LiveKit swapped in meanwhile is not overwritten on restore", async () => { + const { room, sender, original } = makePolicyRoom([2]); + const stop = startLotusAudioInject(mockVm([room])); + + send({ url: "https://example.com/clip.mp3" }); + await flush(); + const newer = makeTrack(); + sender.track = newer; // e.g. a device switch while the clip plays + const source = contextInstances[0].createBufferSource.mock.results[0].value; + source.onended(); + await flush(); + expect(sender.track).toBe(newer); + expect(sender.replaceTrack).not.toHaveBeenCalledWith(original); + + stop(); +}); + +test("[EC #47] no microphone grant at all is still refused up front", async () => { + const { room, publishTrack, sender } = makePolicyRoom([3]); + const stop = startLotusAudioInject(mockVm([room])); + + send({ url: "https://example.com/clip.mp3" }); + await flush(); + + expect(reply).toHaveBeenCalledWith(expect.anything(), { + played: false, + reason: "not_permitted", + }); + expect(publishTrack).not.toHaveBeenCalled(); + expect(sender.replaceTrack).not.toHaveBeenCalled(); + + stop(); +}); diff --git a/src/lotus/lotusAudioInject.ts b/src/lotus/lotusAudioInject.ts index 19a16485..1da0c3c4 100644 --- a/src/lotus/lotusAudioInject.ts +++ b/src/lotus/lotusAudioInject.ts @@ -5,7 +5,12 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { type Room as LivekitRoom, Track } from "livekit-client"; +import { + type LocalTrack, + type Room as LivekitRoom, + Track, + TrackEvent, +} from "livekit-client"; import { logger } from "matrix-js-sdk/lib/logger"; import { type IWidgetApiRequest } from "matrix-widget-api"; @@ -52,6 +57,10 @@ function acquireSharedAudio(): { * that was impossible against the prebuilt EC bundle (LiveKit's * LocalParticipant lived in EC's module scope). * + * [EC #47] Exception: rooms with a call policy issue tokens that only allow + * listed sources, which refuse that separate track. There the clip is mixed + * into the published microphone for its duration (`playIntoMicrophone`). + * * Action data: `{ url?: string, audio?: ArrayBuffer, volume?: number }`. * `audio` is the clip's bytes (cinny #43: a host `blob:` URL can't be fetched * once EC is served from its own origin, so the host sends the bytes and they @@ -76,6 +85,25 @@ export function canPublishSoundboard( return sources.length === 0 || sources.includes(0); } +/** + * Pure: how a clip can reach the call under these permissions (EC #47). + * - `"track"`: as its own Unknown-source track (unrestricted rooms). + * - `"mic"`: mixed into the published microphone track. Rooms with a call + * policy grant only listed sources (microphone, maybe screenshare), which + * refuse the separate track, but the mic is allowed. + * - `null`: neither (no publishing at all). + */ +export function soundboardMode( + perm: + | { canPublish?: boolean; canPublishSources?: readonly number[] } + | undefined, +): "track" | "mic" | null { + if (canPublishSoundboard(perm)) return "track"; + if (perm?.canPublish === false) return null; + // TrackSource.MICROPHONE = 2 + return perm?.canPublishSources?.includes(2) ? "mic" : null; +} + export function startLotusAudioInject(vm: CallViewModel): () => void { const w = widget; if (!w) return () => undefined; @@ -132,21 +160,24 @@ export function startLotusAudioInject(vm: CallViewModel): () => void { return; } - // [EC #47] Clips publish as an Unknown-source track. Audio-only rooms get - // a token limited to the microphone source, where LiveKit refuses that - // publish *after* we replied; say so up front instead of failing silently. - if ( - rooms.some((r) => !canPublishSoundboard(r.localParticipant.permissions)) - ) { + // [EC #47] Clips publish as an Unknown-source track, which rooms with a + // call policy refuse (their tokens list allowed sources). There the clip + // is mixed into the published microphone instead; only when neither is + // possible say so up front instead of failing silently. + const modes = rooms.map((r) => + soundboardMode(r.localParticipant.permissions), + ); + if (modes.some((m) => m === null)) { w.api.transport.reply(ev.detail, { played: false, reason: "not_permitted", }); return; } + const mode = modes.includes("mic") ? "mic" : "track"; w.api.transport.reply(ev.detail, {}); - void playInjectedClip(clip, volume, rooms, activeClips).catch((e) => + void playInjectedClip(clip, volume, rooms, activeClips, mode).catch((e) => logger.warn("[lotus] inject_audio failed", e), ); }; @@ -211,6 +242,7 @@ async function playInjectedClip( volume: number, rooms: LivekitRoom[], activeClips: Set<() => void>, + mode: "track" | "mic" = "track", ): Promise { if (rooms.length === 0) { logger.warn("[lotus] inject_audio: no connected rooms"); @@ -295,8 +327,23 @@ async function playInjectedClip( gain.gain.value = volume; const source = ctx.createBufferSource(); source.buffer = buffer; - source.connect(gain).connect(dest); + source.connect(gain); + if (mode === "mic") { + playIntoMicrophone( + ctx, + source, + gain, + buffer, + rooms, + activeClips, + placeholder, + () => aborted, + ); + return; + } + + gain.connect(dest); const mst = dest.stream.getAudioTracks()[0]; if (!mst) { source.disconnect(); @@ -375,3 +422,114 @@ async function playInjectedClip( source.start(); } + +/** + * [EC #47] Mix the clip into the published microphone for its duration. For + * every room, the track currently going out on the mic's RTCRtpSender (the + * denoised one, when noise suppression is on) and the clip are mixed in a + * MediaStreamAudioDestinationNode, and the sender switches to the mix with + * `replaceTrack`. The server still sees one ordinary microphone track, which + * rooms with a call policy allow. When the clip ends the original track goes + * back. Muting mid-clip stops the clip (#13: muted means silent), and if + * LiveKit swapped its own track in meanwhile (device change, denoise toggle) + * that one is left alone. + */ +function playIntoMicrophone( + ctx: AudioContext, + source: AudioBufferSourceNode, + gain: GainNode, + buffer: AudioBuffer, + rooms: LivekitRoom[], + activeClips: Set<() => void>, + placeholder: () => void, + isAborted: () => boolean, +): void { + type Mix = { + sender: RTCRtpSender; + original: MediaStreamTrack; + mixed: MediaStreamTrack; + mic: MediaStreamAudioSourceNode; + dest: MediaStreamAudioDestinationNode; + track: LocalTrack; + }; + const mixes: Mix[] = []; + let cleanedUp = false; + const cleanup = (): void => { + if (cleanedUp) return; + cleanedUp = true; + activeClips.delete(cleanup); + activeClips.delete(placeholder); + try { + source.stop(); + } catch { + /* not started or already stopped */ + } + source.disconnect(); + gain.disconnect(); + for (const m of mixes) { + m.track.off(TrackEvent.Muted, cleanup); + m.mic.disconnect(); + // Only undo our own swap: LiveKit may have replaced the track since. + if (m.sender.track === m.mixed) + void m.sender.replaceTrack(m.original).catch(() => undefined); + m.mixed.stop(); + } + }; + + for (const room of rooms) { + const track = room.localParticipant.getTrackPublication( + Track.Source.Microphone, + )?.track; + const sender = track?.sender; + const original = sender?.track; + if (!track || !sender || !original) continue; + const dest = ctx.createMediaStreamDestination(); + const mic = ctx.createMediaStreamSource(new MediaStream([original])); + mic.connect(dest); + gain.connect(dest); + const mixed = dest.stream.getAudioTracks()[0]; + if (!mixed) { + mic.disconnect(); + continue; + } + mixes.push({ sender, original, mixed, mic, dest, track }); + track.on(TrackEvent.Muted, cleanup); + } + + activeClips.delete(placeholder); + activeClips.add(cleanup); + if (mixes.length === 0) { + logger.warn("[lotus] inject_audio: no published microphone to mix into"); + cleanup(); + return; + } + if (isAborted()) { + cleanup(); + return; + } + + // Switch every sender to its mix, then start the clip once they're live so + // its start isn't clipped. + void Promise.all( + mixes.map(async (m) => + m.sender.replaceTrack(m.mixed).catch((e) => { + logger.warn("[lotus] inject_audio: replaceTrack failed", e); + }), + ), + ).then(() => { + if (cleanedUp || isAborted()) { + cleanup(); + return; + } + source.onended = cleanup; + const durationMs = Number.isFinite(buffer.duration) + ? buffer.duration * 1000 + 500 + : MAX_CLIP_MS; + const guard = setTimeout( + cleanup, + Math.min(MAX_CLIP_MS, Math.max(0, durationMs)), + ); + source.addEventListener("ended", () => clearTimeout(guard)); + source.start(); + }); +} From e12b3e5979d5407aa17712d35924b6f8de60cfff Mon Sep 17 00:00:00 2001 From: Lotus CI Date: Mon, 19 Oct 2026 23:30:56 -0400 Subject: [PATCH 2/2] style(lotus): format LOTUS.md table Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_017PCb59uY3UKqDqt7B6hzd5 --- LOTUS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LOTUS.md b/LOTUS.md index c7703705..130f43fc 100644 --- a/LOTUS.md +++ b/LOTUS.md @@ -37,7 +37,7 @@ checked against cinny's `lotus` branch. | ---------------------------------- | ----------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------- | | `io.lotus.call_state` | call → host | in-call per-participant speaking / mute state. NOTE: matrix-widget-api `transport.send` is request/response — the host MUST register a handler that replies/acks each one (cinny's `listenAction` does, replying `{}`). If the host has no handler, ClientWidgetApi immediately error-replies "unsupported from-widget action", so the data is silently dropped and each throttled send rejects (caught) — functional miss + log churn, not a hang. | yes | | `io.lotus.focus_participant` | host → call | pin/spotlight (or clear, with userId=null) a participant. | yes | -| `io.lotus.inject_audio` | host → call | play an audio clip into the call: as a separate published track, or mixed into the microphone where the token only allows listed sources (rooms with a call policy, #47). Replies `{ played: false, reason: "muted" \| "not_permitted" }` when it can't. | yes | +| `io.lotus.inject_audio` | host → call | play an audio clip into the call: as a separate published track, or mixed into the microphone where the token only allows listed sources (rooms with a call policy, #47). Replies `{ played: false, reason: "muted" \| "not_permitted" }` when it can't. | yes | | `io.lotus.set_quality` | host → call | set audio/screenshare encoding quality (bitrate/framerate). | yes | | `io.lotus.decorations` | host → call | per-user avatar-decoration image URLs for in-call tiles. | yes | | `io.lotus.set_deafen` | host → call | deafen remote audio (and optionally mute screenshare audio). | yes |