Compare commits
@@ -0,0 +1,126 @@
|
||||
name: CI
|
||||
|
||||
# Build + publish the Lotus fork of Element Call's embedded web bundle.
|
||||
# Models LotusGuild/cinny's .gitea/workflows/ci.yml. Web bundle only, so
|
||||
# Linux-only (the Windows worker is for cinny-desktop / Tauri, not needed here).
|
||||
#
|
||||
# - PRs and pushes to `lotus` -> build + smoke-check the embedded dist
|
||||
# - Pushes of a tag `v*` -> build + publish @lotusguild/element-call-embedded
|
||||
# to the Gitea npm registry
|
||||
#
|
||||
# Versioning: the checked-in embedded/web/package.json tracks the intended next
|
||||
# release (e.g. 0.20.1-lotus.2) for local/manual publishes, but the AUTHORITATIVE
|
||||
# published version is still derived from the git tag at publish time (the
|
||||
# `npm version "$TAG"` step below overwrites it, so a tag always wins).
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [lotus]
|
||||
tags: ['v*']
|
||||
pull_request:
|
||||
branches: [lotus]
|
||||
|
||||
env:
|
||||
# element-call's build:full sets 16384 already; keep parity for safety.
|
||||
NODE_OPTIONS: '--max-old-space-size=16384'
|
||||
# Stamp the build so analytics/rageshakes aren't labelled "dev".
|
||||
VITE_APP_VERSION: ${{ github.ref_name }}
|
||||
|
||||
jobs:
|
||||
build:
|
||||
name: Build embedded bundle
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version-file: '.node-version' # Node 24
|
||||
# registry-url wires up the Gitea-scoped registry + auth for publish
|
||||
registry-url: 'https://code.lotusguild.org/api/packages/LotusGuild/npm/'
|
||||
scope: '@lotusguild'
|
||||
|
||||
- name: Enable corepack (pnpm from packageManager field)
|
||||
run: corepack enable
|
||||
|
||||
- name: Install dependencies
|
||||
run: pnpm install --frozen-lockfile
|
||||
|
||||
- name: Build embedded
|
||||
# build:embedded writes to the repo-root dist/ (vite default outDir).
|
||||
# Stage it into embedded/web/dist — the publish template's "files" entry
|
||||
# — so the smoke-check and npm publish both see the bundle.
|
||||
# (embedded/web/dist is gitignored, hence the explicit copy.)
|
||||
run: |
|
||||
pnpm run build:embedded
|
||||
rm -rf embedded/web/dist
|
||||
cp -r dist embedded/web/dist
|
||||
|
||||
- name: Smoke-check output
|
||||
run: |
|
||||
test -f embedded/web/dist/index.html
|
||||
test -d embedded/web/dist/assets
|
||||
echo "### Embedded bundle" >> "$GITHUB_STEP_SUMMARY"
|
||||
echo "" >> "$GITHUB_STEP_SUMMARY"
|
||||
echo '```' >> "$GITHUB_STEP_SUMMARY"
|
||||
du -sh embedded/web/dist >> "$GITHUB_STEP_SUMMARY"
|
||||
ls embedded/web/dist >> "$GITHUB_STEP_SUMMARY"
|
||||
echo '```' >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
# NOTE: no actions/upload-artifact step — v4 requires GitHub's artifact
|
||||
# backend, which the Gitea act_runner does not implement (it fails with
|
||||
# exit 1). The publish job rebuilds from source anyway, so the artifact
|
||||
# was only a convenience; re-add a Gitea-compatible action here if per-run
|
||||
# dist downloads are ever needed.
|
||||
|
||||
publish:
|
||||
name: Publish to Gitea npm registry
|
||||
needs: build
|
||||
# Only on a version tag push, never on PRs.
|
||||
if: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') }}
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version-file: '.node-version'
|
||||
registry-url: 'https://code.lotusguild.org/api/packages/LotusGuild/npm/'
|
||||
scope: '@lotusguild'
|
||||
|
||||
- name: Enable corepack
|
||||
run: corepack enable
|
||||
|
||||
- name: Install dependencies
|
||||
run: pnpm install --frozen-lockfile
|
||||
|
||||
- name: Build embedded
|
||||
# build:embedded writes to the repo-root dist/ (vite default outDir).
|
||||
# Stage it into embedded/web/dist — the publish template's "files" entry
|
||||
# — so the smoke-check and npm publish both see the bundle.
|
||||
# (embedded/web/dist is gitignored, hence the explicit copy.)
|
||||
run: |
|
||||
pnpm run build:embedded
|
||||
rm -rf embedded/web/dist
|
||||
cp -r dist embedded/web/dist
|
||||
|
||||
- name: Set published version from tag
|
||||
working-directory: embedded/web
|
||||
# Versioning scheme: reserve bare vX.Y.Z for upstream-parity points only
|
||||
# (e.g. v0.20.1 == upstream 0.20.1). For Lotus-only iterations on top of
|
||||
# an upstream base, tag a semver prerelease — v0.20.1-lotus.1, -lotus.2,
|
||||
# … — so we never collide with an already-published upstream-parity
|
||||
# version on the registry.
|
||||
run: |
|
||||
TAG="${GITHUB_REF_NAME#v}" # v0.20.1-lotus.1 -> 0.20.1-lotus.1
|
||||
npm version "$TAG" --no-git-tag-version --allow-same-version
|
||||
|
||||
- name: Publish
|
||||
working-directory: embedded/web
|
||||
env:
|
||||
NODE_AUTH_TOKEN: ${{ secrets.GITEA_NPM_TOKEN }}
|
||||
run: npm publish --access public
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@element-hq/element-call-embedded",
|
||||
"version": "0.0.0",
|
||||
"name": "@lotusguild/element-call-embedded",
|
||||
"version": "0.20.1-lotus.2",
|
||||
"files": [
|
||||
"README.md",
|
||||
"LICENSE-AGPL-3.0",
|
||||
@@ -9,6 +9,6 @@
|
||||
],
|
||||
"repository": {
|
||||
"type": "git",
|
||||
"url": "git+https://github.com/element-hq/element-call.git"
|
||||
"url": "git+https://code.lotusguild.org/LotusGuild/element-call.git"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -391,6 +391,21 @@ describe("UrlParams", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("autoGainControl", () => {
|
||||
it("defaults to true", () => {
|
||||
expect(computeUrlParams().autoGainControl).toBe(true);
|
||||
});
|
||||
|
||||
it("is parsed", () => {
|
||||
expect(computeUrlParams("?autoGainControl=true").autoGainControl).toBe(
|
||||
true,
|
||||
);
|
||||
expect(computeUrlParams("?autoGainControl=false").autoGainControl).toBe(
|
||||
false,
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe("header", () => {
|
||||
it("uses header if provided", () => {
|
||||
expect(computeUrlParams("?header=app_bar&hideHeader=true").header).toBe(
|
||||
|
||||
@@ -239,6 +239,12 @@ export interface UrlConfiguration {
|
||||
* Defaults to true.
|
||||
*/
|
||||
noiseSuppression?: boolean;
|
||||
/**
|
||||
* Whether to enable auto gain control for audio capture.
|
||||
* Defaults to true. Lotus turns this OFF for the in-source ML denoise tier so
|
||||
* the browser's dynamic gain doesn't fight the ML model (pumping artifacts).
|
||||
*/
|
||||
autoGainControl?: boolean;
|
||||
|
||||
callIntent?: RTCCallIntent;
|
||||
}
|
||||
@@ -485,6 +491,7 @@ export const computeUrlParams = (search = "", hash = ""): UrlParams => {
|
||||
autoLeaveWhenOthersLeft: parser.getFlag("autoLeave"),
|
||||
noiseSuppression: parser.getFlagParam("noiseSuppression", true),
|
||||
echoCancellation: parser.getFlagParam("echoCancellation", true),
|
||||
autoGainControl: parser.getFlagParam("autoGainControl", true),
|
||||
};
|
||||
|
||||
// Log the final configuration for debugging purposes.
|
||||
|
||||
@@ -12,8 +12,10 @@
|
||||
|
||||
@media (max-width: 420px) {
|
||||
.reactionPopupMenu {
|
||||
--reaction-button-padding: 8px;
|
||||
--reaction-button-fontsize: 16px;
|
||||
/* 20px glyph + 2×12px padding = a 44px square touch target. Five buttons
|
||||
still fit a ~360px drawer row; previously these were ~32px. */
|
||||
--reaction-button-padding: 12px;
|
||||
--reaction-button-fontsize: 20px;
|
||||
--reaction-button-gap: 6px;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -102,6 +102,20 @@ Once we exceed 500 we hide everything except the buttons.
|
||||
@media (max-width: 500px) {
|
||||
.footer {
|
||||
grid-template-areas: "buttons buttons buttons";
|
||||
/* Reclaim horizontal budget so the control row fits on phones. */
|
||||
padding-left: calc(env(safe-area-inset-left) + var(--cpd-space-3x));
|
||||
padding-right: calc(env(safe-area-inset-right) + var(--cpd-space-3x));
|
||||
}
|
||||
|
||||
.buttons {
|
||||
/* Fill the row and wrap instead of overflowing: on the narrowest phones the
|
||||
control set (mic/video/loudspeaker/hangup) exceeds the viewport, and the
|
||||
parent's overflow-x: hidden would otherwise clip the outer buttons —
|
||||
including the destructive hangup. Wrapping keeps every control reachable. */
|
||||
justify-self: stretch;
|
||||
flex-wrap: wrap;
|
||||
justify-content: center;
|
||||
gap: var(--cpd-space-2x);
|
||||
}
|
||||
|
||||
.settingsOnlyShowNarrow {
|
||||
|
||||
@@ -16,7 +16,14 @@ Please see LICENSE in the repository root for full details.
|
||||
|
||||
.pip {
|
||||
position: absolute;
|
||||
inset: var(--cpd-space-4x);
|
||||
/* Keep the self-view within the safe area (home indicator) and the inline
|
||||
content insets, and clear of the floating footer — matching the
|
||||
reference-correct SpotlightExpandedLayout. A bare 16px inset let the
|
||||
bottom-right self-tile sit under the controls / device chrome on phones. */
|
||||
inset: calc(env(safe-area-inset-top) + var(--cpd-space-4x))
|
||||
var(--content-inset-right)
|
||||
calc(env(safe-area-inset-bottom) + var(--cpd-space-4x))
|
||||
var(--content-inset-left);
|
||||
}
|
||||
|
||||
.pip[data-size="sm"] {
|
||||
|
||||
@@ -43,3 +43,17 @@ unconditionally select the container so we can use cq units */
|
||||
inline-size: 180px;
|
||||
block-size: 135px;
|
||||
}
|
||||
|
||||
/* On a landscape phone the fixed 180px filmstrip rail squeezes the spotlight to
|
||||
a sliver; shrink the rail (keeping the 4:3 tile ratio) to give the spotlight
|
||||
room. Desktop landscape (height > 400px) is unaffected. */
|
||||
@media (max-height: 400px) {
|
||||
.layer {
|
||||
--grid-slot-width: 132px;
|
||||
}
|
||||
|
||||
.grid > .slot {
|
||||
inline-size: 132px;
|
||||
block-size: 99px;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -85,6 +85,25 @@ body {
|
||||
-webkit-tap-highlight-color: transparent;
|
||||
}
|
||||
|
||||
/* [lotus] When embedded in the Lotus host with lotusTransparent=1, make the
|
||||
app background transparent so the host's wallpaper shows through natively —
|
||||
replacing the host's injected `html, body { background: none !important }`
|
||||
override. */
|
||||
body.lotus-transparent,
|
||||
body.lotus-transparent #root {
|
||||
background: transparent !important;
|
||||
}
|
||||
|
||||
/* [lotus] Native Lotus/TDS theme, applied when lotusTheme=1, instead of the
|
||||
host injecting CSS into the iframe after load. Overrides Compound design tokens
|
||||
with Lotus values at the source so theming is complete and flash-free. Extend
|
||||
this block with the full Lotus token map from the design system; the canvas
|
||||
override below is a safe starting point that matches the Lotus dark surface. */
|
||||
body.lotus-theme {
|
||||
--cpd-color-bg-canvas-default: #0c0d10;
|
||||
--video-tile-background: var(--cpd-color-bg-subtle-secondary);
|
||||
}
|
||||
|
||||
/* This prohibits the view to scroll for pages smaller than 122px in width
|
||||
we use this for mobile pip webviews */
|
||||
.no-scroll-body {
|
||||
|
||||
@@ -0,0 +1,43 @@
|
||||
/*
|
||||
Copyright 2026 Lotus Guild
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
/**
|
||||
* Custom widget actions exchanged between the Lotus host (cinny) and this fork.
|
||||
* Kept dependency-free so both `widget.ts` and `lotus/*` can import it without
|
||||
* a circular dependency.
|
||||
*/
|
||||
export enum LotusWidgetActions {
|
||||
/**
|
||||
* fromWidget: in-call per-participant speaking / mute state.
|
||||
* NOTE: matrix-widget-api `transport.send` is request/response — the host
|
||||
* MUST register a handler that replies/acks each one (cinny's `listenAction`
|
||||
* does, replying `{}`). If the host has no handler, ClientWidgetApi
|
||||
* immediately error-replies "unsupported from-widget action", so the data is
|
||||
* silently dropped and each throttled send rejects (caught) — functional miss
|
||||
* + log churn, not a hang.
|
||||
*/
|
||||
CallState = "io.lotus.call_state",
|
||||
/** toWidget: pin/spotlight (or clear, with userId=null) a participant. */
|
||||
FocusParticipant = "io.lotus.focus_participant",
|
||||
/** toWidget: play an audio clip into the call as a separate published track. */
|
||||
InjectAudio = "io.lotus.inject_audio",
|
||||
/** toWidget: set audio/screenshare encoding quality (bitrate/framerate). */
|
||||
SetQuality = "io.lotus.set_quality",
|
||||
/** toWidget: per-user avatar-decoration image URLs for in-call tiles. */
|
||||
Decorations = "io.lotus.decorations",
|
||||
/** toWidget: deafen remote audio (and optionally mute screenshare audio). */
|
||||
SetDeafen = "io.lotus.set_deafen",
|
||||
}
|
||||
|
||||
/** toWidget Lotus actions that must be allow-listed in `initializeWidget`. */
|
||||
export const LOTUS_TO_WIDGET_ACTIONS: LotusWidgetActions[] = [
|
||||
LotusWidgetActions.FocusParticipant,
|
||||
LotusWidgetActions.InjectAudio,
|
||||
LotusWidgetActions.SetQuality,
|
||||
LotusWidgetActions.Decorations,
|
||||
LotusWidgetActions.SetDeafen,
|
||||
];
|
||||
@@ -0,0 +1,247 @@
|
||||
/*
|
||||
Copyright 2026 Lotus Guild
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { type Room as LivekitRoom, Track } from "livekit-client";
|
||||
import { logger } from "matrix-js-sdk/lib/logger";
|
||||
import { type IWidgetApiRequest } from "matrix-widget-api";
|
||||
|
||||
import { type CallViewModel } from "../state/CallViewModel/CallViewModel";
|
||||
import { widget } from "../widget";
|
||||
import { LotusWidgetActions } from "./lotusActions";
|
||||
import { lotusFlag } from "./lotusWidget";
|
||||
|
||||
/** Hard cap so a malformed/huge clip can't hold a published track open forever. */
|
||||
const MAX_CLIP_MS = 30_000;
|
||||
|
||||
/**
|
||||
* Handle the host's `io.lotus.inject_audio` toWidget action (#3): mix a
|
||||
* soundboard clip into the call so other participants hear it.
|
||||
*
|
||||
* Rather than splice into the local mic track (which would fight the denoise
|
||||
* pipeline), we publish the clip as a separate `Unknown`-source LiveKit audio
|
||||
* track — which `MatrixAudioRenderer` already renders for valid call members —
|
||||
* and unpublish it when the clip ends. This is the real call-audio injection
|
||||
* that was impossible against the prebuilt EC bundle (LiveKit's
|
||||
* LocalParticipant lived in EC's module scope).
|
||||
*
|
||||
* Action data: `{ url: string, volume?: number }`. `url` must be an https/blob
|
||||
* URL (the host resolves mxc → media URL).
|
||||
*
|
||||
* No effect unless the host sends the action. Returns a teardown function that
|
||||
* also aborts any clip still playing.
|
||||
*/
|
||||
export function startLotusAudioInject(vm: CallViewModel): () => void {
|
||||
const w = widget;
|
||||
if (!w) return () => undefined;
|
||||
|
||||
// Track the set of connected LiveKit rooms to publish into. Drive off the
|
||||
// LOCAL participant's connection(s), not `livekitRoomItems$` — that stream
|
||||
// omits rooms with no remote members, so inject would no-op while you're
|
||||
// alone. Map the connections to their livekit rooms like `lotusDenoise.ts`.
|
||||
let rooms: LivekitRoom[] = [];
|
||||
const sub = vm.allConnections$.subscribe((data) => {
|
||||
rooms = data.getConnections().map((c) => c.livekitRoom);
|
||||
});
|
||||
|
||||
// In-flight clips, so we can abort them on teardown (unmount / vm change /
|
||||
// call leave) instead of leaving audio blasting to peers.
|
||||
const activeClips = new Set<() => void>();
|
||||
|
||||
const handler = (ev: CustomEvent<IWidgetApiRequest>): void => {
|
||||
// Always ack so the transport doesn't hang, but only act when the host has
|
||||
// explicitly opted in: audio-inject publishes under the local user's
|
||||
// identity, so it must not be silently armed for every call.
|
||||
void w.api.transport.reply(ev.detail, {});
|
||||
if (!lotusFlag("lotusAudioInject")) return;
|
||||
const data = ev.detail.data as
|
||||
| { url?: unknown; volume?: unknown }
|
||||
| undefined;
|
||||
const url = typeof data?.url === "string" ? safeMediaUrl(data.url) : null;
|
||||
if (!url) {
|
||||
logger.warn("[lotus] inject_audio: missing/invalid url");
|
||||
return;
|
||||
}
|
||||
const volume =
|
||||
typeof data?.volume === "number" && data.volume >= 0 && data.volume <= 1
|
||||
? data.volume
|
||||
: 1;
|
||||
void playInjectedClip(url, volume, rooms, activeClips).catch((e) =>
|
||||
logger.warn("[lotus] inject_audio failed", e),
|
||||
);
|
||||
};
|
||||
|
||||
w.lazyActions.on(LotusWidgetActions.InjectAudio, handler);
|
||||
return () => {
|
||||
sub.unsubscribe();
|
||||
w.lazyActions.off(LotusWidgetActions.InjectAudio, handler);
|
||||
// Abort anything still playing.
|
||||
for (const abort of [...activeClips]) abort();
|
||||
};
|
||||
}
|
||||
|
||||
/** Only allow fetchable media URLs; never same-origin credentialed GETs etc. */
|
||||
function safeMediaUrl(raw: string): string | null {
|
||||
try {
|
||||
const u = new URL(raw, window.location.href);
|
||||
return u.protocol === "https:" || u.protocol === "blob:" ? u.href : null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async function playInjectedClip(
|
||||
url: string,
|
||||
volume: number,
|
||||
rooms: LivekitRoom[],
|
||||
activeClips: Set<() => void>,
|
||||
): Promise<void> {
|
||||
if (rooms.length === 0) {
|
||||
logger.warn("[lotus] inject_audio: no connected rooms");
|
||||
return;
|
||||
}
|
||||
|
||||
// Max ONE clip at a time (replace mode): stop any in-flight or playing clip
|
||||
// before starting a new one, so clips can't overlap or be spammed.
|
||||
for (const abort of [...activeClips]) abort();
|
||||
|
||||
// A second inject action can arrive while THIS one is still awaiting its
|
||||
// fetch/decode/publish — before its real cleanup() exists. Register a
|
||||
// synchronous placeholder abort NOW, BEFORE the first await, so the
|
||||
// replace-mode loop above (run by that later action) cancels this one;
|
||||
// otherwise both clips would sail past their awaits and DOUBLE-PUBLISH. The
|
||||
// placeholder aborts the in-flight fetch and flips `aborted`, which we check
|
||||
// after every await; the real cleanup() replaces it once the track is live.
|
||||
let aborted = false;
|
||||
const controller = new AbortController();
|
||||
const placeholder = (): void => {
|
||||
aborted = true;
|
||||
controller.abort();
|
||||
activeClips.delete(placeholder);
|
||||
};
|
||||
activeClips.add(placeholder);
|
||||
|
||||
let resp: Response;
|
||||
try {
|
||||
resp = await fetch(url, {
|
||||
credentials: "omit",
|
||||
mode: "cors",
|
||||
signal: controller.signal,
|
||||
});
|
||||
} catch (e) {
|
||||
// Superseded by a newer clip mid-fetch — expected, not a failure.
|
||||
if (aborted) return;
|
||||
throw e;
|
||||
}
|
||||
if (aborted) return;
|
||||
if (!resp.ok) throw new Error(`fetch ${url} -> ${resp.status}`);
|
||||
const arrayBuffer = await resp.arrayBuffer();
|
||||
if (aborted) return;
|
||||
|
||||
const ctx = new AudioContext();
|
||||
// The action arrives via host postMessage, not a gesture in this iframe, so
|
||||
// the context may start suspended — resume it or the clip is silent and
|
||||
// `onended` never fires.
|
||||
try {
|
||||
await ctx.resume();
|
||||
} catch {
|
||||
/* best effort */
|
||||
}
|
||||
if (aborted) {
|
||||
void ctx.close();
|
||||
return;
|
||||
}
|
||||
if (ctx.state !== "running")
|
||||
logger.warn(`[lotus] inject_audio: AudioContext is ${ctx.state}`);
|
||||
|
||||
let buffer: AudioBuffer;
|
||||
try {
|
||||
buffer = await ctx.decodeAudioData(arrayBuffer);
|
||||
} catch (e) {
|
||||
void ctx.close();
|
||||
throw e;
|
||||
}
|
||||
if (aborted) {
|
||||
void ctx.close();
|
||||
return;
|
||||
}
|
||||
|
||||
const dest = ctx.createMediaStreamDestination();
|
||||
const gain = ctx.createGain();
|
||||
gain.gain.value = volume;
|
||||
const source = ctx.createBufferSource();
|
||||
source.buffer = buffer;
|
||||
source.connect(gain).connect(dest);
|
||||
|
||||
const mst = dest.stream.getAudioTracks()[0];
|
||||
if (!mst) {
|
||||
void ctx.close();
|
||||
throw new Error("no audio track from destination");
|
||||
}
|
||||
|
||||
// Publish (a clone of) the clip track to every connected room.
|
||||
const publications = await Promise.all(
|
||||
rooms.map(async (room) => {
|
||||
const clone = mst.clone();
|
||||
try {
|
||||
const pub = await room.localParticipant.publishTrack(clone, {
|
||||
source: Track.Source.Unknown,
|
||||
name: "lotus-soundboard",
|
||||
dtx: false,
|
||||
red: false,
|
||||
});
|
||||
return { room, pub };
|
||||
} catch (e) {
|
||||
clone.stop(); // don't leak the clone if publish failed
|
||||
logger.warn("[lotus] inject_audio: publish failed", e);
|
||||
return null;
|
||||
}
|
||||
}),
|
||||
);
|
||||
|
||||
let cleanedUp = false;
|
||||
const cleanup = (): void => {
|
||||
if (cleanedUp) return;
|
||||
cleanedUp = true;
|
||||
activeClips.delete(cleanup);
|
||||
try {
|
||||
source.stop();
|
||||
} catch {
|
||||
/* already stopped */
|
||||
}
|
||||
for (const entry of publications) {
|
||||
if (entry?.pub.track)
|
||||
void entry.room.localParticipant
|
||||
.unpublishTrack(entry.pub.track, true)
|
||||
.catch(() => undefined);
|
||||
}
|
||||
void ctx.close().catch(() => undefined);
|
||||
};
|
||||
// Swap the synchronous placeholder for the real cleanup: from here an abort
|
||||
// (teardown or a newer clip) must unpublish the LIVE track, not just cancel a
|
||||
// fetch. This delete+add is synchronous (no await), so a newer clip's
|
||||
// replace-mode loop always sees exactly one of {placeholder, cleanup}.
|
||||
activeClips.delete(placeholder);
|
||||
activeClips.add(cleanup);
|
||||
|
||||
// If a newer clip aborted us WHILE we were publishing, tear down now so we
|
||||
// don't leave an orphan track published after it ran its replace-mode loop.
|
||||
if (aborted) {
|
||||
cleanup();
|
||||
return;
|
||||
}
|
||||
|
||||
source.onended = cleanup;
|
||||
// Safety net: clip metadata can lie (NaN/huge duration), so force teardown
|
||||
// after a sane, capped delay.
|
||||
const durationMs = Number.isFinite(buffer.duration)
|
||||
? buffer.duration * 1000 + 500
|
||||
: MAX_CLIP_MS;
|
||||
const guard = setTimeout(cleanup, Math.min(MAX_CLIP_MS, Math.max(0, durationMs)));
|
||||
source.addEventListener("ended", () => clearTimeout(guard));
|
||||
|
||||
source.start();
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
/*
|
||||
Copyright 2026 Lotus Guild
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { combineLatest, of, type Subscription } from "rxjs";
|
||||
import { distinctUntilChanged, map, switchMap, throttleTime } from "rxjs/operators";
|
||||
|
||||
import { type CallViewModel } from "../state/CallViewModel/CallViewModel";
|
||||
import { LotusWidgetActions, lotusFlag, lotusSendToHost } from "./lotusWidget";
|
||||
|
||||
interface ParticipantState {
|
||||
/** EC media id (`${userId}:${deviceId}`), stable per participant device. */
|
||||
id: string;
|
||||
/** Matrix user id this media belongs to. */
|
||||
userId: string;
|
||||
speaking: boolean;
|
||||
audioEnabled: boolean;
|
||||
videoEnabled: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Stream per-participant speaking / mute / camera state to the Lotus host
|
||||
* (cinny) over the widget API, so the host can drive speaking rings, mute
|
||||
* badges and PiP from real events instead of scraping Element Call's rendered
|
||||
* DOM (`useCallSpeakers.ts`).
|
||||
*
|
||||
* Opt-in: does nothing unless the host set `lotusCallState=1` on the widget
|
||||
* URL. Returns a teardown function.
|
||||
*/
|
||||
export function startLotusCallState(vm: CallViewModel): () => void {
|
||||
if (!lotusFlag("lotusCallState")) return () => undefined;
|
||||
|
||||
const sub: Subscription = vm.userMedia$
|
||||
.pipe(
|
||||
switchMap((members) =>
|
||||
members.length === 0
|
||||
? of([] as ParticipantState[])
|
||||
: combineLatest(
|
||||
members.map((m) =>
|
||||
combineLatest([
|
||||
m.speaking$,
|
||||
m.audioEnabled$,
|
||||
m.videoEnabled$,
|
||||
]).pipe(
|
||||
map(
|
||||
([speaking, audioEnabled, videoEnabled]): ParticipantState => ({
|
||||
id: m.id,
|
||||
userId: m.userId,
|
||||
speaking,
|
||||
audioEnabled,
|
||||
videoEnabled,
|
||||
}),
|
||||
),
|
||||
),
|
||||
),
|
||||
),
|
||||
),
|
||||
// `speaking` flips rapidly; cap the send rate and drop no-op repeats.
|
||||
// 250ms is plenty for speaking rings / mute badges and keeps the
|
||||
// request/response widget traffic modest.
|
||||
throttleTime(250, undefined, { leading: true, trailing: true }),
|
||||
distinctUntilChanged((a, b) => JSON.stringify(a) === JSON.stringify(b)),
|
||||
)
|
||||
.subscribe((participants) => {
|
||||
lotusSendToHost(LotusWidgetActions.CallState, { participants });
|
||||
});
|
||||
|
||||
return () => sub.unsubscribe();
|
||||
}
|
||||
@@ -0,0 +1,120 @@
|
||||
/*
|
||||
Copyright 2026 Lotus Guild
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import {
|
||||
type RemoteParticipant,
|
||||
type Room as LivekitRoom,
|
||||
RoomEvent,
|
||||
Track,
|
||||
} from "livekit-client";
|
||||
import { logger } from "matrix-js-sdk/lib/logger";
|
||||
import { type IWidgetApiRequest } from "matrix-widget-api";
|
||||
|
||||
import { type CallViewModel } from "../state/CallViewModel/CallViewModel";
|
||||
import { widget } from "../widget";
|
||||
import { LotusWidgetActions } from "./lotusActions";
|
||||
|
||||
/**
|
||||
* Handle the host's `io.lotus.set_deafen` toWidget action: silence remote audio
|
||||
* at the LiveKit source, replacing cinny's brittle iframe-DOM `.muted` hack
|
||||
* (which fought MatrixAudioRenderer and broke on re-render / late tracks).
|
||||
*
|
||||
* `deafened` mutes every remote participant's microphone AND screenshare audio;
|
||||
* `screenshareAudioMuted` mutes only the screenshare audio (so the host can
|
||||
* drop shared-tab/game audio while still hearing voices). Volume is set PER
|
||||
* SOURCE via `RemoteParticipant.setVolume(volume, source)` — whose verified
|
||||
* signature in livekit-client ^2.18.1 is
|
||||
* `setVolume(volume, source?: Track.Source.Microphone | Track.Source.ScreenShareAudio)`
|
||||
* and DEFAULTS `source` to `Microphone` (NOT "all audio"), so each source must
|
||||
* be set explicitly. setVolume records the value in the participant's
|
||||
* `volumeMap`, so a track that (re)subscribes later re-applies it automatically.
|
||||
*
|
||||
* State is closure-scoped (per invocation, matching the sibling lotus modules)
|
||||
* and re-applied to every current room's participants on change, and to LATE
|
||||
* JOINERS via `RoomEvent.ParticipantConnected`. The cinny host re-sends the
|
||||
* current state on every call join (CallControl.forceState), so a fresh call
|
||||
* never inherits a previous call's deafen state.
|
||||
*
|
||||
* No effect unless the host sends the action. Returns a teardown function.
|
||||
*/
|
||||
export function startLotusDeafen(vm: CallViewModel): () => void {
|
||||
const w = widget;
|
||||
if (!w) return () => undefined;
|
||||
|
||||
let deafened = false;
|
||||
let screenshareAudioMuted = false;
|
||||
|
||||
const applyToParticipant = (p: RemoteParticipant): void => {
|
||||
p.setVolume(deafened ? 0 : 1, Track.Source.Microphone);
|
||||
p.setVolume(
|
||||
deafened || screenshareAudioMuted ? 0 : 1,
|
||||
Track.Source.ScreenShareAudio,
|
||||
);
|
||||
// NOTE: injected/soundboard audio (published as `Track.Source.Unknown`) is
|
||||
// deliberately NOT silenced here. The verified `setVolume` type signature
|
||||
// only accepts `Track.Source.Microphone | Track.Source.ScreenShareAudio`,
|
||||
// so passing `Unknown` would fail the fork's `tsc` gate and require an
|
||||
// unsafe cast. Soundboard clips are short, host-triggered content the host
|
||||
// already controls at the inject source, so leaving them audible is the
|
||||
// type-clean, safe choice (a full-parity "mute everything" is not needed
|
||||
// for the deafen semantics: don't-hear-other-people's-voices).
|
||||
};
|
||||
|
||||
const applyToRoom = (room: LivekitRoom): void =>
|
||||
room.remoteParticipants.forEach(applyToParticipant);
|
||||
|
||||
// Per-room ParticipantConnected listeners, so LATE JOINERS pick up the
|
||||
// current deafen state the moment they connect.
|
||||
const roomListeners = new Map<LivekitRoom, () => void>();
|
||||
let rooms: LivekitRoom[] = [];
|
||||
|
||||
const sub = vm.livekitRoomItems$.subscribe((items) => {
|
||||
const next = items.map((i) => i.livekitRoom);
|
||||
rooms = next;
|
||||
// Detach listeners for rooms that went away.
|
||||
for (const [room, off] of roomListeners) {
|
||||
if (!next.includes(room)) {
|
||||
off();
|
||||
roomListeners.delete(room);
|
||||
}
|
||||
}
|
||||
// Attach to new rooms + apply the current state to their participants.
|
||||
for (const room of next) {
|
||||
if (!roomListeners.has(room)) {
|
||||
room.on(RoomEvent.ParticipantConnected, applyToParticipant);
|
||||
roomListeners.set(room, () =>
|
||||
room.off(RoomEvent.ParticipantConnected, applyToParticipant),
|
||||
);
|
||||
applyToRoom(room);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
const handler = (ev: CustomEvent<IWidgetApiRequest>): void => {
|
||||
void w.api.transport.reply(ev.detail, {});
|
||||
const data = ev.detail.data as
|
||||
| { deafened?: boolean; screenshareAudioMuted?: boolean }
|
||||
| undefined;
|
||||
// Missing fields default to their CURRENT value, so a partial payload only
|
||||
// moves the flag it actually names.
|
||||
if (typeof data?.deafened === "boolean") deafened = data.deafened;
|
||||
if (typeof data?.screenshareAudioMuted === "boolean")
|
||||
screenshareAudioMuted = data.screenshareAudioMuted;
|
||||
logger.debug(
|
||||
`[lotus] set_deafen: deafened=${deafened} screenshareAudioMuted=${screenshareAudioMuted}`,
|
||||
);
|
||||
rooms.forEach(applyToRoom);
|
||||
};
|
||||
|
||||
w.lazyActions.on(LotusWidgetActions.SetDeafen, handler);
|
||||
return () => {
|
||||
sub.unsubscribe();
|
||||
for (const off of roomListeners.values()) off();
|
||||
roomListeners.clear();
|
||||
w.lazyActions.off(LotusWidgetActions.SetDeafen, handler);
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,106 @@
|
||||
/*
|
||||
Copyright 2026 Lotus Guild
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { useSyncExternalStore } from "react";
|
||||
import { type IWidgetApiRequest } from "matrix-widget-api";
|
||||
|
||||
import { widget } from "../widget";
|
||||
import { LotusWidgetActions } from "./lotusActions";
|
||||
|
||||
/**
|
||||
* Avatar decorations (#6 / A6). The Lotus host (cinny) owns the decoration
|
||||
* roster (MSC4133 profile decorations) and can't draw them on EC's in-call
|
||||
* video tiles from outside the iframe. So it pushes a `userId -> image URL`
|
||||
* map via the `io.lotus.decorations` widget action, and the tile avatar
|
||||
* component renders the overlay natively.
|
||||
*/
|
||||
|
||||
let decorations: Readonly<Record<string, string>> = {};
|
||||
const listeners = new Set<() => void>();
|
||||
|
||||
function emit(): void {
|
||||
for (const l of listeners) l();
|
||||
}
|
||||
|
||||
// Stable module-scope subscribe reference, so `useSyncExternalStore` doesn't
|
||||
// re-subscribe (add/remove the listener) on every render of a tile.
|
||||
function subscribe(cb: () => void): () => void {
|
||||
listeners.add(cb);
|
||||
return () => listeners.delete(cb);
|
||||
}
|
||||
|
||||
/** Subscribe a tile avatar to its participant's decoration URL (or undefined). */
|
||||
export function useLotusDecoration(userId: string): string | undefined {
|
||||
return useSyncExternalStore(subscribe, () => decorations[userId]);
|
||||
}
|
||||
|
||||
function safeImageUrl(raw: unknown): string | null {
|
||||
if (typeof raw !== "string") return null;
|
||||
try {
|
||||
const u = new URL(raw, window.location.href);
|
||||
return u.protocol === "https:" || u.protocol === "blob:" ? u.href : null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
// Ref-counted single registration: the decoration roster is app-wide (keyed by
|
||||
// userId), so multiple tile/InCallView mounts must share ONE handler — otherwise
|
||||
// each would reply to the same widget request (double-reply) and a transient
|
||||
// remount would tear it down.
|
||||
let registrations = 0;
|
||||
let unregister: (() => void) | null = null;
|
||||
|
||||
/**
|
||||
* Register the `io.lotus.decorations` handler (ref-counted). No effect unless
|
||||
* the host sends the action. Returns a teardown function.
|
||||
*/
|
||||
export function startLotusDecorations(): () => void {
|
||||
const w = widget;
|
||||
if (!w) return () => undefined;
|
||||
|
||||
if (registrations === 0) {
|
||||
const handler = (ev: CustomEvent<IWidgetApiRequest>): void => {
|
||||
void w.api.transport.reply(ev.detail, {});
|
||||
const data = ev.detail.data as
|
||||
| { decorations?: Record<string, unknown> }
|
||||
| undefined;
|
||||
const next: Record<string, string> = {};
|
||||
if (data?.decorations && typeof data.decorations === "object") {
|
||||
// Cap the roster so a pathological map can't spawn unbounded overlays.
|
||||
for (const [userId, url] of Object.entries(data.decorations).slice(
|
||||
0,
|
||||
512,
|
||||
)) {
|
||||
const safe = safeImageUrl(url);
|
||||
if (safe) next[userId] = safe;
|
||||
}
|
||||
}
|
||||
decorations = next;
|
||||
emit();
|
||||
};
|
||||
w.lazyActions.on(LotusWidgetActions.Decorations, handler);
|
||||
unregister = () =>
|
||||
w.lazyActions.off(LotusWidgetActions.Decorations, handler);
|
||||
}
|
||||
registrations += 1;
|
||||
|
||||
return () => {
|
||||
registrations -= 1;
|
||||
if (registrations <= 0) {
|
||||
registrations = 0;
|
||||
unregister?.();
|
||||
unregister = null;
|
||||
// Reset the roster once the last registration goes away, so a decoration
|
||||
// pushed in call A can't leak onto a shared user in call B before the
|
||||
// host re-pushes. Notify listeners so any still-mounted tile drops the
|
||||
// now-stale overlay via the render path.
|
||||
decorations = {};
|
||||
emit();
|
||||
}
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,154 @@
|
||||
/*
|
||||
Copyright 2026 Lotus Guild
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import {
|
||||
type LocalAudioTrack,
|
||||
ParticipantEvent,
|
||||
type Room as LivekitRoom,
|
||||
Track,
|
||||
} from "livekit-client";
|
||||
import { logger } from "matrix-js-sdk/lib/logger";
|
||||
|
||||
import { type CallViewModel } from "../state/CallViewModel/CallViewModel";
|
||||
import { lotusFlag, lotusParam } from "./lotusWidget";
|
||||
import {
|
||||
type LotusDenoiseConfig,
|
||||
LotusDenoiseProcessor,
|
||||
} from "./lotusDenoiseProcessor";
|
||||
|
||||
/**
|
||||
* Apply Lotus ML noise suppression to the local mic track as a first-class
|
||||
* Element Call audio TrackProcessor (#1), replacing the host's build-time
|
||||
* `getUserMedia` monkeypatch.
|
||||
*
|
||||
* Opt-in via `lotusDenoiseSource=1` (+ `lotusModel`, `lotusGate`,
|
||||
* `lotusGateThreshold`, `lotusDenoiseBase`). Because the processor is attached
|
||||
* to the mic publication and re-attached on every (re)publish, denoise
|
||||
* survives EC's mid-call reconnect — fixing the A7 "mic dead after reconnect"
|
||||
* bug. Additive: no-op without the flag.
|
||||
*/
|
||||
/**
|
||||
* Resolve the denoise asset base, forcing it to be SAME-ORIGIN. The base is fed
|
||||
* to `audioWorklet.addModule()`, which executes the target as code in the
|
||||
* worklet scope (processing the live mic) — so a cross-origin base from a
|
||||
* crafted call link would be arbitrary code execution. Any non-same-origin or
|
||||
* malformed value falls back to the bundled "./denoise/".
|
||||
*/
|
||||
function safeAssetBase(raw: string | null): string {
|
||||
// Resolve to an ABSOLUTE same-origin href against the document. Absolute is
|
||||
// required because native dynamic `import()` (DTLN/DeepFilterNet) resolves a
|
||||
// relative specifier against the JS chunk's URL, not the document — so a
|
||||
// relative "./denoise/" would 404. addModule()/fetch() work with absolute
|
||||
// too, so this keeps all three asset-load paths consistent.
|
||||
const fallback = new URL("./denoise/", window.location.href).href;
|
||||
if (!raw) return fallback;
|
||||
try {
|
||||
const u = new URL(raw, window.location.href);
|
||||
if (u.origin !== window.location.origin) return fallback;
|
||||
return u.href.endsWith("/") ? u.href : `${u.href}/`;
|
||||
} catch {
|
||||
return fallback;
|
||||
}
|
||||
}
|
||||
|
||||
export function startLotusDenoise(vm: CallViewModel): () => void {
|
||||
// Gate on a DEDICATED flag — NOT the existing `lotusDenoise=ml` that the
|
||||
// host's build-time getUserMedia shim already uses. Otherwise simply shipping
|
||||
// this fork (while the host still injects its shim and still sets
|
||||
// lotusDenoise=ml) would denoise twice. The host opts into the in-source
|
||||
// engine with `lotusDenoiseSource=1` AND stops injecting the shim at the same
|
||||
// time. Default off ⇒ the fork is inert and behaviour is unchanged.
|
||||
if (!lotusFlag("lotusDenoiseSource")) return () => undefined;
|
||||
|
||||
const requested = lotusParam("lotusModel");
|
||||
const model: LotusDenoiseConfig["model"] =
|
||||
requested === "speex" ||
|
||||
requested === "dtln" ||
|
||||
requested === "deepfilternet"
|
||||
? requested
|
||||
: "rnnoise";
|
||||
|
||||
const rawThreshold = lotusParam("lotusGateThreshold");
|
||||
const rawFloor = lotusParam("lotusDenoiseFloor");
|
||||
const config: LotusDenoiseConfig = {
|
||||
model,
|
||||
assetBase: safeAssetBase(lotusParam("lotusDenoiseBase")),
|
||||
gate: lotusFlag("lotusGate"),
|
||||
// Default -45 (matches the reference shim); accept an explicit 0 (don't
|
||||
// coerce it away via `|| default`).
|
||||
gateThreshold:
|
||||
rawThreshold !== null && Number.isFinite(Number(rawThreshold))
|
||||
? Number(rawThreshold)
|
||||
: -45,
|
||||
// Dry/wet attenuation floor. Default 0.15 (~-16 dB) tames the
|
||||
// over-suppression "underwater"/pumping artifact; host can tune via
|
||||
// `lotusDenoiseFloor` (0 = full suppression, no floor).
|
||||
floor:
|
||||
rawFloor !== null && Number.isFinite(Number(rawFloor))
|
||||
? Math.min(0.5, Math.max(0, Number(rawFloor)))
|
||||
: 0.15,
|
||||
};
|
||||
|
||||
const micOf = (room: LivekitRoom): LocalAudioTrack | undefined =>
|
||||
room.localParticipant.getTrackPublication(Track.Source.Microphone)
|
||||
?.track as LocalAudioTrack | undefined;
|
||||
|
||||
const apply = (room: LivekitRoom): void => {
|
||||
const mic = micOf(room);
|
||||
if (mic && !mic.getProcessor()) {
|
||||
void mic
|
||||
.setProcessor(new LotusDenoiseProcessor(config))
|
||||
.catch((e) => logger.warn("[lotus] denoise setProcessor failed", e));
|
||||
}
|
||||
};
|
||||
|
||||
const roomListeners = new Map<LivekitRoom, () => void>();
|
||||
let rooms: LivekitRoom[] = [];
|
||||
|
||||
// Drive activation off the LOCAL participant's connection(s), not
|
||||
// `livekitRoomItems$` — that stream excludes the local participant and only
|
||||
// surfaces rooms with ≥1 remote member, so it wouldn't denoise you while
|
||||
// you're alone and is a fragile coupling to a remote-render concern.
|
||||
const sub = vm.allConnections$.subscribe((data) => {
|
||||
const next = data.getConnections().map((c) => c.livekitRoom);
|
||||
rooms = next;
|
||||
for (const [room, off] of roomListeners) {
|
||||
if (!next.includes(room)) {
|
||||
off();
|
||||
roomListeners.delete(room);
|
||||
}
|
||||
}
|
||||
for (const room of next) {
|
||||
if (!roomListeners.has(room)) {
|
||||
// Re-attach on every (re)publish — this is what makes denoise survive
|
||||
// reconnects (A7), unlike the old getUserMedia patch.
|
||||
const onPublished = (): void => apply(room);
|
||||
room.localParticipant.on(
|
||||
ParticipantEvent.LocalTrackPublished,
|
||||
onPublished,
|
||||
);
|
||||
roomListeners.set(room, () =>
|
||||
room.localParticipant.off(
|
||||
ParticipantEvent.LocalTrackPublished,
|
||||
onPublished,
|
||||
),
|
||||
);
|
||||
apply(room);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
return () => {
|
||||
sub.unsubscribe();
|
||||
for (const off of roomListeners.values()) off();
|
||||
roomListeners.clear();
|
||||
for (const room of rooms) {
|
||||
const mic = micOf(room);
|
||||
if (mic?.getProcessor()) void mic.stopProcessor();
|
||||
}
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,400 @@
|
||||
/*
|
||||
Copyright 2026 Lotus Guild
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import {
|
||||
type AudioProcessorOptions,
|
||||
type Track,
|
||||
type TrackProcessor,
|
||||
} from "livekit-client";
|
||||
import { logger } from "matrix-js-sdk/lib/logger";
|
||||
|
||||
export type LotusDenoiseModel =
|
||||
| "rnnoise"
|
||||
| "speex"
|
||||
| "dtln"
|
||||
| "deepfilternet";
|
||||
|
||||
export interface LotusDenoiseConfig {
|
||||
model: LotusDenoiseModel;
|
||||
/** Base URL the worklet scripts/wasm/ESM are served from (e.g. "./denoise/"). */
|
||||
assetBase: string;
|
||||
gate: boolean;
|
||||
gateThreshold: number;
|
||||
/**
|
||||
* Attenuation floor as a dry/wet mix: the fraction (0..~0.3) of the ORIGINAL
|
||||
* mic blended back under the denoised signal so full suppression never fully
|
||||
* collapses the noise floor — this is what kills the "underwater"/pumping
|
||||
* artifact. 0.15 ≈ a -16 dB floor. 0 = full suppression (no floor).
|
||||
*/
|
||||
floor: number;
|
||||
}
|
||||
|
||||
// Flat sapphi worklets (RNNoise/Speex): each registers a processor under `name`
|
||||
// when its `script` module is added; we feed it the fetched wasm binary.
|
||||
// ⚠️ CONTRACT: these assets are NOT bundled by the fork build — cinny's
|
||||
// vite.config.js `lotusDenoise()` plugin copies them from
|
||||
// `@sapphi-red/web-noise-suppressor` / `@workadventure/noise-suppression` /
|
||||
// `deepfilternet3-noise-filter` into public/element-call/denoise/. The
|
||||
// worklet/wasm/ESM versions must match what this processor expects. An
|
||||
// integration smoke-check should assert GET .../denoise/rnnoise.wasm == 200.
|
||||
const FLAT: Record<
|
||||
"rnnoise" | "speex",
|
||||
{ name: string; script: string; wasm: string; simdWasm?: string }
|
||||
> = {
|
||||
rnnoise: {
|
||||
name: "@sapphi-red/web-noise-suppressor/rnnoise",
|
||||
script: "rnnoiseWorklet.js",
|
||||
wasm: "rnnoise.wasm",
|
||||
simdWasm: "rnnoise_simd.wasm",
|
||||
},
|
||||
speex: {
|
||||
name: "@sapphi-red/web-noise-suppressor/speex",
|
||||
script: "speexWorklet.js",
|
||||
wasm: "speex.wasm",
|
||||
},
|
||||
};
|
||||
// The sapphi gate worklet registers under "noise-gate" (hyphenated).
|
||||
const GATE = {
|
||||
name: "@sapphi-red/web-noise-suppressor/noise-gate",
|
||||
script: "noiseGateWorklet.js",
|
||||
};
|
||||
|
||||
// DTLN (@workadventure) targets 16kHz and doesn't resample; RNNoise/Speex and
|
||||
// DeepFilterNet are 48kHz fullband. The worklets don't resample, so the whole
|
||||
// graph must run at the model's native rate.
|
||||
const sampleRateFor = (model: LotusDenoiseModel): number =>
|
||||
model === "dtln" ? 16_000 : 48_000;
|
||||
|
||||
// Cache fetched wasm per URL so a reconnect/device-switch doesn't re-download.
|
||||
const wasmCache = new Map<string, Promise<ArrayBuffer>>();
|
||||
async function fetchWasmUncached(url: string): Promise<ArrayBuffer> {
|
||||
const r = await fetch(url);
|
||||
if (!r.ok) throw new Error(`denoise wasm ${url} -> ${r.status}`);
|
||||
return r.arrayBuffer();
|
||||
}
|
||||
async function fetchWasm(url: string): Promise<ArrayBuffer> {
|
||||
let p = wasmCache.get(url);
|
||||
if (!p) {
|
||||
p = fetchWasmUncached(url);
|
||||
// Never cache a REJECTED fetch: a transient failure (e.g. a blip during a
|
||||
// reconnect) must not permanently disable denoise for the whole session.
|
||||
// Evict on failure so the next restart/device-switch retries.
|
||||
void p.catch(() => wasmCache.delete(url));
|
||||
wasmCache.set(url, p);
|
||||
}
|
||||
return p;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resume an AudioContext, but never block indefinitely. A suspended context can
|
||||
* only resume after a user gesture; the denoise action can arrive via host
|
||||
* postMessage (no gesture), so `resume()` may stay pending forever. Since this
|
||||
* runs inside LiveKit's per-track change lock, a hung resume() would deadlock
|
||||
* every later mute/unmute/device-switch. Race it against a timeout and proceed
|
||||
* either way — the processor's `statechange` watcher resumes it once a gesture
|
||||
* lands, and a still-suspended context degrades to (temporary) silence that the
|
||||
* watcher heals, not a hang.
|
||||
*/
|
||||
async function resumeCtx(ctx: AudioContext, timeoutMs = 3_000): Promise<void> {
|
||||
await Promise.race([
|
||||
ctx.resume().catch(() => undefined),
|
||||
new Promise<void>((resolve) => setTimeout(resolve, timeoutMs)),
|
||||
]);
|
||||
}
|
||||
|
||||
function supportsSimd(): boolean {
|
||||
try {
|
||||
// Minimal SIMD module (v128) — validates only where SIMD is supported.
|
||||
return WebAssembly.validate(
|
||||
new Uint8Array([
|
||||
0, 97, 115, 109, 1, 0, 0, 0, 1, 5, 1, 96, 0, 1, 123, 3, 2, 1, 0, 10,
|
||||
10, 1, 8, 0, 65, 0, 253, 15, 253, 98, 11,
|
||||
]),
|
||||
);
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
interface MlNode {
|
||||
node: AudioNode;
|
||||
dispose?: () => void;
|
||||
}
|
||||
interface Graph {
|
||||
source: MediaStreamAudioSourceNode;
|
||||
nodes: AudioNode[];
|
||||
disposes: (() => void)[];
|
||||
track: MediaStreamTrack;
|
||||
}
|
||||
|
||||
/**
|
||||
* A LiveKit audio TrackProcessor that runs Lotus ML noise suppression
|
||||
* (RNNoise / Speex / DTLN / DeepFilterNet) on the local microphone track, as a
|
||||
* first-class stage in Element Call's publish pipeline — replacing the host's
|
||||
* `getUserMedia` monkeypatch.
|
||||
*
|
||||
* Because it's a real LiveKit processor, EC re-applies it on every
|
||||
* (re)publish/restart, so denoise survives EC's mid-call reconnect — the root
|
||||
* cause of A7. It owns a dedicated AudioContext at the model's required sample
|
||||
* rate (LiveKit does NOT pass an audioContext to restart()), reused across
|
||||
* restarts and closed on destroy. restart() never throws and never leaves a
|
||||
* stopped track on the sender: on failure it degrades to the RAW mic track
|
||||
* rather than silence.
|
||||
*/
|
||||
export class LotusDenoiseProcessor
|
||||
implements TrackProcessor<Track.Kind.Audio, AudioProcessorOptions>
|
||||
{
|
||||
public readonly name = "lotus-denoise";
|
||||
public processedTrack?: MediaStreamTrack;
|
||||
|
||||
private ctx?: AudioContext;
|
||||
private graph?: Graph;
|
||||
private ctxStateHandler?: () => void;
|
||||
|
||||
public constructor(private readonly config: LotusDenoiseConfig) {}
|
||||
|
||||
public async init(_opts: AudioProcessorOptions): Promise<void> {
|
||||
try {
|
||||
await this.ensureContext();
|
||||
this.graph = await this.buildGraph(_opts.track);
|
||||
this.processedTrack = this.graph.track;
|
||||
} catch (e) {
|
||||
// Don't orphan the owned context if graph construction fails (browsers
|
||||
// cap live AudioContexts, so repeated failed inits could exhaust them).
|
||||
// The caller degrades to the raw mic; we just release our resources.
|
||||
await this.closeContext();
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
public async restart(opts: AudioProcessorOptions): Promise<void> {
|
||||
try {
|
||||
await this.ensureContext();
|
||||
const next = await this.buildGraph(opts.track);
|
||||
this.disposeGraph(this.graph);
|
||||
this.graph = next;
|
||||
this.processedTrack = next.track;
|
||||
} catch (e) {
|
||||
// Never go silent on the A7/device-switch path: fall back to raw audio.
|
||||
logger.warn("[lotus] denoise restart failed; using raw mic", e);
|
||||
this.disposeGraph(this.graph);
|
||||
this.graph = undefined;
|
||||
this.processedTrack = opts.track;
|
||||
}
|
||||
}
|
||||
|
||||
public async destroy(): Promise<void> {
|
||||
this.disposeGraph(this.graph);
|
||||
this.graph = undefined;
|
||||
this.processedTrack = undefined;
|
||||
await this.closeContext();
|
||||
}
|
||||
|
||||
/** Remove the state watcher and close the owned context, if any. */
|
||||
private async closeContext(): Promise<void> {
|
||||
const ctx = this.ctx;
|
||||
if (!ctx) return;
|
||||
if (this.ctxStateHandler) {
|
||||
ctx.removeEventListener("statechange", this.ctxStateHandler);
|
||||
this.ctxStateHandler = undefined;
|
||||
}
|
||||
this.ctx = undefined;
|
||||
if (ctx.state !== "closed") await ctx.close().catch(() => undefined);
|
||||
}
|
||||
|
||||
/** Create (once) the model-rate context + register the flat worklet modules. */
|
||||
private async ensureContext(): Promise<void> {
|
||||
const rate = sampleRateFor(this.config.model);
|
||||
if (this.ctx && this.ctx.state !== "closed" && this.ctx.sampleRate === rate) {
|
||||
if (this.ctx.state === "suspended") await resumeCtx(this.ctx);
|
||||
return;
|
||||
}
|
||||
await this.closeContext();
|
||||
|
||||
const ctx = new AudioContext({ sampleRate: rate });
|
||||
try {
|
||||
if (ctx.sampleRate !== rate)
|
||||
throw new Error(`denoise: got ${ctx.sampleRate}Hz, need ${rate}Hz`);
|
||||
|
||||
// Auto-resume if the OS/browser suspends the context mid-call (mobile
|
||||
// backgrounding, audio interruption): the dest node otherwise emits
|
||||
// silence with no recovery. Only resume while a graph is live.
|
||||
const onStateChange = (): void => {
|
||||
if (ctx.state === "suspended" && this.graph)
|
||||
void ctx.resume().catch(() => undefined);
|
||||
};
|
||||
ctx.addEventListener("statechange", onStateChange);
|
||||
|
||||
// Flat models register via addModule here; DTLN/DeepFilterNet bring their
|
||||
// own processor via the dynamic-imported helper (see buildMlNode).
|
||||
if (this.config.model === "rnnoise" || this.config.model === "speex")
|
||||
await ctx.audioWorklet.addModule(
|
||||
this.config.assetBase + FLAT[this.config.model].script,
|
||||
);
|
||||
if (this.config.gate)
|
||||
await ctx.audioWorklet.addModule(this.config.assetBase + GATE.script);
|
||||
// The action can arrive via host postMessage, not a gesture in this
|
||||
// iframe, so the context can start suspended — resume without hanging.
|
||||
if (ctx.state === "suspended") await resumeCtx(ctx);
|
||||
|
||||
this.ctx = ctx;
|
||||
this.ctxStateHandler = onStateChange;
|
||||
} catch (e) {
|
||||
// Don't leak a half-initialised context on any failure path.
|
||||
await ctx.close().catch(() => undefined);
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
private async buildGraph(track: MediaStreamTrack): Promise<Graph> {
|
||||
const ctx = this.ctx!;
|
||||
const source = ctx.createMediaStreamSource(new MediaStream([track]));
|
||||
const dest = ctx.createMediaStreamDestination();
|
||||
const nodes: AudioNode[] = [];
|
||||
const disposes: (() => void)[] = [];
|
||||
|
||||
try {
|
||||
// Wet (denoised) path: source → ml → [gate] → wetGain.
|
||||
const ml = await this.buildMlNode(ctx);
|
||||
source.connect(ml.node);
|
||||
nodes.push(ml.node);
|
||||
if (ml.dispose) disposes.push(ml.dispose);
|
||||
let wetHead: AudioNode = ml.node;
|
||||
|
||||
// Gate AFTER the ML model, not before: gating the raw noisy signal fed
|
||||
// hard-zeroed frames into the model (discontinuities it must fight) and
|
||||
// made the threshold operate on pre-denoise levels. Gate the residual.
|
||||
if (this.config.gate) {
|
||||
const gate = new AudioWorkletNode(ctx, GATE.name, {
|
||||
processorOptions: {
|
||||
openThreshold: this.config.gateThreshold,
|
||||
closeThreshold: this.config.gateThreshold - 5,
|
||||
holdMs: 150,
|
||||
maxChannels: 1,
|
||||
},
|
||||
});
|
||||
wetHead.connect(gate);
|
||||
wetHead = gate;
|
||||
nodes.push(gate);
|
||||
}
|
||||
|
||||
// Only mix a dry floor for the LOW-LATENCY flat models (RNNoise/Speex).
|
||||
// DTLN/DeepFilterNet add tens of ms of algorithmic latency, so summing an
|
||||
// undelayed dry copy would comb-filter the voice — for those we rely on
|
||||
// the model's own level (e.g. DFN noiseReductionLevel) instead. RNNoise is
|
||||
// also where the "robotic/underwater" reports come from, so this targets it.
|
||||
const lowLatency =
|
||||
this.config.model === "rnnoise" || this.config.model === "speex";
|
||||
const floor = lowLatency
|
||||
? Math.min(0.5, Math.max(0, this.config.floor))
|
||||
: 0;
|
||||
if (floor > 0) {
|
||||
// Dry/wet mix: blend a small amount of the ORIGINAL mic under the
|
||||
// denoised signal so suppression can't fully collapse the noise floor
|
||||
// (kills the "underwater"/pumping artifact). During speech (denoised ≈
|
||||
// original) the two sum back to ~unity; in noise-only gaps the output
|
||||
// floors at `floor` × original instead of digital silence.
|
||||
const wetGain = ctx.createGain();
|
||||
wetGain.gain.value = 1 - floor;
|
||||
wetHead.connect(wetGain);
|
||||
wetGain.connect(dest);
|
||||
nodes.push(wetGain);
|
||||
|
||||
const dryGain = ctx.createGain();
|
||||
dryGain.gain.value = floor;
|
||||
source.connect(dryGain);
|
||||
dryGain.connect(dest);
|
||||
nodes.push(dryGain);
|
||||
} else {
|
||||
wetHead.connect(dest);
|
||||
}
|
||||
|
||||
logger.info(
|
||||
`[lotus] denoise processor active (${this.config.model}, floor=${floor})`,
|
||||
);
|
||||
return { source, nodes, disposes, track: dest.stream.getAudioTracks()[0] };
|
||||
} catch (e) {
|
||||
// A node constructor / model load can throw mid-build; clean up the
|
||||
// partially-built graph so it doesn't leak (init/restart still fall back
|
||||
// to the raw mic on the rejection).
|
||||
this.disposeGraph({
|
||||
source,
|
||||
nodes,
|
||||
disposes,
|
||||
track: dest.stream.getAudioTracks()[0],
|
||||
});
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
private async buildMlNode(ctx: AudioContext): Promise<MlNode> {
|
||||
const base = this.config.assetBase;
|
||||
const model = this.config.model;
|
||||
|
||||
if (model === "dtln") {
|
||||
// Self-contained ESM that resolves its own processor + LiteRT wasm +
|
||||
// TFLite models. bypassUntilReady passes raw audio until the model loads.
|
||||
const mod = await import(/* @vite-ignore */ `${base}workadventure/audio-worklet.js`);
|
||||
return (await mod.createNoiseSuppressionAudioWorklet(ctx, {
|
||||
bypassUntilReady: true,
|
||||
})) as MlNode;
|
||||
}
|
||||
|
||||
if (model === "deepfilternet") {
|
||||
const dfnBase = new URL(`${base}deepfilternet`, window.location.href).href;
|
||||
const mod = await import(/* @vite-ignore */ `${base}deepfilternet/index.esm.js`);
|
||||
const core = new mod.DeepFilterNet3Core({
|
||||
sampleRate: 48_000,
|
||||
// 60, not 80: full-strength suppression is the main source of the
|
||||
// "over-processed" character; a lower level keeps voice natural while
|
||||
// the dry/wet floor handles the noise tail.
|
||||
noiseReductionLevel: 60,
|
||||
assetConfig: { cdnUrl: dfnBase },
|
||||
});
|
||||
await core.initialize();
|
||||
const node = (await core.createAudioWorkletNode(ctx)) as AudioNode;
|
||||
return { node, dispose: () => void safeCall(() => core.destroy()) };
|
||||
}
|
||||
|
||||
// Flat sapphi worklet (rnnoise/speex).
|
||||
const flat = FLAT[model];
|
||||
const useSimd = model === "rnnoise" && !!flat.simdWasm && supportsSimd();
|
||||
const wasmFile = useSimd ? flat.simdWasm! : flat.wasm;
|
||||
let wasmBinary: ArrayBuffer;
|
||||
try {
|
||||
wasmBinary = await fetchWasm(base + wasmFile);
|
||||
} catch (e) {
|
||||
if (useSimd) {
|
||||
wasmCache.delete(base + wasmFile);
|
||||
wasmBinary = await fetchWasm(base + flat.wasm); // fall back to non-SIMD
|
||||
} else throw e;
|
||||
}
|
||||
const node = new AudioWorkletNode(ctx, flat.name, {
|
||||
channelCount: 1,
|
||||
numberOfInputs: 1,
|
||||
numberOfOutputs: 1,
|
||||
processorOptions: { maxChannels: 1, wasmBinary },
|
||||
});
|
||||
return { node, dispose: () => void safeCall(() => node.port.postMessage("destroy")) };
|
||||
}
|
||||
|
||||
private disposeGraph(graph: Graph | undefined): void {
|
||||
if (!graph) return;
|
||||
for (const dispose of graph.disposes) safeCall(dispose);
|
||||
for (const node of graph.nodes) safeCall(() => node.disconnect());
|
||||
safeCall(() => graph.source.disconnect());
|
||||
graph.track.stop();
|
||||
}
|
||||
}
|
||||
|
||||
function safeCall(fn: () => void): void {
|
||||
try {
|
||||
fn();
|
||||
} catch {
|
||||
/* ignore */
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
/*
|
||||
Copyright 2026 Lotus Guild
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { type IWidgetApiRequest } from "matrix-widget-api";
|
||||
|
||||
import { type CallViewModel } from "../state/CallViewModel/CallViewModel";
|
||||
import { widget } from "../widget";
|
||||
import { LotusWidgetActions } from "./lotusActions";
|
||||
|
||||
/**
|
||||
* Handle the host's `io.lotus.focus_participant` toWidget action (#4): pin a
|
||||
* participant to the spotlight by Matrix user id, or clear it with
|
||||
* `{ userId: null }`. This replaces cinny's old DOM `.click()` tile-selector
|
||||
* hack with a real, layout-aware spotlight override.
|
||||
*
|
||||
* No effect unless the host actually sends the action, so registering the
|
||||
* handler whenever we're a widget is safe. Returns a teardown function.
|
||||
*/
|
||||
export function startLotusFocus(vm: CallViewModel): () => void {
|
||||
const w = widget;
|
||||
if (!w) return () => undefined;
|
||||
|
||||
const handler = (ev: CustomEvent<IWidgetApiRequest>): void => {
|
||||
// Always reply so the host transport doesn't time out.
|
||||
void w.api.transport.reply(ev.detail, {});
|
||||
const data = ev.detail.data as { userId?: unknown } | undefined;
|
||||
// Mirror deafen's partial-payload semantics: a payload that OMITS `userId`
|
||||
// must keep the current spotlight, not clear it. Only act when the key is
|
||||
// actually present — an explicit `null` clears, a string pins that user.
|
||||
if (data && "userId" in data) {
|
||||
const userId = typeof data.userId === "string" ? data.userId : null;
|
||||
vm.setManualSpotlight(userId);
|
||||
}
|
||||
};
|
||||
|
||||
w.lazyActions.on(LotusWidgetActions.FocusParticipant, handler);
|
||||
return () =>
|
||||
w.lazyActions.off(LotusWidgetActions.FocusParticipant, handler);
|
||||
}
|
||||
@@ -0,0 +1,184 @@
|
||||
/*
|
||||
Copyright 2026 Lotus Guild
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import {
|
||||
type LocalTrack,
|
||||
ParticipantEvent,
|
||||
type Room as LivekitRoom,
|
||||
Track,
|
||||
} from "livekit-client";
|
||||
import { logger } from "matrix-js-sdk/lib/logger";
|
||||
import { type IWidgetApiRequest } from "matrix-widget-api";
|
||||
|
||||
import { type CallViewModel } from "../state/CallViewModel/CallViewModel";
|
||||
import { widget } from "../widget";
|
||||
import { LotusWidgetActions } from "./lotusActions";
|
||||
|
||||
interface QualitySettings {
|
||||
/** Max audio (mic) bitrate in bits/sec. */
|
||||
audioMaxBitrate?: number;
|
||||
/** Max screenshare video bitrate in bits/sec. */
|
||||
screenshareMaxBitrate?: number;
|
||||
/** Max screenshare framerate in fps. */
|
||||
screenshareMaxFramerate?: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle the host's `io.lotus.set_quality` toWidget action (#7): apply
|
||||
* audio/screenshare encoding limits (bitrate, framerate) to the local
|
||||
* published tracks via `RTCRtpSender.setParameters` — no republish needed.
|
||||
* These controls live in EC's module scope and were unreachable from the host
|
||||
* against the prebuilt bundle.
|
||||
*
|
||||
* Settings are sticky and re-applied whenever a matching local track is
|
||||
* (re)published, so they survive mute/unmute and reconnects. The server-side
|
||||
* voice-limit-guard remains the enforcement backstop.
|
||||
*
|
||||
* No effect unless the host sends the action. Returns a teardown function.
|
||||
*/
|
||||
export function startLotusQuality(vm: CallViewModel): () => void {
|
||||
const w = widget;
|
||||
if (!w) return () => undefined;
|
||||
|
||||
const settings: QualitySettings = {};
|
||||
// Per-room LocalTrackPublished listeners, so sticky settings re-apply on
|
||||
// every (re)publish.
|
||||
const roomListeners = new Map<LivekitRoom, () => void>();
|
||||
// Per-room settle re-apply timers, so we can cancel a pending 500ms re-apply
|
||||
// when a room is removed or on teardown — otherwise it would fire against a
|
||||
// torn-down room.
|
||||
const settleTimers = new Map<LivekitRoom, ReturnType<typeof setTimeout>>();
|
||||
let rooms: LivekitRoom[] = [];
|
||||
|
||||
const applyToRoom = (room: LivekitRoom): void => {
|
||||
const lp = room.localParticipant;
|
||||
|
||||
if (settings.audioMaxBitrate !== undefined) {
|
||||
const mic = lp.getTrackPublication(Track.Source.Microphone)?.track as
|
||||
| LocalTrack
|
||||
| undefined;
|
||||
void patchSender(mic?.sender, { maxBitrate: settings.audioMaxBitrate });
|
||||
}
|
||||
|
||||
const ssPatch: Partial<RTCRtpEncodingParameters> = {};
|
||||
if (settings.screenshareMaxBitrate !== undefined)
|
||||
ssPatch.maxBitrate = settings.screenshareMaxBitrate;
|
||||
if (settings.screenshareMaxFramerate !== undefined)
|
||||
ssPatch.maxFramerate = settings.screenshareMaxFramerate;
|
||||
if (Object.keys(ssPatch).length > 0) {
|
||||
const ss = lp.getTrackPublication(Track.Source.ScreenShare)?.track as
|
||||
| LocalTrack
|
||||
| undefined;
|
||||
void patchSender(ss?.sender, ssPatch);
|
||||
}
|
||||
};
|
||||
|
||||
const applyToAll = (): void => rooms.forEach(applyToRoom);
|
||||
|
||||
// Keep the LocalTrackPublished listeners in sync with the connected rooms.
|
||||
// Drive off the LOCAL participant's connection(s), not `livekitRoomItems$` —
|
||||
// that stream omits rooms with no remote members (returns null for isLocal),
|
||||
// so caps wouldn't apply to the local senders while you're alone. Map the
|
||||
// connections to their livekit rooms exactly like `lotusDenoise.ts` does.
|
||||
const sub = vm.allConnections$.subscribe((data) => {
|
||||
const next = data.getConnections().map((c) => c.livekitRoom);
|
||||
rooms = next;
|
||||
// Remove listeners for rooms that went away.
|
||||
for (const [room, off] of roomListeners) {
|
||||
if (!next.includes(room)) {
|
||||
off();
|
||||
roomListeners.delete(room);
|
||||
}
|
||||
}
|
||||
// Add listeners for new rooms + apply current settings to them.
|
||||
for (const room of next) {
|
||||
if (!roomListeners.has(room)) {
|
||||
// Re-apply on (re)publish AND unmute/track-restart: LiveKit's
|
||||
// refreshSenderEncodings() overwrites maxBitrate/maxFramerate from the
|
||||
// publish presets on replaceTrack (device/source switch, processor
|
||||
// toggle, restart-on-unmute), and those paths don't emit
|
||||
// LocalTrackPublished. The settle re-apply lands after LiveKit's async
|
||||
// recompute so our cap wins.
|
||||
const reapply = (): void => {
|
||||
applyToRoom(room);
|
||||
// Store the settle timer per room and cancel any pending one, so it
|
||||
// can be cleared on removal/teardown and never fires against a
|
||||
// torn-down room.
|
||||
const prev = settleTimers.get(room);
|
||||
if (prev !== undefined) clearTimeout(prev);
|
||||
settleTimers.set(
|
||||
room,
|
||||
setTimeout(() => {
|
||||
settleTimers.delete(room);
|
||||
applyToRoom(room);
|
||||
}, 500),
|
||||
);
|
||||
};
|
||||
const events = [
|
||||
ParticipantEvent.LocalTrackPublished,
|
||||
ParticipantEvent.TrackUnmuted,
|
||||
] as const;
|
||||
for (const e of events) room.localParticipant.on(e, reapply);
|
||||
roomListeners.set(room, () => {
|
||||
for (const e of events) room.localParticipant.off(e, reapply);
|
||||
const t = settleTimers.get(room);
|
||||
if (t !== undefined) clearTimeout(t);
|
||||
settleTimers.delete(room);
|
||||
});
|
||||
applyToRoom(room);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
const handler = (ev: CustomEvent<IWidgetApiRequest>): void => {
|
||||
void w.api.transport.reply(ev.detail, {});
|
||||
const data = ev.detail.data as Record<string, unknown> | undefined;
|
||||
if (!data) return;
|
||||
// Clamp to sane ranges so a typo can't brick the encoder (e.g. a 1 bps mic).
|
||||
const ranges: Record<keyof QualitySettings, [number, number]> = {
|
||||
audioMaxBitrate: [6_000, 510_000],
|
||||
screenshareMaxBitrate: [50_000, 20_000_000],
|
||||
screenshareMaxFramerate: [1, 60],
|
||||
};
|
||||
for (const key of Object.keys(ranges) as (keyof QualitySettings)[]) {
|
||||
const v = data[key];
|
||||
if (v === null) settings[key] = undefined;
|
||||
else if (typeof v === "number" && Number.isFinite(v)) {
|
||||
const [lo, hi] = ranges[key];
|
||||
settings[key] = Math.min(hi, Math.max(lo, v));
|
||||
}
|
||||
}
|
||||
applyToAll();
|
||||
};
|
||||
|
||||
w.lazyActions.on(LotusWidgetActions.SetQuality, handler);
|
||||
return () => {
|
||||
sub.unsubscribe();
|
||||
for (const off of roomListeners.values()) off();
|
||||
roomListeners.clear();
|
||||
w.lazyActions.off(LotusWidgetActions.SetQuality, handler);
|
||||
};
|
||||
}
|
||||
|
||||
async function patchSender(
|
||||
sender: RTCRtpSender | undefined,
|
||||
patch: Partial<RTCRtpEncodingParameters>,
|
||||
): Promise<void> {
|
||||
if (!sender) return;
|
||||
try {
|
||||
const params = sender.getParameters();
|
||||
if (!params.encodings || params.encodings.length === 0)
|
||||
params.encodings = [{}];
|
||||
// Apply to EVERY encoding, not just encodings[0]: screenshare publishes
|
||||
// with simulcast (VP8), so encodings[0] is the small layer and the
|
||||
// full-resolution layer — the real bandwidth hog — is a later encoding.
|
||||
for (const enc of params.encodings) Object.assign(enc, patch);
|
||||
await sender.setParameters(params);
|
||||
} catch (e) {
|
||||
logger.warn("[lotus] set_quality: setParameters failed", e);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,98 @@
|
||||
/*
|
||||
Copyright 2026 Lotus Guild
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { combineLatest, map, type Observable, of, switchMap } from "rxjs";
|
||||
|
||||
import { type UserMediaViewModel } from "../state/media/UserMediaViewModel";
|
||||
import { type ScreenShareViewModel } from "../state/media/ScreenShareViewModel";
|
||||
import { type MediaViewModel } from "../state/media/MediaViewModel";
|
||||
import { type LocalUserMediaViewModel } from "../state/media/LocalUserMediaViewModel";
|
||||
|
||||
interface SpotlightAndPip {
|
||||
spotlight: MediaViewModel[];
|
||||
pip$: Observable<UserMediaViewModel | undefined>;
|
||||
}
|
||||
|
||||
/**
|
||||
* [lotus #4] Manual spotlight override.
|
||||
*
|
||||
* Wraps upstream's auto-selected spotlight speaker so a host can pin a specific
|
||||
* participant (via the `io.lotus.focus_participant` widget action). Kept as a
|
||||
* pure function OUTSIDE CallViewModel so CallViewModel stays byte-close to
|
||||
* upstream and rebases cleanly: CallViewModel keeps its original
|
||||
* `spotlightSpeaker$` auto-selection unchanged and just routes the
|
||||
* screenshare/spotlight computation through this wrapper at one call point.
|
||||
*
|
||||
* Behaviour is IDENTICAL to upstream whenever `manualSpotlightUserId$` stays
|
||||
* `null` (the default). When it names a participant that is still present:
|
||||
* - with no screenshare, that participant is spotlighted instead of the
|
||||
* auto-selected active speaker;
|
||||
* - during a screenshare, that participant's camera is surfaced in the
|
||||
* spotlight ALONGSIDE the shared screen (#4 / A5 "focus camera during
|
||||
* screenshare"), and the redundant PiP is hidden.
|
||||
*
|
||||
* @param autoSpotlightSpeaker$ upstream's speaker-follows auto selection
|
||||
* @param manualSpotlightUserId$ host-pinned userId, or `null` for auto (default)
|
||||
* @param screenShares$ current screen-share view models
|
||||
* @param localUserMediaForPip$ local media suitable for the PiP
|
||||
* @param userMedia$ all user media in the call (to resolve the pinned userId)
|
||||
*/
|
||||
export function overrideSpotlight$(
|
||||
autoSpotlightSpeaker$: Observable<UserMediaViewModel | undefined>,
|
||||
manualSpotlightUserId$: Observable<string | null>,
|
||||
screenShares$: Observable<ScreenShareViewModel[]>,
|
||||
localUserMediaForPip$: Observable<LocalUserMediaViewModel | undefined>,
|
||||
userMedia$: Observable<UserMediaViewModel[]>,
|
||||
): Observable<SpotlightAndPip> {
|
||||
// The effective spotlight speaker: the host-pinned participant when set and
|
||||
// still present, otherwise upstream's auto-selected speaker.
|
||||
const spotlightSpeaker$ = combineLatest([
|
||||
autoSpotlightSpeaker$,
|
||||
manualSpotlightUserId$,
|
||||
userMedia$,
|
||||
]).pipe(
|
||||
map(([auto, manualUserId, mediaItems]) => {
|
||||
if (manualUserId !== null) {
|
||||
const pinned = mediaItems.find((m) => m.userId === manualUserId);
|
||||
if (pinned) return pinned;
|
||||
}
|
||||
return auto;
|
||||
}),
|
||||
);
|
||||
|
||||
return screenShares$.pipe(
|
||||
switchMap((screenShares) => {
|
||||
if (screenShares.length > 0)
|
||||
// During a screenshare, if the host has explicitly pinned a
|
||||
// participant, surface that camera in the spotlight alongside the
|
||||
// shared screen (the whole point of "focus camera during screenshare").
|
||||
// With no manual pin this is unchanged: the screenshare alone is
|
||||
// spotlighted.
|
||||
return combineLatest([manualSpotlightUserId$, userMedia$]).pipe(
|
||||
map(([manualUserId, mediaItems]) => {
|
||||
const pinned =
|
||||
manualUserId !== null
|
||||
? mediaItems.find((m) => m.userId === manualUserId)
|
||||
: undefined;
|
||||
return pinned
|
||||
? { spotlight: [...screenShares, pinned], pip$: of(undefined) }
|
||||
: { spotlight: screenShares, pip$: spotlightSpeaker$ };
|
||||
}),
|
||||
);
|
||||
|
||||
return spotlightSpeaker$.pipe(
|
||||
map((speaker) => ({
|
||||
spotlight: speaker ? [speaker] : [],
|
||||
// Hide PiP if redundant (i.e. if local user is already in spotlight)
|
||||
pip$: localUserMediaForPip$.pipe(
|
||||
map((m) => (m === speaker ? undefined : m)),
|
||||
),
|
||||
})),
|
||||
);
|
||||
}),
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
/*
|
||||
Copyright 2026 Lotus Guild
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
/**
|
||||
* Shared helpers for the Lotus fork's widget extensions.
|
||||
*
|
||||
* Everything here is **opt-in**: each feature is gated behind a `lotus*` URL
|
||||
* param that the Lotus host (cinny) appends to the widget iframe URL. With no
|
||||
* param present, none of this code changes Element Call's behaviour — which
|
||||
* keeps the fork a minimal, additive, easy-to-rebase diff over upstream.
|
||||
*/
|
||||
|
||||
import { logger } from "matrix-js-sdk/lib/logger";
|
||||
|
||||
import { widget } from "../widget";
|
||||
import { LotusWidgetActions } from "./lotusActions";
|
||||
|
||||
export { LotusWidgetActions } from "./lotusActions";
|
||||
|
||||
let cachedParams: URLSearchParams | undefined;
|
||||
|
||||
/**
|
||||
* Read a URL param from either the query string or the hash fragment (Element
|
||||
* Call passes widget params via both depending on host), without depending on
|
||||
* EC's own `getUrlParams` parser (keeps the rebase surface small).
|
||||
*/
|
||||
export function lotusParam(name: string): string | null {
|
||||
if (!cachedParams) {
|
||||
// Match EC's own ParamParser precedence: the hash fragment wins over the
|
||||
// query string. So seed from the fragment first, then fill gaps from query.
|
||||
const hash = window.location.hash.replace(/^#\/?/, "");
|
||||
const hashQuery = hash.includes("?") ? hash.slice(hash.indexOf("?") + 1) : "";
|
||||
cachedParams = new URLSearchParams(hashQuery);
|
||||
for (const [k, v] of new URLSearchParams(window.location.search)) {
|
||||
if (!cachedParams.has(k)) cachedParams.append(k, v);
|
||||
}
|
||||
}
|
||||
return cachedParams.get(name);
|
||||
}
|
||||
|
||||
/** Whether a boolean-ish Lotus feature flag is enabled. */
|
||||
export function lotusFlag(name: string): boolean {
|
||||
const v = lotusParam(name);
|
||||
return v === "1" || v === "true";
|
||||
}
|
||||
|
||||
/**
|
||||
* Send a fromWidget message to the Lotus host, swallowing the inevitable
|
||||
* rejection when the host hasn't (yet) registered a handler for it. Returns
|
||||
* true if the widget transport was available to attempt the send.
|
||||
*/
|
||||
export function lotusSendToHost(action: LotusWidgetActions, data: unknown): boolean {
|
||||
const api = widget?.api;
|
||||
if (!api) return false;
|
||||
void api.transport.send(action, data as Record<string, unknown>).catch((e) => {
|
||||
logger.debug(`[lotus] host did not ack ${action}`, e);
|
||||
});
|
||||
return true;
|
||||
}
|
||||
@@ -29,6 +29,13 @@ import { Header, LeftNav, RightNav, RoomHeaderInfo } from "../Header";
|
||||
import { HeaderStyle, useUrlParams } from "../UrlParams";
|
||||
import { useCallViewKeyboardShortcuts } from "../useCallViewKeyboardShortcuts";
|
||||
import { widget } from "../widget";
|
||||
import { startLotusCallState } from "../lotus/lotusCallState";
|
||||
import { startLotusFocus } from "../lotus/lotusFocus";
|
||||
import { startLotusAudioInject } from "../lotus/lotusAudioInject";
|
||||
import { startLotusQuality } from "../lotus/lotusQuality";
|
||||
import { startLotusDecorations } from "../lotus/lotusDecorations";
|
||||
import { startLotusDenoise } from "../lotus/lotusDenoise";
|
||||
import { startLotusDeafen } from "../lotus/lotusDeafen";
|
||||
import styles from "./InCallView.module.css";
|
||||
import { GridTile } from "../tile/GridTile";
|
||||
import { SettingsModal, defaultSettingsTab } from "../settings/SettingsModal";
|
||||
@@ -279,6 +286,29 @@ export const InCallView: FC<InCallViewProps> = ({
|
||||
const earpieceMode = useBehavior(vm.earpieceMode$);
|
||||
const audioOutputSwitcher = useBehavior(vm.audioOutputSwitcher$);
|
||||
|
||||
// [lotus] Stream per-participant speaking/mute state to the host (cinny) over
|
||||
// the widget API when opted in via lotusCallState=1. No-op otherwise.
|
||||
useEffect(() => startLotusCallState(vm), [vm]);
|
||||
// [lotus] Handle the host's io.lotus.focus_participant action to pin a
|
||||
// participant to the spotlight (#4). No-op unless the host sends it.
|
||||
useEffect(() => startLotusFocus(vm), [vm]);
|
||||
// [lotus] Handle the host's io.lotus.inject_audio action to mix a soundboard
|
||||
// clip into the call as a separate track (#3). No-op unless the host sends it.
|
||||
useEffect(() => startLotusAudioInject(vm), [vm]);
|
||||
// [lotus] Handle the host's io.lotus.set_quality action to cap audio/
|
||||
// screenshare encoding bitrate/framerate (#7). No-op unless the host sends it.
|
||||
useEffect(() => startLotusQuality(vm), [vm]);
|
||||
// [lotus] Receive per-user avatar-decoration URLs from the host and render
|
||||
// them on in-call tile avatars (#6). No-op unless the host sends them.
|
||||
useEffect(() => startLotusDecorations(), []);
|
||||
// [lotus] Apply ML denoise to the mic as a first-class audio processor that
|
||||
// survives reconnects (#1 / A7). No-op unless lotusDenoiseSource=1.
|
||||
useEffect(() => startLotusDenoise(vm), [vm]);
|
||||
// [lotus] Handle the host's io.lotus.set_deafen action to silence remote
|
||||
// audio (and optionally screenshare audio) at the LiveKit source. No-op
|
||||
// unless the host sends the action.
|
||||
useEffect(() => startLotusDeafen(vm), [vm]);
|
||||
|
||||
const fatalCallError = useBehavior(vm.fatalError$);
|
||||
// Stop the rendering and throw for the error boundary
|
||||
if (fatalCallError) {
|
||||
|
||||
@@ -151,6 +151,9 @@ import { type UserMediaViewModel } from "../media/UserMediaViewModel.ts";
|
||||
import { type MediaViewModel } from "../media/MediaViewModel.ts";
|
||||
import { type LocalUserMediaViewModel } from "../media/LocalUserMediaViewModel.ts";
|
||||
import { type RemoteUserMediaViewModel } from "../media/RemoteUserMediaViewModel.ts";
|
||||
// [lotus #4] Manual spotlight override, extracted so this file stays byte-close
|
||||
// to upstream (see the file for the behaviour contract).
|
||||
import { overrideSpotlight$ } from "../../lotus/lotusSpotlight";
|
||||
import {
|
||||
createRingingMedia,
|
||||
type RingingMediaViewModel,
|
||||
@@ -255,6 +258,11 @@ export interface CallViewModel {
|
||||
* Callback to toggle screen sharing. If null, screen sharing is not possible.
|
||||
*/
|
||||
toggleScreenSharing: (() => void) | null;
|
||||
/**
|
||||
* [lotus] Pin a participant to the spotlight by Matrix user id (#4
|
||||
* focus-participant). Pass null to clear and restore speaker-follows.
|
||||
*/
|
||||
setManualSpotlight: (userId: string | null) => void;
|
||||
/**
|
||||
* Whether we are sharing our screen.
|
||||
*/
|
||||
@@ -295,6 +303,8 @@ export interface CallViewModel {
|
||||
* multiple devices.
|
||||
*/
|
||||
participantCount$: Behavior<number>;
|
||||
/** [lotus] All participants' user media, exposed for the Lotus call-state widget bridge. */
|
||||
userMedia$: Behavior<WrappedUserMediaViewModel[]>;
|
||||
allConnections$: Behavior<ConnectionManagerData>;
|
||||
/** Participants sorted by livekit room so they can be used in the audio rendering */
|
||||
livekitRoomItems$: Behavior<LivekitRoomItem[]>;
|
||||
@@ -497,6 +507,7 @@ export function createCallViewModel$(
|
||||
options.livekitRoomFactory,
|
||||
getUrlParams().echoCancellation,
|
||||
getUrlParams().noiseSuppression,
|
||||
getUrlParams().autoGainControl,
|
||||
);
|
||||
|
||||
const connectionManager = createConnectionManager$({
|
||||
@@ -931,6 +942,11 @@ export function createCallViewModel$(
|
||||
),
|
||||
);
|
||||
|
||||
// [lotus #4] Host-pinned spotlight target (io.lotus.focus_participant). null =
|
||||
// follow the active speaker (upstream default), so this is inert unless the
|
||||
// host pins someone. Consumed by overrideSpotlight$ in spotlightAndPip$.
|
||||
const manualSpotlightUserId$ = new BehaviorSubject<string | null>(null);
|
||||
|
||||
const grid$ = scope.behavior<UserMediaViewModel[]>(
|
||||
userMedia$.pipe(
|
||||
switchMap((mediaItems) => {
|
||||
@@ -978,21 +994,17 @@ export function createCallViewModel$(
|
||||
if (ringingMedia.length > 0)
|
||||
return of({ spotlight: ringingMedia, pip$: localUserMediaForPip$ });
|
||||
|
||||
return screenShares$.pipe(
|
||||
switchMap((screenShares) => {
|
||||
if (screenShares.length > 0)
|
||||
return of({ spotlight: screenShares, pip$: spotlightSpeaker$ });
|
||||
|
||||
return spotlightSpeaker$.pipe(
|
||||
map((speaker) => ({
|
||||
spotlight: speaker ? [speaker] : [],
|
||||
// Hide PiP if redundant (i.e. if local user is already in spotlight)
|
||||
pip$: localUserMediaForPip$.pipe(
|
||||
map((m) => (m === speaker ? undefined : m)),
|
||||
),
|
||||
})),
|
||||
);
|
||||
}),
|
||||
// [lotus #4] Route the screenshare/spotlight computation through the
|
||||
// manual-spotlight wrapper. This is byte-for-byte upstream behaviour
|
||||
// unless the host pins a participant via io.lotus.focus_participant;
|
||||
// see src/lotus/lotusSpotlight.ts. Kept out-of-line so CallViewModel
|
||||
// stays close to upstream and rebases cleanly.
|
||||
return overrideSpotlight$(
|
||||
spotlightSpeaker$,
|
||||
manualSpotlightUserId$,
|
||||
screenShares$,
|
||||
localUserMediaForPip$,
|
||||
userMedia$,
|
||||
);
|
||||
}),
|
||||
),
|
||||
@@ -1697,6 +1709,8 @@ export function createCallViewModel$(
|
||||
join: localMembership.requestJoinAndPublish,
|
||||
leave: localMembership.requestDisconnect,
|
||||
toggleScreenSharing: toggleScreenSharing,
|
||||
setManualSpotlight: (userId: string | null): void =>
|
||||
manualSpotlightUserId$.next(userId),
|
||||
sharingScreen$: sharingScreen$,
|
||||
|
||||
tapScreen: (): void => screenTap$.next(),
|
||||
@@ -1721,6 +1735,7 @@ export function createCallViewModel$(
|
||||
),
|
||||
allConnections$,
|
||||
participantCount$: participantCount$,
|
||||
userMedia$,
|
||||
handsRaised$: handsRaised$,
|
||||
reactions$: reactions$,
|
||||
joinSoundEffect$: joinSoundEffect$,
|
||||
|
||||
@@ -55,6 +55,7 @@ export class ECConnectionFactory implements ConnectionFactory {
|
||||
* @param livekitRoomFactory - Optional factory function (for testing) to create LivekitRoom instances. If not provided, a default factory is used.
|
||||
* @param echoCancellation - Whether to enable echo cancellation for audio capture.
|
||||
* @param noiseSuppression - Whether to enable noise suppression for audio capture.
|
||||
* @param autoGainControl - Whether to enable auto gain control for audio capture.
|
||||
*/
|
||||
public constructor(
|
||||
private client: OpenIDClientParts,
|
||||
@@ -66,6 +67,7 @@ export class ECConnectionFactory implements ConnectionFactory {
|
||||
livekitRoomFactory?: () => LivekitRoom,
|
||||
echoCancellation: boolean = true,
|
||||
noiseSuppression: boolean = true,
|
||||
autoGainControl: boolean = true,
|
||||
) {
|
||||
const defaultFactory = (): LivekitRoom =>
|
||||
new LivekitRoom(
|
||||
@@ -81,6 +83,7 @@ export class ECConnectionFactory implements ConnectionFactory {
|
||||
controlledAudioDevices: this.controlledAudioDevices,
|
||||
echoCancellation,
|
||||
noiseSuppression,
|
||||
autoGainControl,
|
||||
}),
|
||||
);
|
||||
this.livekitRoomFactory = livekitRoomFactory ?? defaultFactory;
|
||||
@@ -127,6 +130,7 @@ function generateRoomOption({
|
||||
controlledAudioDevices,
|
||||
echoCancellation,
|
||||
noiseSuppression,
|
||||
autoGainControl,
|
||||
}: {
|
||||
devices: MediaDevices;
|
||||
processorState: ProcessorState;
|
||||
@@ -137,6 +141,7 @@ function generateRoomOption({
|
||||
controlledAudioDevices: boolean;
|
||||
echoCancellation: boolean;
|
||||
noiseSuppression: boolean;
|
||||
autoGainControl: boolean;
|
||||
}): RoomOptions {
|
||||
return {
|
||||
...defaultLiveKitOptions,
|
||||
@@ -150,6 +155,7 @@ function generateRoomOption({
|
||||
deviceId: devices.audioInput.selected$.value?.id,
|
||||
echoCancellation,
|
||||
noiseSuppression,
|
||||
autoGainControl,
|
||||
},
|
||||
audioOutput: {
|
||||
// When using controlled audio devices, we don't want to set the
|
||||
|
||||
@@ -53,14 +53,13 @@ beforeEach(() => {
|
||||
|
||||
describe("ECConnectionFactory - Audio inputs options", () => {
|
||||
test.each([
|
||||
{ echo: true, noise: true },
|
||||
{ echo: true, noise: false },
|
||||
{ echo: false, noise: true },
|
||||
{ echo: false, noise: false },
|
||||
{ echo: true, noise: true, agc: true },
|
||||
{ echo: true, noise: false, agc: false },
|
||||
{ echo: false, noise: true, agc: false },
|
||||
{ echo: false, noise: false, agc: true },
|
||||
])(
|
||||
"it sets echoCancellation=$echo and noiseSuppression=$noise based on constructor parameters",
|
||||
({ echo, noise }) => {
|
||||
// test("it sets echoCancellation and noiseSuppression based on constructor parameters", () => {
|
||||
"it sets echoCancellation=$echo, noiseSuppression=$noise, autoGainControl=$agc based on constructor parameters",
|
||||
({ echo, noise, agc }) => {
|
||||
const RoomConstructor = vi.mocked(LivekitRoom);
|
||||
|
||||
const ecConnectionFactory = new ECConnectionFactory(
|
||||
@@ -76,6 +75,7 @@ describe("ECConnectionFactory - Audio inputs options", () => {
|
||||
undefined,
|
||||
echo,
|
||||
noise,
|
||||
agc,
|
||||
);
|
||||
ecConnectionFactory.createConnection(
|
||||
testScope,
|
||||
@@ -90,6 +90,7 @@ describe("ECConnectionFactory - Audio inputs options", () => {
|
||||
audioCaptureDefaults: expect.objectContaining({
|
||||
echoCancellation: echo,
|
||||
noiseSuppression: noise,
|
||||
autoGainControl: agc,
|
||||
}),
|
||||
}),
|
||||
);
|
||||
|
||||
@@ -12,7 +12,12 @@ Please see LICENSE in the repository root for full details.
|
||||
}
|
||||
|
||||
.tabList {
|
||||
overflow-y: auto;
|
||||
/* The tab row is horizontal, so scroll along the inline axis when the tabs
|
||||
exceed the width (e.g. the Settings modal as a phone drawer). The prior
|
||||
overflow-y:auto expressed the wrong axis; make the horizontal intent
|
||||
explicit and hide the scrollbar. */
|
||||
overflow-x: auto;
|
||||
overflow-y: hidden;
|
||||
max-width: 100%;
|
||||
|
||||
/*no scrollbars*/
|
||||
@@ -24,4 +29,5 @@ Please see LICENSE in the repository root for full details.
|
||||
/*no scrollbars*/
|
||||
background: transparent; /* Chrome/Safari/Webkit */
|
||||
width: 0px;
|
||||
height: 0px;
|
||||
}
|
||||
|
||||
@@ -98,6 +98,14 @@ borders don't support gradients */
|
||||
color: var(--cpd-color-icon-primary);
|
||||
}
|
||||
|
||||
/* The camera-flip control is always visible and primarily used on phones;
|
||||
its inherited 4px padding yields a ~28px box, below the touch minimum. */
|
||||
@media (hover: none), (pointer: coarse) {
|
||||
.tile .switchCamera {
|
||||
padding: var(--cpd-space-3x);
|
||||
}
|
||||
}
|
||||
|
||||
@media (hover) {
|
||||
.tile .switchCamera:hover {
|
||||
background: var(--cpd-color-bg-subtle-secondary);
|
||||
|
||||
@@ -61,6 +61,24 @@ Please see LICENSE in the repository root for full details.
|
||||
mix-blend-mode: multiply;
|
||||
}
|
||||
|
||||
/* [lotus #6] Profile decoration overlaid on the tile avatar. Shares the
|
||||
avatar's centred box and size so frame-style decorations sit around it. */
|
||||
.lotusDecoration {
|
||||
position: absolute;
|
||||
top: 50%;
|
||||
left: 50%;
|
||||
transform: translate(-50%, -50%);
|
||||
pointer-events: none;
|
||||
object-fit: contain;
|
||||
}
|
||||
|
||||
@container mediaView (width > 0) {
|
||||
.lotusDecoration {
|
||||
inline-size: 50cqmin;
|
||||
block-size: 50cqmin;
|
||||
}
|
||||
}
|
||||
|
||||
/* CSS makes us put a condition here, even though all we want to do is
|
||||
unconditionally select the container so we can use cqmin units */
|
||||
@container mediaView (width > 0) {
|
||||
|
||||
@@ -22,6 +22,7 @@ import { ErrorSolidIcon } from "@vector-im/compound-design-tokens/assets/web/ico
|
||||
|
||||
import styles from "./MediaView.module.css";
|
||||
import { Avatar } from "../Avatar";
|
||||
import { useLotusDecoration } from "../lotus/lotusDecorations";
|
||||
import { RaisedHandIndicator } from "../reactions/RaisedHandIndicator";
|
||||
import {
|
||||
showConnectionStats as showConnectionStatsSetting,
|
||||
@@ -91,6 +92,7 @@ export const MediaView: FC<Props> = ({
|
||||
...props
|
||||
}) => {
|
||||
const { t } = useTranslation();
|
||||
const decoration = useLotusDecoration(userId);
|
||||
const [handRaiseTimerVisible] = useSetting(showHandRaisedTimer);
|
||||
const [showConnectionStats] = useSetting(showConnectionStatsSetting);
|
||||
|
||||
@@ -137,6 +139,17 @@ export const MediaView: FC<Props> = ({
|
||||
})}
|
||||
style={{ display: video && videoEnabled ? "none" : "initial" }}
|
||||
/>
|
||||
{decoration && !(video && videoEnabled) && (
|
||||
// [lotus #6] Profile decoration overlay, shown only when the avatar
|
||||
// is visible (i.e. not when live video is showing). Pushed by the
|
||||
// host via io.lotus.decorations; undefined unless opted in.
|
||||
<img
|
||||
className={styles.lotusDecoration}
|
||||
src={decoration}
|
||||
alt=""
|
||||
aria-hidden
|
||||
/>
|
||||
)}
|
||||
{video?.publication !== undefined && (
|
||||
<VideoTrack
|
||||
trackRef={video}
|
||||
|
||||
@@ -178,7 +178,13 @@ export function useAudioContext<S extends string>(
|
||||
if (
|
||||
audioContext &&
|
||||
"setSinkId" in audioContext &&
|
||||
!controlledAudioDevices
|
||||
!controlledAudioDevices &&
|
||||
// Skip until a device is actually selected. audioOutputId is undefined
|
||||
// before MediaDevices resolves (e.g. on the Tauri desktop webview, where
|
||||
// the selected$ observable emits undefined first); setSinkId(undefined)
|
||||
// throws "The provided value is not of type 'AudioSinkOptions'". The
|
||||
// default device is represented by the empty string, which is still valid.
|
||||
typeof audioOutputId === "string"
|
||||
) {
|
||||
// https://developer.mozilla.org/en-US/docs/Web/API/AudioContext/setSinkId
|
||||
// @ts-expect-error - setSinkId doesn't exist yet in types, maybe because it's not supported everywhere.
|
||||
|
||||
@@ -11,6 +11,7 @@ import { type IThemeChangeActionRequest } from "matrix-widget-api";
|
||||
|
||||
import { getUrlParams } from "./UrlParams";
|
||||
import { widget } from "./widget";
|
||||
import { lotusFlag } from "./lotus/lotusWidget";
|
||||
|
||||
export const useTheme = (): void => {
|
||||
const [requestedTheme, setRequestedTheme] = useState(
|
||||
@@ -57,5 +58,10 @@ export const useTheme = (): void => {
|
||||
previousTheme.current = themeString;
|
||||
}
|
||||
document.body.classList.remove("no-theme");
|
||||
// [lotus #5] Native theming hooks, opted in by the host via URL flags, so
|
||||
// it no longer has to inject CSS into the iframe after load.
|
||||
if (lotusFlag("lotusTransparent"))
|
||||
document.body.classList.add("lotus-transparent");
|
||||
if (lotusFlag("lotusTheme")) document.body.classList.add("lotus-theme");
|
||||
}, [previousTheme, requestedTheme]);
|
||||
};
|
||||
|
||||
@@ -22,6 +22,7 @@ import { LazyEventEmitter } from "./LazyEventEmitter";
|
||||
import { getUrlParams } from "./UrlParams";
|
||||
import { Config } from "./config/Config";
|
||||
import { ElementCallReactionEventType } from "./reactions";
|
||||
import { LOTUS_TO_WIDGET_ACTIONS } from "./lotus/lotusActions";
|
||||
|
||||
// Subset of the actions in element-web
|
||||
export enum ElementWidgetActions {
|
||||
@@ -103,6 +104,8 @@ export const initializeWidget = (
|
||||
ElementWidgetActions.JoinCall,
|
||||
ElementWidgetActions.HangupCall,
|
||||
ElementWidgetActions.DeviceMute,
|
||||
// [lotus] custom toWidget actions handled by the fork (focus, audio-inject)
|
||||
...LOTUS_TO_WIDGET_ACTIONS,
|
||||
].forEach((action) => {
|
||||
api.on(`action:${action}`, (ev: CustomEvent<IWidgetApiRequest>) => {
|
||||
ev.preventDefault();
|
||||
|
||||
Reference in New Issue
Block a user