Compare commits
@@ -0,0 +1,126 @@
|
|||||||
|
name: CI
|
||||||
|
|
||||||
|
# Build + publish the Lotus fork of Element Call's embedded web bundle.
|
||||||
|
# Models LotusGuild/cinny's .gitea/workflows/ci.yml. Web bundle only, so
|
||||||
|
# Linux-only (the Windows worker is for cinny-desktop / Tauri, not needed here).
|
||||||
|
#
|
||||||
|
# - PRs and pushes to `lotus` -> build + smoke-check the embedded dist
|
||||||
|
# - Pushes of a tag `v*` -> build + publish @lotusguild/element-call-embedded
|
||||||
|
# to the Gitea npm registry
|
||||||
|
#
|
||||||
|
# Versioning: the checked-in embedded/web/package.json tracks the intended next
|
||||||
|
# release (e.g. 0.20.1-lotus.2) for local/manual publishes, but the AUTHORITATIVE
|
||||||
|
# published version is still derived from the git tag at publish time (the
|
||||||
|
# `npm version "$TAG"` step below overwrites it, so a tag always wins).
|
||||||
|
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
branches: [lotus]
|
||||||
|
tags: ['v*']
|
||||||
|
pull_request:
|
||||||
|
branches: [lotus]
|
||||||
|
|
||||||
|
env:
|
||||||
|
# element-call's build:full sets 16384 already; keep parity for safety.
|
||||||
|
NODE_OPTIONS: '--max-old-space-size=16384'
|
||||||
|
# Stamp the build so analytics/rageshakes aren't labelled "dev".
|
||||||
|
VITE_APP_VERSION: ${{ github.ref_name }}
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
build:
|
||||||
|
name: Build embedded bundle
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- name: Checkout
|
||||||
|
uses: actions/checkout@v4
|
||||||
|
|
||||||
|
- name: Setup Node.js
|
||||||
|
uses: actions/setup-node@v4
|
||||||
|
with:
|
||||||
|
node-version-file: '.node-version' # Node 24
|
||||||
|
# registry-url wires up the Gitea-scoped registry + auth for publish
|
||||||
|
registry-url: 'https://code.lotusguild.org/api/packages/LotusGuild/npm/'
|
||||||
|
scope: '@lotusguild'
|
||||||
|
|
||||||
|
- name: Enable corepack (pnpm from packageManager field)
|
||||||
|
run: corepack enable
|
||||||
|
|
||||||
|
- name: Install dependencies
|
||||||
|
run: pnpm install --frozen-lockfile
|
||||||
|
|
||||||
|
- name: Build embedded
|
||||||
|
# build:embedded writes to the repo-root dist/ (vite default outDir).
|
||||||
|
# Stage it into embedded/web/dist — the publish template's "files" entry
|
||||||
|
# — so the smoke-check and npm publish both see the bundle.
|
||||||
|
# (embedded/web/dist is gitignored, hence the explicit copy.)
|
||||||
|
run: |
|
||||||
|
pnpm run build:embedded
|
||||||
|
rm -rf embedded/web/dist
|
||||||
|
cp -r dist embedded/web/dist
|
||||||
|
|
||||||
|
- name: Smoke-check output
|
||||||
|
run: |
|
||||||
|
test -f embedded/web/dist/index.html
|
||||||
|
test -d embedded/web/dist/assets
|
||||||
|
echo "### Embedded bundle" >> "$GITHUB_STEP_SUMMARY"
|
||||||
|
echo "" >> "$GITHUB_STEP_SUMMARY"
|
||||||
|
echo '```' >> "$GITHUB_STEP_SUMMARY"
|
||||||
|
du -sh embedded/web/dist >> "$GITHUB_STEP_SUMMARY"
|
||||||
|
ls embedded/web/dist >> "$GITHUB_STEP_SUMMARY"
|
||||||
|
echo '```' >> "$GITHUB_STEP_SUMMARY"
|
||||||
|
|
||||||
|
# NOTE: no actions/upload-artifact step — v4 requires GitHub's artifact
|
||||||
|
# backend, which the Gitea act_runner does not implement (it fails with
|
||||||
|
# exit 1). The publish job rebuilds from source anyway, so the artifact
|
||||||
|
# was only a convenience; re-add a Gitea-compatible action here if per-run
|
||||||
|
# dist downloads are ever needed.
|
||||||
|
|
||||||
|
publish:
|
||||||
|
name: Publish to Gitea npm registry
|
||||||
|
needs: build
|
||||||
|
# Only on a version tag push, never on PRs.
|
||||||
|
if: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') }}
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- name: Checkout
|
||||||
|
uses: actions/checkout@v4
|
||||||
|
|
||||||
|
- name: Setup Node.js
|
||||||
|
uses: actions/setup-node@v4
|
||||||
|
with:
|
||||||
|
node-version-file: '.node-version'
|
||||||
|
registry-url: 'https://code.lotusguild.org/api/packages/LotusGuild/npm/'
|
||||||
|
scope: '@lotusguild'
|
||||||
|
|
||||||
|
- name: Enable corepack
|
||||||
|
run: corepack enable
|
||||||
|
|
||||||
|
- name: Install dependencies
|
||||||
|
run: pnpm install --frozen-lockfile
|
||||||
|
|
||||||
|
- name: Build embedded
|
||||||
|
# build:embedded writes to the repo-root dist/ (vite default outDir).
|
||||||
|
# Stage it into embedded/web/dist — the publish template's "files" entry
|
||||||
|
# — so the smoke-check and npm publish both see the bundle.
|
||||||
|
# (embedded/web/dist is gitignored, hence the explicit copy.)
|
||||||
|
run: |
|
||||||
|
pnpm run build:embedded
|
||||||
|
rm -rf embedded/web/dist
|
||||||
|
cp -r dist embedded/web/dist
|
||||||
|
|
||||||
|
- name: Set published version from tag
|
||||||
|
working-directory: embedded/web
|
||||||
|
# Versioning scheme: reserve bare vX.Y.Z for upstream-parity points only
|
||||||
|
# (e.g. v0.20.1 == upstream 0.20.1). For Lotus-only iterations on top of
|
||||||
|
# an upstream base, tag a semver prerelease — v0.20.1-lotus.1, -lotus.2,
|
||||||
|
# … — so we never collide with an already-published upstream-parity
|
||||||
|
# version on the registry.
|
||||||
|
run: |
|
||||||
|
TAG="${GITHUB_REF_NAME#v}" # v0.20.1-lotus.1 -> 0.20.1-lotus.1
|
||||||
|
npm version "$TAG" --no-git-tag-version --allow-same-version
|
||||||
|
|
||||||
|
- name: Publish
|
||||||
|
working-directory: embedded/web
|
||||||
|
env:
|
||||||
|
NODE_AUTH_TOKEN: ${{ secrets.GITEA_NPM_TOKEN }}
|
||||||
|
run: npm publish --access public
|
||||||
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "@element-hq/element-call-embedded",
|
"name": "@lotusguild/element-call-embedded",
|
||||||
"version": "0.0.0",
|
"version": "0.20.1-lotus.2",
|
||||||
"files": [
|
"files": [
|
||||||
"README.md",
|
"README.md",
|
||||||
"LICENSE-AGPL-3.0",
|
"LICENSE-AGPL-3.0",
|
||||||
@@ -9,6 +9,6 @@
|
|||||||
],
|
],
|
||||||
"repository": {
|
"repository": {
|
||||||
"type": "git",
|
"type": "git",
|
||||||
"url": "git+https://github.com/element-hq/element-call.git"
|
"url": "git+https://code.lotusguild.org/LotusGuild/element-call.git"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -391,6 +391,21 @@ describe("UrlParams", () => {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe("autoGainControl", () => {
|
||||||
|
it("defaults to true", () => {
|
||||||
|
expect(computeUrlParams().autoGainControl).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("is parsed", () => {
|
||||||
|
expect(computeUrlParams("?autoGainControl=true").autoGainControl).toBe(
|
||||||
|
true,
|
||||||
|
);
|
||||||
|
expect(computeUrlParams("?autoGainControl=false").autoGainControl).toBe(
|
||||||
|
false,
|
||||||
|
);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
describe("header", () => {
|
describe("header", () => {
|
||||||
it("uses header if provided", () => {
|
it("uses header if provided", () => {
|
||||||
expect(computeUrlParams("?header=app_bar&hideHeader=true").header).toBe(
|
expect(computeUrlParams("?header=app_bar&hideHeader=true").header).toBe(
|
||||||
|
|||||||
@@ -239,6 +239,12 @@ export interface UrlConfiguration {
|
|||||||
* Defaults to true.
|
* Defaults to true.
|
||||||
*/
|
*/
|
||||||
noiseSuppression?: boolean;
|
noiseSuppression?: boolean;
|
||||||
|
/**
|
||||||
|
* Whether to enable auto gain control for audio capture.
|
||||||
|
* Defaults to true. Lotus turns this OFF for the in-source ML denoise tier so
|
||||||
|
* the browser's dynamic gain doesn't fight the ML model (pumping artifacts).
|
||||||
|
*/
|
||||||
|
autoGainControl?: boolean;
|
||||||
|
|
||||||
callIntent?: RTCCallIntent;
|
callIntent?: RTCCallIntent;
|
||||||
}
|
}
|
||||||
@@ -485,6 +491,7 @@ export const computeUrlParams = (search = "", hash = ""): UrlParams => {
|
|||||||
autoLeaveWhenOthersLeft: parser.getFlag("autoLeave"),
|
autoLeaveWhenOthersLeft: parser.getFlag("autoLeave"),
|
||||||
noiseSuppression: parser.getFlagParam("noiseSuppression", true),
|
noiseSuppression: parser.getFlagParam("noiseSuppression", true),
|
||||||
echoCancellation: parser.getFlagParam("echoCancellation", true),
|
echoCancellation: parser.getFlagParam("echoCancellation", true),
|
||||||
|
autoGainControl: parser.getFlagParam("autoGainControl", true),
|
||||||
};
|
};
|
||||||
|
|
||||||
// Log the final configuration for debugging purposes.
|
// Log the final configuration for debugging purposes.
|
||||||
|
|||||||
@@ -12,8 +12,10 @@
|
|||||||
|
|
||||||
@media (max-width: 420px) {
|
@media (max-width: 420px) {
|
||||||
.reactionPopupMenu {
|
.reactionPopupMenu {
|
||||||
--reaction-button-padding: 8px;
|
/* 20px glyph + 2×12px padding = a 44px square touch target. Five buttons
|
||||||
--reaction-button-fontsize: 16px;
|
still fit a ~360px drawer row; previously these were ~32px. */
|
||||||
|
--reaction-button-padding: 12px;
|
||||||
|
--reaction-button-fontsize: 20px;
|
||||||
--reaction-button-gap: 6px;
|
--reaction-button-gap: 6px;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -102,6 +102,20 @@ Once we exceed 500 we hide everything except the buttons.
|
|||||||
@media (max-width: 500px) {
|
@media (max-width: 500px) {
|
||||||
.footer {
|
.footer {
|
||||||
grid-template-areas: "buttons buttons buttons";
|
grid-template-areas: "buttons buttons buttons";
|
||||||
|
/* Reclaim horizontal budget so the control row fits on phones. */
|
||||||
|
padding-left: calc(env(safe-area-inset-left) + var(--cpd-space-3x));
|
||||||
|
padding-right: calc(env(safe-area-inset-right) + var(--cpd-space-3x));
|
||||||
|
}
|
||||||
|
|
||||||
|
.buttons {
|
||||||
|
/* Fill the row and wrap instead of overflowing: on the narrowest phones the
|
||||||
|
control set (mic/video/loudspeaker/hangup) exceeds the viewport, and the
|
||||||
|
parent's overflow-x: hidden would otherwise clip the outer buttons —
|
||||||
|
including the destructive hangup. Wrapping keeps every control reachable. */
|
||||||
|
justify-self: stretch;
|
||||||
|
flex-wrap: wrap;
|
||||||
|
justify-content: center;
|
||||||
|
gap: var(--cpd-space-2x);
|
||||||
}
|
}
|
||||||
|
|
||||||
.settingsOnlyShowNarrow {
|
.settingsOnlyShowNarrow {
|
||||||
|
|||||||
@@ -16,7 +16,14 @@ Please see LICENSE in the repository root for full details.
|
|||||||
|
|
||||||
.pip {
|
.pip {
|
||||||
position: absolute;
|
position: absolute;
|
||||||
inset: var(--cpd-space-4x);
|
/* Keep the self-view within the safe area (home indicator) and the inline
|
||||||
|
content insets, and clear of the floating footer — matching the
|
||||||
|
reference-correct SpotlightExpandedLayout. A bare 16px inset let the
|
||||||
|
bottom-right self-tile sit under the controls / device chrome on phones. */
|
||||||
|
inset: calc(env(safe-area-inset-top) + var(--cpd-space-4x))
|
||||||
|
var(--content-inset-right)
|
||||||
|
calc(env(safe-area-inset-bottom) + var(--cpd-space-4x))
|
||||||
|
var(--content-inset-left);
|
||||||
}
|
}
|
||||||
|
|
||||||
.pip[data-size="sm"] {
|
.pip[data-size="sm"] {
|
||||||
|
|||||||
@@ -43,3 +43,17 @@ unconditionally select the container so we can use cq units */
|
|||||||
inline-size: 180px;
|
inline-size: 180px;
|
||||||
block-size: 135px;
|
block-size: 135px;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* On a landscape phone the fixed 180px filmstrip rail squeezes the spotlight to
|
||||||
|
a sliver; shrink the rail (keeping the 4:3 tile ratio) to give the spotlight
|
||||||
|
room. Desktop landscape (height > 400px) is unaffected. */
|
||||||
|
@media (max-height: 400px) {
|
||||||
|
.layer {
|
||||||
|
--grid-slot-width: 132px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.grid > .slot {
|
||||||
|
inline-size: 132px;
|
||||||
|
block-size: 99px;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -85,6 +85,25 @@ body {
|
|||||||
-webkit-tap-highlight-color: transparent;
|
-webkit-tap-highlight-color: transparent;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* [lotus] When embedded in the Lotus host with lotusTransparent=1, make the
|
||||||
|
app background transparent so the host's wallpaper shows through natively —
|
||||||
|
replacing the host's injected `html, body { background: none !important }`
|
||||||
|
override. */
|
||||||
|
body.lotus-transparent,
|
||||||
|
body.lotus-transparent #root {
|
||||||
|
background: transparent !important;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* [lotus] Native Lotus/TDS theme, applied when lotusTheme=1, instead of the
|
||||||
|
host injecting CSS into the iframe after load. Overrides Compound design tokens
|
||||||
|
with Lotus values at the source so theming is complete and flash-free. Extend
|
||||||
|
this block with the full Lotus token map from the design system; the canvas
|
||||||
|
override below is a safe starting point that matches the Lotus dark surface. */
|
||||||
|
body.lotus-theme {
|
||||||
|
--cpd-color-bg-canvas-default: #0c0d10;
|
||||||
|
--video-tile-background: var(--cpd-color-bg-subtle-secondary);
|
||||||
|
}
|
||||||
|
|
||||||
/* This prohibits the view to scroll for pages smaller than 122px in width
|
/* This prohibits the view to scroll for pages smaller than 122px in width
|
||||||
we use this for mobile pip webviews */
|
we use this for mobile pip webviews */
|
||||||
.no-scroll-body {
|
.no-scroll-body {
|
||||||
|
|||||||
@@ -0,0 +1,43 @@
|
|||||||
|
/*
|
||||||
|
Copyright 2026 Lotus Guild
|
||||||
|
|
||||||
|
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||||
|
Please see LICENSE in the repository root for full details.
|
||||||
|
*/
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Custom widget actions exchanged between the Lotus host (cinny) and this fork.
|
||||||
|
* Kept dependency-free so both `widget.ts` and `lotus/*` can import it without
|
||||||
|
* a circular dependency.
|
||||||
|
*/
|
||||||
|
export enum LotusWidgetActions {
|
||||||
|
/**
|
||||||
|
* fromWidget: in-call per-participant speaking / mute state.
|
||||||
|
* NOTE: matrix-widget-api `transport.send` is request/response — the host
|
||||||
|
* MUST register a handler that replies/acks each one (cinny's `listenAction`
|
||||||
|
* does, replying `{}`). If the host has no handler, ClientWidgetApi
|
||||||
|
* immediately error-replies "unsupported from-widget action", so the data is
|
||||||
|
* silently dropped and each throttled send rejects (caught) — functional miss
|
||||||
|
* + log churn, not a hang.
|
||||||
|
*/
|
||||||
|
CallState = "io.lotus.call_state",
|
||||||
|
/** toWidget: pin/spotlight (or clear, with userId=null) a participant. */
|
||||||
|
FocusParticipant = "io.lotus.focus_participant",
|
||||||
|
/** toWidget: play an audio clip into the call as a separate published track. */
|
||||||
|
InjectAudio = "io.lotus.inject_audio",
|
||||||
|
/** toWidget: set audio/screenshare encoding quality (bitrate/framerate). */
|
||||||
|
SetQuality = "io.lotus.set_quality",
|
||||||
|
/** toWidget: per-user avatar-decoration image URLs for in-call tiles. */
|
||||||
|
Decorations = "io.lotus.decorations",
|
||||||
|
/** toWidget: deafen remote audio (and optionally mute screenshare audio). */
|
||||||
|
SetDeafen = "io.lotus.set_deafen",
|
||||||
|
}
|
||||||
|
|
||||||
|
/** toWidget Lotus actions that must be allow-listed in `initializeWidget`. */
|
||||||
|
export const LOTUS_TO_WIDGET_ACTIONS: LotusWidgetActions[] = [
|
||||||
|
LotusWidgetActions.FocusParticipant,
|
||||||
|
LotusWidgetActions.InjectAudio,
|
||||||
|
LotusWidgetActions.SetQuality,
|
||||||
|
LotusWidgetActions.Decorations,
|
||||||
|
LotusWidgetActions.SetDeafen,
|
||||||
|
];
|
||||||
@@ -0,0 +1,247 @@
|
|||||||
|
/*
|
||||||
|
Copyright 2026 Lotus Guild
|
||||||
|
|
||||||
|
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||||
|
Please see LICENSE in the repository root for full details.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { type Room as LivekitRoom, Track } from "livekit-client";
|
||||||
|
import { logger } from "matrix-js-sdk/lib/logger";
|
||||||
|
import { type IWidgetApiRequest } from "matrix-widget-api";
|
||||||
|
|
||||||
|
import { type CallViewModel } from "../state/CallViewModel/CallViewModel";
|
||||||
|
import { widget } from "../widget";
|
||||||
|
import { LotusWidgetActions } from "./lotusActions";
|
||||||
|
import { lotusFlag } from "./lotusWidget";
|
||||||
|
|
||||||
|
/** Hard cap so a malformed/huge clip can't hold a published track open forever. */
|
||||||
|
const MAX_CLIP_MS = 30_000;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Handle the host's `io.lotus.inject_audio` toWidget action (#3): mix a
|
||||||
|
* soundboard clip into the call so other participants hear it.
|
||||||
|
*
|
||||||
|
* Rather than splice into the local mic track (which would fight the denoise
|
||||||
|
* pipeline), we publish the clip as a separate `Unknown`-source LiveKit audio
|
||||||
|
* track — which `MatrixAudioRenderer` already renders for valid call members —
|
||||||
|
* and unpublish it when the clip ends. This is the real call-audio injection
|
||||||
|
* that was impossible against the prebuilt EC bundle (LiveKit's
|
||||||
|
* LocalParticipant lived in EC's module scope).
|
||||||
|
*
|
||||||
|
* Action data: `{ url: string, volume?: number }`. `url` must be an https/blob
|
||||||
|
* URL (the host resolves mxc → media URL).
|
||||||
|
*
|
||||||
|
* No effect unless the host sends the action. Returns a teardown function that
|
||||||
|
* also aborts any clip still playing.
|
||||||
|
*/
|
||||||
|
export function startLotusAudioInject(vm: CallViewModel): () => void {
|
||||||
|
const w = widget;
|
||||||
|
if (!w) return () => undefined;
|
||||||
|
|
||||||
|
// Track the set of connected LiveKit rooms to publish into. Drive off the
|
||||||
|
// LOCAL participant's connection(s), not `livekitRoomItems$` — that stream
|
||||||
|
// omits rooms with no remote members, so inject would no-op while you're
|
||||||
|
// alone. Map the connections to their livekit rooms like `lotusDenoise.ts`.
|
||||||
|
let rooms: LivekitRoom[] = [];
|
||||||
|
const sub = vm.allConnections$.subscribe((data) => {
|
||||||
|
rooms = data.getConnections().map((c) => c.livekitRoom);
|
||||||
|
});
|
||||||
|
|
||||||
|
// In-flight clips, so we can abort them on teardown (unmount / vm change /
|
||||||
|
// call leave) instead of leaving audio blasting to peers.
|
||||||
|
const activeClips = new Set<() => void>();
|
||||||
|
|
||||||
|
const handler = (ev: CustomEvent<IWidgetApiRequest>): void => {
|
||||||
|
// Always ack so the transport doesn't hang, but only act when the host has
|
||||||
|
// explicitly opted in: audio-inject publishes under the local user's
|
||||||
|
// identity, so it must not be silently armed for every call.
|
||||||
|
void w.api.transport.reply(ev.detail, {});
|
||||||
|
if (!lotusFlag("lotusAudioInject")) return;
|
||||||
|
const data = ev.detail.data as
|
||||||
|
| { url?: unknown; volume?: unknown }
|
||||||
|
| undefined;
|
||||||
|
const url = typeof data?.url === "string" ? safeMediaUrl(data.url) : null;
|
||||||
|
if (!url) {
|
||||||
|
logger.warn("[lotus] inject_audio: missing/invalid url");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const volume =
|
||||||
|
typeof data?.volume === "number" && data.volume >= 0 && data.volume <= 1
|
||||||
|
? data.volume
|
||||||
|
: 1;
|
||||||
|
void playInjectedClip(url, volume, rooms, activeClips).catch((e) =>
|
||||||
|
logger.warn("[lotus] inject_audio failed", e),
|
||||||
|
);
|
||||||
|
};
|
||||||
|
|
||||||
|
w.lazyActions.on(LotusWidgetActions.InjectAudio, handler);
|
||||||
|
return () => {
|
||||||
|
sub.unsubscribe();
|
||||||
|
w.lazyActions.off(LotusWidgetActions.InjectAudio, handler);
|
||||||
|
// Abort anything still playing.
|
||||||
|
for (const abort of [...activeClips]) abort();
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Only allow fetchable media URLs; never same-origin credentialed GETs etc. */
|
||||||
|
function safeMediaUrl(raw: string): string | null {
|
||||||
|
try {
|
||||||
|
const u = new URL(raw, window.location.href);
|
||||||
|
return u.protocol === "https:" || u.protocol === "blob:" ? u.href : null;
|
||||||
|
} catch {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function playInjectedClip(
|
||||||
|
url: string,
|
||||||
|
volume: number,
|
||||||
|
rooms: LivekitRoom[],
|
||||||
|
activeClips: Set<() => void>,
|
||||||
|
): Promise<void> {
|
||||||
|
if (rooms.length === 0) {
|
||||||
|
logger.warn("[lotus] inject_audio: no connected rooms");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Max ONE clip at a time (replace mode): stop any in-flight or playing clip
|
||||||
|
// before starting a new one, so clips can't overlap or be spammed.
|
||||||
|
for (const abort of [...activeClips]) abort();
|
||||||
|
|
||||||
|
// A second inject action can arrive while THIS one is still awaiting its
|
||||||
|
// fetch/decode/publish — before its real cleanup() exists. Register a
|
||||||
|
// synchronous placeholder abort NOW, BEFORE the first await, so the
|
||||||
|
// replace-mode loop above (run by that later action) cancels this one;
|
||||||
|
// otherwise both clips would sail past their awaits and DOUBLE-PUBLISH. The
|
||||||
|
// placeholder aborts the in-flight fetch and flips `aborted`, which we check
|
||||||
|
// after every await; the real cleanup() replaces it once the track is live.
|
||||||
|
let aborted = false;
|
||||||
|
const controller = new AbortController();
|
||||||
|
const placeholder = (): void => {
|
||||||
|
aborted = true;
|
||||||
|
controller.abort();
|
||||||
|
activeClips.delete(placeholder);
|
||||||
|
};
|
||||||
|
activeClips.add(placeholder);
|
||||||
|
|
||||||
|
let resp: Response;
|
||||||
|
try {
|
||||||
|
resp = await fetch(url, {
|
||||||
|
credentials: "omit",
|
||||||
|
mode: "cors",
|
||||||
|
signal: controller.signal,
|
||||||
|
});
|
||||||
|
} catch (e) {
|
||||||
|
// Superseded by a newer clip mid-fetch — expected, not a failure.
|
||||||
|
if (aborted) return;
|
||||||
|
throw e;
|
||||||
|
}
|
||||||
|
if (aborted) return;
|
||||||
|
if (!resp.ok) throw new Error(`fetch ${url} -> ${resp.status}`);
|
||||||
|
const arrayBuffer = await resp.arrayBuffer();
|
||||||
|
if (aborted) return;
|
||||||
|
|
||||||
|
const ctx = new AudioContext();
|
||||||
|
// The action arrives via host postMessage, not a gesture in this iframe, so
|
||||||
|
// the context may start suspended — resume it or the clip is silent and
|
||||||
|
// `onended` never fires.
|
||||||
|
try {
|
||||||
|
await ctx.resume();
|
||||||
|
} catch {
|
||||||
|
/* best effort */
|
||||||
|
}
|
||||||
|
if (aborted) {
|
||||||
|
void ctx.close();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (ctx.state !== "running")
|
||||||
|
logger.warn(`[lotus] inject_audio: AudioContext is ${ctx.state}`);
|
||||||
|
|
||||||
|
let buffer: AudioBuffer;
|
||||||
|
try {
|
||||||
|
buffer = await ctx.decodeAudioData(arrayBuffer);
|
||||||
|
} catch (e) {
|
||||||
|
void ctx.close();
|
||||||
|
throw e;
|
||||||
|
}
|
||||||
|
if (aborted) {
|
||||||
|
void ctx.close();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const dest = ctx.createMediaStreamDestination();
|
||||||
|
const gain = ctx.createGain();
|
||||||
|
gain.gain.value = volume;
|
||||||
|
const source = ctx.createBufferSource();
|
||||||
|
source.buffer = buffer;
|
||||||
|
source.connect(gain).connect(dest);
|
||||||
|
|
||||||
|
const mst = dest.stream.getAudioTracks()[0];
|
||||||
|
if (!mst) {
|
||||||
|
void ctx.close();
|
||||||
|
throw new Error("no audio track from destination");
|
||||||
|
}
|
||||||
|
|
||||||
|
// Publish (a clone of) the clip track to every connected room.
|
||||||
|
const publications = await Promise.all(
|
||||||
|
rooms.map(async (room) => {
|
||||||
|
const clone = mst.clone();
|
||||||
|
try {
|
||||||
|
const pub = await room.localParticipant.publishTrack(clone, {
|
||||||
|
source: Track.Source.Unknown,
|
||||||
|
name: "lotus-soundboard",
|
||||||
|
dtx: false,
|
||||||
|
red: false,
|
||||||
|
});
|
||||||
|
return { room, pub };
|
||||||
|
} catch (e) {
|
||||||
|
clone.stop(); // don't leak the clone if publish failed
|
||||||
|
logger.warn("[lotus] inject_audio: publish failed", e);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
let cleanedUp = false;
|
||||||
|
const cleanup = (): void => {
|
||||||
|
if (cleanedUp) return;
|
||||||
|
cleanedUp = true;
|
||||||
|
activeClips.delete(cleanup);
|
||||||
|
try {
|
||||||
|
source.stop();
|
||||||
|
} catch {
|
||||||
|
/* already stopped */
|
||||||
|
}
|
||||||
|
for (const entry of publications) {
|
||||||
|
if (entry?.pub.track)
|
||||||
|
void entry.room.localParticipant
|
||||||
|
.unpublishTrack(entry.pub.track, true)
|
||||||
|
.catch(() => undefined);
|
||||||
|
}
|
||||||
|
void ctx.close().catch(() => undefined);
|
||||||
|
};
|
||||||
|
// Swap the synchronous placeholder for the real cleanup: from here an abort
|
||||||
|
// (teardown or a newer clip) must unpublish the LIVE track, not just cancel a
|
||||||
|
// fetch. This delete+add is synchronous (no await), so a newer clip's
|
||||||
|
// replace-mode loop always sees exactly one of {placeholder, cleanup}.
|
||||||
|
activeClips.delete(placeholder);
|
||||||
|
activeClips.add(cleanup);
|
||||||
|
|
||||||
|
// If a newer clip aborted us WHILE we were publishing, tear down now so we
|
||||||
|
// don't leave an orphan track published after it ran its replace-mode loop.
|
||||||
|
if (aborted) {
|
||||||
|
cleanup();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
source.onended = cleanup;
|
||||||
|
// Safety net: clip metadata can lie (NaN/huge duration), so force teardown
|
||||||
|
// after a sane, capped delay.
|
||||||
|
const durationMs = Number.isFinite(buffer.duration)
|
||||||
|
? buffer.duration * 1000 + 500
|
||||||
|
: MAX_CLIP_MS;
|
||||||
|
const guard = setTimeout(cleanup, Math.min(MAX_CLIP_MS, Math.max(0, durationMs)));
|
||||||
|
source.addEventListener("ended", () => clearTimeout(guard));
|
||||||
|
|
||||||
|
source.start();
|
||||||
|
}
|
||||||
@@ -0,0 +1,72 @@
|
|||||||
|
/*
|
||||||
|
Copyright 2026 Lotus Guild
|
||||||
|
|
||||||
|
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||||
|
Please see LICENSE in the repository root for full details.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { combineLatest, of, type Subscription } from "rxjs";
|
||||||
|
import { distinctUntilChanged, map, switchMap, throttleTime } from "rxjs/operators";
|
||||||
|
|
||||||
|
import { type CallViewModel } from "../state/CallViewModel/CallViewModel";
|
||||||
|
import { LotusWidgetActions, lotusFlag, lotusSendToHost } from "./lotusWidget";
|
||||||
|
|
||||||
|
interface ParticipantState {
|
||||||
|
/** EC media id (`${userId}:${deviceId}`), stable per participant device. */
|
||||||
|
id: string;
|
||||||
|
/** Matrix user id this media belongs to. */
|
||||||
|
userId: string;
|
||||||
|
speaking: boolean;
|
||||||
|
audioEnabled: boolean;
|
||||||
|
videoEnabled: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Stream per-participant speaking / mute / camera state to the Lotus host
|
||||||
|
* (cinny) over the widget API, so the host can drive speaking rings, mute
|
||||||
|
* badges and PiP from real events instead of scraping Element Call's rendered
|
||||||
|
* DOM (`useCallSpeakers.ts`).
|
||||||
|
*
|
||||||
|
* Opt-in: does nothing unless the host set `lotusCallState=1` on the widget
|
||||||
|
* URL. Returns a teardown function.
|
||||||
|
*/
|
||||||
|
export function startLotusCallState(vm: CallViewModel): () => void {
|
||||||
|
if (!lotusFlag("lotusCallState")) return () => undefined;
|
||||||
|
|
||||||
|
const sub: Subscription = vm.userMedia$
|
||||||
|
.pipe(
|
||||||
|
switchMap((members) =>
|
||||||
|
members.length === 0
|
||||||
|
? of([] as ParticipantState[])
|
||||||
|
: combineLatest(
|
||||||
|
members.map((m) =>
|
||||||
|
combineLatest([
|
||||||
|
m.speaking$,
|
||||||
|
m.audioEnabled$,
|
||||||
|
m.videoEnabled$,
|
||||||
|
]).pipe(
|
||||||
|
map(
|
||||||
|
([speaking, audioEnabled, videoEnabled]): ParticipantState => ({
|
||||||
|
id: m.id,
|
||||||
|
userId: m.userId,
|
||||||
|
speaking,
|
||||||
|
audioEnabled,
|
||||||
|
videoEnabled,
|
||||||
|
}),
|
||||||
|
),
|
||||||
|
),
|
||||||
|
),
|
||||||
|
),
|
||||||
|
),
|
||||||
|
// `speaking` flips rapidly; cap the send rate and drop no-op repeats.
|
||||||
|
// 250ms is plenty for speaking rings / mute badges and keeps the
|
||||||
|
// request/response widget traffic modest.
|
||||||
|
throttleTime(250, undefined, { leading: true, trailing: true }),
|
||||||
|
distinctUntilChanged((a, b) => JSON.stringify(a) === JSON.stringify(b)),
|
||||||
|
)
|
||||||
|
.subscribe((participants) => {
|
||||||
|
lotusSendToHost(LotusWidgetActions.CallState, { participants });
|
||||||
|
});
|
||||||
|
|
||||||
|
return () => sub.unsubscribe();
|
||||||
|
}
|
||||||
@@ -0,0 +1,120 @@
|
|||||||
|
/*
|
||||||
|
Copyright 2026 Lotus Guild
|
||||||
|
|
||||||
|
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||||
|
Please see LICENSE in the repository root for full details.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import {
|
||||||
|
type RemoteParticipant,
|
||||||
|
type Room as LivekitRoom,
|
||||||
|
RoomEvent,
|
||||||
|
Track,
|
||||||
|
} from "livekit-client";
|
||||||
|
import { logger } from "matrix-js-sdk/lib/logger";
|
||||||
|
import { type IWidgetApiRequest } from "matrix-widget-api";
|
||||||
|
|
||||||
|
import { type CallViewModel } from "../state/CallViewModel/CallViewModel";
|
||||||
|
import { widget } from "../widget";
|
||||||
|
import { LotusWidgetActions } from "./lotusActions";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Handle the host's `io.lotus.set_deafen` toWidget action: silence remote audio
|
||||||
|
* at the LiveKit source, replacing cinny's brittle iframe-DOM `.muted` hack
|
||||||
|
* (which fought MatrixAudioRenderer and broke on re-render / late tracks).
|
||||||
|
*
|
||||||
|
* `deafened` mutes every remote participant's microphone AND screenshare audio;
|
||||||
|
* `screenshareAudioMuted` mutes only the screenshare audio (so the host can
|
||||||
|
* drop shared-tab/game audio while still hearing voices). Volume is set PER
|
||||||
|
* SOURCE via `RemoteParticipant.setVolume(volume, source)` — whose verified
|
||||||
|
* signature in livekit-client ^2.18.1 is
|
||||||
|
* `setVolume(volume, source?: Track.Source.Microphone | Track.Source.ScreenShareAudio)`
|
||||||
|
* and DEFAULTS `source` to `Microphone` (NOT "all audio"), so each source must
|
||||||
|
* be set explicitly. setVolume records the value in the participant's
|
||||||
|
* `volumeMap`, so a track that (re)subscribes later re-applies it automatically.
|
||||||
|
*
|
||||||
|
* State is closure-scoped (per invocation, matching the sibling lotus modules)
|
||||||
|
* and re-applied to every current room's participants on change, and to LATE
|
||||||
|
* JOINERS via `RoomEvent.ParticipantConnected`. The cinny host re-sends the
|
||||||
|
* current state on every call join (CallControl.forceState), so a fresh call
|
||||||
|
* never inherits a previous call's deafen state.
|
||||||
|
*
|
||||||
|
* No effect unless the host sends the action. Returns a teardown function.
|
||||||
|
*/
|
||||||
|
export function startLotusDeafen(vm: CallViewModel): () => void {
|
||||||
|
const w = widget;
|
||||||
|
if (!w) return () => undefined;
|
||||||
|
|
||||||
|
let deafened = false;
|
||||||
|
let screenshareAudioMuted = false;
|
||||||
|
|
||||||
|
const applyToParticipant = (p: RemoteParticipant): void => {
|
||||||
|
p.setVolume(deafened ? 0 : 1, Track.Source.Microphone);
|
||||||
|
p.setVolume(
|
||||||
|
deafened || screenshareAudioMuted ? 0 : 1,
|
||||||
|
Track.Source.ScreenShareAudio,
|
||||||
|
);
|
||||||
|
// NOTE: injected/soundboard audio (published as `Track.Source.Unknown`) is
|
||||||
|
// deliberately NOT silenced here. The verified `setVolume` type signature
|
||||||
|
// only accepts `Track.Source.Microphone | Track.Source.ScreenShareAudio`,
|
||||||
|
// so passing `Unknown` would fail the fork's `tsc` gate and require an
|
||||||
|
// unsafe cast. Soundboard clips are short, host-triggered content the host
|
||||||
|
// already controls at the inject source, so leaving them audible is the
|
||||||
|
// type-clean, safe choice (a full-parity "mute everything" is not needed
|
||||||
|
// for the deafen semantics: don't-hear-other-people's-voices).
|
||||||
|
};
|
||||||
|
|
||||||
|
const applyToRoom = (room: LivekitRoom): void =>
|
||||||
|
room.remoteParticipants.forEach(applyToParticipant);
|
||||||
|
|
||||||
|
// Per-room ParticipantConnected listeners, so LATE JOINERS pick up the
|
||||||
|
// current deafen state the moment they connect.
|
||||||
|
const roomListeners = new Map<LivekitRoom, () => void>();
|
||||||
|
let rooms: LivekitRoom[] = [];
|
||||||
|
|
||||||
|
const sub = vm.livekitRoomItems$.subscribe((items) => {
|
||||||
|
const next = items.map((i) => i.livekitRoom);
|
||||||
|
rooms = next;
|
||||||
|
// Detach listeners for rooms that went away.
|
||||||
|
for (const [room, off] of roomListeners) {
|
||||||
|
if (!next.includes(room)) {
|
||||||
|
off();
|
||||||
|
roomListeners.delete(room);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Attach to new rooms + apply the current state to their participants.
|
||||||
|
for (const room of next) {
|
||||||
|
if (!roomListeners.has(room)) {
|
||||||
|
room.on(RoomEvent.ParticipantConnected, applyToParticipant);
|
||||||
|
roomListeners.set(room, () =>
|
||||||
|
room.off(RoomEvent.ParticipantConnected, applyToParticipant),
|
||||||
|
);
|
||||||
|
applyToRoom(room);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
const handler = (ev: CustomEvent<IWidgetApiRequest>): void => {
|
||||||
|
void w.api.transport.reply(ev.detail, {});
|
||||||
|
const data = ev.detail.data as
|
||||||
|
| { deafened?: boolean; screenshareAudioMuted?: boolean }
|
||||||
|
| undefined;
|
||||||
|
// Missing fields default to their CURRENT value, so a partial payload only
|
||||||
|
// moves the flag it actually names.
|
||||||
|
if (typeof data?.deafened === "boolean") deafened = data.deafened;
|
||||||
|
if (typeof data?.screenshareAudioMuted === "boolean")
|
||||||
|
screenshareAudioMuted = data.screenshareAudioMuted;
|
||||||
|
logger.debug(
|
||||||
|
`[lotus] set_deafen: deafened=${deafened} screenshareAudioMuted=${screenshareAudioMuted}`,
|
||||||
|
);
|
||||||
|
rooms.forEach(applyToRoom);
|
||||||
|
};
|
||||||
|
|
||||||
|
w.lazyActions.on(LotusWidgetActions.SetDeafen, handler);
|
||||||
|
return () => {
|
||||||
|
sub.unsubscribe();
|
||||||
|
for (const off of roomListeners.values()) off();
|
||||||
|
roomListeners.clear();
|
||||||
|
w.lazyActions.off(LotusWidgetActions.SetDeafen, handler);
|
||||||
|
};
|
||||||
|
}
|
||||||
@@ -0,0 +1,106 @@
|
|||||||
|
/*
|
||||||
|
Copyright 2026 Lotus Guild
|
||||||
|
|
||||||
|
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||||
|
Please see LICENSE in the repository root for full details.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { useSyncExternalStore } from "react";
|
||||||
|
import { type IWidgetApiRequest } from "matrix-widget-api";
|
||||||
|
|
||||||
|
import { widget } from "../widget";
|
||||||
|
import { LotusWidgetActions } from "./lotusActions";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Avatar decorations (#6 / A6). The Lotus host (cinny) owns the decoration
|
||||||
|
* roster (MSC4133 profile decorations) and can't draw them on EC's in-call
|
||||||
|
* video tiles from outside the iframe. So it pushes a `userId -> image URL`
|
||||||
|
* map via the `io.lotus.decorations` widget action, and the tile avatar
|
||||||
|
* component renders the overlay natively.
|
||||||
|
*/
|
||||||
|
|
||||||
|
let decorations: Readonly<Record<string, string>> = {};
|
||||||
|
const listeners = new Set<() => void>();
|
||||||
|
|
||||||
|
function emit(): void {
|
||||||
|
for (const l of listeners) l();
|
||||||
|
}
|
||||||
|
|
||||||
|
// Stable module-scope subscribe reference, so `useSyncExternalStore` doesn't
|
||||||
|
// re-subscribe (add/remove the listener) on every render of a tile.
|
||||||
|
function subscribe(cb: () => void): () => void {
|
||||||
|
listeners.add(cb);
|
||||||
|
return () => listeners.delete(cb);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Subscribe a tile avatar to its participant's decoration URL (or undefined). */
|
||||||
|
export function useLotusDecoration(userId: string): string | undefined {
|
||||||
|
return useSyncExternalStore(subscribe, () => decorations[userId]);
|
||||||
|
}
|
||||||
|
|
||||||
|
function safeImageUrl(raw: unknown): string | null {
|
||||||
|
if (typeof raw !== "string") return null;
|
||||||
|
try {
|
||||||
|
const u = new URL(raw, window.location.href);
|
||||||
|
return u.protocol === "https:" || u.protocol === "blob:" ? u.href : null;
|
||||||
|
} catch {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Ref-counted single registration: the decoration roster is app-wide (keyed by
|
||||||
|
// userId), so multiple tile/InCallView mounts must share ONE handler — otherwise
|
||||||
|
// each would reply to the same widget request (double-reply) and a transient
|
||||||
|
// remount would tear it down.
|
||||||
|
let registrations = 0;
|
||||||
|
let unregister: (() => void) | null = null;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Register the `io.lotus.decorations` handler (ref-counted). No effect unless
|
||||||
|
* the host sends the action. Returns a teardown function.
|
||||||
|
*/
|
||||||
|
export function startLotusDecorations(): () => void {
|
||||||
|
const w = widget;
|
||||||
|
if (!w) return () => undefined;
|
||||||
|
|
||||||
|
if (registrations === 0) {
|
||||||
|
const handler = (ev: CustomEvent<IWidgetApiRequest>): void => {
|
||||||
|
void w.api.transport.reply(ev.detail, {});
|
||||||
|
const data = ev.detail.data as
|
||||||
|
| { decorations?: Record<string, unknown> }
|
||||||
|
| undefined;
|
||||||
|
const next: Record<string, string> = {};
|
||||||
|
if (data?.decorations && typeof data.decorations === "object") {
|
||||||
|
// Cap the roster so a pathological map can't spawn unbounded overlays.
|
||||||
|
for (const [userId, url] of Object.entries(data.decorations).slice(
|
||||||
|
0,
|
||||||
|
512,
|
||||||
|
)) {
|
||||||
|
const safe = safeImageUrl(url);
|
||||||
|
if (safe) next[userId] = safe;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
decorations = next;
|
||||||
|
emit();
|
||||||
|
};
|
||||||
|
w.lazyActions.on(LotusWidgetActions.Decorations, handler);
|
||||||
|
unregister = () =>
|
||||||
|
w.lazyActions.off(LotusWidgetActions.Decorations, handler);
|
||||||
|
}
|
||||||
|
registrations += 1;
|
||||||
|
|
||||||
|
return () => {
|
||||||
|
registrations -= 1;
|
||||||
|
if (registrations <= 0) {
|
||||||
|
registrations = 0;
|
||||||
|
unregister?.();
|
||||||
|
unregister = null;
|
||||||
|
// Reset the roster once the last registration goes away, so a decoration
|
||||||
|
// pushed in call A can't leak onto a shared user in call B before the
|
||||||
|
// host re-pushes. Notify listeners so any still-mounted tile drops the
|
||||||
|
// now-stale overlay via the render path.
|
||||||
|
decorations = {};
|
||||||
|
emit();
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}
|
||||||
@@ -0,0 +1,154 @@
|
|||||||
|
/*
|
||||||
|
Copyright 2026 Lotus Guild
|
||||||
|
|
||||||
|
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||||
|
Please see LICENSE in the repository root for full details.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import {
|
||||||
|
type LocalAudioTrack,
|
||||||
|
ParticipantEvent,
|
||||||
|
type Room as LivekitRoom,
|
||||||
|
Track,
|
||||||
|
} from "livekit-client";
|
||||||
|
import { logger } from "matrix-js-sdk/lib/logger";
|
||||||
|
|
||||||
|
import { type CallViewModel } from "../state/CallViewModel/CallViewModel";
|
||||||
|
import { lotusFlag, lotusParam } from "./lotusWidget";
|
||||||
|
import {
|
||||||
|
type LotusDenoiseConfig,
|
||||||
|
LotusDenoiseProcessor,
|
||||||
|
} from "./lotusDenoiseProcessor";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Apply Lotus ML noise suppression to the local mic track as a first-class
|
||||||
|
* Element Call audio TrackProcessor (#1), replacing the host's build-time
|
||||||
|
* `getUserMedia` monkeypatch.
|
||||||
|
*
|
||||||
|
* Opt-in via `lotusDenoiseSource=1` (+ `lotusModel`, `lotusGate`,
|
||||||
|
* `lotusGateThreshold`, `lotusDenoiseBase`). Because the processor is attached
|
||||||
|
* to the mic publication and re-attached on every (re)publish, denoise
|
||||||
|
* survives EC's mid-call reconnect — fixing the A7 "mic dead after reconnect"
|
||||||
|
* bug. Additive: no-op without the flag.
|
||||||
|
*/
|
||||||
|
/**
|
||||||
|
* Resolve the denoise asset base, forcing it to be SAME-ORIGIN. The base is fed
|
||||||
|
* to `audioWorklet.addModule()`, which executes the target as code in the
|
||||||
|
* worklet scope (processing the live mic) — so a cross-origin base from a
|
||||||
|
* crafted call link would be arbitrary code execution. Any non-same-origin or
|
||||||
|
* malformed value falls back to the bundled "./denoise/".
|
||||||
|
*/
|
||||||
|
function safeAssetBase(raw: string | null): string {
|
||||||
|
// Resolve to an ABSOLUTE same-origin href against the document. Absolute is
|
||||||
|
// required because native dynamic `import()` (DTLN/DeepFilterNet) resolves a
|
||||||
|
// relative specifier against the JS chunk's URL, not the document — so a
|
||||||
|
// relative "./denoise/" would 404. addModule()/fetch() work with absolute
|
||||||
|
// too, so this keeps all three asset-load paths consistent.
|
||||||
|
const fallback = new URL("./denoise/", window.location.href).href;
|
||||||
|
if (!raw) return fallback;
|
||||||
|
try {
|
||||||
|
const u = new URL(raw, window.location.href);
|
||||||
|
if (u.origin !== window.location.origin) return fallback;
|
||||||
|
return u.href.endsWith("/") ? u.href : `${u.href}/`;
|
||||||
|
} catch {
|
||||||
|
return fallback;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export function startLotusDenoise(vm: CallViewModel): () => void {
|
||||||
|
// Gate on a DEDICATED flag — NOT the existing `lotusDenoise=ml` that the
|
||||||
|
// host's build-time getUserMedia shim already uses. Otherwise simply shipping
|
||||||
|
// this fork (while the host still injects its shim and still sets
|
||||||
|
// lotusDenoise=ml) would denoise twice. The host opts into the in-source
|
||||||
|
// engine with `lotusDenoiseSource=1` AND stops injecting the shim at the same
|
||||||
|
// time. Default off ⇒ the fork is inert and behaviour is unchanged.
|
||||||
|
if (!lotusFlag("lotusDenoiseSource")) return () => undefined;
|
||||||
|
|
||||||
|
const requested = lotusParam("lotusModel");
|
||||||
|
const model: LotusDenoiseConfig["model"] =
|
||||||
|
requested === "speex" ||
|
||||||
|
requested === "dtln" ||
|
||||||
|
requested === "deepfilternet"
|
||||||
|
? requested
|
||||||
|
: "rnnoise";
|
||||||
|
|
||||||
|
const rawThreshold = lotusParam("lotusGateThreshold");
|
||||||
|
const rawFloor = lotusParam("lotusDenoiseFloor");
|
||||||
|
const config: LotusDenoiseConfig = {
|
||||||
|
model,
|
||||||
|
assetBase: safeAssetBase(lotusParam("lotusDenoiseBase")),
|
||||||
|
gate: lotusFlag("lotusGate"),
|
||||||
|
// Default -45 (matches the reference shim); accept an explicit 0 (don't
|
||||||
|
// coerce it away via `|| default`).
|
||||||
|
gateThreshold:
|
||||||
|
rawThreshold !== null && Number.isFinite(Number(rawThreshold))
|
||||||
|
? Number(rawThreshold)
|
||||||
|
: -45,
|
||||||
|
// Dry/wet attenuation floor. Default 0.15 (~-16 dB) tames the
|
||||||
|
// over-suppression "underwater"/pumping artifact; host can tune via
|
||||||
|
// `lotusDenoiseFloor` (0 = full suppression, no floor).
|
||||||
|
floor:
|
||||||
|
rawFloor !== null && Number.isFinite(Number(rawFloor))
|
||||||
|
? Math.min(0.5, Math.max(0, Number(rawFloor)))
|
||||||
|
: 0.15,
|
||||||
|
};
|
||||||
|
|
||||||
|
const micOf = (room: LivekitRoom): LocalAudioTrack | undefined =>
|
||||||
|
room.localParticipant.getTrackPublication(Track.Source.Microphone)
|
||||||
|
?.track as LocalAudioTrack | undefined;
|
||||||
|
|
||||||
|
const apply = (room: LivekitRoom): void => {
|
||||||
|
const mic = micOf(room);
|
||||||
|
if (mic && !mic.getProcessor()) {
|
||||||
|
void mic
|
||||||
|
.setProcessor(new LotusDenoiseProcessor(config))
|
||||||
|
.catch((e) => logger.warn("[lotus] denoise setProcessor failed", e));
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const roomListeners = new Map<LivekitRoom, () => void>();
|
||||||
|
let rooms: LivekitRoom[] = [];
|
||||||
|
|
||||||
|
// Drive activation off the LOCAL participant's connection(s), not
|
||||||
|
// `livekitRoomItems$` — that stream excludes the local participant and only
|
||||||
|
// surfaces rooms with ≥1 remote member, so it wouldn't denoise you while
|
||||||
|
// you're alone and is a fragile coupling to a remote-render concern.
|
||||||
|
const sub = vm.allConnections$.subscribe((data) => {
|
||||||
|
const next = data.getConnections().map((c) => c.livekitRoom);
|
||||||
|
rooms = next;
|
||||||
|
for (const [room, off] of roomListeners) {
|
||||||
|
if (!next.includes(room)) {
|
||||||
|
off();
|
||||||
|
roomListeners.delete(room);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for (const room of next) {
|
||||||
|
if (!roomListeners.has(room)) {
|
||||||
|
// Re-attach on every (re)publish — this is what makes denoise survive
|
||||||
|
// reconnects (A7), unlike the old getUserMedia patch.
|
||||||
|
const onPublished = (): void => apply(room);
|
||||||
|
room.localParticipant.on(
|
||||||
|
ParticipantEvent.LocalTrackPublished,
|
||||||
|
onPublished,
|
||||||
|
);
|
||||||
|
roomListeners.set(room, () =>
|
||||||
|
room.localParticipant.off(
|
||||||
|
ParticipantEvent.LocalTrackPublished,
|
||||||
|
onPublished,
|
||||||
|
),
|
||||||
|
);
|
||||||
|
apply(room);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
return () => {
|
||||||
|
sub.unsubscribe();
|
||||||
|
for (const off of roomListeners.values()) off();
|
||||||
|
roomListeners.clear();
|
||||||
|
for (const room of rooms) {
|
||||||
|
const mic = micOf(room);
|
||||||
|
if (mic?.getProcessor()) void mic.stopProcessor();
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}
|
||||||
@@ -0,0 +1,400 @@
|
|||||||
|
/*
|
||||||
|
Copyright 2026 Lotus Guild
|
||||||
|
|
||||||
|
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||||
|
Please see LICENSE in the repository root for full details.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import {
|
||||||
|
type AudioProcessorOptions,
|
||||||
|
type Track,
|
||||||
|
type TrackProcessor,
|
||||||
|
} from "livekit-client";
|
||||||
|
import { logger } from "matrix-js-sdk/lib/logger";
|
||||||
|
|
||||||
|
export type LotusDenoiseModel =
|
||||||
|
| "rnnoise"
|
||||||
|
| "speex"
|
||||||
|
| "dtln"
|
||||||
|
| "deepfilternet";
|
||||||
|
|
||||||
|
export interface LotusDenoiseConfig {
|
||||||
|
model: LotusDenoiseModel;
|
||||||
|
/** Base URL the worklet scripts/wasm/ESM are served from (e.g. "./denoise/"). */
|
||||||
|
assetBase: string;
|
||||||
|
gate: boolean;
|
||||||
|
gateThreshold: number;
|
||||||
|
/**
|
||||||
|
* Attenuation floor as a dry/wet mix: the fraction (0..~0.3) of the ORIGINAL
|
||||||
|
* mic blended back under the denoised signal so full suppression never fully
|
||||||
|
* collapses the noise floor — this is what kills the "underwater"/pumping
|
||||||
|
* artifact. 0.15 ≈ a -16 dB floor. 0 = full suppression (no floor).
|
||||||
|
*/
|
||||||
|
floor: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Flat sapphi worklets (RNNoise/Speex): each registers a processor under `name`
|
||||||
|
// when its `script` module is added; we feed it the fetched wasm binary.
|
||||||
|
// ⚠️ CONTRACT: these assets are NOT bundled by the fork build — cinny's
|
||||||
|
// vite.config.js `lotusDenoise()` plugin copies them from
|
||||||
|
// `@sapphi-red/web-noise-suppressor` / `@workadventure/noise-suppression` /
|
||||||
|
// `deepfilternet3-noise-filter` into public/element-call/denoise/. The
|
||||||
|
// worklet/wasm/ESM versions must match what this processor expects. An
|
||||||
|
// integration smoke-check should assert GET .../denoise/rnnoise.wasm == 200.
|
||||||
|
const FLAT: Record<
|
||||||
|
"rnnoise" | "speex",
|
||||||
|
{ name: string; script: string; wasm: string; simdWasm?: string }
|
||||||
|
> = {
|
||||||
|
rnnoise: {
|
||||||
|
name: "@sapphi-red/web-noise-suppressor/rnnoise",
|
||||||
|
script: "rnnoiseWorklet.js",
|
||||||
|
wasm: "rnnoise.wasm",
|
||||||
|
simdWasm: "rnnoise_simd.wasm",
|
||||||
|
},
|
||||||
|
speex: {
|
||||||
|
name: "@sapphi-red/web-noise-suppressor/speex",
|
||||||
|
script: "speexWorklet.js",
|
||||||
|
wasm: "speex.wasm",
|
||||||
|
},
|
||||||
|
};
|
||||||
|
// The sapphi gate worklet registers under "noise-gate" (hyphenated).
|
||||||
|
const GATE = {
|
||||||
|
name: "@sapphi-red/web-noise-suppressor/noise-gate",
|
||||||
|
script: "noiseGateWorklet.js",
|
||||||
|
};
|
||||||
|
|
||||||
|
// DTLN (@workadventure) targets 16kHz and doesn't resample; RNNoise/Speex and
|
||||||
|
// DeepFilterNet are 48kHz fullband. The worklets don't resample, so the whole
|
||||||
|
// graph must run at the model's native rate.
|
||||||
|
const sampleRateFor = (model: LotusDenoiseModel): number =>
|
||||||
|
model === "dtln" ? 16_000 : 48_000;
|
||||||
|
|
||||||
|
// Cache fetched wasm per URL so a reconnect/device-switch doesn't re-download.
|
||||||
|
const wasmCache = new Map<string, Promise<ArrayBuffer>>();
|
||||||
|
async function fetchWasmUncached(url: string): Promise<ArrayBuffer> {
|
||||||
|
const r = await fetch(url);
|
||||||
|
if (!r.ok) throw new Error(`denoise wasm ${url} -> ${r.status}`);
|
||||||
|
return r.arrayBuffer();
|
||||||
|
}
|
||||||
|
async function fetchWasm(url: string): Promise<ArrayBuffer> {
|
||||||
|
let p = wasmCache.get(url);
|
||||||
|
if (!p) {
|
||||||
|
p = fetchWasmUncached(url);
|
||||||
|
// Never cache a REJECTED fetch: a transient failure (e.g. a blip during a
|
||||||
|
// reconnect) must not permanently disable denoise for the whole session.
|
||||||
|
// Evict on failure so the next restart/device-switch retries.
|
||||||
|
void p.catch(() => wasmCache.delete(url));
|
||||||
|
wasmCache.set(url, p);
|
||||||
|
}
|
||||||
|
return p;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Resume an AudioContext, but never block indefinitely. A suspended context can
|
||||||
|
* only resume after a user gesture; the denoise action can arrive via host
|
||||||
|
* postMessage (no gesture), so `resume()` may stay pending forever. Since this
|
||||||
|
* runs inside LiveKit's per-track change lock, a hung resume() would deadlock
|
||||||
|
* every later mute/unmute/device-switch. Race it against a timeout and proceed
|
||||||
|
* either way — the processor's `statechange` watcher resumes it once a gesture
|
||||||
|
* lands, and a still-suspended context degrades to (temporary) silence that the
|
||||||
|
* watcher heals, not a hang.
|
||||||
|
*/
|
||||||
|
async function resumeCtx(ctx: AudioContext, timeoutMs = 3_000): Promise<void> {
|
||||||
|
await Promise.race([
|
||||||
|
ctx.resume().catch(() => undefined),
|
||||||
|
new Promise<void>((resolve) => setTimeout(resolve, timeoutMs)),
|
||||||
|
]);
|
||||||
|
}
|
||||||
|
|
||||||
|
function supportsSimd(): boolean {
|
||||||
|
try {
|
||||||
|
// Minimal SIMD module (v128) — validates only where SIMD is supported.
|
||||||
|
return WebAssembly.validate(
|
||||||
|
new Uint8Array([
|
||||||
|
0, 97, 115, 109, 1, 0, 0, 0, 1, 5, 1, 96, 0, 1, 123, 3, 2, 1, 0, 10,
|
||||||
|
10, 1, 8, 0, 65, 0, 253, 15, 253, 98, 11,
|
||||||
|
]),
|
||||||
|
);
|
||||||
|
} catch {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
interface MlNode {
|
||||||
|
node: AudioNode;
|
||||||
|
dispose?: () => void;
|
||||||
|
}
|
||||||
|
interface Graph {
|
||||||
|
source: MediaStreamAudioSourceNode;
|
||||||
|
nodes: AudioNode[];
|
||||||
|
disposes: (() => void)[];
|
||||||
|
track: MediaStreamTrack;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* A LiveKit audio TrackProcessor that runs Lotus ML noise suppression
|
||||||
|
* (RNNoise / Speex / DTLN / DeepFilterNet) on the local microphone track, as a
|
||||||
|
* first-class stage in Element Call's publish pipeline — replacing the host's
|
||||||
|
* `getUserMedia` monkeypatch.
|
||||||
|
*
|
||||||
|
* Because it's a real LiveKit processor, EC re-applies it on every
|
||||||
|
* (re)publish/restart, so denoise survives EC's mid-call reconnect — the root
|
||||||
|
* cause of A7. It owns a dedicated AudioContext at the model's required sample
|
||||||
|
* rate (LiveKit does NOT pass an audioContext to restart()), reused across
|
||||||
|
* restarts and closed on destroy. restart() never throws and never leaves a
|
||||||
|
* stopped track on the sender: on failure it degrades to the RAW mic track
|
||||||
|
* rather than silence.
|
||||||
|
*/
|
||||||
|
export class LotusDenoiseProcessor
|
||||||
|
implements TrackProcessor<Track.Kind.Audio, AudioProcessorOptions>
|
||||||
|
{
|
||||||
|
public readonly name = "lotus-denoise";
|
||||||
|
public processedTrack?: MediaStreamTrack;
|
||||||
|
|
||||||
|
private ctx?: AudioContext;
|
||||||
|
private graph?: Graph;
|
||||||
|
private ctxStateHandler?: () => void;
|
||||||
|
|
||||||
|
public constructor(private readonly config: LotusDenoiseConfig) {}
|
||||||
|
|
||||||
|
public async init(_opts: AudioProcessorOptions): Promise<void> {
|
||||||
|
try {
|
||||||
|
await this.ensureContext();
|
||||||
|
this.graph = await this.buildGraph(_opts.track);
|
||||||
|
this.processedTrack = this.graph.track;
|
||||||
|
} catch (e) {
|
||||||
|
// Don't orphan the owned context if graph construction fails (browsers
|
||||||
|
// cap live AudioContexts, so repeated failed inits could exhaust them).
|
||||||
|
// The caller degrades to the raw mic; we just release our resources.
|
||||||
|
await this.closeContext();
|
||||||
|
throw e;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public async restart(opts: AudioProcessorOptions): Promise<void> {
|
||||||
|
try {
|
||||||
|
await this.ensureContext();
|
||||||
|
const next = await this.buildGraph(opts.track);
|
||||||
|
this.disposeGraph(this.graph);
|
||||||
|
this.graph = next;
|
||||||
|
this.processedTrack = next.track;
|
||||||
|
} catch (e) {
|
||||||
|
// Never go silent on the A7/device-switch path: fall back to raw audio.
|
||||||
|
logger.warn("[lotus] denoise restart failed; using raw mic", e);
|
||||||
|
this.disposeGraph(this.graph);
|
||||||
|
this.graph = undefined;
|
||||||
|
this.processedTrack = opts.track;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public async destroy(): Promise<void> {
|
||||||
|
this.disposeGraph(this.graph);
|
||||||
|
this.graph = undefined;
|
||||||
|
this.processedTrack = undefined;
|
||||||
|
await this.closeContext();
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Remove the state watcher and close the owned context, if any. */
|
||||||
|
private async closeContext(): Promise<void> {
|
||||||
|
const ctx = this.ctx;
|
||||||
|
if (!ctx) return;
|
||||||
|
if (this.ctxStateHandler) {
|
||||||
|
ctx.removeEventListener("statechange", this.ctxStateHandler);
|
||||||
|
this.ctxStateHandler = undefined;
|
||||||
|
}
|
||||||
|
this.ctx = undefined;
|
||||||
|
if (ctx.state !== "closed") await ctx.close().catch(() => undefined);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Create (once) the model-rate context + register the flat worklet modules. */
|
||||||
|
private async ensureContext(): Promise<void> {
|
||||||
|
const rate = sampleRateFor(this.config.model);
|
||||||
|
if (this.ctx && this.ctx.state !== "closed" && this.ctx.sampleRate === rate) {
|
||||||
|
if (this.ctx.state === "suspended") await resumeCtx(this.ctx);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
await this.closeContext();
|
||||||
|
|
||||||
|
const ctx = new AudioContext({ sampleRate: rate });
|
||||||
|
try {
|
||||||
|
if (ctx.sampleRate !== rate)
|
||||||
|
throw new Error(`denoise: got ${ctx.sampleRate}Hz, need ${rate}Hz`);
|
||||||
|
|
||||||
|
// Auto-resume if the OS/browser suspends the context mid-call (mobile
|
||||||
|
// backgrounding, audio interruption): the dest node otherwise emits
|
||||||
|
// silence with no recovery. Only resume while a graph is live.
|
||||||
|
const onStateChange = (): void => {
|
||||||
|
if (ctx.state === "suspended" && this.graph)
|
||||||
|
void ctx.resume().catch(() => undefined);
|
||||||
|
};
|
||||||
|
ctx.addEventListener("statechange", onStateChange);
|
||||||
|
|
||||||
|
// Flat models register via addModule here; DTLN/DeepFilterNet bring their
|
||||||
|
// own processor via the dynamic-imported helper (see buildMlNode).
|
||||||
|
if (this.config.model === "rnnoise" || this.config.model === "speex")
|
||||||
|
await ctx.audioWorklet.addModule(
|
||||||
|
this.config.assetBase + FLAT[this.config.model].script,
|
||||||
|
);
|
||||||
|
if (this.config.gate)
|
||||||
|
await ctx.audioWorklet.addModule(this.config.assetBase + GATE.script);
|
||||||
|
// The action can arrive via host postMessage, not a gesture in this
|
||||||
|
// iframe, so the context can start suspended — resume without hanging.
|
||||||
|
if (ctx.state === "suspended") await resumeCtx(ctx);
|
||||||
|
|
||||||
|
this.ctx = ctx;
|
||||||
|
this.ctxStateHandler = onStateChange;
|
||||||
|
} catch (e) {
|
||||||
|
// Don't leak a half-initialised context on any failure path.
|
||||||
|
await ctx.close().catch(() => undefined);
|
||||||
|
throw e;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private async buildGraph(track: MediaStreamTrack): Promise<Graph> {
|
||||||
|
const ctx = this.ctx!;
|
||||||
|
const source = ctx.createMediaStreamSource(new MediaStream([track]));
|
||||||
|
const dest = ctx.createMediaStreamDestination();
|
||||||
|
const nodes: AudioNode[] = [];
|
||||||
|
const disposes: (() => void)[] = [];
|
||||||
|
|
||||||
|
try {
|
||||||
|
// Wet (denoised) path: source → ml → [gate] → wetGain.
|
||||||
|
const ml = await this.buildMlNode(ctx);
|
||||||
|
source.connect(ml.node);
|
||||||
|
nodes.push(ml.node);
|
||||||
|
if (ml.dispose) disposes.push(ml.dispose);
|
||||||
|
let wetHead: AudioNode = ml.node;
|
||||||
|
|
||||||
|
// Gate AFTER the ML model, not before: gating the raw noisy signal fed
|
||||||
|
// hard-zeroed frames into the model (discontinuities it must fight) and
|
||||||
|
// made the threshold operate on pre-denoise levels. Gate the residual.
|
||||||
|
if (this.config.gate) {
|
||||||
|
const gate = new AudioWorkletNode(ctx, GATE.name, {
|
||||||
|
processorOptions: {
|
||||||
|
openThreshold: this.config.gateThreshold,
|
||||||
|
closeThreshold: this.config.gateThreshold - 5,
|
||||||
|
holdMs: 150,
|
||||||
|
maxChannels: 1,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
wetHead.connect(gate);
|
||||||
|
wetHead = gate;
|
||||||
|
nodes.push(gate);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Only mix a dry floor for the LOW-LATENCY flat models (RNNoise/Speex).
|
||||||
|
// DTLN/DeepFilterNet add tens of ms of algorithmic latency, so summing an
|
||||||
|
// undelayed dry copy would comb-filter the voice — for those we rely on
|
||||||
|
// the model's own level (e.g. DFN noiseReductionLevel) instead. RNNoise is
|
||||||
|
// also where the "robotic/underwater" reports come from, so this targets it.
|
||||||
|
const lowLatency =
|
||||||
|
this.config.model === "rnnoise" || this.config.model === "speex";
|
||||||
|
const floor = lowLatency
|
||||||
|
? Math.min(0.5, Math.max(0, this.config.floor))
|
||||||
|
: 0;
|
||||||
|
if (floor > 0) {
|
||||||
|
// Dry/wet mix: blend a small amount of the ORIGINAL mic under the
|
||||||
|
// denoised signal so suppression can't fully collapse the noise floor
|
||||||
|
// (kills the "underwater"/pumping artifact). During speech (denoised ≈
|
||||||
|
// original) the two sum back to ~unity; in noise-only gaps the output
|
||||||
|
// floors at `floor` × original instead of digital silence.
|
||||||
|
const wetGain = ctx.createGain();
|
||||||
|
wetGain.gain.value = 1 - floor;
|
||||||
|
wetHead.connect(wetGain);
|
||||||
|
wetGain.connect(dest);
|
||||||
|
nodes.push(wetGain);
|
||||||
|
|
||||||
|
const dryGain = ctx.createGain();
|
||||||
|
dryGain.gain.value = floor;
|
||||||
|
source.connect(dryGain);
|
||||||
|
dryGain.connect(dest);
|
||||||
|
nodes.push(dryGain);
|
||||||
|
} else {
|
||||||
|
wetHead.connect(dest);
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.info(
|
||||||
|
`[lotus] denoise processor active (${this.config.model}, floor=${floor})`,
|
||||||
|
);
|
||||||
|
return { source, nodes, disposes, track: dest.stream.getAudioTracks()[0] };
|
||||||
|
} catch (e) {
|
||||||
|
// A node constructor / model load can throw mid-build; clean up the
|
||||||
|
// partially-built graph so it doesn't leak (init/restart still fall back
|
||||||
|
// to the raw mic on the rejection).
|
||||||
|
this.disposeGraph({
|
||||||
|
source,
|
||||||
|
nodes,
|
||||||
|
disposes,
|
||||||
|
track: dest.stream.getAudioTracks()[0],
|
||||||
|
});
|
||||||
|
throw e;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private async buildMlNode(ctx: AudioContext): Promise<MlNode> {
|
||||||
|
const base = this.config.assetBase;
|
||||||
|
const model = this.config.model;
|
||||||
|
|
||||||
|
if (model === "dtln") {
|
||||||
|
// Self-contained ESM that resolves its own processor + LiteRT wasm +
|
||||||
|
// TFLite models. bypassUntilReady passes raw audio until the model loads.
|
||||||
|
const mod = await import(/* @vite-ignore */ `${base}workadventure/audio-worklet.js`);
|
||||||
|
return (await mod.createNoiseSuppressionAudioWorklet(ctx, {
|
||||||
|
bypassUntilReady: true,
|
||||||
|
})) as MlNode;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (model === "deepfilternet") {
|
||||||
|
const dfnBase = new URL(`${base}deepfilternet`, window.location.href).href;
|
||||||
|
const mod = await import(/* @vite-ignore */ `${base}deepfilternet/index.esm.js`);
|
||||||
|
const core = new mod.DeepFilterNet3Core({
|
||||||
|
sampleRate: 48_000,
|
||||||
|
// 60, not 80: full-strength suppression is the main source of the
|
||||||
|
// "over-processed" character; a lower level keeps voice natural while
|
||||||
|
// the dry/wet floor handles the noise tail.
|
||||||
|
noiseReductionLevel: 60,
|
||||||
|
assetConfig: { cdnUrl: dfnBase },
|
||||||
|
});
|
||||||
|
await core.initialize();
|
||||||
|
const node = (await core.createAudioWorkletNode(ctx)) as AudioNode;
|
||||||
|
return { node, dispose: () => void safeCall(() => core.destroy()) };
|
||||||
|
}
|
||||||
|
|
||||||
|
// Flat sapphi worklet (rnnoise/speex).
|
||||||
|
const flat = FLAT[model];
|
||||||
|
const useSimd = model === "rnnoise" && !!flat.simdWasm && supportsSimd();
|
||||||
|
const wasmFile = useSimd ? flat.simdWasm! : flat.wasm;
|
||||||
|
let wasmBinary: ArrayBuffer;
|
||||||
|
try {
|
||||||
|
wasmBinary = await fetchWasm(base + wasmFile);
|
||||||
|
} catch (e) {
|
||||||
|
if (useSimd) {
|
||||||
|
wasmCache.delete(base + wasmFile);
|
||||||
|
wasmBinary = await fetchWasm(base + flat.wasm); // fall back to non-SIMD
|
||||||
|
} else throw e;
|
||||||
|
}
|
||||||
|
const node = new AudioWorkletNode(ctx, flat.name, {
|
||||||
|
channelCount: 1,
|
||||||
|
numberOfInputs: 1,
|
||||||
|
numberOfOutputs: 1,
|
||||||
|
processorOptions: { maxChannels: 1, wasmBinary },
|
||||||
|
});
|
||||||
|
return { node, dispose: () => void safeCall(() => node.port.postMessage("destroy")) };
|
||||||
|
}
|
||||||
|
|
||||||
|
private disposeGraph(graph: Graph | undefined): void {
|
||||||
|
if (!graph) return;
|
||||||
|
for (const dispose of graph.disposes) safeCall(dispose);
|
||||||
|
for (const node of graph.nodes) safeCall(() => node.disconnect());
|
||||||
|
safeCall(() => graph.source.disconnect());
|
||||||
|
graph.track.stop();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function safeCall(fn: () => void): void {
|
||||||
|
try {
|
||||||
|
fn();
|
||||||
|
} catch {
|
||||||
|
/* ignore */
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,43 @@
|
|||||||
|
/*
|
||||||
|
Copyright 2026 Lotus Guild
|
||||||
|
|
||||||
|
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||||
|
Please see LICENSE in the repository root for full details.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { type IWidgetApiRequest } from "matrix-widget-api";
|
||||||
|
|
||||||
|
import { type CallViewModel } from "../state/CallViewModel/CallViewModel";
|
||||||
|
import { widget } from "../widget";
|
||||||
|
import { LotusWidgetActions } from "./lotusActions";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Handle the host's `io.lotus.focus_participant` toWidget action (#4): pin a
|
||||||
|
* participant to the spotlight by Matrix user id, or clear it with
|
||||||
|
* `{ userId: null }`. This replaces cinny's old DOM `.click()` tile-selector
|
||||||
|
* hack with a real, layout-aware spotlight override.
|
||||||
|
*
|
||||||
|
* No effect unless the host actually sends the action, so registering the
|
||||||
|
* handler whenever we're a widget is safe. Returns a teardown function.
|
||||||
|
*/
|
||||||
|
export function startLotusFocus(vm: CallViewModel): () => void {
|
||||||
|
const w = widget;
|
||||||
|
if (!w) return () => undefined;
|
||||||
|
|
||||||
|
const handler = (ev: CustomEvent<IWidgetApiRequest>): void => {
|
||||||
|
// Always reply so the host transport doesn't time out.
|
||||||
|
void w.api.transport.reply(ev.detail, {});
|
||||||
|
const data = ev.detail.data as { userId?: unknown } | undefined;
|
||||||
|
// Mirror deafen's partial-payload semantics: a payload that OMITS `userId`
|
||||||
|
// must keep the current spotlight, not clear it. Only act when the key is
|
||||||
|
// actually present — an explicit `null` clears, a string pins that user.
|
||||||
|
if (data && "userId" in data) {
|
||||||
|
const userId = typeof data.userId === "string" ? data.userId : null;
|
||||||
|
vm.setManualSpotlight(userId);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
w.lazyActions.on(LotusWidgetActions.FocusParticipant, handler);
|
||||||
|
return () =>
|
||||||
|
w.lazyActions.off(LotusWidgetActions.FocusParticipant, handler);
|
||||||
|
}
|
||||||
@@ -0,0 +1,184 @@
|
|||||||
|
/*
|
||||||
|
Copyright 2026 Lotus Guild
|
||||||
|
|
||||||
|
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||||
|
Please see LICENSE in the repository root for full details.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import {
|
||||||
|
type LocalTrack,
|
||||||
|
ParticipantEvent,
|
||||||
|
type Room as LivekitRoom,
|
||||||
|
Track,
|
||||||
|
} from "livekit-client";
|
||||||
|
import { logger } from "matrix-js-sdk/lib/logger";
|
||||||
|
import { type IWidgetApiRequest } from "matrix-widget-api";
|
||||||
|
|
||||||
|
import { type CallViewModel } from "../state/CallViewModel/CallViewModel";
|
||||||
|
import { widget } from "../widget";
|
||||||
|
import { LotusWidgetActions } from "./lotusActions";
|
||||||
|
|
||||||
|
interface QualitySettings {
|
||||||
|
/** Max audio (mic) bitrate in bits/sec. */
|
||||||
|
audioMaxBitrate?: number;
|
||||||
|
/** Max screenshare video bitrate in bits/sec. */
|
||||||
|
screenshareMaxBitrate?: number;
|
||||||
|
/** Max screenshare framerate in fps. */
|
||||||
|
screenshareMaxFramerate?: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Handle the host's `io.lotus.set_quality` toWidget action (#7): apply
|
||||||
|
* audio/screenshare encoding limits (bitrate, framerate) to the local
|
||||||
|
* published tracks via `RTCRtpSender.setParameters` — no republish needed.
|
||||||
|
* These controls live in EC's module scope and were unreachable from the host
|
||||||
|
* against the prebuilt bundle.
|
||||||
|
*
|
||||||
|
* Settings are sticky and re-applied whenever a matching local track is
|
||||||
|
* (re)published, so they survive mute/unmute and reconnects. The server-side
|
||||||
|
* voice-limit-guard remains the enforcement backstop.
|
||||||
|
*
|
||||||
|
* No effect unless the host sends the action. Returns a teardown function.
|
||||||
|
*/
|
||||||
|
export function startLotusQuality(vm: CallViewModel): () => void {
|
||||||
|
const w = widget;
|
||||||
|
if (!w) return () => undefined;
|
||||||
|
|
||||||
|
const settings: QualitySettings = {};
|
||||||
|
// Per-room LocalTrackPublished listeners, so sticky settings re-apply on
|
||||||
|
// every (re)publish.
|
||||||
|
const roomListeners = new Map<LivekitRoom, () => void>();
|
||||||
|
// Per-room settle re-apply timers, so we can cancel a pending 500ms re-apply
|
||||||
|
// when a room is removed or on teardown — otherwise it would fire against a
|
||||||
|
// torn-down room.
|
||||||
|
const settleTimers = new Map<LivekitRoom, ReturnType<typeof setTimeout>>();
|
||||||
|
let rooms: LivekitRoom[] = [];
|
||||||
|
|
||||||
|
const applyToRoom = (room: LivekitRoom): void => {
|
||||||
|
const lp = room.localParticipant;
|
||||||
|
|
||||||
|
if (settings.audioMaxBitrate !== undefined) {
|
||||||
|
const mic = lp.getTrackPublication(Track.Source.Microphone)?.track as
|
||||||
|
| LocalTrack
|
||||||
|
| undefined;
|
||||||
|
void patchSender(mic?.sender, { maxBitrate: settings.audioMaxBitrate });
|
||||||
|
}
|
||||||
|
|
||||||
|
const ssPatch: Partial<RTCRtpEncodingParameters> = {};
|
||||||
|
if (settings.screenshareMaxBitrate !== undefined)
|
||||||
|
ssPatch.maxBitrate = settings.screenshareMaxBitrate;
|
||||||
|
if (settings.screenshareMaxFramerate !== undefined)
|
||||||
|
ssPatch.maxFramerate = settings.screenshareMaxFramerate;
|
||||||
|
if (Object.keys(ssPatch).length > 0) {
|
||||||
|
const ss = lp.getTrackPublication(Track.Source.ScreenShare)?.track as
|
||||||
|
| LocalTrack
|
||||||
|
| undefined;
|
||||||
|
void patchSender(ss?.sender, ssPatch);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const applyToAll = (): void => rooms.forEach(applyToRoom);
|
||||||
|
|
||||||
|
// Keep the LocalTrackPublished listeners in sync with the connected rooms.
|
||||||
|
// Drive off the LOCAL participant's connection(s), not `livekitRoomItems$` —
|
||||||
|
// that stream omits rooms with no remote members (returns null for isLocal),
|
||||||
|
// so caps wouldn't apply to the local senders while you're alone. Map the
|
||||||
|
// connections to their livekit rooms exactly like `lotusDenoise.ts` does.
|
||||||
|
const sub = vm.allConnections$.subscribe((data) => {
|
||||||
|
const next = data.getConnections().map((c) => c.livekitRoom);
|
||||||
|
rooms = next;
|
||||||
|
// Remove listeners for rooms that went away.
|
||||||
|
for (const [room, off] of roomListeners) {
|
||||||
|
if (!next.includes(room)) {
|
||||||
|
off();
|
||||||
|
roomListeners.delete(room);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Add listeners for new rooms + apply current settings to them.
|
||||||
|
for (const room of next) {
|
||||||
|
if (!roomListeners.has(room)) {
|
||||||
|
// Re-apply on (re)publish AND unmute/track-restart: LiveKit's
|
||||||
|
// refreshSenderEncodings() overwrites maxBitrate/maxFramerate from the
|
||||||
|
// publish presets on replaceTrack (device/source switch, processor
|
||||||
|
// toggle, restart-on-unmute), and those paths don't emit
|
||||||
|
// LocalTrackPublished. The settle re-apply lands after LiveKit's async
|
||||||
|
// recompute so our cap wins.
|
||||||
|
const reapply = (): void => {
|
||||||
|
applyToRoom(room);
|
||||||
|
// Store the settle timer per room and cancel any pending one, so it
|
||||||
|
// can be cleared on removal/teardown and never fires against a
|
||||||
|
// torn-down room.
|
||||||
|
const prev = settleTimers.get(room);
|
||||||
|
if (prev !== undefined) clearTimeout(prev);
|
||||||
|
settleTimers.set(
|
||||||
|
room,
|
||||||
|
setTimeout(() => {
|
||||||
|
settleTimers.delete(room);
|
||||||
|
applyToRoom(room);
|
||||||
|
}, 500),
|
||||||
|
);
|
||||||
|
};
|
||||||
|
const events = [
|
||||||
|
ParticipantEvent.LocalTrackPublished,
|
||||||
|
ParticipantEvent.TrackUnmuted,
|
||||||
|
] as const;
|
||||||
|
for (const e of events) room.localParticipant.on(e, reapply);
|
||||||
|
roomListeners.set(room, () => {
|
||||||
|
for (const e of events) room.localParticipant.off(e, reapply);
|
||||||
|
const t = settleTimers.get(room);
|
||||||
|
if (t !== undefined) clearTimeout(t);
|
||||||
|
settleTimers.delete(room);
|
||||||
|
});
|
||||||
|
applyToRoom(room);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
const handler = (ev: CustomEvent<IWidgetApiRequest>): void => {
|
||||||
|
void w.api.transport.reply(ev.detail, {});
|
||||||
|
const data = ev.detail.data as Record<string, unknown> | undefined;
|
||||||
|
if (!data) return;
|
||||||
|
// Clamp to sane ranges so a typo can't brick the encoder (e.g. a 1 bps mic).
|
||||||
|
const ranges: Record<keyof QualitySettings, [number, number]> = {
|
||||||
|
audioMaxBitrate: [6_000, 510_000],
|
||||||
|
screenshareMaxBitrate: [50_000, 20_000_000],
|
||||||
|
screenshareMaxFramerate: [1, 60],
|
||||||
|
};
|
||||||
|
for (const key of Object.keys(ranges) as (keyof QualitySettings)[]) {
|
||||||
|
const v = data[key];
|
||||||
|
if (v === null) settings[key] = undefined;
|
||||||
|
else if (typeof v === "number" && Number.isFinite(v)) {
|
||||||
|
const [lo, hi] = ranges[key];
|
||||||
|
settings[key] = Math.min(hi, Math.max(lo, v));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
applyToAll();
|
||||||
|
};
|
||||||
|
|
||||||
|
w.lazyActions.on(LotusWidgetActions.SetQuality, handler);
|
||||||
|
return () => {
|
||||||
|
sub.unsubscribe();
|
||||||
|
for (const off of roomListeners.values()) off();
|
||||||
|
roomListeners.clear();
|
||||||
|
w.lazyActions.off(LotusWidgetActions.SetQuality, handler);
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
async function patchSender(
|
||||||
|
sender: RTCRtpSender | undefined,
|
||||||
|
patch: Partial<RTCRtpEncodingParameters>,
|
||||||
|
): Promise<void> {
|
||||||
|
if (!sender) return;
|
||||||
|
try {
|
||||||
|
const params = sender.getParameters();
|
||||||
|
if (!params.encodings || params.encodings.length === 0)
|
||||||
|
params.encodings = [{}];
|
||||||
|
// Apply to EVERY encoding, not just encodings[0]: screenshare publishes
|
||||||
|
// with simulcast (VP8), so encodings[0] is the small layer and the
|
||||||
|
// full-resolution layer — the real bandwidth hog — is a later encoding.
|
||||||
|
for (const enc of params.encodings) Object.assign(enc, patch);
|
||||||
|
await sender.setParameters(params);
|
||||||
|
} catch (e) {
|
||||||
|
logger.warn("[lotus] set_quality: setParameters failed", e);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,98 @@
|
|||||||
|
/*
|
||||||
|
Copyright 2026 Lotus Guild
|
||||||
|
|
||||||
|
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||||
|
Please see LICENSE in the repository root for full details.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { combineLatest, map, type Observable, of, switchMap } from "rxjs";
|
||||||
|
|
||||||
|
import { type UserMediaViewModel } from "../state/media/UserMediaViewModel";
|
||||||
|
import { type ScreenShareViewModel } from "../state/media/ScreenShareViewModel";
|
||||||
|
import { type MediaViewModel } from "../state/media/MediaViewModel";
|
||||||
|
import { type LocalUserMediaViewModel } from "../state/media/LocalUserMediaViewModel";
|
||||||
|
|
||||||
|
interface SpotlightAndPip {
|
||||||
|
spotlight: MediaViewModel[];
|
||||||
|
pip$: Observable<UserMediaViewModel | undefined>;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* [lotus #4] Manual spotlight override.
|
||||||
|
*
|
||||||
|
* Wraps upstream's auto-selected spotlight speaker so a host can pin a specific
|
||||||
|
* participant (via the `io.lotus.focus_participant` widget action). Kept as a
|
||||||
|
* pure function OUTSIDE CallViewModel so CallViewModel stays byte-close to
|
||||||
|
* upstream and rebases cleanly: CallViewModel keeps its original
|
||||||
|
* `spotlightSpeaker$` auto-selection unchanged and just routes the
|
||||||
|
* screenshare/spotlight computation through this wrapper at one call point.
|
||||||
|
*
|
||||||
|
* Behaviour is IDENTICAL to upstream whenever `manualSpotlightUserId$` stays
|
||||||
|
* `null` (the default). When it names a participant that is still present:
|
||||||
|
* - with no screenshare, that participant is spotlighted instead of the
|
||||||
|
* auto-selected active speaker;
|
||||||
|
* - during a screenshare, that participant's camera is surfaced in the
|
||||||
|
* spotlight ALONGSIDE the shared screen (#4 / A5 "focus camera during
|
||||||
|
* screenshare"), and the redundant PiP is hidden.
|
||||||
|
*
|
||||||
|
* @param autoSpotlightSpeaker$ upstream's speaker-follows auto selection
|
||||||
|
* @param manualSpotlightUserId$ host-pinned userId, or `null` for auto (default)
|
||||||
|
* @param screenShares$ current screen-share view models
|
||||||
|
* @param localUserMediaForPip$ local media suitable for the PiP
|
||||||
|
* @param userMedia$ all user media in the call (to resolve the pinned userId)
|
||||||
|
*/
|
||||||
|
export function overrideSpotlight$(
|
||||||
|
autoSpotlightSpeaker$: Observable<UserMediaViewModel | undefined>,
|
||||||
|
manualSpotlightUserId$: Observable<string | null>,
|
||||||
|
screenShares$: Observable<ScreenShareViewModel[]>,
|
||||||
|
localUserMediaForPip$: Observable<LocalUserMediaViewModel | undefined>,
|
||||||
|
userMedia$: Observable<UserMediaViewModel[]>,
|
||||||
|
): Observable<SpotlightAndPip> {
|
||||||
|
// The effective spotlight speaker: the host-pinned participant when set and
|
||||||
|
// still present, otherwise upstream's auto-selected speaker.
|
||||||
|
const spotlightSpeaker$ = combineLatest([
|
||||||
|
autoSpotlightSpeaker$,
|
||||||
|
manualSpotlightUserId$,
|
||||||
|
userMedia$,
|
||||||
|
]).pipe(
|
||||||
|
map(([auto, manualUserId, mediaItems]) => {
|
||||||
|
if (manualUserId !== null) {
|
||||||
|
const pinned = mediaItems.find((m) => m.userId === manualUserId);
|
||||||
|
if (pinned) return pinned;
|
||||||
|
}
|
||||||
|
return auto;
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
return screenShares$.pipe(
|
||||||
|
switchMap((screenShares) => {
|
||||||
|
if (screenShares.length > 0)
|
||||||
|
// During a screenshare, if the host has explicitly pinned a
|
||||||
|
// participant, surface that camera in the spotlight alongside the
|
||||||
|
// shared screen (the whole point of "focus camera during screenshare").
|
||||||
|
// With no manual pin this is unchanged: the screenshare alone is
|
||||||
|
// spotlighted.
|
||||||
|
return combineLatest([manualSpotlightUserId$, userMedia$]).pipe(
|
||||||
|
map(([manualUserId, mediaItems]) => {
|
||||||
|
const pinned =
|
||||||
|
manualUserId !== null
|
||||||
|
? mediaItems.find((m) => m.userId === manualUserId)
|
||||||
|
: undefined;
|
||||||
|
return pinned
|
||||||
|
? { spotlight: [...screenShares, pinned], pip$: of(undefined) }
|
||||||
|
: { spotlight: screenShares, pip$: spotlightSpeaker$ };
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
return spotlightSpeaker$.pipe(
|
||||||
|
map((speaker) => ({
|
||||||
|
spotlight: speaker ? [speaker] : [],
|
||||||
|
// Hide PiP if redundant (i.e. if local user is already in spotlight)
|
||||||
|
pip$: localUserMediaForPip$.pipe(
|
||||||
|
map((m) => (m === speaker ? undefined : m)),
|
||||||
|
),
|
||||||
|
})),
|
||||||
|
);
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
}
|
||||||
@@ -0,0 +1,63 @@
|
|||||||
|
/*
|
||||||
|
Copyright 2026 Lotus Guild
|
||||||
|
|
||||||
|
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||||
|
Please see LICENSE in the repository root for full details.
|
||||||
|
*/
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Shared helpers for the Lotus fork's widget extensions.
|
||||||
|
*
|
||||||
|
* Everything here is **opt-in**: each feature is gated behind a `lotus*` URL
|
||||||
|
* param that the Lotus host (cinny) appends to the widget iframe URL. With no
|
||||||
|
* param present, none of this code changes Element Call's behaviour — which
|
||||||
|
* keeps the fork a minimal, additive, easy-to-rebase diff over upstream.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { logger } from "matrix-js-sdk/lib/logger";
|
||||||
|
|
||||||
|
import { widget } from "../widget";
|
||||||
|
import { LotusWidgetActions } from "./lotusActions";
|
||||||
|
|
||||||
|
export { LotusWidgetActions } from "./lotusActions";
|
||||||
|
|
||||||
|
let cachedParams: URLSearchParams | undefined;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Read a URL param from either the query string or the hash fragment (Element
|
||||||
|
* Call passes widget params via both depending on host), without depending on
|
||||||
|
* EC's own `getUrlParams` parser (keeps the rebase surface small).
|
||||||
|
*/
|
||||||
|
export function lotusParam(name: string): string | null {
|
||||||
|
if (!cachedParams) {
|
||||||
|
// Match EC's own ParamParser precedence: the hash fragment wins over the
|
||||||
|
// query string. So seed from the fragment first, then fill gaps from query.
|
||||||
|
const hash = window.location.hash.replace(/^#\/?/, "");
|
||||||
|
const hashQuery = hash.includes("?") ? hash.slice(hash.indexOf("?") + 1) : "";
|
||||||
|
cachedParams = new URLSearchParams(hashQuery);
|
||||||
|
for (const [k, v] of new URLSearchParams(window.location.search)) {
|
||||||
|
if (!cachedParams.has(k)) cachedParams.append(k, v);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return cachedParams.get(name);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Whether a boolean-ish Lotus feature flag is enabled. */
|
||||||
|
export function lotusFlag(name: string): boolean {
|
||||||
|
const v = lotusParam(name);
|
||||||
|
return v === "1" || v === "true";
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Send a fromWidget message to the Lotus host, swallowing the inevitable
|
||||||
|
* rejection when the host hasn't (yet) registered a handler for it. Returns
|
||||||
|
* true if the widget transport was available to attempt the send.
|
||||||
|
*/
|
||||||
|
export function lotusSendToHost(action: LotusWidgetActions, data: unknown): boolean {
|
||||||
|
const api = widget?.api;
|
||||||
|
if (!api) return false;
|
||||||
|
void api.transport.send(action, data as Record<string, unknown>).catch((e) => {
|
||||||
|
logger.debug(`[lotus] host did not ack ${action}`, e);
|
||||||
|
});
|
||||||
|
return true;
|
||||||
|
}
|
||||||
@@ -29,6 +29,13 @@ import { Header, LeftNav, RightNav, RoomHeaderInfo } from "../Header";
|
|||||||
import { HeaderStyle, useUrlParams } from "../UrlParams";
|
import { HeaderStyle, useUrlParams } from "../UrlParams";
|
||||||
import { useCallViewKeyboardShortcuts } from "../useCallViewKeyboardShortcuts";
|
import { useCallViewKeyboardShortcuts } from "../useCallViewKeyboardShortcuts";
|
||||||
import { widget } from "../widget";
|
import { widget } from "../widget";
|
||||||
|
import { startLotusCallState } from "../lotus/lotusCallState";
|
||||||
|
import { startLotusFocus } from "../lotus/lotusFocus";
|
||||||
|
import { startLotusAudioInject } from "../lotus/lotusAudioInject";
|
||||||
|
import { startLotusQuality } from "../lotus/lotusQuality";
|
||||||
|
import { startLotusDecorations } from "../lotus/lotusDecorations";
|
||||||
|
import { startLotusDenoise } from "../lotus/lotusDenoise";
|
||||||
|
import { startLotusDeafen } from "../lotus/lotusDeafen";
|
||||||
import styles from "./InCallView.module.css";
|
import styles from "./InCallView.module.css";
|
||||||
import { GridTile } from "../tile/GridTile";
|
import { GridTile } from "../tile/GridTile";
|
||||||
import { SettingsModal, defaultSettingsTab } from "../settings/SettingsModal";
|
import { SettingsModal, defaultSettingsTab } from "../settings/SettingsModal";
|
||||||
@@ -279,6 +286,29 @@ export const InCallView: FC<InCallViewProps> = ({
|
|||||||
const earpieceMode = useBehavior(vm.earpieceMode$);
|
const earpieceMode = useBehavior(vm.earpieceMode$);
|
||||||
const audioOutputSwitcher = useBehavior(vm.audioOutputSwitcher$);
|
const audioOutputSwitcher = useBehavior(vm.audioOutputSwitcher$);
|
||||||
|
|
||||||
|
// [lotus] Stream per-participant speaking/mute state to the host (cinny) over
|
||||||
|
// the widget API when opted in via lotusCallState=1. No-op otherwise.
|
||||||
|
useEffect(() => startLotusCallState(vm), [vm]);
|
||||||
|
// [lotus] Handle the host's io.lotus.focus_participant action to pin a
|
||||||
|
// participant to the spotlight (#4). No-op unless the host sends it.
|
||||||
|
useEffect(() => startLotusFocus(vm), [vm]);
|
||||||
|
// [lotus] Handle the host's io.lotus.inject_audio action to mix a soundboard
|
||||||
|
// clip into the call as a separate track (#3). No-op unless the host sends it.
|
||||||
|
useEffect(() => startLotusAudioInject(vm), [vm]);
|
||||||
|
// [lotus] Handle the host's io.lotus.set_quality action to cap audio/
|
||||||
|
// screenshare encoding bitrate/framerate (#7). No-op unless the host sends it.
|
||||||
|
useEffect(() => startLotusQuality(vm), [vm]);
|
||||||
|
// [lotus] Receive per-user avatar-decoration URLs from the host and render
|
||||||
|
// them on in-call tile avatars (#6). No-op unless the host sends them.
|
||||||
|
useEffect(() => startLotusDecorations(), []);
|
||||||
|
// [lotus] Apply ML denoise to the mic as a first-class audio processor that
|
||||||
|
// survives reconnects (#1 / A7). No-op unless lotusDenoiseSource=1.
|
||||||
|
useEffect(() => startLotusDenoise(vm), [vm]);
|
||||||
|
// [lotus] Handle the host's io.lotus.set_deafen action to silence remote
|
||||||
|
// audio (and optionally screenshare audio) at the LiveKit source. No-op
|
||||||
|
// unless the host sends the action.
|
||||||
|
useEffect(() => startLotusDeafen(vm), [vm]);
|
||||||
|
|
||||||
const fatalCallError = useBehavior(vm.fatalError$);
|
const fatalCallError = useBehavior(vm.fatalError$);
|
||||||
// Stop the rendering and throw for the error boundary
|
// Stop the rendering and throw for the error boundary
|
||||||
if (fatalCallError) {
|
if (fatalCallError) {
|
||||||
|
|||||||
@@ -151,6 +151,9 @@ import { type UserMediaViewModel } from "../media/UserMediaViewModel.ts";
|
|||||||
import { type MediaViewModel } from "../media/MediaViewModel.ts";
|
import { type MediaViewModel } from "../media/MediaViewModel.ts";
|
||||||
import { type LocalUserMediaViewModel } from "../media/LocalUserMediaViewModel.ts";
|
import { type LocalUserMediaViewModel } from "../media/LocalUserMediaViewModel.ts";
|
||||||
import { type RemoteUserMediaViewModel } from "../media/RemoteUserMediaViewModel.ts";
|
import { type RemoteUserMediaViewModel } from "../media/RemoteUserMediaViewModel.ts";
|
||||||
|
// [lotus #4] Manual spotlight override, extracted so this file stays byte-close
|
||||||
|
// to upstream (see the file for the behaviour contract).
|
||||||
|
import { overrideSpotlight$ } from "../../lotus/lotusSpotlight";
|
||||||
import {
|
import {
|
||||||
createRingingMedia,
|
createRingingMedia,
|
||||||
type RingingMediaViewModel,
|
type RingingMediaViewModel,
|
||||||
@@ -255,6 +258,11 @@ export interface CallViewModel {
|
|||||||
* Callback to toggle screen sharing. If null, screen sharing is not possible.
|
* Callback to toggle screen sharing. If null, screen sharing is not possible.
|
||||||
*/
|
*/
|
||||||
toggleScreenSharing: (() => void) | null;
|
toggleScreenSharing: (() => void) | null;
|
||||||
|
/**
|
||||||
|
* [lotus] Pin a participant to the spotlight by Matrix user id (#4
|
||||||
|
* focus-participant). Pass null to clear and restore speaker-follows.
|
||||||
|
*/
|
||||||
|
setManualSpotlight: (userId: string | null) => void;
|
||||||
/**
|
/**
|
||||||
* Whether we are sharing our screen.
|
* Whether we are sharing our screen.
|
||||||
*/
|
*/
|
||||||
@@ -295,6 +303,8 @@ export interface CallViewModel {
|
|||||||
* multiple devices.
|
* multiple devices.
|
||||||
*/
|
*/
|
||||||
participantCount$: Behavior<number>;
|
participantCount$: Behavior<number>;
|
||||||
|
/** [lotus] All participants' user media, exposed for the Lotus call-state widget bridge. */
|
||||||
|
userMedia$: Behavior<WrappedUserMediaViewModel[]>;
|
||||||
allConnections$: Behavior<ConnectionManagerData>;
|
allConnections$: Behavior<ConnectionManagerData>;
|
||||||
/** Participants sorted by livekit room so they can be used in the audio rendering */
|
/** Participants sorted by livekit room so they can be used in the audio rendering */
|
||||||
livekitRoomItems$: Behavior<LivekitRoomItem[]>;
|
livekitRoomItems$: Behavior<LivekitRoomItem[]>;
|
||||||
@@ -497,6 +507,7 @@ export function createCallViewModel$(
|
|||||||
options.livekitRoomFactory,
|
options.livekitRoomFactory,
|
||||||
getUrlParams().echoCancellation,
|
getUrlParams().echoCancellation,
|
||||||
getUrlParams().noiseSuppression,
|
getUrlParams().noiseSuppression,
|
||||||
|
getUrlParams().autoGainControl,
|
||||||
);
|
);
|
||||||
|
|
||||||
const connectionManager = createConnectionManager$({
|
const connectionManager = createConnectionManager$({
|
||||||
@@ -931,6 +942,11 @@ export function createCallViewModel$(
|
|||||||
),
|
),
|
||||||
);
|
);
|
||||||
|
|
||||||
|
// [lotus #4] Host-pinned spotlight target (io.lotus.focus_participant). null =
|
||||||
|
// follow the active speaker (upstream default), so this is inert unless the
|
||||||
|
// host pins someone. Consumed by overrideSpotlight$ in spotlightAndPip$.
|
||||||
|
const manualSpotlightUserId$ = new BehaviorSubject<string | null>(null);
|
||||||
|
|
||||||
const grid$ = scope.behavior<UserMediaViewModel[]>(
|
const grid$ = scope.behavior<UserMediaViewModel[]>(
|
||||||
userMedia$.pipe(
|
userMedia$.pipe(
|
||||||
switchMap((mediaItems) => {
|
switchMap((mediaItems) => {
|
||||||
@@ -978,21 +994,17 @@ export function createCallViewModel$(
|
|||||||
if (ringingMedia.length > 0)
|
if (ringingMedia.length > 0)
|
||||||
return of({ spotlight: ringingMedia, pip$: localUserMediaForPip$ });
|
return of({ spotlight: ringingMedia, pip$: localUserMediaForPip$ });
|
||||||
|
|
||||||
return screenShares$.pipe(
|
// [lotus #4] Route the screenshare/spotlight computation through the
|
||||||
switchMap((screenShares) => {
|
// manual-spotlight wrapper. This is byte-for-byte upstream behaviour
|
||||||
if (screenShares.length > 0)
|
// unless the host pins a participant via io.lotus.focus_participant;
|
||||||
return of({ spotlight: screenShares, pip$: spotlightSpeaker$ });
|
// see src/lotus/lotusSpotlight.ts. Kept out-of-line so CallViewModel
|
||||||
|
// stays close to upstream and rebases cleanly.
|
||||||
return spotlightSpeaker$.pipe(
|
return overrideSpotlight$(
|
||||||
map((speaker) => ({
|
spotlightSpeaker$,
|
||||||
spotlight: speaker ? [speaker] : [],
|
manualSpotlightUserId$,
|
||||||
// Hide PiP if redundant (i.e. if local user is already in spotlight)
|
screenShares$,
|
||||||
pip$: localUserMediaForPip$.pipe(
|
localUserMediaForPip$,
|
||||||
map((m) => (m === speaker ? undefined : m)),
|
userMedia$,
|
||||||
),
|
|
||||||
})),
|
|
||||||
);
|
|
||||||
}),
|
|
||||||
);
|
);
|
||||||
}),
|
}),
|
||||||
),
|
),
|
||||||
@@ -1697,6 +1709,8 @@ export function createCallViewModel$(
|
|||||||
join: localMembership.requestJoinAndPublish,
|
join: localMembership.requestJoinAndPublish,
|
||||||
leave: localMembership.requestDisconnect,
|
leave: localMembership.requestDisconnect,
|
||||||
toggleScreenSharing: toggleScreenSharing,
|
toggleScreenSharing: toggleScreenSharing,
|
||||||
|
setManualSpotlight: (userId: string | null): void =>
|
||||||
|
manualSpotlightUserId$.next(userId),
|
||||||
sharingScreen$: sharingScreen$,
|
sharingScreen$: sharingScreen$,
|
||||||
|
|
||||||
tapScreen: (): void => screenTap$.next(),
|
tapScreen: (): void => screenTap$.next(),
|
||||||
@@ -1721,6 +1735,7 @@ export function createCallViewModel$(
|
|||||||
),
|
),
|
||||||
allConnections$,
|
allConnections$,
|
||||||
participantCount$: participantCount$,
|
participantCount$: participantCount$,
|
||||||
|
userMedia$,
|
||||||
handsRaised$: handsRaised$,
|
handsRaised$: handsRaised$,
|
||||||
reactions$: reactions$,
|
reactions$: reactions$,
|
||||||
joinSoundEffect$: joinSoundEffect$,
|
joinSoundEffect$: joinSoundEffect$,
|
||||||
|
|||||||
@@ -55,6 +55,7 @@ export class ECConnectionFactory implements ConnectionFactory {
|
|||||||
* @param livekitRoomFactory - Optional factory function (for testing) to create LivekitRoom instances. If not provided, a default factory is used.
|
* @param livekitRoomFactory - Optional factory function (for testing) to create LivekitRoom instances. If not provided, a default factory is used.
|
||||||
* @param echoCancellation - Whether to enable echo cancellation for audio capture.
|
* @param echoCancellation - Whether to enable echo cancellation for audio capture.
|
||||||
* @param noiseSuppression - Whether to enable noise suppression for audio capture.
|
* @param noiseSuppression - Whether to enable noise suppression for audio capture.
|
||||||
|
* @param autoGainControl - Whether to enable auto gain control for audio capture.
|
||||||
*/
|
*/
|
||||||
public constructor(
|
public constructor(
|
||||||
private client: OpenIDClientParts,
|
private client: OpenIDClientParts,
|
||||||
@@ -66,6 +67,7 @@ export class ECConnectionFactory implements ConnectionFactory {
|
|||||||
livekitRoomFactory?: () => LivekitRoom,
|
livekitRoomFactory?: () => LivekitRoom,
|
||||||
echoCancellation: boolean = true,
|
echoCancellation: boolean = true,
|
||||||
noiseSuppression: boolean = true,
|
noiseSuppression: boolean = true,
|
||||||
|
autoGainControl: boolean = true,
|
||||||
) {
|
) {
|
||||||
const defaultFactory = (): LivekitRoom =>
|
const defaultFactory = (): LivekitRoom =>
|
||||||
new LivekitRoom(
|
new LivekitRoom(
|
||||||
@@ -81,6 +83,7 @@ export class ECConnectionFactory implements ConnectionFactory {
|
|||||||
controlledAudioDevices: this.controlledAudioDevices,
|
controlledAudioDevices: this.controlledAudioDevices,
|
||||||
echoCancellation,
|
echoCancellation,
|
||||||
noiseSuppression,
|
noiseSuppression,
|
||||||
|
autoGainControl,
|
||||||
}),
|
}),
|
||||||
);
|
);
|
||||||
this.livekitRoomFactory = livekitRoomFactory ?? defaultFactory;
|
this.livekitRoomFactory = livekitRoomFactory ?? defaultFactory;
|
||||||
@@ -127,6 +130,7 @@ function generateRoomOption({
|
|||||||
controlledAudioDevices,
|
controlledAudioDevices,
|
||||||
echoCancellation,
|
echoCancellation,
|
||||||
noiseSuppression,
|
noiseSuppression,
|
||||||
|
autoGainControl,
|
||||||
}: {
|
}: {
|
||||||
devices: MediaDevices;
|
devices: MediaDevices;
|
||||||
processorState: ProcessorState;
|
processorState: ProcessorState;
|
||||||
@@ -137,6 +141,7 @@ function generateRoomOption({
|
|||||||
controlledAudioDevices: boolean;
|
controlledAudioDevices: boolean;
|
||||||
echoCancellation: boolean;
|
echoCancellation: boolean;
|
||||||
noiseSuppression: boolean;
|
noiseSuppression: boolean;
|
||||||
|
autoGainControl: boolean;
|
||||||
}): RoomOptions {
|
}): RoomOptions {
|
||||||
return {
|
return {
|
||||||
...defaultLiveKitOptions,
|
...defaultLiveKitOptions,
|
||||||
@@ -150,6 +155,7 @@ function generateRoomOption({
|
|||||||
deviceId: devices.audioInput.selected$.value?.id,
|
deviceId: devices.audioInput.selected$.value?.id,
|
||||||
echoCancellation,
|
echoCancellation,
|
||||||
noiseSuppression,
|
noiseSuppression,
|
||||||
|
autoGainControl,
|
||||||
},
|
},
|
||||||
audioOutput: {
|
audioOutput: {
|
||||||
// When using controlled audio devices, we don't want to set the
|
// When using controlled audio devices, we don't want to set the
|
||||||
|
|||||||
@@ -53,14 +53,13 @@ beforeEach(() => {
|
|||||||
|
|
||||||
describe("ECConnectionFactory - Audio inputs options", () => {
|
describe("ECConnectionFactory - Audio inputs options", () => {
|
||||||
test.each([
|
test.each([
|
||||||
{ echo: true, noise: true },
|
{ echo: true, noise: true, agc: true },
|
||||||
{ echo: true, noise: false },
|
{ echo: true, noise: false, agc: false },
|
||||||
{ echo: false, noise: true },
|
{ echo: false, noise: true, agc: false },
|
||||||
{ echo: false, noise: false },
|
{ echo: false, noise: false, agc: true },
|
||||||
])(
|
])(
|
||||||
"it sets echoCancellation=$echo and noiseSuppression=$noise based on constructor parameters",
|
"it sets echoCancellation=$echo, noiseSuppression=$noise, autoGainControl=$agc based on constructor parameters",
|
||||||
({ echo, noise }) => {
|
({ echo, noise, agc }) => {
|
||||||
// test("it sets echoCancellation and noiseSuppression based on constructor parameters", () => {
|
|
||||||
const RoomConstructor = vi.mocked(LivekitRoom);
|
const RoomConstructor = vi.mocked(LivekitRoom);
|
||||||
|
|
||||||
const ecConnectionFactory = new ECConnectionFactory(
|
const ecConnectionFactory = new ECConnectionFactory(
|
||||||
@@ -76,6 +75,7 @@ describe("ECConnectionFactory - Audio inputs options", () => {
|
|||||||
undefined,
|
undefined,
|
||||||
echo,
|
echo,
|
||||||
noise,
|
noise,
|
||||||
|
agc,
|
||||||
);
|
);
|
||||||
ecConnectionFactory.createConnection(
|
ecConnectionFactory.createConnection(
|
||||||
testScope,
|
testScope,
|
||||||
@@ -90,6 +90,7 @@ describe("ECConnectionFactory - Audio inputs options", () => {
|
|||||||
audioCaptureDefaults: expect.objectContaining({
|
audioCaptureDefaults: expect.objectContaining({
|
||||||
echoCancellation: echo,
|
echoCancellation: echo,
|
||||||
noiseSuppression: noise,
|
noiseSuppression: noise,
|
||||||
|
autoGainControl: agc,
|
||||||
}),
|
}),
|
||||||
}),
|
}),
|
||||||
);
|
);
|
||||||
|
|||||||
@@ -12,7 +12,12 @@ Please see LICENSE in the repository root for full details.
|
|||||||
}
|
}
|
||||||
|
|
||||||
.tabList {
|
.tabList {
|
||||||
overflow-y: auto;
|
/* The tab row is horizontal, so scroll along the inline axis when the tabs
|
||||||
|
exceed the width (e.g. the Settings modal as a phone drawer). The prior
|
||||||
|
overflow-y:auto expressed the wrong axis; make the horizontal intent
|
||||||
|
explicit and hide the scrollbar. */
|
||||||
|
overflow-x: auto;
|
||||||
|
overflow-y: hidden;
|
||||||
max-width: 100%;
|
max-width: 100%;
|
||||||
|
|
||||||
/*no scrollbars*/
|
/*no scrollbars*/
|
||||||
@@ -24,4 +29,5 @@ Please see LICENSE in the repository root for full details.
|
|||||||
/*no scrollbars*/
|
/*no scrollbars*/
|
||||||
background: transparent; /* Chrome/Safari/Webkit */
|
background: transparent; /* Chrome/Safari/Webkit */
|
||||||
width: 0px;
|
width: 0px;
|
||||||
|
height: 0px;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -98,6 +98,14 @@ borders don't support gradients */
|
|||||||
color: var(--cpd-color-icon-primary);
|
color: var(--cpd-color-icon-primary);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* The camera-flip control is always visible and primarily used on phones;
|
||||||
|
its inherited 4px padding yields a ~28px box, below the touch minimum. */
|
||||||
|
@media (hover: none), (pointer: coarse) {
|
||||||
|
.tile .switchCamera {
|
||||||
|
padding: var(--cpd-space-3x);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
@media (hover) {
|
@media (hover) {
|
||||||
.tile .switchCamera:hover {
|
.tile .switchCamera:hover {
|
||||||
background: var(--cpd-color-bg-subtle-secondary);
|
background: var(--cpd-color-bg-subtle-secondary);
|
||||||
|
|||||||
@@ -61,6 +61,24 @@ Please see LICENSE in the repository root for full details.
|
|||||||
mix-blend-mode: multiply;
|
mix-blend-mode: multiply;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* [lotus #6] Profile decoration overlaid on the tile avatar. Shares the
|
||||||
|
avatar's centred box and size so frame-style decorations sit around it. */
|
||||||
|
.lotusDecoration {
|
||||||
|
position: absolute;
|
||||||
|
top: 50%;
|
||||||
|
left: 50%;
|
||||||
|
transform: translate(-50%, -50%);
|
||||||
|
pointer-events: none;
|
||||||
|
object-fit: contain;
|
||||||
|
}
|
||||||
|
|
||||||
|
@container mediaView (width > 0) {
|
||||||
|
.lotusDecoration {
|
||||||
|
inline-size: 50cqmin;
|
||||||
|
block-size: 50cqmin;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/* CSS makes us put a condition here, even though all we want to do is
|
/* CSS makes us put a condition here, even though all we want to do is
|
||||||
unconditionally select the container so we can use cqmin units */
|
unconditionally select the container so we can use cqmin units */
|
||||||
@container mediaView (width > 0) {
|
@container mediaView (width > 0) {
|
||||||
|
|||||||
@@ -22,6 +22,7 @@ import { ErrorSolidIcon } from "@vector-im/compound-design-tokens/assets/web/ico
|
|||||||
|
|
||||||
import styles from "./MediaView.module.css";
|
import styles from "./MediaView.module.css";
|
||||||
import { Avatar } from "../Avatar";
|
import { Avatar } from "../Avatar";
|
||||||
|
import { useLotusDecoration } from "../lotus/lotusDecorations";
|
||||||
import { RaisedHandIndicator } from "../reactions/RaisedHandIndicator";
|
import { RaisedHandIndicator } from "../reactions/RaisedHandIndicator";
|
||||||
import {
|
import {
|
||||||
showConnectionStats as showConnectionStatsSetting,
|
showConnectionStats as showConnectionStatsSetting,
|
||||||
@@ -91,6 +92,7 @@ export const MediaView: FC<Props> = ({
|
|||||||
...props
|
...props
|
||||||
}) => {
|
}) => {
|
||||||
const { t } = useTranslation();
|
const { t } = useTranslation();
|
||||||
|
const decoration = useLotusDecoration(userId);
|
||||||
const [handRaiseTimerVisible] = useSetting(showHandRaisedTimer);
|
const [handRaiseTimerVisible] = useSetting(showHandRaisedTimer);
|
||||||
const [showConnectionStats] = useSetting(showConnectionStatsSetting);
|
const [showConnectionStats] = useSetting(showConnectionStatsSetting);
|
||||||
|
|
||||||
@@ -137,6 +139,17 @@ export const MediaView: FC<Props> = ({
|
|||||||
})}
|
})}
|
||||||
style={{ display: video && videoEnabled ? "none" : "initial" }}
|
style={{ display: video && videoEnabled ? "none" : "initial" }}
|
||||||
/>
|
/>
|
||||||
|
{decoration && !(video && videoEnabled) && (
|
||||||
|
// [lotus #6] Profile decoration overlay, shown only when the avatar
|
||||||
|
// is visible (i.e. not when live video is showing). Pushed by the
|
||||||
|
// host via io.lotus.decorations; undefined unless opted in.
|
||||||
|
<img
|
||||||
|
className={styles.lotusDecoration}
|
||||||
|
src={decoration}
|
||||||
|
alt=""
|
||||||
|
aria-hidden
|
||||||
|
/>
|
||||||
|
)}
|
||||||
{video?.publication !== undefined && (
|
{video?.publication !== undefined && (
|
||||||
<VideoTrack
|
<VideoTrack
|
||||||
trackRef={video}
|
trackRef={video}
|
||||||
|
|||||||
@@ -178,7 +178,13 @@ export function useAudioContext<S extends string>(
|
|||||||
if (
|
if (
|
||||||
audioContext &&
|
audioContext &&
|
||||||
"setSinkId" in audioContext &&
|
"setSinkId" in audioContext &&
|
||||||
!controlledAudioDevices
|
!controlledAudioDevices &&
|
||||||
|
// Skip until a device is actually selected. audioOutputId is undefined
|
||||||
|
// before MediaDevices resolves (e.g. on the Tauri desktop webview, where
|
||||||
|
// the selected$ observable emits undefined first); setSinkId(undefined)
|
||||||
|
// throws "The provided value is not of type 'AudioSinkOptions'". The
|
||||||
|
// default device is represented by the empty string, which is still valid.
|
||||||
|
typeof audioOutputId === "string"
|
||||||
) {
|
) {
|
||||||
// https://developer.mozilla.org/en-US/docs/Web/API/AudioContext/setSinkId
|
// https://developer.mozilla.org/en-US/docs/Web/API/AudioContext/setSinkId
|
||||||
// @ts-expect-error - setSinkId doesn't exist yet in types, maybe because it's not supported everywhere.
|
// @ts-expect-error - setSinkId doesn't exist yet in types, maybe because it's not supported everywhere.
|
||||||
|
|||||||
@@ -11,6 +11,7 @@ import { type IThemeChangeActionRequest } from "matrix-widget-api";
|
|||||||
|
|
||||||
import { getUrlParams } from "./UrlParams";
|
import { getUrlParams } from "./UrlParams";
|
||||||
import { widget } from "./widget";
|
import { widget } from "./widget";
|
||||||
|
import { lotusFlag } from "./lotus/lotusWidget";
|
||||||
|
|
||||||
export const useTheme = (): void => {
|
export const useTheme = (): void => {
|
||||||
const [requestedTheme, setRequestedTheme] = useState(
|
const [requestedTheme, setRequestedTheme] = useState(
|
||||||
@@ -57,5 +58,10 @@ export const useTheme = (): void => {
|
|||||||
previousTheme.current = themeString;
|
previousTheme.current = themeString;
|
||||||
}
|
}
|
||||||
document.body.classList.remove("no-theme");
|
document.body.classList.remove("no-theme");
|
||||||
|
// [lotus #5] Native theming hooks, opted in by the host via URL flags, so
|
||||||
|
// it no longer has to inject CSS into the iframe after load.
|
||||||
|
if (lotusFlag("lotusTransparent"))
|
||||||
|
document.body.classList.add("lotus-transparent");
|
||||||
|
if (lotusFlag("lotusTheme")) document.body.classList.add("lotus-theme");
|
||||||
}, [previousTheme, requestedTheme]);
|
}, [previousTheme, requestedTheme]);
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -22,6 +22,7 @@ import { LazyEventEmitter } from "./LazyEventEmitter";
|
|||||||
import { getUrlParams } from "./UrlParams";
|
import { getUrlParams } from "./UrlParams";
|
||||||
import { Config } from "./config/Config";
|
import { Config } from "./config/Config";
|
||||||
import { ElementCallReactionEventType } from "./reactions";
|
import { ElementCallReactionEventType } from "./reactions";
|
||||||
|
import { LOTUS_TO_WIDGET_ACTIONS } from "./lotus/lotusActions";
|
||||||
|
|
||||||
// Subset of the actions in element-web
|
// Subset of the actions in element-web
|
||||||
export enum ElementWidgetActions {
|
export enum ElementWidgetActions {
|
||||||
@@ -103,6 +104,8 @@ export const initializeWidget = (
|
|||||||
ElementWidgetActions.JoinCall,
|
ElementWidgetActions.JoinCall,
|
||||||
ElementWidgetActions.HangupCall,
|
ElementWidgetActions.HangupCall,
|
||||||
ElementWidgetActions.DeviceMute,
|
ElementWidgetActions.DeviceMute,
|
||||||
|
// [lotus] custom toWidget actions handled by the fork (focus, audio-inject)
|
||||||
|
...LOTUS_TO_WIDGET_ACTIONS,
|
||||||
].forEach((action) => {
|
].forEach((action) => {
|
||||||
api.on(`action:${action}`, (ev: CustomEvent<IWidgetApiRequest>) => {
|
api.on(`action:${action}`, (ev: CustomEvent<IWidgetApiRequest>) => {
|
||||||
ev.preventDefault();
|
ev.preventDefault();
|
||||||
|
|||||||
Reference in New Issue
Block a user