Compare commits

...
Author SHA1 Message Date
Lotus CIandClaude Opus 5.5 dfb72bd9a5 chore(lotus): 0.25.0-lotus.19
CI / Build embedded bundle (push) Successful in 3m27s
CI / Publish to Gitea npm registry (push) Successful in 1m16s
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01PPmy3tPq869XDW4njjVaKA
2026-09-26 13:59:34 -04:00
Lotus CIandClaude Opus 5.5 e0eecc5271 feat(lotus): report host call hotkeys from inside the frame (cinny #43)
The host used to add PTT/deafen key listeners to this frame's window,
which needs same-origin access. Now it sends `io.lotus.set_hotkeys
{ codes }` and the fork reports those keys as `io.lotus.hotkey`
(keydown/keyup with modifiers + whether the target is a text field or a
button), plus window focus changes so a held PTT is released. The
default action is cancelled here with the host's rules (not while typing,
not on a focused button). `controls_state` carries `hotkeys: true`.

Fixes a live bug: EC's own shortcuts (M = toggle mic, Space = PTT) also
fired for the host's keys, so with focus in the call and the default
deafen key M, the first press turned the MIC ON instead of deafening
(even in push-to-talk mode). EC's shortcuts now ignore keys the host owns.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01PPmy3tPq869XDW4njjVaKA
2026-09-26 13:59:34 -04:00
Lotus CIandClaude Opus 5.5 c4f6193888 chore(lotus): 0.25.0-lotus.18
CI / Build embedded bundle (push) Successful in 3m9s
CI / Publish to Gitea npm registry (push) Successful in 1m17s
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01PPmy3tPq869XDW4njjVaKA
2026-09-26 13:39:18 -04:00
Lotus CIandClaude Opus 5.5 7855a0c22f feat(lotus): hide the footer + root color-scheme natively (cinny #43)
- `lotusHostControls=1`: body.lotus-host-controls hides EC's footer
  (position absolute + visibility hidden, still in the DOM so a host on an
  engine without Capability Delegation can click its screenshare button).
- With lotusTheme/lotusTransparent, `:root` color-scheme follows the
  theme, so the transparent frame gets no opaque backdrop.
Together these replace the CSS the host injected into this frame.

Also fixes unreadable tile name tags in the light theme: the Lotus
near-black canvas override now applies to dark themes only (it sat behind
light-theme dark text).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01PPmy3tPq869XDW4njjVaKA
2026-09-26 13:39:18 -04:00
Lotus CIandClaude Opus 5.5 aeb72c22b9 chore(lotus): 0.25.0-lotus.17
CI / Build embedded bundle (push) Successful in 2m45s
CI / Publish to Gitea npm registry (push) Successful in 1m17s
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01PPmy3tPq869XDW4njjVaKA
2026-09-26 12:49:58 -04:00
Lotus CIandClaude Opus 5.5 6fa34911ef feat(lotus): io.lotus.set_screenshare for the host call bar (cinny #43)
toWidget `io.lotus.set_screenshare { on?: boolean }` (omit to toggle)
starts/stops sharing via the view model instead of the host clicking EC's
hidden button. The host sends it with Capability Delegation
(`delegate: "display-capture"`) where supported, so getDisplayMedia keeps
the user's click on engines that require it. `controls_state` now carries
`screenshareAction: true` so the host only uses it on a fork that has it.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01PPmy3tPq869XDW4njjVaKA
2026-09-26 12:49:58 -04:00
Lotus CIandClaude Opus 5.5 35bc7a1e98 chore(lotus): 0.25.0-lotus.16
CI / Build embedded bundle (push) Successful in 2m4s
CI / Publish to Gitea npm registry (push) Successful in 56s
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01PPmy3tPq869XDW4njjVaKA
2026-09-25 12:08:57 -04:00
Lotus CIandClaude Opus 5.5 cee6bab9a6 feat(lotus): report the local mic level to the host (cinny #146)
The host had only a boolean `speaking` for yourself, throttled to 500 ms, so
nothing on the call bar showed that the mic was actually picking you up.

- lotusMicLevel.ts: one shared local mic sampler (a clone of the published
  track, ~10 Hz RMS, while a mic is published, muted or not), shared per
  call view model so there is one AudioContext.
- MicLevelQuantizer: 0–3 bars (≈ −46/−36/−26 dBFS), rising at once and
  falling one bar per sample so it doesn't flicker between words.
- fromWidget io.lotus.mic_level { bars } only when the value changes; 0 while
  muted or with no mic. Opt-in with the host state stream (lotusCallState=1).
- "Talking while muted" (#37) now reads the same sampler instead of running
  its own; same gate and behaviour.

Verified in a local call with a fake-tone mic: the host receives 0–3 as the
tone rises/falls (about 4 messages/s while it changes), 0 on mute and levels
again on unmute; muted + tone still raises speakingWhileMuted and the host's
"You're muted" notice. Lotus/room suites pass on Node 22 (148 tests).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01PPmy3tPq869XDW4njjVaKA
2026-09-25 12:08:57 -04:00
14 changed files with 575 additions and 97 deletions
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lotusguild/element-call-embedded",
"version": "0.25.0-lotus.15",
"version": "0.25.0-lotus.19",
"files": [
"README.md",
"LICENSE-AGPL-3.0",
+14 -1
View File
@@ -90,16 +90,29 @@ body.lotus-transparent[data-background="gradient"]::before {
display: none !important;
}
/* [cinny #43] lotusHostControls=1: the host renders its own call bar, so EC's
footer is hidden but stays in the DOM and in layout-independent position. */
body.lotus-host-controls [data-testid="footer-container"] {
position: absolute !important;
visibility: hidden !important;
}
/* [lotus] Native Lotus/TDS theme, applied when lotusTheme=1, instead of the
host injecting CSS into the iframe after load. Overrides Compound design tokens
with Lotus values at the source so theming is complete and flash-free. Extend
this block with the full Lotus token map from the design system; the canvas
override below is a safe starting point that matches the Lotus dark surface. */
body.lotus-theme {
--cpd-color-bg-canvas-default: #0c0d10;
--video-tile-background: var(--cpd-color-bg-subtle-secondary);
}
/* Dark themes only: in the light theme this near-black canvas sat behind
light-theme (dark) text, e.g. the tile name tags were unreadable. */
body.lotus-theme.cpd-theme-dark,
body.lotus-theme.cpd-theme-dark-hc {
--cpd-color-bg-canvas-default: #0c0d10;
}
/* [lotus #21] Subtle contrast guard for elements that sit directly on the
transparent canvas (no opaque tile background behind them) when
lotusTransparent is set: the host's real wallpaper is unknown to us, so a
+4
View File
@@ -30,6 +30,8 @@ describe("LotusWidgetActions", () => {
LotusWidgetActions.SetLayout,
LotusWidgetActions.OpenSettings,
LotusWidgetActions.ToggleReactions,
LotusWidgetActions.SetScreenshare,
LotusWidgetActions.SetHotkeys,
];
expect(new Set(LOTUS_TO_WIDGET_ACTIONS)).toEqual(new Set(expectedToWidget));
@@ -50,5 +52,7 @@ describe("LotusWidgetActions", () => {
expect(LOTUS_TO_WIDGET_ACTIONS).not.toContain(
LotusWidgetActions.ControlsState,
);
expect(LOTUS_TO_WIDGET_ACTIONS).not.toContain(LotusWidgetActions.MicLevel);
expect(LOTUS_TO_WIDGET_ACTIONS).not.toContain(LotusWidgetActions.Hotkey);
});
});
+26 -1
View File
@@ -75,11 +75,34 @@ export enum LotusWidgetActions {
/** toWidget: toggle the reactions / raise-hand menu (cinny #43). */
ToggleReactions = "io.lotus.toggle_reactions",
/**
* fromWidget: `{ screensharing: boolean, layout: "grid" | "spotlight" | null }`
* toWidget: start/stop sharing `{ on?: boolean }` (omit to toggle; cinny #43).
* Starting calls getDisplayMedia, which needs the user's click: the host
* sends this with Capability Delegation (`delegate: "display-capture"`), and
* without it the browser rejects the share.
*/
SetScreenshare = "io.lotus.set_screenshare",
/**
* toWidget: `{ codes: string[] }` — KeyboardEvent.code values (PTT, deafen)
* to report while focus is inside this frame (cinny #43). `[]` stops.
*/
SetHotkeys = "io.lotus.set_hotkeys",
/**
* fromWidget: a watched key went down/up here, or this window's focus
* changed; see `LotusHotkeyReport` (cinny #43).
*/
Hotkey = "io.lotus.hotkey",
/**
* fromWidget: `{ screensharing: boolean, layout: "grid" | "spotlight" | null,
* screenshareAction: true, hotkeys: true }`
* whenever either changes (cinny #43), so the host stops reading EC's DOM for
* them. Its arrival also tells the host this fork supports the actions above.
*/
ControlsState = "io.lotus.controls_state",
/**
* fromWidget: local mic level `{ bars: 0 | 1 | 2 | 3 }` (cinny #146), sent
* only when it changes (≤ 10 Hz); 0 while muted or with no mic.
*/
MicLevel = "io.lotus.mic_level",
}
/** toWidget Lotus actions that must be allow-listed in `initializeWidget`. */
@@ -93,4 +116,6 @@ export const LOTUS_TO_WIDGET_ACTIONS: LotusWidgetActions[] = [
LotusWidgetActions.SetLayout,
LotusWidgetActions.OpenSettings,
LotusWidgetActions.ToggleReactions,
LotusWidgetActions.SetScreenshare,
LotusWidgetActions.SetHotkeys,
];
+19 -1
View File
@@ -7,7 +7,11 @@ Please see LICENSE in the repository root for full details.
import { describe, expect, it } from "vitest";
import { parseLayoutPayload, parseSettingsPayload } from "./lotusControls";
import {
parseLayoutPayload,
parseSettingsPayload,
shouldToggleScreenshare,
} from "./lotusControls";
describe("parseLayoutPayload", () => {
it("accepts grid and spotlight", () => {
@@ -33,3 +37,17 @@ describe("parseSettingsPayload", () => {
expect(parseSettingsPayload({ open: "yes" }, true)).toBe(false);
});
});
describe("shouldToggleScreenshare", () => {
it("toggles only when an explicit on differs from the current state", () => {
expect(shouldToggleScreenshare({ on: true }, false)).toBe(true);
expect(shouldToggleScreenshare({ on: true }, true)).toBe(false);
expect(shouldToggleScreenshare({ on: false }, true)).toBe(true);
expect(shouldToggleScreenshare({ on: false }, false)).toBe(false);
});
it("toggles when on is missing or not a boolean", () => {
expect(shouldToggleScreenshare({}, false)).toBe(true);
expect(shouldToggleScreenshare({ on: "yes" }, true)).toBe(true);
expect(shouldToggleScreenshare(null, true)).toBe(true);
});
});
+34 -3
View File
@@ -21,6 +21,10 @@ export interface LotusControlsState {
screensharing: boolean;
/** Null while EC offers no layout switch (e.g. PiP or a 1:1 layout). */
layout: LayoutMode | null;
/** This fork handles `io.lotus.set_screenshare` (always true). */
screenshareAction: true;
/** This fork handles `io.lotus.set_hotkeys` (always true). */
hotkeys: true;
}
/** `{ layout }` payload → a layout mode, or undefined if invalid. Exported for tests. */
@@ -39,13 +43,28 @@ export function parseSettingsPayload(data: unknown, current: boolean): boolean {
return !current;
}
/**
* `{ on? }` payload + current sharing state → whether to call the toggle.
* Exported for tests.
*/
export function shouldToggleScreenshare(
data: unknown,
sharing: boolean,
): boolean {
if (typeof data === "object" && data !== null && "on" in data) {
const { on } = data as { on?: unknown };
if (typeof on === "boolean") return on !== sharing;
}
return true;
}
/**
* [cinny #43] Widget-API replacements for the host's DOM access to EC's
* controls: layout switch, settings modal and reactions menu, plus a
* `controls_state` report (screensharing + layout) so the host no longer reads
* EC's DOM for them. Screensharing itself stays host-DOM driven for now:
* `getDisplayMedia` needs the user's click to reach this frame (Capability
* Delegation), which a plain widget message doesn't carry.
* EC's DOM for them. Screenshare start/stop too: `getDisplayMedia` needs the
* user's click in this frame, so the host sends `set_screenshare` with
* Capability Delegation (Chromium). Elsewhere it keeps clicking EC's button.
*
* No effect unless the host sends the actions; registering is safe whenever
* we're a widget. Returns a teardown function.
@@ -69,10 +88,19 @@ export function startLotusControls(vm: CallViewModel): () => void {
w.api.transport.reply(ev.detail, {});
window.dispatchEvent(new Event(LOTUS_TOGGLE_REACTIONS_EVENT));
};
const onSetScreenshare = (ev: CustomEvent<IWidgetApiRequest>): void => {
w.api.transport.reply(ev.detail, {});
// Call synchronously: the delegated activation is only good for a few
// seconds after the message arrived.
if (shouldToggleScreenshare(ev.detail.data, vm.sharingScreen$.value)) {
vm.toggleScreenSharing?.();
}
};
w.lazyActions.on(LotusWidgetActions.SetLayout, onSetLayout);
w.lazyActions.on(LotusWidgetActions.OpenSettings, onOpenSettings);
w.lazyActions.on(LotusWidgetActions.ToggleReactions, onToggleReactions);
w.lazyActions.on(LotusWidgetActions.SetScreenshare, onSetScreenshare);
const sub: Subscription = combineLatest([
vm.sharingScreen$,
@@ -85,6 +113,8 @@ export function startLotusControls(vm: CallViewModel): () => void {
([screensharing, layout]): LotusControlsState => ({
screensharing,
layout,
screenshareAction: true,
hotkeys: true,
}),
),
distinctUntilChanged(
@@ -100,5 +130,6 @@ export function startLotusControls(vm: CallViewModel): () => void {
w.lazyActions.off(LotusWidgetActions.SetLayout, onSetLayout);
w.lazyActions.off(LotusWidgetActions.OpenSettings, onOpenSettings);
w.lazyActions.off(LotusWidgetActions.ToggleReactions, onToggleReactions);
w.lazyActions.off(LotusWidgetActions.SetScreenshare, onSetScreenshare);
};
}
+50
View File
@@ -0,0 +1,50 @@
/*
Copyright 2026 Lotus Guild
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { describe, expect, it } from "vitest";
import {
isEditableTarget,
isInteractiveTarget,
parseHotkeyCodes,
} from "./lotusHotkeys";
describe("parseHotkeyCodes", () => {
it("keeps KeyboardEvent.code-like strings", () => {
expect(parseHotkeyCodes({ codes: ["Space", "KeyM", "F13"] })).toEqual([
"Space",
"KeyM",
"F13",
]);
});
it("drops junk and caps the list", () => {
expect(parseHotkeyCodes({ codes: ["Space", 3, "", "a b", "<x>"] })).toEqual(
["Space"],
);
expect(parseHotkeyCodes({ codes: Array(20).fill("KeyA") })).toHaveLength(8);
expect(parseHotkeyCodes({})).toEqual([]);
expect(parseHotkeyCodes(null)).toEqual([]);
});
});
describe("target checks", () => {
it("treats inputs and contenteditable as editable", () => {
document.body.innerHTML =
'<input id="i"><div contenteditable="true"><span id="s">x</span></div><div id="d"></div>';
expect(isEditableTarget(document.getElementById("i"))).toBe(true);
expect(isEditableTarget(document.getElementById("s"))).toBe(true);
expect(isEditableTarget(document.getElementById("d"))).toBe(false);
expect(isEditableTarget(null)).toBe(false);
});
it("treats buttons, links and button roles as interactive", () => {
document.body.innerHTML =
'<button id="b">b</button><div role="menuitem"><span id="m">m</span></div><p id="p"></p>';
expect(isInteractiveTarget(document.getElementById("b"))).toBe(true);
expect(isInteractiveTarget(document.getElementById("m"))).toBe(true);
expect(isInteractiveTarget(document.getElementById("p"))).toBe(false);
});
});
+159
View File
@@ -0,0 +1,159 @@
/*
Copyright 2026 Lotus Guild
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { type IWidgetApiRequest } from "matrix-widget-api";
import { widget } from "../widget";
import { LotusWidgetActions, lotusSendToHost } from "./lotusWidget";
/** fromWidget `io.lotus.hotkey` payload. */
export type LotusHotkeyReport =
| {
type: "keydown" | "keyup";
code: string;
repeat: boolean;
ctrlKey: boolean;
altKey: boolean;
metaKey: boolean;
shiftKey: boolean;
/** The key went to a text field (typing, not a hotkey). */
editable: boolean;
/** The key went to a button/link/etc. (Space activates it). */
interactive: boolean;
}
/** This window gained or lost focus: the host releases a held PTT. */
| { type: "focus" };
const MAX_CODES = 8;
// Codes the host currently owns. Module-level so EC's own shortcuts
// (useCallViewKeyboardShortcuts) can stand down for them.
let hostCodes = new Set<string>();
/**
* Whether `code` is one of the host's call hotkeys (its PTT or deafen key).
* EC's built-in shortcuts ignore such keys: with the defaults, M would
* otherwise both toggle EC's mic and the host's deafen, and Space would open
* EC's own push-to-talk alongside the host's.
*/
export function isLotusHostHotkey(code: string): boolean {
return hostCodes.has(code);
}
/** `{ codes }` payload → the KeyboardEvent.code values to watch. Exported for tests. */
export function parseHotkeyCodes(data: unknown): string[] {
if (typeof data !== "object" || data === null) return [];
const { codes } = data as { codes?: unknown };
if (!Array.isArray(codes)) return [];
return codes
.filter(
(c): c is string =>
typeof c === "string" && /^[A-Za-z0-9]{1,32}$/.test(c),
)
.slice(0, MAX_CODES);
}
/** Same rules as the host's useCallHotkeys. Exported for tests. */
export function isEditableTarget(el: Element | null): boolean {
if (!(el instanceof HTMLElement)) return false;
const tag = el.tagName;
if (tag === "INPUT" || tag === "TEXTAREA" || tag === "SELECT") return true;
let node: HTMLElement | null = el;
while (node && node !== el.ownerDocument.body) {
const ce = node.getAttribute("contenteditable");
if (ce === "" || ce === "true" || ce === "plaintext-only") return true;
if (ce === "false") return false;
node = node.parentElement;
}
return false;
}
/** Exported for tests. */
export function isInteractiveTarget(el: Element | null): boolean {
if (!(el instanceof HTMLElement)) return false;
const tag = el.tagName;
if (tag === "BUTTON" || tag === "A" || tag === "SELECT") return true;
let node: HTMLElement | null = el;
while (node && node !== el.ownerDocument.body) {
const role = node.getAttribute("role");
if (
role === "button" ||
role === "link" ||
role === "menuitem" ||
role === "tab"
)
return true;
node = node.parentElement;
}
return false;
}
/**
* [cinny #43] Call hotkeys (push-to-talk, deafen) while focus is inside this
* frame. The host used to add key listeners to this window directly, which
* needs same-origin access. Now it sends `io.lotus.set_hotkeys { codes }` and
* we report those keys back as `io.lotus.hotkey`. The default action is
* cancelled here, synchronously (it can't be across frames), with the host's
* rules: never while typing in a field, and not on a focused button/link so
* keyboard users can still activate it. Returns a teardown function.
*/
export function startLotusHotkeys(): () => void {
const w = widget;
if (!w) return (): void => undefined;
hostCodes = new Set<string>();
const onSetHotkeys = (ev: CustomEvent<IWidgetApiRequest>): void => {
w.api.transport.reply(ev.detail, {});
hostCodes = new Set(parseHotkeyCodes(ev.detail.data));
};
const onKey = (e: KeyboardEvent): void => {
if (!hostCodes.has(e.code)) return;
const target = e.target instanceof Element ? e.target : null;
const editable = isEditableTarget(target);
const interactive = isInteractiveTarget(target);
if (
e.type === "keydown" &&
!editable &&
!interactive &&
!e.ctrlKey &&
!e.altKey &&
!e.metaKey
) {
e.preventDefault();
}
const report: LotusHotkeyReport = {
type: e.type === "keyup" ? "keyup" : "keydown",
code: e.code,
repeat: e.repeat,
ctrlKey: e.ctrlKey,
altKey: e.altKey,
metaKey: e.metaKey,
shiftKey: e.shiftKey,
editable,
interactive,
};
lotusSendToHost(LotusWidgetActions.Hotkey, report);
};
const onFocusChange = (): void => {
if (hostCodes.size > 0)
lotusSendToHost(LotusWidgetActions.Hotkey, { type: "focus" });
};
w.lazyActions.on(LotusWidgetActions.SetHotkeys, onSetHotkeys);
window.addEventListener("keydown", onKey, true);
window.addEventListener("keyup", onKey, true);
window.addEventListener("blur", onFocusChange);
window.addEventListener("focus", onFocusChange);
return (): void => {
w.lazyActions.off(LotusWidgetActions.SetHotkeys, onSetHotkeys);
window.removeEventListener("keydown", onKey, true);
window.removeEventListener("keyup", onKey, true);
window.removeEventListener("blur", onFocusChange);
window.removeEventListener("focus", onFocusChange);
hostCodes = new Set<string>();
};
}
+37
View File
@@ -0,0 +1,37 @@
/*
Copyright 2026 Lotus Guild
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { describe, expect, it } from "vitest";
import { MicLevelQuantizer } from "./lotusMicLevel";
describe("MicLevelQuantizer", () => {
it("maps RMS to 0–3 bars", () => {
expect(new MicLevelQuantizer().push(0.001)).toBe(0);
expect(new MicLevelQuantizer().push(0.008)).toBe(1);
expect(new MicLevelQuantizer().push(0.02)).toBe(2);
expect(new MicLevelQuantizer().push(0.2)).toBe(3);
});
it("rises at once and falls one bar per sample", () => {
const q = new MicLevelQuantizer();
expect(q.push(0.2)).toBe(3);
expect(q.push(0)).toBe(2);
expect(q.push(0)).toBe(1);
expect(q.push(0.02)).toBe(2);
expect(q.push(0)).toBe(1);
expect(q.push(0)).toBe(0);
expect(q.push(0)).toBe(0);
});
it("reset drops straight to 0", () => {
const q = new MicLevelQuantizer();
q.push(0.2);
q.reset();
expect(q.push(0)).toBe(0);
});
});
+192
View File
@@ -0,0 +1,192 @@
/*
Copyright 2026 Lotus Guild
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import {
type LocalTrackPublication,
type Room as LivekitRoom,
RoomEvent,
Track,
} from "livekit-client";
import { Observable, type Subscription, share, switchMap } from "rxjs";
import { distinctUntilChanged, map } from "rxjs/operators";
import { type CallViewModel } from "../state/CallViewModel/CallViewModel";
import { widget } from "../widget";
import { LotusWidgetActions, lotusFlag, lotusSendToHost } from "./lotusWidget";
/**
* [lotus #146] One local mic sampler shared by the host's mic level meter
* (`io.lotus.mic_level`) and "talking while muted" (#37, lotusMutedSpeech).
* It taps a CLONE of the published mic track (the post-processor track when
* the in-source denoiser is active, so what's measured is what's sent) and
* reads RMS at ~10 Hz while a mic track is published, muted or not.
* Local only: nothing here reaches other participants.
*/
const SAMPLE_MS = 100;
export interface LocalMicSample {
rms: number;
/** The mic is published but muted (the clone still hears it). */
muted: boolean;
}
const micPublication = (
room: LivekitRoom,
): { track: MediaStreamTrack; muted: boolean } | null => {
const pub: LocalTrackPublication | undefined =
room.localParticipant.getTrackPublication(Track.Source.Microphone);
const track = pub?.track?.mediaStreamTrack;
return track && track.readyState === "live"
? { track, muted: pub?.isMuted ?? false }
: null;
};
const samplers = new WeakMap<
CallViewModel,
Observable<LocalMicSample | null>
>();
/**
* RMS samples of the local mic, or `null` while no mic track is published.
* Shared per call view model, so the meter and the muted-speech detector use
* one AudioContext between them.
*/
export function observeLocalMicSample$(
vm: CallViewModel,
): Observable<LocalMicSample | null> {
const cached = samplers.get(vm);
if (cached) return cached;
const sampler$ = vm.allConnections$.pipe(
switchMap(
(data) =>
new Observable<LocalMicSample | null>((subscriber) => {
const rooms = data.getConnections().map((c) => c.livekitRoom);
let ctx: AudioContext | null = null;
let clone: MediaStreamTrack | null = null;
let timer: ReturnType<typeof setInterval> | undefined;
let tapped: MediaStreamTrack | null = null;
let muted = false;
const stopTap = (): void => {
if (timer !== undefined) clearInterval(timer);
timer = undefined;
clone?.stop();
clone = null;
void ctx?.close().catch(() => undefined);
ctx = null;
tapped = null;
subscriber.next(null);
};
const startTap = (source: MediaStreamTrack): void => {
try {
clone = source.clone();
// Muting disables the published track; the clone must still hear.
clone.enabled = true;
ctx = new AudioContext();
const analyser = ctx.createAnalyser();
analyser.fftSize = 1024;
ctx
.createMediaStreamSource(new MediaStream([clone]))
.connect(analyser);
const buf = new Float32Array(analyser.fftSize);
tapped = source;
timer = setInterval(() => {
analyser.getFloatTimeDomainData(buf);
let sum = 0;
for (let i = 0; i < buf.length; i += 1) sum += buf[i] * buf[i];
subscriber.next({ rms: Math.sqrt(sum / buf.length), muted });
}, SAMPLE_MS);
} catch {
stopTap();
}
};
const reconcile = (): void => {
const pub =
rooms.map(micPublication).find((p) => p !== null) ?? null;
muted = pub?.muted ?? false;
const track = pub?.track ?? null;
if (track === tapped) return;
if (tapped) stopTap();
if (track) startTap(track);
};
const events = [
RoomEvent.TrackMuted,
RoomEvent.TrackUnmuted,
RoomEvent.LocalTrackPublished,
RoomEvent.LocalTrackUnpublished,
RoomEvent.Reconnected,
RoomEvent.Disconnected,
] as const;
rooms.forEach((room) =>
events.forEach((ev) => room.on(ev, reconcile)),
);
subscriber.next(null);
reconcile();
return (): void => {
rooms.forEach((room) =>
events.forEach((ev) => room.off(ev, reconcile)),
);
if (tapped) stopTap();
};
}),
),
share(),
);
samplers.set(vm, sampler$);
return sampler$;
}
/** RMS at which each bar lights: ≈ −46, −36 and −26 dBFS. */
export const BAR_THRESHOLDS = [0.005, 0.015, 0.05] as const;
/**
* Quantise RMS to 0–3 bars with a little hysteresis: rises at once, falls one
* bar per sample, so the meter doesn't flicker between words. Unit-tested.
*/
export class MicLevelQuantizer {
private bars = 0;
public reset(): void {
this.bars = 0;
}
public push(rms: number): number {
const target = BAR_THRESHOLDS.filter((t) => rms >= t).length;
this.bars = target >= this.bars ? target : this.bars - 1;
return this.bars;
}
}
/**
* Send the host `io.lotus.mic_level { bars }` (0–3) whenever the quantised
* level changes; 0 while muted or with no mic. At most one message per sample
* (10 Hz) and none during steady silence. Opt-in with the rest of the host
* state stream (`lotusCallState=1`). Returns a teardown function.
*/
export function startLotusMicLevel(vm: CallViewModel): () => void {
if (!lotusFlag("lotusCallState") || !widget) return (): void => undefined;
const quantizer = new MicLevelQuantizer();
const sub: Subscription = observeLocalMicSample$(vm)
.pipe(
map((sample) => {
if (!sample || sample.muted) {
quantizer.reset();
return 0;
}
return quantizer.push(sample.rms);
}),
distinctUntilChanged(),
)
.subscribe((bars) => {
lotusSendToHost(LotusWidgetActions.MicLevel, { bars });
});
return (): void => sub.unsubscribe();
}
+19 -89
View File
@@ -6,14 +6,15 @@ Please see LICENSE in the repository root for full details.
*/
import {
type LocalTrackPublication,
type Room as LivekitRoom,
RoomEvent,
Track,
} from "livekit-client";
import { Observable, distinctUntilChanged, switchMap } from "rxjs";
type Observable,
distinctUntilChanged,
map,
scan,
startWith,
} from "rxjs";
import { type CallViewModel } from "../state/CallViewModel/CallViewModel";
import { type LocalMicSample, observeLocalMicSample$ } from "./lotusMicLevel";
/**
* [lotus #37] "Talking while muted" detection for the LOCAL participant.
@@ -23,13 +24,12 @@ import { type CallViewModel } from "../state/CallViewModel/CallViewModel";
* tells the user they are talking into a muted mic. This taps a CLONE of the
* published track (the post-processor track when the in-source denoiser is
* active, so keyboard noise doesn't count), samples RMS at ~10 Hz and emits a
* debounced boolean. Zero cost when unmuted (tap torn down), local-only —
* debounced boolean. Local-only —
* the flag rides `io.lotus.call_state` to the host and never reaches other
* participants.
*/
export const MUTED_SPEECH_RMS = 0.015; // ≈ −36 dBFS; normal speech into a headset is 0.05–0.3
const SAMPLE_MS = 100;
const ON_SAMPLES = 3; // 300 ms of voice before we say "talking"
const OFF_SAMPLES = 8; // 800 ms of quiet before we drop it
@@ -61,94 +61,24 @@ export class MutedSpeechGate {
}
}
const mutedMicTrack = (room: LivekitRoom): MediaStreamTrack | null => {
const pub: LocalTrackPublication | undefined =
room.localParticipant.getTrackPublication(Track.Source.Microphone);
const track = pub?.track?.mediaStreamTrack;
return pub?.isMuted && track && track.readyState === "live" ? track : null;
};
/**
* Emits while the local mic is muted and voice is detected on it. Emits
* `false` whenever the mic is unmuted, unpublished or the connection changes.
* [lotus #146] Fed by the shared local mic sampler (lotusMicLevel.ts), which
* also drives the host's mic level meter while unmuted.
*/
export function observeSpeakingWhileMuted$(
vm: CallViewModel,
): Observable<boolean> {
return vm.allConnections$.pipe(
switchMap(
(data) =>
new Observable<boolean>((subscriber) => {
const rooms = data.getConnections().map((c) => c.livekitRoom);
let ctx: AudioContext | null = null;
let clone: MediaStreamTrack | null = null;
let timer: ReturnType<typeof setInterval> | undefined;
let tapped: MediaStreamTrack | null = null;
const stopTap = (): void => {
if (timer !== undefined) clearInterval(timer);
timer = undefined;
clone?.stop();
clone = null;
void ctx?.close().catch(() => undefined);
ctx = null;
tapped = null;
subscriber.next(false);
};
const startTap = (source: MediaStreamTrack): void => {
try {
clone = source.clone();
clone.enabled = true; // the source is disabled by the mute — the clone must not be
ctx = new AudioContext();
const analyser = ctx.createAnalyser();
analyser.fftSize = 1024;
ctx
.createMediaStreamSource(new MediaStream([clone]))
.connect(analyser);
const buf = new Float32Array(analyser.fftSize);
const gate = new MutedSpeechGate();
tapped = source;
timer = setInterval(() => {
analyser.getFloatTimeDomainData(buf);
let sum = 0;
for (let i = 0; i < buf.length; i += 1) sum += buf[i] * buf[i];
subscriber.next(gate.push(Math.sqrt(sum / buf.length)));
}, SAMPLE_MS);
} catch {
stopTap();
}
};
const reconcile = (): void => {
const track =
rooms.map(mutedMicTrack).find((t) => t !== null) ?? null;
if (track === tapped) return;
if (tapped) stopTap();
if (track) startTap(track);
};
const events = [
RoomEvent.TrackMuted,
RoomEvent.TrackUnmuted,
RoomEvent.LocalTrackPublished,
RoomEvent.LocalTrackUnpublished,
RoomEvent.Reconnected,
RoomEvent.Disconnected,
] as const;
rooms.forEach((room) =>
events.forEach((ev) => room.on(ev, reconcile)),
);
subscriber.next(false);
reconcile();
return () => {
rooms.forEach((room) =>
events.forEach((ev) => room.off(ev, reconcile)),
);
if (tapped) stopTap();
};
}),
),
return observeLocalMicSample$(vm).pipe(
scan((gate: MutedSpeechGate | null, sample: LocalMicSample | null) => {
if (!sample?.muted) return null;
const g = gate ?? new MutedSpeechGate();
g.push(sample.rms);
return g;
}, null),
map((gate) => gate?.value ?? false),
startWith(false),
distinctUntilChanged(),
);
}
+5
View File
@@ -32,6 +32,8 @@ import { widget } from "../widget";
import { startLotusCallState } from "../lotus/lotusCallState";
import { startLotusFocus } from "../lotus/lotusFocus";
import { startLotusControls } from "../lotus/lotusControls";
import { startLotusHotkeys } from "../lotus/lotusHotkeys";
import { startLotusMicLevel } from "../lotus/lotusMicLevel";
import { startLotusAudioInject } from "../lotus/lotusAudioInject";
import { startLotusQuality } from "../lotus/lotusQuality";
import { startLotusDecorations } from "../lotus/lotusDecorations";
@@ -304,6 +306,9 @@ export const InCallView: FC<InCallViewProps> = ({
// [cinny #43] layout / settings / reactions over the widget API, plus a
// screensharing + layout report, replacing the host's DOM access.
useEffect(() => startLotusControls(vm), [vm]);
useEffect(() => startLotusHotkeys(), []);
// [cinny #146] Local mic level for the host's mute-button meter.
useEffect(() => startLotusMicLevel(vm), [vm]);
// [lotus] Handle the host's io.lotus.inject_audio action to mix a soundboard
// clip into the call as a separate track (#3). No-op unless the host sends it.
useEffect(() => startLotusAudioInject(vm), [vm]);
+4
View File
@@ -14,6 +14,7 @@ import {
ReactionSet,
ReactionsRowSize,
} from "./reactions";
import { isLotusHostHotkey } from "./lotus/lotusHotkeys";
/**
* Determines whether focus is in the same part of the tree as the given
@@ -86,6 +87,8 @@ export function useCallViewKeyboardShortcuts(
useCallback(
(event: KeyboardEvent) => {
logger.info("Keydown event", event);
// [lotus] The Lotus host owns its PTT / deafen keys (cinny #43).
if (isLotusHostHotkey(event.code)) return;
if (!mayReceiveKeyEvents()) return;
if (event.altKey || event.ctrlKey || event.metaKey || event.shiftKey)
return;
@@ -132,6 +135,7 @@ export function useCallViewKeyboardShortcuts(
"keyup",
useCallback(
(event: KeyboardEvent) => {
if (isLotusHostHotkey(event.code)) return;
if (!mayReceiveKeyEvents() || !mayReceiveSpaceKeyEvents()) return;
if (event.key === " ") {
spacebarHeld.current = false;
+11 -1
View File
@@ -76,7 +76,17 @@ export const useTheme = (): void => {
"lotusTheme anyway since the two flags are only meaningful together.",
);
}
if (lotusTheme || lotusTransparent)
if (lotusTheme || lotusTransparent) {
document.body.classList.add("lotus-theme");
// [cinny #43] The frame's root colour scheme must match the host's, or
// the browser paints an opaque backdrop behind the transparent frame.
// (The host used to inject `:root { color-scheme }` for this.)
document.documentElement.style.colorScheme = theme;
}
// [cinny #43] The host draws its own call bar: hide EC's footer, but keep
// it in the DOM (the host still clicks its screenshare button on engines
// without Capability Delegation). Replaces the host's injected styles.
if (lotusFlag("lotusHostControls"))
document.body.classList.add("lotus-host-controls");
}, [previousTheme, requestedTheme]);
};