Compare commits

..
Author SHA1 Message Date
Lotus CIandClaude Opus 5.5 35bc7a1e98 chore(lotus): 0.25.0-lotus.16
CI / Publish to Gitea npm registry (push) Successful in 56s
CI / Build embedded bundle (push) Successful in 2m4s
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01PPmy3tPq869XDW4njjVaKA
2026-09-25 12:08:57 -04:00
Lotus CIandClaude Opus 5.5 cee6bab9a6 feat(lotus): report the local mic level to the host (cinny #146)
The host had only a boolean `speaking` for yourself, throttled to 500 ms, so
nothing on the call bar showed that the mic was actually picking you up.

- lotusMicLevel.ts: one shared local mic sampler (a clone of the published
  track, ~10 Hz RMS, while a mic is published, muted or not), shared per
  call view model so there is one AudioContext.
- MicLevelQuantizer: 0–3 bars (≈ −46/−36/−26 dBFS), rising at once and
  falling one bar per sample so it doesn't flicker between words.
- fromWidget io.lotus.mic_level { bars } only when the value changes; 0 while
  muted or with no mic. Opt-in with the host state stream (lotusCallState=1).
- "Talking while muted" (#37) now reads the same sampler instead of running
  its own; same gate and behaviour.

Verified in a local call with a fake-tone mic: the host receives 0–3 as the
tone rises/falls (about 4 messages/s while it changes), 0 on mute and levels
again on unmute; muted + tone still raises speakingWhileMuted and the host's
"You're muted" notice. Lotus/room suites pass on Node 22 (148 tests).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01PPmy3tPq869XDW4njjVaKA
2026-09-25 12:08:57 -04:00
7 changed files with 258 additions and 90 deletions
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lotusguild/element-call-embedded",
"version": "0.25.0-lotus.15",
"version": "0.25.0-lotus.16",
"files": [
"README.md",
"LICENSE-AGPL-3.0",
+1
View File
@@ -50,5 +50,6 @@ describe("LotusWidgetActions", () => {
expect(LOTUS_TO_WIDGET_ACTIONS).not.toContain(
LotusWidgetActions.ControlsState,
);
expect(LOTUS_TO_WIDGET_ACTIONS).not.toContain(LotusWidgetActions.MicLevel);
});
});
+5
View File
@@ -80,6 +80,11 @@ export enum LotusWidgetActions {
* them. Its arrival also tells the host this fork supports the actions above.
*/
ControlsState = "io.lotus.controls_state",
/**
* fromWidget: local mic level `{ bars: 0 | 1 | 2 | 3 }` (cinny #146), sent
* only when it changes (≤ 10 Hz); 0 while muted or with no mic.
*/
MicLevel = "io.lotus.mic_level",
}
/** toWidget Lotus actions that must be allow-listed in `initializeWidget`. */
+37
View File
@@ -0,0 +1,37 @@
/*
Copyright 2026 Lotus Guild
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { describe, expect, it } from "vitest";
import { MicLevelQuantizer } from "./lotusMicLevel";
describe("MicLevelQuantizer", () => {
it("maps RMS to 0–3 bars", () => {
expect(new MicLevelQuantizer().push(0.001)).toBe(0);
expect(new MicLevelQuantizer().push(0.008)).toBe(1);
expect(new MicLevelQuantizer().push(0.02)).toBe(2);
expect(new MicLevelQuantizer().push(0.2)).toBe(3);
});
it("rises at once and falls one bar per sample", () => {
const q = new MicLevelQuantizer();
expect(q.push(0.2)).toBe(3);
expect(q.push(0)).toBe(2);
expect(q.push(0)).toBe(1);
expect(q.push(0.02)).toBe(2);
expect(q.push(0)).toBe(1);
expect(q.push(0)).toBe(0);
expect(q.push(0)).toBe(0);
});
it("reset drops straight to 0", () => {
const q = new MicLevelQuantizer();
q.push(0.2);
q.reset();
expect(q.push(0)).toBe(0);
});
});
+192
View File
@@ -0,0 +1,192 @@
/*
Copyright 2026 Lotus Guild
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import {
type LocalTrackPublication,
type Room as LivekitRoom,
RoomEvent,
Track,
} from "livekit-client";
import { Observable, type Subscription, share, switchMap } from "rxjs";
import { distinctUntilChanged, map } from "rxjs/operators";
import { type CallViewModel } from "../state/CallViewModel/CallViewModel";
import { widget } from "../widget";
import { LotusWidgetActions, lotusFlag, lotusSendToHost } from "./lotusWidget";
/**
* [lotus #146] One local mic sampler shared by the host's mic level meter
* (`io.lotus.mic_level`) and "talking while muted" (#37, lotusMutedSpeech).
* It taps a CLONE of the published mic track (the post-processor track when
* the in-source denoiser is active, so what's measured is what's sent) and
* reads RMS at ~10 Hz while a mic track is published, muted or not.
* Local only: nothing here reaches other participants.
*/
const SAMPLE_MS = 100;
export interface LocalMicSample {
rms: number;
/** The mic is published but muted (the clone still hears it). */
muted: boolean;
}
const micPublication = (
room: LivekitRoom,
): { track: MediaStreamTrack; muted: boolean } | null => {
const pub: LocalTrackPublication | undefined =
room.localParticipant.getTrackPublication(Track.Source.Microphone);
const track = pub?.track?.mediaStreamTrack;
return track && track.readyState === "live"
? { track, muted: pub?.isMuted ?? false }
: null;
};
const samplers = new WeakMap<
CallViewModel,
Observable<LocalMicSample | null>
>();
/**
* RMS samples of the local mic, or `null` while no mic track is published.
* Shared per call view model, so the meter and the muted-speech detector use
* one AudioContext between them.
*/
export function observeLocalMicSample$(
vm: CallViewModel,
): Observable<LocalMicSample | null> {
const cached = samplers.get(vm);
if (cached) return cached;
const sampler$ = vm.allConnections$.pipe(
switchMap(
(data) =>
new Observable<LocalMicSample | null>((subscriber) => {
const rooms = data.getConnections().map((c) => c.livekitRoom);
let ctx: AudioContext | null = null;
let clone: MediaStreamTrack | null = null;
let timer: ReturnType<typeof setInterval> | undefined;
let tapped: MediaStreamTrack | null = null;
let muted = false;
const stopTap = (): void => {
if (timer !== undefined) clearInterval(timer);
timer = undefined;
clone?.stop();
clone = null;
void ctx?.close().catch(() => undefined);
ctx = null;
tapped = null;
subscriber.next(null);
};
const startTap = (source: MediaStreamTrack): void => {
try {
clone = source.clone();
// Muting disables the published track; the clone must still hear.
clone.enabled = true;
ctx = new AudioContext();
const analyser = ctx.createAnalyser();
analyser.fftSize = 1024;
ctx
.createMediaStreamSource(new MediaStream([clone]))
.connect(analyser);
const buf = new Float32Array(analyser.fftSize);
tapped = source;
timer = setInterval(() => {
analyser.getFloatTimeDomainData(buf);
let sum = 0;
for (let i = 0; i < buf.length; i += 1) sum += buf[i] * buf[i];
subscriber.next({ rms: Math.sqrt(sum / buf.length), muted });
}, SAMPLE_MS);
} catch {
stopTap();
}
};
const reconcile = (): void => {
const pub =
rooms.map(micPublication).find((p) => p !== null) ?? null;
muted = pub?.muted ?? false;
const track = pub?.track ?? null;
if (track === tapped) return;
if (tapped) stopTap();
if (track) startTap(track);
};
const events = [
RoomEvent.TrackMuted,
RoomEvent.TrackUnmuted,
RoomEvent.LocalTrackPublished,
RoomEvent.LocalTrackUnpublished,
RoomEvent.Reconnected,
RoomEvent.Disconnected,
] as const;
rooms.forEach((room) =>
events.forEach((ev) => room.on(ev, reconcile)),
);
subscriber.next(null);
reconcile();
return (): void => {
rooms.forEach((room) =>
events.forEach((ev) => room.off(ev, reconcile)),
);
if (tapped) stopTap();
};
}),
),
share(),
);
samplers.set(vm, sampler$);
return sampler$;
}
/** RMS at which each bar lights: ≈ −46, −36 and −26 dBFS. */
export const BAR_THRESHOLDS = [0.005, 0.015, 0.05] as const;
/**
* Quantise RMS to 0–3 bars with a little hysteresis: rises at once, falls one
* bar per sample, so the meter doesn't flicker between words. Unit-tested.
*/
export class MicLevelQuantizer {
private bars = 0;
public reset(): void {
this.bars = 0;
}
public push(rms: number): number {
const target = BAR_THRESHOLDS.filter((t) => rms >= t).length;
this.bars = target >= this.bars ? target : this.bars - 1;
return this.bars;
}
}
/**
* Send the host `io.lotus.mic_level { bars }` (0–3) whenever the quantised
* level changes; 0 while muted or with no mic. At most one message per sample
* (10 Hz) and none during steady silence. Opt-in with the rest of the host
* state stream (`lotusCallState=1`). Returns a teardown function.
*/
export function startLotusMicLevel(vm: CallViewModel): () => void {
if (!lotusFlag("lotusCallState") || !widget) return (): void => undefined;
const quantizer = new MicLevelQuantizer();
const sub: Subscription = observeLocalMicSample$(vm)
.pipe(
map((sample) => {
if (!sample || sample.muted) {
quantizer.reset();
return 0;
}
return quantizer.push(sample.rms);
}),
distinctUntilChanged(),
)
.subscribe((bars) => {
lotusSendToHost(LotusWidgetActions.MicLevel, { bars });
});
return (): void => sub.unsubscribe();
}
+19 -89
View File
@@ -6,14 +6,15 @@ Please see LICENSE in the repository root for full details.
*/
import {
type LocalTrackPublication,
type Room as LivekitRoom,
RoomEvent,
Track,
} from "livekit-client";
import { Observable, distinctUntilChanged, switchMap } from "rxjs";
type Observable,
distinctUntilChanged,
map,
scan,
startWith,
} from "rxjs";
import { type CallViewModel } from "../state/CallViewModel/CallViewModel";
import { type LocalMicSample, observeLocalMicSample$ } from "./lotusMicLevel";
/**
* [lotus #37] "Talking while muted" detection for the LOCAL participant.
@@ -23,13 +24,12 @@ import { type CallViewModel } from "../state/CallViewModel/CallViewModel";
* tells the user they are talking into a muted mic. This taps a CLONE of the
* published track (the post-processor track when the in-source denoiser is
* active, so keyboard noise doesn't count), samples RMS at ~10 Hz and emits a
* debounced boolean. Zero cost when unmuted (tap torn down), local-only —
* debounced boolean. Local-only —
* the flag rides `io.lotus.call_state` to the host and never reaches other
* participants.
*/
export const MUTED_SPEECH_RMS = 0.015; // ≈ −36 dBFS; normal speech into a headset is 0.05–0.3
const SAMPLE_MS = 100;
const ON_SAMPLES = 3; // 300 ms of voice before we say "talking"
const OFF_SAMPLES = 8; // 800 ms of quiet before we drop it
@@ -61,94 +61,24 @@ export class MutedSpeechGate {
}
}
const mutedMicTrack = (room: LivekitRoom): MediaStreamTrack | null => {
const pub: LocalTrackPublication | undefined =
room.localParticipant.getTrackPublication(Track.Source.Microphone);
const track = pub?.track?.mediaStreamTrack;
return pub?.isMuted && track && track.readyState === "live" ? track : null;
};
/**
* Emits while the local mic is muted and voice is detected on it. Emits
* `false` whenever the mic is unmuted, unpublished or the connection changes.
* [lotus #146] Fed by the shared local mic sampler (lotusMicLevel.ts), which
* also drives the host's mic level meter while unmuted.
*/
export function observeSpeakingWhileMuted$(
vm: CallViewModel,
): Observable<boolean> {
return vm.allConnections$.pipe(
switchMap(
(data) =>
new Observable<boolean>((subscriber) => {
const rooms = data.getConnections().map((c) => c.livekitRoom);
let ctx: AudioContext | null = null;
let clone: MediaStreamTrack | null = null;
let timer: ReturnType<typeof setInterval> | undefined;
let tapped: MediaStreamTrack | null = null;
const stopTap = (): void => {
if (timer !== undefined) clearInterval(timer);
timer = undefined;
clone?.stop();
clone = null;
void ctx?.close().catch(() => undefined);
ctx = null;
tapped = null;
subscriber.next(false);
};
const startTap = (source: MediaStreamTrack): void => {
try {
clone = source.clone();
clone.enabled = true; // the source is disabled by the mute — the clone must not be
ctx = new AudioContext();
const analyser = ctx.createAnalyser();
analyser.fftSize = 1024;
ctx
.createMediaStreamSource(new MediaStream([clone]))
.connect(analyser);
const buf = new Float32Array(analyser.fftSize);
const gate = new MutedSpeechGate();
tapped = source;
timer = setInterval(() => {
analyser.getFloatTimeDomainData(buf);
let sum = 0;
for (let i = 0; i < buf.length; i += 1) sum += buf[i] * buf[i];
subscriber.next(gate.push(Math.sqrt(sum / buf.length)));
}, SAMPLE_MS);
} catch {
stopTap();
}
};
const reconcile = (): void => {
const track =
rooms.map(mutedMicTrack).find((t) => t !== null) ?? null;
if (track === tapped) return;
if (tapped) stopTap();
if (track) startTap(track);
};
const events = [
RoomEvent.TrackMuted,
RoomEvent.TrackUnmuted,
RoomEvent.LocalTrackPublished,
RoomEvent.LocalTrackUnpublished,
RoomEvent.Reconnected,
RoomEvent.Disconnected,
] as const;
rooms.forEach((room) =>
events.forEach((ev) => room.on(ev, reconcile)),
);
subscriber.next(false);
reconcile();
return () => {
rooms.forEach((room) =>
events.forEach((ev) => room.off(ev, reconcile)),
);
if (tapped) stopTap();
};
}),
),
return observeLocalMicSample$(vm).pipe(
scan((gate: MutedSpeechGate | null, sample: LocalMicSample | null) => {
if (!sample?.muted) return null;
const g = gate ?? new MutedSpeechGate();
g.push(sample.rms);
return g;
}, null),
map((gate) => gate?.value ?? false),
startWith(false),
distinctUntilChanged(),
);
}
+3
View File
@@ -32,6 +32,7 @@ import { widget } from "../widget";
import { startLotusCallState } from "../lotus/lotusCallState";
import { startLotusFocus } from "../lotus/lotusFocus";
import { startLotusControls } from "../lotus/lotusControls";
import { startLotusMicLevel } from "../lotus/lotusMicLevel";
import { startLotusAudioInject } from "../lotus/lotusAudioInject";
import { startLotusQuality } from "../lotus/lotusQuality";
import { startLotusDecorations } from "../lotus/lotusDecorations";
@@ -304,6 +305,8 @@ export const InCallView: FC<InCallViewProps> = ({
// [cinny #43] layout / settings / reactions over the widget API, plus a
// screensharing + layout report, replacing the host's DOM access.
useEffect(() => startLotusControls(vm), [vm]);
// [cinny #146] Local mic level for the host's mute-button meter.
useEffect(() => startLotusMicLevel(vm), [vm]);
// [lotus] Handle the host's io.lotus.inject_audio action to mix a soundboard
// clip into the call as a separate track (#3). No-op unless the host sends it.
useEffect(() => startLotusAudioInject(vm), [vm]);