Compare commits
12
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
b9d0ac7cac | ||
|
|
1e34923f6a | ||
|
|
fcb7f8456d | ||
|
|
c2267800be | ||
|
|
5421d545c4 | ||
|
|
2fafa3cf36 | ||
|
|
667230f6e3 | ||
|
|
746917a4c6 | ||
|
|
d881833491 | ||
|
|
021b1881e5 | ||
|
|
1b609d997b | ||
|
|
d9ac9a0fa4 |
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@lotusguild/element-call-embedded",
|
||||
"version": "0.25.0-lotus.4",
|
||||
"version": "0.25.0-lotus.10",
|
||||
"files": [
|
||||
"README.md",
|
||||
"LICENSE-AGPL-3.0",
|
||||
|
||||
@@ -116,6 +116,7 @@
|
||||
"peer_connection_timeout_description": "Connection to the media server timed out. Try switching to a different network or disabling your VPN. If the problem persists, see our <0>troubleshooting guide</0> or contact your server administrator.",
|
||||
"room_creation_restricted": "Failed to create call",
|
||||
"room_creation_restricted_description": "Call creation might be restricted to authorized users only. Try again later, or contact your server admin if the problem persists.",
|
||||
"sfu_token_refused": "Can't join this call",
|
||||
"sticky_events_required": "Homeserver does not support Matrix 2.0 calls",
|
||||
"sticky_events_required_description": "This deployment is configured to use Matrix 2.0 call mode, but the homeserver does not advertise support for sticky events (MSC4354). Ask your server admin to upgrade, or switch the deployment to a compatible mode.",
|
||||
"unexpected_ec_error": "An unexpected error occurred (<0>Error Code:</0> <1>{{ errorCode }}</1>). Please contact your server admin."
|
||||
|
||||
@@ -20,7 +20,7 @@ import { MatrixError } from "matrix-js-sdk";
|
||||
import { getSFUConfigWithOpenID, type OpenIDClientParts } from "./openIDSFU";
|
||||
import { testJWTToken } from "../utils/test-fixtures";
|
||||
import { ownMemberMock } from "../utils/test";
|
||||
import { FailToGetOpenIdToken } from "../utils/errors";
|
||||
import { FailToGetOpenIdToken, SFUTokenRefusedError } from "../utils/errors";
|
||||
|
||||
const sfuUrl = "https://sfu.example.org";
|
||||
|
||||
@@ -91,6 +91,31 @@ describe("getSFUConfigWithOpenID", () => {
|
||||
expect.fail("Expected test to throw;");
|
||||
});
|
||||
|
||||
it("[lotus] surfaces a 403 refusal's reason instead of the generic error", async () => {
|
||||
fetchMock.post("https://sfu.example.org/sfu/get", () => {
|
||||
return {
|
||||
status: 403,
|
||||
body: { errcode: "M_FORBIDDEN", error: "This voice channel is full." },
|
||||
};
|
||||
});
|
||||
try {
|
||||
await getSFUConfigWithOpenID(
|
||||
matrixClient,
|
||||
ownMemberMock,
|
||||
"https://sfu.example.org",
|
||||
"!example_room_id",
|
||||
);
|
||||
} catch (ex: unknown) {
|
||||
expect(ex).toBeInstanceOf(SFUTokenRefusedError);
|
||||
expect((ex as SFUTokenRefusedError).localisedMessage).toEqual(
|
||||
"This voice channel is full.",
|
||||
);
|
||||
void (await fetchMock.flush());
|
||||
return;
|
||||
}
|
||||
expect.fail("Expected test to throw;");
|
||||
});
|
||||
|
||||
it("should retry without delay params if the JWT service legacy endpoint returns M_BAD_JSON 400", async () => {
|
||||
let callCount = 0;
|
||||
|
||||
|
||||
@@ -8,6 +8,7 @@ Please see LICENSE in the repository root for full details.
|
||||
import {
|
||||
type IOpenIDToken,
|
||||
type MatrixClient,
|
||||
MatrixError,
|
||||
parseErrorResponse,
|
||||
} from "matrix-js-sdk";
|
||||
import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager";
|
||||
@@ -16,6 +17,7 @@ import { type Logger } from "matrix-js-sdk/lib/logger";
|
||||
import {
|
||||
FailToGetOpenIdToken,
|
||||
NoMatrix2AuthorizationService,
|
||||
SFUTokenRefusedError,
|
||||
} from "../utils/errors";
|
||||
import { doNetworkOperationWithRetry } from "../utils/matrix";
|
||||
import { Config } from "../config/Config";
|
||||
@@ -165,6 +167,14 @@ export async function getSFUConfigWithOpenID(
|
||||
logger?.info(`Got JWT from call's active focus URL.`);
|
||||
return extractFullConfigFromToken(sfuConfig);
|
||||
} catch (ex) {
|
||||
// [lotus] A 403 from the token service carries the reason the user needs
|
||||
// ("This voice channel is full.") — surface it instead of the generic error.
|
||||
if (ex instanceof MatrixError && ex.httpStatus === 403) {
|
||||
const reason = (ex.data as { error?: unknown } | undefined)?.error;
|
||||
if (typeof reason === "string" && reason.trim()) {
|
||||
throw new SFUTokenRefusedError(reason, ex);
|
||||
}
|
||||
}
|
||||
throw new FailToGetOpenIdToken(
|
||||
ex instanceof Error ? ex : new Error(`Unknown error ${ex}`),
|
||||
);
|
||||
|
||||
@@ -46,6 +46,12 @@ export enum LotusWidgetActions {
|
||||
* toggle can reflect reality rather than the requested state.
|
||||
*/
|
||||
DenoiseState = "io.lotus.denoise_state",
|
||||
/**
|
||||
* fromWidget: one-shot end-of-call readout for the local participant —
|
||||
* `{ durationMs, reconnects, poorMs, verdict }` (#143) — sent on SFU
|
||||
* disconnect or in-call teardown, whichever comes first.
|
||||
*/
|
||||
CallSummary = "io.lotus.call_summary",
|
||||
}
|
||||
|
||||
/** toWidget Lotus actions that must be allow-listed in `initializeWidget`. */
|
||||
|
||||
@@ -53,7 +53,11 @@ function mockMember(id: string, userId: string): Member {
|
||||
}
|
||||
|
||||
function mockVm(members: Member[]): CallViewModel {
|
||||
return { userMedia$: of(members) } as unknown as CallViewModel;
|
||||
return {
|
||||
userMedia$: of(members),
|
||||
// no livekit connections → the muted-speech tap never starts
|
||||
allConnections$: of({ getConnections: () => [] }),
|
||||
} as unknown as CallViewModel;
|
||||
}
|
||||
|
||||
function participantsOf(call: number): unknown[] {
|
||||
|
||||
+41
-28
@@ -5,7 +5,7 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { combineLatest, of, type Subscription } from "rxjs";
|
||||
import { combineLatest, of, startWith, type Subscription } from "rxjs";
|
||||
import {
|
||||
distinctUntilChanged,
|
||||
map,
|
||||
@@ -16,6 +16,7 @@ import {
|
||||
import { type CallViewModel } from "../state/CallViewModel/CallViewModel";
|
||||
import { widget } from "../widget";
|
||||
import { LotusWidgetActions, lotusFlag, lotusSendToHost } from "./lotusWidget";
|
||||
import { observeSpeakingWhileMuted$ } from "./lotusMutedSpeech";
|
||||
|
||||
interface ParticipantState {
|
||||
/** EC media id (`${userId}:${deviceId}`), stable per participant device. */
|
||||
@@ -25,6 +26,11 @@ interface ParticipantState {
|
||||
speaking: boolean;
|
||||
audioEnabled: boolean;
|
||||
videoEnabled: boolean;
|
||||
/**
|
||||
* [lotus #37] LOCAL participant only: voice detected on the mic while it is
|
||||
* muted (see lotusMutedSpeech.ts). Absent for remote participants.
|
||||
*/
|
||||
speakingWhileMuted?: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -45,7 +51,8 @@ function participantsEqual(
|
||||
p.userId === b[i].userId &&
|
||||
p.speaking === b[i].speaking &&
|
||||
p.audioEnabled === b[i].audioEnabled &&
|
||||
p.videoEnabled === b[i].videoEnabled,
|
||||
p.videoEnabled === b[i].videoEnabled &&
|
||||
p.speakingWhileMuted === b[i].speakingWhileMuted,
|
||||
)
|
||||
);
|
||||
}
|
||||
@@ -66,34 +73,40 @@ export function startLotusCallState(vm: CallViewModel): () => void {
|
||||
// lotusDecorations.ts.
|
||||
if (!widget) return () => undefined;
|
||||
|
||||
const sub: Subscription = vm.userMedia$
|
||||
.pipe(
|
||||
switchMap((members) =>
|
||||
members.length === 0
|
||||
? of([] as ParticipantState[])
|
||||
: combineLatest(
|
||||
members.map((m) =>
|
||||
combineLatest([
|
||||
m.speaking$,
|
||||
m.audioEnabled$,
|
||||
m.videoEnabled$,
|
||||
]).pipe(
|
||||
map(
|
||||
([
|
||||
speaking,
|
||||
audioEnabled,
|
||||
videoEnabled,
|
||||
]): ParticipantState => ({
|
||||
id: m.id,
|
||||
userId: m.userId,
|
||||
speaking,
|
||||
audioEnabled,
|
||||
videoEnabled,
|
||||
}),
|
||||
),
|
||||
),
|
||||
const participants$ = vm.userMedia$.pipe(
|
||||
switchMap((members) =>
|
||||
members.length === 0
|
||||
? of([] as (ParticipantState & { local: boolean })[])
|
||||
: combineLatest(
|
||||
members.map((m) =>
|
||||
combineLatest([
|
||||
m.speaking$,
|
||||
m.audioEnabled$,
|
||||
m.videoEnabled$,
|
||||
]).pipe(
|
||||
map(([speaking, audioEnabled, videoEnabled]) => ({
|
||||
id: m.id,
|
||||
userId: m.userId,
|
||||
speaking,
|
||||
audioEnabled,
|
||||
videoEnabled,
|
||||
local: m.local === true,
|
||||
})),
|
||||
),
|
||||
),
|
||||
),
|
||||
),
|
||||
);
|
||||
|
||||
const sub: Subscription = combineLatest([
|
||||
participants$,
|
||||
observeSpeakingWhileMuted$(vm).pipe(startWith(false)),
|
||||
])
|
||||
.pipe(
|
||||
map(([members, speakingWhileMuted]): ParticipantState[] =>
|
||||
members.map(({ local, ...p }) =>
|
||||
local ? { ...p, speakingWhileMuted } : p,
|
||||
),
|
||||
),
|
||||
// `speaking` flips rapidly; drop no-op repeats BEFORE throttling so
|
||||
// the throttle window isn't spent re-emitting an unchanged value.
|
||||
|
||||
@@ -0,0 +1,93 @@
|
||||
/*
|
||||
Copyright 2026 Lotus Guild
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { ConnectionQuality } from "livekit-client";
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import { CallQualityTracker } from "./lotusCallSummary";
|
||||
|
||||
const clock = (): { now: () => number; tick: (ms: number) => void } => {
|
||||
let t = 1_000_000;
|
||||
return {
|
||||
now: () => t,
|
||||
tick: (ms) => {
|
||||
t += ms;
|
||||
},
|
||||
};
|
||||
};
|
||||
|
||||
describe("CallQualityTracker", () => {
|
||||
it("reports unknown when nothing was sampled", () => {
|
||||
const c = clock();
|
||||
const tr = new CallQualityTracker(c.now);
|
||||
tr.start();
|
||||
c.tick(60_000);
|
||||
expect(tr.summary()).toEqual({
|
||||
durationMs: 60_000,
|
||||
reconnects: 0,
|
||||
poorMs: 0,
|
||||
verdict: "unknown",
|
||||
});
|
||||
});
|
||||
|
||||
it("is good when quality stayed fine", () => {
|
||||
const c = clock();
|
||||
const tr = new CallQualityTracker(c.now);
|
||||
tr.start();
|
||||
tr.setQuality(ConnectionQuality.Excellent);
|
||||
c.tick(30 * 60_000);
|
||||
expect(tr.summary().verdict).toBe("good");
|
||||
});
|
||||
|
||||
it("accumulates poor time across episodes, including an open one", () => {
|
||||
const c = clock();
|
||||
const tr = new CallQualityTracker(c.now);
|
||||
tr.start();
|
||||
tr.setQuality(ConnectionQuality.Good);
|
||||
c.tick(60_000);
|
||||
tr.setQuality(ConnectionQuality.Poor);
|
||||
c.tick(10_000);
|
||||
tr.setQuality(ConnectionQuality.Good);
|
||||
c.tick(60_000);
|
||||
tr.setQuality(ConnectionQuality.Lost);
|
||||
c.tick(5_000);
|
||||
const s = tr.summary();
|
||||
expect(s.poorMs).toBe(15_000);
|
||||
expect(s.durationMs).toBe(135_000);
|
||||
expect(s.verdict).toBe("fair");
|
||||
});
|
||||
|
||||
it("is poor with many reconnects or mostly-poor quality", () => {
|
||||
const c = clock();
|
||||
const tr = new CallQualityTracker(c.now);
|
||||
tr.start();
|
||||
tr.setQuality(ConnectionQuality.Good);
|
||||
for (let i = 0; i < 4; i += 1) tr.reconnect();
|
||||
c.tick(60_000);
|
||||
expect(tr.summary()).toMatchObject({ reconnects: 4, verdict: "poor" });
|
||||
|
||||
const tr2 = new CallQualityTracker(c.now);
|
||||
tr2.start();
|
||||
tr2.setQuality(ConnectionQuality.Poor);
|
||||
c.tick(60_000);
|
||||
expect(tr2.summary().verdict).toBe("poor");
|
||||
});
|
||||
|
||||
it("ignores unknown samples and only starts once", () => {
|
||||
const c = clock();
|
||||
const tr = new CallQualityTracker(c.now);
|
||||
tr.start();
|
||||
c.tick(1_000);
|
||||
tr.start();
|
||||
tr.setQuality(ConnectionQuality.Unknown);
|
||||
c.tick(1_000);
|
||||
expect(tr.summary()).toMatchObject({
|
||||
durationMs: 2_000,
|
||||
verdict: "unknown",
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,144 @@
|
||||
/*
|
||||
Copyright 2026 Lotus Guild
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import {
|
||||
ConnectionQuality,
|
||||
type Participant,
|
||||
type Room as LivekitRoom,
|
||||
RoomEvent,
|
||||
} from "livekit-client";
|
||||
|
||||
import { type CallViewModel } from "../state/CallViewModel/CallViewModel";
|
||||
import { widget } from "../widget";
|
||||
import { LotusWidgetActions } from "./lotusActions";
|
||||
import { lotusSendToHost } from "./lotusWidget";
|
||||
|
||||
export type CallQualityVerdict = "good" | "fair" | "poor" | "unknown";
|
||||
|
||||
export interface CallSummary {
|
||||
/** Wall-clock time connected to the SFU, ms. */
|
||||
durationMs: number;
|
||||
/** LiveKit reconnect attempts during the call. */
|
||||
reconnects: number;
|
||||
/** Time the local connection quality was reported poor/lost, ms. */
|
||||
poorMs: number;
|
||||
verdict: CallQualityVerdict;
|
||||
}
|
||||
|
||||
/**
|
||||
* [Gitea #143] Per-call connection quality, kept in memory for the local
|
||||
* participant only and summarised once at hangup. Nothing is stored or sent
|
||||
* anywhere but to the host at the end.
|
||||
*/
|
||||
export class CallQualityTracker {
|
||||
private startedAt: number | undefined;
|
||||
|
||||
private poorSince: number | undefined;
|
||||
|
||||
private poorMs = 0;
|
||||
|
||||
private reconnects = 0;
|
||||
|
||||
private sampled = false;
|
||||
|
||||
public constructor(private readonly now: () => number = () => Date.now()) {}
|
||||
|
||||
public start(): void {
|
||||
if (this.startedAt === undefined) this.startedAt = this.now();
|
||||
}
|
||||
|
||||
public setQuality(quality: ConnectionQuality): void {
|
||||
if (quality === ConnectionQuality.Unknown) return;
|
||||
this.sampled = true;
|
||||
const bad =
|
||||
quality === ConnectionQuality.Poor || quality === ConnectionQuality.Lost;
|
||||
if (bad && this.poorSince === undefined) this.poorSince = this.now();
|
||||
if (!bad && this.poorSince !== undefined) {
|
||||
this.poorMs += this.now() - this.poorSince;
|
||||
this.poorSince = undefined;
|
||||
}
|
||||
}
|
||||
|
||||
public reconnect(): void {
|
||||
this.reconnects += 1;
|
||||
}
|
||||
|
||||
public summary(): CallSummary {
|
||||
const end = this.now();
|
||||
const durationMs =
|
||||
this.startedAt === undefined ? 0 : Math.max(0, end - this.startedAt);
|
||||
const poorMs =
|
||||
this.poorMs + (this.poorSince === undefined ? 0 : end - this.poorSince);
|
||||
let verdict: CallQualityVerdict = "unknown";
|
||||
if (this.sampled && durationMs > 0) {
|
||||
const poorShare = poorMs / durationMs;
|
||||
if (poorShare < 0.05 && this.reconnects <= 1) verdict = "good";
|
||||
else if (poorShare < 0.25 && this.reconnects <= 3) verdict = "fair";
|
||||
else verdict = "poor";
|
||||
}
|
||||
return { durationMs, reconnects: this.reconnects, poorMs, verdict };
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Track the local participant's LiveKit connection quality and reconnects
|
||||
* for the life of the in-call view, and send `io.lotus.call_summary` to the
|
||||
* host once — on the first SFU disconnect or on teardown, whichever comes
|
||||
* first — so the host can show "41 min · connection was good" at hangup.
|
||||
*/
|
||||
export function startLotusCallSummary(vm: CallViewModel): () => void {
|
||||
if (!widget) return () => undefined;
|
||||
const tracker = new CallQualityTracker();
|
||||
let sent = false;
|
||||
const send = (): void => {
|
||||
if (sent) return;
|
||||
sent = true;
|
||||
lotusSendToHost(LotusWidgetActions.CallSummary, tracker.summary());
|
||||
};
|
||||
|
||||
const listeners = new Map<LivekitRoom, () => void>();
|
||||
const attach = (room: LivekitRoom): void => {
|
||||
const onQuality = (
|
||||
quality: ConnectionQuality,
|
||||
participant: Participant,
|
||||
): void => {
|
||||
if (participant.isLocal) tracker.setQuality(quality);
|
||||
};
|
||||
const onConnected = (): void => tracker.start();
|
||||
const onReconnecting = (): void => tracker.reconnect();
|
||||
const onDisconnected = (): void => send();
|
||||
room.on(RoomEvent.ConnectionQualityChanged, onQuality);
|
||||
room.on(RoomEvent.Connected, onConnected);
|
||||
room.on(RoomEvent.Reconnecting, onReconnecting);
|
||||
room.on(RoomEvent.Disconnected, onDisconnected);
|
||||
if (room.state === "connected") tracker.start();
|
||||
listeners.set(room, () => {
|
||||
room.off(RoomEvent.ConnectionQualityChanged, onQuality);
|
||||
room.off(RoomEvent.Connected, onConnected);
|
||||
room.off(RoomEvent.Reconnecting, onReconnecting);
|
||||
room.off(RoomEvent.Disconnected, onDisconnected);
|
||||
});
|
||||
};
|
||||
|
||||
const sub = vm.allConnections$.subscribe((data) => {
|
||||
const next = data.getConnections().map((c) => c.livekitRoom);
|
||||
for (const [room, off] of listeners) {
|
||||
if (!next.includes(room)) {
|
||||
off();
|
||||
listeners.delete(room);
|
||||
}
|
||||
}
|
||||
for (const room of next) if (!listeners.has(room)) attach(room);
|
||||
});
|
||||
|
||||
return () => {
|
||||
sub.unsubscribe();
|
||||
for (const off of listeners.values()) off();
|
||||
listeners.clear();
|
||||
send();
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
/*
|
||||
Copyright 2026 Lotus Guild
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { expect, test } from "vitest";
|
||||
|
||||
import { MutedSpeechGate } from "./lotusMutedSpeech";
|
||||
|
||||
test("needs 300 ms of voice to switch on and 800 ms of quiet to switch off", () => {
|
||||
const g = new MutedSpeechGate(0.015);
|
||||
expect(g.push(0.1)).toBe(false);
|
||||
expect(g.push(0.1)).toBe(false);
|
||||
expect(g.push(0.1)).toBe(true);
|
||||
// a short dip does not drop it
|
||||
for (let i = 0; i < 7; i++) expect(g.push(0.0)).toBe(true);
|
||||
expect(g.push(0.0)).toBe(false);
|
||||
// one loud sample after quiet does not re-trigger
|
||||
expect(g.push(0.2)).toBe(false);
|
||||
});
|
||||
|
||||
test("keyboard-level noise below the threshold never triggers", () => {
|
||||
const g = new MutedSpeechGate(0.015);
|
||||
for (let i = 0; i < 50; i++) expect(g.push(0.01)).toBe(false);
|
||||
});
|
||||
@@ -0,0 +1,154 @@
|
||||
/*
|
||||
Copyright 2026 Lotus Guild
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import {
|
||||
type LocalTrackPublication,
|
||||
type Room as LivekitRoom,
|
||||
RoomEvent,
|
||||
Track,
|
||||
} from "livekit-client";
|
||||
import { Observable, distinctUntilChanged, switchMap } from "rxjs";
|
||||
|
||||
import { type CallViewModel } from "../state/CallViewModel/CallViewModel";
|
||||
|
||||
/**
|
||||
* [lotus #37] "Talking while muted" detection for the LOCAL participant.
|
||||
*
|
||||
* While the mic is muted LiveKit keeps the capture alive and merely disables
|
||||
* the published MediaStreamTrack, so VAD/`speaking` is false and nothing
|
||||
* tells the user they are talking into a muted mic. This taps a CLONE of the
|
||||
* published track (the post-processor track when the in-source denoiser is
|
||||
* active, so keyboard noise doesn't count), samples RMS at ~10 Hz and emits a
|
||||
* debounced boolean. Zero cost when unmuted (tap torn down), local-only —
|
||||
* the flag rides `io.lotus.call_state` to the host and never reaches other
|
||||
* participants.
|
||||
*/
|
||||
|
||||
export const MUTED_SPEECH_RMS = 0.015; // ≈ −36 dBFS; normal speech into a headset is 0.05–0.3
|
||||
const SAMPLE_MS = 100;
|
||||
const ON_SAMPLES = 3; // 300 ms of voice before we say "talking"
|
||||
const OFF_SAMPLES = 8; // 800 ms of quiet before we drop it
|
||||
|
||||
/** Pure hysteresis gate over successive RMS samples (unit-tested). */
|
||||
export class MutedSpeechGate {
|
||||
private above = 0;
|
||||
|
||||
private below = 0;
|
||||
|
||||
private on = false;
|
||||
|
||||
public constructor(private readonly threshold = MUTED_SPEECH_RMS) {}
|
||||
|
||||
public push(rms: number): boolean {
|
||||
if (rms >= this.threshold) {
|
||||
this.above += 1;
|
||||
this.below = 0;
|
||||
if (!this.on && this.above >= ON_SAMPLES) this.on = true;
|
||||
} else {
|
||||
this.below += 1;
|
||||
this.above = 0;
|
||||
if (this.on && this.below >= OFF_SAMPLES) this.on = false;
|
||||
}
|
||||
return this.on;
|
||||
}
|
||||
|
||||
public get value(): boolean {
|
||||
return this.on;
|
||||
}
|
||||
}
|
||||
|
||||
const mutedMicTrack = (room: LivekitRoom): MediaStreamTrack | null => {
|
||||
const pub: LocalTrackPublication | undefined =
|
||||
room.localParticipant.getTrackPublication(Track.Source.Microphone);
|
||||
const track = pub?.track?.mediaStreamTrack;
|
||||
return pub?.isMuted && track && track.readyState === "live" ? track : null;
|
||||
};
|
||||
|
||||
/**
|
||||
* Emits while the local mic is muted and voice is detected on it. Emits
|
||||
* `false` whenever the mic is unmuted, unpublished or the connection changes.
|
||||
*/
|
||||
export function observeSpeakingWhileMuted$(
|
||||
vm: CallViewModel,
|
||||
): Observable<boolean> {
|
||||
return vm.allConnections$.pipe(
|
||||
switchMap(
|
||||
(data) =>
|
||||
new Observable<boolean>((subscriber) => {
|
||||
const rooms = data.getConnections().map((c) => c.livekitRoom);
|
||||
let ctx: AudioContext | null = null;
|
||||
let clone: MediaStreamTrack | null = null;
|
||||
let timer: ReturnType<typeof setInterval> | undefined;
|
||||
let tapped: MediaStreamTrack | null = null;
|
||||
|
||||
const stopTap = (): void => {
|
||||
if (timer !== undefined) clearInterval(timer);
|
||||
timer = undefined;
|
||||
clone?.stop();
|
||||
clone = null;
|
||||
void ctx?.close().catch(() => undefined);
|
||||
ctx = null;
|
||||
tapped = null;
|
||||
subscriber.next(false);
|
||||
};
|
||||
|
||||
const startTap = (source: MediaStreamTrack): void => {
|
||||
try {
|
||||
clone = source.clone();
|
||||
clone.enabled = true; // the source is disabled by the mute — the clone must not be
|
||||
ctx = new AudioContext();
|
||||
const analyser = ctx.createAnalyser();
|
||||
analyser.fftSize = 1024;
|
||||
ctx
|
||||
.createMediaStreamSource(new MediaStream([clone]))
|
||||
.connect(analyser);
|
||||
const buf = new Float32Array(analyser.fftSize);
|
||||
const gate = new MutedSpeechGate();
|
||||
tapped = source;
|
||||
timer = setInterval(() => {
|
||||
analyser.getFloatTimeDomainData(buf);
|
||||
let sum = 0;
|
||||
for (let i = 0; i < buf.length; i += 1) sum += buf[i] * buf[i];
|
||||
subscriber.next(gate.push(Math.sqrt(sum / buf.length)));
|
||||
}, SAMPLE_MS);
|
||||
} catch {
|
||||
stopTap();
|
||||
}
|
||||
};
|
||||
|
||||
const reconcile = (): void => {
|
||||
const track =
|
||||
rooms.map(mutedMicTrack).find((t) => t !== null) ?? null;
|
||||
if (track === tapped) return;
|
||||
if (tapped) stopTap();
|
||||
if (track) startTap(track);
|
||||
};
|
||||
|
||||
const events = [
|
||||
RoomEvent.TrackMuted,
|
||||
RoomEvent.TrackUnmuted,
|
||||
RoomEvent.LocalTrackPublished,
|
||||
RoomEvent.LocalTrackUnpublished,
|
||||
RoomEvent.Reconnected,
|
||||
RoomEvent.Disconnected,
|
||||
] as const;
|
||||
rooms.forEach((room) =>
|
||||
events.forEach((ev) => room.on(ev, reconcile)),
|
||||
);
|
||||
subscriber.next(false);
|
||||
reconcile();
|
||||
return () => {
|
||||
rooms.forEach((room) =>
|
||||
events.forEach((ev) => room.off(ev, reconcile)),
|
||||
);
|
||||
if (tapped) stopTap();
|
||||
};
|
||||
}),
|
||||
),
|
||||
distinctUntilChanged(),
|
||||
);
|
||||
}
|
||||
@@ -35,6 +35,7 @@ import { startLotusAudioInject } from "../lotus/lotusAudioInject";
|
||||
import { startLotusQuality } from "../lotus/lotusQuality";
|
||||
import { startLotusDecorations } from "../lotus/lotusDecorations";
|
||||
import { startLotusDenoise } from "../lotus/lotusDenoise";
|
||||
import { startLotusCallSummary } from "../lotus/lotusCallSummary";
|
||||
import { startLotusDeafen } from "../lotus/lotusDeafen";
|
||||
import styles from "./InCallView.module.css";
|
||||
import { GridTile } from "../tile/GridTile";
|
||||
@@ -309,6 +310,7 @@ export const InCallView: FC<InCallViewProps> = ({
|
||||
// [lotus] Apply ML denoise to the mic as a first-class audio processor that
|
||||
// survives reconnects (#1 / A7). No-op unless lotusDenoiseSource=1.
|
||||
useEffect(() => startLotusDenoise(vm), [vm]);
|
||||
useEffect(() => startLotusCallSummary(vm), [vm]);
|
||||
// [lotus] Handle the host's io.lotus.set_deafen action to silence remote
|
||||
// audio (and optionally screenshare audio) at the LiveKit source. No-op
|
||||
// unless the host sends the action.
|
||||
|
||||
@@ -37,6 +37,7 @@ import { Epoch, ObservableScope } from "../../ObservableScope";
|
||||
import {
|
||||
MatrixRTCTransportMissingError,
|
||||
FailToGetOpenIdToken,
|
||||
SFUTokenRefusedError,
|
||||
} from "../../../utils/errors";
|
||||
import * as openIDSFU from "../../../livekit/openIDSFU";
|
||||
import { customLivekitUrl } from "../../../settings/settings";
|
||||
@@ -125,6 +126,46 @@ describe("LocalTransport", () => {
|
||||
expect(() => active$.value).toThrow(expectedError);
|
||||
});
|
||||
|
||||
it("[lotus] passes SFUTokenRefusedError through untouched", async () => {
|
||||
const scope = new ObservableScope();
|
||||
mockConfig({
|
||||
livekit: { livekit_service_url: "https://lk.example.org" },
|
||||
});
|
||||
const refused = new SFUTokenRefusedError("This voice channel is full.");
|
||||
vi.spyOn(openIDSFU, "getSFUConfigWithOpenID").mockImplementation(
|
||||
async () => {
|
||||
await Promise.resolve();
|
||||
throw refused;
|
||||
},
|
||||
);
|
||||
const errors: Error[] = [];
|
||||
const { active$ } = createLocalTransport$({
|
||||
scope,
|
||||
roomId: "!example_room_id",
|
||||
memberships$: constant(new Epoch<CallMembership[]>([])),
|
||||
client: {
|
||||
baseUrl: "https://example.org",
|
||||
getDomain: () => "example.org",
|
||||
// eslint-disable-next-line @typescript-eslint/naming-convention
|
||||
_unstable_getRTCTransports: async () => Promise.resolve([]),
|
||||
getOpenIdToken: vi.fn(),
|
||||
getDeviceId: vi.fn(),
|
||||
},
|
||||
ownMembershipIdentity: ownMemberMock,
|
||||
forceJwtEndpoint: JwtEndpointVersion.Legacy,
|
||||
delayId$: constant("delay_id_mock"),
|
||||
});
|
||||
active$.subscribe(
|
||||
() => undefined,
|
||||
(e) => errors.push(e),
|
||||
);
|
||||
await flushPromises();
|
||||
expect(errors).toStrictEqual([refused]);
|
||||
expect((errors[0] as SFUTokenRefusedError).localisedMessage).toBe(
|
||||
"This voice channel is full.",
|
||||
);
|
||||
});
|
||||
|
||||
it("emits preferred transport after OpenID resolves", async () => {
|
||||
// Use config so transport discovery succeeds, but delay OpenID JWT fetch
|
||||
mockConfig({
|
||||
|
||||
@@ -26,6 +26,7 @@ import { type Epoch, type ObservableScope } from "../../ObservableScope.ts";
|
||||
import { Config } from "../../../config/Config.ts";
|
||||
import {
|
||||
FailToGetOpenIdToken,
|
||||
SFUTokenRefusedError,
|
||||
MatrixRTCTransportMissingError,
|
||||
NoMatrix2AuthorizationService,
|
||||
} from "../../../utils/errors.ts";
|
||||
@@ -261,7 +262,9 @@ async function doOpenIdAndJWTFromUrl(
|
||||
function mapAuthErrorToUserFriendlyError(e: unknown): Error {
|
||||
if (
|
||||
e instanceof FailToGetOpenIdToken ||
|
||||
e instanceof NoMatrix2AuthorizationService
|
||||
e instanceof NoMatrix2AuthorizationService ||
|
||||
// [lotus] carries the token service's own refusal reason — keep it.
|
||||
e instanceof SFUTokenRefusedError
|
||||
) {
|
||||
// rethrow as is
|
||||
return e;
|
||||
|
||||
@@ -23,6 +23,8 @@ export enum ErrorCode {
|
||||
E2EE_NOT_SUPPORTED = "E2EE_NOT_SUPPORTED",
|
||||
STICKY_EVENTS_NOT_SUPPORTED = "STICKY_EVENTS_NOT_SUPPORTED",
|
||||
OPEN_ID_ERROR = "OPEN_ID_ERROR",
|
||||
/** [lotus] The SFU token service refused us with a human-readable reason (e.g. the voice-limit guard: channel full / no permission). */
|
||||
SFU_TOKEN_REFUSED = "SFU_TOKEN_REFUSED",
|
||||
NO_MATRIX_2_AUTHORIZATION_SERVICE = "NO_MATRIX_2_0_AUTHORIZATION_SERVICE",
|
||||
SFU_ERROR = "SFU_ERROR",
|
||||
UNKNOWN_ERROR = "UNKNOWN_ERROR",
|
||||
@@ -234,6 +236,24 @@ export class FailToStartLivekitConnection extends ElementCallError {
|
||||
/**
|
||||
* Error indicating that a LiveKit's server has hit its track limits.
|
||||
*/
|
||||
/**
|
||||
* [lotus] The SFU token service answered 403 with a reason we can show verbatim
|
||||
* — the voice-limit guard says things like "This voice channel is full." or
|
||||
* "You don't have permission to share your screen here." Without this the user
|
||||
* only ever saw "Something went wrong (OPEN_ID_ERROR)".
|
||||
*/
|
||||
export class SFUTokenRefusedError extends ElementCallError {
|
||||
public constructor(reason: string, cause?: Error) {
|
||||
super(
|
||||
t("error.sfu_token_refused"),
|
||||
ErrorCode.SFU_TOKEN_REFUSED,
|
||||
ErrorCategory.CONFIGURATION_ISSUE,
|
||||
reason,
|
||||
cause,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
export class InsufficientCapacityError extends ElementCallError {
|
||||
public constructor() {
|
||||
super(
|
||||
|
||||
Reference in New Issue
Block a user