From a366243836247f4a81ca323f8604eee5a0d6d2ce Mon Sep 17 00:00:00 2001 From: Lotus CI Date: Sat, 24 Oct 2026 19:42:23 -0400 Subject: [PATCH] feat(lotus): expected-device check before joining (cinny #121) EC remembers the picked mic / output by id and silently falls back to the first device when it is missing, so a user who always joins with a headset lands on the laptop mic. With lotusDeviceCheck=1: - the last-picked device's NAME is remembered next to its id (lotus-audio-{input,output}-name), since ids can change on re-plug; - before the automatic join, a usual device that is gone is reported to the host (io.lotus.device_missing) and EC waits for io.lotus.device_decision { join }; one that came back under a new id is just re-selected; - in a call, losing it is reported (io.lotus.device_lost) for a toast. Fails open (no names / no host / nothing remembered). Without the flag the join stays synchronous, as upstream. The gate runs once per call: CallView's effect re-runs while its inputs settle and must not restart it. Embedded version 0.28.0-lotus.2. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_017PCb59uY3UKqDqt7B6hzd5 --- embedded/web/package.json | 2 +- src/lotus/lotusActions.test.ts | 1 + src/lotus/lotusActions.ts | 15 +++ src/lotus/lotusDevices.test.ts | 188 ++++++++++++++++++++++++++ src/lotus/lotusDevices.ts | 236 +++++++++++++++++++++++++++++++++ src/room/CallView.tsx | 20 ++- src/room/InCallView.tsx | 3 + 7 files changed, 463 insertions(+), 2 deletions(-) create mode 100644 src/lotus/lotusDevices.test.ts create mode 100644 src/lotus/lotusDevices.ts diff --git a/embedded/web/package.json b/embedded/web/package.json index a22af56f..7c791233 100644 --- a/embedded/web/package.json +++ b/embedded/web/package.json @@ -1,6 +1,6 @@ { "name": "@lotusguild/element-call-embedded", - "version": "0.28.0-lotus.1", + "version": "0.28.0-lotus.2", "files": [ "README.md", "LICENSE-AGPL-3.0", diff --git a/src/lotus/lotusActions.test.ts b/src/lotus/lotusActions.test.ts index 25216b9a..a0dc93bb 100644 --- a/src/lotus/lotusActions.test.ts +++ b/src/lotus/lotusActions.test.ts @@ -35,6 +35,7 @@ describe("LotusWidgetActions", () => { LotusWidgetActions.SetFrameScreenshare, LotusWidgetActions.PromptScreenshare, LotusWidgetActions.Reconnect, + LotusWidgetActions.DeviceDecision, ]; expect(new Set(LOTUS_TO_WIDGET_ACTIONS)).toEqual(new Set(expectedToWidget)); diff --git a/src/lotus/lotusActions.ts b/src/lotus/lotusActions.ts index 65b6875d..75cf559b 100644 --- a/src/lotus/lotusActions.ts +++ b/src/lotus/lotusActions.ts @@ -130,6 +130,20 @@ export enum LotusWidgetActions { CallError = "io.lotus.call_error", /** toWidget: press the error page's Reconnect / Try again (cinny #318). */ Reconnect = "io.lotus.reconnect", + /** + * fromWidget: before the automatic join (`lotusDeviceCheck=1`), the user's + * usual mic / output is missing — `{ missing: [{ kind: "audioinput" | + * "audiooutput", expected: name, fallback: name | null }] }`. EC waits for + * `DeviceDecision` (cinny #121). + */ + DeviceMissing = "io.lotus.device_missing", + /** toWidget: the user's answer, `{ join: boolean }` (cinny #121). */ + DeviceDecision = "io.lotus.device_decision", + /** + * fromWidget: in a call, the usual device disappeared — `{ kind, expected, + * fallback }` — for a toast (cinny #121). + */ + DeviceLost = "io.lotus.device_lost", } /** toWidget Lotus actions that must be allow-listed in `initializeWidget`. */ @@ -148,4 +162,5 @@ export const LOTUS_TO_WIDGET_ACTIONS: LotusWidgetActions[] = [ LotusWidgetActions.SetFrameScreenshare, LotusWidgetActions.PromptScreenshare, LotusWidgetActions.Reconnect, + LotusWidgetActions.DeviceDecision, ]; diff --git a/src/lotus/lotusDevices.test.ts b/src/lotus/lotusDevices.test.ts new file mode 100644 index 00000000..54c3537c --- /dev/null +++ b/src/lotus/lotusDevices.test.ts @@ -0,0 +1,188 @@ +/* +Copyright 2026 Lotus Guild + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { BehaviorSubject } from "rxjs"; +import { afterEach, describe, expect, it, vi } from "vitest"; + +const sent: { action: string; data: unknown }[] = []; +const handlers = new Map void>(); +vi.mock("../widget", () => ({ + widget: { + api: { transport: { reply: vi.fn() } }, + lazyActions: { + on: (a: string, h: (ev: CustomEvent) => void): void => + void handlers.set(a, h), + off: (a: string): void => void handlers.delete(a), + }, + }, +})); +vi.mock("./lotusWidget", () => ({ + lotusFlag: (name: string): boolean => name === "lotusDeviceCheck", + lotusSendToHost: (action: string, data: unknown): boolean => { + sent.push({ action, data }); + return true; + }, +})); + +const { + checkDevice, + lotusDeviceGate, + startLotusDevices, + audioInputName, + audioOutputName, +} = await import("./lotusDevices"); +const { audioInput, audioOutput } = await import("../settings/settings"); + +const named = ( + ...names: [string, string][] +): Map => + new Map(names.map(([id, name]) => [id, { type: "name", name }])); + +type Labels = Map; +const source = ( + labels: Labels, +): { available$: BehaviorSubject; select: (id: string) => void } => ({ + available$: new BehaviorSubject(labels), + select: vi.fn<(id: string) => void>(), +}); +const devices = ( + inputs: Labels, + outputs: Labels = named(["", "Default"]), +): { + audioInput: ReturnType; + audioOutput: ReturnType; +} => ({ audioInput: source(inputs), audioOutput: source(outputs) }); + +afterEach(() => { + sent.length = 0; + handlers.clear(); + audioInput.setValue(undefined); + audioOutput.setValue(undefined); + audioInputName.setValue(undefined); + audioOutputName.setValue(undefined); +}); + +describe("[cinny #121] checkDevice", () => { + const avail = named(["laptop", "Laptop Mic"], ["arctis2", "Arctis 7"]); + it("is fine when the usual device is there, or nothing is remembered", () => { + expect(checkDevice("audioinput", "arctis2", "Arctis 7", avail).result).toBe( + "ok", + ); + expect(checkDevice("audioinput", undefined, undefined, avail).result).toBe( + "ok", + ); + expect(checkDevice("audioinput", "default", "x", avail).result).toBe("ok"); + }); + it("re-selects a device that came back under a new id", () => { + expect(checkDevice("audioinput", "arctis1", "Arctis 7", avail)).toEqual({ + kind: "audioinput", + result: "reselect", + id: "arctis2", + }); + }); + it("reports a missing device and what EC would use instead", () => { + expect( + checkDevice( + "audioinput", + "arctis1", + "Arctis 7", + named(["laptop", "Laptop Mic"]), + ), + ).toEqual({ + kind: "audioinput", + result: "missing", + expected: "Arctis 7", + fallback: "Laptop Mic", + }); + }); + it("fails open without device names (no permission yet)", () => { + const unnamed = new Map([["x", { type: "number" }]]); + expect( + checkDevice("audioinput", "arctis1", "Arctis 7", unnamed).result, + ).toBe("ok"); + }); +}); + +describe("[cinny #121] lotusDeviceGate", () => { + it("joins straight away when everything is there", async () => { + audioInput.setValue("arctis"); + audioInputName.setValue("Arctis 7"); + await expect( + lotusDeviceGate(devices(named(["arctis", "Arctis 7"]))), + ).resolves.toBe(true); + expect(sent).toEqual([]); + }); + + it("asks the host and waits for the answer", async () => { + audioInput.setValue("arctis"); + audioInputName.setValue("Arctis 7"); + const gate = lotusDeviceGate(devices(named(["laptop", "Laptop Mic"]))); + await vi.waitFor(() => expect(sent).toHaveLength(1)); + expect(sent[0]).toEqual({ + action: "io.lotus.device_missing", + data: { + missing: [ + { kind: "audioinput", expected: "Arctis 7", fallback: "Laptop Mic" }, + ], + }, + }); + handlers.get("io.lotus.device_decision")!( + new CustomEvent("x", { detail: { data: { join: false } } }), + ); + await expect(gate).resolves.toBe(false); + }); + + it("re-selects a re-plugged device without asking", async () => { + audioInput.setValue("old-id"); + audioInputName.setValue("Arctis 7"); + const d = devices(named(["new-id", "Arctis 7"])); + await expect(lotusDeviceGate(d)).resolves.toBe(true); + expect(d.audioInput.select).toHaveBeenCalledWith("new-id"); + expect(sent).toEqual([]); + }); + + it("joins when device names never arrive", async () => { + audioInput.setValue("arctis"); + audioInputName.setValue("Arctis 7"); + const d = devices( + new Map([["x", { type: "number" }]]), + new Map([["y", { type: "number" }]]), + ); + await expect(lotusDeviceGate(d, 50)).resolves.toBe(true); + }); +}); + +describe("[cinny #121] startLotusDevices", () => { + it("remembers the picked device's name and reports when it goes", () => { + const d = devices(named(["laptop", "Laptop Mic"], ["arctis", "Arctis 7"])); + const stop = startLotusDevices(d); + audioInput.setValue("arctis"); + expect(audioInputName.getValue()).toBe("Arctis 7"); + d.audioInput.available$.next(named(["laptop", "Laptop Mic"])); + expect(sent).toEqual([ + { + action: "io.lotus.device_lost", + data: { + kind: "audioinput", + expected: "Arctis 7", + fallback: "Laptop Mic", + }, + }, + ]); + // Name kept, so the next join can still recognise it. + expect(audioInputName.getValue()).toBe("Arctis 7"); + stop(); + }); + + it("choosing the system default clears the remembered name", () => { + audioInputName.setValue("Arctis 7"); + const stop = startLotusDevices(devices(named(["", "Default"]))); + audioInput.setValue(""); + expect(audioInputName.getValue()).toBeUndefined(); + stop(); + }); +}); diff --git a/src/lotus/lotusDevices.ts b/src/lotus/lotusDevices.ts new file mode 100644 index 00000000..d5065cd4 --- /dev/null +++ b/src/lotus/lotusDevices.ts @@ -0,0 +1,236 @@ +/* +Copyright 2026 Lotus Guild + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { logger } from "matrix-js-sdk/lib/logger"; +import { type IWidgetApiRequest } from "matrix-widget-api"; +import { + combineLatest, + filter, + firstValueFrom, + map, + type Observable, + timeout, +} from "rxjs"; + +import { + audioInput as audioInputSetting, + audioOutput as audioOutputSetting, + Setting, +} from "../settings/settings"; +import { widget } from "../widget"; +import { LotusWidgetActions } from "./lotusActions"; +import { lotusFlag, lotusSendToHost } from "./lotusWidget"; + +/** + * [cinny #121] Expected-device check. EC remembers the chosen mic / output by + * id and, when that device is missing, silently uses the first one it finds, + * so a user who always joins with a headset ends up on the laptop mic. + * + * - "Usual" = the device the user last picked (EC's own audio-input / + * audio-output settings). Its NAME is remembered alongside (ids can change + * when a device is re-plugged on Windows). + * - Before the automatic join (`lotusDeviceCheck=1`), a missing usual device + * is reported to the host (`io.lotus.device_missing`), which asks the user + * and answers `io.lotus.device_decision { join }`. A device that came back + * under a new id but the same name is just re-selected. + * - In a call, losing the usual device is reported (`io.lotus.device_lost`) + * for a toast; no question mid-call. + * Fails open: no names (no permission yet), no host, or nothing remembered + * means no check. + */ + +export type DeviceKind = "audioinput" | "audiooutput"; + +/** Any device label with an optional human name (EC's label unions). */ +type AnyLabel = { type: string; name?: string | null }; + +export type DeviceCheck = + | { kind: DeviceKind; result: "ok" } + | { kind: DeviceKind; result: "reselect"; id: string } + | { + kind: DeviceKind; + result: "missing"; + expected: string; + fallback: string | null; + }; + +const nameOf = (label: AnyLabel | undefined): string | null => + label && typeof label.name === "string" && label.name ? label.name : null; + +/** Whether the device list carries real names (permission granted). */ +export const hasNames = (available: Map): boolean => + [...available.values()].some((l) => l.type === "name"); + +/** The id EC is going to (or does) use as the "system default" choice. */ +const isDefaultId = (id: string): boolean => id === "" || id === "default"; + +/** + * Pure: compare the remembered device with what is plugged in now. + * `fallback` names the device EC falls back to (the first in the list, as + * MediaDevices' selectDevice$ does). + */ +export function checkDevice( + kind: DeviceKind, + preferredId: string | undefined, + expectedName: string | undefined, + available: Map, +): DeviceCheck { + if (available.size === 0 || !hasNames(available)) + return { kind, result: "ok" }; + if (preferredId === undefined || isDefaultId(preferredId)) + return { kind, result: "ok" }; + if (available.has(preferredId)) return { kind, result: "ok" }; + if (!expectedName) return { kind, result: "ok" }; + for (const [id, label] of available) { + if (nameOf(label) === expectedName) return { kind, result: "reselect", id }; + } + const first = available.values().next().value; + return { + kind, + result: "missing", + expected: expectedName, + fallback: nameOf(first), + }; +} + +/** The remembered NAME of the last-picked device of each kind. */ +export const audioInputName = new Setting( + "lotus-audio-input-name", + undefined, +); +export const audioOutputName = new Setting( + "lotus-audio-output-name", + undefined, +); + +interface DeviceSource { + available$: Observable>; + select(id: string): void; +} +export interface LotusMediaDevices { + audioInput: DeviceSource; + audioOutput: DeviceSource; +} + +const kinds = ( + devices: LotusMediaDevices, +): { + kind: DeviceKind; + source: DeviceSource; + preferred: Setting; + name: Setting; +}[] => [ + { + kind: "audioinput", + source: devices.audioInput, + preferred: audioInputSetting, + name: audioInputName, + }, + { + kind: "audiooutput", + source: devices.audioOutput, + preferred: audioOutputSetting, + name: audioOutputName, + }, +]; + +/** Whether the host asked for the check (a Lotus host with the flag). */ +export const lotusDeviceCheckWanted = (): boolean => + !!widget && lotusFlag("lotusDeviceCheck"); + +/** + * Before EC's automatic join: true = join now, false = the user cancelled. + * Waits up to `namesTimeoutMs` for named device lists, then checks both kinds. + */ +export async function lotusDeviceGate( + devices: LotusMediaDevices, + namesTimeoutMs = 3000, +): Promise { + const w = widget; + if (!w || !lotusDeviceCheckWanted()) return true; + const entries = kinds(devices); + let lists: Map[]; + try { + lists = await firstValueFrom( + combineLatest(entries.map((e) => e.source.available$)).pipe( + filter((all) => all.some(hasNames)), + timeout(namesTimeoutMs), + ), + ); + } catch { + return true; // no names (yet): nothing to compare by + } + const missing: Extract[] = []; + entries.forEach((e, i) => { + const check = checkDevice( + e.kind, + e.preferred.getValue(), + e.name.getValue(), + lists[i], + ); + if (check.result === "reselect") { + logger.info(`[lotus] ${e.kind}: usual device is back under a new id`); + e.source.select(check.id); + } else if (check.result === "missing") missing.push(check); + }); + if (missing.length === 0) return true; + + return new Promise((resolve) => { + const onDecision = (ev: CustomEvent): void => { + w.api.transport.reply(ev.detail, {}); + w.lazyActions.off(LotusWidgetActions.DeviceDecision, onDecision); + resolve((ev.detail.data as { join?: unknown }).join !== false); + }; + w.lazyActions.on(LotusWidgetActions.DeviceDecision, onDecision); + const sent = lotusSendToHost(LotusWidgetActions.DeviceMissing, { + missing: missing.map(({ kind, expected, fallback }) => ({ + kind, + expected, + fallback, + })), + }); + if (!sent) { + w.lazyActions.off(LotusWidgetActions.DeviceDecision, onDecision); + resolve(true); + } + }); +} + +/** + * In a call: remember the name of every device the user picks, and report + * when the picked device disappears. Returns a teardown function. + */ +export function startLotusDevices(devices: LotusMediaDevices): () => void { + const subs = kinds(devices).map((e) => { + let wasPresent: boolean | undefined; + return combineLatest([e.preferred.value$, e.source.available$]) + .pipe(map(([id, available]) => ({ id, available }))) + .subscribe(({ id, available }) => { + if (id === undefined || isDefaultId(id) || !hasNames(available)) { + if (id !== undefined && isDefaultId(id)) e.name.setValue(undefined); + wasPresent = undefined; + return; + } + const label = available.get(id); + if (label) { + const name = nameOf(label); + if (name && e.name.getValue() !== name) e.name.setValue(name); + wasPresent = true; + return; + } + if (wasPresent && e.name.getValue()) { + lotusSendToHost(LotusWidgetActions.DeviceLost, { + kind: e.kind, + expected: e.name.getValue(), + fallback: nameOf(available.values().next().value), + }); + } + wasPresent = false; + }); + }); + return (): void => subs.forEach((s) => s.unsubscribe()); +} diff --git a/src/room/CallView.tsx b/src/room/CallView.tsx index 3c34dc84..60150469 100644 --- a/src/room/CallView.tsx +++ b/src/room/CallView.tsx @@ -10,6 +10,7 @@ import { type ReactNode, useCallback, useEffect, + useRef, useMemo, useState, } from "react"; @@ -39,6 +40,7 @@ import { findDeviceByName } from "../utils/media"; import { ActiveCall } from "./InCallView"; import { type MuteStates } from "../state/MuteStates"; import { useMediaDevices } from "../MediaDevicesContext"; +import { lotusDeviceCheckWanted, lotusDeviceGate } from "../lotus/lotusDevices"; import { useMatrixRTCSessionMemberships } from "../useMatrixRTCSessionMemberships"; import { saveKeyForRoom, @@ -277,6 +279,7 @@ const LoadedCallView: FC = ({ ); const mediaDevices = useMediaDevices(); + const lotusGate = useRef | null>(null); const latestMuteStates = useLatest(muteStates); const enterRTCSessionOrError = useCallback( @@ -357,7 +360,22 @@ const LoadedCallView: FC = ({ return (): void => subscription.unsubscribe(); } else { // No lobby and no preload: we enter the rtc session right away - setJoined(true); + // ([lotus #121] after checking the usual mic / output are present when + // the Lotus host asked for it; it may ask the user and cancel). + if (!lotusDeviceCheckWanted()) { + setJoined(true); + return; + } + // Started once per call: this effect re-runs as its inputs settle, + // and restarting the check each time could hold the join forever. + lotusGate.current ??= lotusDeviceGate(mediaDevices); + let cancelled = false; + void lotusGate.current.then((join) => { + if (!cancelled && join) setJoined(true); + }); + return (): void => { + cancelled = true; + }; } } }, [ diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index d15f3cb6..69dc6ab7 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -40,6 +40,7 @@ import { startLotusCallSummary } from "../lotus/lotusCallSummary"; import { startLotusAudioOutput } from "../lotus/lotusAudioOutput"; import { startLotusScreenshareWatch } from "../lotus/lotusScreenshareWatch"; import { startLotusDeafen } from "../lotus/lotusDeafen"; +import { startLotusDevices } from "../lotus/lotusDevices"; import { useHostBridge } from "../HostBridge.ts"; import { useRootElement } from "../RootElementContext"; import { observeElementSize$ } from "../utils/elementSize"; @@ -361,6 +362,8 @@ export const InCallView: FC = ({ useEffect(() => startLotusDeafen(), []); // [lotus #119] Let the host pick the audio output device from its call bar. const lotusMediaDevices = useMediaDevices(); + // [cinny #121] Remember the picked mic / output by name; report if it goes. + useEffect(() => startLotusDevices(lotusMediaDevices), [lotusMediaDevices]); useEffect( () => startLotusAudioOutput(lotusMediaDevices), [lotusMediaDevices],