Merge pull request 'feat(lotus): expected-device check before joining (cinny #121)' (#62) from feat/expected-devices into lotus

embedded/web/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lotusguild/element-call-embedded",
-  "version": "0.28.0-lotus.1",
+  "version": "0.28.0-lotus.2",
   "files": [
     "README.md",
     "LICENSE-AGPL-3.0",

src/lotus/lotusActions.test.ts

@@ -35,6 +35,7 @@ describe("LotusWidgetActions", () => {
       LotusWidgetActions.SetFrameScreenshare,
       LotusWidgetActions.PromptScreenshare,
       LotusWidgetActions.Reconnect,
+      LotusWidgetActions.DeviceDecision,
     ];
 
     expect(new Set(LOTUS_TO_WIDGET_ACTIONS)).toEqual(new Set(expectedToWidget));

src/lotus/lotusActions.ts

@@ -130,6 +130,20 @@ export enum LotusWidgetActions {
   CallError = "io.lotus.call_error",
   /** toWidget: press the error page's Reconnect / Try again (cinny #318). */
   Reconnect = "io.lotus.reconnect",
+  /**
+   * fromWidget: before the automatic join (`lotusDeviceCheck=1`), the user's
+   * usual mic / output is missing — `{ missing: [{ kind: "audioinput" |
+   * "audiooutput", expected: name, fallback: name | null }] }`. EC waits for
+   * `DeviceDecision` (cinny #121).
+   */
+  DeviceMissing = "io.lotus.device_missing",
+  /** toWidget: the user's answer, `{ join: boolean }` (cinny #121). */
+  DeviceDecision = "io.lotus.device_decision",
+  /**
+   * fromWidget: in a call, the usual device disappeared — `{ kind, expected,
+   * fallback }` — for a toast (cinny #121).
+   */
+  DeviceLost = "io.lotus.device_lost",
 }
 
 /** toWidget Lotus actions that must be allow-listed in `initializeWidget`. */
@@ -148,4 +162,5 @@ export const LOTUS_TO_WIDGET_ACTIONS: LotusWidgetActions[] = [
   LotusWidgetActions.SetFrameScreenshare,
   LotusWidgetActions.PromptScreenshare,
   LotusWidgetActions.Reconnect,
+  LotusWidgetActions.DeviceDecision,
 ];

src/lotus/lotusDevices.test.ts

@@ -0,0 +1,188 @@
+/*
+Copyright 2026 Lotus Guild
+
+SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
+Please see LICENSE in the repository root for full details.
+*/
+
+import { BehaviorSubject } from "rxjs";
+import { afterEach, describe, expect, it, vi } from "vitest";
+
+const sent: { action: string; data: unknown }[] = [];
+const handlers = new Map<string, (ev: CustomEvent) => void>();
+vi.mock("../widget", () => ({
+  widget: {
+    api: { transport: { reply: vi.fn() } },
+    lazyActions: {
+      on: (a: string, h: (ev: CustomEvent) => void): void =>
+        void handlers.set(a, h),
+      off: (a: string): void => void handlers.delete(a),
+    },
+  },
+}));
+vi.mock("./lotusWidget", () => ({
+  lotusFlag: (name: string): boolean => name === "lotusDeviceCheck",
+  lotusSendToHost: (action: string, data: unknown): boolean => {
+    sent.push({ action, data });
+    return true;
+  },
+}));
+
+const {
+  checkDevice,
+  lotusDeviceGate,
+  startLotusDevices,
+  audioInputName,
+  audioOutputName,
+} = await import("./lotusDevices");
+const { audioInput, audioOutput } = await import("../settings/settings");
+
+const named = (
+  ...names: [string, string][]
+): Map<string, { type: string; name?: string }> =>
+  new Map(names.map(([id, name]) => [id, { type: "name", name }]));
+
+type Labels = Map<string, { type: string; name?: string }>;
+const source = (
+  labels: Labels,
+): { available$: BehaviorSubject<Labels>; select: (id: string) => void } => ({
+  available$: new BehaviorSubject(labels),
+  select: vi.fn<(id: string) => void>(),
+});
+const devices = (
+  inputs: Labels,
+  outputs: Labels = named(["", "Default"]),
+): {
+  audioInput: ReturnType<typeof source>;
+  audioOutput: ReturnType<typeof source>;
+} => ({ audioInput: source(inputs), audioOutput: source(outputs) });
+
+afterEach(() => {
+  sent.length = 0;
+  handlers.clear();
+  audioInput.setValue(undefined);
+  audioOutput.setValue(undefined);
+  audioInputName.setValue(undefined);
+  audioOutputName.setValue(undefined);
+});
+
+describe("[cinny #121] checkDevice", () => {
+  const avail = named(["laptop", "Laptop Mic"], ["arctis2", "Arctis 7"]);
+  it("is fine when the usual device is there, or nothing is remembered", () => {
+    expect(checkDevice("audioinput", "arctis2", "Arctis 7", avail).result).toBe(
+      "ok",
+    );
+    expect(checkDevice("audioinput", undefined, undefined, avail).result).toBe(
+      "ok",
+    );
+    expect(checkDevice("audioinput", "default", "x", avail).result).toBe("ok");
+  });
+  it("re-selects a device that came back under a new id", () => {
+    expect(checkDevice("audioinput", "arctis1", "Arctis 7", avail)).toEqual({
+      kind: "audioinput",
+      result: "reselect",
+      id: "arctis2",
+    });
+  });
+  it("reports a missing device and what EC would use instead", () => {
+    expect(
+      checkDevice(
+        "audioinput",
+        "arctis1",
+        "Arctis 7",
+        named(["laptop", "Laptop Mic"]),
+      ),
+    ).toEqual({
+      kind: "audioinput",
+      result: "missing",
+      expected: "Arctis 7",
+      fallback: "Laptop Mic",
+    });
+  });
+  it("fails open without device names (no permission yet)", () => {
+    const unnamed = new Map([["x", { type: "number" }]]);
+    expect(
+      checkDevice("audioinput", "arctis1", "Arctis 7", unnamed).result,
+    ).toBe("ok");
+  });
+});
+
+describe("[cinny #121] lotusDeviceGate", () => {
+  it("joins straight away when everything is there", async () => {
+    audioInput.setValue("arctis");
+    audioInputName.setValue("Arctis 7");
+    await expect(
+      lotusDeviceGate(devices(named(["arctis", "Arctis 7"]))),
+    ).resolves.toBe(true);
+    expect(sent).toEqual([]);
+  });
+
+  it("asks the host and waits for the answer", async () => {
+    audioInput.setValue("arctis");
+    audioInputName.setValue("Arctis 7");
+    const gate = lotusDeviceGate(devices(named(["laptop", "Laptop Mic"])));
+    await vi.waitFor(() => expect(sent).toHaveLength(1));
+    expect(sent[0]).toEqual({
+      action: "io.lotus.device_missing",
+      data: {
+        missing: [
+          { kind: "audioinput", expected: "Arctis 7", fallback: "Laptop Mic" },
+        ],
+      },
+    });
+    handlers.get("io.lotus.device_decision")!(
+      new CustomEvent("x", { detail: { data: { join: false } } }),
+    );
+    await expect(gate).resolves.toBe(false);
+  });
+
+  it("re-selects a re-plugged device without asking", async () => {
+    audioInput.setValue("old-id");
+    audioInputName.setValue("Arctis 7");
+    const d = devices(named(["new-id", "Arctis 7"]));
+    await expect(lotusDeviceGate(d)).resolves.toBe(true);
+    expect(d.audioInput.select).toHaveBeenCalledWith("new-id");
+    expect(sent).toEqual([]);
+  });
+
+  it("joins when device names never arrive", async () => {
+    audioInput.setValue("arctis");
+    audioInputName.setValue("Arctis 7");
+    const d = devices(
+      new Map([["x", { type: "number" }]]),
+      new Map([["y", { type: "number" }]]),
+    );
+    await expect(lotusDeviceGate(d, 50)).resolves.toBe(true);
+  });
+});
+
+describe("[cinny #121] startLotusDevices", () => {
+  it("remembers the picked device's name and reports when it goes", () => {
+    const d = devices(named(["laptop", "Laptop Mic"], ["arctis", "Arctis 7"]));
+    const stop = startLotusDevices(d);
+    audioInput.setValue("arctis");
+    expect(audioInputName.getValue()).toBe("Arctis 7");
+    d.audioInput.available$.next(named(["laptop", "Laptop Mic"]));
+    expect(sent).toEqual([
+      {
+        action: "io.lotus.device_lost",
+        data: {
+          kind: "audioinput",
+          expected: "Arctis 7",
+          fallback: "Laptop Mic",
+        },
+      },
+    ]);
+    // Name kept, so the next join can still recognise it.
+    expect(audioInputName.getValue()).toBe("Arctis 7");
+    stop();
+  });
+
+  it("choosing the system default clears the remembered name", () => {
+    audioInputName.setValue("Arctis 7");
+    const stop = startLotusDevices(devices(named(["", "Default"])));
+    audioInput.setValue("");
+    expect(audioInputName.getValue()).toBeUndefined();
+    stop();
+  });
+});

src/lotus/lotusDevices.ts

@@ -0,0 +1,236 @@
+/*
+Copyright 2026 Lotus Guild
+
+SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
+Please see LICENSE in the repository root for full details.
+*/
+
+import { logger } from "matrix-js-sdk/lib/logger";
+import { type IWidgetApiRequest } from "matrix-widget-api";
+import {
+  combineLatest,
+  filter,
+  firstValueFrom,
+  map,
+  type Observable,
+  timeout,
+} from "rxjs";
+
+import {
+  audioInput as audioInputSetting,
+  audioOutput as audioOutputSetting,
+  Setting,
+} from "../settings/settings";
+import { widget } from "../widget";
+import { LotusWidgetActions } from "./lotusActions";
+import { lotusFlag, lotusSendToHost } from "./lotusWidget";
+
+/**
+ * [cinny #121] Expected-device check. EC remembers the chosen mic / output by
+ * id and, when that device is missing, silently uses the first one it finds,
+ * so a user who always joins with a headset ends up on the laptop mic.
+ *
+ * - "Usual" = the device the user last picked (EC's own audio-input /
+ *   audio-output settings). Its NAME is remembered alongside (ids can change
+ *   when a device is re-plugged on Windows).
+ * - Before the automatic join (`lotusDeviceCheck=1`), a missing usual device
+ *   is reported to the host (`io.lotus.device_missing`), which asks the user
+ *   and answers `io.lotus.device_decision { join }`. A device that came back
+ *   under a new id but the same name is just re-selected.
+ * - In a call, losing the usual device is reported (`io.lotus.device_lost`)
+ *   for a toast; no question mid-call.
+ * Fails open: no names (no permission yet), no host, or nothing remembered
+ * means no check.
+ */
+
+export type DeviceKind = "audioinput" | "audiooutput";
+
+/** Any device label with an optional human name (EC's label unions). */
+type AnyLabel = { type: string; name?: string | null };
+
+export type DeviceCheck =
+  | { kind: DeviceKind; result: "ok" }
+  | { kind: DeviceKind; result: "reselect"; id: string }
+  | {
+      kind: DeviceKind;
+      result: "missing";
+      expected: string;
+      fallback: string | null;
+    };
+
+const nameOf = (label: AnyLabel | undefined): string | null =>
+  label && typeof label.name === "string" && label.name ? label.name : null;
+
+/** Whether the device list carries real names (permission granted). */
+export const hasNames = (available: Map<string, AnyLabel>): boolean =>
+  [...available.values()].some((l) => l.type === "name");
+
+/** The id EC is going to (or does) use as the "system default" choice. */
+const isDefaultId = (id: string): boolean => id === "" || id === "default";
+
+/**
+ * Pure: compare the remembered device with what is plugged in now.
+ * `fallback` names the device EC falls back to (the first in the list, as
+ * MediaDevices' selectDevice$ does).
+ */
+export function checkDevice(
+  kind: DeviceKind,
+  preferredId: string | undefined,
+  expectedName: string | undefined,
+  available: Map<string, AnyLabel>,
+): DeviceCheck {
+  if (available.size === 0 || !hasNames(available))
+    return { kind, result: "ok" };
+  if (preferredId === undefined || isDefaultId(preferredId))
+    return { kind, result: "ok" };
+  if (available.has(preferredId)) return { kind, result: "ok" };
+  if (!expectedName) return { kind, result: "ok" };
+  for (const [id, label] of available) {
+    if (nameOf(label) === expectedName) return { kind, result: "reselect", id };
+  }
+  const first = available.values().next().value;
+  return {
+    kind,
+    result: "missing",
+    expected: expectedName,
+    fallback: nameOf(first),
+  };
+}
+
+/** The remembered NAME of the last-picked device of each kind. */
+export const audioInputName = new Setting<string | undefined>(
+  "lotus-audio-input-name",
+  undefined,
+);
+export const audioOutputName = new Setting<string | undefined>(
+  "lotus-audio-output-name",
+  undefined,
+);
+
+interface DeviceSource {
+  available$: Observable<Map<string, AnyLabel>>;
+  select(id: string): void;
+}
+export interface LotusMediaDevices {
+  audioInput: DeviceSource;
+  audioOutput: DeviceSource;
+}
+
+const kinds = (
+  devices: LotusMediaDevices,
+): {
+  kind: DeviceKind;
+  source: DeviceSource;
+  preferred: Setting<string | undefined>;
+  name: Setting<string | undefined>;
+}[] => [
+  {
+    kind: "audioinput",
+    source: devices.audioInput,
+    preferred: audioInputSetting,
+    name: audioInputName,
+  },
+  {
+    kind: "audiooutput",
+    source: devices.audioOutput,
+    preferred: audioOutputSetting,
+    name: audioOutputName,
+  },
+];
+
+/** Whether the host asked for the check (a Lotus host with the flag). */
+export const lotusDeviceCheckWanted = (): boolean =>
+  !!widget && lotusFlag("lotusDeviceCheck");
+
+/**
+ * Before EC's automatic join: true = join now, false = the user cancelled.
+ * Waits up to `namesTimeoutMs` for named device lists, then checks both kinds.
+ */
+export async function lotusDeviceGate(
+  devices: LotusMediaDevices,
+  namesTimeoutMs = 3000,
+): Promise<boolean> {
+  const w = widget;
+  if (!w || !lotusDeviceCheckWanted()) return true;
+  const entries = kinds(devices);
+  let lists: Map<string, AnyLabel>[];
+  try {
+    lists = await firstValueFrom(
+      combineLatest(entries.map((e) => e.source.available$)).pipe(
+        filter((all) => all.some(hasNames)),
+        timeout(namesTimeoutMs),
+      ),
+    );
+  } catch {
+    return true; // no names (yet): nothing to compare by
+  }
+  const missing: Extract<DeviceCheck, { result: "missing" }>[] = [];
+  entries.forEach((e, i) => {
+    const check = checkDevice(
+      e.kind,
+      e.preferred.getValue(),
+      e.name.getValue(),
+      lists[i],
+    );
+    if (check.result === "reselect") {
+      logger.info(`[lotus] ${e.kind}: usual device is back under a new id`);
+      e.source.select(check.id);
+    } else if (check.result === "missing") missing.push(check);
+  });
+  if (missing.length === 0) return true;
+
+  return new Promise<boolean>((resolve) => {
+    const onDecision = (ev: CustomEvent<IWidgetApiRequest>): void => {
+      w.api.transport.reply(ev.detail, {});
+      w.lazyActions.off(LotusWidgetActions.DeviceDecision, onDecision);
+      resolve((ev.detail.data as { join?: unknown }).join !== false);
+    };
+    w.lazyActions.on(LotusWidgetActions.DeviceDecision, onDecision);
+    const sent = lotusSendToHost(LotusWidgetActions.DeviceMissing, {
+      missing: missing.map(({ kind, expected, fallback }) => ({
+        kind,
+        expected,
+        fallback,
+      })),
+    });
+    if (!sent) {
+      w.lazyActions.off(LotusWidgetActions.DeviceDecision, onDecision);
+      resolve(true);
+    }
+  });
+}
+
+/**
+ * In a call: remember the name of every device the user picks, and report
+ * when the picked device disappears. Returns a teardown function.
+ */
+export function startLotusDevices(devices: LotusMediaDevices): () => void {
+  const subs = kinds(devices).map((e) => {
+    let wasPresent: boolean | undefined;
+    return combineLatest([e.preferred.value$, e.source.available$])
+      .pipe(map(([id, available]) => ({ id, available })))
+      .subscribe(({ id, available }) => {
+        if (id === undefined || isDefaultId(id) || !hasNames(available)) {
+          if (id !== undefined && isDefaultId(id)) e.name.setValue(undefined);
+          wasPresent = undefined;
+          return;
+        }
+        const label = available.get(id);
+        if (label) {
+          const name = nameOf(label);
+          if (name && e.name.getValue() !== name) e.name.setValue(name);
+          wasPresent = true;
+          return;
+        }
+        if (wasPresent && e.name.getValue()) {
+          lotusSendToHost(LotusWidgetActions.DeviceLost, {
+            kind: e.kind,
+            expected: e.name.getValue(),
+            fallback: nameOf(available.values().next().value),
+          });
+        }
+        wasPresent = false;
+      });
+  });
+  return (): void => subs.forEach((s) => s.unsubscribe());
+}

src/room/CallView.tsx

@@ -10,6 +10,7 @@ import {
   type ReactNode,
   useCallback,
   useEffect,
+  useRef,
   useMemo,
   useState,
 } from "react";
@@ -39,6 +40,7 @@ import { findDeviceByName } from "../utils/media";
 import { ActiveCall } from "./InCallView";
 import { type MuteStates } from "../state/MuteStates";
 import { useMediaDevices } from "../MediaDevicesContext";
+import { lotusDeviceCheckWanted, lotusDeviceGate } from "../lotus/lotusDevices";
 import { useMatrixRTCSessionMemberships } from "../useMatrixRTCSessionMemberships";
 import {
   saveKeyForRoom,
@@ -277,6 +279,7 @@ const LoadedCallView: FC<LoadedProps> = ({
   );
 
   const mediaDevices = useMediaDevices();
+  const lotusGate = useRef<Promise<boolean> | null>(null);
   const latestMuteStates = useLatest(muteStates);
 
   const enterRTCSessionOrError = useCallback(
@@ -357,7 +360,22 @@ const LoadedCallView: FC<LoadedProps> = ({
         return (): void => subscription.unsubscribe();
       } else {
         // No lobby and no preload: we enter the rtc session right away
-        setJoined(true);
+        // ([lotus #121] after checking the usual mic / output are present when
+        // the Lotus host asked for it; it may ask the user and cancel).
+        if (!lotusDeviceCheckWanted()) {
+          setJoined(true);
+          return;
+        }
+        // Started once per call: this effect re-runs as its inputs settle,
+        // and restarting the check each time could hold the join forever.
+        lotusGate.current ??= lotusDeviceGate(mediaDevices);
+        let cancelled = false;
+        void lotusGate.current.then((join) => {
+          if (!cancelled && join) setJoined(true);
+        });
+        return (): void => {
+          cancelled = true;
+        };
       }
     }
   }, [

src/room/InCallView.tsx

@@ -40,6 +40,7 @@ import { startLotusCallSummary } from "../lotus/lotusCallSummary";
 import { startLotusAudioOutput } from "../lotus/lotusAudioOutput";
 import { startLotusScreenshareWatch } from "../lotus/lotusScreenshareWatch";
 import { startLotusDeafen } from "../lotus/lotusDeafen";
+import { startLotusDevices } from "../lotus/lotusDevices";
 import { useHostBridge } from "../HostBridge.ts";
 import { useRootElement } from "../RootElementContext";
 import { observeElementSize$ } from "../utils/elementSize";
@@ -361,6 +362,8 @@ export const InCallView: FC<InCallViewProps> = ({
   useEffect(() => startLotusDeafen(), []);
   // [lotus #119] Let the host pick the audio output device from its call bar.
   const lotusMediaDevices = useMediaDevices();
+  // [cinny #121] Remember the picked mic / output by name; report if it goes.
+  useEffect(() => startLotusDevices(lotusMediaDevices), [lotusMediaDevices]);
   useEffect(
     () => startLotusAudioOutput(lotusMediaDevices),
     [lotusMediaDevices],