Compare commits
10
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
9e6fe75f14 | ||
|
|
f180f76196 | ||
|
|
4ca3b0695c | ||
|
|
283b231c7a | ||
|
|
16f5f3d1a3 | ||
|
|
117467b1fb | ||
|
|
43cbd8409d | ||
|
|
4c84c3a4f3 | ||
|
|
a366243836 | ||
|
|
83a700b7f6 |
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@lotusguild/element-call-embedded",
|
||||
"version": "0.28.0-lotus.1",
|
||||
"version": "0.28.0-lotus.2",
|
||||
"files": [
|
||||
"README.md",
|
||||
"LICENSE-AGPL-3.0",
|
||||
|
||||
+8
-8
@@ -64,9 +64,9 @@
|
||||
"@rolldown/plugin-babel": "^0.2.3",
|
||||
"@sentry/react": "^8.0.0",
|
||||
"@sentry/vite-plugin": "^3.0.0",
|
||||
"@storybook/addon-docs": "^10.3.6",
|
||||
"@storybook/addon-vitest": "^10.3.6",
|
||||
"@storybook/react-vite": "^10.3.6",
|
||||
"@storybook/addon-docs": "^10.6.3",
|
||||
"@storybook/addon-vitest": "^10.6.3",
|
||||
"@storybook/react-vite": "^10.6.3",
|
||||
"@testing-library/dom": "^10.1.0",
|
||||
"@testing-library/jest-dom": "^6.6.3",
|
||||
"@testing-library/react": "^16.0.0",
|
||||
@@ -86,9 +86,9 @@
|
||||
"@vector-im/compound-design-tokens": "^11.0.0",
|
||||
"@vector-im/compound-web": "^10.0.0",
|
||||
"@vitejs/plugin-react": "^6.0.2",
|
||||
"@vitest/browser-playwright": "^4.1.5",
|
||||
"@vitest/browser-playwright": "^4.1.12",
|
||||
"@vitest/coverage-v8": "^4.0.18",
|
||||
"@vitest/ui": "4.1.11",
|
||||
"@vitest/ui": "^4.1.12",
|
||||
"babel-plugin-react-compiler": "^1.0.0",
|
||||
"classnames": "^2.3.1",
|
||||
"copy-to-clipboard": "^3.3.3",
|
||||
@@ -123,18 +123,18 @@
|
||||
"react-use-measure": "^2.1.1",
|
||||
"rxjs": "^7.8.1",
|
||||
"sass": "^1.42.1",
|
||||
"storybook": "^10.3.6",
|
||||
"storybook": "^10.6.3",
|
||||
"typescript": "^5.8.3",
|
||||
"unique-names-generator": "^4.6.0",
|
||||
"uuid": "^14.0.0",
|
||||
"vaul": "^1.0.0",
|
||||
"vite": "^8.0.0",
|
||||
"vite": "^8.3.5",
|
||||
"vite-plugin-html": "^3.2.2",
|
||||
"vite-plugin-node-polyfills": "^0.28.0",
|
||||
"vite-plugin-node-stdlib-browser": "^0.2.1",
|
||||
"vite-plugin-svgr": "^4.0.0",
|
||||
"vite-plugin-wasm": "^3.6.0",
|
||||
"vitest": "^4.1.5",
|
||||
"vitest": "^4.1.12",
|
||||
"vitest-axe": "^1.0.0-pre.3"
|
||||
},
|
||||
"packageManager": "pnpm@11.21.0+sha512.521705bce689924eac72f5a3587122f362689ef6571e55ba80076fd637c11132ecffada26fad4ea79c485bfddbfd3d5a2a5b05805a77e893de71ec8a6cca3bb1"
|
||||
|
||||
Generated
+418
-348
File diff suppressed because it is too large
Load Diff
+18
-4
@@ -28,24 +28,38 @@ overrides:
|
||||
# - https://github.com/advisories/GHSA-7r86-cg39-jmmj
|
||||
# - https://github.com/advisories/GHSA-23c5-xmqv-rm74
|
||||
"minimatch": "^10.2.3"
|
||||
# Security fixes for transitive build/test tooling (none of these ship in the
|
||||
# embedded call bundle; checked against the npm advisory database 2026-10-25).
|
||||
"tinypool": "^2.1.2"
|
||||
"@vitest/mocker": "^4.1.12"
|
||||
"brace-expansion@>=4": "^5.0.12"
|
||||
"browserslist": "^4.28.7"
|
||||
"smol-toml": "^1.8.1"
|
||||
"source-map-js": "^1.2.2"
|
||||
"postcss-selector-parser@>=7": "^7.1.6"
|
||||
"baseline-browser-mapping": "^2.11.0"
|
||||
"fflate@<0.5": "^0.4.9"
|
||||
"fflate@>=0.8": "^0.8.3"
|
||||
"pbkdf2": "^3.1.7"
|
||||
"zod@>=4": "^4.6.6"
|
||||
# Security fix: https://github.com/element-hq/element-call/security/dependabot/109
|
||||
"glob": "^10.5.0"
|
||||
# Security fixes:
|
||||
# - https://github.com/element-hq/element-call/security/dependabot/110
|
||||
# - https://github.com/element-hq/element-call/security/dependabot/122
|
||||
# - https://github.com/element-hq/element-call/security/dependabot/179
|
||||
"qs": "^6.15.2"
|
||||
"qs": "^6.16.0"
|
||||
# Security fixes:
|
||||
# - https://github.com/element-hq/element-call/security/dependabot/106
|
||||
# - https://github.com/element-hq/element-call/security/dependabot/203
|
||||
"js-yaml": "^4.2.0"
|
||||
"js-yaml": "^4.3.2"
|
||||
# Storybook declares support for 0.27.0 only but empirically works fine with 0.28.0.
|
||||
"esbuild": "^0.28.0"
|
||||
# Security fixes:
|
||||
# - https://github.com/nodejs/undici/releases/tag/v6.24.0
|
||||
# - https://github.com/element-hq/element-call/security/dependabot/201
|
||||
# - https://github.com/element-hq/element-call/security/dependabot/199
|
||||
"undici": "^6.27.0"
|
||||
"undici": "^6.29.1"
|
||||
# Security fix: https://github.com/advisories/GHSA-rf6f-7fwh-wjgh
|
||||
"flatted": "^3.4.2"
|
||||
# Security fix: https://github.com/element-hq/element-call/security/dependabot/184
|
||||
@@ -59,4 +73,4 @@ overrides:
|
||||
# - https://github.com/element-hq/element-call/security/dependabot/196
|
||||
# - https://github.com/element-hq/element-call/security/dependabot/195
|
||||
# - https://github.com/element-hq/element-call/security/dependabot/193
|
||||
"dompurify": "^3.4.11"
|
||||
"dompurify": "^3.4.17"
|
||||
|
||||
@@ -35,6 +35,7 @@ describe("LotusWidgetActions", () => {
|
||||
LotusWidgetActions.SetFrameScreenshare,
|
||||
LotusWidgetActions.PromptScreenshare,
|
||||
LotusWidgetActions.Reconnect,
|
||||
LotusWidgetActions.DeviceDecision,
|
||||
];
|
||||
|
||||
expect(new Set(LOTUS_TO_WIDGET_ACTIONS)).toEqual(new Set(expectedToWidget));
|
||||
|
||||
@@ -130,6 +130,20 @@ export enum LotusWidgetActions {
|
||||
CallError = "io.lotus.call_error",
|
||||
/** toWidget: press the error page's Reconnect / Try again (cinny #318). */
|
||||
Reconnect = "io.lotus.reconnect",
|
||||
/**
|
||||
* fromWidget: before the automatic join (`lotusDeviceCheck=1`), the user's
|
||||
* usual mic / output is missing — `{ missing: [{ kind: "audioinput" |
|
||||
* "audiooutput", expected: name, fallback: name | null }] }`. EC waits for
|
||||
* `DeviceDecision` (cinny #121).
|
||||
*/
|
||||
DeviceMissing = "io.lotus.device_missing",
|
||||
/** toWidget: the user's answer, `{ join: boolean }` (cinny #121). */
|
||||
DeviceDecision = "io.lotus.device_decision",
|
||||
/**
|
||||
* fromWidget: in a call, the usual device disappeared — `{ kind, expected,
|
||||
* fallback }` — for a toast (cinny #121).
|
||||
*/
|
||||
DeviceLost = "io.lotus.device_lost",
|
||||
}
|
||||
|
||||
/** toWidget Lotus actions that must be allow-listed in `initializeWidget`. */
|
||||
@@ -148,4 +162,5 @@ export const LOTUS_TO_WIDGET_ACTIONS: LotusWidgetActions[] = [
|
||||
LotusWidgetActions.SetFrameScreenshare,
|
||||
LotusWidgetActions.PromptScreenshare,
|
||||
LotusWidgetActions.Reconnect,
|
||||
LotusWidgetActions.DeviceDecision,
|
||||
];
|
||||
|
||||
@@ -11,6 +11,7 @@ import { type AudioProcessorOptions } from "livekit-client";
|
||||
import {
|
||||
assertModuleExport,
|
||||
type LotusDenoiseConfig,
|
||||
normalizeDfnModelBytes,
|
||||
LotusDenoiseProcessor,
|
||||
prepareDenoiseAssets,
|
||||
releasePreparedDenoiseAssets,
|
||||
@@ -245,3 +246,37 @@ describe("assertModuleExport (#26)", () => {
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe("normalizeDfnModelBytes", () => {
|
||||
const tar = (): Uint8Array => {
|
||||
const b = new Uint8Array(1024);
|
||||
b.set(new TextEncoder().encode("ustar"), 257);
|
||||
b.set(new TextEncoder().encode("model.onnx"), 0);
|
||||
return b;
|
||||
};
|
||||
const gunzip = async (buf: ArrayBuffer): Promise<Uint8Array> =>
|
||||
new Uint8Array(
|
||||
await new Response(
|
||||
new Response(buf).body!.pipeThrough(new DecompressionStream("gzip")),
|
||||
).arrayBuffer(),
|
||||
);
|
||||
|
||||
test("passes a gzipped model through untouched", async () => {
|
||||
const gz = new Uint8Array([0x1f, 0x8b, 8, 0, 1, 2, 3]).buffer;
|
||||
expect(await normalizeDfnModelBytes(gz)).toBe(gz);
|
||||
});
|
||||
|
||||
test("re-gzips a tar that a Content-Encoding server already unpacked", async () => {
|
||||
const out = await normalizeDfnModelBytes(tar().buffer as ArrayBuffer);
|
||||
const u8 = new Uint8Array(out);
|
||||
expect([u8[0], u8[1]]).toEqual([0x1f, 0x8b]);
|
||||
expect(await gunzip(out)).toEqual(tar());
|
||||
});
|
||||
|
||||
test("rejects anything else (e.g. an HTML error page)", async () => {
|
||||
const html = new TextEncoder().encode("<!doctype html><title>404</title>");
|
||||
await expect(
|
||||
normalizeDfnModelBytes(html.buffer as ArrayBuffer),
|
||||
).rejects.toThrow(/neither gzip nor tar/);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -144,6 +144,8 @@ interface DfnCore {
|
||||
initialize: () => Promise<void>;
|
||||
createAudioWorkletNode: (ctx: AudioContext) => Promise<AudioNode>;
|
||||
destroy: () => void;
|
||||
/** Set by `initialize()`; the model bytes are handed to the worklet as-is. */
|
||||
assets?: { wasmModule: WebAssembly.Module; modelBytes: ArrayBuffer } | null;
|
||||
}
|
||||
interface DfnModule {
|
||||
DeepFilterNet3Core: new (opts: {
|
||||
@@ -167,6 +169,33 @@ export function assertModuleExport<T>(
|
||||
return mod as T;
|
||||
}
|
||||
|
||||
/**
|
||||
* Make sure the DeepFilterNet model is the gzipped tarball its wasm expects.
|
||||
*
|
||||
* A server or proxy that sends the `.tar.gz` with `Content-Encoding: gzip`
|
||||
* (vite preview does; some CDNs and nginx `gzip_static` setups do too) makes
|
||||
* `fetch()` hand back the inner tar. The library's worklet then fails inside
|
||||
* `df_create`, swallows the error and passes the raw mic straight through,
|
||||
* while we report DeepFilterNet as active. An already-decompressed tar is
|
||||
* re-gzipped; anything else throws, so the rnnoise fallback runs and the host
|
||||
* is told the truth.
|
||||
*/
|
||||
export async function normalizeDfnModelBytes(
|
||||
bytes: ArrayBuffer,
|
||||
): Promise<ArrayBuffer> {
|
||||
const u8 = new Uint8Array(bytes);
|
||||
if (u8.length > 2 && u8[0] === 0x1f && u8[1] === 0x8b) return bytes;
|
||||
const isTar =
|
||||
u8.length > 262 &&
|
||||
String.fromCharCode(...u8.subarray(257, 262)) === "ustar";
|
||||
if (!isTar)
|
||||
throw new Error(
|
||||
`denoise: DeepFilterNet model is neither gzip nor tar (${u8.length} bytes)`,
|
||||
);
|
||||
const gz = new Response(u8).body!.pipeThrough(new CompressionStream("gzip"));
|
||||
return new Response(gz).arrayBuffer();
|
||||
}
|
||||
|
||||
async function loadDfnCore(config: LotusDenoiseConfig): Promise<DfnCore> {
|
||||
const base = config.assetBase;
|
||||
const url = `${base}deepfilternet/index.esm.js`;
|
||||
@@ -178,13 +207,19 @@ async function loadDfnCore(config: LotusDenoiseConfig): Promise<DfnCore> {
|
||||
);
|
||||
const core = new mod.DeepFilterNet3Core({
|
||||
sampleRate: 48_000,
|
||||
// 60, not 80: full-strength suppression is the main source of the
|
||||
// "over-processed" character; a lower level keeps voice natural while
|
||||
// the dry/wet floor handles the noise tail.
|
||||
noiseReductionLevel: 60,
|
||||
// Attenuation limit in dB. At 60 the background is muted outright
|
||||
// (measured -61 to -64 dB), so every word drops into dead silence and
|
||||
// listeners hear the mic "cutting out" at word ends. 25 keeps a quiet
|
||||
// background bed and scored best on room-echo speech with fan and
|
||||
// keyboard noise (PESQ 2.72 / 2.38 vs 2.33 / 2.04 at 60).
|
||||
noiseReductionLevel: 25,
|
||||
assetConfig: { cdnUrl: dfnBase },
|
||||
});
|
||||
await core.initialize();
|
||||
if (core.assets)
|
||||
core.assets.modelBytes = await normalizeDfnModelBytes(
|
||||
core.assets.modelBytes,
|
||||
);
|
||||
return core;
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,188 @@
|
||||
/*
|
||||
Copyright 2026 Lotus Guild
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { BehaviorSubject } from "rxjs";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
const sent: { action: string; data: unknown }[] = [];
|
||||
const handlers = new Map<string, (ev: CustomEvent) => void>();
|
||||
vi.mock("../widget", () => ({
|
||||
widget: {
|
||||
api: { transport: { reply: vi.fn() } },
|
||||
lazyActions: {
|
||||
on: (a: string, h: (ev: CustomEvent) => void): void =>
|
||||
void handlers.set(a, h),
|
||||
off: (a: string): void => void handlers.delete(a),
|
||||
},
|
||||
},
|
||||
}));
|
||||
vi.mock("./lotusWidget", () => ({
|
||||
lotusFlag: (name: string): boolean => name === "lotusDeviceCheck",
|
||||
lotusSendToHost: (action: string, data: unknown): boolean => {
|
||||
sent.push({ action, data });
|
||||
return true;
|
||||
},
|
||||
}));
|
||||
|
||||
const {
|
||||
checkDevice,
|
||||
lotusDeviceGate,
|
||||
startLotusDevices,
|
||||
audioInputName,
|
||||
audioOutputName,
|
||||
} = await import("./lotusDevices");
|
||||
const { audioInput, audioOutput } = await import("../settings/settings");
|
||||
|
||||
const named = (
|
||||
...names: [string, string][]
|
||||
): Map<string, { type: string; name?: string }> =>
|
||||
new Map(names.map(([id, name]) => [id, { type: "name", name }]));
|
||||
|
||||
type Labels = Map<string, { type: string; name?: string }>;
|
||||
const source = (
|
||||
labels: Labels,
|
||||
): { available$: BehaviorSubject<Labels>; select: (id: string) => void } => ({
|
||||
available$: new BehaviorSubject(labels),
|
||||
select: vi.fn<(id: string) => void>(),
|
||||
});
|
||||
const devices = (
|
||||
inputs: Labels,
|
||||
outputs: Labels = named(["", "Default"]),
|
||||
): {
|
||||
audioInput: ReturnType<typeof source>;
|
||||
audioOutput: ReturnType<typeof source>;
|
||||
} => ({ audioInput: source(inputs), audioOutput: source(outputs) });
|
||||
|
||||
afterEach(() => {
|
||||
sent.length = 0;
|
||||
handlers.clear();
|
||||
audioInput.setValue(undefined);
|
||||
audioOutput.setValue(undefined);
|
||||
audioInputName.setValue(undefined);
|
||||
audioOutputName.setValue(undefined);
|
||||
});
|
||||
|
||||
describe("[cinny #121] checkDevice", () => {
|
||||
const avail = named(["laptop", "Laptop Mic"], ["arctis2", "Arctis 7"]);
|
||||
it("is fine when the usual device is there, or nothing is remembered", () => {
|
||||
expect(checkDevice("audioinput", "arctis2", "Arctis 7", avail).result).toBe(
|
||||
"ok",
|
||||
);
|
||||
expect(checkDevice("audioinput", undefined, undefined, avail).result).toBe(
|
||||
"ok",
|
||||
);
|
||||
expect(checkDevice("audioinput", "default", "x", avail).result).toBe("ok");
|
||||
});
|
||||
it("re-selects a device that came back under a new id", () => {
|
||||
expect(checkDevice("audioinput", "arctis1", "Arctis 7", avail)).toEqual({
|
||||
kind: "audioinput",
|
||||
result: "reselect",
|
||||
id: "arctis2",
|
||||
});
|
||||
});
|
||||
it("reports a missing device and what EC would use instead", () => {
|
||||
expect(
|
||||
checkDevice(
|
||||
"audioinput",
|
||||
"arctis1",
|
||||
"Arctis 7",
|
||||
named(["laptop", "Laptop Mic"]),
|
||||
),
|
||||
).toEqual({
|
||||
kind: "audioinput",
|
||||
result: "missing",
|
||||
expected: "Arctis 7",
|
||||
fallback: "Laptop Mic",
|
||||
});
|
||||
});
|
||||
it("fails open without device names (no permission yet)", () => {
|
||||
const unnamed = new Map([["x", { type: "number" }]]);
|
||||
expect(
|
||||
checkDevice("audioinput", "arctis1", "Arctis 7", unnamed).result,
|
||||
).toBe("ok");
|
||||
});
|
||||
});
|
||||
|
||||
describe("[cinny #121] lotusDeviceGate", () => {
|
||||
it("joins straight away when everything is there", async () => {
|
||||
audioInput.setValue("arctis");
|
||||
audioInputName.setValue("Arctis 7");
|
||||
await expect(
|
||||
lotusDeviceGate(devices(named(["arctis", "Arctis 7"]))),
|
||||
).resolves.toBe(true);
|
||||
expect(sent).toEqual([]);
|
||||
});
|
||||
|
||||
it("asks the host and waits for the answer", async () => {
|
||||
audioInput.setValue("arctis");
|
||||
audioInputName.setValue("Arctis 7");
|
||||
const gate = lotusDeviceGate(devices(named(["laptop", "Laptop Mic"])));
|
||||
await vi.waitFor(() => expect(sent).toHaveLength(1));
|
||||
expect(sent[0]).toEqual({
|
||||
action: "io.lotus.device_missing",
|
||||
data: {
|
||||
missing: [
|
||||
{ kind: "audioinput", expected: "Arctis 7", fallback: "Laptop Mic" },
|
||||
],
|
||||
},
|
||||
});
|
||||
handlers.get("io.lotus.device_decision")!(
|
||||
new CustomEvent("x", { detail: { data: { join: false } } }),
|
||||
);
|
||||
await expect(gate).resolves.toBe(false);
|
||||
});
|
||||
|
||||
it("re-selects a re-plugged device without asking", async () => {
|
||||
audioInput.setValue("old-id");
|
||||
audioInputName.setValue("Arctis 7");
|
||||
const d = devices(named(["new-id", "Arctis 7"]));
|
||||
await expect(lotusDeviceGate(d)).resolves.toBe(true);
|
||||
expect(d.audioInput.select).toHaveBeenCalledWith("new-id");
|
||||
expect(sent).toEqual([]);
|
||||
});
|
||||
|
||||
it("joins when device names never arrive", async () => {
|
||||
audioInput.setValue("arctis");
|
||||
audioInputName.setValue("Arctis 7");
|
||||
const d = devices(
|
||||
new Map([["x", { type: "number" }]]),
|
||||
new Map([["y", { type: "number" }]]),
|
||||
);
|
||||
await expect(lotusDeviceGate(d, 50)).resolves.toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe("[cinny #121] startLotusDevices", () => {
|
||||
it("remembers the picked device's name and reports when it goes", () => {
|
||||
const d = devices(named(["laptop", "Laptop Mic"], ["arctis", "Arctis 7"]));
|
||||
const stop = startLotusDevices(d);
|
||||
audioInput.setValue("arctis");
|
||||
expect(audioInputName.getValue()).toBe("Arctis 7");
|
||||
d.audioInput.available$.next(named(["laptop", "Laptop Mic"]));
|
||||
expect(sent).toEqual([
|
||||
{
|
||||
action: "io.lotus.device_lost",
|
||||
data: {
|
||||
kind: "audioinput",
|
||||
expected: "Arctis 7",
|
||||
fallback: "Laptop Mic",
|
||||
},
|
||||
},
|
||||
]);
|
||||
// Name kept, so the next join can still recognise it.
|
||||
expect(audioInputName.getValue()).toBe("Arctis 7");
|
||||
stop();
|
||||
});
|
||||
|
||||
it("choosing the system default clears the remembered name", () => {
|
||||
audioInputName.setValue("Arctis 7");
|
||||
const stop = startLotusDevices(devices(named(["", "Default"])));
|
||||
audioInput.setValue("");
|
||||
expect(audioInputName.getValue()).toBeUndefined();
|
||||
stop();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,236 @@
|
||||
/*
|
||||
Copyright 2026 Lotus Guild
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { logger } from "matrix-js-sdk/lib/logger";
|
||||
import { type IWidgetApiRequest } from "matrix-widget-api";
|
||||
import {
|
||||
combineLatest,
|
||||
filter,
|
||||
firstValueFrom,
|
||||
map,
|
||||
type Observable,
|
||||
timeout,
|
||||
} from "rxjs";
|
||||
|
||||
import {
|
||||
audioInput as audioInputSetting,
|
||||
audioOutput as audioOutputSetting,
|
||||
Setting,
|
||||
} from "../settings/settings";
|
||||
import { widget } from "../widget";
|
||||
import { LotusWidgetActions } from "./lotusActions";
|
||||
import { lotusFlag, lotusSendToHost } from "./lotusWidget";
|
||||
|
||||
/**
|
||||
* [cinny #121] Expected-device check. EC remembers the chosen mic / output by
|
||||
* id and, when that device is missing, silently uses the first one it finds,
|
||||
* so a user who always joins with a headset ends up on the laptop mic.
|
||||
*
|
||||
* - "Usual" = the device the user last picked (EC's own audio-input /
|
||||
* audio-output settings). Its NAME is remembered alongside (ids can change
|
||||
* when a device is re-plugged on Windows).
|
||||
* - Before the automatic join (`lotusDeviceCheck=1`), a missing usual device
|
||||
* is reported to the host (`io.lotus.device_missing`), which asks the user
|
||||
* and answers `io.lotus.device_decision { join }`. A device that came back
|
||||
* under a new id but the same name is just re-selected.
|
||||
* - In a call, losing the usual device is reported (`io.lotus.device_lost`)
|
||||
* for a toast; no question mid-call.
|
||||
* Fails open: no names (no permission yet), no host, or nothing remembered
|
||||
* means no check.
|
||||
*/
|
||||
|
||||
export type DeviceKind = "audioinput" | "audiooutput";
|
||||
|
||||
/** Any device label with an optional human name (EC's label unions). */
|
||||
type AnyLabel = { type: string; name?: string | null };
|
||||
|
||||
export type DeviceCheck =
|
||||
| { kind: DeviceKind; result: "ok" }
|
||||
| { kind: DeviceKind; result: "reselect"; id: string }
|
||||
| {
|
||||
kind: DeviceKind;
|
||||
result: "missing";
|
||||
expected: string;
|
||||
fallback: string | null;
|
||||
};
|
||||
|
||||
const nameOf = (label: AnyLabel | undefined): string | null =>
|
||||
label && typeof label.name === "string" && label.name ? label.name : null;
|
||||
|
||||
/** Whether the device list carries real names (permission granted). */
|
||||
export const hasNames = (available: Map<string, AnyLabel>): boolean =>
|
||||
[...available.values()].some((l) => l.type === "name");
|
||||
|
||||
/** The id EC is going to (or does) use as the "system default" choice. */
|
||||
const isDefaultId = (id: string): boolean => id === "" || id === "default";
|
||||
|
||||
/**
|
||||
* Pure: compare the remembered device with what is plugged in now.
|
||||
* `fallback` names the device EC falls back to (the first in the list, as
|
||||
* MediaDevices' selectDevice$ does).
|
||||
*/
|
||||
export function checkDevice(
|
||||
kind: DeviceKind,
|
||||
preferredId: string | undefined,
|
||||
expectedName: string | undefined,
|
||||
available: Map<string, AnyLabel>,
|
||||
): DeviceCheck {
|
||||
if (available.size === 0 || !hasNames(available))
|
||||
return { kind, result: "ok" };
|
||||
if (preferredId === undefined || isDefaultId(preferredId))
|
||||
return { kind, result: "ok" };
|
||||
if (available.has(preferredId)) return { kind, result: "ok" };
|
||||
if (!expectedName) return { kind, result: "ok" };
|
||||
for (const [id, label] of available) {
|
||||
if (nameOf(label) === expectedName) return { kind, result: "reselect", id };
|
||||
}
|
||||
const first = available.values().next().value;
|
||||
return {
|
||||
kind,
|
||||
result: "missing",
|
||||
expected: expectedName,
|
||||
fallback: nameOf(first),
|
||||
};
|
||||
}
|
||||
|
||||
/** The remembered NAME of the last-picked device of each kind. */
|
||||
export const audioInputName = new Setting<string | undefined>(
|
||||
"lotus-audio-input-name",
|
||||
undefined,
|
||||
);
|
||||
export const audioOutputName = new Setting<string | undefined>(
|
||||
"lotus-audio-output-name",
|
||||
undefined,
|
||||
);
|
||||
|
||||
interface DeviceSource {
|
||||
available$: Observable<Map<string, AnyLabel>>;
|
||||
select(id: string): void;
|
||||
}
|
||||
export interface LotusMediaDevices {
|
||||
audioInput: DeviceSource;
|
||||
audioOutput: DeviceSource;
|
||||
}
|
||||
|
||||
const kinds = (
|
||||
devices: LotusMediaDevices,
|
||||
): {
|
||||
kind: DeviceKind;
|
||||
source: DeviceSource;
|
||||
preferred: Setting<string | undefined>;
|
||||
name: Setting<string | undefined>;
|
||||
}[] => [
|
||||
{
|
||||
kind: "audioinput",
|
||||
source: devices.audioInput,
|
||||
preferred: audioInputSetting,
|
||||
name: audioInputName,
|
||||
},
|
||||
{
|
||||
kind: "audiooutput",
|
||||
source: devices.audioOutput,
|
||||
preferred: audioOutputSetting,
|
||||
name: audioOutputName,
|
||||
},
|
||||
];
|
||||
|
||||
/** Whether the host asked for the check (a Lotus host with the flag). */
|
||||
export const lotusDeviceCheckWanted = (): boolean =>
|
||||
!!widget && lotusFlag("lotusDeviceCheck");
|
||||
|
||||
/**
|
||||
* Before EC's automatic join: true = join now, false = the user cancelled.
|
||||
* Waits up to `namesTimeoutMs` for named device lists, then checks both kinds.
|
||||
*/
|
||||
export async function lotusDeviceGate(
|
||||
devices: LotusMediaDevices,
|
||||
namesTimeoutMs = 3000,
|
||||
): Promise<boolean> {
|
||||
const w = widget;
|
||||
if (!w || !lotusDeviceCheckWanted()) return true;
|
||||
const entries = kinds(devices);
|
||||
let lists: Map<string, AnyLabel>[];
|
||||
try {
|
||||
lists = await firstValueFrom(
|
||||
combineLatest(entries.map((e) => e.source.available$)).pipe(
|
||||
filter((all) => all.some(hasNames)),
|
||||
timeout(namesTimeoutMs),
|
||||
),
|
||||
);
|
||||
} catch {
|
||||
return true; // no names (yet): nothing to compare by
|
||||
}
|
||||
const missing: Extract<DeviceCheck, { result: "missing" }>[] = [];
|
||||
entries.forEach((e, i) => {
|
||||
const check = checkDevice(
|
||||
e.kind,
|
||||
e.preferred.getValue(),
|
||||
e.name.getValue(),
|
||||
lists[i],
|
||||
);
|
||||
if (check.result === "reselect") {
|
||||
logger.info(`[lotus] ${e.kind}: usual device is back under a new id`);
|
||||
e.source.select(check.id);
|
||||
} else if (check.result === "missing") missing.push(check);
|
||||
});
|
||||
if (missing.length === 0) return true;
|
||||
|
||||
return new Promise<boolean>((resolve) => {
|
||||
const onDecision = (ev: CustomEvent<IWidgetApiRequest>): void => {
|
||||
w.api.transport.reply(ev.detail, {});
|
||||
w.lazyActions.off(LotusWidgetActions.DeviceDecision, onDecision);
|
||||
resolve((ev.detail.data as { join?: unknown }).join !== false);
|
||||
};
|
||||
w.lazyActions.on(LotusWidgetActions.DeviceDecision, onDecision);
|
||||
const sent = lotusSendToHost(LotusWidgetActions.DeviceMissing, {
|
||||
missing: missing.map(({ kind, expected, fallback }) => ({
|
||||
kind,
|
||||
expected,
|
||||
fallback,
|
||||
})),
|
||||
});
|
||||
if (!sent) {
|
||||
w.lazyActions.off(LotusWidgetActions.DeviceDecision, onDecision);
|
||||
resolve(true);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* In a call: remember the name of every device the user picks, and report
|
||||
* when the picked device disappears. Returns a teardown function.
|
||||
*/
|
||||
export function startLotusDevices(devices: LotusMediaDevices): () => void {
|
||||
const subs = kinds(devices).map((e) => {
|
||||
let wasPresent: boolean | undefined;
|
||||
return combineLatest([e.preferred.value$, e.source.available$])
|
||||
.pipe(map(([id, available]) => ({ id, available })))
|
||||
.subscribe(({ id, available }) => {
|
||||
if (id === undefined || isDefaultId(id) || !hasNames(available)) {
|
||||
if (id !== undefined && isDefaultId(id)) e.name.setValue(undefined);
|
||||
wasPresent = undefined;
|
||||
return;
|
||||
}
|
||||
const label = available.get(id);
|
||||
if (label) {
|
||||
const name = nameOf(label);
|
||||
if (name && e.name.getValue() !== name) e.name.setValue(name);
|
||||
wasPresent = true;
|
||||
return;
|
||||
}
|
||||
if (wasPresent && e.name.getValue()) {
|
||||
lotusSendToHost(LotusWidgetActions.DeviceLost, {
|
||||
kind: e.kind,
|
||||
expected: e.name.getValue(),
|
||||
fallback: nameOf(available.values().next().value),
|
||||
});
|
||||
}
|
||||
wasPresent = false;
|
||||
});
|
||||
});
|
||||
return (): void => subs.forEach((s) => s.unsubscribe());
|
||||
}
|
||||
+19
-1
@@ -10,6 +10,7 @@ import {
|
||||
type ReactNode,
|
||||
useCallback,
|
||||
useEffect,
|
||||
useRef,
|
||||
useMemo,
|
||||
useState,
|
||||
} from "react";
|
||||
@@ -39,6 +40,7 @@ import { findDeviceByName } from "../utils/media";
|
||||
import { ActiveCall } from "./InCallView";
|
||||
import { type MuteStates } from "../state/MuteStates";
|
||||
import { useMediaDevices } from "../MediaDevicesContext";
|
||||
import { lotusDeviceCheckWanted, lotusDeviceGate } from "../lotus/lotusDevices";
|
||||
import { useMatrixRTCSessionMemberships } from "../useMatrixRTCSessionMemberships";
|
||||
import {
|
||||
saveKeyForRoom,
|
||||
@@ -277,6 +279,7 @@ const LoadedCallView: FC<LoadedProps> = ({
|
||||
);
|
||||
|
||||
const mediaDevices = useMediaDevices();
|
||||
const lotusGate = useRef<Promise<boolean> | null>(null);
|
||||
const latestMuteStates = useLatest(muteStates);
|
||||
|
||||
const enterRTCSessionOrError = useCallback(
|
||||
@@ -357,7 +360,22 @@ const LoadedCallView: FC<LoadedProps> = ({
|
||||
return (): void => subscription.unsubscribe();
|
||||
} else {
|
||||
// No lobby and no preload: we enter the rtc session right away
|
||||
setJoined(true);
|
||||
// ([lotus #121] after checking the usual mic / output are present when
|
||||
// the Lotus host asked for it; it may ask the user and cancel).
|
||||
if (!lotusDeviceCheckWanted()) {
|
||||
setJoined(true);
|
||||
return;
|
||||
}
|
||||
// Started once per call: this effect re-runs as its inputs settle,
|
||||
// and restarting the check each time could hold the join forever.
|
||||
lotusGate.current ??= lotusDeviceGate(mediaDevices);
|
||||
let cancelled = false;
|
||||
void lotusGate.current.then((join) => {
|
||||
if (!cancelled && join) setJoined(true);
|
||||
});
|
||||
return (): void => {
|
||||
cancelled = true;
|
||||
};
|
||||
}
|
||||
}
|
||||
}, [
|
||||
|
||||
@@ -40,6 +40,7 @@ import { startLotusCallSummary } from "../lotus/lotusCallSummary";
|
||||
import { startLotusAudioOutput } from "../lotus/lotusAudioOutput";
|
||||
import { startLotusScreenshareWatch } from "../lotus/lotusScreenshareWatch";
|
||||
import { startLotusDeafen } from "../lotus/lotusDeafen";
|
||||
import { startLotusDevices } from "../lotus/lotusDevices";
|
||||
import { useHostBridge } from "../HostBridge.ts";
|
||||
import { useRootElement } from "../RootElementContext";
|
||||
import { observeElementSize$ } from "../utils/elementSize";
|
||||
@@ -361,6 +362,8 @@ export const InCallView: FC<InCallViewProps> = ({
|
||||
useEffect(() => startLotusDeafen(), []);
|
||||
// [lotus #119] Let the host pick the audio output device from its call bar.
|
||||
const lotusMediaDevices = useMediaDevices();
|
||||
// [cinny #121] Remember the picked mic / output by name; report if it goes.
|
||||
useEffect(() => startLotusDevices(lotusMediaDevices), [lotusMediaDevices]);
|
||||
useEffect(
|
||||
() => startLotusAudioOutput(lotusMediaDevices),
|
||||
[lotusMediaDevices],
|
||||
|
||||
Reference in New Issue
Block a user