From 01254291e569f9e1f6a5e69b6fa3bb6b8a70482d Mon Sep 17 00:00:00 2001 From: Jared Vititoe Date: Mon, 9 Nov 2026 17:25:36 -0500 Subject: [PATCH] feat(calls): in-game voice overlay for the desktop app (cinny-desktop #25) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Settings → Calls → "In-game overlay (beta)", desktop only, off by default, device-local. While a call is joined it sends the call snapshot to the desktop's new `set_overlay_state` command, which shows a small always-on-top, click-through list while Lotus isn't focused. - utils/callOverlay: pure snapshot builder. Everyone in the call, you first, one row per person across devices (speaking if any device is, muted only if all are), your own mute/deafen/PTT. In PTT mode you're not shown as Muted while the key is up. - useTauriOverlay: live io.lotus.call_state participants plus control state, coalesced to at most one send per 100 ms (latest always lands), unchanged snapshots skipped, cleared on unmount. - Overlay position setting (four corners). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PPmy3tPq869XDW4njjVaKA --- src/app/components/TauriDesktopFeatures.tsx | 2 + src/app/features/settings/general/General.tsx | 28 ++++ src/app/hooks/useTauriOverlay.ts | 101 ++++++++++++++ src/app/state/settings.ts | 10 ++ src/app/utils/callOverlay.test.ts | 124 ++++++++++++++++++ src/app/utils/callOverlay.ts | 121 +++++++++++++++++ src/app/utils/settingsSync.ts | 2 + 7 files changed, 388 insertions(+) create mode 100644 src/app/hooks/useTauriOverlay.ts create mode 100644 src/app/utils/callOverlay.test.ts create mode 100644 src/app/utils/callOverlay.ts diff --git a/src/app/components/TauriDesktopFeatures.tsx b/src/app/components/TauriDesktopFeatures.tsx index 328053ed7..a31966803 100644 --- a/src/app/components/TauriDesktopFeatures.tsx +++ b/src/app/components/TauriDesktopFeatures.tsx @@ -7,6 +7,7 @@ import { useTauriNetwork } from '../hooks/useTauriNetwork'; import { useTauriToastActions } from '../hooks/useTauriToastActions'; import { useTauriFocusAssist } from '../hooks/useTauriFocusAssist'; import { useTauriDnd } from '../hooks/useTauriDnd'; +import { useTauriOverlay } from '../hooks/useTauriOverlay'; /** * Mounts the client-scoped native desktop feature hooks (call/room aware). Each @@ -25,5 +26,6 @@ export function TauriDesktopFeatures(): null { useTauriToastActions(); // P5-41/35 rich toast click → open room, quick reply → send useTauriFocusAssist(); // P5-56 Windows Focus Assist → DND suppression atom useTauriDnd(); // P6-1 tray "Do Not Disturb" → notification suppression atom + useTauriOverlay(); // cinny-desktop #25 in-game voice overlay return null; } diff --git a/src/app/features/settings/general/General.tsx b/src/app/features/settings/general/General.tsx index 5c55984d0..1767dc246 100644 --- a/src/app/features/settings/general/General.tsx +++ b/src/app/features/settings/general/General.tsx @@ -134,6 +134,7 @@ import { playCallJoinSound } from '../../../utils/callSounds'; import { previewRingtone, RINGTONE_OPTIONS } from '../../../utils/ringtones'; import { DenoiseTester } from './DenoiseTester'; import { SettingsSelect } from '../../../components/settings-select/SettingsSelect'; +import type { OverlayCorner } from '../../../utils/callOverlay'; import { isBindableCallKey } from '../../../utils/callKeybind'; import { StorageUsage } from './StorageUsage'; import { hapticsSupported } from '../../../utils/haptics'; @@ -1894,6 +1895,8 @@ export function Calls() { const [deafenKey, setDeafenKey] = useSetting(settingsAtom, 'deafenKey'); const [deafenHotkey, setDeafenHotkey] = useSetting(settingsAtom, 'deafenHotkey'); const [globalCallHotkeys, setGlobalCallHotkeys] = useSetting(settingsAtom, 'globalCallHotkeys'); + const [callOverlay, setCallOverlay] = useSetting(settingsAtom, 'callOverlay'); + const [callOverlayCorner, setCallOverlayCorner] = useSetting(settingsAtom, 'callOverlayCorner'); const [afkAutoMute, setAfkAutoMute] = useSetting(settingsAtom, 'afkAutoMute'); const [mutedTalkWarning, setMutedTalkWarning] = useSetting(settingsAtom, 'mutedTalkWarning'); const [afkTimeoutMinutes, setAfkTimeoutMinutes] = useSetting(settingsAtom, 'afkTimeoutMinutes'); @@ -2345,6 +2348,31 @@ export function Calls() { } /> )} + {isTauriEnv() && ( + } + /> + )} + {isTauriEnv() && callOverlay && ( + + value={callOverlayCorner} + onChange={setCallOverlayCorner} + options={[ + { value: 'top-left', label: 'Top left' }, + { value: 'top-right', label: 'Top right' }, + { value: 'bottom-left', label: 'Bottom left' }, + { value: 'bottom-right', label: 'Bottom right' }, + ]} + /> + } + /> + )} {hapticsSupported() && ( ([]); + const tracking = isTauri() && enabled && joined && callEmbed !== undefined; + useEffect(() => { + if (!tracking || !callEmbed) { + setParticipants([]); + return undefined; + } + const sync = (): void => setParticipants(callEmbed.getLotusParticipants() ?? []); + sync(); + return callEmbed.onLotusCallState(sync); + }, [tracking, callEmbed]); + + const snapshot: OverlaySnapshot = useMemo(() => { + if (!tracking || !callEmbed) return inactiveOverlay(corner); + const { room } = callEmbed; + return buildOverlaySnapshot({ + enabled, + joined, + corner, + myUserId: mx.getUserId(), + participants, + nameOf: (userId) => getMemberName(room, userId), + muted: !microphone, + deafened: !sound, + pttMode, + }); + }, [tracking, callEmbed, enabled, joined, corner, mx, participants, microphone, sound, pttMode]); + + // Coalesced send: skip unchanged snapshots, send at most every + // SEND_INTERVAL_MS, and always deliver the latest one. + const key = overlaySnapshotKey(snapshot); + const latest = useRef(snapshot); + latest.current = snapshot; + const sentKey = useRef(null); + const lastSentAt = useRef(0); + const timer = useRef | undefined>(undefined); + useEffect(() => { + if (!isTauri() || key === sentKey.current) return; + const send = (): void => { + timer.current = undefined; + const s = latest.current; + sentKey.current = overlaySnapshotKey(s); + lastSentAt.current = Date.now(); + invokeTauri('set_overlay_state', { state: s }); + }; + if (timer.current !== undefined) return; + const wait = SEND_INTERVAL_MS - (Date.now() - lastSentAt.current); + if (wait <= 0) send(); + else timer.current = setTimeout(send, wait); + }, [key]); + + // Hide the overlay when this unmounts (logout, client teardown). + useEffect( + () => () => { + if (timer.current !== undefined) clearTimeout(timer.current); + if (isTauri()) { + invokeTauri('set_overlay_state', { state: inactiveOverlay(latest.current.corner) }); + } + }, + [], + ); +} diff --git a/src/app/state/settings.ts b/src/app/state/settings.ts index b512d5725..944b657ef 100644 --- a/src/app/state/settings.ts +++ b/src/app/state/settings.ts @@ -1,5 +1,6 @@ import { atom } from 'jotai'; import { isSupportedTargetLang } from '../utils/translation/langUtils'; +import type { OverlayCorner } from '../utils/callOverlay'; const STORAGE_KEY = 'settings'; export type DateFormat = @@ -287,6 +288,12 @@ export interface Settings { // (a game) has focus, via a non-consuming key poll. Device-local. globalCallHotkeys: boolean; + // [cinny-desktop #25] Desktop (Windows) only: the in-game voice overlay, a + // small always-on-top list of who's in the call and who's talking, shown + // while Lotus isn't focused. Off by default while in beta. Device-local. + callOverlay: boolean; + callOverlayCorner: OverlayCorner; + pauseAnimations: boolean; composerToolbarButtons: ComposerToolbarSettings; @@ -413,6 +420,9 @@ const defaultSettings: Settings = { globalCallHotkeys: true, + callOverlay: false, + callOverlayCorner: 'top-left', + pauseAnimations: false, composerToolbarButtons: DEFAULT_COMPOSER_TOOLBAR, diff --git a/src/app/utils/callOverlay.test.ts b/src/app/utils/callOverlay.test.ts new file mode 100644 index 000000000..912d97668 --- /dev/null +++ b/src/app/utils/callOverlay.test.ts @@ -0,0 +1,124 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { buildOverlaySnapshot, inactiveOverlay, OverlayInput } from './callOverlay'; + +const names: Record = { '@me:x': 'Me', '@a:x': 'Alice', '@b:x': 'Bob' }; + +const base = (over: Partial = {}): OverlayInput => ({ + enabled: true, + joined: true, + corner: 'top-left', + myUserId: '@me:x', + participants: [ + { userId: '@me:x', speaking: false, audioEnabled: true }, + { userId: '@a:x', speaking: true, audioEnabled: true }, + { userId: '@b:x', speaking: false, audioEnabled: false }, + ], + nameOf: (id) => names[id] ?? id, + muted: false, + deafened: false, + pttMode: false, + ...over, +}); + +test('inactive when the setting is off or there is no joined call', () => { + assert.deepEqual(buildOverlaySnapshot(base({ enabled: false })), inactiveOverlay('top-left')); + assert.deepEqual( + buildOverlaySnapshot(base({ joined: false, corner: 'bottom-right' })), + inactiveOverlay('bottom-right'), + ); +}); + +test('everyone is listed, you first, speakers and muted people marked', () => { + const s = buildOverlaySnapshot(base()); + assert.equal(s.active, true); + assert.deepEqual( + s.people.map((p) => [p.name, p.speaking, p.muted, p.isSelf]), + [ + ['Me', false, false, true], + ['Alice', true, false, false], + ['Bob', false, true, false], + ], + ); +}); + +test('you are listed first even when the fork reports you later', () => { + const s = buildOverlaySnapshot( + base({ + participants: [ + { userId: '@a:x', speaking: false, audioEnabled: true }, + { userId: '@me:x', speaking: true, audioEnabled: true }, + ], + }), + ); + assert.deepEqual( + s.people.map((p) => p.userId), + ['@me:x', '@a:x'], + ); + assert.equal(s.people[0].speaking, true); +}); + +test('several devices of one person become one row', () => { + const s = buildOverlaySnapshot( + base({ + participants: [ + { userId: '@a:x', speaking: false, audioEnabled: false }, + { userId: '@a:x', speaking: true, audioEnabled: true }, + { userId: '@b:x', speaking: false, audioEnabled: false }, + { userId: '@b:x', speaking: false, audioEnabled: false }, + ], + }), + ); + const alice = s.people.find((p) => p.userId === '@a:x'); + const bob = s.people.find((p) => p.userId === '@b:x'); + assert.equal(s.people.filter((p) => p.userId === '@a:x').length, 1); + assert.deepEqual([alice?.speaking, alice?.muted], [true, false]); + assert.deepEqual([bob?.speaking, bob?.muted], [false, true]); +}); + +test('your own mute and deafen state', () => { + const muted = buildOverlaySnapshot(base({ muted: true })); + assert.equal(muted.muted, true); + assert.equal(muted.people[0].muted, true); + + const deaf = buildOverlaySnapshot( + base({ + deafened: true, + participants: [{ userId: '@me:x', speaking: true, audioEnabled: true }], + }), + ); + assert.equal(deaf.deafened, true); + assert.equal(deaf.people[0].muted, true); + assert.equal(deaf.people[0].speaking, false); +}); + +test('push-to-talk: not shown as muted while the key is up', () => { + const s = buildOverlaySnapshot(base({ pttMode: true, muted: true })); + assert.equal(s.ptt, true); + assert.equal(s.muted, false); + assert.equal(s.people[0].muted, false); +}); + +test('you are still listed before the fork reports you', () => { + const s = buildOverlaySnapshot(base({ participants: [] })); + assert.deepEqual( + s.people.map((p) => [p.name, p.isSelf]), + [['Me', true]], + ); +}); + +test('non-user ids from the fork are ignored', () => { + const s = buildOverlaySnapshot( + base({ + myUserId: null, + participants: [ + { userId: 'screen:1', speaking: true, audioEnabled: true }, + { userId: '@a:x', speaking: false, audioEnabled: true }, + ], + }), + ); + assert.deepEqual( + s.people.map((p) => p.userId), + ['@a:x'], + ); +}); diff --git a/src/app/utils/callOverlay.ts b/src/app/utils/callOverlay.ts new file mode 100644 index 000000000..6ed5e047a --- /dev/null +++ b/src/app/utils/callOverlay.ts @@ -0,0 +1,121 @@ +/** + * [cinny-desktop #25] In-game voice overlay: the call snapshot the desktop app + * draws in its always-on-top overlay window. Pure, so it can be tested without + * a call; `useTauriOverlay` feeds it live state and sends the result to the + * native `set_overlay_state` command. + */ + +export type OverlayCorner = 'top-left' | 'top-right' | 'bottom-left' | 'bottom-right'; + +export const OVERLAY_CORNERS: readonly OverlayCorner[] = [ + 'top-left', + 'top-right', + 'bottom-left', + 'bottom-right', +]; + +export type OverlayPerson = { + userId: string; + name: string; + speaking: boolean; + muted: boolean; + isSelf: boolean; +}; + +/** Mirrors `OverlayState` in cinny-desktop `src-tauri/src/native/overlay.rs`. */ +export type OverlaySnapshot = { + active: boolean; + corner: OverlayCorner; + muted: boolean; + deafened: boolean; + ptt: boolean; + people: OverlayPerson[]; +}; + +/** The subset of an `io.lotus.call_state` participant the overlay needs. */ +export type OverlayParticipant = { + userId: string; + speaking: boolean; + audioEnabled: boolean; +}; + +export type OverlayInput = { + /** Settings → Calls → In-game overlay. */ + enabled: boolean; + /** In a joined call. */ + joined: boolean; + corner: OverlayCorner; + myUserId: string | null; + /** Per device, in the fork's order (stable while people stay). */ + participants: readonly OverlayParticipant[]; + nameOf: (userId: string) => string; + /** Own mic is off (`microphone === false`). */ + muted: boolean; + /** Own call audio is off (`sound === false`). */ + deafened: boolean; + pttMode: boolean; +}; + +export const inactiveOverlay = (corner: OverlayCorner): OverlaySnapshot => ({ + active: false, + corner, + muted: false, + deafened: false, + ptt: false, + people: [], +}); + +/** + * Build the snapshot: everyone in the call (you first, then everyone else in + * call order), one row per person even with several devices (speaking if any + * device is, muted only if every device is), and your own mute / deafen / PTT + * state for the header. + * + * In push-to-talk mode the mic is "off" whenever the key isn't held, so you + * are not shown as Muted then: the header says PTT, and your row lights up + * while you transmit. + */ +export function buildOverlaySnapshot(input: OverlayInput): OverlaySnapshot { + const { corner } = input; + if (!input.enabled || !input.joined) return inactiveOverlay(corner); + + const byUser = new Map(); + input.participants.forEach((p) => { + if (!p.userId.startsWith('@')) return; + const prev = byUser.get(p.userId); + byUser.set(p.userId, { + speaking: (prev?.speaking ?? false) || p.speaking, + muted: (prev?.muted ?? true) && !p.audioEnabled, + }); + }); + + const selfMuted = input.pttMode ? false : input.muted; + const people: OverlayPerson[] = []; + const me = input.myUserId; + if (me) { + const mine = byUser.get(me); + people.push({ + userId: me, + name: input.nameOf(me), + speaking: (mine?.speaking ?? false) && !selfMuted && !input.deafened, + muted: selfMuted || input.deafened, + isSelf: true, + }); + } + byUser.forEach((state, userId) => { + if (userId === me) return; + people.push({ userId, name: input.nameOf(userId), ...state, isSelf: false }); + }); + + return { + active: true, + corner, + muted: selfMuted, + deafened: input.deafened, + ptt: input.pttMode, + people, + }; +} + +/** Key for skipping sends when nothing visible changed. */ +export const overlaySnapshotKey = (s: OverlaySnapshot): string => JSON.stringify(s); diff --git a/src/app/utils/settingsSync.ts b/src/app/utils/settingsSync.ts index 205b24887..6a703d1fd 100644 --- a/src/app/utils/settingsSync.ts +++ b/src/app/utils/settingsSync.ts @@ -28,6 +28,8 @@ export type SyncedSettingsContent = { export const DEVICE_LOCAL_KEYS: ReadonlySet = new Set([ 'settingsSync', 'globalCallHotkeys', + 'callOverlay', + 'callOverlayCorner', 'pageZoom', 'mediaAutoLoad', 'pauseAnimations', -- 2.47.3