diff --git a/LOTUS.md b/LOTUS.md index 130f43fc..361813b6 100644 --- a/LOTUS.md +++ b/LOTUS.md @@ -43,7 +43,7 @@ checked against cinny's `lotus` branch. | `io.lotus.set_deafen` | host → call | deafen remote audio (and optionally mute screenshare audio). | yes | | `io.lotus.request_state` | call → host | sent on (re)registration of a lotus toWidget handler (e.g. decorations) so the host can re-push state that would otherwise only be sent on change (decorations roster, the current focus pin) after an EC-side reconnect/remount. cinny's `resendForkState()` should respond to this the same way it responds to a fresh join. | yes | | `io.lotus.denoise_state` | call → host | real state of the in-source denoise engine — `{ active: boolean, model: string, error?: string }` — sent once the processor attaches (or after the rnnoise fallback also fails), so the host toggle can reflect reality rather than the requested state. | yes | -| `io.lotus.call_summary` | call → host | one-shot end-of-call readout for the local participant — `{ durationMs, reconnects, poorMs, verdict }` (#143) — sent on SFU disconnect or in-call teardown, whichever comes first. | yes | +| `io.lotus.call_summary` | call → host | one-shot end-of-call readout `{ durationMs, reconnects, poorMs, verdict, details? }` (#143); `details` (cinny #253): participants, peak, per-person poor time, everyone-poor time, RTT/loss/jitter, your talk/mute/screenshare time. | yes | | `io.lotus.set_audio_output` | host → call | select the audio output device `{ deviceId }` (#119). | yes | | `io.lotus.audio_output_state` | call → host | the currently selected output `{ deviceId }` (#119). | cinny #417 | | `io.lotus.screenshare_audio_state` | call → host | `{ present }`: whether any remote screenshare audio is being received, sent on change (cinny #277). | cinny #277 | diff --git a/src/lotus/lotusCallSummary.test.ts b/src/lotus/lotusCallSummary.test.ts index 02621528..b9949246 100644 --- a/src/lotus/lotusCallSummary.test.ts +++ b/src/lotus/lotusCallSummary.test.ts @@ -8,7 +8,11 @@ Please see LICENSE in the repository root for full details. import { ConnectionQuality } from "livekit-client"; import { describe, expect, it } from "vitest"; -import { CallQualityTracker } from "./lotusCallSummary"; +import { + CallDetailsTracker, + CallQualityTracker, + matrixUserOf, +} from "./lotusCallSummary"; const clock = (): { now: () => number; tick: (ms: number) => void } => { let t = 1_000_000; @@ -91,3 +95,92 @@ describe("CallQualityTracker", () => { }); }); }); + +describe("[cinny #253] CallDetailsTracker", () => { + it("participants, peak, and devices of one person counted once", () => { + const c = clock(); + const d = new CallDetailsTracker(c.now); + d.remoteJoined("@bob:x:DEV1"); + d.remoteJoined("@bob:x:DEV2"); + d.remoteJoined("@carol:x:D"); + d.remoteLeft("@carol:x:D"); + d.remoteJoined("@dave:x:D"); + const r = d.details(); + expect(r.participants).toEqual(["@bob:x", "@carol:x", "@dave:x"]); + expect(r.peakParticipants).toBe(3); // you + bob + carol (dave came after carol left) + }); + + it("joining is idempotent per device (seeding everyone present on connect)", () => { + const d = new CallDetailsTracker(); + d.remoteJoined("@bob:x:DEV1"); + d.remoteJoined("@bob:x:DEV1"); // seen again by the on-connect seed + d.remoteLeft("@bob:x:DEV1"); // one leave: bob is gone + d.remoteJoined("@carol:x:D"); + d.remoteJoined("@dave:x:D"); // peak is you + carol + dave, not + a ghost bob + d.setQuality("@bob:x:DEV1", ConnectionQuality.Poor); + expect(d.details().participants).toEqual(["@bob:x", "@carol:x", "@dave:x"]); + expect(d.details().peakParticipants).toBe(3); + }); + + it("poor time per person, and everyone-at-once only while all are poor", () => { + const c = clock(); + const d = new CallDetailsTracker(c.now); + d.remoteJoined("@bob:x:D"); + d.setQuality(undefined, ConnectionQuality.Poor); // you poor + c.tick(10_000); + d.setQuality("@bob:x:D", ConnectionQuality.Lost); // now everyone + c.tick(5_000); + d.setQuality(undefined, ConnectionQuality.Good); + c.tick(3_000); + d.setQuality("@bob:x:D", ConnectionQuality.Excellent); + const q = d.details().quality; + expect(q.youPoorMs).toBe(15_000); + expect(q.othersPoorMs).toEqual({ "@bob:x": 8_000 }); + expect(q.everyonePoorMs).toBe(5_000); + }); + + it("network numbers: RTT median/p95, loss from the latest counters, jitter", () => { + const d = new CallDetailsTracker(); + for (const rtt of [0.02, 0.03, 0.04, 0.05, 0.3]) d.addRtt(rtt); + d.addRtt(Number.NaN); + d.setSent("mic", 900, 5); + d.setSent("mic", 995, 5); // cumulative: latest wins + d.setReceived("bob", 490, 10); + d.addJitter(0.004); + d.addJitter(0.006); + const n = d.details().network; + expect(n.rttMedianMs).toBe(40); + expect(n.rttP95Ms).toBe(300); + expect(n.outboundLossPct).toBe(0.5); + expect(n.inboundLossPct).toBe(2); + expect(n.jitterMs).toBe(5); + }); + + it("no samples: network numbers are absent, not zero", () => { + expect(new CallDetailsTracker().details().network).toEqual({}); + }); + + it("talk, mute and screenshare time", () => { + const c = clock(); + const d = new CallDetailsTracker(c.now); + d.setMuted(true); + c.tick(4_000); + d.setMuted(false); + d.setSpeaking(true); + c.tick(2_000); + d.setSpeaking(false); + d.setScreensharing(true); + c.tick(6_000); // still sharing when summarised + expect(d.details().activity).toEqual({ + speakingMs: 2_000, + mutedMs: 4_000, + screenshareMs: 6_000, + }); + }); + + it("matrixUserOf reduces a LiveKit identity to the Matrix user", () => { + expect(matrixUserOf("@bob:example.org:ABCDEF")).toBe("@bob:example.org"); + expect(matrixUserOf("@bob:example.org")).toBe("@bob:example.org"); + expect(matrixUserOf("hashed-federated-id")).toBe("hashed-federated-id"); + }); +}); diff --git a/src/lotus/lotusCallSummary.ts b/src/lotus/lotusCallSummary.ts index 13e89c97..7945689b 100644 --- a/src/lotus/lotusCallSummary.ts +++ b/src/lotus/lotusCallSummary.ts @@ -7,9 +7,16 @@ Please see LICENSE in the repository root for full details. import { ConnectionQuality, + type LocalAudioTrack, + type LocalTrackPublication, type Participant, + ParticipantEvent, + type RemoteAudioTrack, + type RemoteParticipant, type Room as LivekitRoom, RoomEvent, + Track, + type TrackPublication, } from "livekit-client"; import { type CallViewModel } from "../state/CallViewModel/CallViewModel"; @@ -27,6 +34,241 @@ export interface CallSummary { /** Time the local connection quality was reported poor/lost, ms. */ poorMs: number; verdict: CallQualityVerdict; + /** [cinny #253] The breakdown behind the host's "Details" card. */ + details?: CallSummaryDetails; +} + +/** [cinny #253] Everything the host's post-call Details card shows. */ +export interface CallSummaryDetails { + /** Other Matrix users seen in the call (`@user:server`), first-seen order. */ + participants: string[]; + /** Most people in the call at once, you included. */ + peakParticipants: number; + quality: { + /** Your connection reported poor/lost, ms. */ + youPoorMs: number; + /** Each other person's connection reported poor/lost, ms (only > 0). */ + othersPoorMs: Record; + /** Everyone, you included, poor at the same time (points at the server). */ + everyonePoorMs: number; + }; + network: { + rttMedianMs?: number; + rttP95Ms?: number; + /** Your microphone's packets the SFU reported lost, %. */ + outboundLossPct?: number; + /** Packets lost on the audio you received, %. */ + inboundLossPct?: number; + jitterMs?: number; + }; + activity: { speakingMs: number; mutedMs: number; screenshareMs: number }; +} + +/** Time spent in an on/off state, e.g. muted. */ +class Stopwatch { + private since: number | undefined; + + private total = 0; + + public set(on: boolean, now: number): void { + if (on && this.since === undefined) this.since = now; + if (!on && this.since !== undefined) { + this.total += now - this.since; + this.since = undefined; + } + } + + public get on(): boolean { + return this.since !== undefined; + } + + public ms(now: number): number { + return this.total + (this.since === undefined ? 0 : now - this.since); + } +} + +/** `@user:server:DEVICE` (a LiveKit identity) → `@user:server`. */ +export function matrixUserOf(identity: string): string { + if (!identity.startsWith("@")) return identity; + const first = identity.indexOf(":"); + const second = first === -1 ? -1 : identity.indexOf(":", first + 1); + return second === -1 ? identity : identity.slice(0, second); +} + +const percentile = (sorted: number[], p: number): number => + sorted[Math.min(sorted.length - 1, Math.floor(p * sorted.length))]; + +/** + * [cinny #253] Who was in the call, whose connection was poor and when, + * network numbers, and your own activity. Fed by LiveKit events and periodic + * stats samples; nothing leaves the frame except the final summary, sent to + * the host only. + */ +export class CallDetailsTracker { + // user -> identities (devices) currently in the call + private readonly devices = new Map>(); + + private readonly seen: string[] = []; + + private peak = 1; + + private readonly poor = new Map(); + + private readonly youPoor = new Stopwatch(); + + private readonly everyonePoor = new Stopwatch(); + + private readonly rtts: number[] = []; + + private readonly jitters: number[] = []; + + private readonly sent = new Map(); + + private readonly received = new Map< + string, + { packets: number; lost: number } + >(); + + private readonly speaking = new Stopwatch(); + + private readonly muted = new Stopwatch(); + + private readonly sharing = new Stopwatch(); + + public constructor(private readonly now: () => number = () => Date.now()) {} + + /** Idempotent per device: safe to call for everyone already present too. */ + public remoteJoined(identity: string): void { + const user = matrixUserOf(identity); + let ids = this.devices.get(user); + if (!ids) this.devices.set(user, (ids = new Set())); + ids.add(identity); + if (!this.seen.includes(user)) this.seen.push(user); + this.peak = Math.max(this.peak, this.devices.size + 1); + this.updateEveryone(); + } + + public remoteLeft(identity: string): void { + const user = matrixUserOf(identity); + const ids = this.devices.get(user); + ids?.delete(identity); + if (!ids || ids.size === 0) { + this.devices.delete(user); + this.poor.get(user)?.set(false, this.now()); + } + this.updateEveryone(); + } + + /** `identity` undefined = the local participant. */ + public setQuality( + identity: string | undefined, + quality: ConnectionQuality, + ): void { + if (quality === ConnectionQuality.Unknown) return; + const bad = + quality === ConnectionQuality.Poor || quality === ConnectionQuality.Lost; + if (identity === undefined) this.youPoor.set(bad, this.now()); + else { + const user = matrixUserOf(identity); + let sw = this.poor.get(user); + if (!sw) this.poor.set(user, (sw = new Stopwatch())); + sw.set(bad, this.now()); + } + this.updateEveryone(); + } + + private updateEveryone(): void { + const others = [...this.devices.keys()]; + const all = + this.youPoor.on && + others.length > 0 && + others.every((u) => this.poor.get(u)?.on); + this.everyonePoor.set(all, this.now()); + } + + public addRtt(seconds: number | undefined): void { + if (typeof seconds === "number" && Number.isFinite(seconds) && seconds >= 0) + this.rtts.push(seconds * 1000); + } + + public addJitter(seconds: number | undefined): void { + if (typeof seconds === "number" && Number.isFinite(seconds) && seconds >= 0) + this.jitters.push(seconds * 1000); + } + + /** Cumulative counters for one sent/received track (latest value wins). */ + public setSent(key: string, packets?: number, lost?: number): void { + if (typeof packets === "number") + this.sent.set(key, { packets, lost: Math.max(0, lost ?? 0) }); + } + + public setReceived(key: string, packets?: number, lost?: number): void { + if (typeof packets === "number") + this.received.set(key, { packets, lost: Math.max(0, lost ?? 0) }); + } + + public setSpeaking(on: boolean): void { + this.speaking.set(on, this.now()); + } + + public setMuted(on: boolean): void { + this.muted.set(on, this.now()); + } + + public setScreensharing(on: boolean): void { + this.sharing.set(on, this.now()); + } + + public details(): CallSummaryDetails { + const now = this.now(); + const othersPoorMs: Record = {}; + for (const [user, sw] of this.poor) { + const ms = sw.ms(now); + if (ms > 0) othersPoorMs[user] = ms; + } + const lossPct = ( + m: Map, + ): number | undefined => { + let packets = 0; + let lost = 0; + for (const v of m.values()) { + packets += v.packets; + lost += v.lost; + } + return packets + lost > 0 + ? Math.round((lost / (packets + lost)) * 1000) / 10 + : undefined; + }; + const rtts = [...this.rtts].sort((a, b) => a - b); + const network: CallSummaryDetails["network"] = {}; + if (rtts.length) { + network.rttMedianMs = Math.round(percentile(rtts, 0.5)); + network.rttP95Ms = Math.round(percentile(rtts, 0.95)); + } + const out = lossPct(this.sent); + if (out !== undefined) network.outboundLossPct = out; + const inn = lossPct(this.received); + if (inn !== undefined) network.inboundLossPct = inn; + if (this.jitters.length) + network.jitterMs = Math.round( + this.jitters.reduce((a, b) => a + b, 0) / this.jitters.length, + ); + return { + participants: [...this.seen], + peakParticipants: this.peak, + quality: { + youPoorMs: this.youPoor.ms(now), + othersPoorMs, + everyonePoorMs: this.everyonePoor.ms(now), + }, + network, + activity: { + speakingMs: this.speaking.ms(now), + mutedMs: this.muted.ms(now), + screenshareMs: this.sharing.ms(now), + }, + }; + } } /** @@ -93,13 +335,48 @@ export class CallQualityTracker { export function startLotusCallSummary(vm: CallViewModel): () => void { if (!widget) return () => undefined; const tracker = new CallQualityTracker(); + const details = new CallDetailsTracker(); let sent = false; const send = (): void => { if (sent) return; sent = true; - lotusSendToHost(LotusWidgetActions.CallSummary, tracker.summary()); + clearInterval(sampler); + lotusSendToHost(LotusWidgetActions.CallSummary, { + ...tracker.summary(), + details: details.details(), + }); }; + // [cinny #253] Network numbers: sample sender/receiver stats every 5 s. + const sample = async (room: LivekitRoom): Promise => { + const mic = room.localParticipant.getTrackPublication( + Track.Source.Microphone, + )?.track as LocalAudioTrack | undefined; + const s = await mic?.getSenderStats?.().catch(() => undefined); + if (s) { + details.addRtt(s.roundTripTime); + details.setSent(`${room.name}:mic`, s.packetsSent, s.packetsLost); + } + for (const p of room.remoteParticipants.values()) { + for (const pub of p.audioTrackPublications.values()) { + const t = pub.track as RemoteAudioTrack | undefined; + const r = await t?.getReceiverStats?.().catch(() => undefined); + if (r) { + details.addJitter(r.jitter); + details.setReceived( + `${room.name}:${p.identity}:${pub.trackSid}`, + r.packetsReceived, + r.packetsLost, + ); + } + } + } + }; + const sampler = setInterval(() => { + for (const room of listeners.keys()) + void sample(room).catch(() => undefined); + }, 5_000); + const listeners = new Map void>(); const attach = (room: LivekitRoom): void => { const onQuality = ( @@ -107,20 +384,69 @@ export function startLotusCallSummary(vm: CallViewModel): () => void { participant: Participant, ): void => { if (participant.isLocal) tracker.setQuality(quality); + details.setQuality( + participant.isLocal ? undefined : participant.identity, + quality, + ); + }; + // LiveKit doesn't send ParticipantConnected for people already in the + // room when we connect, and the mic isn't published yet at attach time: + // record both once the connection is up. + const onConnected = (): void => { + tracker.start(); + for (const p of room.remoteParticipants.values()) + details.remoteJoined(p.identity); + details.setMuted(!room.localParticipant.isMicrophoneEnabled); }; - const onConnected = (): void => tracker.start(); const onReconnecting = (): void => tracker.reconnect(); const onDisconnected = (): void => send(); + const onJoined = (p: RemoteParticipant): void => + details.remoteJoined(p.identity); + const onLeft = (p: RemoteParticipant): void => + details.remoteLeft(p.identity); + const isLocalMic = (pub: TrackPublication, p: Participant): boolean => + p.isLocal && pub.source === Track.Source.Microphone; + const onMuted = (pub: TrackPublication, p: Participant): void => { + if (isLocalMic(pub, p)) details.setMuted(true); + }; + const onUnmuted = (pub: TrackPublication, p: Participant): void => { + if (isLocalMic(pub, p)) details.setMuted(false); + }; + const onLocalPublished = (pub: LocalTrackPublication): void => { + if (pub.source === Track.Source.ScreenShare) + details.setScreensharing(true); + if (pub.source === Track.Source.Microphone) details.setMuted(pub.isMuted); + }; + const onLocalUnpublished = (pub: LocalTrackPublication): void => { + if (pub.source === Track.Source.ScreenShare) + details.setScreensharing(false); + }; + const onSpeaking = (speaking: boolean): void => + details.setSpeaking(speaking); room.on(RoomEvent.ConnectionQualityChanged, onQuality); room.on(RoomEvent.Connected, onConnected); room.on(RoomEvent.Reconnecting, onReconnecting); room.on(RoomEvent.Disconnected, onDisconnected); - if (room.state === "connected") tracker.start(); + room.on(RoomEvent.ParticipantConnected, onJoined); + room.on(RoomEvent.ParticipantDisconnected, onLeft); + room.on(RoomEvent.TrackMuted, onMuted); + room.on(RoomEvent.TrackUnmuted, onUnmuted); + room.on(RoomEvent.LocalTrackPublished, onLocalPublished); + room.on(RoomEvent.LocalTrackUnpublished, onLocalUnpublished); + room.localParticipant.on(ParticipantEvent.IsSpeakingChanged, onSpeaking); + if (room.state === "connected") onConnected(); listeners.set(room, () => { room.off(RoomEvent.ConnectionQualityChanged, onQuality); room.off(RoomEvent.Connected, onConnected); room.off(RoomEvent.Reconnecting, onReconnecting); room.off(RoomEvent.Disconnected, onDisconnected); + room.off(RoomEvent.ParticipantConnected, onJoined); + room.off(RoomEvent.ParticipantDisconnected, onLeft); + room.off(RoomEvent.TrackMuted, onMuted); + room.off(RoomEvent.TrackUnmuted, onUnmuted); + room.off(RoomEvent.LocalTrackPublished, onLocalPublished); + room.off(RoomEvent.LocalTrackUnpublished, onLocalUnpublished); + room.localParticipant.off(ParticipantEvent.IsSpeakingChanged, onSpeaking); }); };