Merge pull request 'feat(lotus): detailed call summary for the host (cinny #253)' (#60) from feat/call-summary-details into lotus

LOTUS.md

@@ -43,7 +43,7 @@ checked against cinny's `lotus` branch.
 | `io.lotus.set_deafen`              | host → call | deafen remote audio (and optionally mute screenshare audio).                                                                                                                                                                                                                                                                                                                                                                                        | yes                        |
 | `io.lotus.request_state`           | call → host | sent on (re)registration of a lotus toWidget handler (e.g. decorations) so the host can re-push state that would otherwise only be sent on change (decorations roster, the current focus pin) after an EC-side reconnect/remount. cinny's `resendForkState()` should respond to this the same way it responds to a fresh join.                                                                                                                      | yes                        |
 | `io.lotus.denoise_state`           | call → host | real state of the in-source denoise engine — `{ active: boolean, model: string, error?: string }` — sent once the processor attaches (or after the rnnoise fallback also fails), so the host toggle can reflect reality rather than the requested state.                                                                                                                                                                                            | yes                        |
-| `io.lotus.call_summary`            | call → host | one-shot end-of-call readout for the local participant — `{ durationMs, reconnects, poorMs, verdict }` (#143) — sent on SFU disconnect or in-call teardown, whichever comes first.                                                                                                                                                                                                                                                                  | yes                        |
+| `io.lotus.call_summary`            | call → host | one-shot end-of-call readout `{ durationMs, reconnects, poorMs, verdict, details? }` (#143); `details` (cinny #253): participants, peak, per-person poor time, everyone-poor time, RTT/loss/jitter, your talk/mute/screenshare time.                                                                                                                                                                                                                | yes                        |
 | `io.lotus.set_audio_output`        | host → call | select the audio output device `{ deviceId }` (#119).                                                                                                                                                                                                                                                                                                                                                                                               | yes                        |
 | `io.lotus.audio_output_state`      | call → host | the currently selected output `{ deviceId }` (#119).                                                                                                                                                                                                                                                                                                                                                                                                | cinny #417                 |
 | `io.lotus.screenshare_audio_state` | call → host | `{ present }`: whether any remote screenshare audio is being received, sent on change (cinny #277).                                                                                                                                                                                                                                                                                                                                                 | cinny #277                 |

src/lotus/lotusCallSummary.test.ts

@@ -8,7 +8,11 @@ Please see LICENSE in the repository root for full details.
 import { ConnectionQuality } from "livekit-client";
 import { describe, expect, it } from "vitest";
 
-import { CallQualityTracker } from "./lotusCallSummary";
+import {
+  CallDetailsTracker,
+  CallQualityTracker,
+  matrixUserOf,
+} from "./lotusCallSummary";
 
 const clock = (): { now: () => number; tick: (ms: number) => void } => {
   let t = 1_000_000;
@@ -91,3 +95,92 @@ describe("CallQualityTracker", () => {
     });
   });
 });
+
+describe("[cinny #253] CallDetailsTracker", () => {
+  it("participants, peak, and devices of one person counted once", () => {
+    const c = clock();
+    const d = new CallDetailsTracker(c.now);
+    d.remoteJoined("@bob:x:DEV1");
+    d.remoteJoined("@bob:x:DEV2");
+    d.remoteJoined("@carol:x:D");
+    d.remoteLeft("@carol:x:D");
+    d.remoteJoined("@dave:x:D");
+    const r = d.details();
+    expect(r.participants).toEqual(["@bob:x", "@carol:x", "@dave:x"]);
+    expect(r.peakParticipants).toBe(3); // you + bob + carol (dave came after carol left)
+  });
+
+  it("joining is idempotent per device (seeding everyone present on connect)", () => {
+    const d = new CallDetailsTracker();
+    d.remoteJoined("@bob:x:DEV1");
+    d.remoteJoined("@bob:x:DEV1"); // seen again by the on-connect seed
+    d.remoteLeft("@bob:x:DEV1"); // one leave: bob is gone
+    d.remoteJoined("@carol:x:D");
+    d.remoteJoined("@dave:x:D"); // peak is you + carol + dave, not + a ghost bob
+    d.setQuality("@bob:x:DEV1", ConnectionQuality.Poor);
+    expect(d.details().participants).toEqual(["@bob:x", "@carol:x", "@dave:x"]);
+    expect(d.details().peakParticipants).toBe(3);
+  });
+
+  it("poor time per person, and everyone-at-once only while all are poor", () => {
+    const c = clock();
+    const d = new CallDetailsTracker(c.now);
+    d.remoteJoined("@bob:x:D");
+    d.setQuality(undefined, ConnectionQuality.Poor); // you poor
+    c.tick(10_000);
+    d.setQuality("@bob:x:D", ConnectionQuality.Lost); // now everyone
+    c.tick(5_000);
+    d.setQuality(undefined, ConnectionQuality.Good);
+    c.tick(3_000);
+    d.setQuality("@bob:x:D", ConnectionQuality.Excellent);
+    const q = d.details().quality;
+    expect(q.youPoorMs).toBe(15_000);
+    expect(q.othersPoorMs).toEqual({ "@bob:x": 8_000 });
+    expect(q.everyonePoorMs).toBe(5_000);
+  });
+
+  it("network numbers: RTT median/p95, loss from the latest counters, jitter", () => {
+    const d = new CallDetailsTracker();
+    for (const rtt of [0.02, 0.03, 0.04, 0.05, 0.3]) d.addRtt(rtt);
+    d.addRtt(Number.NaN);
+    d.setSent("mic", 900, 5);
+    d.setSent("mic", 995, 5); // cumulative: latest wins
+    d.setReceived("bob", 490, 10);
+    d.addJitter(0.004);
+    d.addJitter(0.006);
+    const n = d.details().network;
+    expect(n.rttMedianMs).toBe(40);
+    expect(n.rttP95Ms).toBe(300);
+    expect(n.outboundLossPct).toBe(0.5);
+    expect(n.inboundLossPct).toBe(2);
+    expect(n.jitterMs).toBe(5);
+  });
+
+  it("no samples: network numbers are absent, not zero", () => {
+    expect(new CallDetailsTracker().details().network).toEqual({});
+  });
+
+  it("talk, mute and screenshare time", () => {
+    const c = clock();
+    const d = new CallDetailsTracker(c.now);
+    d.setMuted(true);
+    c.tick(4_000);
+    d.setMuted(false);
+    d.setSpeaking(true);
+    c.tick(2_000);
+    d.setSpeaking(false);
+    d.setScreensharing(true);
+    c.tick(6_000); // still sharing when summarised
+    expect(d.details().activity).toEqual({
+      speakingMs: 2_000,
+      mutedMs: 4_000,
+      screenshareMs: 6_000,
+    });
+  });
+
+  it("matrixUserOf reduces a LiveKit identity to the Matrix user", () => {
+    expect(matrixUserOf("@bob:example.org:ABCDEF")).toBe("@bob:example.org");
+    expect(matrixUserOf("@bob:example.org")).toBe("@bob:example.org");
+    expect(matrixUserOf("hashed-federated-id")).toBe("hashed-federated-id");
+  });
+});

src/lotus/lotusCallSummary.ts

@@ -7,9 +7,16 @@ Please see LICENSE in the repository root for full details.
 
 import {
   ConnectionQuality,
+  type LocalAudioTrack,
+  type LocalTrackPublication,
   type Participant,
+  ParticipantEvent,
+  type RemoteAudioTrack,
+  type RemoteParticipant,
   type Room as LivekitRoom,
   RoomEvent,
+  Track,
+  type TrackPublication,
 } from "livekit-client";
 
 import { type CallViewModel } from "../state/CallViewModel/CallViewModel";
@@ -27,6 +34,241 @@ export interface CallSummary {
   /** Time the local connection quality was reported poor/lost, ms. */
   poorMs: number;
   verdict: CallQualityVerdict;
+  /** [cinny #253] The breakdown behind the host's "Details" card. */
+  details?: CallSummaryDetails;
+}
+
+/** [cinny #253] Everything the host's post-call Details card shows. */
+export interface CallSummaryDetails {
+  /** Other Matrix users seen in the call (`@user:server`), first-seen order. */
+  participants: string[];
+  /** Most people in the call at once, you included. */
+  peakParticipants: number;
+  quality: {
+    /** Your connection reported poor/lost, ms. */
+    youPoorMs: number;
+    /** Each other person's connection reported poor/lost, ms (only > 0). */
+    othersPoorMs: Record<string, number>;
+    /** Everyone, you included, poor at the same time (points at the server). */
+    everyonePoorMs: number;
+  };
+  network: {
+    rttMedianMs?: number;
+    rttP95Ms?: number;
+    /** Your microphone's packets the SFU reported lost, %. */
+    outboundLossPct?: number;
+    /** Packets lost on the audio you received, %. */
+    inboundLossPct?: number;
+    jitterMs?: number;
+  };
+  activity: { speakingMs: number; mutedMs: number; screenshareMs: number };
+}
+
+/** Time spent in an on/off state, e.g. muted. */
+class Stopwatch {
+  private since: number | undefined;
+
+  private total = 0;
+
+  public set(on: boolean, now: number): void {
+    if (on && this.since === undefined) this.since = now;
+    if (!on && this.since !== undefined) {
+      this.total += now - this.since;
+      this.since = undefined;
+    }
+  }
+
+  public get on(): boolean {
+    return this.since !== undefined;
+  }
+
+  public ms(now: number): number {
+    return this.total + (this.since === undefined ? 0 : now - this.since);
+  }
+}
+
+/** `@user:server:DEVICE` (a LiveKit identity) → `@user:server`. */
+export function matrixUserOf(identity: string): string {
+  if (!identity.startsWith("@")) return identity;
+  const first = identity.indexOf(":");
+  const second = first === -1 ? -1 : identity.indexOf(":", first + 1);
+  return second === -1 ? identity : identity.slice(0, second);
+}
+
+const percentile = (sorted: number[], p: number): number =>
+  sorted[Math.min(sorted.length - 1, Math.floor(p * sorted.length))];
+
+/**
+ * [cinny #253] Who was in the call, whose connection was poor and when,
+ * network numbers, and your own activity. Fed by LiveKit events and periodic
+ * stats samples; nothing leaves the frame except the final summary, sent to
+ * the host only.
+ */
+export class CallDetailsTracker {
+  // user -> identities (devices) currently in the call
+  private readonly devices = new Map<string, Set<string>>();
+
+  private readonly seen: string[] = [];
+
+  private peak = 1;
+
+  private readonly poor = new Map<string, Stopwatch>();
+
+  private readonly youPoor = new Stopwatch();
+
+  private readonly everyonePoor = new Stopwatch();
+
+  private readonly rtts: number[] = [];
+
+  private readonly jitters: number[] = [];
+
+  private readonly sent = new Map<string, { packets: number; lost: number }>();
+
+  private readonly received = new Map<
+    string,
+    { packets: number; lost: number }
+  >();
+
+  private readonly speaking = new Stopwatch();
+
+  private readonly muted = new Stopwatch();
+
+  private readonly sharing = new Stopwatch();
+
+  public constructor(private readonly now: () => number = () => Date.now()) {}
+
+  /** Idempotent per device: safe to call for everyone already present too. */
+  public remoteJoined(identity: string): void {
+    const user = matrixUserOf(identity);
+    let ids = this.devices.get(user);
+    if (!ids) this.devices.set(user, (ids = new Set()));
+    ids.add(identity);
+    if (!this.seen.includes(user)) this.seen.push(user);
+    this.peak = Math.max(this.peak, this.devices.size + 1);
+    this.updateEveryone();
+  }
+
+  public remoteLeft(identity: string): void {
+    const user = matrixUserOf(identity);
+    const ids = this.devices.get(user);
+    ids?.delete(identity);
+    if (!ids || ids.size === 0) {
+      this.devices.delete(user);
+      this.poor.get(user)?.set(false, this.now());
+    }
+    this.updateEveryone();
+  }
+
+  /** `identity` undefined = the local participant. */
+  public setQuality(
+    identity: string | undefined,
+    quality: ConnectionQuality,
+  ): void {
+    if (quality === ConnectionQuality.Unknown) return;
+    const bad =
+      quality === ConnectionQuality.Poor || quality === ConnectionQuality.Lost;
+    if (identity === undefined) this.youPoor.set(bad, this.now());
+    else {
+      const user = matrixUserOf(identity);
+      let sw = this.poor.get(user);
+      if (!sw) this.poor.set(user, (sw = new Stopwatch()));
+      sw.set(bad, this.now());
+    }
+    this.updateEveryone();
+  }
+
+  private updateEveryone(): void {
+    const others = [...this.devices.keys()];
+    const all =
+      this.youPoor.on &&
+      others.length > 0 &&
+      others.every((u) => this.poor.get(u)?.on);
+    this.everyonePoor.set(all, this.now());
+  }
+
+  public addRtt(seconds: number | undefined): void {
+    if (typeof seconds === "number" && Number.isFinite(seconds) && seconds >= 0)
+      this.rtts.push(seconds * 1000);
+  }
+
+  public addJitter(seconds: number | undefined): void {
+    if (typeof seconds === "number" && Number.isFinite(seconds) && seconds >= 0)
+      this.jitters.push(seconds * 1000);
+  }
+
+  /** Cumulative counters for one sent/received track (latest value wins). */
+  public setSent(key: string, packets?: number, lost?: number): void {
+    if (typeof packets === "number")
+      this.sent.set(key, { packets, lost: Math.max(0, lost ?? 0) });
+  }
+
+  public setReceived(key: string, packets?: number, lost?: number): void {
+    if (typeof packets === "number")
+      this.received.set(key, { packets, lost: Math.max(0, lost ?? 0) });
+  }
+
+  public setSpeaking(on: boolean): void {
+    this.speaking.set(on, this.now());
+  }
+
+  public setMuted(on: boolean): void {
+    this.muted.set(on, this.now());
+  }
+
+  public setScreensharing(on: boolean): void {
+    this.sharing.set(on, this.now());
+  }
+
+  public details(): CallSummaryDetails {
+    const now = this.now();
+    const othersPoorMs: Record<string, number> = {};
+    for (const [user, sw] of this.poor) {
+      const ms = sw.ms(now);
+      if (ms > 0) othersPoorMs[user] = ms;
+    }
+    const lossPct = (
+      m: Map<string, { packets: number; lost: number }>,
+    ): number | undefined => {
+      let packets = 0;
+      let lost = 0;
+      for (const v of m.values()) {
+        packets += v.packets;
+        lost += v.lost;
+      }
+      return packets + lost > 0
+        ? Math.round((lost / (packets + lost)) * 1000) / 10
+        : undefined;
+    };
+    const rtts = [...this.rtts].sort((a, b) => a - b);
+    const network: CallSummaryDetails["network"] = {};
+    if (rtts.length) {
+      network.rttMedianMs = Math.round(percentile(rtts, 0.5));
+      network.rttP95Ms = Math.round(percentile(rtts, 0.95));
+    }
+    const out = lossPct(this.sent);
+    if (out !== undefined) network.outboundLossPct = out;
+    const inn = lossPct(this.received);
+    if (inn !== undefined) network.inboundLossPct = inn;
+    if (this.jitters.length)
+      network.jitterMs = Math.round(
+        this.jitters.reduce((a, b) => a + b, 0) / this.jitters.length,
+      );
+    return {
+      participants: [...this.seen],
+      peakParticipants: this.peak,
+      quality: {
+        youPoorMs: this.youPoor.ms(now),
+        othersPoorMs,
+        everyonePoorMs: this.everyonePoor.ms(now),
+      },
+      network,
+      activity: {
+        speakingMs: this.speaking.ms(now),
+        mutedMs: this.muted.ms(now),
+        screenshareMs: this.sharing.ms(now),
+      },
+    };
+  }
 }
 
 /**
@@ -93,13 +335,48 @@ export class CallQualityTracker {
 export function startLotusCallSummary(vm: CallViewModel): () => void {
   if (!widget) return () => undefined;
   const tracker = new CallQualityTracker();
+  const details = new CallDetailsTracker();
   let sent = false;
   const send = (): void => {
     if (sent) return;
     sent = true;
-    lotusSendToHost(LotusWidgetActions.CallSummary, tracker.summary());
+    clearInterval(sampler);
+    lotusSendToHost(LotusWidgetActions.CallSummary, {
+      ...tracker.summary(),
+      details: details.details(),
+    });
   };
 
+  // [cinny #253] Network numbers: sample sender/receiver stats every 5 s.
+  const sample = async (room: LivekitRoom): Promise<void> => {
+    const mic = room.localParticipant.getTrackPublication(
+      Track.Source.Microphone,
+    )?.track as LocalAudioTrack | undefined;
+    const s = await mic?.getSenderStats?.().catch(() => undefined);
+    if (s) {
+      details.addRtt(s.roundTripTime);
+      details.setSent(`${room.name}:mic`, s.packetsSent, s.packetsLost);
+    }
+    for (const p of room.remoteParticipants.values()) {
+      for (const pub of p.audioTrackPublications.values()) {
+        const t = pub.track as RemoteAudioTrack | undefined;
+        const r = await t?.getReceiverStats?.().catch(() => undefined);
+        if (r) {
+          details.addJitter(r.jitter);
+          details.setReceived(
+            `${room.name}:${p.identity}:${pub.trackSid}`,
+            r.packetsReceived,
+            r.packetsLost,
+          );
+        }
+      }
+    }
+  };
+  const sampler = setInterval(() => {
+    for (const room of listeners.keys())
+      void sample(room).catch(() => undefined);
+  }, 5_000);
+
   const listeners = new Map<LivekitRoom, () => void>();
   const attach = (room: LivekitRoom): void => {
     const onQuality = (
@@ -107,20 +384,69 @@ export function startLotusCallSummary(vm: CallViewModel): () => void {
       participant: Participant,
     ): void => {
       if (participant.isLocal) tracker.setQuality(quality);
+      details.setQuality(
+        participant.isLocal ? undefined : participant.identity,
+        quality,
+      );
+    };
+    // LiveKit doesn't send ParticipantConnected for people already in the
+    // room when we connect, and the mic isn't published yet at attach time:
+    // record both once the connection is up.
+    const onConnected = (): void => {
+      tracker.start();
+      for (const p of room.remoteParticipants.values())
+        details.remoteJoined(p.identity);
+      details.setMuted(!room.localParticipant.isMicrophoneEnabled);
     };
-    const onConnected = (): void => tracker.start();
     const onReconnecting = (): void => tracker.reconnect();
     const onDisconnected = (): void => send();
+    const onJoined = (p: RemoteParticipant): void =>
+      details.remoteJoined(p.identity);
+    const onLeft = (p: RemoteParticipant): void =>
+      details.remoteLeft(p.identity);
+    const isLocalMic = (pub: TrackPublication, p: Participant): boolean =>
+      p.isLocal && pub.source === Track.Source.Microphone;
+    const onMuted = (pub: TrackPublication, p: Participant): void => {
+      if (isLocalMic(pub, p)) details.setMuted(true);
+    };
+    const onUnmuted = (pub: TrackPublication, p: Participant): void => {
+      if (isLocalMic(pub, p)) details.setMuted(false);
+    };
+    const onLocalPublished = (pub: LocalTrackPublication): void => {
+      if (pub.source === Track.Source.ScreenShare)
+        details.setScreensharing(true);
+      if (pub.source === Track.Source.Microphone) details.setMuted(pub.isMuted);
+    };
+    const onLocalUnpublished = (pub: LocalTrackPublication): void => {
+      if (pub.source === Track.Source.ScreenShare)
+        details.setScreensharing(false);
+    };
+    const onSpeaking = (speaking: boolean): void =>
+      details.setSpeaking(speaking);
     room.on(RoomEvent.ConnectionQualityChanged, onQuality);
     room.on(RoomEvent.Connected, onConnected);
     room.on(RoomEvent.Reconnecting, onReconnecting);
     room.on(RoomEvent.Disconnected, onDisconnected);
-    if (room.state === "connected") tracker.start();
+    room.on(RoomEvent.ParticipantConnected, onJoined);
+    room.on(RoomEvent.ParticipantDisconnected, onLeft);
+    room.on(RoomEvent.TrackMuted, onMuted);
+    room.on(RoomEvent.TrackUnmuted, onUnmuted);
+    room.on(RoomEvent.LocalTrackPublished, onLocalPublished);
+    room.on(RoomEvent.LocalTrackUnpublished, onLocalUnpublished);
+    room.localParticipant.on(ParticipantEvent.IsSpeakingChanged, onSpeaking);
+    if (room.state === "connected") onConnected();
     listeners.set(room, () => {
       room.off(RoomEvent.ConnectionQualityChanged, onQuality);
       room.off(RoomEvent.Connected, onConnected);
       room.off(RoomEvent.Reconnecting, onReconnecting);
       room.off(RoomEvent.Disconnected, onDisconnected);
+      room.off(RoomEvent.ParticipantConnected, onJoined);
+      room.off(RoomEvent.ParticipantDisconnected, onLeft);
+      room.off(RoomEvent.TrackMuted, onMuted);
+      room.off(RoomEvent.TrackUnmuted, onUnmuted);
+      room.off(RoomEvent.LocalTrackPublished, onLocalPublished);
+      room.off(RoomEvent.LocalTrackUnpublished, onLocalUnpublished);
+      room.localParticipant.off(ParticipantEvent.IsSpeakingChanged, onSpeaking);
     });
   };