fix(lotus): screenshare bitrate cap is a budget across simulcast layers

The same maxBitrate was written into every encoding, so a 1.5 Mbps cap
could mean 4.5 Mbps aggregate. Distribute proportionally to the layers'
existing ratios (floor, so never over). Tested with a 3-layer case.

Fixes #12

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01PPmy3tPq869XDW4njjVaKA

src/lotus/lotusQuality.test.ts

@@ -7,7 +7,11 @@ Please see LICENSE in the repository root for full details.
 
 import { describe, expect, test, vi } from "vitest";
 
-import { buildPatch, patchSender } from "./lotusQuality";
+import {
+  applyProportionalMaxBitrate,
+  buildPatch,
+  patchSender,
+} from "./lotusQuality";
 
 function makeSender(
   initialEncodings: RTCRtpEncodingParameters[] = [{}],
@@ -91,3 +95,65 @@ describe("lotusQuality set_quality -> clear (#11)", () => {
     );
   });
 });
+
+describe("lotusQuality screenshare simulcast cap (#12)", () => {
+  test("caps the aggregate across a 3-layer simulcast publication, not per-layer", async () => {
+    // A typical VP8 simulcast screenshare publication: low/medium/high
+    // layers with existing presets, ordered low-to-high like LiveKit
+    // publishes them.
+    const sender = makeSender([
+      { maxBitrate: 150_000 },
+      { maxBitrate: 500_000 },
+      { maxBitrate: 1_500_000 },
+    ]);
+    const writtenKeys = new WeakMap<
+      RTCRtpSender,
+      Set<keyof RTCRtpEncodingParameters>
+    >();
+
+    const cap = 750_000;
+    const patch = buildPatch(sender, { maxBitrate: cap }, writtenKeys);
+    await patchSender(sender, patch, writtenKeys);
+
+    const encodings = sender.getParameters().encodings;
+    expect(encodings).toHaveLength(3);
+    // Each layer keeps its old proportion of the total, but the aggregate
+    // across all layers must not exceed the requested cap — that's the bug:
+    // writing `cap` into every layer let the aggregate reach ~3x the cap.
+    const oldTotal = 150_000 + 500_000 + 1_500_000;
+    expect(encodings[0].maxBitrate).toBe(
+      Math.floor((150_000 / oldTotal) * cap),
+    );
+    expect(encodings[1].maxBitrate).toBe(
+      Math.floor((500_000 / oldTotal) * cap),
+    );
+    expect(encodings[2].maxBitrate).toBe(
+      Math.floor((1_500_000 / oldTotal) * cap),
+    );
+    // The highest layer still ends up with the largest share.
+    expect(encodings[2].maxBitrate).toBeGreaterThan(encodings[1].maxBitrate!);
+    expect(encodings[1].maxBitrate).toBeGreaterThan(encodings[0].maxBitrate!);
+
+    const aggregate = encodings.reduce(
+      (sum, enc) => sum + (enc.maxBitrate ?? 0),
+      0,
+    );
+    expect(aggregate).toBeLessThanOrEqual(cap);
+  });
+
+  test("a single encoding keeps the previous (non-scaled) behaviour", () => {
+    const encodings: RTCRtpEncodingParameters[] = [{ maxBitrate: 2_000_000 }];
+    applyProportionalMaxBitrate(encodings, 500_000);
+    expect(encodings).toEqual([{ maxBitrate: 500_000 }]);
+  });
+
+  test("falls back to capping only the highest layer when there are no prior ratios", () => {
+    const encodings: RTCRtpEncodingParameters[] = [
+      {}, // no prior cap on any layer to derive a ratio from
+      {},
+    ];
+    applyProportionalMaxBitrate(encodings, 500_000);
+    expect(encodings[0].maxBitrate).toBeUndefined();
+    expect(encodings[1].maxBitrate).toBe(500_000);
+  });
+});

src/lotus/lotusQuality.ts

@@ -198,6 +198,43 @@ export function buildPatch(
   return patch;
 }
 
+// [lotus] Treat `cap` as a budget for the WHOLE simulcast publication rather
+// than a per-encoding value, and distribute it across encodings in
+// proportion to their existing (pre-patch) ratios — instead of writing `cap`
+// into every encoding (#12), which let the aggregate reach up to N× the
+// requested cap (and also raised the low layers far above their presets).
+// With a single encoding this reduces to the previous behaviour: that
+// encoding simply gets `cap`.
+// Exported for unit testing only (see lotusQuality.test.ts) — not part of
+// the module's public surface used by callers.
+export function applyProportionalMaxBitrate(
+  encodings: RTCRtpEncodingParameters[],
+  cap: number,
+): void {
+  if (encodings.length <= 1) {
+    for (const enc of encodings) enc.maxBitrate = cap;
+    return;
+  }
+  const total = encodings.reduce(
+    (sum, enc) =>
+      sum + (typeof enc.maxBitrate === "number" ? enc.maxBitrate : 0),
+    0,
+  );
+  if (total <= 0) {
+    // No existing ratios to scale from: fall back to capping only the
+    // highest layer (LiveKit orders simulcast encodings low-to-high, so
+    // that's the last one) and leave the others alone rather than guess.
+    encodings[encodings.length - 1].maxBitrate = cap;
+    return;
+  }
+  // Math.floor (not round) so rounding error can only push the aggregate
+  // under the cap, never over it.
+  for (const enc of encodings) {
+    const prev = typeof enc.maxBitrate === "number" ? enc.maxBitrate : 0;
+    enc.maxBitrate = Math.floor((prev / total) * cap);
+  }
+}
+
 // Exported for unit testing only (see lotusQuality.test.ts) — not part of
 // the module's public surface used by callers.
 export async function patchSender(
@@ -210,10 +247,26 @@ export async function patchSender(
     const params = sender.getParameters();
     if (!params.encodings || params.encodings.length === 0)
       params.encodings = [{}];
-    // Apply to EVERY encoding, not just encodings[0]: screenshare publishes
-    // with simulcast (VP8), so encodings[0] is the small layer and the
-    // full-resolution layer — the real bandwidth hog — is a later encoding.
-    for (const enc of params.encodings) Object.assign(enc, patch);
+    // Screenshare publishes with simulcast (VP8): encodings[0] is the small
+    // layer and the full-resolution layer — the real bandwidth hog — is a
+    // later encoding. maxFramerate (and an `undefined` clear) still apply to
+    // every encoding, but maxBitrate is special-cased (#12): writing the same
+    // value into every encoding let the aggregate reach N× the requested cap
+    // and also raised the low layers far above their presets, so instead we
+    // treat the requested value as a budget for the whole publication and
+    // distribute it across encodings proportionally to their existing
+    // ratios.
+    const { maxBitrate, ...rest } = patch;
+    for (const enc of params.encodings) Object.assign(enc, rest);
+    if ("maxBitrate" in patch) {
+      if (maxBitrate === undefined) {
+        // Clearing the cap (#11): drop it from every encoding rather than
+        // scaling — there's no budget to distribute.
+        for (const enc of params.encodings) enc.maxBitrate = undefined;
+      } else {
+        applyProportionalMaxBitrate(params.encodings, maxBitrate);
+      }
+    }
     await sender.setParameters(params);
     // Remember only the caps that are still active (defined) after this
     // write, so a later clear knows exactly which keys to unset.