feat(e2ee): tell the user when their device's key uploads are stuck (#201 KE-1)

When the server keeps rejecting this device's /keys/upload with 400 "One
time key ... already exists", the device's encryption store has diverged
and the SDK retries forever (matrix-rust-sdk#5200, still open). Synapse
saw 474k of these from May 30 to Jul 8 2026. The fix is a fresh login, so:

- keyUploadWatch observes /keys/upload responses through the client's
  fetchFn (the response the SDK gets is untouched; a clone is read).
  Three conflicts within 15 min = stuck; any successful upload clears it.
- KeyUploadBanner (critical strip, same slot as the sync/clock banners):
  "Encryption on this device is out of sync" with Details and
  "Fix: sign out…", which opens the normal logout dialog (it already
  warns when the device isn't verified / backed up).

Server side, Grafana now alerts on the same signal (E2EE Key Upload
Conflicts) and on delayed-event 5xx (Call Delayed-Event Errors).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_017PCb59uY3UKqDqt7B6hzd5

src/app/pages/client/ClientRoot.tsx

@@ -46,6 +46,7 @@ import { useSyncState } from '../../hooks/useSyncState';
 import { stopPropagation } from '../../utils/keyboard';
 import { SyncStatus } from './SyncStatus';
 import { ClockSkewBanner } from './ClockSkewBanner';
+import { KeyUploadBanner } from './KeyUploadBanner';
 import {
   ServerStatusBanner,
   ServerStatusFeature,
@@ -260,6 +261,12 @@ export function ClientRoot({ children }: ClientRootProps) {
         {mx && !syncError && <ServerStatusBanner />}
         {mx && !syncError && <SyncStatus mx={mx} />}
         {mx && !syncError && <ClockSkewBanner />}
+        {mx && !syncError && (
+          // The fix opens LogoutDialog, which reads the client from context.
+          <MatrixClientProvider value={mx}>
+            <KeyUploadBanner />
+          </MatrixClientProvider>
+        )}
         {loading && <ClientRootOptions mx={mx} />}
         {(loadState.status === AsyncStatus.Error ||
           startState.status === AsyncStatus.Error ||

src/app/pages/client/KeyUploadBanner.tsx

@@ -0,0 +1,104 @@
+import React, { useState, useSyncExternalStore } from 'react';
+import FocusTrap from 'focus-trap-react';
+import { Box, Button, config, Line, Overlay, OverlayBackdrop, OverlayCenter, Text } from 'folds';
+import { ContainerColor } from '../../styles/ContainerColor.css';
+import { getKeyUploadWatch, subscribeKeyUploadWatch } from '../../utils/keyUploadWatch';
+import { LogoutDialog } from '../../components/LogoutDialog';
+import { stopPropagation } from '../../utils/keyboard';
+
+/**
+ * [Gitea #201 KE-1] "Encryption on this device is out of sync." Shown while
+ * the server keeps rejecting this device's key uploads (keyUploadWatch). The
+ * fix is a fresh login, so the action opens the normal logout dialog, which
+ * already warns if the device isn't verified / backed up. Hideable for this
+ * session only: the problem doesn't go away by itself.
+ */
+export function KeyUploadBanner() {
+  const { stuck, lastError } = useSyncExternalStore(subscribeKeyUploadWatch, getKeyUploadWatch);
+  const [hidden, setHidden] = useState(false);
+  const [details, setDetails] = useState(false);
+  const [logout, setLogout] = useState(false);
+
+  if (!stuck || hidden) return null;
+
+  return (
+    <Box direction="Column" shrink="No">
+      <Box
+        className={ContainerColor({ variant: 'Critical' })}
+        style={{ padding: `${config.space.S100} ${config.space.S300}` }}
+        direction="Column"
+        alignItems="Center"
+        gap="100"
+        role="alert"
+      >
+        <Box alignItems="Center" gap="300" wrap="Wrap" justifyContent="Center">
+          <Text size="L400" align="Center">
+            Encryption on this device is out of sync with the server. New encrypted messages and
+            calls may fail until you sign out and back in.
+          </Text>
+          <Button
+            size="300"
+            variant="Critical"
+            fill="Solid"
+            radii="300"
+            onClick={() => setLogout(true)}
+          >
+            <Text size="B300">Fix: sign out…</Text>
+          </Button>
+          <Button
+            size="300"
+            variant="Critical"
+            fill="Soft"
+            radii="300"
+            onClick={() => setDetails((v) => !v)}
+            aria-expanded={details}
+          >
+            <Text size="B300">Details</Text>
+          </Button>
+          <Button
+            size="300"
+            variant="Critical"
+            fill="None"
+            radii="300"
+            onClick={() => setHidden(true)}
+          >
+            <Text size="B300">Hide</Text>
+          </Button>
+        </Box>
+        {details && (
+          <Box direction="Column" gap="100" style={{ maxWidth: 640 }}>
+            <Text size="T200" align="Center">
+              The server already has encryption keys this device thinks it hasn&apos;t sent, so it
+              rejects every new upload and other people can&apos;t start fresh encrypted sessions
+              with you. This usually happens after the browser cleared this site&apos;s storage
+              while you stayed signed in. Before signing out, make sure this device is verified or
+              key backup is on (Settings → Devices), so your message history comes back when you
+              sign in again.
+            </Text>
+            {lastError && (
+              <Text size="T200" align="Center" priority="300" style={{ wordBreak: 'break-word' }}>
+                Server said: {lastError}
+              </Text>
+            )}
+          </Box>
+        )}
+      </Box>
+      <Line variant="Critical" size="300" />
+      {logout && (
+        <Overlay open backdrop={<OverlayBackdrop />}>
+          <OverlayCenter>
+            <FocusTrap
+              focusTrapOptions={{
+                onDeactivate: () => setLogout(false),
+                clickOutsideDeactivates: true,
+                escapeDeactivates: stopPropagation,
+              }}
+            >
+              <LogoutDialog handleClose={() => setLogout(false)} />
+            </FocusTrap>
+          </OverlayCenter>
+        </Overlay>
+      )}
+    </Box>
+  );
+}

src/app/utils/keyUploadWatch.test.ts

@@ -0,0 +1,86 @@
+import { test } from 'node:test';
+import assert from 'node:assert/strict';
+import {
+  getKeyUploadWatch,
+  isKeyUploadUrl,
+  isOtkConflict,
+  recordKeyUpload,
+  resetKeyUploadWatch,
+  STUCK_AFTER,
+  subscribeKeyUploadWatch,
+  WINDOW_MS,
+  watchKeyUploads,
+} from './keyUploadWatch';
+
+const CONFLICT = JSON.stringify({
+  errcode: 'M_UNKNOWN',
+  error:
+    'One time key signed_curve25519:AAAAAAAAAA0 already exists. Old key: {...}; new key: {...}',
+});
+
+test('[#201] recognises the upload endpoint and the conflict body', () => {
+  assert.equal(isKeyUploadUrl('https://hs.example/_matrix/client/v3/keys/upload'), true);
+  assert.equal(isKeyUploadUrl('https://hs.example/_matrix/client/v3/keys/upload/'), true);
+  assert.equal(isKeyUploadUrl('https://hs.example/_matrix/client/v3/keys/query'), false);
+  assert.equal(isKeyUploadUrl('https://hs.example/_matrix/client/v3/room_keys/keys'), false);
+  assert.equal(isOtkConflict(400, CONFLICT), true);
+  assert.equal(isOtkConflict(400, '{"errcode":"M_BAD_JSON"}'), false);
+  assert.equal(isOtkConflict(500, CONFLICT), false);
+});
+
+test('[#201] stuck only after repeated conflicts inside the window', () => {
+  resetKeyUploadWatch();
+  const t0 = 1_000_000;
+  for (let i = 0; i < STUCK_AFTER - 1; i += 1) recordKeyUpload(400, CONFLICT, t0 + i * 1000);
+  assert.equal(getKeyUploadWatch().stuck, false, 'a couple of races is not divergence');
+  recordKeyUpload(400, CONFLICT, t0 + 5000);
+  assert.equal(getKeyUploadWatch().stuck, true);
+  assert.match(getKeyUploadWatch().lastError ?? '', /already exists/);
+});
+
+test('[#201] old conflicts age out; other 400s are ignored', () => {
+  resetKeyUploadWatch();
+  recordKeyUpload(400, CONFLICT, 0);
+  recordKeyUpload(400, CONFLICT, 1000);
+  recordKeyUpload(400, CONFLICT, WINDOW_MS + 60_000);
+  assert.equal(getKeyUploadWatch().stuck, false);
+  recordKeyUpload(400, '{"errcode":"M_BAD_JSON"}', WINDOW_MS + 61_000);
+  assert.equal(getKeyUploadWatch().conflicts, 1);
+});
+
+test('[#201] a successful upload clears it and notifies', () => {
+  resetKeyUploadWatch();
+  let calls = 0;
+  const off = subscribeKeyUploadWatch(() => {
+    calls += 1;
+  });
+  for (let i = 0; i < STUCK_AFTER; i += 1) recordKeyUpload(400, CONFLICT, i);
+  assert.equal(getKeyUploadWatch().stuck, true);
+  recordKeyUpload(200, '');
+  assert.equal(getKeyUploadWatch().stuck, false);
+  assert.ok(calls >= 2);
+  off();
+});
+
+test('[#201] the fetch wrapper observes without changing the response', async () => {
+  resetKeyUploadWatch();
+  const fake: typeof fetch = async (input) =>
+    String(input).endsWith('/keys/upload')
+      ? new Response(CONFLICT, { status: 400 })
+      : new Response('{}', { status: 200 });
+  const f = watchKeyUploads(fake);
+  const responses = await Promise.all(
+    Array.from({ length: STUCK_AFTER }, () =>
+      f('https://hs.example/_matrix/client/v3/keys/upload', { method: 'POST' }),
+    ),
+  );
+  const bodies = await Promise.all(responses.map((res) => res.text()));
+  responses.forEach((res) => assert.equal(res.status, 400));
+  bodies.forEach((b) => assert.match(b, /already exists/, 'the SDK can still read the body'));
+  await new Promise((r) => {
+    setTimeout(r, 10);
+  });
+  assert.equal(getKeyUploadWatch().stuck, true);
+  await f('https://hs.example/_matrix/client/v3/sync');
+  assert.equal(getKeyUploadWatch().stuck, true, 'unrelated requests change nothing');
+});

src/app/utils/keyUploadWatch.ts

@@ -0,0 +1,102 @@
+/**
+ * [Gitea #201 KE-1] Notice when this device's encryption store has diverged
+ * from the server: every POST /keys/upload answers 400 "One time key ...
+ * already exists", and the SDK retries forever (matrix-rust-sdk#5200, open),
+ * so new encrypted messages and calls quietly degrade. The server saw 474k of
+ * these between May 30 and Jul 8 2026. The only fix is a fresh login on that
+ * device, so we tell the user (KeyUploadBanner) instead of looping silently.
+ *
+ * Watched at the fetch layer (MatrixClient `fetchFn`), so it doesn't depend on
+ * SDK log wording. Pure state + a tiny subscribe API; no network I/O of its own.
+ */
+
+/** Conflicts needed before we call the device stuck. The SDK retries within
+ * seconds, so a real divergence crosses this quickly; a one-off race doesn't. */
+export const STUCK_AFTER = 3;
+/** Conflicts older than this don't count (a long-ago blip isn't "now"). */
+export const WINDOW_MS = 15 * 60 * 1000;
+
+export type KeyUploadWatchState = { stuck: boolean; conflicts: number; lastError?: string };
+
+let conflictTimes: number[] = [];
+let lastError: string | undefined;
+let state: KeyUploadWatchState = { stuck: false, conflicts: 0 };
+const listeners = new Set<() => void>();
+
+const publish = (next: KeyUploadWatchState) => {
+  if (next.stuck === state.stuck && next.conflicts === state.conflicts) return;
+  state = next;
+  listeners.forEach((l) => l());
+};
+
+export const isKeyUploadUrl = (url: string): boolean => {
+  try {
+    return /\/_matrix\/client\/[^/]+\/keys\/upload\/?$/.test(new URL(url, 'http://x').pathname);
+  } catch {
+    return false;
+  }
+};
+
+/** The 400 body that marks a diverged one-time-key store. */
+export const isOtkConflict = (status: number, body: string): boolean =>
+  status === 400 && /one time key .*already exists/i.test(body);
+
+/** Feed one /keys/upload response. `now` is injectable for tests. */
+export const recordKeyUpload = (status: number, body: string, now = Date.now()): void => {
+  if (status >= 200 && status < 300) {
+    // A successful upload means the stores agree again.
+    conflictTimes = [];
+    lastError = undefined;
+    publish({ stuck: false, conflicts: 0 });
+    return;
+  }
+  if (!isOtkConflict(status, body)) return;
+  conflictTimes = [...conflictTimes, now].filter((t) => now - t <= WINDOW_MS);
+  lastError = body.slice(0, 300);
+  publish({
+    stuck: conflictTimes.length >= STUCK_AFTER,
+    conflicts: conflictTimes.length,
+    lastError,
+  });
+};
+
+export const getKeyUploadWatch = (): KeyUploadWatchState => state;
+
+export const subscribeKeyUploadWatch = (cb: () => void): (() => void) => {
+  listeners.add(cb);
+  return () => listeners.delete(cb);
+};
+
+/** Test/logout helper: forget everything (a new session starts clean). */
+export const resetKeyUploadWatch = (): void => {
+  conflictTimes = [];
+  lastError = undefined;
+  publish({ stuck: false, conflicts: 0 });
+};
+
+const urlOf = (input: RequestInfo | URL): string =>
+  typeof input === 'string' ? input : input instanceof URL ? input.href : input.url;
+
+/**
+ * Wrap a fetch so /keys/upload responses are observed. The response the SDK
+ * gets is untouched; a clone is read in the background, never awaited.
+ */
+export const watchKeyUploads =
+  (inner: typeof fetch): typeof fetch =>
+  async (input, init) => {
+    const res = await inner(input, init);
+    try {
+      if (isKeyUploadUrl(urlOf(input))) {
+        if (res.ok) recordKeyUpload(res.status, '');
+        else if (res.status === 400)
+          res
+            .clone()
+            .text()
+            .then((body) => recordKeyUpload(res.status, body))
+            .catch(() => undefined);
+      }
+    } catch {
+      // observation must never break the request
+    }
+    return res;
+  };

src/client/initMatrix.ts

@@ -8,6 +8,7 @@ import { revokeOidcTokens } from './oidcLogout';
 import { pushSessionToSW } from '../sw-session';
 import { deleteSearchCacheDatabase, shouldRunBootCleanup } from '../app/utils/searchCache';
 import { clearPlaintextCaches } from '../app/state/plaintextCaches';
+import { watchKeyUploads } from '../app/utils/keyUploadWatch';
 import { clearSoundboardClipCache } from '../app/utils/soundboardClips';
 import { clearSharedPayload } from '../swShare';
 
@@ -76,6 +77,9 @@ export const initClient = async (session: Session): Promise<MatrixClient> => {
     store: indexedDBStore,
     cryptoStore: legacyCryptoStore,
     deviceId: session.deviceId,
+    // [Gitea #201] Observe /keys/upload so a diverged OTK store is reported to
+    // the user (KeyUploadBanner) instead of retrying silently forever.
+    fetchFn: watchKeyUploads((input, init) => globalThis.fetch(input, init)),
     timelineSupport: true,
     cryptoCallbacks: cryptoCallbacks as any,
     // SAS (emoji) + QR-code verification (show/scan/reciprocate).