From c2c43eda99b8d347845b015e73458cf3207ddd61 Mon Sep 17 00:00:00 2001 From: Lotus CI Date: Fri, 23 Oct 2026 23:51:15 -0400 Subject: [PATCH] feat(e2ee): tell the user when their device's key uploads are stuck (#201 KE-1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When the server keeps rejecting this device's /keys/upload with 400 "One time key ... already exists", the device's encryption store has diverged and the SDK retries forever (matrix-rust-sdk#5200, still open). Synapse saw 474k of these from May 30 to Jul 8 2026. The fix is a fresh login, so: - keyUploadWatch observes /keys/upload responses through the client's fetchFn (the response the SDK gets is untouched; a clone is read). Three conflicts within 15 min = stuck; any successful upload clears it. - KeyUploadBanner (critical strip, same slot as the sync/clock banners): "Encryption on this device is out of sync" with Details and "Fix: sign out…", which opens the normal logout dialog (it already warns when the device isn't verified / backed up). Server side, Grafana now alerts on the same signal (E2EE Key Upload Conflicts) and on delayed-event 5xx (Call Delayed-Event Errors). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_017PCb59uY3UKqDqt7B6hzd5 --- e2e/key-upload-recovery.spec.ts | 49 +++++++++++ src/app/pages/client/ClientRoot.tsx | 7 ++ src/app/pages/client/KeyUploadBanner.tsx | 104 +++++++++++++++++++++++ src/app/utils/keyUploadWatch.test.ts | 86 +++++++++++++++++++ src/app/utils/keyUploadWatch.ts | 102 ++++++++++++++++++++++ src/client/initMatrix.ts | 4 + 6 files changed, 352 insertions(+) create mode 100644 e2e/key-upload-recovery.spec.ts create mode 100644 src/app/pages/client/KeyUploadBanner.tsx create mode 100644 src/app/utils/keyUploadWatch.test.ts create mode 100644 src/app/utils/keyUploadWatch.ts diff --git a/e2e/key-upload-recovery.spec.ts b/e2e/key-upload-recovery.spec.ts new file mode 100644 index 000000000..e2a646eaa --- /dev/null +++ b/e2e/key-upload-recovery.spec.ts @@ -0,0 +1,49 @@ +import { test, expect } from '@playwright/test'; +import { ensureUser, hsReachable, loginUI, uniq } from './localHs'; + +// [Gitea #201 KE-1] When the server keeps rejecting this device's key uploads +// with "One time key … already exists", the app says so and offers the fix. +const CONFLICT = { + errcode: 'M_UNKNOWN', + error: 'One time key signed_curve25519:AAAAAAAAAA0 already exists. Old key: {}; new key: {}', +}; + +test.describe('key upload recovery (#201)', () => { + test.beforeAll(async () => { + test.skip(!(await hsReachable()), 'local homeserver not running'); + }); + + test('a diverged device sees the banner, and Fix opens the logout dialog', async ({ page }) => { + test.setTimeout(240_000); + const user = await ensureUser(uniq('kup')); + let rejected = 0; + await page.route(/\/_matrix\/client\/v3\/keys\/upload/, async (route) => { + rejected += 1; + await route.fulfill({ status: 400, json: CONFLICT }); + }); + await loginUI(page, user); + + const banner = page + .getByRole('alert') + .filter({ hasText: /Encryption on this device is out of sync/ }); + await expect(banner).toBeVisible({ timeout: 200_000 }); + console.log(`key uploads rejected: ${rejected}`); + expect(rejected).toBeGreaterThanOrEqual(3); + + await banner.getByRole('button', { name: 'Details' }).click(); + await expect(banner).toContainText('Settings → Devices'); + await expect(banner).toContainText('already exists'); + + await banner.getByRole('button', { name: /Fix: sign out/ }).click(); + await expect(page.getByRole('dialog').getByText('Logout').first()).toBeVisible(); + }); + + test('a healthy device never sees it', async ({ page }) => { + test.setTimeout(120_000); + const user = await ensureUser(uniq('kok')); + await loginUI(page, user); + await page.getByText('Connecting...').first().waitFor({ state: 'detached', timeout: 90_000 }); + await page.waitForTimeout(5000); + await expect(page.getByText(/Encryption on this device is out of sync/)).toHaveCount(0); + }); +}); diff --git a/src/app/pages/client/ClientRoot.tsx b/src/app/pages/client/ClientRoot.tsx index 486217c10..d7d2fa0b8 100644 --- a/src/app/pages/client/ClientRoot.tsx +++ b/src/app/pages/client/ClientRoot.tsx @@ -46,6 +46,7 @@ import { useSyncState } from '../../hooks/useSyncState'; import { stopPropagation } from '../../utils/keyboard'; import { SyncStatus } from './SyncStatus'; import { ClockSkewBanner } from './ClockSkewBanner'; +import { KeyUploadBanner } from './KeyUploadBanner'; import { ServerStatusBanner, ServerStatusFeature, @@ -260,6 +261,12 @@ export function ClientRoot({ children }: ClientRootProps) { {mx && !syncError && } {mx && !syncError && } {mx && !syncError && } + {mx && !syncError && ( + // The fix opens LogoutDialog, which reads the client from context. + + + + )} {loading && } {(loadState.status === AsyncStatus.Error || startState.status === AsyncStatus.Error || diff --git a/src/app/pages/client/KeyUploadBanner.tsx b/src/app/pages/client/KeyUploadBanner.tsx new file mode 100644 index 000000000..7fd2e5d98 --- /dev/null +++ b/src/app/pages/client/KeyUploadBanner.tsx @@ -0,0 +1,104 @@ +import React, { useState, useSyncExternalStore } from 'react'; +import FocusTrap from 'focus-trap-react'; +import { Box, Button, config, Line, Overlay, OverlayBackdrop, OverlayCenter, Text } from 'folds'; +import { ContainerColor } from '../../styles/ContainerColor.css'; +import { getKeyUploadWatch, subscribeKeyUploadWatch } from '../../utils/keyUploadWatch'; +import { LogoutDialog } from '../../components/LogoutDialog'; +import { stopPropagation } from '../../utils/keyboard'; + +/** + * [Gitea #201 KE-1] "Encryption on this device is out of sync." Shown while + * the server keeps rejecting this device's key uploads (keyUploadWatch). The + * fix is a fresh login, so the action opens the normal logout dialog, which + * already warns if the device isn't verified / backed up. Hideable for this + * session only: the problem doesn't go away by itself. + */ +export function KeyUploadBanner() { + const { stuck, lastError } = useSyncExternalStore(subscribeKeyUploadWatch, getKeyUploadWatch); + const [hidden, setHidden] = useState(false); + const [details, setDetails] = useState(false); + const [logout, setLogout] = useState(false); + + if (!stuck || hidden) return null; + + return ( + + + + + Encryption on this device is out of sync with the server. New encrypted messages and + calls may fail until you sign out and back in. + + + + + + {details && ( + + + The server already has encryption keys this device thinks it hasn't sent, so it + rejects every new upload and other people can't start fresh encrypted sessions + with you. This usually happens after the browser cleared this site's storage + while you stayed signed in. Before signing out, make sure this device is verified or + key backup is on (Settings → Devices), so your message history comes back when you + sign in again. + + {lastError && ( + + Server said: {lastError} + + )} + + )} + + + {logout && ( + }> + + setLogout(false), + clickOutsideDeactivates: true, + escapeDeactivates: stopPropagation, + }} + > + setLogout(false)} /> + + + + )} + + ); +} diff --git a/src/app/utils/keyUploadWatch.test.ts b/src/app/utils/keyUploadWatch.test.ts new file mode 100644 index 000000000..7518ec0ac --- /dev/null +++ b/src/app/utils/keyUploadWatch.test.ts @@ -0,0 +1,86 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { + getKeyUploadWatch, + isKeyUploadUrl, + isOtkConflict, + recordKeyUpload, + resetKeyUploadWatch, + STUCK_AFTER, + subscribeKeyUploadWatch, + WINDOW_MS, + watchKeyUploads, +} from './keyUploadWatch'; + +const CONFLICT = JSON.stringify({ + errcode: 'M_UNKNOWN', + error: + 'One time key signed_curve25519:AAAAAAAAAA0 already exists. Old key: {...}; new key: {...}', +}); + +test('[#201] recognises the upload endpoint and the conflict body', () => { + assert.equal(isKeyUploadUrl('https://hs.example/_matrix/client/v3/keys/upload'), true); + assert.equal(isKeyUploadUrl('https://hs.example/_matrix/client/v3/keys/upload/'), true); + assert.equal(isKeyUploadUrl('https://hs.example/_matrix/client/v3/keys/query'), false); + assert.equal(isKeyUploadUrl('https://hs.example/_matrix/client/v3/room_keys/keys'), false); + assert.equal(isOtkConflict(400, CONFLICT), true); + assert.equal(isOtkConflict(400, '{"errcode":"M_BAD_JSON"}'), false); + assert.equal(isOtkConflict(500, CONFLICT), false); +}); + +test('[#201] stuck only after repeated conflicts inside the window', () => { + resetKeyUploadWatch(); + const t0 = 1_000_000; + for (let i = 0; i < STUCK_AFTER - 1; i += 1) recordKeyUpload(400, CONFLICT, t0 + i * 1000); + assert.equal(getKeyUploadWatch().stuck, false, 'a couple of races is not divergence'); + recordKeyUpload(400, CONFLICT, t0 + 5000); + assert.equal(getKeyUploadWatch().stuck, true); + assert.match(getKeyUploadWatch().lastError ?? '', /already exists/); +}); + +test('[#201] old conflicts age out; other 400s are ignored', () => { + resetKeyUploadWatch(); + recordKeyUpload(400, CONFLICT, 0); + recordKeyUpload(400, CONFLICT, 1000); + recordKeyUpload(400, CONFLICT, WINDOW_MS + 60_000); + assert.equal(getKeyUploadWatch().stuck, false); + recordKeyUpload(400, '{"errcode":"M_BAD_JSON"}', WINDOW_MS + 61_000); + assert.equal(getKeyUploadWatch().conflicts, 1); +}); + +test('[#201] a successful upload clears it and notifies', () => { + resetKeyUploadWatch(); + let calls = 0; + const off = subscribeKeyUploadWatch(() => { + calls += 1; + }); + for (let i = 0; i < STUCK_AFTER; i += 1) recordKeyUpload(400, CONFLICT, i); + assert.equal(getKeyUploadWatch().stuck, true); + recordKeyUpload(200, ''); + assert.equal(getKeyUploadWatch().stuck, false); + assert.ok(calls >= 2); + off(); +}); + +test('[#201] the fetch wrapper observes without changing the response', async () => { + resetKeyUploadWatch(); + const fake: typeof fetch = async (input) => + String(input).endsWith('/keys/upload') + ? new Response(CONFLICT, { status: 400 }) + : new Response('{}', { status: 200 }); + const f = watchKeyUploads(fake); + const responses = await Promise.all( + Array.from({ length: STUCK_AFTER }, () => + f('https://hs.example/_matrix/client/v3/keys/upload', { method: 'POST' }), + ), + ); + const bodies = await Promise.all(responses.map((res) => res.text())); + responses.forEach((res) => assert.equal(res.status, 400)); + bodies.forEach((b) => assert.match(b, /already exists/, 'the SDK can still read the body')); + await new Promise((r) => { + setTimeout(r, 10); + }); + assert.equal(getKeyUploadWatch().stuck, true); + await f('https://hs.example/_matrix/client/v3/sync'); + assert.equal(getKeyUploadWatch().stuck, true, 'unrelated requests change nothing'); +}); diff --git a/src/app/utils/keyUploadWatch.ts b/src/app/utils/keyUploadWatch.ts new file mode 100644 index 000000000..c09718bdf --- /dev/null +++ b/src/app/utils/keyUploadWatch.ts @@ -0,0 +1,102 @@ +/** + * [Gitea #201 KE-1] Notice when this device's encryption store has diverged + * from the server: every POST /keys/upload answers 400 "One time key ... + * already exists", and the SDK retries forever (matrix-rust-sdk#5200, open), + * so new encrypted messages and calls quietly degrade. The server saw 474k of + * these between May 30 and Jul 8 2026. The only fix is a fresh login on that + * device, so we tell the user (KeyUploadBanner) instead of looping silently. + * + * Watched at the fetch layer (MatrixClient `fetchFn`), so it doesn't depend on + * SDK log wording. Pure state + a tiny subscribe API; no network I/O of its own. + */ + +/** Conflicts needed before we call the device stuck. The SDK retries within + * seconds, so a real divergence crosses this quickly; a one-off race doesn't. */ +export const STUCK_AFTER = 3; +/** Conflicts older than this don't count (a long-ago blip isn't "now"). */ +export const WINDOW_MS = 15 * 60 * 1000; + +export type KeyUploadWatchState = { stuck: boolean; conflicts: number; lastError?: string }; + +let conflictTimes: number[] = []; +let lastError: string | undefined; +let state: KeyUploadWatchState = { stuck: false, conflicts: 0 }; +const listeners = new Set<() => void>(); + +const publish = (next: KeyUploadWatchState) => { + if (next.stuck === state.stuck && next.conflicts === state.conflicts) return; + state = next; + listeners.forEach((l) => l()); +}; + +export const isKeyUploadUrl = (url: string): boolean => { + try { + return /\/_matrix\/client\/[^/]+\/keys\/upload\/?$/.test(new URL(url, 'http://x').pathname); + } catch { + return false; + } +}; + +/** The 400 body that marks a diverged one-time-key store. */ +export const isOtkConflict = (status: number, body: string): boolean => + status === 400 && /one time key .*already exists/i.test(body); + +/** Feed one /keys/upload response. `now` is injectable for tests. */ +export const recordKeyUpload = (status: number, body: string, now = Date.now()): void => { + if (status >= 200 && status < 300) { + // A successful upload means the stores agree again. + conflictTimes = []; + lastError = undefined; + publish({ stuck: false, conflicts: 0 }); + return; + } + if (!isOtkConflict(status, body)) return; + conflictTimes = [...conflictTimes, now].filter((t) => now - t <= WINDOW_MS); + lastError = body.slice(0, 300); + publish({ + stuck: conflictTimes.length >= STUCK_AFTER, + conflicts: conflictTimes.length, + lastError, + }); +}; + +export const getKeyUploadWatch = (): KeyUploadWatchState => state; + +export const subscribeKeyUploadWatch = (cb: () => void): (() => void) => { + listeners.add(cb); + return () => listeners.delete(cb); +}; + +/** Test/logout helper: forget everything (a new session starts clean). */ +export const resetKeyUploadWatch = (): void => { + conflictTimes = []; + lastError = undefined; + publish({ stuck: false, conflicts: 0 }); +}; + +const urlOf = (input: RequestInfo | URL): string => + typeof input === 'string' ? input : input instanceof URL ? input.href : input.url; + +/** + * Wrap a fetch so /keys/upload responses are observed. The response the SDK + * gets is untouched; a clone is read in the background, never awaited. + */ +export const watchKeyUploads = + (inner: typeof fetch): typeof fetch => + async (input, init) => { + const res = await inner(input, init); + try { + if (isKeyUploadUrl(urlOf(input))) { + if (res.ok) recordKeyUpload(res.status, ''); + else if (res.status === 400) + res + .clone() + .text() + .then((body) => recordKeyUpload(res.status, body)) + .catch(() => undefined); + } + } catch { + // observation must never break the request + } + return res; + }; diff --git a/src/client/initMatrix.ts b/src/client/initMatrix.ts index 14f418556..883fb1ebd 100644 --- a/src/client/initMatrix.ts +++ b/src/client/initMatrix.ts @@ -8,6 +8,7 @@ import { revokeOidcTokens } from './oidcLogout'; import { pushSessionToSW } from '../sw-session'; import { deleteSearchCacheDatabase, shouldRunBootCleanup } from '../app/utils/searchCache'; import { clearPlaintextCaches } from '../app/state/plaintextCaches'; +import { watchKeyUploads } from '../app/utils/keyUploadWatch'; import { clearSoundboardClipCache } from '../app/utils/soundboardClips'; import { clearSharedPayload } from '../swShare'; @@ -76,6 +77,9 @@ export const initClient = async (session: Session): Promise => { store: indexedDBStore, cryptoStore: legacyCryptoStore, deviceId: session.deviceId, + // [Gitea #201] Observe /keys/upload so a diverged OTK store is reported to + // the user (KeyUploadBanner) instead of retrying silently forever. + fetchFn: watchKeyUploads((input, init) => globalThis.fetch(input, init)), timelineSupport: true, cryptoCallbacks: cryptoCallbacks as any, // SAS (emoji) + QR-code verification (show/scan/reciprocate).