/** * [Gitea #158] Local-clock-vs-homeserver skew detection. * * Incident 2026-09-17: a wrong Windows clock made every MatrixRTC membership * look expired locally (calls failed, media keys rejected) while the server * answered 200 to everything, and nothing in the UI hinted at the cause. * * Measurement needs no extra requests and no CORS-exposed headers: every event * a `/sync` delivers carries `origin_server_ts` (stamped by its origin server) * and `unsigned.age` (= OUR server's `now − origin_server_ts` at the moment it * built the response). So `origin_server_ts + age` is our server's clock at * response time, and `receivedAt − (origin_server_ts + age)` is our skew plus * the download latency (tens of ms; ignored). matrix-js-sdk already computes * `localTimestamp = Date.now() − age` at event construction, so a sample is * simply `localTimestamp − origin_server_ts`. * * Only LIVE events count (`RoomEvent.Timeline` data.liveEvent, which the SDK * already sets false for events replayed from the IndexedDB cache — those carry * a stale `age` that would read as hours of skew). The initial network sync's * events qualify, so a wrong clock is flagged within seconds of startup. */ export const SKEW_WARN_MS = 30_000; export const SKEW_CLEAR_MS = 15_000; export const SKEW_MIN_SAMPLES = 3; /** Samples older than this are forgotten. */ export const SKEW_WINDOW_MS = 5 * 60 * 1000; export const SKEW_MAX_SAMPLES = 30; /** * "Ahead" must hold across samples received at least this far apart. * * Incident 2026-09-29: the homeserver's host ran out of memory and stalled for * ~2 minutes; the /sync that finally went out carried events whose `age` was * computed ~30 s before it arrived, so every client read "your clock is 30 s * ahead" — while the real problem was the server. A late delivery can only make * the local clock look AHEAD (never behind), so the estimate is the LOWEST * recent sample (the one delivered fastest), and "ahead" has to persist across * a minute of fresh samples before it is reported. "Behind" can't come from a * delay and is reported as soon as there are enough samples. */ export const SKEW_AHEAD_SPAN_MS = 60_000; /** * Sanity cap on `age`. Old events are still valid samples (the server computes * `age` at response time, so `ts + age` is its clock regardless of the event's * own age) — this only rejects garbage. */ export const SKEW_MAX_AGE_MS = 30 * 24 * 60 * 60 * 1000; export type ClockSkewState = { /** Median of the recent samples, ms; positive = local clock is AHEAD. */ skewMs: number | null; /** Over the threshold (with hysteresis). */ warning: boolean; }; /** A wall-clock change larger than this (vs the monotonic clock) resets the samples. */ export const CLOCK_JUMP_MS = 5_000; type Sample = { skew: number; at: number }; const currentClock = (): { wall: number; mono: number } => { const wall = Date.now(); const mono = typeof performance !== 'undefined' ? performance.now() : wall; return { wall, mono }; }; export class ClockSkewMonitor { private samples: Sample[] = []; private clockOffset: number | undefined; private state: ClockSkewState = { skewMs: null, warning: false }; private listeners = new Set<(state: ClockSkewState) => void>(); public getState(): ClockSkewState { return this.state; } public subscribe(cb: (state: ClockSkewState) => void): () => void { this.listeners.add(cb); return () => { this.listeners.delete(cb); }; } /** * Feed one live event. `originServerTs` + `age` come from the event; * `localTimestamp` is the SDK's `Date.now() − age` at construction; `clock` * is when the sample was taken: wall clock and a monotonic clock * (performance.now()), so samples are aged by real elapsed time and a change * of the local clock (someone fixing it) starts the measurement afresh. * Only feed events stamped by OUR homeserver: another server's * `origin_server_ts` carries that server's clock. * Returns the new state (unchanged object when nothing moved). */ public sample( originServerTs: number, age: number | undefined, localTimestamp: number, clock: { wall: number; mono: number } = currentClock(), ): ClockSkewState { if (age === undefined || !Number.isFinite(age) || age < 0 || age > SKEW_MAX_AGE_MS) { return this.state; } if (!Number.isFinite(originServerTs) || !Number.isFinite(localTimestamp)) return this.state; const now = clock.mono; const offset = clock.wall - clock.mono; if (this.clockOffset !== undefined && Math.abs(offset - this.clockOffset) > CLOCK_JUMP_MS) { // The local clock was changed: earlier samples measured the old clock. this.reset(); } this.clockOffset = offset; this.samples.push({ skew: localTimestamp - originServerTs, at: now }); this.samples = this.samples.filter((s) => now - s.at <= SKEW_WINDOW_MS); if (this.samples.length > SKEW_MAX_SAMPLES) this.samples.shift(); if (this.samples.length < SKEW_MIN_SAMPLES) return this.state; // Delivery delay only ever adds to a sample: the smallest is the truest. const skewMs = Math.min(...this.samples.map((s) => s.skew)); const abs = Math.abs(skewMs); let warning: boolean; if (this.state.warning) warning = abs >= SKEW_CLEAR_MS; else if (skewMs < -SKEW_WARN_MS) warning = true; else if (skewMs > SKEW_WARN_MS) { // Ahead: only if the fastest-delivered samples stayed high for a minute. const span = now - Math.min(...this.samples.map((s) => s.at)); warning = span >= SKEW_AHEAD_SPAN_MS; } else warning = false; if (skewMs === this.state.skewMs && warning === this.state.warning) return this.state; this.state = { skewMs, warning }; this.listeners.forEach((cb) => cb(this.state)); return this.state; } public reset(): void { this.samples = []; this.clockOffset = undefined; if (this.state.skewMs !== null || this.state.warning) { this.state = { skewMs: null, warning: false }; this.listeners.forEach((cb) => cb(this.state)); } } } /** "14 minutes ahead" / "2 hours behind" / "45 seconds ahead". */ export const formatSkew = (skewMs: number): string => { const abs = Math.abs(skewMs); const dir = skewMs > 0 ? 'ahead' : 'behind'; const unit = (n: number, word: string) => `${n} ${word}${n === 1 ? '' : 's'}`; if (abs >= 36 * 60 * 60 * 1000) return `${unit(Math.round(abs / (24 * 60 * 60 * 1000)), 'day')} ${dir}`; if (abs >= 90 * 60 * 1000) return `${unit(Math.round(abs / (60 * 60 * 1000)), 'hour')} ${dir}`; if (abs >= 90 * 1000) return `${unit(Math.round(abs / 60_000), 'minute')} ${dir}`; return `${unit(Math.round(abs / 1000), 'second')} ${dir}`; }; /** "14 minutes ahead of the server" / "3 hours behind the server". */ export const describeSkewVsServer = (skewMs: number): string => formatSkew(skewMs) .replace(/ ahead$/, ' ahead of the server') .replace(/ behind$/, ' behind the server'); export type ClockFixPlatform = 'windows' | 'mac' | 'linux' | 'ios' | 'android' | 'other'; export const detectClockFixPlatform = (ua: string): ClockFixPlatform => { if (/iPhone|iPad|iPod/i.test(ua)) return 'ios'; if (/Android/i.test(ua)) return 'android'; if (/Windows/i.test(ua)) return 'windows'; if (/Mac OS X|Macintosh/i.test(ua)) return 'mac'; if (/Linux|X11/i.test(ua)) return 'linux'; return 'other'; }; export const clockFixHint = (platform: ClockFixPlatform): string => { switch (platform) { case 'windows': return 'Windows: Settings → Time & language → Date & time → turn on "Set time automatically", then Sync now.'; case 'mac': return 'macOS: System Settings → General → Date & Time → turn on "Set time and date automatically".'; case 'linux': return "Linux: enable NTP (e.g. `timedatectl set-ntp true`) or your desktop's Date & Time → Automatic."; case 'ios': return 'iOS: Settings → General → Date & Time → Set Automatically.'; case 'android': return 'Android: Settings → System → Date & time → Set time automatically.'; default: return "Turn on automatic (network) time in your operating system's date & time settings."; } };