2026-09-19 12:52:03 -04:00
|
|
|
|
/**
|
|
|
|
|
|
* [Gitea #158] Local-clock-vs-homeserver skew detection.
|
|
|
|
|
|
*
|
|
|
|
|
|
* Incident 2026-09-17: a wrong Windows clock made every MatrixRTC membership
|
|
|
|
|
|
* look expired locally (calls failed, media keys rejected) while the server
|
|
|
|
|
|
* answered 200 to everything, and nothing in the UI hinted at the cause.
|
|
|
|
|
|
*
|
|
|
|
|
|
* Measurement needs no extra requests and no CORS-exposed headers: every event
|
|
|
|
|
|
* a `/sync` delivers carries `origin_server_ts` (stamped by its origin server)
|
|
|
|
|
|
* and `unsigned.age` (= OUR server's `now − origin_server_ts` at the moment it
|
|
|
|
|
|
* built the response). So `origin_server_ts + age` is our server's clock at
|
|
|
|
|
|
* response time, and `receivedAt − (origin_server_ts + age)` is our skew plus
|
|
|
|
|
|
* the download latency (tens of ms; ignored). matrix-js-sdk already computes
|
|
|
|
|
|
* `localTimestamp = Date.now() − age` at event construction, so a sample is
|
|
|
|
|
|
* simply `localTimestamp − origin_server_ts`.
|
|
|
|
|
|
*
|
|
|
|
|
|
* Only LIVE events count (`RoomEvent.Timeline` data.liveEvent, which the SDK
|
|
|
|
|
|
* already sets false for events replayed from the IndexedDB cache — those carry
|
|
|
|
|
|
* a stale `age` that would read as hours of skew). The initial network sync's
|
|
|
|
|
|
* events qualify, so a wrong clock is flagged within seconds of startup.
|
|
|
|
|
|
*/
|
|
|
|
|
|
|
|
|
|
|
|
export const SKEW_WARN_MS = 30_000;
|
|
|
|
|
|
export const SKEW_CLEAR_MS = 15_000;
|
|
|
|
|
|
export const SKEW_MIN_SAMPLES = 3;
|
2026-09-28 21:40:02 -04:00
|
|
|
|
/** Samples older than this are forgotten. */
|
|
|
|
|
|
export const SKEW_WINDOW_MS = 5 * 60 * 1000;
|
|
|
|
|
|
export const SKEW_MAX_SAMPLES = 30;
|
|
|
|
|
|
/**
|
|
|
|
|
|
* "Ahead" must hold across samples received at least this far apart.
|
|
|
|
|
|
*
|
|
|
|
|
|
* Incident 2026-09-29: the homeserver's host ran out of memory and stalled for
|
|
|
|
|
|
* ~2 minutes; the /sync that finally went out carried events whose `age` was
|
|
|
|
|
|
* computed ~30 s before it arrived, so every client read "your clock is 30 s
|
|
|
|
|
|
* ahead" — while the real problem was the server. A late delivery can only make
|
|
|
|
|
|
* the local clock look AHEAD (never behind), so the estimate is the LOWEST
|
|
|
|
|
|
* recent sample (the one delivered fastest), and "ahead" has to persist across
|
|
|
|
|
|
* a minute of fresh samples before it is reported. "Behind" can't come from a
|
|
|
|
|
|
* delay and is reported as soon as there are enough samples.
|
|
|
|
|
|
*/
|
|
|
|
|
|
export const SKEW_AHEAD_SPAN_MS = 60_000;
|
2026-09-19 12:52:03 -04:00
|
|
|
|
/**
|
|
|
|
|
|
* Sanity cap on `age`. Old events are still valid samples (the server computes
|
|
|
|
|
|
* `age` at response time, so `ts + age` is its clock regardless of the event's
|
|
|
|
|
|
* own age) — this only rejects garbage.
|
|
|
|
|
|
*/
|
|
|
|
|
|
export const SKEW_MAX_AGE_MS = 30 * 24 * 60 * 60 * 1000;
|
|
|
|
|
|
|
|
|
|
|
|
export type ClockSkewState = {
|
|
|
|
|
|
/** Median of the recent samples, ms; positive = local clock is AHEAD. */
|
|
|
|
|
|
skewMs: number | null;
|
|
|
|
|
|
/** Over the threshold (with hysteresis). */
|
|
|
|
|
|
warning: boolean;
|
|
|
|
|
|
};
|
|
|
|
|
|
|
2026-09-28 21:40:02 -04:00
|
|
|
|
/** A wall-clock change larger than this (vs the monotonic clock) resets the samples. */
|
|
|
|
|
|
export const CLOCK_JUMP_MS = 5_000;
|
|
|
|
|
|
|
|
|
|
|
|
type Sample = { skew: number; at: number };
|
|
|
|
|
|
|
|
|
|
|
|
const currentClock = (): { wall: number; mono: number } => {
|
|
|
|
|
|
const wall = Date.now();
|
|
|
|
|
|
const mono = typeof performance !== 'undefined' ? performance.now() : wall;
|
|
|
|
|
|
return { wall, mono };
|
2026-09-19 12:52:03 -04:00
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
|
|
export class ClockSkewMonitor {
|
2026-09-28 21:40:02 -04:00
|
|
|
|
private samples: Sample[] = [];
|
|
|
|
|
|
|
|
|
|
|
|
private clockOffset: number | undefined;
|
2026-09-19 12:52:03 -04:00
|
|
|
|
|
|
|
|
|
|
private state: ClockSkewState = { skewMs: null, warning: false };
|
|
|
|
|
|
|
|
|
|
|
|
private listeners = new Set<(state: ClockSkewState) => void>();
|
|
|
|
|
|
|
|
|
|
|
|
public getState(): ClockSkewState {
|
|
|
|
|
|
return this.state;
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
public subscribe(cb: (state: ClockSkewState) => void): () => void {
|
|
|
|
|
|
this.listeners.add(cb);
|
|
|
|
|
|
return () => {
|
|
|
|
|
|
this.listeners.delete(cb);
|
|
|
|
|
|
};
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
|
* Feed one live event. `originServerTs` + `age` come from the event;
|
2026-09-28 21:40:02 -04:00
|
|
|
|
* `localTimestamp` is the SDK's `Date.now() − age` at construction; `clock`
|
|
|
|
|
|
* is when the sample was taken: wall clock and a monotonic clock
|
|
|
|
|
|
* (performance.now()), so samples are aged by real elapsed time and a change
|
|
|
|
|
|
* of the local clock (someone fixing it) starts the measurement afresh.
|
|
|
|
|
|
* Only feed events stamped by OUR homeserver: another server's
|
|
|
|
|
|
* `origin_server_ts` carries that server's clock.
|
2026-09-19 12:52:03 -04:00
|
|
|
|
* Returns the new state (unchanged object when nothing moved).
|
|
|
|
|
|
*/
|
|
|
|
|
|
public sample(
|
|
|
|
|
|
originServerTs: number,
|
|
|
|
|
|
age: number | undefined,
|
|
|
|
|
|
localTimestamp: number,
|
2026-09-28 21:40:02 -04:00
|
|
|
|
clock: { wall: number; mono: number } = currentClock(),
|
2026-09-19 12:52:03 -04:00
|
|
|
|
): ClockSkewState {
|
|
|
|
|
|
if (age === undefined || !Number.isFinite(age) || age < 0 || age > SKEW_MAX_AGE_MS) {
|
|
|
|
|
|
return this.state;
|
|
|
|
|
|
}
|
|
|
|
|
|
if (!Number.isFinite(originServerTs) || !Number.isFinite(localTimestamp)) return this.state;
|
2026-09-28 21:40:02 -04:00
|
|
|
|
const now = clock.mono;
|
|
|
|
|
|
const offset = clock.wall - clock.mono;
|
|
|
|
|
|
if (this.clockOffset !== undefined && Math.abs(offset - this.clockOffset) > CLOCK_JUMP_MS) {
|
|
|
|
|
|
// The local clock was changed: earlier samples measured the old clock.
|
|
|
|
|
|
this.reset();
|
|
|
|
|
|
}
|
|
|
|
|
|
this.clockOffset = offset;
|
|
|
|
|
|
this.samples.push({ skew: localTimestamp - originServerTs, at: now });
|
|
|
|
|
|
this.samples = this.samples.filter((s) => now - s.at <= SKEW_WINDOW_MS);
|
|
|
|
|
|
if (this.samples.length > SKEW_MAX_SAMPLES) this.samples.shift();
|
2026-09-19 12:52:03 -04:00
|
|
|
|
if (this.samples.length < SKEW_MIN_SAMPLES) return this.state;
|
|
|
|
|
|
|
2026-09-28 21:40:02 -04:00
|
|
|
|
// Delivery delay only ever adds to a sample: the smallest is the truest.
|
|
|
|
|
|
const skewMs = Math.min(...this.samples.map((s) => s.skew));
|
2026-09-19 12:52:03 -04:00
|
|
|
|
const abs = Math.abs(skewMs);
|
2026-09-28 21:40:02 -04:00
|
|
|
|
let warning: boolean;
|
|
|
|
|
|
if (this.state.warning) warning = abs >= SKEW_CLEAR_MS;
|
|
|
|
|
|
else if (skewMs < -SKEW_WARN_MS) warning = true;
|
|
|
|
|
|
else if (skewMs > SKEW_WARN_MS) {
|
|
|
|
|
|
// Ahead: only if the fastest-delivered samples stayed high for a minute.
|
|
|
|
|
|
const span = now - Math.min(...this.samples.map((s) => s.at));
|
|
|
|
|
|
warning = span >= SKEW_AHEAD_SPAN_MS;
|
|
|
|
|
|
} else warning = false;
|
2026-09-19 12:52:03 -04:00
|
|
|
|
if (skewMs === this.state.skewMs && warning === this.state.warning) return this.state;
|
|
|
|
|
|
this.state = { skewMs, warning };
|
|
|
|
|
|
this.listeners.forEach((cb) => cb(this.state));
|
|
|
|
|
|
return this.state;
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
public reset(): void {
|
|
|
|
|
|
this.samples = [];
|
2026-09-28 21:40:02 -04:00
|
|
|
|
this.clockOffset = undefined;
|
2026-09-19 12:52:03 -04:00
|
|
|
|
if (this.state.skewMs !== null || this.state.warning) {
|
|
|
|
|
|
this.state = { skewMs: null, warning: false };
|
|
|
|
|
|
this.listeners.forEach((cb) => cb(this.state));
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
/** "14 minutes ahead" / "2 hours behind" / "45 seconds ahead". */
|
|
|
|
|
|
export const formatSkew = (skewMs: number): string => {
|
|
|
|
|
|
const abs = Math.abs(skewMs);
|
|
|
|
|
|
const dir = skewMs > 0 ? 'ahead' : 'behind';
|
|
|
|
|
|
const unit = (n: number, word: string) => `${n} ${word}${n === 1 ? '' : 's'}`;
|
|
|
|
|
|
if (abs >= 36 * 60 * 60 * 1000)
|
|
|
|
|
|
return `${unit(Math.round(abs / (24 * 60 * 60 * 1000)), 'day')} ${dir}`;
|
|
|
|
|
|
if (abs >= 90 * 60 * 1000) return `${unit(Math.round(abs / (60 * 60 * 1000)), 'hour')} ${dir}`;
|
|
|
|
|
|
if (abs >= 90 * 1000) return `${unit(Math.round(abs / 60_000), 'minute')} ${dir}`;
|
|
|
|
|
|
return `${unit(Math.round(abs / 1000), 'second')} ${dir}`;
|
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
|
|
/** "14 minutes ahead of the server" / "3 hours behind the server". */
|
|
|
|
|
|
export const describeSkewVsServer = (skewMs: number): string =>
|
|
|
|
|
|
formatSkew(skewMs)
|
|
|
|
|
|
.replace(/ ahead$/, ' ahead of the server')
|
|
|
|
|
|
.replace(/ behind$/, ' behind the server');
|
|
|
|
|
|
|
|
|
|
|
|
export type ClockFixPlatform = 'windows' | 'mac' | 'linux' | 'ios' | 'android' | 'other';
|
|
|
|
|
|
|
|
|
|
|
|
export const detectClockFixPlatform = (ua: string): ClockFixPlatform => {
|
|
|
|
|
|
if (/iPhone|iPad|iPod/i.test(ua)) return 'ios';
|
|
|
|
|
|
if (/Android/i.test(ua)) return 'android';
|
|
|
|
|
|
if (/Windows/i.test(ua)) return 'windows';
|
|
|
|
|
|
if (/Mac OS X|Macintosh/i.test(ua)) return 'mac';
|
|
|
|
|
|
if (/Linux|X11/i.test(ua)) return 'linux';
|
|
|
|
|
|
return 'other';
|
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
|
|
export const clockFixHint = (platform: ClockFixPlatform): string => {
|
|
|
|
|
|
switch (platform) {
|
|
|
|
|
|
case 'windows':
|
|
|
|
|
|
return 'Windows: Settings → Time & language → Date & time → turn on "Set time automatically", then Sync now.';
|
|
|
|
|
|
case 'mac':
|
|
|
|
|
|
return 'macOS: System Settings → General → Date & Time → turn on "Set time and date automatically".';
|
|
|
|
|
|
case 'linux':
|
|
|
|
|
|
return "Linux: enable NTP (e.g. `timedatectl set-ntp true`) or your desktop's Date & Time → Automatic.";
|
|
|
|
|
|
case 'ios':
|
|
|
|
|
|
return 'iOS: Settings → General → Date & Time → Set Automatically.';
|
|
|
|
|
|
case 'android':
|
|
|
|
|
|
return 'Android: Settings → System → Date & time → Set time automatically.';
|
|
|
|
|
|
default:
|
|
|
|
|
|
return "Turn on automatic (network) time in your operating system's date & time settings.";
|
|
|
|
|
|
}
|
|
|
|
|
|
};
|