jaredandClaude Opus 5
d929143f7d
RoomTimeline renders a numeric index window into the live timeline's event arrays; SDK back-pagination prepends, so any side panel calling paginateEventTimeline(room.getLiveTimeline()) shifted the visible messages into the past on the next render and broke at-bottom tracking. New utils/detachedTimeline.ts builds a timeline set that mirrors the already-loaded history and paginates independently: a room-registered filtered set (server-side contains_url / types filter) when the filter is usable, else a private EventTimelineSet seeded from the live timeline. useRoomMediaTimeline wraps it for the gallery (live events + redactions handled); RoomActivityLog uses a type filter (safe in encrypted rooms); ExportRoomHistory pages a private set so a full export no longer parks thousands of events in the live timeline. Verified with Playwright against a local Synapse in a 400-message plain room and a 200-message encrypted room: timeline stays at the bottom through gallery pages, activity load-more and a full export; live messages keep auto-scrolling; all media found in both rooms. Also adds scripts/dev-homeserver.sh + scripts/dev-seed.py (local throwaway Synapse for driving the real UI) and documents them. Closes #163 Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01PPmy3tPq869XDW4njjVaKA
442 lines
17 KiB
TypeScript
442 lines
17 KiB
TypeScript
import React, { useCallback, useEffect, useRef, useState } from 'react';
|
||
import { Box, Button, Icon, IconButton, Icons, Input, Scroll, Text } from 'folds';
|
||
import { EventType } from 'matrix-js-sdk';
|
||
import { Page, PageContent, PageHeader } from '../../components/page';
|
||
import { useMatrixClient } from '../../hooks/useMatrixClient';
|
||
import { useRoom } from '../../hooks/useRoom';
|
||
import { createDetachedTimelineSet } from '../../utils/detachedTimeline';
|
||
import { useRoomName } from '../../hooks/useRoomMeta';
|
||
import { SequenceCard } from '../../components/sequence-card';
|
||
import { SequenceCardStyle } from '../common-settings/styles.css';
|
||
import { RawExportRecord, formatExportBody, resolveMessageEdits } from './exportRoomHistory.utils';
|
||
|
||
type ExportFormat = 'txt' | 'json' | 'html';
|
||
|
||
const FORMAT_LABELS: Record<ExportFormat, string> = {
|
||
txt: 'Plain Text',
|
||
json: 'JSON',
|
||
html: 'HTML',
|
||
};
|
||
|
||
const PAGE_LIMIT = 100;
|
||
// Hard cap on back-pagination requests. Without a fromDate, "export all" would
|
||
// otherwise decrypt and hold every message in the room, hammering the server and
|
||
// risking an OOM/freeze with no way to stop. 200 pages × 100 ≈ 20,000 events.
|
||
const MAX_EXPORT_PAGES = 200;
|
||
|
||
type ExportRoomHistoryProps = {
|
||
requestClose: () => void;
|
||
};
|
||
|
||
export function ExportRoomHistory({ requestClose }: ExportRoomHistoryProps) {
|
||
const mx = useMatrixClient();
|
||
const room = useRoom();
|
||
const roomName = useRoomName(room);
|
||
|
||
const [format, setFormat] = useState<ExportFormat>('txt');
|
||
const [fromDate, setFromDate] = useState('');
|
||
const [toDate, setToDate] = useState('');
|
||
const [exporting, setExporting] = useState(false);
|
||
const [exportCount, setExportCount] = useState(0);
|
||
const [notice, setNotice] = useState('');
|
||
const cancelledRef = useRef(false);
|
||
|
||
const handleCancel = useCallback(() => {
|
||
cancelledRef.current = true;
|
||
}, []);
|
||
|
||
// Stop an in-flight export if the panel unmounts (closing settings mid-export
|
||
// would otherwise keep paginating + decrypting in the background).
|
||
useEffect(
|
||
() => () => {
|
||
cancelledRef.current = true;
|
||
},
|
||
[],
|
||
);
|
||
|
||
const handleExport = useCallback(async () => {
|
||
if (exporting) return;
|
||
cancelledRef.current = false;
|
||
setExporting(true);
|
||
setExportCount(0);
|
||
setNotice('');
|
||
|
||
try {
|
||
const fromTs = fromDate ? new Date(`${fromDate}T00:00:00`).getTime() : null;
|
||
const toTs = toDate ? new Date(`${toDate}T23:59:59`).getTime() : null;
|
||
|
||
const rawRecords: RawExportRecord[] = [];
|
||
// mxc/media-file URL for message events that carry one, keyed by eventId -
|
||
// surfaced in the JSON export since the export never includes the actual
|
||
// media (see the UI note below the Export button).
|
||
const mediaUrlByEventId = new Map<string, string>();
|
||
// timeline.getEvents() returns the entire growing window on every call,
|
||
// so we must deduplicate by eventId to avoid re-adding the same events
|
||
// on each pagination step.
|
||
const seen = new Set<string>();
|
||
// [Gitea #163] Paginate a private timeline set, not the live one: the
|
||
// export can pull thousands of events and the room behind this modal
|
||
// renders the live timeline by index.
|
||
const timeline = createDetachedTimelineSet(mx, room).set.getLiveTimeline();
|
||
let canLoadMore = true;
|
||
// Track the oldest collected timestamp incrementally so the fromTs check
|
||
// doesn't rescan the whole `collected` array on every pagination step.
|
||
let oldestTs = Number.POSITIVE_INFINITY;
|
||
// Oldest RAW message ts paginated (tracked BEFORE the fromTs filter). The
|
||
// date-range early-break must use this — oldestTs only ever holds collected
|
||
// events (all >= fromTs), so it can never fall below fromTs and the export
|
||
// would over-paginate to the page cap and show a misleading "truncated".
|
||
let oldestRawTs = Number.POSITIVE_INFINITY;
|
||
|
||
const addEvents = async (events: ReturnType<typeof timeline.getEvents>) => {
|
||
for (const ev of events) {
|
||
const evId = ev.getId();
|
||
if (!evId || seen.has(evId)) continue;
|
||
seen.add(evId);
|
||
// Advance the raw pagination boundary for EVERY event (any type,
|
||
// decrypted or not) — getTs() is unencrypted metadata. Gating this on
|
||
// a decrypted m.room.message let undecryptable/non-message old events
|
||
// stall oldestRawTs, so the fromTs break never fired → over-paginate
|
||
// and a false "truncated".
|
||
const ts = ev.getTs();
|
||
// Require a positive ts: an event with a bogus 0/negative
|
||
// origin_server_ts must not collapse the boundary and trigger an early
|
||
// break (silent under-pagination in the export).
|
||
if (ts > 0 && ts < oldestRawTs) oldestRawTs = ts;
|
||
// Attempt decryption for events that haven't been decrypted yet
|
||
// (paginateEventTimeline may fetch events before the SDK decrypts them)
|
||
if (ev.isEncrypted() && !ev.getClearContent()) {
|
||
// eslint-disable-next-line no-await-in-loop
|
||
await mx.decryptEventIfNeeded(ev).catch(() => undefined);
|
||
}
|
||
if (ev.getType() !== EventType.RoomMessage) continue;
|
||
if (ev.isDecryptionFailure()) continue;
|
||
if (fromTs !== null && ts < fromTs) continue;
|
||
if (toTs !== null && ts > toTs) continue;
|
||
const content = ev.getContent();
|
||
|
||
// m.replace (edit) events must not become their own row — collect
|
||
// them separately and substitute the new content into the original
|
||
// event's row below, instead of adding a stale original + a garbled
|
||
// "* new text" duplicate line.
|
||
const relatesTo = content['m.relates_to'] as
|
||
| { rel_type?: string; event_id?: string }
|
||
| undefined;
|
||
if (relatesTo?.rel_type === 'm.replace' && relatesTo.event_id) {
|
||
const newContent = content['m.new_content'] as { body?: string } | undefined;
|
||
const newBody = newContent?.body ?? '';
|
||
if (!newBody) continue;
|
||
rawRecords.push({
|
||
eventId: evId,
|
||
ts,
|
||
sender: ev.getSender() ?? '',
|
||
body: '',
|
||
msgtype: '',
|
||
editsEventId: relatesTo.event_id,
|
||
newBody,
|
||
});
|
||
continue;
|
||
}
|
||
|
||
const body: string = content.body ?? '';
|
||
const msgtype: string = content.msgtype ?? '';
|
||
if (!body) continue;
|
||
if (ts < oldestTs) oldestTs = ts;
|
||
const mediaUrl =
|
||
(content.url as string | undefined) ??
|
||
(content.file as { url?: string } | undefined)?.url;
|
||
if (mediaUrl) mediaUrlByEventId.set(evId, mediaUrl);
|
||
rawRecords.push({
|
||
ts,
|
||
sender: ev.getSender() ?? '',
|
||
body,
|
||
eventId: evId,
|
||
msgtype,
|
||
});
|
||
}
|
||
// Approximate progress — excludes edit rows, which never become their
|
||
// own line in the final (edit-resolved) output.
|
||
setExportCount(rawRecords.filter((r) => !r.editsEventId).length);
|
||
};
|
||
|
||
await addEvents(timeline.getEvents());
|
||
|
||
// Paginate backwards until start, date range exceeded, cap hit, or cancel
|
||
let pageCount = 0;
|
||
let truncated = false;
|
||
let cancelled = false;
|
||
while (canLoadMore) {
|
||
if (cancelledRef.current) {
|
||
cancelled = true;
|
||
break;
|
||
}
|
||
// If we've paginated back past the fromTs boundary, there's nothing more
|
||
// in range to fetch (use the raw paginated ts, not the collected one).
|
||
if (fromTs !== null && oldestRawTs < fromTs) break;
|
||
// Hard cap so "export all" can't run away and OOM the tab.
|
||
if (pageCount >= MAX_EXPORT_PAGES) {
|
||
truncated = true;
|
||
break;
|
||
}
|
||
pageCount += 1;
|
||
|
||
// eslint-disable-next-line no-await-in-loop
|
||
canLoadMore = await mx.paginateEventTimeline(timeline, {
|
||
backwards: true,
|
||
limit: PAGE_LIMIT,
|
||
});
|
||
|
||
// eslint-disable-next-line no-await-in-loop
|
||
await addEvents(timeline.getEvents());
|
||
}
|
||
|
||
// Resolve m.replace edits against their target row (drops the edit rows,
|
||
// substitutes m.new_content into the original) before sorting/rendering.
|
||
const collected = resolveMessageEdits(rawRecords);
|
||
|
||
if (cancelled) {
|
||
setNotice(`Export cancelled after ${collected.length} messages.`);
|
||
return;
|
||
}
|
||
|
||
// Sort chronologically (oldest first)
|
||
collected.sort((a, b) => a.ts - b.ts);
|
||
|
||
const exportedAt = new Date().toISOString();
|
||
const dateStr = exportedAt.slice(0, 10);
|
||
const safeRoomName = roomName.replace(/[^a-z0-9_-]/gi, '_').toLowerCase();
|
||
|
||
let content = '';
|
||
let mimeType = 'text/plain';
|
||
let ext: string = format;
|
||
|
||
if (format === 'txt') {
|
||
const lines: string[] = [
|
||
`# Export: ${roomName}`,
|
||
`# Exported: ${exportedAt}`,
|
||
`# Messages: ${collected.length}`,
|
||
'',
|
||
];
|
||
for (const msg of collected) {
|
||
const d = new Date(msg.ts);
|
||
const pad = (n: number) => String(n).padStart(2, '0');
|
||
const dateLabel = `${d.getFullYear()}-${pad(d.getMonth() + 1)}-${pad(d.getDate())} ${pad(d.getHours())}:${pad(d.getMinutes())}`;
|
||
lines.push(`[${dateLabel}] ${msg.sender}: ${formatExportBody(msg)}`);
|
||
}
|
||
content = lines.join('\n');
|
||
mimeType = 'text/plain';
|
||
ext = 'txt';
|
||
} else if (format === 'json') {
|
||
const payload = {
|
||
room: roomName,
|
||
exportedAt,
|
||
messages: collected.map((m) => ({
|
||
ts: m.ts,
|
||
sender: m.sender,
|
||
body: m.body,
|
||
eventId: m.eventId,
|
||
type: m.msgtype,
|
||
edited: m.edited,
|
||
mediaUrl: mediaUrlByEventId.get(m.eventId),
|
||
})),
|
||
};
|
||
content = JSON.stringify(payload, null, 2);
|
||
mimeType = 'application/json';
|
||
ext = 'json';
|
||
} else {
|
||
// HTML
|
||
const esc = (s: string) =>
|
||
s
|
||
.replace(/&/g, '&')
|
||
.replace(/</g, '<')
|
||
.replace(/>/g, '>')
|
||
.replace(/"/g, '"');
|
||
|
||
const msgRows = collected
|
||
.map((msg) => {
|
||
const d = new Date(msg.ts);
|
||
const pad = (n: number) => String(n).padStart(2, '0');
|
||
const dateLabel = `${d.getFullYear()}-${pad(d.getMonth() + 1)}-${pad(d.getDate())} ${pad(d.getHours())}:${pad(d.getMinutes())}`;
|
||
return `<div class="msg"><span class="ts">[${esc(dateLabel)}]</span> <span class="sender">${esc(msg.sender)}</span><span class="body">: ${esc(formatExportBody(msg))}</span></div>`;
|
||
})
|
||
.join('\n');
|
||
|
||
content = `<!DOCTYPE html>
|
||
<html lang="en"><head><meta charset="UTF-8"><title>Export: ${esc(roomName)}</title>
|
||
<style>body{background:#0d0d0d;color:#c4d9ee;font-family:monospace;padding:20px}
|
||
.msg{margin:4px 0;line-height:1.5}
|
||
.ts{color:#555;font-size:0.85em}
|
||
.sender{color:#FF6B00;font-weight:bold}
|
||
.body{color:#c4d9ee}</style></head>
|
||
<body><h2 style="color:#00D4FF">Room: ${esc(roomName)}</h2>
|
||
<p style="color:#555">Exported ${esc(exportedAt)} — ${collected.length} messages</p>
|
||
<div class="messages">
|
||
${msgRows}
|
||
</div></body></html>`;
|
||
mimeType = 'text/html';
|
||
ext = 'html';
|
||
}
|
||
|
||
const blob = new Blob([content], { type: mimeType });
|
||
const url = URL.createObjectURL(blob);
|
||
const a = document.createElement('a');
|
||
a.href = url;
|
||
a.download = `export-${safeRoomName}-${dateStr}.${ext}`;
|
||
a.click();
|
||
URL.revokeObjectURL(url);
|
||
|
||
if (truncated) {
|
||
setNotice(
|
||
`Export truncated to ${collected.length} messages (reached the ${MAX_EXPORT_PAGES}-page limit). Narrow the date range to export older history.`,
|
||
);
|
||
}
|
||
} finally {
|
||
setExporting(false);
|
||
}
|
||
}, [exporting, format, fromDate, toDate, mx, room, roomName]);
|
||
|
||
return (
|
||
<Page>
|
||
<PageHeader outlined={false}>
|
||
<Box grow="Yes" gap="200">
|
||
<Box grow="Yes" alignItems="Center" gap="200">
|
||
<Text as="h2" size="H3" truncate>
|
||
Export
|
||
</Text>
|
||
</Box>
|
||
<Box shrink="No">
|
||
<IconButton onClick={requestClose} variant="Surface" aria-label="Close">
|
||
<Icon src={Icons.Cross} />
|
||
</IconButton>
|
||
</Box>
|
||
</Box>
|
||
</PageHeader>
|
||
<Box grow="Yes">
|
||
<Scroll hideTrack visibility="Hover">
|
||
<PageContent>
|
||
<Box direction="Column" gap="700">
|
||
{/* Format */}
|
||
<Box direction="Column" gap="100">
|
||
<Text size="L400">Format</Text>
|
||
<SequenceCard
|
||
className={SequenceCardStyle}
|
||
variant="SurfaceVariant"
|
||
direction="Column"
|
||
gap="400"
|
||
>
|
||
<Box gap="200" wrap="Wrap">
|
||
{(Object.keys(FORMAT_LABELS) as ExportFormat[]).map((f) => (
|
||
<Button
|
||
key={f}
|
||
size="300"
|
||
variant={format === f ? 'Primary' : 'Secondary'}
|
||
fill={format === f ? 'Soft' : 'None'}
|
||
radii="300"
|
||
onClick={() => setFormat(f)}
|
||
aria-pressed={format === f}
|
||
>
|
||
<Text size="B300">{FORMAT_LABELS[f]}</Text>
|
||
</Button>
|
||
))}
|
||
</Box>
|
||
</SequenceCard>
|
||
</Box>
|
||
|
||
{/* Date range */}
|
||
<Box direction="Column" gap="100">
|
||
<Text size="L400">Date Range (optional)</Text>
|
||
<SequenceCard
|
||
className={SequenceCardStyle}
|
||
variant="SurfaceVariant"
|
||
direction="Column"
|
||
gap="400"
|
||
>
|
||
<Box gap="400" wrap="Wrap">
|
||
<Box direction="Column" gap="100" style={{ flex: 1, minWidth: 160 }}>
|
||
<Text size="T300">From</Text>
|
||
<Input
|
||
type="date"
|
||
variant="Secondary"
|
||
size="400"
|
||
radii="300"
|
||
value={fromDate}
|
||
onChange={(e) => setFromDate(e.target.value)}
|
||
/>
|
||
</Box>
|
||
<Box direction="Column" gap="100" style={{ flex: 1, minWidth: 160 }}>
|
||
<Text size="T300">To</Text>
|
||
<Input
|
||
type="date"
|
||
variant="Secondary"
|
||
size="400"
|
||
radii="300"
|
||
value={toDate}
|
||
onChange={(e) => setToDate(e.target.value)}
|
||
/>
|
||
</Box>
|
||
</Box>
|
||
<Text size="T200" priority="300">
|
||
Leave blank to export all available history.
|
||
</Text>
|
||
</SequenceCard>
|
||
</Box>
|
||
|
||
{/* Export */}
|
||
<Box direction="Column" gap="100">
|
||
<Text size="L400">Download</Text>
|
||
<SequenceCard
|
||
className={SequenceCardStyle}
|
||
variant="SurfaceVariant"
|
||
direction="Column"
|
||
gap="400"
|
||
>
|
||
<Box alignItems="Center" gap="400" justifyContent="SpaceBetween">
|
||
<Text size="T300" priority="300">
|
||
{exporting
|
||
? `Exporting… ${exportCount} messages`
|
||
: 'Export will download automatically.'}
|
||
</Text>
|
||
{exporting ? (
|
||
<Button
|
||
size="400"
|
||
variant="Critical"
|
||
fill="Soft"
|
||
radii="300"
|
||
onClick={handleCancel}
|
||
before={<Icon src={Icons.Cross} size="100" />}
|
||
>
|
||
<Text size="B400">Cancel</Text>
|
||
</Button>
|
||
) : (
|
||
<Button
|
||
size="400"
|
||
variant="Primary"
|
||
fill="Solid"
|
||
radii="300"
|
||
onClick={handleExport}
|
||
before={<Icon src={Icons.Download} size="100" />}
|
||
>
|
||
<Text size="B400">Export</Text>
|
||
</Button>
|
||
)}
|
||
</Box>
|
||
{notice && (
|
||
<Text size="T200" priority="400">
|
||
{notice}
|
||
</Text>
|
||
)}
|
||
<Text size="T200" priority="300">
|
||
Attachments (images, videos, audio, files) are not included in the export -
|
||
media messages are marked with a type label (e.g. "[Image]") but only
|
||
their filename is exported, not the file itself.
|
||
</Text>
|
||
</SequenceCard>
|
||
</Box>
|
||
</Box>
|
||
</PageContent>
|
||
</Scroll>
|
||
</Box>
|
||
</Page>
|
||
);
|
||
}
|