diff --git a/e2e/your-data.spec.ts b/e2e/your-data.spec.ts new file mode 100644 index 000000000..379ae5e37 --- /dev/null +++ b/e2e/your-data.spec.ts @@ -0,0 +1,98 @@ +import fs from 'node:fs'; +import { test, expect } from '@playwright/test'; +import { api, ensureUser, hsReachable, loginUI, uniq } from './localHs'; + +// [Gitea #114] Export / import "your data": a round trip through the file, the +// merge summary, and the keep-mine / use-the-file's choice. +const enc = encodeURIComponent; + +test.describe('your data (#114)', () => { + test.beforeAll(async () => { + test.skip(!(await hsReachable()), 'local homeserver not running'); + }); + + test('export, change things, import merges with the chosen conflict rule', async ({ page }) => { + test.setTimeout(150_000); + const user = await ensureUser(uniq('yd')); + const put = (type: string, content: unknown) => + api( + 'PUT', + `/_matrix/client/v3/user/${enc(user.userId)}/account_data/${type}`, + user.token, + content, + ); + const get = (type: string) => + api>( + 'GET', + `/_matrix/client/v3/user/${enc(user.userId)}/account_data/${type}`, + user.token, + ); + const bm = (eventId: string) => ({ roomId: '!r:localhost', eventId, savedAt: 1 }); + await put('io.lotus.bookmarks', { bookmarks: [bm('$one'), bm('$two')] }); + await put('io.lotus.user_notes', { '@friend:localhost': 'met at the LAN party' }); + + await loginUI(page, user); + await page + .getByText('Connecting...') + .first() + .waitFor({ state: 'detached', timeout: 90_000 }) + .catch(() => undefined); + const openGeneral = async () => { + await page.getByRole('button', { name: 'User Settings' }).click(); + await page.getByRole('dialog').getByText('General', { exact: true }).first().click(); + await page + .getByRole('dialog') + .getByText('Your data', { exact: true }) + .scrollIntoViewIfNeeded(); + }; + await openGeneral(); + + // Export. + const [download] = await Promise.all([ + page.waitForEvent('download'), + page.getByRole('dialog').getByRole('button', { name: 'Export', exact: true }).click(), + ]); + const path = await download.path(); + const file = JSON.parse(fs.readFileSync(path!, 'utf8')); + expect(file.format).toBe('lotus-chat-data'); + expect(file.accountData['io.lotus.bookmarks'].bookmarks).toHaveLength(2); + expect(file.local.settings).toBeTruthy(); + + // Lose a bookmark, change the note. + await put('io.lotus.bookmarks', { bookmarks: [bm('$one')] }); + await put('io.lotus.user_notes', { '@friend:localhost': 'changed my mind' }); + await page.waitForTimeout(3000); // let sync deliver the new account data + + const importFile = async () => { + await page + .locator('input[type="file"][aria-label="Lotus Chat data file"]') + .setInputFiles(path!); + await expect(page.getByRole('region', { name: 'Import summary' })).toBeVisible(); + }; + await importFile(); + const summary = page.getByRole('region', { name: 'Import summary' }); + await expect(summary).toContainText('Bookmarks: 1 new'); + await expect(summary).toContainText('Private notes about people: 1 differ'); + + // Keep mine: the bookmark comes back, the note stays as it is now. + await summary.getByRole('button', { name: 'Import', exact: true }).click(); + await expect(page.getByText('Imported.')).toBeVisible({ timeout: 20_000 }); + expect(((await get('io.lotus.bookmarks')).bookmarks as unknown[]).length).toBe(2); + expect((await get('io.lotus.user_notes'))['@friend:localhost']).toBe('changed my mind'); + + // Again, using the file's: the note goes back. + await page.waitForTimeout(2000); + await importFile(); + await summary.getByText("Use the file's").click(); + await summary.getByRole('button', { name: 'Import', exact: true }).click(); + await expect(page.getByText('Imported.')).toBeVisible({ timeout: 20_000 }); + expect((await get('io.lotus.user_notes'))['@friend:localhost']).toBe('met at the LAN party'); + + // Wrong files are refused with a plain message. + fs.writeFileSync(`${path}.bad`, '{"format":"something-else"}'); + await page + .locator('input[type="file"][aria-label="Lotus Chat data file"]') + .setInputFiles(`${path}.bad`); + await expect(page.getByText("This isn't a Lotus Chat data file.")).toBeVisible(); + }); +}); diff --git a/src/app/features/settings/general/General.tsx b/src/app/features/settings/general/General.tsx index d1c9be22d..bc551da7c 100644 --- a/src/app/features/settings/general/General.tsx +++ b/src/app/features/settings/general/General.tsx @@ -61,6 +61,7 @@ import { } from '../../../utils/lotusDenoiseUtils'; import { useSetting } from '../../../state/hooks/settings'; import { useSettingsSyncActions } from '../../../hooks/useSettingsSync'; +import { YourDataSection } from './YourData'; import { CallAudioBitrate, ChatBackground, @@ -3007,6 +3008,7 @@ export function General({ requestClose }: GeneralProps) { + diff --git a/src/app/features/settings/general/YourData.tsx b/src/app/features/settings/general/YourData.tsx new file mode 100644 index 000000000..1a9ea2e40 --- /dev/null +++ b/src/app/features/settings/general/YourData.tsx @@ -0,0 +1,285 @@ +import React, { useCallback, useRef, useState } from 'react'; +import { useAtom } from 'jotai'; +import { Method } from 'matrix-js-sdk'; +import { Box, Button, Text, color } from 'folds'; +import { SequenceCard } from '../../../components/sequence-card'; +import { SequenceCardStyle } from '../styles.css'; +import { SettingTile } from '../../../components/setting-tile'; +import { useMatrixClient } from '../../../hooks/useMatrixClient'; +import { Settings, settingsAtom } from '../../../state/settings'; +import { + buildExport, + EXPORT_ACCOUNT_DATA, + ExportFile, + ImportPlanItem, + mergeData, + parseExport, + planImport, +} from '../../../utils/dataExport'; +import { invalidateDecorationCache } from '../../../hooks/useAvatarDecoration'; + +const DECORATION_FIELD = 'io.lotus.avatar_decoration'; + +type Current = { + accountData: Record; + decoration: string | null; + localSettings: Record | null; +}; + +/** + * [Gitea #114] "Your data": export the account's Lotus data (settings, + * bookmarks, notes, sections, ...) to a file and import it here or on another + * account. Import merges: new things are added; for things that differ the + * user picks "keep mine" (default) or "use the file's". Message history and + * encryption keys have their own exports. + */ +export function YourDataSection() { + const mx = useMatrixClient(); + const userId = mx.getSafeUserId(); + const [settings, setSettings] = useAtom(settingsAtom); + const fileInput = useRef(null); + const [pending, setPending] = useState<{ file: ExportFile; plan: ImportPlanItem[] }>(); + const [preferFile, setPreferFile] = useState(false); + const [message, setMessage] = useState<{ tone: 'ok' | 'error'; text: string }>(); + const [busy, setBusy] = useState(false); + + const readCurrent = useCallback(async (): Promise => { + const accountData: Record = {}; + EXPORT_ACCOUNT_DATA.forEach(({ type }) => { + const c = mx.getAccountData(type as never)?.getContent(); + if (c && Object.keys(c).length > 0) accountData[type] = c; + }); + let decoration: string | null = null; + try { + const profile = await mx.http.authedRequest>( + Method.Get, + `/profile/${encodeURIComponent(userId)}`, + ); + const v = profile[DECORATION_FIELD]; + decoration = typeof v === 'string' && v ? v : null; + } catch { + /* no decoration */ + } + return { + accountData, + decoration, + localSettings: settings as unknown as Record, + }; + }, [mx, userId, settings]); + + const exportData = async () => { + setBusy(true); + setMessage(undefined); + try { + const cur = await readCurrent(); + const file = buildExport({ userId, ...cur }); + const blob = new Blob([JSON.stringify(file, null, 2)], { type: 'application/json' }); + const url = URL.createObjectURL(blob); + const a = document.createElement('a'); + a.href = url; + a.download = `lotus-chat-data-${userId.replace(/[^a-z0-9]+/gi, '-')}-${file.exportedAt.slice(0, 10)}.json`; + a.click(); + URL.revokeObjectURL(url); + setMessage({ tone: 'ok', text: 'Exported. Keep the file somewhere safe.' }); + } catch { + setMessage({ tone: 'error', text: "Couldn't export your data. Please try again." }); + } finally { + setBusy(false); + } + }; + + const chooseFile = async (f: File | undefined) => { + if (!f) return; + setMessage(undefined); + const parsed = parseExport(await f.text()); + if (!parsed.ok) { + setMessage({ tone: 'error', text: parsed.error }); + return; + } + const plan = planImport(parsed.file, await readCurrent()); + setPreferFile(false); + setPending({ file: parsed.file, plan }); + }; + + const applyImport = async () => { + if (!pending) return; + setBusy(true); + try { + const cur = await readCurrent(); + const { file } = pending; + await Object.entries(file.accountData).reduce>( + async (prev, [type, incoming]) => { + await prev; + const r = mergeData(cur.accountData[type], incoming, preferFile); + if (r.added > 0 || (preferFile && r.conflicts > 0)) { + await mx.setAccountData(type as never, r.value as never); + } + }, + Promise.resolve(), + ); + const deco = file.profile.decoration; + if (deco && deco !== cur.decoration && (!cur.decoration || preferFile)) { + await mx.http.authedRequest( + Method.Put, + `/profile/${encodeURIComponent(userId)}/${DECORATION_FIELD}`, + undefined, + { [DECORATION_FIELD]: deco }, + ); + invalidateDecorationCache(userId); + } + if (file.local.settings) { + const r = mergeData(cur.localSettings ?? {}, file.local.settings, preferFile); + setSettings({ ...settings, ...(r.value as Partial) }); + } + setPending(undefined); + setMessage({ tone: 'ok', text: 'Imported.' }); + } catch { + setMessage({ tone: 'error', text: "Couldn't finish the import. Nothing else was changed." }); + } finally { + setBusy(false); + } + }; + + const totals = pending?.plan.reduce( + (t, p) => ({ added: t.added + p.added, conflicts: t.conflicts + p.conflicts }), + { added: 0, conflicts: 0 }, + ); + + return ( + + Your data + + + Export + + } + /> + fileInput.current?.click()} + > + Import… + + } + /> + { + chooseFile(e.target.files?.[0]).catch(() => undefined); + e.target.value = ''; + }} + /> + {pending && totals && ( + + {pending.file.userId && pending.file.userId !== userId && ( + + This file is from {pending.file.userId}. + + )} + {pending.plan.map((p) => ( + + {p.label}:{' '} + {p.added === 0 && p.conflicts === 0 + ? 'already up to date' + : [p.added > 0 && `${p.added} new`, p.conflicts > 0 && `${p.conflicts} differ`] + .filter(Boolean) + .join(', ')} + + ))} + {totals.conflicts > 0 && ( + + + {totals.conflicts}{' '} + {totals.conflicts === 1 ? 'thing differs' : 'things differ'} from what's + here: + + + + + )} + + + + + + )} + {message && ( + + {message.text} + + )} + + + ); +} diff --git a/src/app/utils/dataExport.test.ts b/src/app/utils/dataExport.test.ts new file mode 100644 index 000000000..6090bacdb --- /dev/null +++ b/src/app/utils/dataExport.test.ts @@ -0,0 +1,76 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { buildExport, mergeData, parseExport, planImport } from './dataExport'; + +const file = buildExport({ + userId: '@a:hs', + accountData: { + 'io.lotus.bookmarks': { bookmarks: [{ id: 'b1', note: 'x' }] }, + 'io.lotus.user_notes': { notes: { '@b:hs': 'met at LAN party' } }, + 'm.direct': { secret: true }, + 'io.lotus.settings': { theme: 'dark', fontSize: 16 }, + }, + decoration: 'halo', + localSettings: { callNoiseSuppression: 'ml' }, + now: new Date('2026-10-25T00:00:00Z'), +}); + +test('[#114] export keeps only our own types', () => { + assert.equal(file.format, 'lotus-chat-data'); + assert.equal(file.version, 1); + assert.deepEqual(Object.keys(file.accountData).sort(), [ + 'io.lotus.bookmarks', + 'io.lotus.settings', + 'io.lotus.user_notes', + ]); + assert.equal(file.profile.decoration, 'halo'); +}); + +test('[#114] parse: round trip, wrong files, newer versions, foreign types', () => { + const r = parseExport(JSON.stringify(file)); + assert.ok(r.ok); + assert.equal(parseExport('nope').ok, false); + assert.equal(parseExport('{"format":"other"}').ok, false); + const newer = parseExport(JSON.stringify({ ...file, version: 2 })); + assert.ok(!newer.ok && /newer Lotus Chat/.test(newer.error)); + const sneaky = parseExport( + JSON.stringify({ ...file, accountData: { 'm.push_rules': {}, 'io.lotus.bookmarks': {} } }), + ); + assert.ok(sneaky.ok); + assert.deepEqual(Object.keys(sneaky.file.accountData), ['io.lotus.bookmarks']); +}); + +test('[#114] merge: lists by id, maps by key, conflicts kept unless preferred', () => { + const mine = { bookmarks: [{ id: 'b1', note: 'mine' }, { id: 'b2' }] }; + const theirs = { bookmarks: [{ id: 'b1', note: 'theirs' }, { id: 'b3' }] }; + const keep = mergeData(mine, theirs); + assert.equal(keep.added, 1); + assert.equal(keep.conflicts, 1); + assert.deepEqual(keep.value, { + bookmarks: [{ id: 'b1', note: 'mine' }, { id: 'b2' }, { id: 'b3' }], + }); + const take = mergeData(mine, theirs, true); + assert.deepEqual((take.value as typeof mine).bookmarks[0], { id: 'b1', note: 'theirs' }); + + const notes = mergeData({ a: 1, b: 2 }, { b: 3, c: 4 }); + assert.deepEqual(notes, { value: { a: 1, b: 2, c: 4 }, added: 1, conflicts: 1 }); + assert.deepEqual(mergeData(undefined, { x: 1 }), { value: { x: 1 }, added: 1, conflicts: 0 }); + assert.deepEqual(mergeData(['a'], ['a', 'b']).value, ['a', 'b']); +}); + +test('[#114] plan summarises per category', () => { + const plan = planImport(file, { + accountData: { 'io.lotus.settings': { theme: 'light' } }, + decoration: null, + localSettings: { callNoiseSuppression: 'ml' }, + }); + const by = Object.fromEntries(plan.map((p) => [p.key, p])); + assert.deepEqual( + [by['io.lotus.settings'].added, by['io.lotus.settings'].conflicts], + [1, 1], + 'fontSize new, theme differs', + ); + assert.equal(by['io.lotus.bookmarks'].added, 1); + assert.equal(by['profile.decoration'].added, 1); + assert.deepEqual([by['local.settings'].added, by['local.settings'].conflicts], [0, 0]); +}); diff --git a/src/app/utils/dataExport.ts b/src/app/utils/dataExport.ts new file mode 100644 index 000000000..629f968ee --- /dev/null +++ b/src/app/utils/dataExport.ts @@ -0,0 +1,219 @@ +/** + * [Gitea #114] Export / import "your data": the Lotus-side personal data of + * an account (not message history, not encryption keys: those have their own + * exports, and keys are secret). One JSON file: + * + * { format: 'lotus-chat-data', version: 1, exportedAt, userId, + * accountData: { [type]: content }, profile: { decoration }, local: { settings } } + * + * Import MERGES: anything missing here is added; a value that differs is a + * conflict, kept as it is unless the user chooses the file's. Pure helpers. + */ + +export const EXPORT_FORMAT = 'lotus-chat-data'; +export const EXPORT_VERSION = 1; + +/** The account-data types that are the user's own Lotus data, with labels. */ +export const EXPORT_ACCOUNT_DATA: { type: string; label: string }[] = [ + { type: 'io.lotus.settings', label: 'Synced settings' }, + { type: 'io.lotus.bookmarks', label: 'Bookmarks' }, + { type: 'io.lotus.user_notes', label: 'Private notes about people' }, + { type: 'io.lotus.room_names', label: 'Your names for rooms' }, + { type: 'in.cinny.spaces', label: 'Room sections and sidebar' }, + { type: 'io.lotus.reminders', label: 'Reminders' }, + { type: 'io.lotus.status_presets', label: 'Status presets' }, + { type: 'io.lotus.thread_notifications', label: 'Thread notification settings' }, + { type: 'io.lotus.mute_timers', label: 'Timed mutes' }, + { type: 'io.lotus.soundboard', label: 'Soundboard (clips stay on the server)' }, + { type: 'io.lotus.soundboard_rooms', label: 'Soundboards enabled everywhere' }, + { type: 'im.ponies.user_emotes', label: 'Your emoji and stickers' }, + { type: 'im.ponies.emote_rooms', label: 'Emoji packs enabled everywhere' }, +]; + +export type ExportFile = { + format: typeof EXPORT_FORMAT; + version: number; + exportedAt: string; + userId: string; + accountData: Record; + profile: { decoration: string | null }; + local: { settings: Record | null }; +}; + +export const buildExport = (input: { + userId: string; + accountData: Record; + decoration: string | null; + localSettings: Record | null; + now?: Date; +}): ExportFile => { + const accountData: Record = {}; + EXPORT_ACCOUNT_DATA.forEach(({ type }) => { + const c = input.accountData[type]; + if (c !== undefined && c !== null) accountData[type] = c; + }); + return { + format: EXPORT_FORMAT, + version: EXPORT_VERSION, + exportedAt: (input.now ?? new Date()).toISOString(), + userId: input.userId, + accountData, + profile: { decoration: input.decoration }, + local: { settings: input.localSettings }, + }; +}; + +const isObject = (v: unknown): v is Record => + !!v && typeof v === 'object' && !Array.isArray(v); + +export type ParseResult = { ok: true; file: ExportFile } | { ok: false; error: string }; + +/** Validate an imported file; older versions would be migrated here. */ +export const parseExport = (text: string): ParseResult => { + let v: unknown; + try { + v = JSON.parse(text); + } catch { + return { ok: false, error: "This isn't a Lotus Chat data file (it isn't valid JSON)." }; + } + if (!isObject(v) || v.format !== EXPORT_FORMAT) + return { ok: false, error: "This isn't a Lotus Chat data file." }; + if (typeof v.version !== 'number' || v.version < 1) + return { ok: false, error: 'This data file is damaged (no version).' }; + if (v.version > EXPORT_VERSION) + return { + ok: false, + error: 'This file was made by a newer Lotus Chat. Update the app, then import it again.', + }; + const accountData = isObject(v.accountData) ? v.accountData : {}; + const known = new Set(EXPORT_ACCOUNT_DATA.map((t) => t.type)); + const filtered: Record = {}; + Object.entries(accountData).forEach(([type, content]) => { + // Only our own types: a file can't be used to write arbitrary account data. + if (known.has(type) && content !== null && content !== undefined) filtered[type] = content; + }); + const profile = isObject(v.profile) ? v.profile : {}; + const local = isObject(v.local) ? v.local : {}; + return { + ok: true, + file: { + format: EXPORT_FORMAT, + version: v.version, + exportedAt: typeof v.exportedAt === 'string' ? v.exportedAt : '', + userId: typeof v.userId === 'string' ? v.userId : '', + accountData: filtered, + profile: { + decoration: typeof profile.decoration === 'string' ? profile.decoration : null, + }, + local: { settings: isObject(local.settings) ? local.settings : null }, + }, + }; +}; + +const same = (a: unknown, b: unknown): boolean => JSON.stringify(a) === JSON.stringify(b); + +const itemKey = (v: unknown): string => { + if (isObject(v)) { + const id = v.id ?? v.eventId ?? v.event_id ?? v.key ?? v.roomId ?? v.room_id ?? v.mxc; + if (typeof id === 'string' || typeof id === 'number') return `id:${String(id)}`; + } + return `json:${JSON.stringify(v)}`; +}; + +export type MergeResult = { value: unknown; added: number; conflicts: number }; + +/** + * Pure: merge `incoming` into `current`. Lists: add items that are missing + * (matched by id, else by content). Objects: add missing keys, recurse into + * shared ones. A differing leaf is a conflict: `preferIncoming` decides. + */ +export const mergeData = ( + current: unknown, + incoming: unknown, + preferIncoming = false, +): MergeResult => { + if (current === undefined || current === null) { + return { value: incoming, added: incoming === undefined ? 0 : 1, conflicts: 0 }; + } + if (Array.isArray(current) && Array.isArray(incoming)) { + const have = new Map(current.map((v, i) => [itemKey(v), i])); + const out = [...current]; + let added = 0; + let conflicts = 0; + incoming.forEach((item) => { + const k = itemKey(item); + const at = have.get(k); + if (at === undefined) { + out.push(item); + have.set(k, out.length - 1); + added += 1; + } else if (!same(out[at], item)) { + conflicts += 1; + if (preferIncoming) out[at] = item; + } + }); + return { value: out, added, conflicts }; + } + if (isObject(current) && isObject(incoming)) { + const out: Record = { ...current }; + let added = 0; + let conflicts = 0; + Object.entries(incoming).forEach(([k, v]) => { + if (!(k in current)) { + out[k] = v; + added += 1; + return; + } + const r = mergeData(current[k], v, preferIncoming); + out[k] = r.value; + added += r.added; + conflicts += r.conflicts; + }); + return { value: out, added, conflicts }; + } + if (same(current, incoming)) return { value: current, added: 0, conflicts: 0 }; + return { value: preferIncoming ? incoming : current, added: 0, conflicts: 1 }; +}; + +export type ImportPlanItem = { + key: string; + label: string; + added: number; + conflicts: number; +}; + +/** What importing `file` into the current data would change, per category. */ +export const planImport = ( + file: ExportFile, + current: { + accountData: Record; + decoration: string | null; + localSettings: Record | null; + }, +): ImportPlanItem[] => { + const items: ImportPlanItem[] = []; + EXPORT_ACCOUNT_DATA.forEach(({ type, label }) => { + if (!(type in file.accountData)) return; + const r = mergeData(current.accountData[type], file.accountData[type]); + items.push({ key: type, label, added: r.added, conflicts: r.conflicts }); + }); + if (file.profile.decoration !== null) { + const differs = current.decoration !== file.profile.decoration; + items.push({ + key: 'profile.decoration', + label: 'Avatar decoration', + added: !current.decoration && differs ? 1 : 0, + conflicts: current.decoration && differs ? 1 : 0, + }); + } + if (file.local.settings) { + const r = mergeData(current.localSettings ?? {}, file.local.settings); + items.push({ + key: 'local.settings', + label: "This device's settings", + added: r.added, + conflicts: r.conflicts, + }); + } + return items; +};