From 378b40fabe56767342181f4c39081b7e371e8ec2 Mon Sep 17 00:00:00 2001 From: Lotus CI Date: Sun, 25 Oct 2026 01:08:28 -0400 Subject: [PATCH] feat(settings): export / import your data (#114) Settings -> General -> "Your data": Export writes one JSON file of the account's Lotus data (synced settings, bookmarks, private notes, room names and sections, reminders, status presets, thread notification settings, timed mutes, soundboard (clips stay on the server), emoji packs, avatar decoration, and this device's settings). Not messages or encryption keys: they have their own exports, and keys are secret. Import merges: new items are added; things that differ are listed per category and kept unless the user picks "Use the file's". Only our own account-data types are read from a file. Files from a newer Lotus Chat are refused with a plain message; the version stamp leaves room to migrate older ones. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_017PCb59uY3UKqDqt7B6hzd5 --- e2e/your-data.spec.ts | 98 ++++++ src/app/features/settings/general/General.tsx | 2 + .../features/settings/general/YourData.tsx | 285 ++++++++++++++++++ src/app/utils/dataExport.test.ts | 76 +++++ src/app/utils/dataExport.ts | 219 ++++++++++++++ 5 files changed, 680 insertions(+) create mode 100644 e2e/your-data.spec.ts create mode 100644 src/app/features/settings/general/YourData.tsx create mode 100644 src/app/utils/dataExport.test.ts create mode 100644 src/app/utils/dataExport.ts diff --git a/e2e/your-data.spec.ts b/e2e/your-data.spec.ts new file mode 100644 index 000000000..379ae5e37 --- /dev/null +++ b/e2e/your-data.spec.ts @@ -0,0 +1,98 @@ +import fs from 'node:fs'; +import { test, expect } from '@playwright/test'; +import { api, ensureUser, hsReachable, loginUI, uniq } from './localHs'; + +// [Gitea #114] Export / import "your data": a round trip through the file, the +// merge summary, and the keep-mine / use-the-file's choice. +const enc = encodeURIComponent; + +test.describe('your data (#114)', () => { + test.beforeAll(async () => { + test.skip(!(await hsReachable()), 'local homeserver not running'); + }); + + test('export, change things, import merges with the chosen conflict rule', async ({ page }) => { + test.setTimeout(150_000); + const user = await ensureUser(uniq('yd')); + const put = (type: string, content: unknown) => + api( + 'PUT', + `/_matrix/client/v3/user/${enc(user.userId)}/account_data/${type}`, + user.token, + content, + ); + const get = (type: string) => + api>( + 'GET', + `/_matrix/client/v3/user/${enc(user.userId)}/account_data/${type}`, + user.token, + ); + const bm = (eventId: string) => ({ roomId: '!r:localhost', eventId, savedAt: 1 }); + await put('io.lotus.bookmarks', { bookmarks: [bm('$one'), bm('$two')] }); + await put('io.lotus.user_notes', { '@friend:localhost': 'met at the LAN party' }); + + await loginUI(page, user); + await page + .getByText('Connecting...') + .first() + .waitFor({ state: 'detached', timeout: 90_000 }) + .catch(() => undefined); + const openGeneral = async () => { + await page.getByRole('button', { name: 'User Settings' }).click(); + await page.getByRole('dialog').getByText('General', { exact: true }).first().click(); + await page + .getByRole('dialog') + .getByText('Your data', { exact: true }) + .scrollIntoViewIfNeeded(); + }; + await openGeneral(); + + // Export. + const [download] = await Promise.all([ + page.waitForEvent('download'), + page.getByRole('dialog').getByRole('button', { name: 'Export', exact: true }).click(), + ]); + const path = await download.path(); + const file = JSON.parse(fs.readFileSync(path!, 'utf8')); + expect(file.format).toBe('lotus-chat-data'); + expect(file.accountData['io.lotus.bookmarks'].bookmarks).toHaveLength(2); + expect(file.local.settings).toBeTruthy(); + + // Lose a bookmark, change the note. + await put('io.lotus.bookmarks', { bookmarks: [bm('$one')] }); + await put('io.lotus.user_notes', { '@friend:localhost': 'changed my mind' }); + await page.waitForTimeout(3000); // let sync deliver the new account data + + const importFile = async () => { + await page + .locator('input[type="file"][aria-label="Lotus Chat data file"]') + .setInputFiles(path!); + await expect(page.getByRole('region', { name: 'Import summary' })).toBeVisible(); + }; + await importFile(); + const summary = page.getByRole('region', { name: 'Import summary' }); + await expect(summary).toContainText('Bookmarks: 1 new'); + await expect(summary).toContainText('Private notes about people: 1 differ'); + + // Keep mine: the bookmark comes back, the note stays as it is now. + await summary.getByRole('button', { name: 'Import', exact: true }).click(); + await expect(page.getByText('Imported.')).toBeVisible({ timeout: 20_000 }); + expect(((await get('io.lotus.bookmarks')).bookmarks as unknown[]).length).toBe(2); + expect((await get('io.lotus.user_notes'))['@friend:localhost']).toBe('changed my mind'); + + // Again, using the file's: the note goes back. + await page.waitForTimeout(2000); + await importFile(); + await summary.getByText("Use the file's").click(); + await summary.getByRole('button', { name: 'Import', exact: true }).click(); + await expect(page.getByText('Imported.')).toBeVisible({ timeout: 20_000 }); + expect((await get('io.lotus.user_notes'))['@friend:localhost']).toBe('met at the LAN party'); + + // Wrong files are refused with a plain message. + fs.writeFileSync(`${path}.bad`, '{"format":"something-else"}'); + await page + .locator('input[type="file"][aria-label="Lotus Chat data file"]') + .setInputFiles(`${path}.bad`); + await expect(page.getByText("This isn't a Lotus Chat data file.")).toBeVisible(); + }); +}); diff --git a/src/app/features/settings/general/General.tsx b/src/app/features/settings/general/General.tsx index d1c9be22d..bc551da7c 100644 --- a/src/app/features/settings/general/General.tsx +++ b/src/app/features/settings/general/General.tsx @@ -61,6 +61,7 @@ import { } from '../../../utils/lotusDenoiseUtils'; import { useSetting } from '../../../state/hooks/settings'; import { useSettingsSyncActions } from '../../../hooks/useSettingsSync'; +import { YourDataSection } from './YourData'; import { CallAudioBitrate, ChatBackground, @@ -3007,6 +3008,7 @@ export function General({ requestClose }: GeneralProps) { + diff --git a/src/app/features/settings/general/YourData.tsx b/src/app/features/settings/general/YourData.tsx new file mode 100644 index 000000000..1a9ea2e40 --- /dev/null +++ b/src/app/features/settings/general/YourData.tsx @@ -0,0 +1,285 @@ +import React, { useCallback, useRef, useState } from 'react'; +import { useAtom } from 'jotai'; +import { Method } from 'matrix-js-sdk'; +import { Box, Button, Text, color } from 'folds'; +import { SequenceCard } from '../../../components/sequence-card'; +import { SequenceCardStyle } from '../styles.css'; +import { SettingTile } from '../../../components/setting-tile'; +import { useMatrixClient } from '../../../hooks/useMatrixClient'; +import { Settings, settingsAtom } from '../../../state/settings'; +import { + buildExport, + EXPORT_ACCOUNT_DATA, + ExportFile, + ImportPlanItem, + mergeData, + parseExport, + planImport, +} from '../../../utils/dataExport'; +import { invalidateDecorationCache } from '../../../hooks/useAvatarDecoration'; + +const DECORATION_FIELD = 'io.lotus.avatar_decoration'; + +type Current = { + accountData: Record; + decoration: string | null; + localSettings: Record | null; +}; + +/** + * [Gitea #114] "Your data": export the account's Lotus data (settings, + * bookmarks, notes, sections, ...) to a file and import it here or on another + * account. Import merges: new things are added; for things that differ the + * user picks "keep mine" (default) or "use the file's". Message history and + * encryption keys have their own exports. + */ +export function YourDataSection() { + const mx = useMatrixClient(); + const userId = mx.getSafeUserId(); + const [settings, setSettings] = useAtom(settingsAtom); + const fileInput = useRef(null); + const [pending, setPending] = useState<{ file: ExportFile; plan: ImportPlanItem[] }>(); + const [preferFile, setPreferFile] = useState(false); + const [message, setMessage] = useState<{ tone: 'ok' | 'error'; text: string }>(); + const [busy, setBusy] = useState(false); + + const readCurrent = useCallback(async (): Promise => { + const accountData: Record = {}; + EXPORT_ACCOUNT_DATA.forEach(({ type }) => { + const c = mx.getAccountData(type as never)?.getContent(); + if (c && Object.keys(c).length > 0) accountData[type] = c; + }); + let decoration: string | null = null; + try { + const profile = await mx.http.authedRequest>( + Method.Get, + `/profile/${encodeURIComponent(userId)}`, + ); + const v = profile[DECORATION_FIELD]; + decoration = typeof v === 'string' && v ? v : null; + } catch { + /* no decoration */ + } + return { + accountData, + decoration, + localSettings: settings as unknown as Record, + }; + }, [mx, userId, settings]); + + const exportData = async () => { + setBusy(true); + setMessage(undefined); + try { + const cur = await readCurrent(); + const file = buildExport({ userId, ...cur }); + const blob = new Blob([JSON.stringify(file, null, 2)], { type: 'application/json' }); + const url = URL.createObjectURL(blob); + const a = document.createElement('a'); + a.href = url; + a.download = `lotus-chat-data-${userId.replace(/[^a-z0-9]+/gi, '-')}-${file.exportedAt.slice(0, 10)}.json`; + a.click(); + URL.revokeObjectURL(url); + setMessage({ tone: 'ok', text: 'Exported. Keep the file somewhere safe.' }); + } catch { + setMessage({ tone: 'error', text: "Couldn't export your data. Please try again." }); + } finally { + setBusy(false); + } + }; + + const chooseFile = async (f: File | undefined) => { + if (!f) return; + setMessage(undefined); + const parsed = parseExport(await f.text()); + if (!parsed.ok) { + setMessage({ tone: 'error', text: parsed.error }); + return; + } + const plan = planImport(parsed.file, await readCurrent()); + setPreferFile(false); + setPending({ file: parsed.file, plan }); + }; + + const applyImport = async () => { + if (!pending) return; + setBusy(true); + try { + const cur = await readCurrent(); + const { file } = pending; + await Object.entries(file.accountData).reduce>( + async (prev, [type, incoming]) => { + await prev; + const r = mergeData(cur.accountData[type], incoming, preferFile); + if (r.added > 0 || (preferFile && r.conflicts > 0)) { + await mx.setAccountData(type as never, r.value as never); + } + }, + Promise.resolve(), + ); + const deco = file.profile.decoration; + if (deco && deco !== cur.decoration && (!cur.decoration || preferFile)) { + await mx.http.authedRequest( + Method.Put, + `/profile/${encodeURIComponent(userId)}/${DECORATION_FIELD}`, + undefined, + { [DECORATION_FIELD]: deco }, + ); + invalidateDecorationCache(userId); + } + if (file.local.settings) { + const r = mergeData(cur.localSettings ?? {}, file.local.settings, preferFile); + setSettings({ ...settings, ...(r.value as Partial) }); + } + setPending(undefined); + setMessage({ tone: 'ok', text: 'Imported.' }); + } catch { + setMessage({ tone: 'error', text: "Couldn't finish the import. Nothing else was changed." }); + } finally { + setBusy(false); + } + }; + + const totals = pending?.plan.reduce( + (t, p) => ({ added: t.added + p.added, conflicts: t.conflicts + p.conflicts }), + { added: 0, conflicts: 0 }, + ); + + return ( + + Your data + + + Export + + } + /> + fileInput.current?.click()} + > + Import… + + } + /> + { + chooseFile(e.target.files?.[0]).catch(() => undefined); + e.target.value = ''; + }} + /> + {pending && totals && ( + + {pending.file.userId && pending.file.userId !== userId && ( + + This file is from {pending.file.userId}. + + )} + {pending.plan.map((p) => ( + + {p.label}:{' '} + {p.added === 0 && p.conflicts === 0 + ? 'already up to date' + : [p.added > 0 && `${p.added} new`, p.conflicts > 0 && `${p.conflicts} differ`] + .filter(Boolean) + .join(', ')} + + ))} + {totals.conflicts > 0 && ( + + + {totals.conflicts}{' '} + {totals.conflicts === 1 ? 'thing differs' : 'things differ'} from what's + here: + + + + + )} + + + + + + )} + {message && ( + + {message.text} + + )} + + + ); +} diff --git a/src/app/utils/dataExport.test.ts b/src/app/utils/dataExport.test.ts new file mode 100644 index 000000000..6090bacdb --- /dev/null +++ b/src/app/utils/dataExport.test.ts @@ -0,0 +1,76 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { buildExport, mergeData, parseExport, planImport } from './dataExport'; + +const file = buildExport({ + userId: '@a:hs', + accountData: { + 'io.lotus.bookmarks': { bookmarks: [{ id: 'b1', note: 'x' }] }, + 'io.lotus.user_notes': { notes: { '@b:hs': 'met at LAN party' } }, + 'm.direct': { secret: true }, + 'io.lotus.settings': { theme: 'dark', fontSize: 16 }, + }, + decoration: 'halo', + localSettings: { callNoiseSuppression: 'ml' }, + now: new Date('2026-10-25T00:00:00Z'), +}); + +test('[#114] export keeps only our own types', () => { + assert.equal(file.format, 'lotus-chat-data'); + assert.equal(file.version, 1); + assert.deepEqual(Object.keys(file.accountData).sort(), [ + 'io.lotus.bookmarks', + 'io.lotus.settings', + 'io.lotus.user_notes', + ]); + assert.equal(file.profile.decoration, 'halo'); +}); + +test('[#114] parse: round trip, wrong files, newer versions, foreign types', () => { + const r = parseExport(JSON.stringify(file)); + assert.ok(r.ok); + assert.equal(parseExport('nope').ok, false); + assert.equal(parseExport('{"format":"other"}').ok, false); + const newer = parseExport(JSON.stringify({ ...file, version: 2 })); + assert.ok(!newer.ok && /newer Lotus Chat/.test(newer.error)); + const sneaky = parseExport( + JSON.stringify({ ...file, accountData: { 'm.push_rules': {}, 'io.lotus.bookmarks': {} } }), + ); + assert.ok(sneaky.ok); + assert.deepEqual(Object.keys(sneaky.file.accountData), ['io.lotus.bookmarks']); +}); + +test('[#114] merge: lists by id, maps by key, conflicts kept unless preferred', () => { + const mine = { bookmarks: [{ id: 'b1', note: 'mine' }, { id: 'b2' }] }; + const theirs = { bookmarks: [{ id: 'b1', note: 'theirs' }, { id: 'b3' }] }; + const keep = mergeData(mine, theirs); + assert.equal(keep.added, 1); + assert.equal(keep.conflicts, 1); + assert.deepEqual(keep.value, { + bookmarks: [{ id: 'b1', note: 'mine' }, { id: 'b2' }, { id: 'b3' }], + }); + const take = mergeData(mine, theirs, true); + assert.deepEqual((take.value as typeof mine).bookmarks[0], { id: 'b1', note: 'theirs' }); + + const notes = mergeData({ a: 1, b: 2 }, { b: 3, c: 4 }); + assert.deepEqual(notes, { value: { a: 1, b: 2, c: 4 }, added: 1, conflicts: 1 }); + assert.deepEqual(mergeData(undefined, { x: 1 }), { value: { x: 1 }, added: 1, conflicts: 0 }); + assert.deepEqual(mergeData(['a'], ['a', 'b']).value, ['a', 'b']); +}); + +test('[#114] plan summarises per category', () => { + const plan = planImport(file, { + accountData: { 'io.lotus.settings': { theme: 'light' } }, + decoration: null, + localSettings: { callNoiseSuppression: 'ml' }, + }); + const by = Object.fromEntries(plan.map((p) => [p.key, p])); + assert.deepEqual( + [by['io.lotus.settings'].added, by['io.lotus.settings'].conflicts], + [1, 1], + 'fontSize new, theme differs', + ); + assert.equal(by['io.lotus.bookmarks'].added, 1); + assert.equal(by['profile.decoration'].added, 1); + assert.deepEqual([by['local.settings'].added, by['local.settings'].conflicts], [0, 0]); +}); diff --git a/src/app/utils/dataExport.ts b/src/app/utils/dataExport.ts new file mode 100644 index 000000000..629f968ee --- /dev/null +++ b/src/app/utils/dataExport.ts @@ -0,0 +1,219 @@ +/** + * [Gitea #114] Export / import "your data": the Lotus-side personal data of + * an account (not message history, not encryption keys: those have their own + * exports, and keys are secret). One JSON file: + * + * { format: 'lotus-chat-data', version: 1, exportedAt, userId, + * accountData: { [type]: content }, profile: { decoration }, local: { settings } } + * + * Import MERGES: anything missing here is added; a value that differs is a + * conflict, kept as it is unless the user chooses the file's. Pure helpers. + */ + +export const EXPORT_FORMAT = 'lotus-chat-data'; +export const EXPORT_VERSION = 1; + +/** The account-data types that are the user's own Lotus data, with labels. */ +export const EXPORT_ACCOUNT_DATA: { type: string; label: string }[] = [ + { type: 'io.lotus.settings', label: 'Synced settings' }, + { type: 'io.lotus.bookmarks', label: 'Bookmarks' }, + { type: 'io.lotus.user_notes', label: 'Private notes about people' }, + { type: 'io.lotus.room_names', label: 'Your names for rooms' }, + { type: 'in.cinny.spaces', label: 'Room sections and sidebar' }, + { type: 'io.lotus.reminders', label: 'Reminders' }, + { type: 'io.lotus.status_presets', label: 'Status presets' }, + { type: 'io.lotus.thread_notifications', label: 'Thread notification settings' }, + { type: 'io.lotus.mute_timers', label: 'Timed mutes' }, + { type: 'io.lotus.soundboard', label: 'Soundboard (clips stay on the server)' }, + { type: 'io.lotus.soundboard_rooms', label: 'Soundboards enabled everywhere' }, + { type: 'im.ponies.user_emotes', label: 'Your emoji and stickers' }, + { type: 'im.ponies.emote_rooms', label: 'Emoji packs enabled everywhere' }, +]; + +export type ExportFile = { + format: typeof EXPORT_FORMAT; + version: number; + exportedAt: string; + userId: string; + accountData: Record; + profile: { decoration: string | null }; + local: { settings: Record | null }; +}; + +export const buildExport = (input: { + userId: string; + accountData: Record; + decoration: string | null; + localSettings: Record | null; + now?: Date; +}): ExportFile => { + const accountData: Record = {}; + EXPORT_ACCOUNT_DATA.forEach(({ type }) => { + const c = input.accountData[type]; + if (c !== undefined && c !== null) accountData[type] = c; + }); + return { + format: EXPORT_FORMAT, + version: EXPORT_VERSION, + exportedAt: (input.now ?? new Date()).toISOString(), + userId: input.userId, + accountData, + profile: { decoration: input.decoration }, + local: { settings: input.localSettings }, + }; +}; + +const isObject = (v: unknown): v is Record => + !!v && typeof v === 'object' && !Array.isArray(v); + +export type ParseResult = { ok: true; file: ExportFile } | { ok: false; error: string }; + +/** Validate an imported file; older versions would be migrated here. */ +export const parseExport = (text: string): ParseResult => { + let v: unknown; + try { + v = JSON.parse(text); + } catch { + return { ok: false, error: "This isn't a Lotus Chat data file (it isn't valid JSON)." }; + } + if (!isObject(v) || v.format !== EXPORT_FORMAT) + return { ok: false, error: "This isn't a Lotus Chat data file." }; + if (typeof v.version !== 'number' || v.version < 1) + return { ok: false, error: 'This data file is damaged (no version).' }; + if (v.version > EXPORT_VERSION) + return { + ok: false, + error: 'This file was made by a newer Lotus Chat. Update the app, then import it again.', + }; + const accountData = isObject(v.accountData) ? v.accountData : {}; + const known = new Set(EXPORT_ACCOUNT_DATA.map((t) => t.type)); + const filtered: Record = {}; + Object.entries(accountData).forEach(([type, content]) => { + // Only our own types: a file can't be used to write arbitrary account data. + if (known.has(type) && content !== null && content !== undefined) filtered[type] = content; + }); + const profile = isObject(v.profile) ? v.profile : {}; + const local = isObject(v.local) ? v.local : {}; + return { + ok: true, + file: { + format: EXPORT_FORMAT, + version: v.version, + exportedAt: typeof v.exportedAt === 'string' ? v.exportedAt : '', + userId: typeof v.userId === 'string' ? v.userId : '', + accountData: filtered, + profile: { + decoration: typeof profile.decoration === 'string' ? profile.decoration : null, + }, + local: { settings: isObject(local.settings) ? local.settings : null }, + }, + }; +}; + +const same = (a: unknown, b: unknown): boolean => JSON.stringify(a) === JSON.stringify(b); + +const itemKey = (v: unknown): string => { + if (isObject(v)) { + const id = v.id ?? v.eventId ?? v.event_id ?? v.key ?? v.roomId ?? v.room_id ?? v.mxc; + if (typeof id === 'string' || typeof id === 'number') return `id:${String(id)}`; + } + return `json:${JSON.stringify(v)}`; +}; + +export type MergeResult = { value: unknown; added: number; conflicts: number }; + +/** + * Pure: merge `incoming` into `current`. Lists: add items that are missing + * (matched by id, else by content). Objects: add missing keys, recurse into + * shared ones. A differing leaf is a conflict: `preferIncoming` decides. + */ +export const mergeData = ( + current: unknown, + incoming: unknown, + preferIncoming = false, +): MergeResult => { + if (current === undefined || current === null) { + return { value: incoming, added: incoming === undefined ? 0 : 1, conflicts: 0 }; + } + if (Array.isArray(current) && Array.isArray(incoming)) { + const have = new Map(current.map((v, i) => [itemKey(v), i])); + const out = [...current]; + let added = 0; + let conflicts = 0; + incoming.forEach((item) => { + const k = itemKey(item); + const at = have.get(k); + if (at === undefined) { + out.push(item); + have.set(k, out.length - 1); + added += 1; + } else if (!same(out[at], item)) { + conflicts += 1; + if (preferIncoming) out[at] = item; + } + }); + return { value: out, added, conflicts }; + } + if (isObject(current) && isObject(incoming)) { + const out: Record = { ...current }; + let added = 0; + let conflicts = 0; + Object.entries(incoming).forEach(([k, v]) => { + if (!(k in current)) { + out[k] = v; + added += 1; + return; + } + const r = mergeData(current[k], v, preferIncoming); + out[k] = r.value; + added += r.added; + conflicts += r.conflicts; + }); + return { value: out, added, conflicts }; + } + if (same(current, incoming)) return { value: current, added: 0, conflicts: 0 }; + return { value: preferIncoming ? incoming : current, added: 0, conflicts: 1 }; +}; + +export type ImportPlanItem = { + key: string; + label: string; + added: number; + conflicts: number; +}; + +/** What importing `file` into the current data would change, per category. */ +export const planImport = ( + file: ExportFile, + current: { + accountData: Record; + decoration: string | null; + localSettings: Record | null; + }, +): ImportPlanItem[] => { + const items: ImportPlanItem[] = []; + EXPORT_ACCOUNT_DATA.forEach(({ type, label }) => { + if (!(type in file.accountData)) return; + const r = mergeData(current.accountData[type], file.accountData[type]); + items.push({ key: type, label, added: r.added, conflicts: r.conflicts }); + }); + if (file.profile.decoration !== null) { + const differs = current.decoration !== file.profile.decoration; + items.push({ + key: 'profile.decoration', + label: 'Avatar decoration', + added: !current.decoration && differs ? 1 : 0, + conflicts: current.decoration && differs ? 1 : 0, + }); + } + if (file.local.settings) { + const r = mergeData(current.localSettings ?? {}, file.local.settings); + items.push({ + key: 'local.settings', + label: "This device's settings", + added: r.added, + conflicts: r.conflicts, + }); + } + return items; +}; -- 2.47.3