feat(settings): export / import your data (#114) #454

Merged
jared merged 1 commits from feat/data-export into lotus 2026-10-25 02:16:05 -04:00
5 changed files with 680 additions and 0 deletions
+98
View File
@@ -0,0 +1,98 @@
import fs from 'node:fs';
import { test, expect } from '@playwright/test';
import { api, ensureUser, hsReachable, loginUI, uniq } from './localHs';
// [Gitea #114] Export / import "your data": a round trip through the file, the
// merge summary, and the keep-mine / use-the-file's choice.
const enc = encodeURIComponent;
test.describe('your data (#114)', () => {
test.beforeAll(async () => {
test.skip(!(await hsReachable()), 'local homeserver not running');
});
test('export, change things, import merges with the chosen conflict rule', async ({ page }) => {
test.setTimeout(150_000);
const user = await ensureUser(uniq('yd'));
const put = (type: string, content: unknown) =>
api(
'PUT',
`/_matrix/client/v3/user/${enc(user.userId)}/account_data/${type}`,
user.token,
content,
);
const get = (type: string) =>
api<Record<string, unknown>>(
'GET',
`/_matrix/client/v3/user/${enc(user.userId)}/account_data/${type}`,
user.token,
);
const bm = (eventId: string) => ({ roomId: '!r:localhost', eventId, savedAt: 1 });
await put('io.lotus.bookmarks', { bookmarks: [bm('$one'), bm('$two')] });
await put('io.lotus.user_notes', { '@friend:localhost': 'met at the LAN party' });
await loginUI(page, user);
await page
.getByText('Connecting...')
.first()
.waitFor({ state: 'detached', timeout: 90_000 })
.catch(() => undefined);
const openGeneral = async () => {
await page.getByRole('button', { name: 'User Settings' }).click();
await page.getByRole('dialog').getByText('General', { exact: true }).first().click();
await page
.getByRole('dialog')
.getByText('Your data', { exact: true })
.scrollIntoViewIfNeeded();
};
await openGeneral();
// Export.
const [download] = await Promise.all([
page.waitForEvent('download'),
page.getByRole('dialog').getByRole('button', { name: 'Export', exact: true }).click(),
]);
const path = await download.path();
const file = JSON.parse(fs.readFileSync(path!, 'utf8'));
expect(file.format).toBe('lotus-chat-data');
expect(file.accountData['io.lotus.bookmarks'].bookmarks).toHaveLength(2);
expect(file.local.settings).toBeTruthy();
// Lose a bookmark, change the note.
await put('io.lotus.bookmarks', { bookmarks: [bm('$one')] });
await put('io.lotus.user_notes', { '@friend:localhost': 'changed my mind' });
await page.waitForTimeout(3000); // let sync deliver the new account data
const importFile = async () => {
await page
.locator('input[type="file"][aria-label="Lotus Chat data file"]')
.setInputFiles(path!);
await expect(page.getByRole('region', { name: 'Import summary' })).toBeVisible();
};
await importFile();
const summary = page.getByRole('region', { name: 'Import summary' });
await expect(summary).toContainText('Bookmarks: 1 new');
await expect(summary).toContainText('Private notes about people: 1 differ');
// Keep mine: the bookmark comes back, the note stays as it is now.
await summary.getByRole('button', { name: 'Import', exact: true }).click();
await expect(page.getByText('Imported.')).toBeVisible({ timeout: 20_000 });
expect(((await get('io.lotus.bookmarks')).bookmarks as unknown[]).length).toBe(2);
expect((await get('io.lotus.user_notes'))['@friend:localhost']).toBe('changed my mind');
// Again, using the file's: the note goes back.
await page.waitForTimeout(2000);
await importFile();
await summary.getByText("Use the file's").click();
await summary.getByRole('button', { name: 'Import', exact: true }).click();
await expect(page.getByText('Imported.')).toBeVisible({ timeout: 20_000 });
expect((await get('io.lotus.user_notes'))['@friend:localhost']).toBe('met at the LAN party');
// Wrong files are refused with a plain message.
fs.writeFileSync(`${path}.bad`, '{"format":"something-else"}');
await page
.locator('input[type="file"][aria-label="Lotus Chat data file"]')
.setInputFiles(`${path}.bad`);
await expect(page.getByText("This isn't a Lotus Chat data file.")).toBeVisible();
});
});
@@ -61,6 +61,7 @@ import {
} from '../../../utils/lotusDenoiseUtils';
import { useSetting } from '../../../state/hooks/settings';
import { useSettingsSyncActions } from '../../../hooks/useSettingsSync';
import { YourDataSection } from './YourData';
import {
CallAudioBitrate,
ChatBackground,
@@ -3007,6 +3008,7 @@ export function General({ requestClose }: GeneralProps) {
<Privacy />
<StorageUsage />
<SettingsSyncSection />
<YourDataSection />
<Calls />
<DesktopSettings />
<AppUpdates />
@@ -0,0 +1,285 @@
import React, { useCallback, useRef, useState } from 'react';
import { useAtom } from 'jotai';
import { Method } from 'matrix-js-sdk';
import { Box, Button, Text, color } from 'folds';
import { SequenceCard } from '../../../components/sequence-card';
import { SequenceCardStyle } from '../styles.css';
import { SettingTile } from '../../../components/setting-tile';
import { useMatrixClient } from '../../../hooks/useMatrixClient';
import { Settings, settingsAtom } from '../../../state/settings';
import {
buildExport,
EXPORT_ACCOUNT_DATA,
ExportFile,
ImportPlanItem,
mergeData,
parseExport,
planImport,
} from '../../../utils/dataExport';
import { invalidateDecorationCache } from '../../../hooks/useAvatarDecoration';
const DECORATION_FIELD = 'io.lotus.avatar_decoration';
type Current = {
accountData: Record<string, unknown>;
decoration: string | null;
localSettings: Record<string, unknown> | null;
};
/**
* [Gitea #114] "Your data": export the account's Lotus data (settings,
* bookmarks, notes, sections, ...) to a file and import it here or on another
* account. Import merges: new things are added; for things that differ the
* user picks "keep mine" (default) or "use the file's". Message history and
* encryption keys have their own exports.
*/
export function YourDataSection() {
const mx = useMatrixClient();
const userId = mx.getSafeUserId();
const [settings, setSettings] = useAtom(settingsAtom);
const fileInput = useRef<HTMLInputElement>(null);
const [pending, setPending] = useState<{ file: ExportFile; plan: ImportPlanItem[] }>();
const [preferFile, setPreferFile] = useState(false);
const [message, setMessage] = useState<{ tone: 'ok' | 'error'; text: string }>();
const [busy, setBusy] = useState(false);
const readCurrent = useCallback(async (): Promise<Current> => {
const accountData: Record<string, unknown> = {};
EXPORT_ACCOUNT_DATA.forEach(({ type }) => {
const c = mx.getAccountData(type as never)?.getContent();
if (c && Object.keys(c).length > 0) accountData[type] = c;
});
let decoration: string | null = null;
try {
const profile = await mx.http.authedRequest<Record<string, unknown>>(
Method.Get,
`/profile/${encodeURIComponent(userId)}`,
);
const v = profile[DECORATION_FIELD];
decoration = typeof v === 'string' && v ? v : null;
} catch {
/* no decoration */
}
return {
accountData,
decoration,
localSettings: settings as unknown as Record<string, unknown>,
};
}, [mx, userId, settings]);
const exportData = async () => {
setBusy(true);
setMessage(undefined);
try {
const cur = await readCurrent();
const file = buildExport({ userId, ...cur });
const blob = new Blob([JSON.stringify(file, null, 2)], { type: 'application/json' });
const url = URL.createObjectURL(blob);
const a = document.createElement('a');
a.href = url;
a.download = `lotus-chat-data-${userId.replace(/[^a-z0-9]+/gi, '-')}-${file.exportedAt.slice(0, 10)}.json`;
a.click();
URL.revokeObjectURL(url);
setMessage({ tone: 'ok', text: 'Exported. Keep the file somewhere safe.' });
} catch {
setMessage({ tone: 'error', text: "Couldn't export your data. Please try again." });
} finally {
setBusy(false);
}
};
const chooseFile = async (f: File | undefined) => {
if (!f) return;
setMessage(undefined);
const parsed = parseExport(await f.text());
if (!parsed.ok) {
setMessage({ tone: 'error', text: parsed.error });
return;
}
const plan = planImport(parsed.file, await readCurrent());
setPreferFile(false);
setPending({ file: parsed.file, plan });
};
const applyImport = async () => {
if (!pending) return;
setBusy(true);
try {
const cur = await readCurrent();
const { file } = pending;
await Object.entries(file.accountData).reduce<Promise<void>>(
async (prev, [type, incoming]) => {
await prev;
const r = mergeData(cur.accountData[type], incoming, preferFile);
if (r.added > 0 || (preferFile && r.conflicts > 0)) {
await mx.setAccountData(type as never, r.value as never);
}
},
Promise.resolve(),
);
const deco = file.profile.decoration;
if (deco && deco !== cur.decoration && (!cur.decoration || preferFile)) {
await mx.http.authedRequest(
Method.Put,
`/profile/${encodeURIComponent(userId)}/${DECORATION_FIELD}`,
undefined,
{ [DECORATION_FIELD]: deco },
);
invalidateDecorationCache(userId);
}
if (file.local.settings) {
const r = mergeData(cur.localSettings ?? {}, file.local.settings, preferFile);
setSettings({ ...settings, ...(r.value as Partial<Settings>) });
}
setPending(undefined);
setMessage({ tone: 'ok', text: 'Imported.' });
} catch {
setMessage({ tone: 'error', text: "Couldn't finish the import. Nothing else was changed." });
} finally {
setBusy(false);
}
};
const totals = pending?.plan.reduce(
(t, p) => ({ added: t.added + p.added, conflicts: t.conflicts + p.conflicts }),
{ added: 0, conflicts: 0 },
);
return (
<Box direction="Column" gap="100">
<Text size="L400">Your data</Text>
<SequenceCard
className={SequenceCardStyle}
variant="SurfaceVariant"
direction="Column"
gap="400"
>
<SettingTile
title="Export your data"
description="Settings, bookmarks, private notes, room sections and names, reminders, status presets, soundboard and emoji, as one file. Not your messages or encryption keys (they have their own exports)."
after={
<Button
size="300"
variant="Secondary"
fill="Soft"
radii="300"
outlined
disabled={busy}
onClick={exportData}
>
<Text size="B300">Export</Text>
</Button>
}
/>
<SettingTile
title="Import data"
description="Load a Lotus Chat data file into this account. New items are added; you choose what happens to things that differ."
after={
<Button
size="300"
variant="Secondary"
fill="Soft"
radii="300"
outlined
disabled={busy}
onClick={() => fileInput.current?.click()}
>
<Text size="B300">Import…</Text>
</Button>
}
/>
<input
ref={fileInput}
type="file"
accept="application/json,.json"
hidden
aria-label="Lotus Chat data file"
onChange={(e) => {
chooseFile(e.target.files?.[0]).catch(() => undefined);
e.target.value = '';
}}
/>
{pending && totals && (
<Box direction="Column" gap="200" role="region" aria-label="Import summary">
{pending.file.userId && pending.file.userId !== userId && (
<Text size="T200">
This file is from <b>{pending.file.userId}</b>.
</Text>
)}
{pending.plan.map((p) => (
<Text key={p.key} size="T200">
{p.label}:{' '}
{p.added === 0 && p.conflicts === 0
? 'already up to date'
: [p.added > 0 && `${p.added} new`, p.conflicts > 0 && `${p.conflicts} differ`]
.filter(Boolean)
.join(', ')}
</Text>
))}
{totals.conflicts > 0 && (
<Box direction="Column" gap="100" role="radiogroup" aria-label="Things that differ">
<Text size="T200">
<b>{totals.conflicts}</b>{' '}
{totals.conflicts === 1 ? 'thing differs' : 'things differ'} from what&apos;s
here:
</Text>
<label>
<input
type="radio"
name="your-data-conflicts"
checked={!preferFile}
onChange={() => setPreferFile(false)}
/>{' '}
<Text as="span" size="T200">
Keep mine
</Text>
</label>
<label>
<input
type="radio"
name="your-data-conflicts"
checked={preferFile}
onChange={() => setPreferFile(true)}
/>{' '}
<Text as="span" size="T200">
Use the file&apos;s
</Text>
</label>
</Box>
)}
<Box gap="200">
<Button
size="300"
variant="Primary"
radii="300"
disabled={busy || (totals.added === 0 && !(preferFile && totals.conflicts > 0))}
onClick={applyImport}
>
<Text size="B300">Import</Text>
</Button>
<Button
size="300"
variant="Secondary"
fill="Soft"
radii="300"
disabled={busy}
onClick={() => setPending(undefined)}
>
<Text size="B300">Cancel</Text>
</Button>
</Box>
</Box>
)}
{message && (
<Text
size="T200"
role="status"
style={{ color: message.tone === 'error' ? color.Critical.Main : undefined }}
>
{message.text}
</Text>
)}
</SequenceCard>
</Box>
);
}
+76
View File
@@ -0,0 +1,76 @@
import { test } from 'node:test';
import assert from 'node:assert/strict';
import { buildExport, mergeData, parseExport, planImport } from './dataExport';
const file = buildExport({
userId: '@a:hs',
accountData: {
'io.lotus.bookmarks': { bookmarks: [{ id: 'b1', note: 'x' }] },
'io.lotus.user_notes': { notes: { '@b:hs': 'met at LAN party' } },
'm.direct': { secret: true },
'io.lotus.settings': { theme: 'dark', fontSize: 16 },
},
decoration: 'halo',
localSettings: { callNoiseSuppression: 'ml' },
now: new Date('2026-10-25T00:00:00Z'),
});
test('[#114] export keeps only our own types', () => {
assert.equal(file.format, 'lotus-chat-data');
assert.equal(file.version, 1);
assert.deepEqual(Object.keys(file.accountData).sort(), [
'io.lotus.bookmarks',
'io.lotus.settings',
'io.lotus.user_notes',
]);
assert.equal(file.profile.decoration, 'halo');
});
test('[#114] parse: round trip, wrong files, newer versions, foreign types', () => {
const r = parseExport(JSON.stringify(file));
assert.ok(r.ok);
assert.equal(parseExport('nope').ok, false);
assert.equal(parseExport('{"format":"other"}').ok, false);
const newer = parseExport(JSON.stringify({ ...file, version: 2 }));
assert.ok(!newer.ok && /newer Lotus Chat/.test(newer.error));
const sneaky = parseExport(
JSON.stringify({ ...file, accountData: { 'm.push_rules': {}, 'io.lotus.bookmarks': {} } }),
);
assert.ok(sneaky.ok);
assert.deepEqual(Object.keys(sneaky.file.accountData), ['io.lotus.bookmarks']);
});
test('[#114] merge: lists by id, maps by key, conflicts kept unless preferred', () => {
const mine = { bookmarks: [{ id: 'b1', note: 'mine' }, { id: 'b2' }] };
const theirs = { bookmarks: [{ id: 'b1', note: 'theirs' }, { id: 'b3' }] };
const keep = mergeData(mine, theirs);
assert.equal(keep.added, 1);
assert.equal(keep.conflicts, 1);
assert.deepEqual(keep.value, {
bookmarks: [{ id: 'b1', note: 'mine' }, { id: 'b2' }, { id: 'b3' }],
});
const take = mergeData(mine, theirs, true);
assert.deepEqual((take.value as typeof mine).bookmarks[0], { id: 'b1', note: 'theirs' });
const notes = mergeData({ a: 1, b: 2 }, { b: 3, c: 4 });
assert.deepEqual(notes, { value: { a: 1, b: 2, c: 4 }, added: 1, conflicts: 1 });
assert.deepEqual(mergeData(undefined, { x: 1 }), { value: { x: 1 }, added: 1, conflicts: 0 });
assert.deepEqual(mergeData(['a'], ['a', 'b']).value, ['a', 'b']);
});
test('[#114] plan summarises per category', () => {
const plan = planImport(file, {
accountData: { 'io.lotus.settings': { theme: 'light' } },
decoration: null,
localSettings: { callNoiseSuppression: 'ml' },
});
const by = Object.fromEntries(plan.map((p) => [p.key, p]));
assert.deepEqual(
[by['io.lotus.settings'].added, by['io.lotus.settings'].conflicts],
[1, 1],
'fontSize new, theme differs',
);
assert.equal(by['io.lotus.bookmarks'].added, 1);
assert.equal(by['profile.decoration'].added, 1);
assert.deepEqual([by['local.settings'].added, by['local.settings'].conflicts], [0, 0]);
});
+219
View File
@@ -0,0 +1,219 @@
/**
* [Gitea #114] Export / import "your data": the Lotus-side personal data of
* an account (not message history, not encryption keys: those have their own
* exports, and keys are secret). One JSON file:
*
* { format: 'lotus-chat-data', version: 1, exportedAt, userId,
* accountData: { [type]: content }, profile: { decoration }, local: { settings } }
*
* Import MERGES: anything missing here is added; a value that differs is a
* conflict, kept as it is unless the user chooses the file's. Pure helpers.
*/
export const EXPORT_FORMAT = 'lotus-chat-data';
export const EXPORT_VERSION = 1;
/** The account-data types that are the user's own Lotus data, with labels. */
export const EXPORT_ACCOUNT_DATA: { type: string; label: string }[] = [
{ type: 'io.lotus.settings', label: 'Synced settings' },
{ type: 'io.lotus.bookmarks', label: 'Bookmarks' },
{ type: 'io.lotus.user_notes', label: 'Private notes about people' },
{ type: 'io.lotus.room_names', label: 'Your names for rooms' },
{ type: 'in.cinny.spaces', label: 'Room sections and sidebar' },
{ type: 'io.lotus.reminders', label: 'Reminders' },
{ type: 'io.lotus.status_presets', label: 'Status presets' },
{ type: 'io.lotus.thread_notifications', label: 'Thread notification settings' },
{ type: 'io.lotus.mute_timers', label: 'Timed mutes' },
{ type: 'io.lotus.soundboard', label: 'Soundboard (clips stay on the server)' },
{ type: 'io.lotus.soundboard_rooms', label: 'Soundboards enabled everywhere' },
{ type: 'im.ponies.user_emotes', label: 'Your emoji and stickers' },
{ type: 'im.ponies.emote_rooms', label: 'Emoji packs enabled everywhere' },
];
export type ExportFile = {
format: typeof EXPORT_FORMAT;
version: number;
exportedAt: string;
userId: string;
accountData: Record<string, unknown>;
profile: { decoration: string | null };
local: { settings: Record<string, unknown> | null };
};
export const buildExport = (input: {
userId: string;
accountData: Record<string, unknown>;
decoration: string | null;
localSettings: Record<string, unknown> | null;
now?: Date;
}): ExportFile => {
const accountData: Record<string, unknown> = {};
EXPORT_ACCOUNT_DATA.forEach(({ type }) => {
const c = input.accountData[type];
if (c !== undefined && c !== null) accountData[type] = c;
});
return {
format: EXPORT_FORMAT,
version: EXPORT_VERSION,
exportedAt: (input.now ?? new Date()).toISOString(),
userId: input.userId,
accountData,
profile: { decoration: input.decoration },
local: { settings: input.localSettings },
};
};
const isObject = (v: unknown): v is Record<string, unknown> =>
!!v && typeof v === 'object' && !Array.isArray(v);
export type ParseResult = { ok: true; file: ExportFile } | { ok: false; error: string };
/** Validate an imported file; older versions would be migrated here. */
export const parseExport = (text: string): ParseResult => {
let v: unknown;
try {
v = JSON.parse(text);
} catch {
return { ok: false, error: "This isn't a Lotus Chat data file (it isn't valid JSON)." };
}
if (!isObject(v) || v.format !== EXPORT_FORMAT)
return { ok: false, error: "This isn't a Lotus Chat data file." };
if (typeof v.version !== 'number' || v.version < 1)
return { ok: false, error: 'This data file is damaged (no version).' };
if (v.version > EXPORT_VERSION)
return {
ok: false,
error: 'This file was made by a newer Lotus Chat. Update the app, then import it again.',
};
const accountData = isObject(v.accountData) ? v.accountData : {};
const known = new Set(EXPORT_ACCOUNT_DATA.map((t) => t.type));
const filtered: Record<string, unknown> = {};
Object.entries(accountData).forEach(([type, content]) => {
// Only our own types: a file can't be used to write arbitrary account data.
if (known.has(type) && content !== null && content !== undefined) filtered[type] = content;
});
const profile = isObject(v.profile) ? v.profile : {};
const local = isObject(v.local) ? v.local : {};
return {
ok: true,
file: {
format: EXPORT_FORMAT,
version: v.version,
exportedAt: typeof v.exportedAt === 'string' ? v.exportedAt : '',
userId: typeof v.userId === 'string' ? v.userId : '',
accountData: filtered,
profile: {
decoration: typeof profile.decoration === 'string' ? profile.decoration : null,
},
local: { settings: isObject(local.settings) ? local.settings : null },
},
};
};
const same = (a: unknown, b: unknown): boolean => JSON.stringify(a) === JSON.stringify(b);
const itemKey = (v: unknown): string => {
if (isObject(v)) {
const id = v.id ?? v.eventId ?? v.event_id ?? v.key ?? v.roomId ?? v.room_id ?? v.mxc;
if (typeof id === 'string' || typeof id === 'number') return `id:${String(id)}`;
}
return `json:${JSON.stringify(v)}`;
};
export type MergeResult = { value: unknown; added: number; conflicts: number };
/**
* Pure: merge `incoming` into `current`. Lists: add items that are missing
* (matched by id, else by content). Objects: add missing keys, recurse into
* shared ones. A differing leaf is a conflict: `preferIncoming` decides.
*/
export const mergeData = (
current: unknown,
incoming: unknown,
preferIncoming = false,
): MergeResult => {
if (current === undefined || current === null) {
return { value: incoming, added: incoming === undefined ? 0 : 1, conflicts: 0 };
}
if (Array.isArray(current) && Array.isArray(incoming)) {
const have = new Map(current.map((v, i) => [itemKey(v), i]));
const out = [...current];
let added = 0;
let conflicts = 0;
incoming.forEach((item) => {
const k = itemKey(item);
const at = have.get(k);
if (at === undefined) {
out.push(item);
have.set(k, out.length - 1);
added += 1;
} else if (!same(out[at], item)) {
conflicts += 1;
if (preferIncoming) out[at] = item;
}
});
return { value: out, added, conflicts };
}
if (isObject(current) && isObject(incoming)) {
const out: Record<string, unknown> = { ...current };
let added = 0;
let conflicts = 0;
Object.entries(incoming).forEach(([k, v]) => {
if (!(k in current)) {
out[k] = v;
added += 1;
return;
}
const r = mergeData(current[k], v, preferIncoming);
out[k] = r.value;
added += r.added;
conflicts += r.conflicts;
});
return { value: out, added, conflicts };
}
if (same(current, incoming)) return { value: current, added: 0, conflicts: 0 };
return { value: preferIncoming ? incoming : current, added: 0, conflicts: 1 };
};
export type ImportPlanItem = {
key: string;
label: string;
added: number;
conflicts: number;
};
/** What importing `file` into the current data would change, per category. */
export const planImport = (
file: ExportFile,
current: {
accountData: Record<string, unknown>;
decoration: string | null;
localSettings: Record<string, unknown> | null;
},
): ImportPlanItem[] => {
const items: ImportPlanItem[] = [];
EXPORT_ACCOUNT_DATA.forEach(({ type, label }) => {
if (!(type in file.accountData)) return;
const r = mergeData(current.accountData[type], file.accountData[type]);
items.push({ key: type, label, added: r.added, conflicts: r.conflicts });
});
if (file.profile.decoration !== null) {
const differs = current.decoration !== file.profile.decoration;
items.push({
key: 'profile.decoration',
label: 'Avatar decoration',
added: !current.decoration && differs ? 1 : 0,
conflicts: current.decoration && differs ? 1 : 0,
});
}
if (file.local.settings) {
const r = mergeData(current.localSettings ?? {}, file.local.settings);
items.push({
key: 'local.settings',
label: "This device's settings",
added: r.added,
conflicts: r.conflicts,
});
}
return items;
};