feat(search): opt-in persistent index for encrypted-room search (P4-8)
Raw-IndexedDB cache (lotus-search-cache: messages keyed [roomId,eventId] + per-room coverage) merged into local search with in-memory-wins dedupe. OPT-IN (default off) via a standalone atom — stores decrypted text at rest, so it ships with a privacy note, a Clear button, and an unconditional wipe on logout (initMatrix). All IDB errors degrade to cache-miss. +8 tests (1 IDB skip in node). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,130 @@
|
||||
import { test } from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import {
|
||||
computeCoverage,
|
||||
mergeSearchResults,
|
||||
putRows,
|
||||
queryRoom,
|
||||
getCoverage,
|
||||
saveRoomIndex,
|
||||
clearRoom,
|
||||
clearAll,
|
||||
deleteSearchCacheDatabase,
|
||||
SearchCacheRow,
|
||||
} from './searchCache';
|
||||
|
||||
// --- Pure helpers: mergeSearchResults ---------------------------------------
|
||||
|
||||
type Item = { event: { event_id: string; origin_server_ts?: number } };
|
||||
const item = (eventId: string, ts?: number): Item => ({
|
||||
event: { event_id: eventId, origin_server_ts: ts },
|
||||
});
|
||||
|
||||
test('mergeSearchResults: sorts by origin_server_ts descending', () => {
|
||||
const out = mergeSearchResults([item('$a', 10), item('$b', 30), item('$c', 20)], []);
|
||||
assert.deepEqual(
|
||||
out.map((i) => i.event.event_id),
|
||||
['$b', '$c', '$a'],
|
||||
);
|
||||
});
|
||||
|
||||
test('mergeSearchResults: dedupes by event_id with in-memory winning', () => {
|
||||
const memory = [{ event: { event_id: '$dup', origin_server_ts: 5 }, tag: 'memory' }];
|
||||
const cached = [
|
||||
{ event: { event_id: '$dup', origin_server_ts: 5 }, tag: 'cached' },
|
||||
{ event: { event_id: '$only', origin_server_ts: 9 }, tag: 'cached' },
|
||||
];
|
||||
const out = mergeSearchResults(memory, cached);
|
||||
assert.equal(out.length, 2);
|
||||
const dup = out.find((i) => i.event.event_id === '$dup');
|
||||
assert.equal(dup?.tag, 'memory');
|
||||
});
|
||||
|
||||
test('mergeSearchResults: cached-only hits are included', () => {
|
||||
const out = mergeSearchResults<Item>([], [item('$c1', 1), item('$c2', 2)]);
|
||||
assert.equal(out.length, 2);
|
||||
});
|
||||
|
||||
test('mergeSearchResults: missing ts sorts as 0 (last)', () => {
|
||||
const out = mergeSearchResults([item('$noTs'), item('$withTs', 100)], []);
|
||||
assert.deepEqual(
|
||||
out.map((i) => i.event.event_id),
|
||||
['$withTs', '$noTs'],
|
||||
);
|
||||
});
|
||||
|
||||
// --- Pure helpers: computeCoverage ------------------------------------------
|
||||
|
||||
const row = (ts: number): Pick<SearchCacheRow, 'ts'> => ({ ts });
|
||||
|
||||
test('computeCoverage: derives oldest/newest from rows', () => {
|
||||
const cov = computeCoverage('!r', [row(30), row(10), row(20)], 3);
|
||||
assert.deepEqual(cov, { roomId: '!r', oldestTs: 10, newestTs: 30, count: 3 });
|
||||
});
|
||||
|
||||
test('computeCoverage: widens the window against previous coverage', () => {
|
||||
const prev = { roomId: '!r', oldestTs: 5, newestTs: 25, count: 2 };
|
||||
const cov = computeCoverage('!r', [row(15), row(40)], 4, prev);
|
||||
assert.equal(cov.oldestTs, 5); // previous oldest kept
|
||||
assert.equal(cov.newestTs, 40); // batch newest wins
|
||||
assert.equal(cov.count, 4); // authoritative count from caller
|
||||
});
|
||||
|
||||
test('computeCoverage: empty rows with no previous yields zeroed window', () => {
|
||||
const cov = computeCoverage('!r', [], 0);
|
||||
assert.deepEqual(cov, { roomId: '!r', oldestTs: 0, newestTs: 0, count: 0 });
|
||||
});
|
||||
|
||||
// --- IDB round-trip: skip when IndexedDB is unavailable (e.g. node --test) ---
|
||||
|
||||
const hasIdb = typeof indexedDB !== 'undefined';
|
||||
|
||||
test('searchCache IDB round-trip', { skip: !hasIdb }, async () => {
|
||||
await clearAll();
|
||||
const rows: SearchCacheRow[] = [
|
||||
{ roomId: '!r1', eventId: '$1', ts: 100, sender: '@a', body: 'hello world' },
|
||||
{
|
||||
roomId: '!r1',
|
||||
eventId: '$2',
|
||||
ts: 200,
|
||||
sender: '@b',
|
||||
body: 'goodbye',
|
||||
formattedBody: '<b>x</b>',
|
||||
},
|
||||
{ roomId: '!r2', eventId: '$3', ts: 300, sender: '@a', body: 'other room' },
|
||||
];
|
||||
await putRows(rows);
|
||||
|
||||
const r1 = await queryRoom('!r1');
|
||||
assert.equal(r1.length, 2);
|
||||
assert.deepEqual(r1.map((x) => x.eventId).sort(), ['$1', '$2']);
|
||||
|
||||
await saveRoomIndex(
|
||||
'!r1',
|
||||
rows.filter((x) => x.roomId === '!r1'),
|
||||
);
|
||||
const cov = await getCoverage('!r1');
|
||||
assert.equal(cov?.count, 2);
|
||||
assert.equal(cov?.oldestTs, 100);
|
||||
assert.equal(cov?.newestTs, 200);
|
||||
|
||||
await clearRoom('!r1');
|
||||
assert.equal((await queryRoom('!r1')).length, 0);
|
||||
assert.equal((await queryRoom('!r2')).length, 1);
|
||||
|
||||
await deleteSearchCacheDatabase();
|
||||
});
|
||||
|
||||
test('resilient helpers never throw when IDB is unavailable', { skip: hasIdb }, async () => {
|
||||
// In this environment IndexedDB is absent; every call must degrade to a
|
||||
// cache-miss rather than throwing.
|
||||
await assert.doesNotReject(
|
||||
putRows([{ roomId: '!r', eventId: '$1', ts: 1, sender: '@a', body: 'x' }]),
|
||||
);
|
||||
assert.deepEqual(await queryRoom('!r'), []);
|
||||
assert.equal(await getCoverage('!r'), null);
|
||||
await assert.doesNotReject(saveRoomIndex('!r', []));
|
||||
await assert.doesNotReject(clearRoom('!r'));
|
||||
await assert.doesNotReject(clearAll());
|
||||
await assert.doesNotReject(deleteSearchCacheDatabase());
|
||||
});
|
||||
@@ -0,0 +1,308 @@
|
||||
/**
|
||||
* P4-8 — persistent encrypted-search cache (raw IndexedDB, no new deps).
|
||||
*
|
||||
* The homeserver cannot search E2EE message content, so encrypted-room search
|
||||
* only ever covers what the client has paginated + decrypted this session. This
|
||||
* module persists a local plaintext index so coverage survives reloads.
|
||||
*
|
||||
* PRIVACY: this stores decrypted plaintext at rest. It is opt-in (default OFF),
|
||||
* clearable, and wiped on logout via `deleteSearchCacheDatabase()`.
|
||||
*
|
||||
* Resilience contract: every entry point swallows IndexedDB errors and behaves
|
||||
* as a cache-miss. Nothing here ever throws to the UI.
|
||||
*/
|
||||
|
||||
const DB_NAME = 'lotus-search-cache';
|
||||
const DB_VERSION = 1;
|
||||
const MESSAGES_STORE = 'messages';
|
||||
const COVERAGE_STORE = 'coverage';
|
||||
const ROOM_TS_INDEX = 'roomTs';
|
||||
|
||||
/** A single cached, decrypted message row. Keyed on `[roomId, eventId]`. */
|
||||
export type SearchCacheRow = {
|
||||
roomId: string;
|
||||
eventId: string;
|
||||
ts: number;
|
||||
sender: string;
|
||||
body: string;
|
||||
formattedBody?: string;
|
||||
pollText?: string;
|
||||
};
|
||||
|
||||
/** Per-room coverage stats for the "X / Y cached" UI counters. */
|
||||
export type SearchCacheCoverage = {
|
||||
roomId: string;
|
||||
oldestTs: number;
|
||||
newestTs: number;
|
||||
count: number;
|
||||
};
|
||||
|
||||
// A key range that matches every `[roomId, *]` entry in a composite-key store
|
||||
// or `[roomId, ts]` index: an empty array sorts after all other key types, so
|
||||
// `[roomId]` .. `[roomId, []]` brackets the whole room partition.
|
||||
const roomRange = (roomId: string): IDBKeyRange => IDBKeyRange.bound([roomId], [roomId, []]);
|
||||
|
||||
let dbPromise: Promise<IDBDatabase | null> | null = null;
|
||||
|
||||
const openDb = (): Promise<IDBDatabase | null> => {
|
||||
if (dbPromise) return dbPromise;
|
||||
dbPromise = new Promise<IDBDatabase | null>((resolve) => {
|
||||
try {
|
||||
if (typeof indexedDB === 'undefined') {
|
||||
resolve(null);
|
||||
return;
|
||||
}
|
||||
const req = indexedDB.open(DB_NAME, DB_VERSION);
|
||||
req.onupgradeneeded = () => {
|
||||
const db = req.result;
|
||||
if (!db.objectStoreNames.contains(MESSAGES_STORE)) {
|
||||
const store = db.createObjectStore(MESSAGES_STORE, {
|
||||
keyPath: ['roomId', 'eventId'],
|
||||
});
|
||||
store.createIndex(ROOM_TS_INDEX, ['roomId', 'ts']);
|
||||
}
|
||||
if (!db.objectStoreNames.contains(COVERAGE_STORE)) {
|
||||
db.createObjectStore(COVERAGE_STORE, { keyPath: 'roomId' });
|
||||
}
|
||||
};
|
||||
req.onsuccess = () => resolve(req.result);
|
||||
req.onerror = () => {
|
||||
dbPromise = null; // allow a later retry
|
||||
resolve(null);
|
||||
};
|
||||
req.onblocked = () => {
|
||||
dbPromise = null;
|
||||
resolve(null);
|
||||
};
|
||||
} catch {
|
||||
dbPromise = null;
|
||||
resolve(null);
|
||||
}
|
||||
});
|
||||
return dbPromise;
|
||||
};
|
||||
|
||||
/** Resolve once a write transaction commits (or reject/abort → caller swallows). */
|
||||
const awaitTx = (tx: IDBTransaction): Promise<void> =>
|
||||
new Promise<void>((resolve, reject) => {
|
||||
tx.oncomplete = () => resolve();
|
||||
tx.onerror = () => reject(tx.error);
|
||||
tx.onabort = () => reject(tx.error);
|
||||
});
|
||||
|
||||
/** Upsert message rows. No-op on empty input or when IDB is unavailable. */
|
||||
export const putRows = async (rows: SearchCacheRow[]): Promise<void> => {
|
||||
if (rows.length === 0) return;
|
||||
const db = await openDb();
|
||||
if (!db) return;
|
||||
try {
|
||||
const tx = db.transaction(MESSAGES_STORE, 'readwrite');
|
||||
const store = tx.objectStore(MESSAGES_STORE);
|
||||
rows.forEach((row) => store.put(row));
|
||||
await awaitTx(tx);
|
||||
} catch {
|
||||
// Cache write failures must never surface to the UI.
|
||||
}
|
||||
};
|
||||
|
||||
/** All cached rows for a room, ordered oldest→newest by the `[roomId, ts]` index. */
|
||||
export const queryRoom = async (roomId: string): Promise<SearchCacheRow[]> => {
|
||||
const db = await openDb();
|
||||
if (!db) return [];
|
||||
try {
|
||||
return await new Promise<SearchCacheRow[]>((resolve, reject) => {
|
||||
const tx = db.transaction(MESSAGES_STORE, 'readonly');
|
||||
const index = tx.objectStore(MESSAGES_STORE).index(ROOM_TS_INDEX);
|
||||
const req = index.getAll(roomRange(roomId));
|
||||
req.onsuccess = () => resolve((req.result as SearchCacheRow[]) ?? []);
|
||||
req.onerror = () => reject(req.error);
|
||||
});
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
};
|
||||
|
||||
/** Cursor variant: stream a room's rows through a matcher, collecting hits. */
|
||||
export const searchRoom = async (
|
||||
roomId: string,
|
||||
matcher: (row: SearchCacheRow) => boolean,
|
||||
): Promise<SearchCacheRow[]> => {
|
||||
const db = await openDb();
|
||||
if (!db) return [];
|
||||
try {
|
||||
return await new Promise<SearchCacheRow[]>((resolve, reject) => {
|
||||
const hits: SearchCacheRow[] = [];
|
||||
const tx = db.transaction(MESSAGES_STORE, 'readonly');
|
||||
const index = tx.objectStore(MESSAGES_STORE).index(ROOM_TS_INDEX);
|
||||
const req = index.openCursor(roomRange(roomId));
|
||||
req.onsuccess = () => {
|
||||
const cursor = req.result;
|
||||
if (!cursor) {
|
||||
resolve(hits);
|
||||
return;
|
||||
}
|
||||
const row = cursor.value as SearchCacheRow;
|
||||
if (matcher(row)) hits.push(row);
|
||||
cursor.continue();
|
||||
};
|
||||
req.onerror = () => reject(req.error);
|
||||
});
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
};
|
||||
|
||||
/** Number of cached rows for a room. */
|
||||
export const countRoom = async (roomId: string): Promise<number> => {
|
||||
const db = await openDb();
|
||||
if (!db) return 0;
|
||||
try {
|
||||
return await new Promise<number>((resolve, reject) => {
|
||||
const tx = db.transaction(MESSAGES_STORE, 'readonly');
|
||||
const index = tx.objectStore(MESSAGES_STORE).index(ROOM_TS_INDEX);
|
||||
const req = index.count(roomRange(roomId));
|
||||
req.onsuccess = () => resolve(req.result);
|
||||
req.onerror = () => reject(req.error);
|
||||
});
|
||||
} catch {
|
||||
return 0;
|
||||
}
|
||||
};
|
||||
|
||||
export const getCoverage = async (roomId: string): Promise<SearchCacheCoverage | null> => {
|
||||
const db = await openDb();
|
||||
if (!db) return null;
|
||||
try {
|
||||
return await new Promise<SearchCacheCoverage | null>((resolve, reject) => {
|
||||
const tx = db.transaction(COVERAGE_STORE, 'readonly');
|
||||
const req = tx.objectStore(COVERAGE_STORE).get(roomId);
|
||||
req.onsuccess = () => resolve((req.result as SearchCacheCoverage) ?? null);
|
||||
req.onerror = () => reject(req.error);
|
||||
});
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
};
|
||||
|
||||
export const putCoverage = async (coverage: SearchCacheCoverage): Promise<void> => {
|
||||
const db = await openDb();
|
||||
if (!db) return;
|
||||
try {
|
||||
const tx = db.transaction(COVERAGE_STORE, 'readwrite');
|
||||
tx.objectStore(COVERAGE_STORE).put(coverage);
|
||||
await awaitTx(tx);
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* Pure helper: fold a batch of rows into a coverage record, widening the
|
||||
* `oldestTs`/`newestTs` window against any previous coverage. `count` is
|
||||
* supplied by the caller (authoritative store count) so dedup across sessions
|
||||
* is handled correctly. Exported for testing without IDB.
|
||||
*/
|
||||
export const computeCoverage = (
|
||||
roomId: string,
|
||||
rows: ReadonlyArray<Pick<SearchCacheRow, 'ts'>>,
|
||||
count: number,
|
||||
previous?: SearchCacheCoverage | null,
|
||||
): SearchCacheCoverage => {
|
||||
let oldestTs = previous?.oldestTs ?? Number.POSITIVE_INFINITY;
|
||||
let newestTs = previous?.newestTs ?? Number.NEGATIVE_INFINITY;
|
||||
rows.forEach((row) => {
|
||||
if (row.ts < oldestTs) oldestTs = row.ts;
|
||||
if (row.ts > newestTs) newestTs = row.ts;
|
||||
});
|
||||
if (!Number.isFinite(oldestTs)) oldestTs = 0;
|
||||
if (!Number.isFinite(newestTs)) newestTs = 0;
|
||||
return { roomId, oldestTs, newestTs, count };
|
||||
};
|
||||
|
||||
/**
|
||||
* Convenience persist path used by the search hook: upsert a batch of rows for
|
||||
* a room, then recompute + store the room's coverage from the authoritative
|
||||
* store count. Fire-and-forget; never throws.
|
||||
*/
|
||||
export const saveRoomIndex = async (roomId: string, rows: SearchCacheRow[]): Promise<void> => {
|
||||
if (rows.length === 0) return;
|
||||
await putRows(rows);
|
||||
const [count, previous] = await Promise.all([countRoom(roomId), getCoverage(roomId)]);
|
||||
await putCoverage(computeCoverage(roomId, rows, count, previous));
|
||||
};
|
||||
|
||||
/**
|
||||
* Pure helper: merge in-memory result items with cache-derived result items,
|
||||
* deduping by `event.event_id` (in-memory wins), sorted by `origin_server_ts`
|
||||
* descending. Generic over the minimal shape it reads so it is fully testable
|
||||
* without matrix-js-sdk types. Exported for testing.
|
||||
*/
|
||||
export const mergeSearchResults = <
|
||||
T extends { event: { event_id: string; origin_server_ts?: number } },
|
||||
>(
|
||||
memory: ReadonlyArray<T>,
|
||||
cached: ReadonlyArray<T>,
|
||||
): T[] => {
|
||||
const byId = new Map<string, T>();
|
||||
// Seed with cached, then let in-memory overwrite so in-memory always wins.
|
||||
cached.forEach((item) => byId.set(item.event.event_id, item));
|
||||
memory.forEach((item) => byId.set(item.event.event_id, item));
|
||||
return Array.from(byId.values()).sort(
|
||||
(a, b) => (b.event.origin_server_ts ?? 0) - (a.event.origin_server_ts ?? 0),
|
||||
);
|
||||
};
|
||||
|
||||
export const clearRoom = async (roomId: string): Promise<void> => {
|
||||
const db = await openDb();
|
||||
if (!db) return;
|
||||
try {
|
||||
const tx = db.transaction([MESSAGES_STORE, COVERAGE_STORE], 'readwrite');
|
||||
tx.objectStore(MESSAGES_STORE).delete(roomRange(roomId));
|
||||
tx.objectStore(COVERAGE_STORE).delete(roomId);
|
||||
await awaitTx(tx);
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
};
|
||||
|
||||
export const clearAll = async (): Promise<void> => {
|
||||
const db = await openDb();
|
||||
if (!db) return;
|
||||
try {
|
||||
const tx = db.transaction([MESSAGES_STORE, COVERAGE_STORE], 'readwrite');
|
||||
tx.objectStore(MESSAGES_STORE).clear();
|
||||
tx.objectStore(COVERAGE_STORE).clear();
|
||||
await awaitTx(tx);
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* Drop the entire on-disk database. Wired into the logout path by the
|
||||
* coordinator (initMatrix) so no decrypted plaintext lingers after sign-out.
|
||||
* Closes any open handle first so the delete is not blocked. Never throws.
|
||||
*/
|
||||
export const deleteSearchCacheDatabase = async (): Promise<void> => {
|
||||
try {
|
||||
const existing = dbPromise ? await dbPromise : null;
|
||||
if (existing) existing.close();
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
dbPromise = null;
|
||||
return new Promise<void>((resolve) => {
|
||||
try {
|
||||
if (typeof indexedDB === 'undefined') {
|
||||
resolve();
|
||||
return;
|
||||
}
|
||||
const req = indexedDB.deleteDatabase(DB_NAME);
|
||||
req.onsuccess = () => resolve();
|
||||
req.onerror = () => resolve();
|
||||
req.onblocked = () => resolve();
|
||||
} catch {
|
||||
resolve();
|
||||
}
|
||||
});
|
||||
};
|
||||
Reference in New Issue
Block a user