LotusGuild/cinny
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(search): opt-in persistent index for encrypted-room search (P4-8)

Browse Source 
Raw-IndexedDB cache (lotus-search-cache: messages keyed [roomId,eventId] +
per-room coverage) merged into local search with in-memory-wins dedupe. OPT-IN
(default off) via a standalone atom — stores decrypted text at rest, so it ships
with a privacy note, a Clear button, and an unconditional wipe on logout
(initMatrix). All IDB errors degrade to cache-miss. +8 tests (1 IDB skip in node).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Jared Vititoe
2026-07-01 21:19:02 -04:00
parent ed51c39fe7
commit 7da960ac8c
6 changed files with 700 additions and 78 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/app/features/message-search/MessageSearch.tsx
+65 -7
View File
@@ -11,6 +11,8 @@ import {
Line,
toRem,
Button,
Switch,
Chip,
} from 'folds';
import { useAtom, useAtomValue } from 'jotai';
import { useVirtualizer } from '@tanstack/react-virtual';
@@ -41,7 +43,9 @@ import {
ResultGroup,
useMessageSearch,
} from './useMessageSearch';
import { useLocalMessageSearch } from './useLocalMessageSearch';
import { LocalSearchResult, useLocalMessageSearch } from './useLocalMessageSearch';
import { searchCacheEnabledAtom } from '../../state/searchCacheEnabled';
import { clearAll as clearSearchCache } from '../../utils/searchCache';
import { addRecentSearch, recentSearchesAtom } from '../../state/recentSearches';
import { SearchResultGroup } from './SearchResultGroup';
import { SearchInput } from './SearchInput';
@@ -240,6 +244,10 @@ export function MessageSearch({
// Bump this whenever more messages are loaded so localResult re-computes
const [cacheVersion, setCacheVersion] = useState(0);
const handleCacheLoaded = useCallback(() => setCacheVersion((v) => v + 1), []);
// Explicit wipe of the persistent on-disk index, then re-run the merge.
const handleClearSearchCache = useCallback(() => {
clearSearchCache().then(() => setCacheVersion((v) => v + 1));
}, []);
// The rooms actually in scope for this search (mirrors server-side logic)
const localSearchRooms = useMemo(
@@ -253,24 +261,43 @@ export function MessageSearch({
const hasActiveSearch = msgSearchParams.term !== undefined || !!msgSearchParams.senders?.length;
const senderOnlyMode = !msgSearchParams.term && !!msgSearchParams.senders?.length;
// Run synchronous client-side search immediately.
// Run the client-side search whenever inputs change.
// In text-search mode: covers encrypted rooms only (server handles plaintext).
// In sender-only mode: covers all rooms (server has no sender-only search).
// cacheVersion in deps so it re-runs after "Load more" paginates new events.
const localResult = useMemo(() => {
if (!hasActiveSearch) return null;
return searchLocalMessages({
// The scan is async because — when the persistent cache is enabled — it also
// reads cached rows from IndexedDB and merges them with the in-memory hits.
// cacheVersion in deps so it re-runs after "Load more" paginates new events;
// searchCacheEnabled so toggling the cache re-runs the merge.
const [searchCacheEnabled, setSearchCacheEnabled] = useAtom(searchCacheEnabledAtom);
const [localResult, setLocalResult] = useState<LocalSearchResult | null>(null);
useEffect(() => {
if (!hasActiveSearch) {
setLocalResult(null);
return undefined;
}
let cancelled = false;
searchLocalMessages({
term: msgSearchParams.term ?? '',
roomIds: localSearchRooms,
senders: msgSearchParams.senders,
fromTs: msgSearchParams.fromTs,
toTs: msgSearchParams.toTs,
}).then((result) => {
if (!cancelled) setLocalResult(result);
});
// eslint-disable-next-line react-hooks/exhaustive-deps
return () => {
cancelled = true;
};
}, [
searchLocalMessages,
localSearchRooms,
msgSearchParams.term,
msgSearchParams.senders,
msgSearchParams.fromTs,
msgSearchParams.toTs,
hasActiveSearch,
cacheVersion,
searchCacheEnabled,
]);
const { status, data, error, fetchNextPage, hasNextPage, isFetchingNextPage } = useInfiniteQuery({
@@ -668,6 +695,37 @@ export function MessageSearch({
? `Showing locally cached messages from ${localResult.searchedRoomsCount} encrypted room${localResult.searchedRoomsCount !== 1 ? 's' : ''}. Load more history below to extend coverage.`
: `No matches in your local cache. Load messages below to search further back.`}
</Text>
<Box
alignItems="Center"
gap="200"
style={{
padding: config.space.S200,
background: color.SurfaceVariant.Container,
borderRadius: config.radii.R300,
}}
>
<Switch
variant="Primary"
value={searchCacheEnabled}
onChange={setSearchCacheEnabled}
/>
<Box grow="Yes" direction="Column" style={{ minWidth: 0 }}>
<Text size="T300">Persist search index on this device</Text>
<Text size="T200" priority="300">
Stores decrypted text on this device
</Text>
</Box>
{searchCacheEnabled && (
<Chip
variant="Secondary"
radii="Pill"
onClick={handleClearSearchCache}
before={<Icon size="100" src={Icons.Delete} />}
>
<Text size="T200">Clear cached index</Text>
</Chip>
)}
</Box>
<Line size="300" variant="Surface" />
</Box>
{localGroups.length > 0 && (
src/app/features/message-search/useLocalMessageSearch.ts
+169 -71
View File
@@ -1,12 +1,23 @@
import { EventType } from 'matrix-js-sdk';
import { EventType, MatrixEvent } from 'matrix-js-sdk';
import { useCallback } from 'react';
import { useAtomValue } from 'jotai';
import { useMatrixClient } from '../../hooks/useMatrixClient';
import { ResultGroup, ResultItem } from './useMessageSearch';
import { searchCacheEnabledAtom } from '../../state/searchCacheEnabled';
import {
mergeSearchResults,
queryRoom,
saveRoomIndex,
SearchCacheRow,
} from '../../utils/searchCache';
export type LocalSearchParams = {
term: string;
roomIds: string[];
senders?: string[];
/** Optional date-range filter (ms). Applied to both memory and cached rows. */
fromTs?: number;
toTs?: number;
};
export type LocalSearchResult = {
@@ -17,19 +28,110 @@ export type LocalSearchResult = {
searchedRoomsCount: number;
};
/** Extracted, searchable plaintext for a single message event. */
type ExtractedText = {
body: string;
formattedBody: string;
pollText: string;
};
const POLL_START_TYPES = ['m.poll.start', 'org.matrix.msc3381.poll.start'];
/**
* Pull the text we index/search from a decrypted event's content. Returns
* `null` for events that carry no searchable text (e.g. stickers).
*/
const extractText = (event: MatrixEvent): ExtractedText | null => {
const evType = event.getType();
const content = event.getContent();
if (POLL_START_TYPES.includes(evType)) {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const poll = (content['m.poll'] ?? content['org.matrix.msc3381.poll.start']) as any;
if (!poll) return null;
const qBody =
(poll.question?.['m.text'] as Array<{ body: string }> | undefined)?.[0]?.body ??
(poll.question?.body as string | undefined) ??
'';
const answerBodies = ((poll.answers ?? []) as Array<Record<string, unknown>>)
.map(
(a) =>
((a['m.text'] as Array<{ body: string }> | undefined)?.[0]?.body ??
// eslint-disable-next-line @typescript-eslint/no-explicit-any
(a['org.matrix.msc3381.poll.answer'] as any)?.body ??
'') as string,
)
.join(' ');
const pollText = `${qBody} ${answerBodies}`.trim();
return pollText ? { body: '', formattedBody: '', pollText } : null;
}
if (evType !== EventType.RoomMessage) return null;
const body = (content.body as string | undefined) ?? '';
const formattedBody = (content.formatted_body as string | undefined) ?? '';
if (!body && !formattedBody) return null;
return { body, formattedBody, pollText: '' };
};
/** Does the extracted text contain the (already-lowercased) term? */
const matchesTerm = (text: ExtractedText, termLower: string): boolean =>
text.body.toLowerCase().includes(termLower) ||
text.formattedBody.toLowerCase().includes(termLower) ||
text.pollText.toLowerCase().includes(termLower);
const rowMatchesTerm = (row: SearchCacheRow, termLower: string): boolean =>
row.body.toLowerCase().includes(termLower) ||
(row.formattedBody ?? '').toLowerCase().includes(termLower) ||
(row.pollText ?? '').toLowerCase().includes(termLower);
/** Build the synthetic result item a cached row renders as (text message). */
const rowToResultItem = (row: SearchCacheRow): ResultItem => {
const bodyText = row.body || row.pollText || '';
const content: Record<string, unknown> = { msgtype: 'm.text', body: bodyText };
if (row.formattedBody) {
content.format = 'org.matrix.custom.html';
content.formatted_body = row.formattedBody;
}
const syntheticEvent = {
room_id: row.roomId,
event_id: row.eventId,
type: EventType.RoomMessage,
sender: row.sender,
origin_server_ts: row.ts,
content,
unsigned: {},
};
return {
rank: 0,
// eslint-disable-next-line @typescript-eslint/no-explicit-any
event: syntheticEvent as any,
context: { events_before: [], events_after: [], profile_info: {} },
};
};
/**
* Client-side full-text search over locally cached events in encrypted rooms.
* The homeserver cannot search E2EE message content, so we scan whatever the
* client has already received and decrypted in memory.
*
* Limitation: only messages present in the live timeline window are covered.
* Rooms that haven't been opened yet will return no results.
* When the persistent search cache is enabled (opt-in), the in-memory scan is
* also persisted to IndexedDB (fire-and-forget) and merged with prior cached
* coverage so results survive reloads. When disabled, zero cache reads/writes
* occur.
*/
export const useLocalMessageSearch = () => {
const mx = useMatrixClient();
const cacheEnabled = useAtomValue(searchCacheEnabledAtom);
const search = useCallback(
({ term, roomIds, senders }: LocalSearchParams): LocalSearchResult => {
async ({
term,
roomIds,
senders,
fromTs,
toTs,
}: LocalSearchParams): Promise<LocalSearchResult> => {
const trimmedTerm = term.trim();
const senderSet = senders && senders.length > 0 ? new Set(senders) : null;
@@ -41,6 +143,9 @@ export const useLocalMessageSearch = () => {
}
const termLower = trimmedTerm.toLowerCase();
const inRange = (ts: number): boolean =>
(fromTs === undefined || ts >= fromTs) && (toTs === undefined || ts <= toTs);
const groups: ResultGroup[] = [];
let encryptedRoomsCount = 0;
let searchedRoomsCount = 0;
@@ -61,106 +166,99 @@ export const useLocalMessageSearch = () => {
.getUnfilteredTimelineSet()
.getTimelines()
.flatMap((tl) => tl.getEvents());
if (events.length === 0) continue;
// eslint-disable-next-line no-await-in-loop
const cachedRows = cacheEnabled ? await queryRoom(roomId) : [];
if (events.length === 0 && cachedRows.length === 0) continue;
searchedRoomsCount += 1;
const items: ResultItem[] = [];
const memoryItems: ResultItem[] = [];
const rowsToPersist: SearchCacheRow[] = [];
for (let i = 0; i < events.length; i += 1) {
const event = events[i];
// In sender-only mode: include all message types; skip non-message events
if (event.getType() !== EventType.RoomMessage) {
if (senderOnlyMode) continue;
const evType = event.getType();
const isSticker = evType === 'm.sticker';
const isPoll = evType === 'm.poll.start' || evType === 'org.matrix.msc3381.poll.start';
if (!isSticker && !isPoll) continue;
}
if (event.isDecryptionFailure()) continue;
if (event.isRedacted()) continue;
if (senderSet && !senderSet.has(event.getSender() ?? '')) continue;
// getContent() returns decrypted plaintext regardless of encryption
const content = event.getContent();
const evType = event.getType();
const isSticker = evType === 'm.sticker';
const isMessageLike =
evType === EventType.RoomMessage || POLL_START_TYPES.includes(evType);
// Sender-only mode: no text filter needed
if (!senderOnlyMode) {
const evType = event.getType();
const isPoll = evType === 'm.poll.start' || evType === 'org.matrix.msc3381.poll.start';
// Sender-only mode indexes/returns all message types; text mode needs text.
if (!senderOnlyMode && !isMessageLike && !isSticker) continue;
let body = '';
let formattedBody = '';
if (!isPoll) {
body = (content.body as string | undefined) ?? '';
formattedBody = (content.formatted_body as string | undefined) ?? '';
} else {
// Poll — index question text and all answer options
const poll = (content['m.poll'] ??
// eslint-disable-next-line @typescript-eslint/no-explicit-any
content['org.matrix.msc3381.poll.start']) as any;
if (poll) {
const qBody =
// eslint-disable-next-line @typescript-eslint/no-explicit-any
(poll.question?.['m.text'] as Array<{ body: string }> | undefined)?.[0]?.body ??
(poll.question?.body as string | undefined) ??
'';
const answerBodies = ((poll.answers ?? []) as Array<Record<string, unknown>>)
.map(
(a) =>
// eslint-disable-next-line @typescript-eslint/no-explicit-any
((a['m.text'] as Array<{ body: string }> | undefined)?.[0]?.body ??
// eslint-disable-next-line @typescript-eslint/no-explicit-any
(a['org.matrix.msc3381.poll.answer'] as any)?.body ??
'') as string,
)
.join(' ');
body = `${qBody} ${answerBodies}`.trim();
}
}
const sender = event.getSender() ?? '';
const ts = event.getTs();
const text = extractText(event);
if (
!body.toLowerCase().includes(termLower) &&
!formattedBody.toLowerCase().includes(termLower)
)
continue;
// Persist every indexable (text-bearing) event we scanned, regardless
// of whether it matches the current term — future searches benefit.
if (cacheEnabled && text && event.getId()) {
rowsToPersist.push({
roomId,
eventId: event.getId() as string,
ts,
sender,
body: text.body,
...(text.formattedBody ? { formattedBody: text.formattedBody } : {}),
...(text.pollText ? { pollText: text.pollText } : {}),
});
}
// Build a synthetic IEventWithRoomId using decrypted content so the
// existing SearchResultGroup renderer works without modification.
if (senderSet && !senderSet.has(sender)) continue;
if (!inRange(ts)) continue;
if (!senderOnlyMode) {
if (!text || !matchesTerm(text, termLower)) continue;
}
const content = event.getContent();
const syntheticEvent = {
room_id: roomId,
event_id: event.getId() ?? '',
type: event.getType(),
sender: event.getSender() ?? '',
origin_server_ts: event.getTs(),
type: evType,
sender,
origin_server_ts: ts,
content,
unsigned: event.getUnsigned(),
};
items.push({
memoryItems.push({
rank: 0,
// eslint-disable-next-line @typescript-eslint/no-explicit-any
event: syntheticEvent as any,
context: {
events_before: [],
events_after: [],
profile_info: {},
},
context: { events_before: [], events_after: [], profile_info: {} },
});
}
// Match cached rows (skip ids already present in memory happens in merge).
const cachedItems: ResultItem[] = [];
cachedRows.forEach((row) => {
if (senderSet && !senderSet.has(row.sender)) return;
if (!inRange(row.ts)) return;
if (!senderOnlyMode && !rowMatchesTerm(row, termLower)) return;
cachedItems.push(rowToResultItem(row));
});
const items = mergeSearchResults(memoryItems, cachedItems);
if (items.length > 0) {
items.sort((a, b) => (b.event.origin_server_ts ?? 0) - (a.event.origin_server_ts ?? 0));
groups.push({ roomId, items });
}
// Fire-and-forget persist of freshly scanned rows + coverage.
// saveRoomIndex swallows all errors internally, so a floating promise
// here can never reject.
if (cacheEnabled && rowsToPersist.length > 0) {
saveRoomIndex(roomId, rowsToPersist);
}
}
return { groups, encryptedRoomsCount, searchedRoomsCount };
},
[mx],
[mx, cacheEnabled],
);
return search;