diff --git a/LOTUS_FEATURES.md b/LOTUS_FEATURES.md index ed9e168a5..cf23ba7a2 100644 --- a/LOTUS_FEATURES.md +++ b/LOTUS_FEATURES.md @@ -845,9 +845,12 @@ player.kick). - **Accessibility** — `radiogroup`/`radio` (single, with arrow-key roving) or `group`/`checkbox` (multi) semantics, `aria-checked`/`aria-disabled`, winner announced to AT. - Pure tally/visibility/winner + wire-format parsing live in `utils/poll.ts` (+ `poll.test.ts`, 14 tests incl. the stable/unstable round-trip). -### Voice Message Playback Speed +### Voice Message Playback (waveform + speed) -`AudioContent.tsx` adds a playback speed cycle button to voice message players. Available speeds: `[0.75, 1, 1.5, 2]×`. A `useEffect` sets `audioElement.playbackRate` whenever the speed selection changes. +`AudioContent.tsx` is the shared audio player (timeline + Media Gallery Audio tab): + +- **Waveform scrubbing** — voice messages carry an MSC1767 waveform (`org.matrix.msc1767.audio.waveform`, sent by the recorder). Playback renders it as bars that fill with the accent as the clip plays (TDS green under Lotus Terminal), and the waveform **is** the seek control — click, drag, or keyboard (arrows ±5s, Home/End) to scrub (`role="slider"`, ARIA value text). Threaded through `MAudio` (`MsgTypeRenderers.tsx`) + passed directly by the gallery. Regular audio with no waveform keeps the plain seek bar. +- **Playback speed** — a cycle button (`[0.75, 1, 1.5, 2]×`); a `useEffect` sets `audioElement.playbackRate` and re-applies it on (re)load (the browser resets it). --- @@ -1084,7 +1087,7 @@ A toggle in **Settings → Privacy** switches between sending `m.read` (public r - Four tabs: **Images**, **Videos**, **Audio**, **Files** (each with a live count) - **Images/Videos** — a month-grouped grid; tiles decrypt on demand (lazy, near-viewport), open a keyboard-navigable **lightbox** (←/→/Esc, prev/next) -- **Audio** — voice messages + audio files (`m.audio`) with an inline player (reuses `AudioContent`: play/seek/**speed control**; decrypts on play) +- **Audio** — voice messages + audio files (`m.audio`) with an inline player (reuses `AudioContent`: **waveform scrubbing** for voice messages, play/seek/**speed control**; decrypts on play) - **Files** — name/size/sender rows with download - **Jump to message** — a "Go to message" action on file rows, audio rows, and in the lightbox navigates the timeline to the source event (`useRoomNavigate`) and closes the drawer - Encrypted media is decrypted client-side on demand (no lock placeholder); download works for all types diff --git a/src/app/components/message/MsgTypeRenderers.tsx b/src/app/components/message/MsgTypeRenderers.tsx index 2a71e27db..51df01293 100644 --- a/src/app/components/message/MsgTypeRenderers.tsx +++ b/src/app/components/message/MsgTypeRenderers.tsx @@ -411,6 +411,7 @@ type RenderAudioContentProps = { mimeType: string; url: string; encInfo?: IEncryptedFile; + waveform?: number[]; }; type MAudioProps = { content: IAudioContent; @@ -431,6 +432,9 @@ export function MAudio({ content, renderAsFile, renderAudioContent, outlined }: } const filename = content.filename ?? content.body ?? 'Audio'; + const waveform = ( + content as unknown as { 'org.matrix.msc1767.audio'?: { waveform?: number[] } } + )['org.matrix.msc1767.audio']?.waveform; return ( @@ -454,6 +458,7 @@ export function MAudio({ content, renderAsFile, renderAudioContent, outlined }: mimeType: safeMimeType, url: mxcUrl, encInfo: content.file, + waveform, })} diff --git a/src/app/components/message/content/AudioContent.tsx b/src/app/components/message/content/AudioContent.tsx index 71c6aca4a..fd9a87721 100644 --- a/src/app/components/message/content/AudioContent.tsx +++ b/src/app/components/message/content/AudioContent.tsx @@ -1,8 +1,21 @@ -import React, { ReactNode, useCallback, useEffect, useRef, useState } from 'react'; -import { Badge, Chip, Icon, IconButton, Icons, ProgressBar, Spinner, Text, toRem } from 'folds'; +import React, { ReactNode, useCallback, useEffect, useMemo, useRef, useState } from 'react'; +import { + Badge, + Chip, + color, + Icon, + IconButton, + Icons, + ProgressBar, + Spinner, + Text, + toRem, +} from 'folds'; import { EncryptedAttachmentInfo } from 'browser-encrypt-attachment'; import { Range } from 'react-range'; import { useMatrixClient } from '../../../hooks/useMatrixClient'; +import { useSetting } from '../../../state/hooks/settings'; +import { settingsAtom } from '../../../state/settings'; import { AsyncStatus, useAsyncCallback } from '../../../hooks/useAsyncCallback'; import { IAudioInfo } from '../../../../types/matrix/common'; import { @@ -28,6 +41,123 @@ const PLAY_TIME_THROTTLE_OPS = { immediate: true, }; +const WAVEFORM_DISPLAY_BARS = 48; + +/** Reduce a stored MSC1767 waveform (up to ~100 samples) to a fixed display count. */ +export function downsampleWaveform(waveform: number[], count: number): number[] { + if (waveform.length <= count) return waveform; + const out: number[] = []; + const step = waveform.length / count; + for (let i = 0; i < count; i += 1) { + out.push(waveform[Math.floor(i * step)] ?? 0); + } + return out; +} + +// Voice-message waveform that doubles as a seek control: bars fill with the accent +// as the clip plays; click/drag/keyboard scrubs. Mirrors the recorder's bar styling +// (VoiceMessageRecorder). Falls back to the plain seek Range when there's no waveform. +function WaveformSeek({ + waveform, + currentTime, + duration, + onSeek, +}: { + waveform: number[]; + currentTime: number; + duration: number; + onSeek: (time: number) => void; +}) { + const [lotusTerminal] = useSetting(settingsAtom, 'lotusTerminal'); + const containerRef = useRef(null); + const bars = useMemo(() => downsampleWaveform(waveform, WAVEFORM_DISPLAY_BARS), [waveform]); + const barMax = useMemo(() => Math.max(...bars, 1), [bars]); + const progress = duration > 0 ? Math.min(1, Math.max(0, currentTime / duration)) : 0; + + const seekFromClientX = useCallback( + (clientX: number) => { + const el = containerRef.current; + if (!el || duration <= 0) return; + const rect = el.getBoundingClientRect(); + const ratio = rect.width > 0 ? (clientX - rect.left) / rect.width : 0; + onSeek(Math.min(duration, Math.max(0, ratio * duration))); + }, + [duration, onSeek], + ); + + const handlePointerDown = (evt: React.PointerEvent) => { + evt.currentTarget.setPointerCapture(evt.pointerId); + seekFromClientX(evt.clientX); + }; + const handlePointerMove = (evt: React.PointerEvent) => { + if (evt.currentTarget.hasPointerCapture(evt.pointerId)) seekFromClientX(evt.clientX); + }; + const handleKeyDown = (evt: React.KeyboardEvent) => { + if (duration <= 0) return; + if (evt.key === 'ArrowRight' || evt.key === 'ArrowUp') { + evt.preventDefault(); + onSeek(Math.min(duration, currentTime + 5)); + } else if (evt.key === 'ArrowLeft' || evt.key === 'ArrowDown') { + evt.preventDefault(); + onSeek(Math.max(0, currentTime - 5)); + } else if (evt.key === 'Home') { + evt.preventDefault(); + onSeek(0); + } else if (evt.key === 'End') { + evt.preventDefault(); + onSeek(duration); + } + }; + + return ( +
+ {bars.map((v, i) => { + const played = bars.length > 0 && i / bars.length <= progress; + return ( +
+ ); + })} +
+ ); +} + type RenderMediaControlProps = { after: ReactNode; leftControl: ReactNode; @@ -39,6 +169,8 @@ export type AudioContentProps = { url: string; info: IAudioInfo; encInfo?: EncryptedAttachmentInfo; + /** MSC1767 voice waveform (0–1024 ints); when present, the seek bar renders it. */ + waveform?: number[]; renderMediaControl: (props: RenderMediaControlProps) => ReactNode; }; export function AudioContent({ @@ -46,6 +178,7 @@ export function AudioContent({ url, info, encInfo, + waveform, renderMediaControl, }: AudioContentProps) { const mx = useMatrixClient(); @@ -132,8 +265,17 @@ export function AudioContent({ } }; + const hasWaveform = !!waveform && waveform.length > 0 && duration > 0; + return renderMediaControl({ - after: ( + after: hasWaveform ? ( + + ) : ( } />