From 56863d26494f5f963d3a280282726381f8d192aa Mon Sep 17 00:00:00 2001 From: Jared Vititoe Date: Thu, 9 Jul 2026 21:49:34 -0400 Subject: [PATCH] feat(voice): render the waveform on playback with click/drag/keyboard scrubbing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Voice messages carry an MSC1767 waveform (org.matrix.msc1767.audio.waveform) and the recorder draws a live one, but AudioContent playback only showed a plain seek bar. Now the player renders the waveform as bars that fill with the accent (TDS green under Lotus Terminal) as the clip plays, and the waveform itself is the seek control — click, drag, or keyboard (arrows +/-5s, Home/End) with role=slider + ARIA value text. - AudioContent: new optional "waveform" prop + a WaveformSeek sub-component (downsamples to 48 bars, mirrors the recorder's bar styling); falls back to the plain Range seek bar when there's no waveform. - Threaded through MAudio (RenderAudioContentProps) so timeline voice messages get it automatically; the Media Gallery Audio tab passes it directly. Improves both the timeline and the new gallery Audio tab at once. Co-Authored-By: Claude Opus 4.8 --- LOTUS_FEATURES.md | 9 +- .../components/message/MsgTypeRenderers.tsx | 5 + .../message/content/AudioContent.tsx | 148 +++++++++++++++++- src/app/features/room/MediaGallery.tsx | 4 + 4 files changed, 160 insertions(+), 6 deletions(-) diff --git a/LOTUS_FEATURES.md b/LOTUS_FEATURES.md index ed9e168a5..cf23ba7a2 100644 --- a/LOTUS_FEATURES.md +++ b/LOTUS_FEATURES.md @@ -845,9 +845,12 @@ player.kick). - **Accessibility** — `radiogroup`/`radio` (single, with arrow-key roving) or `group`/`checkbox` (multi) semantics, `aria-checked`/`aria-disabled`, winner announced to AT. - Pure tally/visibility/winner + wire-format parsing live in `utils/poll.ts` (+ `poll.test.ts`, 14 tests incl. the stable/unstable round-trip). -### Voice Message Playback Speed +### Voice Message Playback (waveform + speed) -`AudioContent.tsx` adds a playback speed cycle button to voice message players. Available speeds: `[0.75, 1, 1.5, 2]×`. A `useEffect` sets `audioElement.playbackRate` whenever the speed selection changes. +`AudioContent.tsx` is the shared audio player (timeline + Media Gallery Audio tab): + +- **Waveform scrubbing** — voice messages carry an MSC1767 waveform (`org.matrix.msc1767.audio.waveform`, sent by the recorder). Playback renders it as bars that fill with the accent as the clip plays (TDS green under Lotus Terminal), and the waveform **is** the seek control — click, drag, or keyboard (arrows ±5s, Home/End) to scrub (`role="slider"`, ARIA value text). Threaded through `MAudio` (`MsgTypeRenderers.tsx`) + passed directly by the gallery. Regular audio with no waveform keeps the plain seek bar. +- **Playback speed** — a cycle button (`[0.75, 1, 1.5, 2]×`); a `useEffect` sets `audioElement.playbackRate` and re-applies it on (re)load (the browser resets it). --- @@ -1084,7 +1087,7 @@ A toggle in **Settings → Privacy** switches between sending `m.read` (public r - Four tabs: **Images**, **Videos**, **Audio**, **Files** (each with a live count) - **Images/Videos** — a month-grouped grid; tiles decrypt on demand (lazy, near-viewport), open a keyboard-navigable **lightbox** (←/→/Esc, prev/next) -- **Audio** — voice messages + audio files (`m.audio`) with an inline player (reuses `AudioContent`: play/seek/**speed control**; decrypts on play) +- **Audio** — voice messages + audio files (`m.audio`) with an inline player (reuses `AudioContent`: **waveform scrubbing** for voice messages, play/seek/**speed control**; decrypts on play) - **Files** — name/size/sender rows with download - **Jump to message** — a "Go to message" action on file rows, audio rows, and in the lightbox navigates the timeline to the source event (`useRoomNavigate`) and closes the drawer - Encrypted media is decrypted client-side on demand (no lock placeholder); download works for all types diff --git a/src/app/components/message/MsgTypeRenderers.tsx b/src/app/components/message/MsgTypeRenderers.tsx index 2a71e27db..51df01293 100644 --- a/src/app/components/message/MsgTypeRenderers.tsx +++ b/src/app/components/message/MsgTypeRenderers.tsx @@ -411,6 +411,7 @@ type RenderAudioContentProps = { mimeType: string; url: string; encInfo?: IEncryptedFile; + waveform?: number[]; }; type MAudioProps = { content: IAudioContent; @@ -431,6 +432,9 @@ export function MAudio({ content, renderAsFile, renderAudioContent, outlined }: } const filename = content.filename ?? content.body ?? 'Audio'; + const waveform = ( + content as unknown as { 'org.matrix.msc1767.audio'?: { waveform?: number[] } } + )['org.matrix.msc1767.audio']?.waveform; return ( @@ -454,6 +458,7 @@ export function MAudio({ content, renderAsFile, renderAudioContent, outlined }: mimeType: safeMimeType, url: mxcUrl, encInfo: content.file, + waveform, })} diff --git a/src/app/components/message/content/AudioContent.tsx b/src/app/components/message/content/AudioContent.tsx index 71c6aca4a..fd9a87721 100644 --- a/src/app/components/message/content/AudioContent.tsx +++ b/src/app/components/message/content/AudioContent.tsx @@ -1,8 +1,21 @@ -import React, { ReactNode, useCallback, useEffect, useRef, useState } from 'react'; -import { Badge, Chip, Icon, IconButton, Icons, ProgressBar, Spinner, Text, toRem } from 'folds'; +import React, { ReactNode, useCallback, useEffect, useMemo, useRef, useState } from 'react'; +import { + Badge, + Chip, + color, + Icon, + IconButton, + Icons, + ProgressBar, + Spinner, + Text, + toRem, +} from 'folds'; import { EncryptedAttachmentInfo } from 'browser-encrypt-attachment'; import { Range } from 'react-range'; import { useMatrixClient } from '../../../hooks/useMatrixClient'; +import { useSetting } from '../../../state/hooks/settings'; +import { settingsAtom } from '../../../state/settings'; import { AsyncStatus, useAsyncCallback } from '../../../hooks/useAsyncCallback'; import { IAudioInfo } from '../../../../types/matrix/common'; import { @@ -28,6 +41,123 @@ const PLAY_TIME_THROTTLE_OPS = { immediate: true, }; +const WAVEFORM_DISPLAY_BARS = 48; + +/** Reduce a stored MSC1767 waveform (up to ~100 samples) to a fixed display count. */ +export function downsampleWaveform(waveform: number[], count: number): number[] { + if (waveform.length <= count) return waveform; + const out: number[] = []; + const step = waveform.length / count; + for (let i = 0; i < count; i += 1) { + out.push(waveform[Math.floor(i * step)] ?? 0); + } + return out; +} + +// Voice-message waveform that doubles as a seek control: bars fill with the accent +// as the clip plays; click/drag/keyboard scrubs. Mirrors the recorder's bar styling +// (VoiceMessageRecorder). Falls back to the plain seek Range when there's no waveform. +function WaveformSeek({ + waveform, + currentTime, + duration, + onSeek, +}: { + waveform: number[]; + currentTime: number; + duration: number; + onSeek: (time: number) => void; +}) { + const [lotusTerminal] = useSetting(settingsAtom, 'lotusTerminal'); + const containerRef = useRef(null); + const bars = useMemo(() => downsampleWaveform(waveform, WAVEFORM_DISPLAY_BARS), [waveform]); + const barMax = useMemo(() => Math.max(...bars, 1), [bars]); + const progress = duration > 0 ? Math.min(1, Math.max(0, currentTime / duration)) : 0; + + const seekFromClientX = useCallback( + (clientX: number) => { + const el = containerRef.current; + if (!el || duration <= 0) return; + const rect = el.getBoundingClientRect(); + const ratio = rect.width > 0 ? (clientX - rect.left) / rect.width : 0; + onSeek(Math.min(duration, Math.max(0, ratio * duration))); + }, + [duration, onSeek], + ); + + const handlePointerDown = (evt: React.PointerEvent) => { + evt.currentTarget.setPointerCapture(evt.pointerId); + seekFromClientX(evt.clientX); + }; + const handlePointerMove = (evt: React.PointerEvent) => { + if (evt.currentTarget.hasPointerCapture(evt.pointerId)) seekFromClientX(evt.clientX); + }; + const handleKeyDown = (evt: React.KeyboardEvent) => { + if (duration <= 0) return; + if (evt.key === 'ArrowRight' || evt.key === 'ArrowUp') { + evt.preventDefault(); + onSeek(Math.min(duration, currentTime + 5)); + } else if (evt.key === 'ArrowLeft' || evt.key === 'ArrowDown') { + evt.preventDefault(); + onSeek(Math.max(0, currentTime - 5)); + } else if (evt.key === 'Home') { + evt.preventDefault(); + onSeek(0); + } else if (evt.key === 'End') { + evt.preventDefault(); + onSeek(duration); + } + }; + + return ( +
+ {bars.map((v, i) => { + const played = bars.length > 0 && i / bars.length <= progress; + return ( +
+ ); + })} +
+ ); +} + type RenderMediaControlProps = { after: ReactNode; leftControl: ReactNode; @@ -39,6 +169,8 @@ export type AudioContentProps = { url: string; info: IAudioInfo; encInfo?: EncryptedAttachmentInfo; + /** MSC1767 voice waveform (0–1024 ints); when present, the seek bar renders it. */ + waveform?: number[]; renderMediaControl: (props: RenderMediaControlProps) => ReactNode; }; export function AudioContent({ @@ -46,6 +178,7 @@ export function AudioContent({ url, info, encInfo, + waveform, renderMediaControl, }: AudioContentProps) { const mx = useMatrixClient(); @@ -132,8 +265,17 @@ export function AudioContent({ } }; + const hasWaveform = !!waveform && waveform.length > 0 && duration > 0; + return renderMediaControl({ - after: ( + after: hasWaveform ? ( + + ) : ( } />