Skip to content

Commit 94609da

Browse files
miguel-heygenmiguel-heygen
authored
feat(studio): audio pills show real peaks and loudness (#4282)
* feat(studio): audio clips draw real peaks shaded by loudness The pill fill is the track tint at 70 percent, and each bar's opacity follows the cached server peaks. Muting greys the pill in place. Linked video audio shows a 1px tick. * style(studio): format the audio waveform markup * fix(studio): map audio peaks across the full clip width * fix(studio): keep waveform visible through fade wedges * fix(studio): keep silent waveform bins visible * fix(studio): keep waveform baseline visible edge to edge * fix(studio): keep waveform visible over fades * fix(studio): keep quiet waveform bars visible * fix(studio): paint waveform above fade wedges * fix(studio): fit waveform inside audio clip * fix(studio): preserve waveform contrast tokens * test(studio): ratchet waveform color literals * fix(studio): keep loudness floor internal * chore(studio): document intentional fallow findings * refactor(studio): share fade handle rendering * fix(studio): pass fade handle styles explicitly * refactor(studio): split waveform drawing stages * test(studio): cover waveform canvas painting
1 parent 9f19d9d commit 94609da

14 files changed

Lines changed: 491 additions & 109 deletions

‎packages/studio/src/hooks/useRenderClipContent.test.ts‎

Lines changed: 31 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ import { useRenderClipContent } from "./useRenderClipContent";
1313
(globalThis as unknown as { IS_REACT_ACT_ENVIRONMENT: boolean }).IS_REACT_ACT_ENVIRONMENT = true;
1414

1515
afterEach(() => {
16-
usePlayerStore.setState({ thumbnailMode: "hidden" });
16+
usePlayerStore.setState({ thumbnailMode: "hidden", elements: [] });
1717
document.body.innerHTML = "";
1818
});
1919

@@ -144,6 +144,36 @@ describe("useRenderClipContent", () => {
144144
}
145145
});
146146

147+
it("marks audio linked when a video clip uses the same file, and muted when hidden", () => {
148+
usePlayerStore.setState({
149+
thumbnailMode: "hidden",
150+
elements: [
151+
{
152+
id: "picture",
153+
tag: "video",
154+
start: 0,
155+
duration: 4,
156+
track: 0,
157+
src: "assets/clip.mp4",
158+
},
159+
],
160+
});
161+
const linked = renderClipContent({
162+
id: "bed",
163+
tag: "audio",
164+
start: 0,
165+
duration: 4,
166+
track: 1,
167+
src: "assets/clip.mp4",
168+
hidden: true,
169+
});
170+
expect(isValidElement<{ linked: boolean; muted: boolean }>(linked)).toBe(true);
171+
if (isValidElement<{ linked: boolean; muted: boolean }>(linked)) {
172+
expect(linked.props.linked).toBe(true);
173+
expect(linked.props.muted).toBe(true);
174+
}
175+
});
176+
147177
it("passes empty labels to thumbnail content so TimelineClip owns clip names", () => {
148178
usePlayerStore.setState({ thumbnailMode: "adaptive" });
149179

‎packages/studio/src/hooks/useRenderClipContent.ts‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import { createElement } from "react";
44
import { CompositionThumbnail, VideoThumbnail } from "../player";
55
import type { TimelineElement } from "../player";
66
import type { TimelineClipRenderContext } from "../player/components/TimelineTypes";
7+
import { audioPillFlags } from "../player/components/audioClipLink";
78
import { AudioWaveform } from "../player/components/AudioWaveform";
89
import { ImageThumbnail } from "../player/components/ImageThumbnail";
910
import { encodePreviewPath, resolveMediaPreviewUrl } from "../player/components/thumbnailUtils";
@@ -70,6 +71,7 @@ function renderAudioClip(
7071
sessionEpoch: number,
7172
labelColor: string,
7273
context: TimelineClipRenderContext,
74+
elements: readonly TimelineElement[],
7375
): ReactNode {
7476
const audioUrl = resolveMediaPreviewUrl(el.src ?? "", pid, window.location.origin);
7577
const srcRelative = resolvePreviewRelative(audioUrl, pid, window.location.origin);
@@ -91,6 +93,7 @@ function renderAudioClip(
9193
projectId: pid,
9294
sessionEpoch,
9395
priority: context.priority,
96+
...audioPillFlags(el, elements),
9497
});
9598
}
9699

@@ -112,6 +115,7 @@ export function useRenderClipContent({
112115
const effectiveMode = effectiveThumbnailMode(thumbnailMode);
113116
const sessionEpoch = usePlayerStore((s) => s.timelineSessionEpoch);
114117
const contentRevision = usePlayerStore((s) => s.thumbnailContentRevision);
118+
const elements = usePlayerStore((s) => s.elements);
115119
return useCallback(
116120
// Pre-existing clip-content dispatcher; reduced by extracting renderAudioClip.
117121
// fallow-ignore-next-line complexity
@@ -127,7 +131,7 @@ export function useRenderClipContent({
127131
// its waveform (cheap, not a frame thumbnail). Toggle: timeline toolbar.
128132
if (effectiveMode === "hidden") {
129133
return el.tag === "audio"
130-
? renderAudioClip(el, pid, sessionEpoch, style.label, context)
134+
? renderAudioClip(el, pid, sessionEpoch, style.label, context, elements)
131135
: null;
132136
}
133137

@@ -165,7 +169,7 @@ export function useRenderClipContent({
165169
// activePreviewUrl thumbnail branch; audio rows need waveform data, not a
166170
// captured frame from the currently drilled composition preview.
167171
if (el.tag === "audio") {
168-
return renderAudioClip(el, pid, sessionEpoch, style.label, context);
172+
return renderAudioClip(el, pid, sessionEpoch, style.label, context, elements);
169173
}
170174

171175
// When drilled into a composition, render all inner elements via
@@ -251,6 +255,7 @@ export function useRenderClipContent({
251255
effectiveMode,
252256
sessionEpoch,
253257
contentRevision,
258+
elements,
254259
],
255260
);
256261
}

‎packages/studio/src/player/components/AudioWaveform.test.tsx‎

Lines changed: 53 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ vi.mock("../../hooks/useThumbnailLease", () => ({
1111
useThumbnailLease: leaseSpy,
1212
}));
1313

14-
import { AudioWaveform } from "./AudioWaveform";
14+
import { AudioWaveform, drawWaveformCanvas } from "./AudioWaveform";
1515

1616
(globalThis as unknown as { IS_REACT_ACT_ENVIRONMENT: boolean }).IS_REACT_ACT_ENVIRONMENT = true;
1717

@@ -21,6 +21,30 @@ afterEach(() => {
2121
});
2222

2323
describe("AudioWaveform", () => {
24+
it("paints a baseline and peak bar for every mapped waveform bin", () => {
25+
const fillRect = vi.fn();
26+
const context = {
27+
scale: vi.fn(),
28+
clearRect: vi.fn(),
29+
fillRect,
30+
fillStyle: "",
31+
} as unknown as CanvasRenderingContext2D;
32+
const canvas = document.createElement("canvas");
33+
Object.defineProperties(canvas, {
34+
clientWidth: { value: 6 },
35+
clientHeight: { value: 20 },
36+
});
37+
vi.spyOn(canvas, "getContext").mockReturnValue(context);
38+
drawWaveformCanvas(canvas, [0.25, 1], false, 0, 1);
39+
expect(fillRect).toHaveBeenCalledTimes(4);
40+
expect(fillRect.mock.calls.map(([x, y, width, height]) => [x, y, width, height])).toEqual([
41+
[0, 18, 3, 2],
42+
[0, 15, 3, 5],
43+
[3, 18, 3, 2],
44+
[3, 0, 3, 20],
45+
]);
46+
});
47+
2448
it("leases waveform decoding with the clip's project, session, and viewport priority", () => {
2549
const host = document.createElement("div");
2650
document.body.append(host);
@@ -50,4 +74,32 @@ describe("AudioWaveform", () => {
5074

5175
act(() => root.unmount());
5276
});
77+
78+
it("greys the clip in place when muted and draws the parent tick when linked", () => {
79+
const host = document.createElement("div");
80+
host.className = "timeline-clip is-audio";
81+
document.body.append(host);
82+
const root = createRoot(host);
83+
84+
act(() => {
85+
root.render(
86+
<AudioWaveform
87+
audioUrl="/media/voice.wav"
88+
label=""
89+
labelColor="#fff"
90+
projectId="project-a"
91+
sessionEpoch={1}
92+
priority="visible"
93+
muted
94+
linked
95+
/>,
96+
);
97+
});
98+
99+
expect(host.getAttribute("data-audio-muted")).toBe("true");
100+
expect(host.querySelector(".timeline-audio-link")).not.toBeNull();
101+
102+
act(() => root.unmount());
103+
expect(host.hasAttribute("data-audio-muted")).toBe(false);
104+
});
53105
});

‎packages/studio/src/player/components/AudioWaveform.tsx‎

Lines changed: 113 additions & 63 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ import { memo, useCallback, useEffect, useMemo, useRef } from "react";
22
import { useMountEffect } from "../../hooks/useMountEffect";
33
import { useThumbnailLease } from "../../hooks/useThumbnailLease";
44
import { createThumbnailKey, type ThumbnailPriority } from "../lib/thumbnailScheduler";
5+
import { decimatePeaks, loudnessToOpacity } from "./audioWaveformPeaks";
56

67
interface AudioWaveformProps {
78
audioUrl: string;
@@ -13,11 +14,68 @@ interface AudioWaveformProps {
1314
projectId: string;
1415
sessionEpoch: number;
1516
priority: ThumbnailPriority;
17+
/** `data-hidden` or a muted audio group. Greys the pill; the clip stays. */
18+
muted?: boolean;
19+
/** Same media file as a video clip. Draws the 1px parent tick. */
20+
linked?: boolean;
1621
}
1722

18-
const BAR_WIDTH = 2;
1923
const BAR_STEP = 3;
2024

25+
type BarGeometry = { x: number; width: number; height: number };
26+
27+
function paintWaveformBars(
28+
context: CanvasRenderingContext2D,
29+
bars: readonly BarGeometry[],
30+
height: number,
31+
waveformBarRgb: string,
32+
waveformBaselineRgb: string,
33+
amplitudes: readonly number[],
34+
) {
35+
bars.forEach((bar, index) => {
36+
const amplitude = amplitudes[index] ?? 0;
37+
context.fillStyle = `rgb(${waveformBaselineRgb})`;
38+
context.fillRect(bar.x, height - 2, bar.width, 2);
39+
context.fillStyle = `rgba(${waveformBarRgb},${loudnessToOpacity(amplitude).toFixed(2)})`;
40+
context.fillRect(bar.x, height - bar.height, bar.width, bar.height);
41+
});
42+
}
43+
44+
export function drawWaveformCanvas(
45+
canvas: HTMLCanvasElement,
46+
peaks: readonly number[],
47+
muted: boolean,
48+
trimStartFraction: number,
49+
trimEndFraction: number,
50+
) {
51+
const width = Math.max(1, canvas.clientWidth);
52+
const height = Math.max(1, canvas.clientHeight);
53+
const scale = window.devicePixelRatio || 1;
54+
canvas.width = Math.ceil(width * scale);
55+
canvas.height = Math.ceil(height * scale);
56+
const context = canvas.getContext("2d");
57+
if (!context) return;
58+
context.scale(scale, scale);
59+
context.clearRect(0, 0, width, height);
60+
const amplitudes = decimatePeaks(
61+
peaks,
62+
trimStartFraction,
63+
trimEndFraction,
64+
Math.max(1, Math.ceil(width / BAR_STEP)),
65+
);
66+
const bars = amplitudes.map((amplitude, index) => ({
67+
x: (index * width) / amplitudes.length,
68+
width: Math.max(1, width / amplitudes.length),
69+
height: Math.max(3, amplitude * height),
70+
}));
71+
const channelToken = muted ? "--timeline-waveform-muted-rgb" : "--timeline-waveform-bar-rgb";
72+
const waveformBarRgb = getComputedStyle(canvas).getPropertyValue(channelToken);
73+
const waveformBaselineRgb = getComputedStyle(canvas).getPropertyValue(
74+
"--timeline-waveform-baseline-rgb",
75+
);
76+
paintWaveformBars(context, bars, height, waveformBarRgb, waveformBaselineRgb, amplitudes);
77+
}
78+
2179
function extractPeaks(channelData: Float32Array, barCount: number): number[] {
2280
const peaks: number[] = [];
2381
const samplesPerBar = Math.floor(channelData.length / barCount);
@@ -92,7 +150,10 @@ export const AudioWaveform = memo(function AudioWaveform({
92150
projectId,
93151
sessionEpoch,
94152
priority,
153+
muted = false,
154+
linked = false,
95155
}: AudioWaveformProps) {
156+
const rootRef = useRef<HTMLDivElement | null>(null);
96157
const canvasRef = useRef<HTMLCanvasElement | null>(null);
97158
const observerRef = useRef<ResizeObserver | null>(null);
98159
const cacheKey = waveformUrl ?? audioUrl;
@@ -121,30 +182,8 @@ export const AudioWaveform = memo(function AudioWaveform({
121182
const draw = useCallback(() => {
122183
const canvas = canvasRef.current;
123184
if (!canvas || !peaks) return;
124-
const width = Math.max(1, canvas.clientWidth);
125-
const height = Math.max(1, canvas.clientHeight);
126-
const scale = window.devicePixelRatio || 1;
127-
canvas.width = Math.ceil(width * scale);
128-
canvas.height = Math.ceil(height * scale);
129-
const context = canvas.getContext("2d");
130-
if (!context) return;
131-
context.scale(scale, scale);
132-
context.clearRect(0, 0, width, height);
133-
const startFraction = Math.max(0, Math.min(1, trimStartFraction ?? 0));
134-
const endFraction = Math.max(startFraction, Math.min(1, trimEndFraction ?? 1));
135-
const start = Math.floor(startFraction * peaks.length);
136-
const end = Math.max(start + 1, Math.ceil(endFraction * peaks.length));
137-
const span = end - start;
138-
const barCount = Math.floor(width / BAR_STEP);
139-
const waveformBarRgb = getComputedStyle(canvas).getPropertyValue("--timeline-waveform-bar-rgb");
140-
for (let index = 0; index < barCount; index++) {
141-
const peakIndex = start + Math.min(span - 1, Math.floor((index / barCount) * span));
142-
const amplitude = peaks[peakIndex] ?? 0;
143-
const barHeight = Math.max(2, amplitude * height);
144-
context.fillStyle = `rgba(${waveformBarRgb},${(0.45 + amplitude * 0.4).toFixed(2)})`;
145-
context.fillRect(index * BAR_STEP, height - barHeight, BAR_WIDTH, barHeight);
146-
}
147-
}, [peaks, trimEndFraction, trimStartFraction]);
185+
drawWaveformCanvas(canvas, peaks, muted, trimStartFraction ?? 0, trimEndFraction ?? 1);
186+
}, [muted, peaks, trimEndFraction, trimStartFraction]);
148187

149188
const setCanvasRef = useCallback(
150189
(canvas: HTMLCanvasElement | null) => {
@@ -170,51 +209,62 @@ export const AudioWaveform = memo(function AudioWaveform({
170209

171210
useMountEffect(() => () => observerRef.current?.disconnect());
172211

212+
useEffect(() => {
213+
const clip = rootRef.current?.closest(".timeline-clip");
214+
if (!(clip instanceof HTMLElement)) return;
215+
if (muted) clip.setAttribute("data-audio-muted", "true");
216+
else clip.removeAttribute("data-audio-muted");
217+
return () => clip.removeAttribute("data-audio-muted");
218+
}, [muted]);
219+
173220
return (
174-
<div className="absolute inset-0 overflow-hidden">
175-
<canvas
176-
ref={setCanvasRef}
177-
className="absolute inset-x-0 bottom-0 w-full"
178-
style={{ top: 16 }}
179-
/>
180-
{snapshot.status === "loading" && (
181-
<div
182-
className="absolute inset-x-0 bottom-0 top-4 animate-pulse"
183-
style={{
184-
background: "var(--timeline-thumbnail-shimmer)",
185-
}}
221+
<div ref={rootRef} className="absolute inset-0">
222+
{linked ? <span className="timeline-audio-link" aria-hidden="true" /> : null}
223+
<div className="absolute inset-0 overflow-hidden" style={{ zIndex: 10 }}>
224+
<canvas
225+
ref={setCanvasRef}
226+
className="absolute inset-x-0 bottom-0 w-full"
227+
style={{ top: 16, height: "calc(100% - 16px)" }}
186228
/>
187-
)}
188-
{/* Degraded state — the decode failed; say so rather than paint a
189-
waveform the author could edit against. */}
190-
{snapshot.status === "error" && (
191-
<div
192-
className="absolute inset-x-0 flex items-center justify-center gap-1.5"
193-
style={{ top: 16, bottom: 0 }}
194-
>
229+
{snapshot.status === "loading" && (
195230
<div
196-
className="absolute inset-x-0"
231+
className="absolute inset-x-0 bottom-0 top-4 animate-pulse"
197232
style={{
198-
bottom: "20%",
199-
height: 2,
200-
background: "var(--timeline-waveform-error)",
233+
background: "var(--timeline-thumbnail-shimmer)",
201234
}}
202235
/>
203-
<span className="relative rounded-sm bg-black/50 px-1 text-[8px] text-neutral-500">
204-
waveform unavailable
205-
</span>
206-
</div>
207-
)}
208-
{label && (
209-
<div className="absolute inset-x-0 top-0 z-10 px-1.5 py-0.5">
210-
<span
211-
className="block truncate text-[9px] font-semibold leading-tight"
212-
style={{ color: labelColor, textShadow: "var(--timeline-waveform-label-shadow)" }}
236+
)}
237+
{/* Degraded state — the decode failed; say so rather than paint a
238+
waveform the author could edit against. */}
239+
{snapshot.status === "error" && (
240+
<div
241+
className="absolute inset-x-0 flex items-center justify-center gap-1.5"
242+
style={{ top: 16, bottom: 0 }}
213243
>
214-
{label}
215-
</span>
216-
</div>
217-
)}
244+
<div
245+
className="absolute inset-x-0"
246+
style={{
247+
bottom: "20%",
248+
height: 2,
249+
background: "var(--timeline-waveform-error)",
250+
}}
251+
/>
252+
<span className="relative rounded-sm bg-black/50 px-1 text-[8px] text-neutral-500">
253+
waveform unavailable
254+
</span>
255+
</div>
256+
)}
257+
{label && (
258+
<div className="absolute inset-x-0 top-0 z-10 px-1.5 py-0.5">
259+
<span
260+
className="block truncate text-[9px] font-semibold leading-tight"
261+
style={{ color: labelColor, textShadow: "var(--timeline-waveform-label-shadow)" }}
262+
>
263+
{label}
264+
</span>
265+
</div>
266+
)}
267+
</div>
218268
</div>
219269
);
220270
});

0 commit comments

Comments
 (0)