Skip to content

Commit 824e410

Browse files
vanceingallsclaude
andauthored
feat(studio): waveform strip, clip light, peak marks and video beat source (5/8) (#4817)
* feat(studio): draw a waveform strip under video clips that carry sound An audible video keeps its thumbnails and gains the audio waveform along the bottom 38% of the clip (also when thumbnails are off); muted and silent video are unchanged. The waveform route already decodes video audio; a test now proves it on an mp4. Co-Authored-By: Claude Opus 5.5 <[email protected]> * feat(studio): latch a CLIP light on the master meter at the export ceiling The light turns red when the preview master sample peak reaches -1 dBFS (the export true-peak ceiling) and stays lit until clicked. Mutation-tested threshold and latch. Co-Authored-By: Claude Opus 5.5 <[email protected]> * fix(studio): paint the CLIP light from theme colour tokens Co-Authored-By: Claude Opus 5.5 <[email protected]> * feat(studio): mark where each clip redlines on its waveform studio-server computes an absolute per-50ms sample-peak map once per media file (native channels, cached by size+mtime next to the waveform cache); the timeline paints red marks where the source at the clip's own volume reaches -1 dBFS and names the loudest point. Co-Authored-By: Claude Opus 5.5 <[email protected]> * feat(studio): let a video with sound be the beat source Studio's isMusicSourceElement accepts an audible, unmuted video for the music track and the beat fallback (lane zoning unchanged); core findMusicAudioSrc accepts <video data-has-audio="true"> without muted; the beats CLI message mentions video. Co-Authored-By: Claude Opus 5.5 <[email protected]> * fix(studio): decode each peak map once and never offer Duck on a carve source Concurrent peak requests for one file share a single ffmpeg decode; Duck under voice is withheld on a clip another bed already carves against. Co-Authored-By: Claude Opus 5.5 <[email protected]> * fix(studio-server): decode the same audio stream the peak map probes The channel probe read stream a:0 while the decode left stream choice to ffmpeg, which picks a later stereo stream over a mono first one; every frame was then read at the wrong width, doubling the bins and taking the other stream's level. Both commands now select a:0, and the cache key moves to peaks-v2 so maps built from the wrong stream are rebuilt. Co-Authored-By: Claude Opus 5.5 <[email protected]> * fix(studio-server): terminate the peak map's ffprobe options before the input Co-Authored-By: Claude Opus 5.5 <[email protected]> * fix(studio-server): reject a peak map whose decode exits nonzero A decode that emits some samples and then fails no longer resolves as a complete map, so /peaks cannot cache a truncated clipping map. Co-Authored-By: Claude Opus 5.5 <[email protected]> --------- Co-authored-by: Claude Opus 5.5 <[email protected]>
1 parent 9244c47 commit 824e410

24 files changed

Lines changed: 1063 additions & 19 deletions

‎packages/cli/src/commands/beats.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,8 @@ function resolveMusicTarget(project: ProjectDir): { rel: string; audioPath: stri
2323
const src = findMusicAudioSrc(readFileSync(project.indexPath, "utf-8"));
2424
if (!src) {
2525
fail(
26-
'No music track found. Add data-timeline-role="music" to the <audio> element ' +
26+
'No music track found. Add data-timeline-role="music" to the <audio> element, or to a ' +
27+
'<video data-has-audio="true"> whose own sound is the music ' +
2728
"(or give it an id like music/bgm/soundtrack).",
2829
);
2930
}
Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
import { describe, expect, it } from "vitest";
2+
import { findMusicAudioSrc } from "./beatFile";
3+
4+
describe("findMusicAudioSrc", () => {
5+
it("finds a video with sound tagged as music", () => {
6+
const html = `<video id="clip" src="band.mp4" data-has-audio="true" data-timeline-role="music"></video>`;
7+
expect(findMusicAudioSrc(html)).toBe("band.mp4");
8+
});
9+
10+
it("finds an audible video by a music id", () => {
11+
expect(findMusicAudioSrc(`<video id="soundtrack" src="s.mp4" data-has-audio="true">`)).toBe(
12+
"s.mp4",
13+
);
14+
});
15+
16+
it("skips a muted video and one without declared sound", () => {
17+
const html = [
18+
`<video id="music" src="muted.mp4" data-has-audio="true" muted>`,
19+
`<video id="bgm" src="undeclared.mp4">`,
20+
`<audio id="music-bed" src="bed.mp3">`,
21+
].join("");
22+
expect(findMusicAudioSrc(html)).toBe("bed.mp3");
23+
});
24+
25+
it("keeps document order across audio and video", () => {
26+
const html = `<audio id="music" src="first.mp3"></audio><video id="bgm" src="v.mp4" data-has-audio="true">`;
27+
expect(findMusicAudioSrc(html)).toBe("first.mp3");
28+
});
29+
});

‎packages/core/src/beats/beatFile.ts‎

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -87,18 +87,25 @@ function attr(tag: string, name: string): string | null {
8787
return m ? m[1]! : null;
8888
}
8989

90+
function isSoundingVideoTag(tag: string): boolean {
91+
const unquoted = tag.replace(/"[^"]*"|'[^']*'/g, '""');
92+
return attr(tag, "data-has-audio") === "true" && !/\smuted(?=[\s/>=])/i.test(unquoted);
93+
}
94+
9095
/**
9196
* Find the music track's src in composition HTML, applying the SAME rules as the
92-
* Studio's `isMusicTrack` so the CLI and Studio agree on which `<audio>` is music:
93-
* the FIRST `<audio>` (in document order) where data-timeline-role="music", or —
97+
* Studio's `isMusicTrack` so the CLI and Studio agree on which clip is music:
98+
* the FIRST `<audio>`, or `<video data-has-audio="true">` without `muted` (in
99+
* document order) where data-timeline-role="music", or —
94100
* when no role is set — whose id matches the music regex. An explicit non-music
95101
* role excludes the element. Returns the raw src attribute, or null.
96102
*/
97103
export function findMusicAudioSrc(html: string): string | null {
98104
// `[^>]*` spans newlines (it's a negated class, not `.`), so multi-line opening
99105
// tags are handled. HyperFrames authors src as an attribute on <audio>.
100-
const tags = html.match(/<audio\b[^>]*>/gi) ?? [];
106+
const tags = html.match(/<(?:audio|video)\b[^>]*>/gi) ?? [];
101107
for (const tag of tags) {
108+
if (/^<video/i.test(tag) && !isSoundingVideoTag(tag)) continue;
102109
const src = attr(tag, "src");
103110
if (!src) continue;
104111
const role = attr(tag, "data-timeline-role");

‎packages/studio-server/src/createStudioApi.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import { registerThumbnailRoutes } from "./routes/thumbnail.js";
1010
import { registerWaveformRoutes } from "./routes/waveform.js";
1111
import { registerFreezeFrameRoutes } from "./routes/freezeFrame.js";
1212
import { registerLoudnessRoutes } from "./routes/loudness.js";
13+
import { registerPeakRoutes } from "./routes/peaks.js";
1314
import { registerFontRoutes } from "./routes/fonts.js";
1415
import { registerRegistryRoutes } from "./routes/registry.js";
1516
import { registerSelectionRoutes } from "./routes/selection.js";
@@ -65,6 +66,7 @@ export function createStudioApi(adapter: StudioApiAdapter): Hono {
6566
registerWaveformRoutes(api, adapter);
6667
registerFreezeFrameRoutes(api, adapter);
6768
registerLoudnessRoutes(api, adapter);
69+
registerPeakRoutes(api, adapter);
6870
registerFontRoutes(api);
6971
registerRegistryRoutes(api, adapter);
7072
registerGlobalAssetRoutes(api);
Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
import { chmodSync, mkdtempSync, writeFileSync } from "node:fs";
2+
import { tmpdir } from "node:os";
3+
import { join } from "node:path";
4+
import { describe, expect, it, vi } from "vitest";
5+
6+
// Stand-in binaries: ffprobe reports a stereo stream, ffmpeg writes samples and then fails.
7+
const fakeBin = vi.hoisted(() => ({ dir: "" }));
8+
vi.mock("@hyperframes/parsers/ff-binaries", () => ({
9+
findFfBinary: (name: string) => join(fakeBin.dir, name),
10+
}));
11+
12+
import { decodePeakMap } from "./peakMap";
13+
14+
describe("decodePeakMap, decode failure", () => {
15+
it.skipIf(process.platform === "win32")(
16+
"rejects partial output when ffmpeg exits nonzero",
17+
async () => {
18+
fakeBin.dir = mkdtempSync(join(tmpdir(), "hf-fake-ff-"));
19+
const script = (name: string, body: string) => {
20+
const file = join(fakeBin.dir, name);
21+
writeFileSync(file, `#!/bin/sh\n${body}\n`);
22+
chmodSync(file, 0o755);
23+
};
24+
script("ffprobe", "echo 2");
25+
script("ffmpeg", "head -c 38400 /dev/zero; exit 1");
26+
await expect(decodePeakMap("truncated.mov")).rejects.toThrow(/exit 1/);
27+
},
28+
);
29+
});
Lines changed: 149 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,149 @@
1+
import { execFileSync } from "node:child_process";
2+
import { mkdtempSync, rmSync } from "node:fs";
3+
import { tmpdir } from "node:os";
4+
import { join } from "node:path";
5+
import { describe, expect, it } from "vitest";
6+
import { findFfBinary } from "@hyperframes/parsers/ff-binaries";
7+
import {
8+
accumulatePeaks,
9+
buildPeakMapCacheKey,
10+
createPeakAccumulator,
11+
decodePeakMap,
12+
finishPeaks,
13+
} from "./peakMap.js";
14+
15+
function stereo(frames: Array<[number, number]>): Buffer {
16+
const buf = Buffer.alloc(frames.length * 8);
17+
frames.forEach(([l, r], i) => {
18+
buf.writeFloatLE(l, i * 8);
19+
buf.writeFloatLE(r, i * 8 + 4);
20+
});
21+
return buf;
22+
}
23+
24+
describe("peak accumulation", () => {
25+
it("keeps the absolute max of either channel per bin, un-normalized", () => {
26+
const acc = createPeakAccumulator(4, 0.5, 2);
27+
accumulatePeaks(
28+
acc,
29+
stereo([
30+
[0.1, -0.3],
31+
[0.2, 0],
32+
[0, 0.05],
33+
[-0.04, 0],
34+
]),
35+
);
36+
expect(finishPeaks(acc)).toEqual([0.3, 0.05]);
37+
});
38+
39+
it("carries a frame split across chunks", () => {
40+
const acc = createPeakAccumulator(2, 1, 2);
41+
const whole = stereo([
42+
[0.5, 0],
43+
[0, -0.9],
44+
]);
45+
accumulatePeaks(acc, whole.subarray(0, 5));
46+
accumulatePeaks(acc, whole.subarray(5));
47+
expect(finishPeaks(acc)).toEqual([0.9]);
48+
expect(acc.frame).toBe(2);
49+
});
50+
});
51+
52+
describe("peak accumulation, mono", () => {
53+
it("reads one channel per frame", () => {
54+
const acc = createPeakAccumulator(2, 1, 1);
55+
const buf = Buffer.alloc(8);
56+
buf.writeFloatLE(-1, 0);
57+
buf.writeFloatLE(0.5, 4);
58+
accumulatePeaks(acc, buf);
59+
expect(finishPeaks(acc)).toEqual([1]);
60+
});
61+
});
62+
63+
describe("buildPeakMapCacheKey", () => {
64+
it("changes when the file is replaced and never collides with a waveform entry", () => {
65+
const a = buildPeakMapCacheKey("a/talk.mp4", { size: 10, mtimeMs: 1 });
66+
expect(a).toBe(buildPeakMapCacheKey("a/talk.mp4", { size: 10, mtimeMs: 1 }));
67+
expect(a).not.toBe(buildPeakMapCacheKey("a/talk.mp4", { size: 11, mtimeMs: 1 }));
68+
expect(a).not.toBe(buildPeakMapCacheKey("a/talk.mp4", { size: 10, mtimeMs: 2 }));
69+
expect(a).toMatch(/^peaks-v2_a_talk\.mp4_10-1\.json$/);
70+
});
71+
});
72+
73+
describe("decodePeakMap", () => {
74+
const ffmpeg = findFfBinary("ffmpeg");
75+
it.skipIf(!ffmpeg)(
76+
"reads a 0 dBFS video track near 1.0 and a -12 dBFS one near 0.25",
77+
async () => {
78+
const dir = mkdtempSync(join(tmpdir(), "hf-peaks-"));
79+
try {
80+
const make = (name: string, volume: string) => {
81+
const file = join(dir, name);
82+
execFileSync(ffmpeg ?? "ffmpeg", [
83+
"-v",
84+
"error",
85+
"-f",
86+
"lavfi",
87+
"-i",
88+
"color=c=black:s=32x32:d=1",
89+
"-f",
90+
"lavfi",
91+
"-i",
92+
`sine=frequency=440:duration=1,volume=${volume}`,
93+
"-shortest",
94+
"-c:v",
95+
"libx264",
96+
"-c:a",
97+
"pcm_s16le",
98+
file,
99+
]);
100+
return file;
101+
};
102+
const loud = await decodePeakMap(make("loud.mov", "8"));
103+
const quiet = await decodePeakMap(make("quiet.mov", "2"));
104+
expect(Math.max(...loud)).toBeGreaterThan(0.95);
105+
expect(Math.max(...quiet)).toBeLessThan(0.3);
106+
expect(loud.length).toBeGreaterThanOrEqual(19);
107+
} finally {
108+
rmSync(dir, { recursive: true, force: true });
109+
}
110+
},
111+
);
112+
113+
it.skipIf(!ffmpeg)(
114+
"decodes the probed first audio stream when a louder stereo stream follows it",
115+
async () => {
116+
const dir = mkdtempSync(join(tmpdir(), "hf-peaks-"));
117+
try {
118+
const file = join(dir, "multi.mkv");
119+
execFileSync(ffmpeg ?? "ffmpeg", [
120+
"-v",
121+
"error",
122+
"-f",
123+
"lavfi",
124+
"-i",
125+
"aevalsrc=0.1*sin(2*PI*440*t):s=48000:d=1",
126+
"-f",
127+
"lavfi",
128+
"-i",
129+
"aevalsrc=0.9*sin(2*PI*440*t)|0.9*sin(2*PI*440*t):s=48000:d=1",
130+
"-map",
131+
"0:a",
132+
"-map",
133+
"1:a",
134+
"-c:a",
135+
"pcm_f32le",
136+
"-disposition:a",
137+
"0",
138+
file,
139+
]);
140+
const peaks = await decodePeakMap(file);
141+
expect(peaks.length).toBeGreaterThanOrEqual(19);
142+
expect(peaks.length).toBeLessThanOrEqual(21);
143+
expect(Math.max(...peaks)).toBeCloseTo(0.1, 2);
144+
} finally {
145+
rmSync(dir, { recursive: true, force: true });
146+
}
147+
},
148+
);
149+
});
Lines changed: 131 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,131 @@
1+
import { execFile, spawn } from "node:child_process";
2+
import { promisify } from "node:util";
3+
import { findFfBinary } from "@hyperframes/parsers/ff-binaries";
4+
5+
export const PEAK_BIN_SECONDS = 0.05;
6+
const PEAK_SAMPLE_RATE = 48_000;
7+
const execFileAsync = promisify(execFile);
8+
const PEAK_MAP_CACHE_VERSION = "peaks-v2";
9+
10+
/**
11+
* Running max-abs per bin over interleaved f32le at the source's own channel
12+
* count. Native channels, not `-ac 2`: ffmpeg's mono upmix pans at -3 dB,
13+
* which would hide a clip that really does reach 0 dBFS.
14+
*/
15+
export interface PeakAccumulator {
16+
bins: number[];
17+
frame: number;
18+
carry: Buffer;
19+
framesPerBin: number;
20+
channels: number;
21+
}
22+
23+
export function createPeakAccumulator(
24+
sampleRate = PEAK_SAMPLE_RATE,
25+
binSeconds = PEAK_BIN_SECONDS,
26+
channels = 2,
27+
): PeakAccumulator {
28+
return {
29+
bins: [],
30+
frame: 0,
31+
carry: Buffer.alloc(0),
32+
framesPerBin: Math.max(1, Math.round(sampleRate * binSeconds)),
33+
channels: Math.max(1, channels),
34+
};
35+
}
36+
37+
function framePeak(data: Buffer, offset: number, channels: number): number {
38+
let peak = 0;
39+
for (let ch = 0; ch < channels; ch++) {
40+
peak = Math.max(peak, Math.abs(data.readFloatLE(offset + ch * 4)));
41+
}
42+
return peak;
43+
}
44+
45+
export function accumulatePeaks(acc: PeakAccumulator, chunk: Buffer): void {
46+
const data = acc.carry.length > 0 ? Buffer.concat([acc.carry, chunk]) : chunk;
47+
const frameBytes = 4 * acc.channels;
48+
const whole = data.length - (data.length % frameBytes);
49+
for (let offset = 0; offset < whole; offset += frameBytes) {
50+
const peak = framePeak(data, offset, acc.channels);
51+
const bin = Math.floor(acc.frame / acc.framesPerBin);
52+
if (acc.bins.length <= bin) acc.bins.push(0);
53+
if (peak > (acc.bins[bin] ?? 0)) acc.bins[bin] = peak;
54+
acc.frame += 1;
55+
}
56+
acc.carry = Buffer.from(data.subarray(whole));
57+
}
58+
59+
/** Linear sample peak per bin, NOT normalized — the level is the point. */
60+
export function finishPeaks(acc: PeakAccumulator): number[] {
61+
return acc.bins.map((peak) => Number(peak.toFixed(4)));
62+
}
63+
64+
async function probeChannels(mediaPath: string): Promise<number> {
65+
const { stdout } = await execFileAsync(
66+
findFfBinary("ffprobe") ?? "ffprobe",
67+
[
68+
"-v",
69+
"error",
70+
"-select_streams",
71+
"a:0",
72+
"-show_entries",
73+
"stream=channels",
74+
"-of",
75+
"csv=p=0",
76+
"--",
77+
mediaPath,
78+
],
79+
{ encoding: "utf8", timeout: 30_000, windowsHide: true },
80+
);
81+
const channels = Number.parseInt(stdout.trim(), 10);
82+
if (!Number.isFinite(channels) || channels < 1) throw new Error("no audio stream");
83+
return channels;
84+
}
85+
86+
export async function decodePeakMap(mediaPath: string): Promise<number[]> {
87+
const channels = await probeChannels(mediaPath);
88+
return new Promise((resolvePromise, reject) => {
89+
const proc = spawn(
90+
findFfBinary("ffmpeg") ?? "ffmpeg",
91+
[
92+
"-v",
93+
"error",
94+
"-i",
95+
mediaPath,
96+
"-map",
97+
"0:a:0",
98+
"-vn",
99+
"-ar",
100+
String(PEAK_SAMPLE_RATE),
101+
"-f",
102+
"f32le",
103+
"pipe:1",
104+
],
105+
{ stdio: ["ignore", "pipe", "ignore"], windowsHide: true },
106+
);
107+
const acc = createPeakAccumulator(PEAK_SAMPLE_RATE, PEAK_BIN_SECONDS, channels);
108+
proc.stdout?.on("data", (chunk: Buffer) => accumulatePeaks(acc, chunk));
109+
proc.on("close", (code) => {
110+
if (code !== 0) {
111+
reject(new Error(`ffmpeg failed decoding audio (exit ${code})`));
112+
return;
113+
}
114+
if (acc.frame === 0) {
115+
reject(new Error("ffmpeg produced no audio samples"));
116+
return;
117+
}
118+
resolvePromise(finishPeaks(acc));
119+
});
120+
proc.on("error", reject);
121+
});
122+
}
123+
124+
/** Keyed on content (size + mtime) like the waveform cache, under its own version prefix. */
125+
export function buildPeakMapCacheKey(
126+
assetPath: string,
127+
fingerprint: { size: number; mtimeMs: number },
128+
): string {
129+
const name = assetPath.replace(/[/\\]/g, "_");
130+
return `${PEAK_MAP_CACHE_VERSION}_${name}_${fingerprint.size}-${Math.round(fingerprint.mtimeMs)}.json`;
131+
}

0 commit comments

Comments
 (0)