Skip to content

Commit bdecae0

Browse files
feat(engine): render speed ramps with the same source-time mapping as the preview (#4143)
* feat(engine): render speed ramps with the same source-time mapping as the preview The frame extractor, frame lookup table, coverage check, compile-time duration and audio mixer read a clip's rate lane through sourceTimeAt, so a rendered frame at t shows the source frame the preview shows. Audio bakes a ramp as consecutive source slices stretched by their mean rate, keeping pitch. The compile-time attribute reader now handles single-quoted and entity-encoded JSON attributes. Adds a speed-ramp regression fixture. * test(producer): regenerate the speed-ramp golden audio with the current engine The stored baseline predates the per-slice audio fix, so its audio drifted 1 to 3 percent in pitch from the expected source-time track and the CI render correlated at 0.61. Rendered in the CI test image, the new baseline tracks the expected pitch to 0.1 percent median, and the compare now reports visual 0 failed frames and audio correlation 1.000. * refactor(core): drop data-preserve-pitch, pitch is preserved by construction The engine always time-stretches audio with atempo, so an opt-out read only by the preview made preview and render disagree. Remove the attribute and its reader so both paths keep pitch.
1 parent 45f92e5 commit bdecae0

22 files changed

Lines changed: 502 additions & 87 deletions

‎packages/core/src/compiler/timingCompiler.test.ts‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -468,3 +468,29 @@ describe("shouldClampResolvedMediaDuration", () => {
468468
expect(shouldClampResolvedMediaDuration("audio", 5, 1)).toBe(true);
469469
});
470470
});
471+
472+
describe("rate lane in compiled media", () => {
473+
const lane = {
474+
version: 1,
475+
lanes: [
476+
{
477+
target: "rate",
478+
points: [
479+
{ t: 0, v: 1 },
480+
{ t: 2, v: 3 },
481+
],
482+
},
483+
],
484+
};
485+
const encodings = [
486+
["single-quoted JSON", `data-automation='${JSON.stringify(lane)}'`],
487+
["entity-encoded JSON", `data-automation="${JSON.stringify(lane).replace(/"/g, """)}"`],
488+
];
489+
490+
it.each(encodings)("hands a %s rate lane to the unresolved element", (_name, attr) => {
491+
const { unresolved } = compileTimingAttrs(
492+
`<video id="v" src="a.mp4" data-start="0" ${attr}></video>`,
493+
);
494+
expect(unresolved[0]?.playbackRate).toMatchObject({ target: "rate" });
495+
});
496+
});

‎packages/core/src/compiler/timingCompiler.ts‎

Lines changed: 27 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -21,9 +21,10 @@
2121
import { parseNumeric } from "@hyperframes/parsers/composition-contract";
2222
import {
2323
parseStrictFiniteTimingNumber,
24-
readElementPlaybackRate,
24+
readElementRateSpec,
2525
readMediaStart,
2626
} from "../runtime/playbackRate.js";
27+
import type { RateSpec } from "../speedRamp.js";
2728
// ── Types ────────────────────────────────────────────────────────────────
2829

2930
export interface UnresolvedElement {
@@ -34,7 +35,7 @@ export interface UnresolvedElement {
3435
end?: number;
3536
duration?: number;
3637
mediaStart: number;
37-
playbackRate: number;
38+
playbackRate: RateSpec;
3839
compositionSrc?: string;
3940
}
4041

@@ -50,7 +51,7 @@ export interface ResolvedMediaElement {
5051
start: number;
5152
duration: number;
5253
mediaStart: number;
53-
playbackRate: number;
54+
playbackRate: RateSpec;
5455
loop: boolean;
5556
}
5657

@@ -91,8 +92,25 @@ function getAttr(tag: string, attr: string): string | null {
9192
// made compileTag believe a Studio-stamped `data-hf-id`-only element already
9293
// had an `id`, so it skipped its `hf-video-N` injection — leaving the element
9394
// with no real `el.id`, which the render pipeline keys off of (blank wash).
94-
const match = tag.match(new RegExp(`(?<![\\w-])${attr}=["']([^"']+)["']`));
95-
return match ? (match[1] ?? null) : null;
95+
const match = tag.match(new RegExp(`(?<![\\w-])${attr}=(?:"([^"]+)"|'([^']+)')`));
96+
return match ? (match[1] ?? match[2] ?? null) : null;
97+
}
98+
99+
/** An attribute reader over tag source that decodes the entities the DOM would, for JSON-valued attributes. */
100+
function tagAttrReader(tag: string): Pick<Element, "getAttribute"> {
101+
return {
102+
getAttribute: (name) => {
103+
const raw = getAttr(tag, name);
104+
return raw && name === "data-automation"
105+
? raw
106+
.replace(/&quot;/g, '"')
107+
.replace(/&#39;/g, "'")
108+
.replace(/&lt;/g, "<")
109+
.replace(/&gt;/g, ">")
110+
.replace(/&amp;/g, "&")
111+
: raw;
112+
},
113+
};
96114
}
97115

98116
function hasAttr(tag: string, attr: string): boolean {
@@ -228,9 +246,9 @@ function compileTag(
228246
startStr = "0";
229247
}
230248
const start = parseNumeric(startStr);
231-
const attrReader = { getAttribute: (name: string) => getAttr(result, name) };
249+
const attrReader = tagAttrReader(result);
232250
const mediaStart = readMediaStart(attrReader);
233-
const playbackRate = readElementPlaybackRate(attrReader);
251+
const playbackRate = readElementRateSpec(attrReader);
234252

235253
// 1. Compute data-end from data-start + data-duration. Skip relative id-refs.
236254
if (!hasAttr(result, "data-end")) {
@@ -374,7 +392,7 @@ export function extractResolvedMedia(html: string): ResolvedMediaElement[] {
374392

375393
const isVideo = /^<video/i.test(tag);
376394
const startStr = getAttr(tag, "data-start");
377-
const attrReader = { getAttribute: (name: string) => getAttr(tag, name) };
395+
const attrReader = tagAttrReader(tag);
378396

379397
resolved.push({
380398
id,
@@ -383,7 +401,7 @@ export function extractResolvedMedia(html: string): ResolvedMediaElement[] {
383401
start: parseNumeric(startStr) ?? 0,
384402
duration,
385403
mediaStart: readMediaStart(attrReader),
386-
playbackRate: readElementPlaybackRate(attrReader),
404+
playbackRate: readElementRateSpec(attrReader),
387405
loop: hasAttr(tag, "loop"),
388406
});
389407
}

‎packages/core/src/index.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -321,12 +321,15 @@ export {
321321
export { createRuntimeStartTimeResolver } from "./runtime/startResolver.js";
322322
export {
323323
normalizePlaybackRate,
324+
normalizeRateSpec,
324325
parseStrictFiniteTimingNumber,
325326
readElementPlaybackRate,
327+
readElementRateSpec,
326328
readMediaStart,
327329
resolveNaturalMediaTimelineDuration,
328330
resolveNaturalMediaTimelineDurationFromValues,
329331
} from "./runtime/playbackRate.js";
332+
export { shiftRateLane, sourceTimeAt, timeAtSourceTime, type RateSpec } from "./speedRamp.js";
330333

331334
// Variable validation (CLI / tooling-side)
332335
export {

‎packages/core/src/runtime/media.test.ts‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -380,7 +380,6 @@ describe("syncRuntimeMedia", () => {
380380
expect(clip.el.currentTime).toBeCloseTo(3.641, 2);
381381
syncRuntimeMedia({ clips: [clip], timeSeconds: 3, playing: true, playbackRate: 1 });
382382
expect(clip.el.playbackRate).toBeCloseTo(3, 5);
383-
expect(clip.el.preservesPitch).toBe(true);
384383
});
385384
});
386385

‎packages/core/src/runtime/media.ts‎

Lines changed: 1 addition & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -2,13 +2,7 @@ import { swallow } from "./diagnostics";
22
import { interpolateVolumeGain, type VolumeKeyframe } from "./mediaVolumeEnvelope.js";
33
import { elementVolumeLaneGain } from "./audioAutomationVolume.js";
44
import { readElementPlaybackRate, readElementRateSpec, readMediaStart } from "./playbackRate.js";
5-
import {
6-
rateAt,
7-
readPreservePitch,
8-
sourceTimeAt,
9-
timeAtSourceTime,
10-
type RateSpec,
11-
} from "../speedRamp.js";
5+
import { rateAt, sourceTimeAt, timeAtSourceTime, type RateSpec } from "../speedRamp.js";
126
import { clampAudioGain } from "../audioGain.js";
137
import { isMemberGroupHidden } from "../audioGroups.js";
148
import { findInjectedRenderFrame } from "./renderFrameSibling.js";
@@ -386,7 +380,6 @@ export function syncRuntimeMedia(params: {
386380
try {
387381
// Per-element rate × global transport rate
388382
el.playbackRate = rateAt(clipRate, params.timeSeconds - clip.start) * params.playbackRate;
389-
el.preservesPitch = readPreservePitch(el);
390383
} catch (err) {
391384
// ignore unsupported playbackRate
392385
swallow("runtime.media.site1", err);

‎packages/core/src/runtime/playbackRate.ts‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,11 @@ export function readElementPlaybackRate(el: Pick<Element, "getAttribute">): numb
2424
return normalizePlaybackRate(raw);
2525
}
2626

27+
/** A constant rate clamped to the shared range; a lane is already normalised by its parser. */
28+
export function normalizeRateSpec(spec: RateSpec | undefined): RateSpec {
29+
return typeof spec === "object" ? spec : normalizePlaybackRate(spec ?? 1);
30+
}
31+
2732
/** The clip's rate: its `rate` lane when present, otherwise the constant rate. */
2833
export function readElementRateSpec(el: Pick<Element, "getAttribute">): RateSpec {
2934
return resolveRateSpec(el.getAttribute("data-automation"), readElementPlaybackRate(el));

‎packages/core/src/speedRamp.test.ts‎

Lines changed: 27 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,8 @@ import {
44
SPEED_PRESETS,
55
parseRateLane,
66
rateAt,
7-
readPreservePitch,
87
resolveRateSpec,
8+
shiftRateLane,
99
sourceTimeAt,
1010
speedPresetLane,
1111
timeAtSourceTime,
@@ -71,6 +71,31 @@ describe("rateAt", () => {
7171
});
7272
});
7373

74+
describe("shiftRateLane", () => {
75+
it("sees the lane from dt seconds in: integrating from 1s of a 4s 1x to 3x ramp", () => {
76+
const lane = ramp([
77+
[0, 1],
78+
[4, 3],
79+
]);
80+
// source(3) - source(1) = 4.6586 - 1.1508
81+
expect(sourceTimeAt(shiftRateLane(lane, 1), 2)).toBeCloseTo(3.5078, 2);
82+
});
83+
});
84+
85+
describe("shiftRateLane with a shaped segment", () => {
86+
it("keeps the curve of a segment cut in the middle: shifted integral equals the original's difference", () => {
87+
const lane: HfAutomationLane = {
88+
target: RATE_TARGET,
89+
points: [
90+
{ t: 0, v: 1, curve: 0.8 },
91+
{ t: 4, v: 3 },
92+
],
93+
};
94+
const expected = sourceTimeAt(lane, 4) - sourceTimeAt(lane, 1);
95+
expect(sourceTimeAt(shiftRateLane(lane, 1), 3)).toBeCloseTo(expected, 2);
96+
});
97+
});
98+
7499
describe("lane parsing", () => {
75100
const attr = JSON.stringify({
76101
version: 1,
@@ -88,18 +113,12 @@ describe("lane parsing", () => {
88113
});
89114
});
90115

91-
describe("presets and pitch", () => {
116+
describe("presets", () => {
92117
it("stretches every preset over the clip and keeps it in range", () => {
93118
for (const { id } of SPEED_PRESETS) {
94119
const lane = speedPresetLane(id, 8);
95120
expect(lane.points[lane.points.length - 1]!.t).toBe(8);
96121
for (const p of lane.points) expect(p.v).toBeGreaterThanOrEqual(0.1);
97122
}
98123
});
99-
100-
it("preserves pitch unless the clip opts out", () => {
101-
const el = (v: string | null) => ({ getAttribute: () => v });
102-
expect(readPreservePitch(el(null))).toBe(true);
103-
expect(readPreservePitch(el("false"))).toBe(false);
104-
});
105124
});

‎packages/core/src/speedRamp.ts‎

Lines changed: 19 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,25 @@ import { MAX_PLAYBACK_RATE, MIN_PLAYBACK_RATE } from "./playbackRateBounds.js";
77
/** A constant multiplier, or a lane whose `v` is the multiplier over clip-local time. */
88
export type RateSpec = number | HfAutomationLane;
99

10-
const PRESERVE_PITCH_ATTR = "data-preserve-pitch";
10+
const SHIFT_RESAMPLES = 16;
11+
12+
/**
13+
* The lane as seen from `dt` seconds into the clip: what a renderer sees after it trims the clip's start.
14+
* A shaped segment (`curve`, `viaX`) cut in the middle is resampled, since its shape belongs to its left point.
15+
*/
16+
export function shiftRateLane(spec: RateSpec, dt: number): RateSpec {
17+
if (typeof spec === "number" || dt === 0) return spec;
18+
const next = spec.points.findIndex((p) => p.t > dt);
19+
const later = next < 0 ? [] : spec.points.slice(next).map((p) => ({ ...p, t: p.t - dt }));
20+
const left = next > 0 ? spec.points[next - 1] : undefined;
21+
const shaped = left !== undefined && (left.curve || left.viaX !== undefined);
22+
const cut = shaped && next > 0 ? spec.points[next]!.t - dt : 0;
23+
const samples = Array.from({ length: shaped ? SHIFT_RESAMPLES - 1 : 0 }, (_, k) => {
24+
const t = ((k + 1) * cut) / SHIFT_RESAMPLES;
25+
return { t, v: rateAt(spec, dt + t) };
26+
});
27+
return { ...spec, points: [{ t: 0, v: rateAt(spec, dt) }, ...samples, ...later] };
28+
}
1129

1230
const CELLS_PER_SEGMENT = 48;
1331

@@ -114,11 +132,6 @@ export function resolveRateSpec(
114132
return parseRateLane(automationAttr) ?? constant;
115133
}
116134

117-
/** Pitch is preserved unless the clip opts out with `data-preserve-pitch="false"`. */
118-
export function readPreservePitch(el: Pick<Element, "getAttribute">): boolean {
119-
return el.getAttribute(PRESERVE_PITCH_ATTR) !== "false";
120-
}
121-
122135
interface RatePreset {
123136
id: string;
124137
label: string;

‎packages/engine/src/services/audioMixer.test.ts‎

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -390,6 +390,95 @@ describe("processCompositionAudio", () => {
390390
expect(runFfmpegMock.mock.calls[0]?.[0]).toEqual(expect.arrayContaining(["-af", filter]));
391391
});
392392

393+
it("bakes a rate lane as concatenated source slices whose tempo is the mean rate over each slice", async () => {
394+
const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-"));
395+
const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-"));
396+
tempDirs.push(baseDir, workDir);
397+
writeFileSync(join(baseDir, "timecode.wav"), "stub");
398+
const rate = {
399+
target: "rate",
400+
points: [
401+
{ t: 0, v: 2 },
402+
{ t: 1, v: 2 },
403+
],
404+
};
405+
406+
await processCompositionAudio(
407+
[
408+
{
409+
id: "timecode",
410+
src: "timecode.wav",
411+
start: 0,
412+
end: 1,
413+
mediaStart: 0,
414+
playbackRate: rate,
415+
layer: 0,
416+
volume: 1,
417+
type: "audio",
418+
},
419+
],
420+
baseDir,
421+
workDir,
422+
join(baseDir, "out.m4a"),
423+
1,
424+
);
425+
426+
const args: string[] = runFfmpegMock.mock.calls[0]?.[0] ?? [];
427+
const graph = args[args.indexOf("-filter_complex") + 1] ?? "";
428+
// a constant 2x lane: 1s of composition consumes 2s of source, in 4 quarter-second slices
429+
expect(args).toEqual(expect.arrayContaining(["-t", "2", "-map", "[out]"]));
430+
expect(graph).toContain("asplit=4");
431+
expect(graph).toContain(
432+
"[s0]atrim=start=0:end=0.5,asetpts=PTS-STARTPTS,atempo=2,apad,asetpts=N/SR/TB,atrim=0:0.25[t0]",
433+
);
434+
expect(graph).toContain(
435+
"[s3]atrim=start=1.5:end=2,asetpts=PTS-STARTPTS,atempo=2,apad,asetpts=N/SR/TB,atrim=0:0.25[t3]",
436+
);
437+
expect(graph).toContain("concat=n=4:v=0:a=1");
438+
});
439+
440+
it("stretches each slice of a non-constant lane by its mean rate, and trims a video's audio to the consumed span", async () => {
441+
const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-"));
442+
const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-"));
443+
tempDirs.push(baseDir, workDir);
444+
writeFileSync(join(baseDir, "clip.mp4"), "stub");
445+
const rate = {
446+
target: "rate",
447+
points: [
448+
{ t: 0, v: 1 },
449+
{ t: 1, v: 3 },
450+
],
451+
};
452+
453+
await processCompositionAudio(
454+
[
455+
{
456+
id: "clip",
457+
src: "clip.mp4",
458+
start: 0,
459+
end: 1,
460+
mediaStart: 0,
461+
playbackRate: rate,
462+
layer: 0,
463+
volume: 1,
464+
type: "video",
465+
},
466+
],
467+
baseDir,
468+
workDir,
469+
join(baseDir, "out.m4a"),
470+
1,
471+
);
472+
473+
const args: string[] = runFfmpegMock.mock.calls[0]?.[0] ?? [];
474+
const graph = args[args.indexOf("-filter_complex") + 1] ?? "";
475+
// geometric 1x to 3x over 1s consumes (3-1)/ln 3 = 1.8205 source seconds; quarter-second 0 has mean tempo 1.1508
476+
expect(Number(args[args.indexOf("-t") + 1])).toBeCloseTo(1.8205, 2);
477+
expect(
478+
Number(graph.match(/\[s0\]atrim=[^,]*,asetpts=PTS-STARTPTS,atempo=([\d.]+)/)?.[1]),
479+
).toBeCloseTo(1.1508, 2);
480+
});
481+
393482
it("keeps automation on authored timeline time after constant retiming", async () => {
394483
const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-"));
395484
const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-"));

0 commit comments

Comments
 (0)