Skip to content

Commit 5cc2d9a

Browse files
committed
fix(producer): retain scene trims for discovered speed ramps
1 parent 692ec34 commit 5cc2d9a

6 files changed

Lines changed: 149 additions & 45 deletions

File tree

‎docs/reference/speed-ramps.mdx‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,9 @@ which the preview and the renderer both use, so the render shows the frames the
3939
preview showed. Set `data-duration` on the clip to fix how long it holds the
4040
timeline; without it the length follows from the source and the speed.
4141

42+
If a browser probe discovers a rate lane on an existing clip, export preserves
43+
the scene's trim in both the ramp's position and the source media in-point.
44+
4245
Pitch is kept. The preview leaves the browser's default pitch correction on.
4346
The render time-stretches audio in short constant-speed slices (a quarter
4447
second, coarser on clips longer than a minute), which approximates the curve

‎packages/engine/src/index.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -262,6 +262,7 @@ export {
262262
parseAudioElements,
263263
parseMasterFxChain,
264264
processCompositionAudio,
265+
videoAudioId,
265266
} from "./services/audioMixer.js";
266267
export { audioFailureDiagnostics } from "./services/audioFailureDiagnostic.js";
267268
export { cloneCaptureWarning, cloneCaptureWarnings } from "./services/captureWarning.js";

‎packages/engine/src/services/audioMixer.ts‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -535,6 +535,10 @@ export function parseMasterFxChain(html: string): string | undefined {
535535
return readMasterFxChain(parseHTML(unwrapTemplate(html)).document);
536536
}
537537

538+
export function videoAudioId(videoId: string): string {
539+
return `${videoId}-audio`;
540+
}
541+
538542
export function parseAudioElements(html: string): AudioElement[] {
539543
const elements: AudioElement[] = [];
540544
const { document } = parseHTML(unwrapTemplate(html));
@@ -632,7 +636,7 @@ export function parseAudioElements(html: string): AudioElement[] {
632636
const src = resolveMediaElementSrc(el);
633637
if (!id || !src || isHidden(el) || memberGroupHidden(el)) continue;
634638
if (isKnownInactiveTimelineWindow(el, resolveStart(el))) continue;
635-
elements.push(build(el, `${id}-audio`, src, "video", true));
639+
elements.push(build(el, videoAudioId(id), src, "video", true));
636640
}
637641

638642
return elements;

‎packages/producer/src/services/render/stages/extractVideosStage.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@ import {
5050
resolveProjectRelativeSrc,
5151
runVideoExtractionWithRetry,
5252
safeVideoExtractionSourceIdentity,
53+
videoAudioId,
5354
} from "@hyperframes/engine";
5455
import {
5556
collectVideoMetadataHints,
@@ -573,7 +574,7 @@ export function appendAutoDetectedVideoAudio(
573574
const video = composition.videos.find((v) => v.id === ext.videoId);
574575
if (!video || !video.hasAudio || video.hidden || existingAudioSrcs.has(video.src)) continue;
575576
composition.audios.push({
576-
id: `${video.id}-audio`,
577+
id: videoAudioId(video.id),
577578
src: video.src,
578579
start: video.start,
579580
end: video.end,

‎packages/producer/src/services/render/stages/probeStage.test.ts‎

Lines changed: 109 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,13 @@ import { mkdtempSync, rmSync } from "node:fs";
33
import { tmpdir } from "node:os";
44
import { join } from "node:path";
55
import type { RateSpec } from "@hyperframes/core";
6+
import { parseAudioElements, videoAudioId } from "../../../../../engine/src/services/audioMixer.js";
7+
import {
8+
parseVideoElements,
9+
parseImageElements,
10+
} from "../../../../../engine/src/services/videoFrameExtractor.js";
11+
import { resolveReferencedStart } from "../../../../../engine/src/services/referenceResolver.js";
12+
import { collectRenderMedia } from "../../renderMediaCollector.js";
613

714
const workDir = mkdtempSync(join(tmpdir(), "hf-stage-test-"));
815
afterAll(() => rmSync(workDir, { recursive: true, force: true }));
@@ -100,6 +107,11 @@ mock.module("../../assetMediaType.js", () => ({
100107
}));
101108

102109
mock.module("@hyperframes/engine", () => ({
110+
parseAudioElements,
111+
parseVideoElements,
112+
parseImageElements,
113+
resolveReferencedStart,
114+
videoAudioId,
103115
createCaptureSession: async (
104116
_url: string,
105117
_dir: string,
@@ -479,49 +491,124 @@ describe("runProbeStage — forceScreenshot threading", () => {
479491
},
480492
);
481493

482-
it.each(runtimeRates)(
483-
"preserves a collected speed ramp during browser rate %j reconciliation",
484-
async (browserRate) => {
494+
it.each([
495+
{ tagName: "video", pageStart: 1, expected: 3 },
496+
{ tagName: "video", pageStart: 3, expected: 5 },
497+
{ tagName: "audio", pageStart: 1, expected: 3 },
498+
{ tagName: "audio", pageStart: 3, expected: 5 },
499+
])(
500+
"applies a scene trim when adopting a runtime $tagName lane (page start $pageStart)",
501+
async ({ tagName, pageStart, expected }) => {
485502
resetRetryMocks();
503+
const playbackRate = {
504+
target: "rate",
505+
points: [
506+
{ t: 0, v: 1 },
507+
{ t: 2, v: 1 },
508+
{ t: 4, v: 2 },
509+
],
510+
} satisfies RateSpec;
486511
browserMediaResults = [
487512
{
488513
id: "clip",
489-
tagName: "video",
514+
tagName,
490515
src: "clip.mp4",
491516
start: 0,
492-
end: 3,
493-
duration: 3,
494-
mediaStart: 0,
495-
playbackRate: browserRate,
517+
end: 5,
518+
duration: 5,
519+
mediaStart: pageStart,
520+
playbackRate,
496521
loop: false,
497-
hasAudio: false,
522+
hasAudio: true,
498523
volume: 1,
499524
muted: false,
500525
},
501526
];
502527
const { runProbeStage } = await import("./probeStage.js");
503528
const input = makeProbeInput({});
504-
const playbackRate = {
529+
input.compiled.html = `<div data-composition-id="scene" data-start="5" data-duration="3" data-playback-start="2">
530+
<${tagName} id="clip" src="clip.mp4" data-start="0" data-duration="5" data-playback-start="1" data-playback-rate="2" data-has-audio="true"></${tagName}>
531+
</div>`;
532+
const collected = collectRenderMedia(input.compiled.html);
533+
input.composition.videos = collected.videos;
534+
input.composition.audios = collected.audios;
535+
const before = tagName === "video" ? collected.videos : collected.audios;
536+
expect(before[0]).toMatchObject({ start: 5, end: 8, mediaStart: 5, playbackRate: 2 });
537+
538+
await runProbeStage(input);
539+
540+
const shiftedRate = {
505541
target: "rate",
506542
points: [
507-
{ t: 0, v: 1.25 },
543+
{ t: 0, v: 1 },
508544
{ t: 2, v: 2 },
509545
],
510-
} satisfies RateSpec;
511-
input.composition.videos.push({
512-
id: "clip",
513-
src: "clip.mp4",
514-
start: 0,
515-
end: 3,
516-
mediaStart: 0,
517-
playbackRate,
518-
loop: false,
519-
hasAudio: false,
546+
};
547+
const media = tagName === "video" ? input.composition.videos : input.composition.audios;
548+
expect(media[0]).toMatchObject({
549+
start: 5,
550+
end: 8,
551+
mediaStart: expected,
552+
playbackRate: shiftedRate,
520553
});
554+
if (tagName === "video") {
555+
expect(input.composition.audios[0]).toMatchObject({
556+
mediaStart: expected,
557+
playbackRate: shiftedRate,
558+
});
559+
}
560+
},
561+
);
562+
563+
it.each(
564+
runtimeRates.flatMap((playbackRate) => [
565+
{ tagName: "video", playbackRate },
566+
{ tagName: "audio", playbackRate },
567+
]),
568+
)(
569+
"preserves a collected $tagName speed ramp during reconciliation (browser rate: $playbackRate)",
570+
async ({ tagName, playbackRate }) => {
571+
resetRetryMocks();
572+
browserMediaResults = [
573+
{
574+
id: "clip",
575+
tagName,
576+
src: "clip.mp4",
577+
start: 0,
578+
end: 5,
579+
duration: 5,
580+
mediaStart: 1,
581+
playbackRate,
582+
loop: false,
583+
hasAudio: true,
584+
volume: 1,
585+
muted: false,
586+
},
587+
];
588+
const { runProbeStage } = await import("./probeStage.js");
589+
const input = makeProbeInput({});
590+
const authoredRate = {
591+
target: "rate",
592+
points: [
593+
{ t: 0, v: 1 },
594+
{ t: 2, v: 1 },
595+
{ t: 4, v: 2 },
596+
],
597+
} satisfies RateSpec;
598+
input.compiled.html = `<div data-composition-id="scene" data-start="5" data-duration="3" data-playback-start="2">
599+
<${tagName} id="clip" src="clip.mp4" data-start="0" data-duration="5" data-playback-start="1" data-has-audio="true" data-automation='${JSON.stringify({ version: 1, lanes: [authoredRate] })}'></${tagName}>
600+
</div>`;
601+
const collected = collectRenderMedia(input.compiled.html);
602+
input.composition.videos = collected.videos;
603+
input.composition.audios = collected.audios;
604+
const before = structuredClone({ videos: collected.videos, audios: collected.audios });
605+
const media = tagName === "video" ? collected.videos : collected.audios;
606+
expect(media[0]).toMatchObject({ mediaStart: 3 });
521607

522608
await runProbeStage(input);
523609

524-
expect(input.composition.videos[0]?.playbackRate).toEqual(playbackRate);
610+
expect(input.composition.videos).toEqual(before.videos);
611+
expect(input.composition.audios).toEqual(before.audios);
525612
},
526613
);
527614

‎packages/producer/src/services/render/stages/probeStage.ts‎

Lines changed: 29 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -43,8 +43,17 @@ import {
4343
initializeSession,
4444
isTransientBrowserError,
4545
probeBeginFrameLiveness,
46+
videoAudioId,
4647
} from "@hyperframes/engine";
47-
import { MEDIA_RENDER_ID_ATTR, fpsToNumber, readMediaStart } from "@hyperframes/core";
48+
import {
49+
MEDIA_RENDER_ID_ATTR,
50+
fpsToNumber,
51+
normalizeRateSpec,
52+
readMediaStart,
53+
shiftRateLane,
54+
sourceTimeAt,
55+
timeAtSourceTime,
56+
} from "@hyperframes/core";
4857
import { extractMediaSrcMutations } from "@hyperframes/parsers";
4958
import type { BrowserMediaElement, CompiledComposition } from "../../htmlCompiler.js";
5059
import {
@@ -193,28 +202,39 @@ function authoredMediaStarts(html: string): Map<string, number> {
193202
}
194203

195204
function isVideoAudio(audio: CompositionMetadata["audios"][number], videoId: string): boolean {
196-
return audio.type === "video" && audio.id === `${videoId}-audio`;
205+
return audio.type === "video" && audio.id === videoAudioId(videoId);
197206
}
198207

199208
/**
200-
* Take a start point the page changed by moving the collected one by the same amount, so the
201-
* in-point its scene added survives, and a video's sound track moves with its picture.
209+
* Adopt browser timing without losing the scene trim the collector applied to picture and sound.
202210
*/
203-
function adoptRuntimeMediaStart(
211+
function adoptRuntimeMediaTiming(
204212
composition: Pick<CompositionMetadata, "videos" | "audios">,
205213
el: BrowserMediaElement,
206214
authored: number,
207215
): void {
208216
const shift = el.mediaStart - authored;
209-
if (el.mediaStart <= 0 || Math.abs(shift) <= BROWSER_MEDIA_EPSILON) return;
210217
const entries =
211218
el.tagName === "video"
212219
? [
213220
...composition.videos.filter((video) => video.id === el.id),
214221
...composition.audios.filter((audio) => isVideoAudio(audio, el.id)),
215222
]
216223
: composition.audios.filter((audio) => audio.type !== "video" && audio.id === el.id);
217-
for (const entry of entries) entry.mediaStart += shift;
224+
for (const entry of entries) {
225+
// Collected lanes already carry their scene trim. Only newly discovered lanes need it.
226+
if (typeof entry.playbackRate !== "object") {
227+
if (typeof el.playbackRate === "object") {
228+
const consumed = Math.max(0, entry.mediaStart - authored);
229+
const head = timeAtSourceTime(normalizeRateSpec(entry.playbackRate), consumed);
230+
entry.mediaStart += sourceTimeAt(el.playbackRate, head) - consumed;
231+
entry.playbackRate = shiftRateLane(el.playbackRate, head);
232+
} else {
233+
entry.playbackRate = el.playbackRate;
234+
}
235+
}
236+
if (el.mediaStart > 0 && Math.abs(shift) > BROWSER_MEDIA_EPSILON) entry.mediaStart += shift;
237+
}
218238
}
219239

220240
function reconcileBrowserMediaEnd(
@@ -596,20 +616,11 @@ export async function runProbeStage(input: ProbeStageInput): Promise<ProbeStageR
596616
sourceChanged,
597617
el.durationInferred,
598618
);
599-
adoptRuntimeMediaStart(
619+
adoptRuntimeMediaTiming(
600620
composition,
601621
el,
602622
authoredStarts.get(el.id) ?? existing.mediaStart,
603623
);
604-
// A collected rate lane already includes scene trimming; retain its phase.
605-
if (typeof existing.playbackRate !== "object") {
606-
existing.playbackRate = el.playbackRate;
607-
for (const audio of composition.audios) {
608-
if (isVideoAudio(audio, el.id) && typeof audio.playbackRate !== "object") {
609-
audio.playbackRate = el.playbackRate;
610-
}
611-
}
612-
}
613624
if (el.hasAudio && !el.muted && !existing.hasAudio) {
614625
existing.hasAudio = true;
615626
}
@@ -651,14 +662,11 @@ export async function runProbeStage(input: ProbeStageInput): Promise<ProbeStageR
651662
sourceChanged,
652663
el.durationInferred,
653664
);
654-
adoptRuntimeMediaStart(
665+
adoptRuntimeMediaTiming(
655666
composition,
656667
el,
657668
authoredStarts.get(el.id) ?? existing.mediaStart,
658669
);
659-
if (typeof existing.playbackRate !== "object") {
660-
existing.playbackRate = el.playbackRate;
661-
}
662670
if (
663671
el.volume > 0 &&
664672
Math.abs((existing.volume ?? 1) - el.volume) > BROWSER_MEDIA_EPSILON

0 commit comments

Comments
 (0)