Skip to content

Commit 3def351

Browse files
fix(engine): export draws a video in exactly the frames it is shown in (#4565)
* fix(engine): export draws a video in exactly the frames export visibility shows it * test(producer): guard that chunk frame planning snaps clip windows like export * refactor(engine): set the render fps once per frame table and name its clip window * chore(engine): trim frame table comments
1 parent 31c00d2 commit 3def351

10 files changed

Lines changed: 92 additions & 27 deletions

File tree

‎packages/core/src/index.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -269,6 +269,7 @@ export {
269269
export {
270270
MEDIA_VISUAL_STYLE_PROPERTIES,
271271
copyMediaVisualStyles,
272+
exportClipWindow,
272273
quantizeSeekTime,
273274
quantizeTimeToFrame,
274275
type MediaVisualStyleProperty,

‎packages/core/src/inline-scripts/parityContract.ts‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,18 @@ export function snapTimeToFrameBoundary(timeSeconds: number, fps: number): numbe
8282
: time;
8383
}
8484

85+
/** The window export shows a clip in: both ends snapped like its seeks, so a hair off a frame counts as on it. */
86+
export function exportClipWindow(
87+
start: number,
88+
end: number,
89+
fps: number,
90+
): { start: number; end: number } {
91+
return {
92+
start: snapTimeToFrameBoundary(start, fps),
93+
end: Number.isFinite(end) ? snapTimeToFrameBoundary(end, fps) : end,
94+
};
95+
}
96+
8597
export function copyMediaVisualStyles(
8698
targetStyle: CSSStyleDeclaration,
8799
sourceStyle: CSSStyleDeclaration,

‎packages/core/src/runtime/timeline.ts‎

Lines changed: 6 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ import {
2222
import { resolveCssStackingContextId } from "./stackingContext";
2323
import { createRuntimeStartTimeResolver } from "./startResolver";
2424
import { isClipVisibleAt } from "./clipWindow";
25-
import { snapTimeToFrameBoundary } from "../inline-scripts/parityContract";
25+
import { exportClipWindow } from "../inline-scripts/parityContract";
2626
import { isSceneLikeCompositionId } from "../slideshow/index.js";
2727
import { COMPOSITION_CONTRACT_VERSION } from "../compositionContract.js";
2828
import { runtimeProtocolMetadata } from "./protocol.js";
@@ -79,17 +79,13 @@ export function isRuntimeElementVisibleAt(
7979
const computedEnd =
8080
duration != null && duration > 0 ? start + duration : Number.POSITIVE_INFINITY;
8181
// Export seeks snap to frame boundaries; interactive visibility uses authored seconds.
82-
const visibilityStart = options.exportRenderSeek
83-
? snapTimeToFrameBoundary(start, options.canonicalFps)
84-
: start;
85-
const visibilityEnd =
86-
options.exportRenderSeek && Number.isFinite(computedEnd)
87-
? snapTimeToFrameBoundary(computedEnd, options.canonicalFps)
88-
: computedEnd;
82+
const clipWindow = options.exportRenderSeek
83+
? exportClipWindow(start, computedEnd, options.canonicalFps)
84+
: { start, end: computedEnd };
8985
return isClipVisibleAt(
9086
options.currentTime,
91-
visibilityStart,
92-
visibilityEnd,
87+
clipWindow.start,
88+
clipWindow.end,
9389
options.compositionDuration,
9490
);
9591
}

‎packages/engine/src/services/videoFrameExtractor.test.ts‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1131,6 +1131,35 @@ describe("FrameLookupTable", () => {
11311131
expect(table.getActiveFramePayloads(3.44).get("hero")?.frameIndex).toBe(42);
11321132
});
11331133

1134+
it("gives a frame a hair before two clips meet to the clip export shows there", () => {
1135+
const clip = (id: string, start: number, end: number) => ({
1136+
id,
1137+
src: `${id}.webm`,
1138+
start,
1139+
end,
1140+
mediaStart: 0,
1141+
loop: false,
1142+
hasAudio: false,
1143+
});
1144+
const table = () =>
1145+
createFrameLookupTable(
1146+
[clip("a", 0, 1.00001), clip("b", 1.00001, 2)],
1147+
[
1148+
{ ...fakeExtracted(90, 30), videoId: "a" },
1149+
{ ...fakeExtracted(90, 30), videoId: "b" },
1150+
],
1151+
30,
1152+
);
1153+
const stepping = table();
1154+
stepping.getActiveFramePayloads(29 / 30);
1155+
const payloads = stepping.getActiveFramePayloads(1);
1156+
1157+
expect([...payloads.keys()]).toEqual(["b"]);
1158+
expect(payloads.get("b")?.frameIndex).toBe(0);
1159+
expect([...table().getActiveFramePayloads(1).keys()]).toEqual(["b"]);
1160+
expect(table().getFrame("a", 1)).toBeNull();
1161+
});
1162+
11341163
it.each([
11351164
["an exact", 0.1, 0.3, 0.3],
11361165
["a float-sum", 0.1, 0.1 + 0.2, 9 / 30],

‎packages/engine/src/services/videoFrameExtractor.ts‎

Lines changed: 18 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ import {
2121
sourceTimeAt,
2222
timeAtSourceTime,
2323
type RateSpec,
24+
exportClipWindow,
2425
hasClipStarted,
2526
isInClipWindow,
2627
parseStrictFiniteTimingNumber,
@@ -2199,7 +2200,6 @@ function getFrameIndexAtTime(
21992200
holdLastFrame = false,
22002201
playbackRate: RateSpec = 1,
22012202
): number | null {
2202-
if (!hasClipStarted(globalTime, videoStart)) return null;
22032203
let localTime = Math.max(0, globalTime - videoStart);
22042204
const normalizedPlaybackRate = normalizeRateSpec(playbackRate);
22052205
const loopDuration = timeAtSourceTime(
@@ -2237,6 +2237,7 @@ export function getFrameAtTime(
22372237
loop = false,
22382238
mediaStart = 0,
22392239
): string | null {
2240+
if (!hasClipStarted(globalTime, videoStart)) return null;
22402241
const frameIndex = getFrameIndexAtTime(extracted, globalTime, videoStart, loop, mediaStart);
22412242
return frameIndex == null ? null : extracted.framePaths.get(frameIndex) || null;
22422243
}
@@ -2270,7 +2271,7 @@ export class FrameLookupTable {
22702271
{
22712272
extracted: ExtractedFrames;
22722273
start: number;
2273-
end: number;
2274+
clipWindow: { start: number; end: number };
22742275
mediaStart: number;
22752276
loop: boolean;
22762277
playbackRate: RateSpec;
@@ -2280,7 +2281,7 @@ export class FrameLookupTable {
22802281
videoId: string;
22812282
extracted: ExtractedFrames;
22822283
start: number;
2283-
end: number;
2284+
clipWindow: { start: number; end: number };
22842285
mediaStart: number;
22852286
loop: boolean;
22862287
playbackRate: RateSpec;
@@ -2289,6 +2290,9 @@ export class FrameLookupTable {
22892290
private startCursor = 0;
22902291
private lastTime: number | null = null;
22912292

2293+
/** The render's `fps`, so a clip shows in exactly export visibility's frames; omit for authored times. */
2294+
constructor(private readonly fps?: number) {}
2295+
22922296
addVideo(
22932297
extracted: ExtractedFrames,
22942298
start: number,
@@ -2300,21 +2304,21 @@ export class FrameLookupTable {
23002304
this.videos.set(extracted.videoId, {
23012305
extracted,
23022306
start,
2303-
end,
2307+
clipWindow: this.fps ? exportClipWindow(start, end, this.fps) : { start, end },
23042308
mediaStart,
23052309
loop,
23062310
playbackRate: normalizeRateSpec(playbackRate),
23072311
});
23082312
this.orderedVideos = Array.from(this.videos.entries())
23092313
.map(([videoId, video]) => ({ videoId, ...video }))
2310-
.sort((a, b) => a.start - b.start);
2314+
.sort((a, b) => a.clipWindow.start - b.clipWindow.start);
23112315
this.resetActiveState();
23122316
}
23132317

23142318
getFrame(videoId: string, globalTime: number): string | null {
23152319
const video = this.videos.get(videoId);
23162320
if (!video) return null;
2317-
if (!isInClipWindow(globalTime, video.start, video.end)) return null;
2321+
if (!isInClipWindow(globalTime, video.clipWindow.start, video.clipWindow.end)) return null;
23182322
const frameIndex = getFrameIndexAtTime(
23192323
video.extracted,
23202324
globalTime,
@@ -2334,14 +2338,13 @@ export class FrameLookupTable {
23342338
}
23352339

23362340
private refreshActiveSet(globalTime: number): void {
2337-
// The runtime's half-open window. Rendered times stay below the composition end, so its
2338-
// terminal hold (isClipVisibleAt) never applies here.
2341+
// Half-open exportClipWindow; rendered times stay below the composition end, so no terminal hold here.
23392342
if (this.lastTime == null || globalTime < this.lastTime) {
23402343
this.activeVideoIds.clear();
23412344
this.startCursor = 0;
23422345
for (const entry of this.orderedVideos) {
2343-
if (!hasClipStarted(globalTime, entry.start)) break;
2344-
if (isInClipWindow(globalTime, entry.start, entry.end)) {
2346+
if (!hasClipStarted(globalTime, entry.clipWindow.start)) break;
2347+
if (isInClipWindow(globalTime, entry.clipWindow.start, entry.clipWindow.end)) {
23452348
this.activeVideoIds.add(entry.videoId);
23462349
}
23472350
this.startCursor += 1;
@@ -2353,18 +2356,18 @@ export class FrameLookupTable {
23532356
while (this.startCursor < this.orderedVideos.length) {
23542357
const candidate = this.orderedVideos[this.startCursor];
23552358
if (!candidate) break;
2356-
if (!hasClipStarted(globalTime, candidate.start)) {
2359+
if (!hasClipStarted(globalTime, candidate.clipWindow.start)) {
23572360
break;
23582361
}
2359-
if (isInClipWindow(globalTime, candidate.start, candidate.end)) {
2362+
if (isInClipWindow(globalTime, candidate.clipWindow.start, candidate.clipWindow.end)) {
23602363
this.activeVideoIds.add(candidate.videoId);
23612364
}
23622365
this.startCursor += 1;
23632366
}
23642367

23652368
for (const videoId of Array.from(this.activeVideoIds)) {
23662369
const video = this.videos.get(videoId);
2367-
if (!video || !isInClipWindow(globalTime, video.start, video.end)) {
2370+
if (!video || !isInClipWindow(globalTime, video.clipWindow.start, video.clipWindow.end)) {
23682371
this.activeVideoIds.delete(videoId);
23692372
}
23702373
}
@@ -2424,8 +2427,9 @@ export class FrameLookupTable {
24242427
export function createFrameLookupTable(
24252428
videos: VideoElement[],
24262429
extracted: ExtractedFrames[],
2430+
fps?: number,
24272431
): FrameLookupTable {
2428-
const table = new FrameLookupTable();
2432+
const table = new FrameLookupTable(fps);
24292433
const extractedMap = new Map<string, ExtractedFrames>();
24302434
for (const ext of extracted) extractedMap.set(ext.videoId, ext);
24312435

‎packages/producer/src/services/distributed/planV2.test.ts‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -409,6 +409,17 @@ describe("Plan v2 manifest", () => {
409409
);
410410
});
411411

412+
it("gives a chunk the frames of a video starting a hair after its frame, as export shows it", () => {
413+
const root = tempPath("hf-plan-v2-snapped-start-");
414+
const v1 = createV1Plan(root, { video: true, videoStart: 1 / 30 + 1e-6, videoEnd: 1 });
415+
const manifest = readPlanV2Manifest(createPlanV2FromV1(v1, join(root, "v2")).planDir);
416+
const chunk1 = listPlanV2ArtifactsForTarget(manifest, { role: "chunk", chunkIndex: 1 });
417+
418+
expect(chunk1.some((artifact) => artifact.path === "video-frames/hero/frame_00001.jpg")).toBe(
419+
true,
420+
);
421+
});
422+
412423
it("materializes empty video directories for chunks where the video is inactive", () => {
413424
const root = tempPath("hf-plan-v2-inactive-video-directory-");
414425
const v1 = createV1Plan(root, {

‎packages/producer/src/services/distributed/planV2.ts‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@ import {
4646
} from "./planV2Publisher.js";
4747
import { PLAN_V2_INTEGRITY_UNRECOVERABLE, PlanV2IntegrityError } from "./planV2Errors.js";
4848
import { planV2BlobPath } from "./planV2Layout.js";
49+
import { resolveRenderFpsConfig } from "../fileServer.js";
4950
import {
5051
isPlanAudioArtifactPath,
5152
PLAN_VIDEOS_META_RELATIVE_PATH,
@@ -407,9 +408,13 @@ function buildVideoChunkDependencies(
407408
const parsedVideos = parsePlanVideosJson(videos);
408409
const parsedChunks = parseChunkSlices(chunks);
409410
const extracted = listVideoFramePaths(executionPlanDir, parsedVideos);
410-
const table = createFrameLookupTable(parsedVideos.videos, extracted);
411411
const fpsNum = readPositiveInteger(dimensions.fpsNum, "dimensions.fpsNum");
412412
const fpsDen = readPositiveInteger(dimensions.fpsDen, "dimensions.fpsDen");
413+
const table = createFrameLookupTable(
414+
parsedVideos.videos,
415+
extracted,
416+
resolveRenderFpsConfig({ num: fpsNum, den: fpsDen }).value,
417+
);
413418
const mutable = new Map<string, Set<number>>();
414419

415420
for (const chunk of parsedChunks) {

‎packages/producer/src/services/distributed/renderChunk.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,7 @@ import {
7878
buildVirtualTimeShim,
7979
closeFileServerSafely,
8080
createFileServer,
81+
resolveRenderFpsConfig,
8182
type FileServerHandle,
8283
} from "../fileServer.js";
8384
import {
@@ -678,6 +679,7 @@ export async function renderChunk(
678679
planVideos.extracted,
679680
v2Manifest === null ? "dense-v1" : "sparse-v2",
680681
),
682+
resolveRenderFpsConfig(job.config.fps).value,
681683
)
682684
: null;
683685
const createChunkVideoFrameInjector = createChunkVideoFrameInjectorFactory(videoFrameLookup);

‎packages/producer/src/services/fileServer.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -410,7 +410,7 @@ const RENDER_SEEK_OFFSET_FRACTION = Math.max(
410410
Math.min(0.95, Number(process.env.PRODUCER_RUNTIME_RENDER_SEEK_OFFSET_FRACTION || 0.5)),
411411
);
412412

413-
function resolveRenderFpsConfig(fps: Fps | undefined): {
413+
export function resolveRenderFpsConfig(fps: Fps | undefined): {
414414
value: number;
415415
source: "render-options" | "default";
416416
fallbackReason?: "missing" | "invalid";

‎packages/producer/src/services/render/stages/extractVideosStage.ts‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,7 @@ import {
5656
type RenderJob,
5757
} from "../../renderOrchestrator.js";
5858
import { materializeExtractedFramesForCompiledDir, type CompositionMetadata } from "../shared.js";
59+
import { resolveRenderFpsConfig } from "../../fileServer.js";
5960
import type { ProducerLogger } from "../../../logger.js";
6061
import { encoderFailureError } from "../encoderInterruption.js";
6162
import {
@@ -516,7 +517,11 @@ export async function runExtractVideosStage(
516517
});
517518

518519
if (extractionResult.extracted.length > 0) {
519-
frameLookup = createFrameLookupTable(composition.videos, extractionResult.extracted);
520+
frameLookup = createFrameLookupTable(
521+
composition.videos,
522+
extractionResult.extracted,
523+
resolveRenderFpsConfig(job.config.fps).value,
524+
);
520525
}
521526
videoReadinessSkipIds = collectVideoReadinessSkipIds(
522527
nativeHdrVideoIds,

0 commit comments

Comments
 (0)