Skip to content

Commit 5377b75

Browse files
fix(core): snapshot and preview show a clip on the frame render shows it (#7238)
* fix(core): snapshot and preview show a clip on the frame render shows it * fix(cli): snapshot pulls a video's frame on the same on-screen rule * refactor(cli): drop two doc lines the names already carry
1 parent d40d0b2 commit 5377b75

12 files changed

Lines changed: 168 additions & 45 deletions
Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
2+
import { tmpdir } from "node:os";
3+
import { join } from "node:path";
4+
import sharp from "sharp";
5+
import { afterAll, expect, it, vi } from "vitest";
6+
import snapshotCommand from "./snapshot.js";
7+
8+
const dir = mkdtempSync(join(tmpdir(), "hf-snapshot-held-boundary-"));
9+
10+
afterAll(() => rmSync(dir, { recursive: true, force: true }));
11+
12+
// A title held across two back-to-back clips; the second starts a hair past 100 s, as a start chained after a media
13+
// clip's measured length lands.
14+
writeFileSync(
15+
join(dir, "index.html"),
16+
`<!doctype html><html><body style="margin:0;background:#000">
17+
<div id="root" data-composition-id="main" data-start="0" data-duration="110" data-width="160" data-height="90" style="position:relative;width:160px;height:90px;background:#000">
18+
<div id="held" data-start="90" data-duration="10" data-track-index="0" style="position:absolute;inset:0;background:#f00"></div>
19+
<div id="rest" data-start="100.00002" data-duration="10" data-track-index="0" style="position:absolute;inset:0;background:#f00"></div>
20+
</div>
21+
<script>
22+
window.__timelines = window.__timelines || {};
23+
window.__timelines["main"] = { pause() {}, time() { return 0; }, duration() { return 110; }, seek() {} };
24+
</script></body></html>`,
25+
);
26+
27+
const redAtCentre = async (path: string) => {
28+
const { data, info } = await sharp(path).raw().toBuffer({ resolveWithObject: true });
29+
const at =
30+
(Math.floor(info.height / 2) * info.width + Math.floor(info.width / 2)) * info.channels;
31+
return data[at]! > 200 && data[at + 1]! < 60;
32+
};
33+
34+
// fails without the fix: the still at exactly 100 s came out black, while 98 s, 102 s and the render show the title.
35+
it("keeps a title held across back-to-back clips on screen at the exact second the second clip takes over", async () => {
36+
const log = vi.spyOn(console, "log").mockImplementation(() => undefined);
37+
vi.spyOn(console, "error").mockImplementation(() => undefined);
38+
vi.spyOn(console, "warn").mockImplementation(() => undefined);
39+
try {
40+
await snapshotCommand.run?.({
41+
args: {
42+
dir,
43+
at: "98,100,102",
44+
end: false,
45+
json: true,
46+
output: join(dir, "out"),
47+
describe: "false",
48+
"browser-gpu": false,
49+
},
50+
} as never);
51+
const { frames } = JSON.parse(String(log.mock.calls.at(-1)?.[0])) as {
52+
frames: { path: string; time: number }[];
53+
};
54+
const shown = await Promise.all(
55+
frames.map(async (frame) => [frame.time, await redAtCentre(frame.path)]),
56+
);
57+
expect(shown).toEqual([
58+
[98, true],
59+
[100, true],
60+
[102, true],
61+
]);
62+
} finally {
63+
vi.restoreAllMocks();
64+
}
65+
}, 120_000);

‎packages/cli/src/commands/snapshot.test.ts‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -322,6 +322,21 @@ describe("snapshot video overlays and the blurred frame", () => {
322322
});
323323

324324
describe("resolveSnapshotVideoFrameTime", () => {
325+
// fails without the fix: the element showed at 100 s on render's window, but no frame was pulled for its video.
326+
it("pulls a frame for a video starting a hair past 100 s at exactly 100 s, as render does", () => {
327+
expect(
328+
resolveSnapshotVideoFrameTime({
329+
globalTime: 100,
330+
clipStart: 100.00002,
331+
clipDuration: 10,
332+
relativeTime: -0.00002,
333+
sourceDuration: 10,
334+
compositionDuration: 110,
335+
fps: 30,
336+
}),
337+
).toBe(0);
338+
});
339+
325340
it("holds a clip ending with the composition on its last decodable frame", () => {
326341
expect(
327342
resolveSnapshotVideoFrameTime({

‎packages/cli/src/commands/snapshot.ts‎

Lines changed: 20 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -70,12 +70,14 @@ import {
7070
} from "../capture/captureCompositionFrame.js";
7171
import {
7272
firstFrameColourArgs,
73+
exportClipWindow,
7374
hdrToSdrToneMapFilter,
7475
keyframeTransferArgs,
7576
parseKeyframeTransfers,
7677
type ToneMapFrameColour,
77-
isClipVisibleAt,
78+
isClipOnScreenAt,
7879
isInClipWindow,
80+
parseFpsWithDefault,
7981
MEDIA_RENDER_ID_ATTR,
8082
parseFirstFrameColour,
8183
readsKeyframeTransfers,
@@ -84,6 +86,7 @@ import {
8486
timeAtSourceTime,
8587
type RateSpec,
8688
} from "@hyperframes/core";
89+
import { parseHTML } from "linkedom";
8790
import { resolveProject } from "../utils/project.js";
8891
import {
8992
definitiveEntryMismatchComposition,
@@ -250,15 +253,18 @@ export function resolveSnapshotVideoFrameTime(input: {
250253
relativeTime: number;
251254
sourceDuration: number;
252255
compositionDuration: number;
256+
fps?: number;
253257
}): number | null {
254258
const { globalTime, clipStart, clipDuration, sourceDuration, compositionDuration } = input;
259+
const fps = input.fps ?? 30;
255260
const clipEnd = clipStart + clipDuration;
256-
if (!isClipVisibleAt(globalTime, clipStart, clipEnd, compositionDuration)) return null;
261+
if (!isClipOnScreenAt(globalTime, clipStart, clipEnd, fps, compositionDuration)) return null;
262+
const onFrames = exportClipWindow(clipStart, clipEnd, fps);
257263
const relativeTime =
258264
globalTime < clipStart ? Math.max(0, input.relativeTime) : input.relativeTime;
259265
if (relativeTime < 0) return null;
260266
const pastSource = sourceDuration > 0 && relativeTime >= sourceDuration;
261-
if (isInClipWindow(globalTime, clipStart, clipEnd) && !pastSource) return relativeTime;
267+
if (isInClipWindow(globalTime, onFrames.start, onFrames.end) && !pastSource) return relativeTime;
262268

263269
const sourceEnd = sourceDuration > 0 ? sourceDuration : relativeTime;
264270
return Math.max(0, Math.min(relativeTime, sourceEnd - 1 / 30));
@@ -312,6 +318,7 @@ export function activeVideoAt(
312318
candidate: SnapshotVideoCandidate,
313319
time: number,
314320
compositionDuration: number,
321+
fps: number,
315322
) {
316323
const start = resolveSnapshotVideoClipStart(candidate);
317324
const playbackRate = resolveSnapshotVideoRateSpec(candidate);
@@ -337,6 +344,7 @@ export function activeVideoAt(
337344
relativeTime: relTime,
338345
sourceDuration: candidate.srcDuration,
339346
compositionDuration,
347+
fps,
340348
});
341349
return frameTime === null
342350
? null
@@ -901,6 +909,12 @@ export function computeSnapshotTimes(
901909
return { times: times.map(round), appendedTail: false };
902910
}
903911

912+
function compositionFps(html: string): number {
913+
const root = parseHTML(html).document.querySelector("[data-composition-id]");
914+
const declared = parseFpsWithDefault(root?.getAttribute("data-fps") ?? undefined);
915+
return declared.ok ? declared.value.num / declared.value.den : 30;
916+
}
917+
904918
export function serveCapturePage(projectDir: string, html: string, proxy: boolean | undefined) {
905919
return serveStaticProjectHtml(projectDir, html, undefined, [], proxy === true);
906920
}
@@ -971,6 +985,7 @@ export async function captureSnapshots(
971985
console.warn(` ${c.warn("⚠")} ${swallowed}`);
972986
}
973987
const server = await serveCapturePage(projectDir, html, opts.autoProxy);
988+
const fps = compositionFps(html);
974989
const heldVideoIds = localVideoRenderIds(html);
975990
const heldProbes = new Map<string, Promise<HeldVideoProbe>>();
976991
const heldHints = await heldVideoHints(projectDir, localVideos(html), heldProbes);
@@ -1270,7 +1285,7 @@ export async function captureSnapshots(
12701285
);
12711286
let unreadable = 0;
12721287
const active = candidates.flatMap((candidate) => {
1273-
const video = activeVideoAt(candidate, time, duration);
1288+
const video = activeVideoAt(candidate, time, duration, fps);
12741289
if (!video) return [];
12751290
if (!candidate.id || !candidate.src) {
12761291
unreadable += 1;
@@ -1314,7 +1329,7 @@ export async function captureSnapshots(
13141329
(opts.zoom ? (opts.zoomScale ?? DEFAULT_ZOOM_SCALE) : v.devicePixelRatio),
13151330
);
13161331
const wantAt = (t: number): FrameWant | null => {
1317-
const shown = activeVideoAt(v, t, duration);
1332+
const shown = activeVideoAt(v, t, duration, fps);
13181333
if (!shown) return null;
13191334
const holdLastFrame =
13201335
shown.srcDuration > 0 && shown.relTime >= shown.srcDuration - 1;

‎packages/core/src/compositionReadiness.test.ts‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -97,7 +97,6 @@ describe("scanPendingCompositionAssets", () => {
9797
currentTime: 0,
9898
compositionDuration: Number.POSITIVE_INFINITY,
9999
canonicalFps: 30,
100-
exportRenderSeek: false,
101100
timelineRegistry: {},
102101
resolver,
103102
});

‎packages/core/src/compositionReadiness.ts‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,6 @@ function isActiveAtFirstFrame(
6767
currentTime: 0,
6868
compositionDuration: Number.POSITIVE_INFINITY,
6969
canonicalFps: 30,
70-
exportRenderSeek: false,
7170
timelineRegistry,
7271
resolver,
7372
})

‎packages/core/src/index.ts‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -397,7 +397,12 @@ export {
397397
// (verify:packed-manifests catches exactly that).
398398
export { createRuntimeStartTimeResolver } from "./runtime/startResolver.js";
399399
// Also exposed via the ./runtime/clip-window subpath; re-exported here for the same dist-emit reason.
400-
export { hasClipStarted, isClipVisibleAt, isInClipWindow } from "./runtime/clipWindow.js";
400+
export {
401+
hasClipStarted,
402+
isClipOnScreenAt,
403+
isClipVisibleAt,
404+
isInClipWindow,
405+
} from "./runtime/clipWindow.js";
401406
export { MEDIA_ELEMENT_LEAD_SECONDS, mediaElementTime } from "./runtime/mediaClock.js";
402407
export {
403408
normalizePlaybackRate,

‎packages/core/src/runtime/clipWindow.ts‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import { sameInstant } from "../clipFacts";
2+
import { exportClipWindow } from "../inline-scripts/parityContract";
23

34
export const hasClipStarted = (time: number, start: number) =>
45
time >= start || sameInstant(time, start);
@@ -23,3 +24,15 @@ export const isClipVisibleAt = (
2324
(hasClipStarted(time, start) &&
2425
compositionDuration > 0 &&
2526
end >= compositionDuration - TERMINAL_EPSILON_SECONDS);
27+
28+
/** The one on-screen rule render, snapshot and preview share: edges a hair off a frame count as on it. */
29+
export const isClipOnScreenAt = (
30+
time: number,
31+
start: number,
32+
end: number,
33+
fps: number,
34+
compositionDuration: number,
35+
): boolean => {
36+
const snapped = exportClipWindow(start, end, fps);
37+
return isClipVisibleAt(time, snapped.start, snapped.end, compositionDuration);
38+
};

‎packages/core/src/runtime/frameSources.test.ts‎

Lines changed: 3 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -14,13 +14,12 @@ function deferred() {
1414
}
1515

1616
const adapters: ReturnType<typeof createFrameSourceAdapter>[] = [];
17-
const adapter = (compositionDuration = 20, exportRenderSeek = false) => {
17+
const adapter = (compositionDuration = 20) => {
1818
const runtime = createFrameSourceAdapter({
1919
start: (element) => createRuntimeStartTimeResolver({}).resolveStartForElement(element, 0),
2020
duration: (element) => createRuntimeStartTimeResolver({}).resolveDurationForElement(element),
2121
compositionDuration: () => compositionDuration,
2222
canonicalFps: () => 30,
23-
exportRenderSeek: () => exportRenderSeek,
2423
});
2524
adapters.push(runtime);
2625
return runtime;
@@ -293,10 +292,10 @@ describe("frame sources", () => {
293292
expect(render).toHaveBeenCalledTimes(1);
294293
});
295294

296-
it("matches snapped export visibility at near-frame starts and ends", async () => {
295+
it("shows a source on render's snapped window at near-frame starts and ends, in preview too", async () => {
297296
const render = vi.fn();
298297
disposers.push(registerFrameSource({ element: mount("1.00001", "1"), render }));
299-
const runtime = adapter(3, true);
298+
const runtime = adapter(3);
300299
runtime.seek({ time: 1 });
301300
await waitForSeekCompletion();
302301
expect(render).toHaveBeenLastCalledWith(0, expect.any(AbortSignal));
@@ -308,18 +307,6 @@ describe("frame sources", () => {
308307
expect(render).toHaveBeenCalledTimes(2);
309308
});
310309

311-
it("retains unsnapped authored timing during interactive preview", async () => {
312-
const render = vi.fn();
313-
disposers.push(registerFrameSource({ element: mount("1.00001", "1"), render }));
314-
const runtime = adapter(3);
315-
runtime.seek({ time: 1 });
316-
await waitForSeekCompletion();
317-
expect(render).not.toHaveBeenCalled();
318-
runtime.seek({ time: 2 });
319-
await waitForSeekCompletion();
320-
expect(render).toHaveBeenCalledTimes(1);
321-
});
322-
323310
it("rejects invalid original ranges before registering a source", () => {
324311
const element = mount();
325312
for (const sourceRange of [

‎packages/core/src/runtime/frameSources.ts‎

Lines changed: 9 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,7 @@
1-
import { exportClipWindow } from "../inline-scripts/parityContract";
21
import { sourceTimeAt } from "../speedRamp";
32
import { readElementRateSpec, readMediaStart } from "./playbackRate";
43
import { registerSeekCompletion } from "./adapters/seek-dispatch";
5-
import { isClipVisibleAt } from "./clipWindow";
4+
import { isClipOnScreenAt } from "./clipWindow";
65
import type { RuntimeDeterministicAdapter } from "./types";
76

87
export interface FrameSource {
@@ -93,7 +92,6 @@ export function createFrameSourceAdapter(timing: {
9392
duration: (element: Element) => number | null;
9493
compositionDuration: () => number;
9594
canonicalFps: () => number;
96-
exportRenderSeek: () => boolean;
9795
}): RuntimeDeterministicAdapter {
9896
const owned = new Set<RegisteredSource>();
9997
let readySources: RegisteredSource[] = [];
@@ -132,11 +130,14 @@ export function createFrameSourceAdapter(timing: {
132130
const start = timing.start(element);
133131
const duration = timing.duration(element);
134132
const end = start + (duration ?? Infinity);
135-
const clipWindow = timing.exportRenderSeek()
136-
? exportClipWindow(start, end, timing.canonicalFps())
137-
: { start, end };
138-
if (!isClipVisibleAt(time, clipWindow.start, clipWindow.end, timing.compositionDuration()))
139-
continue;
133+
const onScreen = isClipOnScreenAt(
134+
time,
135+
start,
136+
end,
137+
timing.canonicalFps(),
138+
timing.compositionDuration(),
139+
);
140+
if (!onScreen) continue;
140141
const localTime = Math.max(0, time - start);
141142
const sourceTime =
142143
readMediaStart(element) + sourceTimeAt(readElementRateSpec(element), localTime);

‎packages/core/src/runtime/init.ts‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2941,7 +2941,6 @@ export function initSandboxRuntimeModular(): void {
29412941
currentTime: time,
29422942
compositionDuration,
29432943
canonicalFps: state.canonicalFps,
2944-
exportRenderSeek: Boolean(window.__HF_EXPORT_RENDER_SEEK_CONFIG),
29452944
timelineRegistry: window.__timelines ?? {},
29462945
resolver: timingResolverFor(true),
29472946
});
@@ -5091,7 +5090,6 @@ export function initSandboxRuntimeModular(): void {
50915090
duration: (element) => resolveDurationForElement(element),
50925091
compositionDuration: () => getSafeTimelineDurationSeconds(state.capturedTimeline, 0),
50935092
canonicalFps: () => state.canonicalFps,
5094-
exportRenderSeek: () => Boolean(window.__HF_EXPORT_RENDER_SEEK_CONFIG),
50955093
}),
50965094
createWaapiAdapter(),
50975095
createCssAdapter({

0 commit comments

Comments
 (0)