Skip to content

Commit 078c493

Browse files
perf: the preview loads and decodes only the images of scenes on screen or due next (#4578)
* perf(core): skip hidden clips' content so the browser stops decoding their images Hidden timed clips get content-visibility:hidden while their inline visibility is hidden, from script evaluation on. A clip due within the look-ahead (2 s by default) keeps its content live, so its images load before it shows. * perf(studio-server): load preview images lazily so hidden scenes fetch nothing * fix(core): stop the readiness scan waiting on and decoding later scenes' images * perf(studio-server): keep first-frame images eager so the preview opens as fast as before Only images inside a clip with a numeric start after 0 go lazy; a start the route cannot read stays eager. * perf(core): keep clips that start at 0 laid out before the first pass so the first frame is on time * perf(core): open the preview on time by hiding only lazy images before the first pass GSAP reads the computed style of the elements it animates while timelines build. Inside a content-skipped clip every read forces a style update, which cost about 65 ms at open on the bench composition. Before the first pass the rule now takes only lazy images out of layout, so hidden later scenes still do not load theirs; after it, the skip rule keeps their content skipped. * fix(core): hold a paused jump on the previous picture until the next scene's images decode * fix(studio): swap in a reloaded preview only after its restore seek has painted * fix(core): apply a held paused jump after 1 s when an image never decodes * fix(core): keep hidden-scene image skipping and held jumps to Studio's preview, never renders * fix(studio): reapply manual edits once a held paused jump lands * fix(studio): copy a clip without the preview's lazy-loading marks * fix(studio-server): mark preview images lazy without re-encoding the document's entities * fix(core): share one preview-mark constant between the runtime and the preview route * fix(studio): reapply manual edits a second time only after a seek that was held * refactor(core): share the hidden-ancestor walk between clip visibility and image holds * fix(core): import the studio preview mark from its own subpath * fix(studio-server): find preview images with the linear html scanner * chore(core): drop two doc comments the scanner's names already carry * fix(studio-server): serve the preview mark as its own meta, apart from the GSAP fallback script * fix(studio-server): capture thumbnails and frames from the preview without lazy images or the mark * fix(core): let the full readiness scan wait on every image except lazy ones off the first frame * fix(studio-server): keep an authored loading attribute in any letter case on preview images * fix(core): leave template content out of findStartTags so preview images match the DOM * fix(studio): send Ask agent the live markup without the preview loading marks * chore(core): drop the template note from findStartTags, its test carries it * perf(studio-server): splice lazy attributes into the preview in one pass * fix(studio): keep authored image loading on copy and Ask agent by marking preview-made lazy images * test(studio-server): follow the capture preview URL back to its sub-composition for every project id * refactor: name the upcoming-clip and GSAP fallback marks once * fix(studio-server): keep capture builds out of the on-disk preview store
1 parent bc73c41 commit 078c493

29 files changed

Lines changed: 907 additions & 91 deletions

‎packages/core/package-subpaths.json‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,12 @@
5050
"types": "./dist/compositionContract.d.ts",
5151
"environments": ["browser", "bun", "node"]
5252
},
53+
"./studio-preview-mark": {
54+
"source": "./src/studioPreviewMark.ts",
55+
"runtime": "./dist/studioPreviewMark.js",
56+
"types": "./dist/studioPreviewMark.d.ts",
57+
"environments": ["browser", "bun", "node"]
58+
},
5359
"./media-timing": {
5460
"source": "./src/mediaTiming.ts",
5561
"runtime": "./dist/mediaTiming.js",

‎packages/core/package.json‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,12 @@
6464
"import": "./src/compositionContract.ts",
6565
"types": "./src/compositionContract.ts"
6666
},
67+
"./studio-preview-mark": {
68+
"bun": "./src/studioPreviewMark.ts",
69+
"node": "./dist/studioPreviewMark.js",
70+
"import": "./src/studioPreviewMark.ts",
71+
"types": "./src/studioPreviewMark.ts"
72+
},
6773
"./media-timing": {
6874
"bun": "./src/mediaTiming.ts",
6975
"node": "./dist/mediaTiming.js",
@@ -466,6 +472,10 @@
466472
"import": "./dist/compositionContract.js",
467473
"types": "./dist/compositionContract.d.ts"
468474
},
475+
"./studio-preview-mark": {
476+
"import": "./dist/studioPreviewMark.js",
477+
"types": "./dist/studioPreviewMark.d.ts"
478+
},
469479
"./media-timing": {
470480
"import": "./dist/mediaTiming.js",
471481
"types": "./dist/mediaTiming.d.ts"

‎packages/core/src/compiler/htmlDocument.test.ts‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import { describe, expect, it } from "vitest";
22
import {
3+
findStartTags,
34
injectScriptsAtHeadStart,
45
injectScriptsIntoHtml,
56
injectTagsAtHeadStart,
@@ -221,6 +222,26 @@ describe("htmlDocument helpers", () => {
221222
});
222223
});
223224

225+
describe("findStartTags", () => {
226+
const at = (html: string, name: string) =>
227+
findStartTags(html, name).map((i) => html.slice(i, html.indexOf(">", i) + 1));
228+
229+
it("finds each start tag in any case, and none in comments, raw text or longer names", () => {
230+
const html =
231+
'<!-- <img a> --><script>"<img b>"</script><textarea><img c></textarea>' +
232+
'<img-card></img-card><IMG src=d><img\nsrc=e><div title="<img f>"></div>';
233+
expect(at(html, "img")).toEqual(["<IMG src=d>", "<img\nsrc=e>"]);
234+
});
235+
236+
it("leaves out template content, nested templates included", () => {
237+
const html =
238+
"<img a><template><img b><template><img c></template><img d></template>" +
239+
'<div data-start="5"><template><img e></template><img f></div>';
240+
expect(at(html, "img")).toEqual(["<img a>", "<img f>"]);
241+
expect(at(html, "template")).toEqual(["<template>", "<template>"]);
242+
});
243+
});
244+
224245
describe("injectTagsAtHeadStart on long adversarial input", () => {
225246
const M = 2_000_000;
226247
it.each([

‎packages/core/src/compiler/htmlDocument.ts‎

Lines changed: 23 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -210,19 +210,38 @@ const RAW_TEXT_TAGS = ["script", "style", "title", "textarea"] as const;
210210
type DocumentTag = "<head" | "</head" | "<body" | "</body";
211211
const COMMENT_END = /--!?>/g;
212212

213-
function findDocumentTag(html: string, tag: DocumentTag): number {
214-
const lowered = lowerAscii(html);
213+
function* markupStarts(lowered: string): Generator<number> {
215214
const unclosedRawText = new Set<string>();
216215
let cursor = 0;
217216
while (cursor !== -1) {
218217
const open = lowered.indexOf("<", cursor);
219-
if (open === -1) return -1;
220-
if (isTagAt(lowered, open, tag)) return open;
218+
if (open === -1) return;
219+
yield open;
221220
cursor = skipMarkup(lowered, open, unclosedRawText);
222221
}
222+
}
223+
224+
function findDocumentTag(html: string, tag: DocumentTag): number {
225+
const lowered = lowerAscii(html);
226+
for (const open of markupStarts(lowered)) {
227+
if (isTagAt(lowered, open, tag)) return open;
228+
}
223229
return -1;
224230
}
225231

232+
export function findStartTags(html: string, name: string): number[] {
233+
const lowered = lowerAscii(html);
234+
const token = `<${lowerAscii(name)}`;
235+
const starts: number[] = [];
236+
let templateDepth = 0;
237+
for (const open of markupStarts(lowered)) {
238+
if (templateDepth === 0 && isTagAt(lowered, open, token)) starts.push(open);
239+
if (isTagAt(lowered, open, "<template")) templateDepth++;
240+
else if (templateDepth > 0 && isTagAt(lowered, open, "</template")) templateDepth--;
241+
}
242+
return starts;
243+
}
244+
226245
function isTagAt(lowered: string, at: number, token: string): boolean {
227246
return lowered.startsWith(token, at) && isTagBoundary(lowered.charAt(at + token.length));
228247
}

‎packages/core/src/compositionReadiness.test.ts‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -178,6 +178,25 @@ describe("mediaReadinessInput", () => {
178178
video.dispatchEvent(new Event("canplay"));
179179
await expect(pending).resolves.toBeUndefined();
180180
});
181+
182+
it("waits on a later scene's image in the full scan unless it is lazy", () => {
183+
const doc = docWith(
184+
'<div data-start="30" data-duration="5"><img id="later" src="later.png">' +
185+
'<img id="lazy" loading="LAZY" src="lazy.png"></div>',
186+
);
187+
for (const image of doc.querySelectorAll("img")) {
188+
Object.defineProperty(image, "complete", { value: false });
189+
image.decode = vi.fn().mockResolvedValue(undefined);
190+
}
191+
const later = doc.querySelector<HTMLImageElement>("#later")!;
192+
const lazy = doc.querySelector<HTMLImageElement>("#lazy")!;
193+
194+
expect(scanPendingCompositionAssets(doc, { scope: "all" }).pendingImages).toEqual([later]);
195+
expect(scanPendingCompositionAssets(doc, { scope: "first-frame" }).pendingImages).toEqual([]);
196+
expect(mediaReadinessInput(doc, new AbortController().signal, { scope: "all" })).not.toBeNull();
197+
expect(later.decode).toHaveBeenCalled();
198+
expect(lazy.decode).not.toHaveBeenCalled();
199+
});
181200
});
182201

183202
describe("computeReadinessInput", () => {

‎packages/core/src/compositionReadiness.ts‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -107,8 +107,16 @@ export function scanPendingCompositionAssets(
107107
.filter(isRealmHtmlMediaElement)
108108
.filter((el) => shouldIncludeAsset(el, scope, resolver, runtimeWindow?.__timelines ?? {}))
109109
.filter((el) => el.readyState < HAVE_FUTURE_DATA);
110+
// A lazy image off the first frame loads only once shown, so no scope can wait on it.
110111
const pendingImages = Array.from(doc.querySelectorAll("img"))
111-
.filter((img) => shouldIncludeAsset(img, scope, resolver, runtimeWindow?.__timelines ?? {}))
112+
.filter((img) =>
113+
shouldIncludeAsset(
114+
img,
115+
img.getAttribute("loading")?.toLowerCase() === "lazy" ? FIRST_FRAME_READINESS_SCOPE : scope,
116+
resolver,
117+
runtimeWindow?.__timelines ?? {},
118+
),
119+
)
112120
.filter((img) => !img.complete);
113121
const fontsLoading = doc.fonts?.status === "loading";
114122
return { pendingMedia, pendingImages, fontsLoading };

‎packages/core/src/runtime/entry.test.ts‎

Lines changed: 194 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
// fallow-ignore-file code-duplication
22
import { afterEach, describe, expect, it, vi } from "vitest";
33
import { HF_COLOR_GRADING_ATTR, serializeHfColorGrading } from "../colorGrading";
4+
import { STUDIO_PREVIEW_LAZY_ATTR, STUDIO_PREVIEW_MARK_META } from "../studioPreviewMark";
45
import type { RuntimeTimelineLike } from "./types";
56

67
function pausedTimeline(duration: number): RuntimeTimelineLike {
@@ -52,9 +53,25 @@ async function evaluateRuntime(): Promise<void> {
5253
}
5354

5455
const visibility = (...els: HTMLElement[]) => els.map((el) => getComputedStyle(el).visibility);
56+
const imageSkipped = (...clips: HTMLElement[]) =>
57+
clips.map((clip) => getComputedStyle(clip.querySelector("img")!).display === "none");
58+
const withImage = <T extends HTMLElement>(clip: T): T => {
59+
clip.appendChild(document.createElement("img"));
60+
return clip;
61+
};
62+
// What Studio's preview route serves at the head start; render and player documents never carry it.
63+
const servePreview = () =>
64+
document.head.appendChild(
65+
Object.assign(document.createElement("meta"), { name: STUDIO_PREVIEW_MARK_META }),
66+
);
67+
const neverDecodes = (clip: HTMLElement) => {
68+
withImage(clip).querySelector("img")!.decode = () => new Promise<void>(() => {});
69+
return clip;
70+
};
5571

5672
describe("runtime entry", () => {
5773
afterEach(() => {
74+
vi.useRealTimers();
5875
window.__hfRuntimeTeardown?.();
5976
document.head.innerHTML = "";
6077
document.body.innerHTML = "";
@@ -73,10 +90,14 @@ describe("runtime entry", () => {
7390
});
7491

7592
it("paints no timed clip, from script evaluation until the first visibility pass decides it", async () => {
93+
servePreview();
7694
const root = mountRoot();
7795
const current = timed(root, "div", "0");
7896
const later = timed(root, "div", "5");
7997
const poster = timed(root, "img", "0");
98+
// Studio serves later scenes' images lazy; laid out, they would load before the first pass.
99+
const plate = later.appendChild(document.createElement("img"));
100+
plate.setAttribute("loading", "lazy");
80101
// A composition script may write visibility inline before the runtime runs.
81102
later.style.visibility = "visible";
82103
// Readiness, which runs the first pass, waits while GSAP batches timelines.
@@ -86,6 +107,7 @@ describe("runtime entry", () => {
86107
await evaluateRuntime();
87108
expect(window.__player).toBeUndefined();
88109
expect(visibility(current, later)).toEqual(["hidden", "hidden"]);
110+
expect(getComputedStyle(plate).display).toBe("none");
89111

90112
delete (document as { readyState?: unknown }).readyState;
91113
document.dispatchEvent(new Event("DOMContentLoaded"));
@@ -96,6 +118,178 @@ describe("runtime entry", () => {
96118
window.dispatchEvent(new CustomEvent("hf-timelines-built"));
97119
expect(window.__renderReady).toBe(true);
98120
expect(visibility(current, later, poster)).toEqual(["visible", "hidden", "visible"]);
121+
expect(document.querySelector("style[data-hf-first-pass-hide]")).toBeNull();
122+
});
123+
124+
it("gives a render document neither rule, no look-ahead marks and no held seeks", async () => {
125+
const root = mountRoot();
126+
const current = timed(root, "div", "0");
127+
const later = neverDecodes(timed(root, "div", "5"));
128+
const plate = later.querySelector("img")!;
129+
plate.setAttribute("loading", "lazy");
130+
window.__hfTimelinesBuilding = true;
131+
Object.defineProperty(document, "readyState", { configurable: true, get: () => "loading" });
132+
133+
await evaluateRuntime();
134+
// A setup-time measure of an authored lazy image must see it laid out.
135+
expect(getComputedStyle(plate).display).not.toBe("none");
136+
delete (document as { readyState?: unknown }).readyState;
137+
document.dispatchEvent(new Event("DOMContentLoaded"));
138+
window.__hfTimelinesBuilding = false;
139+
window.dispatchEvent(new CustomEvent("hf-timelines-built"));
140+
window.__player?.seek(3.5);
141+
expect(imageSkipped(later)).toEqual([false]);
142+
expect(document.querySelector("[data-hf-upcoming]")).toBeNull();
143+
window.__player?.seek(5.5);
144+
expect(visibility(current, later)).toEqual(["hidden", "visible"]);
145+
});
146+
147+
it("keys preview mode on the preview meta alone, not on the GSAP fallback script captures keep", async () => {
148+
document.head
149+
.appendChild(document.createElement("script"))
150+
.setAttribute("data-hf-gsap-fallback", "");
151+
const root = mountRoot();
152+
const current = timed(root, "div", "0");
153+
const later = neverDecodes(timed(root, "div", "5"));
154+
const plate = later.querySelector("img")!;
155+
plate.setAttribute("loading", "lazy");
156+
window.__hfTimelinesBuilding = true;
157+
Object.defineProperty(document, "readyState", { configurable: true, get: () => "loading" });
158+
159+
await evaluateRuntime();
160+
expect(getComputedStyle(plate).display).not.toBe("none");
161+
delete (document as { readyState?: unknown }).readyState;
162+
document.dispatchEvent(new Event("DOMContentLoaded"));
163+
window.__hfTimelinesBuilding = false;
164+
window.dispatchEvent(new CustomEvent("hf-timelines-built"));
165+
expect(document.querySelector("style[data-hf-skip-hidden-images]")).toBeNull();
166+
window.__player?.seek(3.5);
167+
expect(imageSkipped(later)).toEqual([false]);
168+
expect(document.querySelector("[data-hf-upcoming]")).toBeNull();
169+
expect(window.__player?.seek(5.5)).toBeUndefined();
170+
expect(visibility(current, later)).toEqual(["hidden", "visible"]);
171+
});
172+
173+
it("skips the images of each hidden clip not due within the look-ahead, until something shows it", async () => {
174+
servePreview();
175+
const root = mountRoot();
176+
const current = withImage(timed(root, "div", "0"));
177+
const soon = withImage(timed(root, "div", "1.5"));
178+
const later = withImage(timed(root, "div", "5"));
179+
// Shorter than the look-ahead: due within it, though already over at its far end.
180+
const brief = withImage(timed(root, "div", "5"));
181+
brief.setAttribute("data-duration", "0.5");
182+
183+
await evaluateRuntime();
184+
expect(imageSkipped(current, soon, later, brief)).toEqual([false, false, true, true]);
185+
window.__player?.seek(3.6);
186+
expect(imageSkipped(later, brief)).toEqual([false, false]);
187+
window.__player?.seek(0);
188+
// How Studio's layer reveal shows a hidden clip.
189+
later.style.visibility = "visible";
190+
expect(imageSkipped(later)).toEqual([false]);
191+
});
192+
193+
it("holds a paused jump on the previous picture until the next scene's image decodes", async () => {
194+
servePreview();
195+
const root = mountRoot();
196+
const current = timed(root, "div", "0");
197+
const later = timed(root, "div", "5");
198+
const plate = later.appendChild(document.createElement("img"));
199+
plate.setAttribute("loading", "lazy");
200+
plate.setAttribute(STUDIO_PREVIEW_LAZY_ATTR, "");
201+
let decoded = () => {};
202+
plate.decode = () => new Promise<void>((resolve) => (decoded = resolve));
203+
const authored = later.appendChild(document.createElement("img"));
204+
authored.setAttribute("loading", "lazy");
205+
authored.decode = () => Promise.resolve();
206+
207+
await evaluateRuntime();
208+
expect(window.__player?.seek(0.5)).toBeUndefined();
209+
const landed = window.__player?.seek(5.5);
210+
expect(landed).toBeInstanceOf(Promise);
211+
// Any frame painted now shows the previous scene, while the next one is unskipped so its image loads.
212+
expect(visibility(current, later)).toEqual(["visible", "hidden"]);
213+
expect(imageSkipped(later)).toEqual([false]);
214+
expect(window.__player?.getTime()).toBe(5.5);
215+
expect([plate, authored].map((img) => img.getAttribute("loading"))).toEqual(["eager", "lazy"]);
216+
decoded();
217+
await landed;
218+
expect(visibility(current, later)).toEqual(["hidden", "visible"]);
219+
await window.__hfWaitForSeekCompletion?.();
220+
});
221+
222+
it("applies a held jump after the cap when an image never decodes", async () => {
223+
servePreview();
224+
const root = mountRoot();
225+
const current = timed(root, "div", "0");
226+
const later = timed(root, "div", "5");
227+
later.appendChild(document.createElement("img")).decode = () => new Promise<void>(() => {});
228+
229+
await evaluateRuntime();
230+
const swallowed: string[] = [];
231+
window.__hf = {
232+
...window.__hf,
233+
onSwallowed: ({ label }: { label: string }) => swallowed.push(label),
234+
};
235+
vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] });
236+
window.__player?.seek(5.5);
237+
await vi.advanceTimersByTimeAsync(999);
238+
expect(visibility(current, later)).toEqual(["visible", "hidden"]);
239+
await vi.advanceTimersByTimeAsync(1);
240+
expect(visibility(current, later)).toEqual(["hidden", "visible"]);
241+
expect(swallowed).toEqual(["runtime.init.seekHoldCap"]);
242+
});
243+
244+
it("drops a held jump when a newer seek or a render seek lands first", async () => {
245+
servePreview();
246+
const root = mountRoot();
247+
const current = timed(root, "div", "0");
248+
const mid = timed(root, "div", "1");
249+
const later = timed(root, "div", "5");
250+
const plate = later.appendChild(document.createElement("img"));
251+
let decoded = () => {};
252+
plate.decode = () => new Promise<void>((resolve) => (decoded = resolve));
253+
254+
await evaluateRuntime();
255+
window.__player?.seek(5.5);
256+
window.__player?.seek(1);
257+
expect(visibility(current, mid, later)).toEqual(["visible", "visible", "hidden"]);
258+
decoded();
259+
await window.__hfWaitForSeekCompletion?.();
260+
expect(visibility(current, mid, later)).toEqual(["visible", "visible", "hidden"]);
261+
expect(window.__player?.getTime()).toBe(1);
262+
263+
window.__player?.seek(0);
264+
window.__player?.seek(5.5);
265+
window.__player?.renderSeek(1);
266+
decoded();
267+
await window.__hfWaitForSeekCompletion?.();
268+
expect(visibility(current, mid, later)).toEqual(["visible", "visible", "hidden"]);
269+
});
270+
271+
it("never holds a render seek", async () => {
272+
servePreview();
273+
const root = mountRoot();
274+
const current = timed(root, "div", "0");
275+
const later = neverDecodes(timed(root, "div", "5"));
276+
277+
await evaluateRuntime();
278+
window.__player?.renderSeek(5.5);
279+
expect(visibility(current, later)).toEqual(["hidden", "visible"]);
280+
});
281+
282+
it("shows the jump target at once when play is pressed during the hold", async () => {
283+
servePreview();
284+
const root = mountRoot();
285+
const current = timed(root, "div", "0");
286+
const later = timed(root, "div", "5");
287+
later.appendChild(document.createElement("img")).decode = () => new Promise<void>(() => {});
288+
289+
await evaluateRuntime();
290+
window.__player?.seek(5.5);
291+
window.__player?.play();
292+
expect(visibility(current, later)).toEqual(["hidden", "visible"]);
99293
});
100294

101295
it("leaves nothing hidden when the runtime is evaluated a second time", async () => {

0 commit comments

Comments
 (0)