Skip to content

Commit 1b8cd79

Browse files
committed
fix(cli): cloud render keeps media scripts may reach and never trims clips scripts retime
- An offloaded file keeps its zip copy when script code holds both its stem and extension, so a path joined at runtime still loads. - Entry GIFs are prepared from a private copy checked against the staged file, so the GIF cache never stores a later edit. - No cut when any script can write a clip's timing attributes, dataset, currentTime, playbackRate, src or loop.
1 parent 0b06284 commit 1b8cd79

3 files changed

Lines changed: 126 additions & 11 deletions

File tree

‎packages/cli/src/cloud/offload.test.ts‎

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -589,6 +589,20 @@ describe("offloadHeavyMedia", () => {
589589
),
590590
"long.mkv": HEAVY,
591591
});
592+
const retimedById = project({
593+
"index.html": html(
594+
`<video id="a" src="long.mkv" data-start="0" data-duration="2" data-media-start="10"></video>` +
595+
`<script>const a = document.getElementById("a"); a.setAttribute("data-media-start", "100"); a.volume = 0.5;</script>`,
596+
),
597+
"long.mkv": HEAVY,
598+
});
599+
const retimedByDataset = project({
600+
"index.html": html(
601+
`<video id="a" src="long.mkv" data-duration="2" data-media-start="10"></video><script src="main.js"></script>`,
602+
),
603+
"main.js": `document.getElementById("a").dataset.mediaStart = "100";`,
604+
"long.mkv": HEAVY,
605+
});
592606
const namedInPage = project({
593607
"index.html": html(
594608
`<video src="long.mkv" data-duration="2" data-media-start="10"></video><p data-note="long.mkv"></p>`,
@@ -603,6 +617,8 @@ describe("offloadHeavyMedia", () => {
603617
automated,
604618
offsetParent,
605619
nested,
620+
retimedById,
621+
retimedByDataset,
606622
namedInPage,
607623
]) {
608624
const run = vi.fn(async () => ({
@@ -663,6 +679,66 @@ describe("offloadHeavyMedia", () => {
663679
expect(fileMap.has("media/clip.mp4")).toBe(true);
664680
});
665681

682+
it("keeps a zip copy of an offloaded file a script can also build from split strings", async () => {
683+
const dir = project({
684+
"index.html": html(
685+
`<video id="v" src="media/clip.mp4"></video>` +
686+
`<script>document.getElementById("v").src = "media/" + "clip" + ".mp4";</script>`,
687+
),
688+
"media/clip.mp4": HEAVY,
689+
});
690+
const fileMap = buildPublishFileMap(dir);
691+
692+
const report = await offloadHeavyMedia(dir, fileMap, {
693+
apiBaseUrl: "https://api.example",
694+
upload: fakeUpload(),
695+
cachePath: cachePath(),
696+
});
697+
698+
expect(report).toEqual([
699+
expect.objectContaining({ path: "media/clip.mp4", kept_in_archive: true }),
700+
]);
701+
expect(fileMap.has("media/clip.mp4")).toBe(true);
702+
});
703+
704+
it("prepares an entry GIF staged on disk from a checked private copy", async () => {
705+
const files = {
706+
"index.html": html(`<img src="media/loop.gif" />`),
707+
"media/loop.gif": HEAVY,
708+
};
709+
const prepared: Buffer[] = [];
710+
const gifPrep = vi.fn<GifPrep>(async (input, options) => {
711+
const source = options.sourceAssets.get("media/loop.gif")!;
712+
// An edit while the renderer hashes and encodes must not reach what it caches.
713+
writeFileSync(join(options.projectDir, "media/loop.gif"), Buffer.alloc(9));
714+
prepared.push(readFileSync(source));
715+
const outputPath = join(options.outputDir, "hfgif-v1-abc.webm");
716+
writeFileSync(outputPath, "webm");
717+
return {
718+
html: input.replace(
719+
`<img src="media/loop.gif" />`,
720+
`<video src="_animated_gif/hfgif-v1-abc.webm"></video>`,
721+
),
722+
preparedGifs: [{ sourceSrc: "media/loop.gif", outputPath }],
723+
};
724+
});
725+
const opts = { apiBaseUrl: "https://api.example", upload: fakeUpload(), gifPrep };
726+
727+
const dir = project(files);
728+
await offloadHeavyMedia(dir, stagePublishFiles(dir, OFFLOAD_MIN_BYTES), {
729+
...opts,
730+
cachePath: cachePath(),
731+
});
732+
expect(prepared[0]?.equals(HEAVY)).toBe(true);
733+
734+
const edited = project(files);
735+
const fileMap = stagePublishFiles(edited, OFFLOAD_MIN_BYTES);
736+
writeFileSync(join(edited, "media/loop.gif"), Buffer.alloc(9));
737+
await expect(
738+
offloadHeavyMedia(edited, fileMap, { ...opts, cachePath: cachePath() }),
739+
).rejects.toThrow(/changed on disk/);
740+
});
741+
666742
it("never decodes a media file as text, since one can exceed the longest string allowed", async () => {
667743
const dir = project({
668744
"index.html": html(`<video src="clip.mp4"></video>`),

‎packages/cli/src/cloud/offload.ts‎

Lines changed: 43 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
import { execFile } from "node:child_process";
77
import { createHash } from "node:crypto";
88
import {
9+
copyFileSync,
910
createReadStream,
1011
mkdirSync,
1112
mkdtempSync,
@@ -457,11 +458,19 @@ async function stagedSha256(file: StagedFile): Promise<string> {
457458
* is used in place (checked unchanged after use), one staged in memory gets a temp copy.
458459
*/
459460
function snapshotPath(path: string, file: StagedFile, tempFiles: string[]): string {
460-
if (!Buffer.isBuffer(file)) return file.absPath;
461+
return Buffer.isBuffer(file) ? stagedCopy(path, file, tempFiles) : file.absPath;
462+
}
463+
464+
/** A private copy of the staged bytes, for readers that cache what they read before any check runs. */
465+
function stagedCopy(path: string, file: StagedFile, tempFiles: string[]): string {
461466
const dir = mkdtempSync(join(tmpdir(), "hf-offload-src-"));
462467
tempFiles.push(dir);
463468
const copy = join(dir, basename(path));
464-
writeFileSync(copy, file);
469+
if (Buffer.isBuffer(file)) writeFileSync(copy, file);
470+
else {
471+
copyFileSync(file.absPath, copy);
472+
assertUnchanged(file);
473+
}
465474
return copy;
466475
}
467476

@@ -470,6 +479,21 @@ function textOf(path: string, file: StagedFile | undefined): string | undefined
470479
return TEXT_FILE_EXTS.test(path) && Buffer.isBuffer(file) ? file.toString("utf-8") : undefined;
471480
}
472481

482+
/** What the page's scripts can read: inline scripts of HTML files, and script and data files. */
483+
function scriptCode(fileMap: Map<string, StagedFile>): string {
484+
const parts: string[] = [];
485+
for (const [path, content] of fileMap) {
486+
const text = textOf(path, content);
487+
if (text === undefined) continue;
488+
if (/\.html?$/i.test(path)) {
489+
for (const script of parseHTML(text).document.querySelectorAll("script:not([src])")) {
490+
parts.push(script.textContent ?? "");
491+
}
492+
} else if (/\.(m?js|cjs|json)$/i.test(path)) parts.push(text);
493+
}
494+
return parts.join("\n");
495+
}
496+
473497
function assertFitsAsset(path: string, filename: string, size: number): void {
474498
const maxBytes = ACCEPTED[extname(filename).toLowerCase()]?.maxBytes ?? 0;
475499
if (size > maxBytes) {
@@ -545,6 +569,7 @@ async function planFileTrim(
545569
([other, content]) =>
546570
other !== entryPath && other !== path && !!textOf(other, content)?.includes(name),
547571
),
572+
scriptCode: scriptCode(ctx.fileMap),
548573
});
549574
if (!plan) return null;
550575
ctx.fileMap.set(entryPath, Buffer.from(plan.html, "utf-8"));
@@ -650,7 +675,7 @@ function entryGifSources(
650675
const key = htmlSrcKey(ctx.absDir, entryPath, src, ctx.fileMap);
651676
if (!key?.toLowerCase().endsWith(".gif")) continue;
652677
const file = ctx.fileMap.get(key)!;
653-
sourceAssets.set(src, snapshotPath(key, file, ctx.tempFiles));
678+
sourceAssets.set(src, stagedCopy(key, file, ctx.tempFiles));
654679
heavy ||= stagedSize(file) >= OFFLOAD_MIN_BYTES;
655680
}
656681
return heavy ? sourceAssets : null;
@@ -707,28 +732,36 @@ function dropReplacedFiles(
707732
preparedGifSources: string[],
708733
assetUrls: string[],
709734
): void {
735+
// Asset URLs end in names like original.mp4 that can collide with a project file's name.
736+
const withoutUrls = (text: string) =>
737+
assetUrls.reduce((acc, url) => acc.replaceAll(url, ""), text);
738+
const code = withoutUrls(scriptCode(fileMap));
710739
for (const item of report) {
711-
if (isStillMentioned(fileMap, item.path, assetUrls)) item.kept_in_archive = true;
740+
if (isStillMentioned(fileMap, item.path, code, withoutUrls)) item.kept_in_archive = true;
712741
else fileMap.delete(item.path);
713742
}
714743
for (const gif of preparedGifSources) {
715-
if (!isStillMentioned(fileMap, gif, assetUrls)) fileMap.delete(gif);
744+
if (!isStillMentioned(fileMap, gif, code, withoutUrls)) fileMap.delete(gif);
716745
}
717746
}
718747

719-
// A file still named after rewriting (say, built by string concatenation) keeps a zip copy.
748+
/**
749+
* Whether a reference rewriting could not replace may still load the file: its name anywhere, or its
750+
* stem and extension both in script code, which can join them at runtime ("clip" + ".mp4").
751+
*/
720752
function isStillMentioned(
721753
fileMap: Map<string, StagedFile>,
722754
path: string,
723-
assetUrls: string[],
755+
code: string,
756+
withoutUrls: (text: string) => string,
724757
): boolean {
758+
const ext = extname(path);
759+
if (code.includes(basename(path, ext)) && code.includes(ext.slice(1))) return true;
725760
const name = basename(path);
726761
for (const [entryPath, content] of fileMap) {
727762
const contentText = textOf(entryPath, content);
728763
if (entryPath === path || !contentText?.includes(name)) continue;
729-
// Asset URLs end in names like original.mp4 that can collide with a project file's name.
730-
const text = assetUrls.reduce((acc, url) => acc.replaceAll(url, ""), contentText);
731-
if (text.includes(name)) return true;
764+
if (withoutUrls(contentText).includes(name)) return true;
732765
}
733766
return false;
734767
}

‎packages/cli/src/cloud/offloadTrim.ts‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,10 @@ export interface PlayedSegment {
1414
end: number;
1515
}
1616

17+
// Script that can write a clip's in-point, length, rate, loop or source moves what it plays.
18+
const RETIMING_SCRIPT_RE =
19+
/data-(?:playback-start|media-start|start|duration|end|playback-rate)|\b(?:dataset|setAttribute|currentTime|playbackRate)\b|\.(?:src|loop)\b/;
20+
1721
type AttrReader = (name: string) => string | null;
1822

1923
function mediaSources(el: Element): string[] {
@@ -97,8 +101,10 @@ export function planTrim(input: {
97101
resolveKey: (rawSrc: string) => string | null;
98102
gridPerSecond: number;
99103
mentionedElsewhere: boolean;
104+
/** Every script the page runs, entry inline scripts included. */
105+
scriptCode: string;
100106
}): { segments: PlayedSegment[]; html: string } | null {
101-
if (input.mentionedElsewhere) return null;
107+
if (input.mentionedElsewhere || RETIMING_SCRIPT_RE.test(input.scriptCode)) return null;
102108
const { document } = parseHTML(input.entryHtml);
103109
const uses: Element[] = [];
104110
let sources = 0;

0 commit comments

Comments
 (0)