Skip to content

Commit d66cd6d

Browse files
authored
fix(catalog): publish cache assets without overwriting entries (#3739)
* fix(catalog): publish cache assets without overwriting entries * fix(catalog): publish cache assets without overwriting entries
1 parent 65cc1d6 commit d66cd6d

3 files changed

Lines changed: 198 additions & 15 deletions

File tree

‎.github/workflows/windows-render.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -489,10 +489,10 @@ jobs:
489489
shell: pwsh
490490
run: bunx vitest run packages/producer/src/services/windowsFrameStaging.test.ts --maxWorkers=2
491491

492-
- name: Verify catalog source reads on Windows
492+
- name: Verify catalog file operations on Windows
493493
if: matrix.lane == 'studio-engine-cli'
494494
shell: pwsh
495-
run: bunx vitest run scripts/catalog/catalog-payload-assets.file-race.test.ts --maxWorkers=2
495+
run: bunx vitest run scripts/catalog/catalog-payload-assets.file-race.test.ts scripts/catalog/catalog-payload-assets.cache-race.test.ts --maxWorkers=2
496496

497497
- name: Run runtime contract test
498498
if: matrix.runtime_contract

‎scripts/catalog-payload-assets.ts‎

Lines changed: 33 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,11 @@
99

1010
import { createHash } from "node:crypto";
1111
import {
12-
existsSync,
1312
mkdirSync,
13+
lstatSync,
14+
linkSync,
15+
mkdtempSync,
16+
rmSync,
1417
readdirSync,
1518
readFileSync,
1619
readSync,
@@ -206,6 +209,33 @@ export interface AssetTarget {
206209
urlBase: string;
207210
}
208211

212+
/** Publish a complete cache entry without replacing a competing file or link. */
213+
function cacheAsset(bytes: Buffer<ArrayBuffer>, ext: string, target: AssetTarget): string {
214+
const name = `${createHash("sha256").update(bytes).digest("hex").slice(0, 16)}${ext}`;
215+
const dest = join(target.dir, name);
216+
// Cache hits need no writable directory. A miss still has to win linkSync.
217+
if (lstatSync(dest, { throwIfNoEntry: false })) return name;
218+
mkdirSync(target.dir, { recursive: true });
219+
if (!lstatSync(target.dir).isDirectory())
220+
throw new Error("Catalog cache must be a real directory");
221+
const staging = mkdtempSync(join(target.dir, ".hf-asset-"));
222+
try {
223+
const staged = join(staging, "content");
224+
writeFileSync(staged, bytes, { flag: "wx" });
225+
try {
226+
linkSync(staged, dest);
227+
} catch (error) {
228+
if (!(error instanceof Error && "code" in error && error.code === "EEXIST")) throw error;
229+
}
230+
} finally {
231+
// Cleanup must not mask a publication error or fail an already published asset.
232+
try {
233+
rmSync(staging, { recursive: true, force: true });
234+
} catch {}
235+
}
236+
return name;
237+
}
238+
209239
/**
210240
* Point every local reference at something the browser can fetch.
211241
*
@@ -252,12 +282,7 @@ export function processAssets(html: string, projectDir: string, target: AssetTar
252282
}
253283

254284
if (HOSTED_EXTENSIONS.has(ext)) {
255-
const name = `${createHash("sha256").update(bytes).digest("hex").slice(0, 16)}${ext}`;
256-
const dest = join(target.dir, name);
257-
if (!existsSync(dest)) {
258-
mkdirSync(target.dir, { recursive: true });
259-
writeFileSync(dest, bytes);
260-
}
285+
const name = cacheAsset(bytes, ext, target);
261286
out = out.split(ref).join(`${target.urlBase}/${name}`);
262287
hosted += 1;
263288
continue;
@@ -305,12 +330,7 @@ export function externalizeDataUris(
305330
const bytes = Buffer.from(blob, "base64");
306331
if (bytes.length < EXTERNALIZE_MIN_BYTES) return whole;
307332

308-
const name = `${createHash("sha256").update(bytes).digest("hex").slice(0, 16)}${ext}`;
309-
const dest = join(target.dir, name);
310-
if (!existsSync(dest)) {
311-
mkdirSync(target.dir, { recursive: true });
312-
writeFileSync(dest, bytes);
313-
}
333+
const name = cacheAsset(bytes, ext, target);
314334
externalized += 1;
315335
return `${target.urlBase}/${name}`;
316336
},
Lines changed: 163 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,163 @@
1+
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
2+
import { createHash } from "node:crypto";
3+
import { join, dirname } from "node:path";
4+
import { tmpdir } from "node:os";
5+
import { processAssets, externalizeDataUris } from "../catalog-payload-assets.ts";
6+
7+
const hooks = vi.hoisted(() => {
8+
const state: {
9+
dest: string;
10+
beforePublish?: () => void;
11+
failWrite: boolean;
12+
failLink: boolean;
13+
failCleanup: boolean;
14+
stagedBytes?: Buffer;
15+
stages: string[];
16+
} = { dest: "", failWrite: false, failLink: false, failCleanup: false, stages: [] };
17+
return state;
18+
});
19+
vi.mock("node:fs", async (importOriginal) => {
20+
const fs = await importOriginal<typeof import("node:fs")>();
21+
function beforePublish(): void {
22+
const hook = hooks.beforePublish;
23+
hooks.beforePublish = undefined;
24+
hook?.();
25+
}
26+
return {
27+
...fs,
28+
mkdtempSync: (...args: Parameters<typeof fs.mkdtempSync>) => {
29+
const dir = fs.mkdtempSync(...args);
30+
hooks.stages.push(String(dir));
31+
return dir;
32+
},
33+
writeFileSync: (...args: Parameters<typeof fs.writeFileSync>) => {
34+
if (String(args[0]) === hooks.dest) beforePublish();
35+
if (hooks.failWrite) {
36+
fs.writeFileSync(args[0], "partial");
37+
throw new Error("write failure");
38+
}
39+
return fs.writeFileSync(...args);
40+
},
41+
linkSync: (...args: Parameters<typeof fs.linkSync>) => {
42+
hooks.stagedBytes = fs.readFileSync(args[0]);
43+
beforePublish();
44+
if (hooks.failLink) throw new Error("link failure");
45+
return fs.linkSync(...args);
46+
},
47+
rmSync: (...args: Parameters<typeof fs.rmSync>) => {
48+
if (hooks.failCleanup) throw new Error("cleanup failure");
49+
return fs.rmSync(...args);
50+
},
51+
};
52+
});
53+
const fs = await vi.importActual<typeof import("node:fs")>("node:fs");
54+
let root: string;
55+
let project: string;
56+
let target: { dir: string; urlBase: string };
57+
const bytes = Buffer.alloc(4096, 42);
58+
const name = `${createHash("sha256").update(bytes).digest("hex").slice(0, 16)}.png`;
59+
beforeEach(() => {
60+
root = fs.mkdtempSync(join(tmpdir(), "hf-catalog-cache-"));
61+
project = join(root, "project");
62+
fs.mkdirSync(project);
63+
fs.writeFileSync(join(project, "asset.png"), bytes);
64+
target = { dir: join(root, "cache"), urlBase: "/assets" };
65+
Object.assign(hooks, {
66+
dest: join(target.dir, name),
67+
beforePublish: undefined,
68+
failWrite: false,
69+
failLink: false,
70+
failCleanup: false,
71+
stagedBytes: undefined,
72+
stages: [],
73+
});
74+
});
75+
afterEach(() => fs.rmSync(root, { recursive: true, force: true }));
76+
function publish(kind: string): string {
77+
if (kind === "reference") {
78+
const result = processAssets('<img src="asset.png">', project, target);
79+
expect(result.hosted).toBe(1);
80+
expect(result.inlined).toBe(0);
81+
expect(result.unresolved).toEqual([]);
82+
return result.html;
83+
}
84+
const result = externalizeDataUris(
85+
`<img src="data:image/png;base64,${bytes.toString("base64")}">`,
86+
target,
87+
);
88+
expect(result.externalized).toBe(1);
89+
return result.html;
90+
}
91+
function assertNoStages(): void {
92+
for (const stage of hooks.stages) expect(fs.existsSync(stage)).toBe(false);
93+
}
94+
95+
describe.each(["reference", "data URI"])("catalog %s cache publication", (kind) => {
96+
it("publishes complete bytes with the existing hash and reuses cache hits without staging", () => {
97+
expect(publish(kind)).toBe(`<img src="/assets/${name}">`);
98+
expect(hooks.stagedBytes).toEqual(bytes);
99+
expect(fs.readFileSync(hooks.dest)).toEqual(bytes);
100+
assertNoStages();
101+
hooks.failWrite = true;
102+
const count = hooks.stages.length;
103+
expect(publish(kind)).toBe(`<img src="/assets/${name}">`);
104+
expect(hooks.stages.length).toBe(count);
105+
});
106+
it.each(["file", "symlink", "dangling symlink"])("does not overwrite a competing %s", (entry) => {
107+
const outside = join(root, "outside.png");
108+
if (entry === "symlink") fs.writeFileSync(outside, "outside");
109+
hooks.beforePublish = () => {
110+
if (entry === "file") fs.writeFileSync(hooks.dest, "winner");
111+
else fs.symlinkSync(outside, hooks.dest, "file");
112+
};
113+
expect(publish(kind)).toBe(`<img src="/assets/${name}">`);
114+
expect(hooks.beforePublish).toBeUndefined();
115+
if (entry === "file") expect(fs.readFileSync(hooks.dest, "utf8")).toBe("winner");
116+
else {
117+
expect(fs.lstatSync(hooks.dest).isSymbolicLink()).toBe(true);
118+
if (entry === "symlink") expect(fs.readFileSync(outside, "utf8")).toBe("outside");
119+
else expect(fs.existsSync(outside)).toBe(false);
120+
}
121+
assertNoStages();
122+
});
123+
it("leaves a pre-existing dangling destination link untouched", () => {
124+
fs.mkdirSync(target.dir);
125+
const outside = join(root, "outside.png");
126+
fs.symlinkSync(outside, hooks.dest, "file");
127+
publish(kind);
128+
expect(fs.existsSync(outside)).toBe(false);
129+
expect(fs.lstatSync(hooks.dest).isSymbolicLink()).toBe(true);
130+
expect(hooks.stages).toEqual([]);
131+
});
132+
it("rejects a cache directory linked outside without writing there", () => {
133+
const outside = join(root, "outside");
134+
fs.mkdirSync(outside);
135+
fs.symlinkSync(outside, target.dir, process.platform === "win32" ? "junction" : "dir");
136+
expect(() => publish(kind)).toThrow("Catalog cache must be a real directory");
137+
expect(fs.readdirSync(outside)).toEqual([]);
138+
});
139+
it("removes a partial staging file after write failure without publishing it", () => {
140+
hooks.failWrite = true;
141+
expect(() => publish(kind)).toThrow("write failure");
142+
expect(fs.existsSync(hooks.dest)).toBe(false);
143+
assertNoStages();
144+
});
145+
it("preserves publication errors and cleans staging", () => {
146+
hooks.failLink = true;
147+
expect(() => publish(kind)).toThrow("link failure");
148+
expect(fs.existsSync(hooks.dest)).toBe(false);
149+
assertNoStages();
150+
});
151+
it("does not replace a publication error with a cleanup error", () => {
152+
hooks.failLink = true;
153+
hooks.failCleanup = true;
154+
expect(() => publish(kind)).toThrow("link failure");
155+
expect(fs.existsSync(hooks.dest)).toBe(false);
156+
});
157+
it("keeps a successful publication successful if cleanup fails", () => {
158+
hooks.failCleanup = true;
159+
expect(publish(kind)).toBe(`<img src="/assets/${name}">`);
160+
expect(fs.readFileSync(hooks.dest)).toEqual(bytes);
161+
expect(hooks.stages.map((stage) => dirname(stage))).toEqual([target.dir]);
162+
});
163+
});

0 commit comments

Comments
 (0)