Skip to content

Commit 7a2a691

Browse files
authored
fix(catalog): bind asset reads to checked project files (#3735)
* fix(catalog): bind asset reads to checked project files * fix(catalog): budget the exact bytes published * fix(catalog): bound directory asset reads
1 parent f1d0c2e commit 7a2a691

3 files changed

Lines changed: 448 additions & 69 deletions

File tree

‎.github/workflows/windows-render.yml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -484,6 +484,11 @@ jobs:
484484
shell: pwsh
485485
run: bunx vitest run packages/producer/src/services/hyperframeLint.file-race.test.ts --maxWorkers=2
486486

487+
- name: Verify catalog source reads on Windows
488+
if: matrix.lane == 'studio-engine-cli'
489+
shell: pwsh
490+
run: bunx vitest run scripts/catalog/catalog-payload-assets.file-race.test.ts --maxWorkers=2
491+
487492
- name: Run runtime contract test
488493
if: matrix.runtime_contract
489494
shell: pwsh

‎scripts/catalog-payload-assets.ts‎

Lines changed: 126 additions & 69 deletions
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,21 @@
88
*/
99

1010
import { createHash } from "node:crypto";
11-
import { existsSync, mkdirSync, readdirSync, readFileSync, statSync, writeFileSync } from "node:fs";
12-
import { extname, join, resolve } from "node:path";
11+
import {
12+
existsSync,
13+
mkdirSync,
14+
readdirSync,
15+
readFileSync,
16+
readSync,
17+
statSync,
18+
writeFileSync,
19+
realpathSync,
20+
openSync,
21+
fstatSync,
22+
closeSync,
23+
constants,
24+
} from "node:fs";
25+
import { extname, join, resolve, relative, isAbsolute, sep } from "node:path";
1326

1427
export const MIME_TYPES: Record<string, string> = {
1528
".png": "image/png",
@@ -127,6 +140,65 @@ export interface AssetResult {
127140
unresolved: string[];
128141
}
129142

143+
function isWithin(root: string, filePath: string): boolean {
144+
const rel = relative(root, filePath);
145+
return rel !== ".." && !rel.startsWith(`..${sep}`) && !isAbsolute(rel);
146+
}
147+
148+
function isMissingFile(error: unknown): boolean {
149+
return (
150+
error instanceof Error &&
151+
"code" in error &&
152+
(error.code === "ENOENT" || error.code === "ENOTDIR")
153+
);
154+
}
155+
156+
/** Keep oversized or concurrently growing directory assets within the read budget. */
157+
function readWithinBudget(fd: number, maxBytes: number): Buffer<ArrayBuffer> {
158+
const chunks: Buffer<ArrayBuffer>[] = [];
159+
let remaining = maxBytes + 1;
160+
while (remaining > 0) {
161+
const chunk = Buffer.alloc(Math.min(64 * 1024, remaining));
162+
const count = readSync(fd, chunk, 0, chunk.length, null);
163+
if (count === 0) break;
164+
chunks.push(chunk.subarray(0, count));
165+
remaining -= count;
166+
}
167+
return Buffer.concat(chunks);
168+
}
169+
170+
/** Read one checked file from the prepared project, including internal links. */
171+
function readProjectFile(
172+
root: string,
173+
filePath: string,
174+
maxBytes?: number,
175+
): Buffer<ArrayBuffer> | null {
176+
if (!isWithin(root, filePath)) return null;
177+
let source: string;
178+
try {
179+
source = realpathSync(filePath);
180+
if (!isWithin(realpathSync(root), source)) return null;
181+
} catch (error) {
182+
if (isMissingFile(error)) return null;
183+
throw error;
184+
}
185+
let fd: number;
186+
try {
187+
// Nonblocking mode lets fstat reject named pipes without waiting for a writer.
188+
fd = openSync(source, constants.O_RDONLY | constants.O_NONBLOCK);
189+
} catch (error) {
190+
if (isMissingFile(error)) return null;
191+
if (!statSync(source, { throwIfNoEntry: false })?.isFile()) return null;
192+
throw error;
193+
}
194+
try {
195+
if (!fstatSync(fd).isFile()) return null;
196+
return maxBytes === undefined ? readFileSync(fd) : readWithinBudget(fd, maxBytes);
197+
} finally {
198+
closeSync(fd);
199+
}
200+
}
201+
130202
export interface AssetTarget {
131203
/** Directory shared by every item, so one font is stored once. */
132204
dir: string;
@@ -164,8 +236,8 @@ export function processAssets(html: string, projectDir: string, target: AssetTar
164236

165237
// A composition reaching outside its own directory would pull an arbitrary
166238
// file from the build machine into a published payload.
167-
const contained = source === root || source.startsWith(`${root}/`);
168-
if (!contained || !existsSync(source) || !statSync(source).isFile()) {
239+
const bytes = readProjectFile(root, source);
240+
if (bytes === null) {
169241
// A name a script passed around that turned out not to be a file is just
170242
// a string; only a reference we are sure about counts as a broken one.
171243
if (strict) unresolved.push(ref);
@@ -179,7 +251,6 @@ export function processAssets(html: string, projectDir: string, target: AssetTar
179251
continue;
180252
}
181253

182-
const bytes = readFileSync(source);
183254
if (HOSTED_EXTENSIONS.has(ext)) {
184255
const name = `${createHash("sha256").update(bytes).digest("hex").slice(0, 16)}${ext}`;
185256
const dest = join(target.dir, name);
@@ -247,21 +318,30 @@ export function externalizeDataUris(
247318
return { html: out, externalized };
248319
}
249320

250-
/** Copy a directory's publishable files into `destDir`, flattening one level. */
251-
function walkInto(from: string, rel: string, destDir: string, onCopy: () => void): void {
321+
/** Enumerate publishable paths without using pathname sizes to authorize reads. */
322+
function* hostedPaths(from: string, rel = ""): Generator<string> {
252323
for (const entry of readdirSync(from, { withFileTypes: true })) {
253324
if (entry.isSymbolicLink()) continue;
254325
const childRel = rel ? `${rel}/${entry.name}` : entry.name;
255-
const childFrom = join(from, entry.name);
256326
if (entry.isDirectory()) {
257-
walkInto(childFrom, childRel, destDir, onCopy);
258-
continue;
327+
yield* hostedPaths(join(from, entry.name), childRel);
328+
} else if (HOSTED_EXTENSIONS.has(extname(entry.name).toLowerCase())) {
329+
yield childRel;
259330
}
260-
if (!HOSTED_EXTENSIONS.has(extname(entry.name).toLowerCase())) continue;
261-
const to = join(destDir, childRel);
262-
mkdirSync(join(to, ".."), { recursive: true });
263-
writeFileSync(to, readFileSync(childFrom));
264-
onCopy();
331+
}
332+
}
333+
334+
/** Internal directory aliases share the buffers already collected at their real paths. */
335+
function downloadMirrorPrefix(projectDir: string): string | null {
336+
try {
337+
const root = realpathSync(projectDir);
338+
const downloads = realpathSync(join(projectDir, "_downloads"));
339+
if (!isWithin(root, downloads) || !statSync(downloads).isDirectory()) return null;
340+
const rel = relative(root, downloads).split(sep).join("/");
341+
return rel ? `${rel}/` : "";
342+
} catch (error) {
343+
if (isMissingFile(error)) return null;
344+
throw error;
265345
}
266346
}
267347

@@ -278,22 +358,6 @@ function walkInto(from: string, rel: string, destDir: string, onCopy: () => void
278358
* Only publishable types are copied; a composition needing something the host
279359
* drops still falls back to inlining, which is handled by the caller.
280360
*/
281-
/** Publishable bytes an item would add, counted before anything is written. */
282-
function directoryBytes(dir: string): number {
283-
let total = 0;
284-
for (const entry of readdirSync(dir, { withFileTypes: true })) {
285-
if (entry.isSymbolicLink()) continue;
286-
const child = join(dir, entry.name);
287-
if (entry.isDirectory()) {
288-
total += directoryBytes(child);
289-
continue;
290-
}
291-
if (!HOSTED_EXTENSIONS.has(extname(entry.name).toLowerCase())) continue;
292-
total += statSync(child).size;
293-
}
294-
return total;
295-
}
296-
297361
/**
298362
* What an item may add by publishing its own directory.
299363
*
@@ -304,45 +368,37 @@ function directoryBytes(dir: string): number {
304368
export const MAX_HOSTED_DIRECTORY_BYTES = 2_000_000;
305369

306370
export function hostItemDirectory(projectDir: string, destDir: string, urlBase: string): string {
307-
if (directoryBytes(projectDir) > MAX_HOSTED_DIRECTORY_BYTES) return "";
308-
let copied = 0;
309-
310-
const walk = (from: string, rel: string): void => {
311-
for (const entry of readdirSync(from, { withFileTypes: true })) {
312-
// Nothing here should ever leave the prepared copy, and a symlink is the
313-
// one entry that could point back out of it.
314-
if (entry.isSymbolicLink()) continue;
315-
const childRel = rel ? `${rel}/${entry.name}` : entry.name;
316-
const childFrom = join(from, entry.name);
317-
if (entry.isDirectory()) {
318-
walk(childFrom, childRel);
319-
continue;
320-
}
321-
if (!HOSTED_EXTENSIONS.has(extname(entry.name).toLowerCase())) continue;
322-
const to = join(destDir, childRel);
323-
mkdirSync(join(to, ".."), { recursive: true });
324-
writeFileSync(to, readFileSync(childFrom));
325-
copied += 1;
326-
}
371+
const mirrorPrefix = downloadMirrorPrefix(projectDir);
372+
const files = new Map<string, Buffer<ArrayBuffer>>();
373+
let total = 0;
374+
for (const path of hostedPaths(projectDir)) {
375+
const bytes = readProjectFile(
376+
projectDir,
377+
join(projectDir, path),
378+
MAX_HOSTED_DIRECTORY_BYTES - total,
379+
);
380+
if (bytes === null) continue;
381+
total += bytes.length;
382+
if (total > MAX_HOSTED_DIRECTORY_BYTES) return "";
383+
files.set(path, bytes);
384+
}
385+
386+
// Charge each source once, as before. Publish only after the entire item fits,
387+
// and reuse the collected bytes for both registry and install-path layouts.
388+
const publish = (path: string, bytes: Buffer<ArrayBuffer>): void => {
389+
const to = join(destDir, path);
390+
mkdirSync(join(to, ".."), { recursive: true });
391+
writeFileSync(to, bytes);
327392
};
393+
for (const [path, bytes] of files) publish(path, bytes);
328394

329-
// Both layouts are published. The prepared copy holds each asset twice, once
330-
// as the registry stores it and once at its install path, and which one a
331-
// composition asks for differs per item: the texture masks use the registry
332-
// spelling, the caption textures the install one. Guessing wrong is a 404 at
333-
// run time, so the budget below is what keeps the cost in check instead.
334-
walk(projectDir, "");
335-
336-
// The compiler pulls remote media into `_downloads/` but rewrites references
337-
// as if the document sat inside it, so `_remote_media/x.woff2` has to resolve
338-
// from the item root too. Mirroring rather than moving keeps both spellings
339-
// working, and the files are content-identical either way.
340-
const downloads = join(projectDir, "_downloads");
341-
if (existsSync(downloads) && statSync(downloads).isDirectory()) {
342-
walkInto(downloads, "", destDir, () => (copied += 1));
395+
// Compiler references omit `_downloads/`. Preserve both spellings and the
396+
// existing mirror-wins collision order without reopening any source file.
397+
for (const [path, bytes] of files) {
398+
if (mirrorPrefix !== null && path.startsWith(mirrorPrefix))
399+
publish(path.slice(mirrorPrefix.length), bytes);
343400
}
344-
345-
return copied > 0 ? urlBase : "";
401+
return files.size > 0 ? urlBase : "";
346402
}
347403

348404
/**
@@ -376,8 +432,9 @@ export function inlineMountedComposition(html: string, projectDir: string): stri
376432
(whole, quote: string, ref: string) => {
377433
if (/^(https?:|data:)/i.test(ref)) return whole;
378434
const source = resolve(projectDir, ref.replace(/^\.\//, "").split(/[?#]/)[0] ?? ref);
379-
if (!source.startsWith(resolve(projectDir)) || !existsSync(source)) return whole;
380-
const encoded = readFileSync(source).toString("base64");
435+
const bytes = readProjectFile(resolve(projectDir), source);
436+
if (bytes === null) return whole;
437+
const encoded = bytes.toString("base64");
381438
return `data-composition-src=${quote}data:text/html;base64,${encoded}${quote}`;
382439
},
383440
);

0 commit comments

Comments
 (0)