Skip to content

Commit cd1a996

Browse files
authored
fix(catalog): confine hosted preview asset downloads (#3843)
* fix(catalog): confine hosted preview asset downloads * test(catalog): pin credential and port refusal with finite streams
1 parent 96637ae commit cd1a996

4 files changed

Lines changed: 251 additions & 41 deletions

File tree

‎package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@
4848
"player:perf": "bun run --filter @hyperframes/player perf",
4949
"format:check": "oxfmt --check .",
5050
"knip": "knip",
51-
"test:scripts": "node --import tsx --test scripts/animejs-v4-guidance.test.mjs scripts/check-tracked-artifacts.test.mjs scripts/check-no-main-deletions.test.mjs scripts/check-docs-snippet-motion.test.mjs scripts/registry-target-paths.test.mjs scripts/check-workspace-contracts.test.mjs scripts/check-package-cycles.test.mjs scripts/check-cli-process-ownership.test.mjs scripts/check-large-files.test.mjs scripts/package-subpaths.test.mjs scripts/validate-release-channel.test.mjs scripts/publish-workflow.test.mjs scripts/install-workspace-dependencies.test.mjs scripts/draft-changelog.test.ts scripts/set-version.test.ts scripts/release-prepare.test.ts scripts/cli-options.test.ts scripts/changelog-weekly.test.ts scripts/claude-plugin-compression.test.ts scripts/catalog-payload-assets.test.ts scripts/host-registry-assets.test.ts scripts/catalog-preview-temp.test.ts scripts/player-cdn-pin.test.ts scripts/studio-runtime-smoke.test.mjs scripts/verify-packed-manifests.test.mjs scripts/lint-skills.test.mjs packages/gcp-cloud-run/check-dockerfile-workspaces.test.mjs packages/core/scripts/writeGeneratedFile.test.ts && vitest run scripts/catalog/",
51+
"test:scripts": "node --import tsx --test scripts/animejs-v4-guidance.test.mjs scripts/check-tracked-artifacts.test.mjs scripts/check-no-main-deletions.test.mjs scripts/check-docs-snippet-motion.test.mjs scripts/registry-target-paths.test.mjs scripts/check-workspace-contracts.test.mjs scripts/check-package-cycles.test.mjs scripts/check-cli-process-ownership.test.mjs scripts/check-large-files.test.mjs scripts/package-subpaths.test.mjs scripts/validate-release-channel.test.mjs scripts/publish-workflow.test.mjs scripts/install-workspace-dependencies.test.mjs scripts/draft-changelog.test.ts scripts/set-version.test.ts scripts/release-prepare.test.ts scripts/cli-options.test.ts scripts/changelog-weekly.test.ts scripts/claude-plugin-compression.test.ts scripts/catalog-payload-assets.test.ts scripts/host-registry-assets.test.ts scripts/catalog-preview-temp.test.ts scripts/catalog-hosted-files.test.ts scripts/player-cdn-pin.test.ts scripts/studio-runtime-smoke.test.mjs scripts/verify-packed-manifests.test.mjs scripts/lint-skills.test.mjs packages/gcp-cloud-run/check-dockerfile-workspaces.test.mjs packages/core/scripts/writeGeneratedFile.test.ts && vitest run scripts/catalog/",
5252
"typecheck:scripts": "tsc --noEmit -p scripts/tsconfig.json",
5353
"test:skills": "node --test 'skills/**/*.test.mjs'",
5454
"generate:previews": "tsx scripts/generate-template-previews.ts",
Lines changed: 155 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,155 @@
1+
import assert from "node:assert/strict";
2+
import { mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from "node:fs";
3+
import { tmpdir } from "node:os";
4+
import { join } from "node:path";
5+
import { test } from "node:test";
6+
import { fetchHostedFiles } from "./catalog-hosted-files.js";
7+
8+
function fixture(url = "https://static.heygen.ai/asset", path = "asset.bin") {
9+
const root = mkdtempSync(join(tmpdir(), "hf-hosted-test-"));
10+
writeFileSync(join(root, "registry-item.json"), JSON.stringify({ files: [{ url, path }] }));
11+
return root;
12+
}
13+
14+
test("downloads exact bytes through a relative CDN redirect", async (t) => {
15+
const root = fixture();
16+
const calls: string[] = [];
17+
t.mock.method(
18+
globalThis,
19+
"fetch",
20+
async (url: Parameters<typeof fetch>[0], options?: RequestInit) => {
21+
calls.push(String(url));
22+
assert.equal(options?.redirect, "manual");
23+
assert.ok(options?.signal);
24+
return calls.length === 1
25+
? new Response(null, { status: 302, headers: { location: "/final" } })
26+
: new Response(new Uint8Array([0, 255, 42]));
27+
},
28+
);
29+
try {
30+
await fetchHostedFiles(root);
31+
assert.deepEqual(calls, ["https://static.heygen.ai/asset", "https://static.heygen.ai/final"]);
32+
assert.deepEqual(readFileSync(join(root, "asset.bin")), Buffer.from([0, 255, 42]));
33+
} finally {
34+
rmSync(root, { recursive: true, force: true });
35+
}
36+
});
37+
38+
for (const location of [
39+
"http://169.254.169.254/meta",
40+
"https://localhost/x",
41+
"https://static.heygen.ai.evil.test/x",
42+
"http://static.heygen.ai/x",
43+
]) {
44+
test(`rejects redirect to ${location} before a second request`, async (t) => {
45+
const root = fixture();
46+
const mock = t.mock.method(
47+
globalThis,
48+
"fetch",
49+
async () => new Response(null, { status: 302, headers: { location } }),
50+
);
51+
try {
52+
await assert.rejects(fetchHostedFiles(root), /HTTPS CDN/);
53+
assert.equal(mock.mock.callCount(), 1);
54+
assert.deepEqual(readdirSync(root), ["registry-item.json"]);
55+
} finally {
56+
rmSync(root, { recursive: true, force: true });
57+
}
58+
});
59+
}
60+
61+
for (const url of [
62+
"https://127.0.0.1/x",
63+
"https://user:[email protected]/x",
64+
"https://static.heygen.ai:8443/x",
65+
]) {
66+
test(`rejects disallowed initial URL ${url} without a request`, async (t) => {
67+
const root = fixture(url);
68+
const mock = t.mock.method(globalThis, "fetch", async () => assert.fail("unexpected request"));
69+
try {
70+
await assert.rejects(fetchHostedFiles(root), /HTTPS CDN/);
71+
assert.equal(mock.mock.callCount(), 0);
72+
} finally {
73+
rmSync(root, { recursive: true, force: true });
74+
}
75+
});
76+
}
77+
78+
test("enforces actual streaming bytes even when Content-Length understates them", async (t) => {
79+
const root = fixture();
80+
let cancelled = false;
81+
let chunks = 0;
82+
const chunk = new Uint8Array(129 * 1024 * 1024);
83+
t.mock.method(
84+
globalThis,
85+
"fetch",
86+
async () =>
87+
new Response(
88+
new ReadableStream({
89+
pull(controller) {
90+
if (chunks++ < 3) controller.enqueue(chunk);
91+
else controller.close();
92+
},
93+
cancel() {
94+
cancelled = true;
95+
},
96+
}),
97+
{ headers: { "content-length": "1" } },
98+
),
99+
);
100+
try {
101+
await assert.rejects(fetchHostedFiles(root), /256 MiB/);
102+
assert.equal(cancelled, true);
103+
assert.deepEqual(readdirSync(root), ["registry-item.json"]);
104+
} finally {
105+
rmSync(root, { recursive: true, force: true });
106+
}
107+
});
108+
109+
for (const mode of ["declared oversize", "loop", "HTTP error", "stream error"]) {
110+
test(`${mode} preserves the existing asset`, async (t) => {
111+
const root = fixture();
112+
writeFileSync(join(root, "asset.bin"), "previous");
113+
let calls = 0;
114+
let cancelled = false;
115+
t.mock.method(globalThis, "fetch", async () => {
116+
calls++;
117+
const body = new ReadableStream({
118+
start(controller) {
119+
if (mode === "stream error") controller.error(new Error("broken stream"));
120+
},
121+
cancel() {
122+
cancelled = true;
123+
},
124+
});
125+
if (mode === "loop")
126+
return new Response(body, { status: 302, headers: { location: "/asset" } });
127+
if (mode === "HTTP error") return new Response(body, { status: 500 });
128+
return new Response(body, {
129+
headers: {
130+
"content-length": mode === "declared oversize" ? String(257 * 1024 * 1024) : "1",
131+
},
132+
});
133+
});
134+
try {
135+
await assert.rejects(fetchHostedFiles(root), /256 MiB|redirect limit|HTTP 500|broken stream/);
136+
assert.equal(readFileSync(join(root, "asset.bin"), "utf8"), "previous");
137+
assert.equal(calls, mode === "loop" ? 6 : 1);
138+
if (mode !== "stream error") assert.equal(cancelled, true);
139+
} finally {
140+
rmSync(root, { recursive: true, force: true });
141+
}
142+
});
143+
}
144+
145+
test("skips manifest paths outside the copied project before fetching", async (t) => {
146+
const root = fixture("https://static.heygen.ai/asset", "../escape.bin");
147+
const mock = t.mock.method(globalThis, "fetch", async () => assert.fail("unexpected request"));
148+
try {
149+
await fetchHostedFiles(root);
150+
assert.equal(mock.mock.callCount(), 0);
151+
assert.deepEqual(readdirSync(root), ["registry-item.json"]);
152+
} finally {
153+
rmSync(root, { recursive: true, force: true });
154+
}
155+
});

‎scripts/catalog-hosted-files.ts‎

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
import { existsSync, readFileSync, realpathSync, mkdirSync, writeFileSync } from "node:fs";
2+
import { dirname, join, resolve } from "node:path";
3+
import { isContainedIn } from "./registry-target-paths.mjs";
4+
5+
// All published registry assets use this CDN. Keep every redirect on the same
6+
// HTTPS origin; a contributor manifest must not select a runner-local service.
7+
const ASSET_ORIGIN = "https://static.heygen.ai";
8+
// Match the media-freeze ceiling to allow future video assets while bounding runner memory.
9+
const MAX_ASSET_BYTES = 256 * 1024 * 1024;
10+
11+
function assertCdnUrl(url: URL): void {
12+
if (url.origin !== ASSET_ORIGIN || url.username || url.password) {
13+
throw new Error("Hosted asset URL must use the registry HTTPS CDN");
14+
}
15+
}
16+
17+
async function readHostedBody(response: Response): Promise<Uint8Array> {
18+
await validateResponse(response);
19+
const chunks: Uint8Array[] = [];
20+
let bytes = 0;
21+
if (response.body) {
22+
for await (const chunk of response.body) {
23+
bytes += chunk.byteLength;
24+
if (bytes > MAX_ASSET_BYTES) throw new Error("Hosted asset exceeds 256 MiB");
25+
chunks.push(chunk);
26+
}
27+
}
28+
return Buffer.concat(chunks, bytes);
29+
}
30+
31+
async function validateResponse(response: Response): Promise<void> {
32+
if (!response.ok) {
33+
await response.body?.cancel();
34+
throw new Error(`Hosted asset fetch failed: HTTP ${response.status}`);
35+
}
36+
if (Number(response.headers.get("content-length")) > MAX_ASSET_BYTES) {
37+
await response.body?.cancel();
38+
throw new Error("Hosted asset exceeds 256 MiB");
39+
}
40+
}
41+
42+
async function redirectTarget(response: Response, current: URL): Promise<URL> {
43+
await response.body?.cancel();
44+
const location = response.headers.get("location");
45+
if (!location) throw new Error("Hosted asset redirect has no Location");
46+
return new URL(location, current);
47+
}
48+
49+
async function fetchHostedAsset(url: string): Promise<Uint8Array> {
50+
const signal = AbortSignal.timeout(120_000);
51+
let current = new URL(url);
52+
for (let hop = 0; hop <= 5; hop++) {
53+
assertCdnUrl(current);
54+
const response = await fetch(current.href, { redirect: "manual", signal });
55+
if (response.status >= 300 && response.status < 400) {
56+
current = await redirectTarget(response, current);
57+
} else {
58+
return readHostedBody(response);
59+
}
60+
}
61+
throw new Error("Hosted asset exceeded redirect limit");
62+
}
63+
64+
interface HostedFile {
65+
path: string;
66+
url: string;
67+
}
68+
69+
function hasHostedFields(file: { path?: string; url?: string }): file is HostedFile {
70+
return typeof file.path === "string" && typeof file.url === "string";
71+
}
72+
73+
function hostedFilesOf(root: string): HostedFile[] {
74+
const manifestPath = join(root, "registry-item.json");
75+
if (!existsSync(manifestPath)) return [];
76+
const manifest = JSON.parse(readFileSync(manifestPath, "utf8")) as {
77+
files?: { path?: string; url?: string }[];
78+
};
79+
return (manifest.files ?? [])
80+
.filter(hasHostedFields)
81+
.filter((file) => file.url.startsWith("https://") && isContainedIn(root, file.path));
82+
}
83+
84+
/** Materialize manifest-hosted bytes only inside the copied preview project. */
85+
export async function fetchHostedFiles(projectDir: string): Promise<void> {
86+
const root = realpathSync(projectDir);
87+
for (const file of hostedFilesOf(root)) {
88+
const bytes = await fetchHostedAsset(file.url);
89+
const destination = resolve(root, file.path);
90+
mkdirSync(dirname(destination), { recursive: true });
91+
writeFileSync(destination, bytes);
92+
}
93+
}

‎scripts/generate-catalog-previews.ts‎

Lines changed: 2 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,8 @@ import {
4242
executeRenderJob,
4343
} from "../packages/producer/src/index.js";
4444
import { compileForRender } from "../packages/producer/src/services/htmlCompiler.js";
45-
import { isContainedIn, resolveContainedCopies } from "./registry-target-paths.mjs";
45+
import { resolveContainedCopies } from "./registry-target-paths.mjs";
46+
import { fetchHostedFiles } from "./catalog-hosted-files.js";
4647
import { withHostedDefaults } from "./registry-hosted-assets.ts";
4748
import type { RegistryItem } from "../packages/core/src/index.js";
4849
import { openOpaqueCapture } from "./preview-capture.js";
@@ -132,45 +133,6 @@ function outputDir(kind: ItemKind): string {
132133
return resolve(repoRoot, "docs/images/catalog", typeDir);
133134
}
134135

135-
/**
136-
* Download the item's CDN-hosted files into the copied project.
137-
*
138-
* `cpSync` only carries what is committed. A file declaring `url` deliberately
139-
* is not, so without this the preview renders the composition with every image
140-
* missing and the failure looks like a layout bug rather than an absent file.
141-
*
142-
* Same containment rule as the target mirror, and for the same reason: this
143-
* runs on `pull_request`, so `files[].path` is a contributor's string deciding
144-
* where bytes land on the runner.
145-
*/
146-
function hostedFilesOf(projectDir: string): { path: string; url: string }[] {
147-
const manifestPath = join(projectDir, "registry-item.json");
148-
if (!existsSync(manifestPath)) return [];
149-
150-
const manifest = JSON.parse(readFileSync(manifestPath, "utf-8")) as {
151-
files?: { path?: string; url?: string }[];
152-
};
153-
return (manifest.files ?? []).filter(
154-
(file): file is { path: string; url: string } =>
155-
typeof file.path === "string" &&
156-
typeof file.url === "string" &&
157-
file.url.startsWith("https://") &&
158-
isContainedIn(projectDir, file.path),
159-
);
160-
}
161-
162-
async function fetchHostedFiles(projectDir: string): Promise<void> {
163-
for (const file of hostedFilesOf(projectDir)) {
164-
const res = await fetch(file.url);
165-
if (!res.ok) {
166-
throw new Error(`Hosted asset fetch failed: ${file.url} — HTTP ${res.status}`);
167-
}
168-
const destPath = resolve(projectDir, file.path);
169-
mkdirSync(dirname(destPath), { recursive: true });
170-
writeFileSync(destPath, new Uint8Array(await res.arrayBuffer()));
171-
}
172-
}
173-
174136
/**
175137
* Rewrite the composition's variable defaults from local names to CDN URLs.
176138
*

0 commit comments

Comments
 (0)