Skip to content

Commit 962c954

Browse files
authored
fix(registry): verify staged asset content before publishing manifests (#3832)
* fix(registry): verify staged asset content before publishing manifests * test(registry): run asset staging regression in script CI * refactor(registry): separate verified manifest publication
1 parent 8bf5b44 commit 962c954

3 files changed

Lines changed: 112 additions & 24 deletions

File tree

‎package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@
4848
"player:perf": "bun run --filter @hyperframes/player perf",
4949
"format:check": "oxfmt --check .",
5050
"knip": "knip",
51-
"test:scripts": "node --import tsx --test scripts/animejs-v4-guidance.test.mjs scripts/check-tracked-artifacts.test.mjs scripts/check-no-main-deletions.test.mjs scripts/check-docs-snippet-motion.test.mjs scripts/registry-target-paths.test.mjs scripts/check-workspace-contracts.test.mjs scripts/check-package-cycles.test.mjs scripts/check-cli-process-ownership.test.mjs scripts/check-large-files.test.mjs scripts/package-subpaths.test.mjs scripts/validate-release-channel.test.mjs scripts/publish-workflow.test.mjs scripts/install-workspace-dependencies.test.mjs scripts/draft-changelog.test.ts scripts/set-version.test.ts scripts/release-prepare.test.ts scripts/cli-options.test.ts scripts/changelog-weekly.test.ts scripts/claude-plugin-compression.test.ts scripts/catalog-payload-assets.test.ts scripts/catalog-preview-temp.test.ts scripts/player-cdn-pin.test.ts scripts/studio-runtime-smoke.test.mjs scripts/verify-packed-manifests.test.mjs scripts/lint-skills.test.mjs packages/gcp-cloud-run/check-dockerfile-workspaces.test.mjs packages/core/scripts/writeGeneratedFile.test.ts && vitest run scripts/catalog/",
51+
"test:scripts": "node --import tsx --test scripts/animejs-v4-guidance.test.mjs scripts/check-tracked-artifacts.test.mjs scripts/check-no-main-deletions.test.mjs scripts/check-docs-snippet-motion.test.mjs scripts/registry-target-paths.test.mjs scripts/check-workspace-contracts.test.mjs scripts/check-package-cycles.test.mjs scripts/check-cli-process-ownership.test.mjs scripts/check-large-files.test.mjs scripts/package-subpaths.test.mjs scripts/validate-release-channel.test.mjs scripts/publish-workflow.test.mjs scripts/install-workspace-dependencies.test.mjs scripts/draft-changelog.test.ts scripts/set-version.test.ts scripts/release-prepare.test.ts scripts/cli-options.test.ts scripts/changelog-weekly.test.ts scripts/claude-plugin-compression.test.ts scripts/catalog-payload-assets.test.ts scripts/host-registry-assets.test.ts scripts/catalog-preview-temp.test.ts scripts/player-cdn-pin.test.ts scripts/studio-runtime-smoke.test.mjs scripts/verify-packed-manifests.test.mjs scripts/lint-skills.test.mjs packages/gcp-cloud-run/check-dockerfile-workspaces.test.mjs packages/core/scripts/writeGeneratedFile.test.ts && vitest run scripts/catalog/",
5252
"typecheck:scripts": "tsc --noEmit -p scripts/tsconfig.json",
5353
"test:skills": "node --test 'skills/**/*.test.mjs'",
5454
"generate:previews": "tsx scripts/generate-template-previews.ts",
Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
import assert from "node:assert/strict";
2+
import { createHash } from "node:crypto";
3+
import fs from "node:fs";
4+
import { syncBuiltinESMExports } from "node:module";
5+
import { tmpdir } from "node:os";
6+
import { join, resolve } from "node:path";
7+
import { test } from "node:test";
8+
import { pathToFileURL } from "node:url";
9+
10+
const repoRoot = resolve(import.meta.dirname, "..");
11+
12+
async function fixture() {
13+
const root = fs.realpathSync(fs.mkdtempSync(join(tmpdir(), "hf-host-assets-")));
14+
const item = join(root, "registry/blocks/test-item");
15+
fs.mkdirSync(item, { recursive: true });
16+
fs.mkdirSync(join(root, "scripts"));
17+
for (const name of ["host-registry-assets.ts", "entrypoint.ts"]) {
18+
fs.copyFileSync(join(repoRoot, "scripts", name), join(root, "scripts", name));
19+
}
20+
for (const name of ["packages", "node_modules"]) {
21+
fs.symlinkSync(join(repoRoot, name), join(root, name), "junction");
22+
}
23+
const source = join(item, "asset.png");
24+
const manifestPath = join(item, "registry-item.json");
25+
const bytes = Buffer.from("original asset bytes");
26+
fs.writeFileSync(source, bytes);
27+
const manifest = JSON.stringify({ name: "test-item", files: [{ path: "asset.png" }] });
28+
fs.writeFileSync(manifestPath, manifest);
29+
const { main } = await import(pathToFileURL(join(root, "scripts/host-registry-assets.ts")).href);
30+
return { root, source, manifestPath, manifest, bytes, main };
31+
}
32+
33+
test("staged bytes match their key, dry-run is unchanged, and no-upload preserves sources", async () => {
34+
const f = await fixture();
35+
try {
36+
await f.main(["test-item", "--dry-run"]);
37+
assert.equal(fs.readFileSync(f.manifestPath, "utf8"), f.manifest);
38+
assert.equal(fs.existsSync(join(f.root, ".registry-assets-staging")), false);
39+
await f.main(["test-item", "--no-upload"]);
40+
const key = `${createHash("sha256").update(f.bytes).digest("hex").slice(0, 16)}.png`;
41+
assert.deepEqual(fs.readFileSync(join(f.root, ".registry-assets-staging", key)), f.bytes);
42+
assert.deepEqual(fs.readFileSync(f.source), f.bytes);
43+
const manifest = JSON.parse(fs.readFileSync(f.manifestPath, "utf8"));
44+
assert.equal(
45+
manifest.files[0].url,
46+
`https://static.heygen.ai/hyperframes-oss/registry-assets/${key}`,
47+
);
48+
} finally {
49+
fs.rmSync(f.root, { recursive: true, force: true });
50+
}
51+
});
52+
53+
test("source replacement between planning and staging aborts before manifest publication", async (t) => {
54+
const f = await fixture();
55+
const copy = fs.copyFileSync;
56+
t.mock.method(fs, "copyFileSync", (...args: Parameters<typeof copy>) => {
57+
if (args[0] === f.source) fs.writeFileSync(f.source, "replacement bytes");
58+
return copy(...args);
59+
});
60+
syncBuiltinESMExports();
61+
try {
62+
await assert.rejects(f.main(["test-item", "--no-upload"]), /Asset changed while staging/);
63+
assert.equal(fs.readFileSync(f.manifestPath, "utf8"), f.manifest);
64+
assert.equal(fs.readFileSync(f.source, "utf8"), "replacement bytes");
65+
assert.deepEqual(fs.readdirSync(join(f.root, ".registry-assets-staging")), []);
66+
} finally {
67+
t.mock.restoreAll();
68+
syncBuiltinESMExports();
69+
fs.rmSync(f.root, { recursive: true, force: true });
70+
}
71+
});

‎scripts/host-registry-assets.ts‎

Lines changed: 40 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -33,12 +33,15 @@
3333
import { createHash } from "node:crypto";
3434
import {
3535
copyFileSync,
36+
constants,
37+
closeSync,
38+
fstatSync,
39+
openSync,
3640
existsSync,
3741
mkdirSync,
3842
readdirSync,
3943
readFileSync,
4044
rmSync,
41-
statSync,
4245
unlinkSync,
4346
writeFileSync,
4447
} from "node:fs";
@@ -87,6 +90,8 @@ interface Plan {
8790
source: string;
8891
/** Content-addressed object key, without the bucket prefix. */
8992
key: string;
93+
/** Full digest of the bytes used to derive the key. */
94+
digest: string;
9095
/** Public URL the manifest will carry. */
9196
url: string;
9297
}
@@ -124,28 +129,41 @@ function needsHosting(file: FileTarget): boolean {
124129
/** Point one file at the CDN, and say which bytes have to get there. */
125130
function hostFile(itemName: string, itemDir: string, file: FileTarget): Plan {
126131
const source = resolve(itemDir, file.path);
127-
if (!existsSync(source) || !statSync(source).isFile()) {
128-
throw new Error(`${itemName}: files[] declares "${file.path}", which is not on disk.`);
132+
const fd = openSync(source, constants.O_RDONLY | constants.O_NONBLOCK);
133+
let digest: string;
134+
try {
135+
if (!fstatSync(fd).isFile()) {
136+
throw new Error(`${itemName}: files[] declares "${file.path}", which is not a file.`);
137+
}
138+
digest = createHash("sha256").update(readFileSync(fd)).digest("hex");
139+
} finally {
140+
closeSync(fd);
129141
}
130-
const digest = createHash("sha256").update(readFileSync(source)).digest("hex").slice(0, 16);
131-
const key = `${digest}${extname(file.path).toLowerCase()}`;
142+
const key = `${digest.slice(0, 16)}${extname(file.path).toLowerCase()}`;
132143
file.url = `${CDN_BASE}/${key}`;
133-
return { source, key, url: file.url };
144+
return { source, key, digest, url: file.url };
134145
}
135146

136-
/** Rewrite one item's manifest, and report which bytes have to reach the CDN. */
137-
function planItem(itemDir: string): Plan[] {
147+
/** Prepare a manifest update and identify the bytes that must reach the CDN. */
148+
function planItem(itemDir: string): {
149+
plans: Plan[];
150+
manifestPath: string;
151+
manifest: RegistryItem;
152+
} {
138153
const manifestPath = join(itemDir, "registry-item.json");
139154
const manifest = JSON.parse(readFileSync(manifestPath, "utf-8")) as RegistryItem;
140155

141156
const plans = (manifest.files ?? [])
142157
.filter(needsHosting)
143158
.map((file) => hostFile(manifest.name, itemDir, file));
144159

145-
if (plans.length > 0) {
160+
return { plans, manifestPath, manifest };
161+
}
162+
163+
function writeManifests(items: ReturnType<typeof planItem>[]): void {
164+
for (const { manifestPath, manifest } of items) {
146165
writeFileSync(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`, "utf-8");
147166
}
148-
return plans;
149167
}
150168

151169
/** Collect every planned object into one flat directory, deduplicated by key. */
@@ -156,7 +174,13 @@ function stage(plans: Plan[]): number {
156174
for (const plan of plans) {
157175
if (seen.has(plan.key)) continue;
158176
seen.add(plan.key);
159-
copyFileSync(plan.source, join(STAGING_DIR, plan.key));
177+
const stagedPath = join(STAGING_DIR, plan.key);
178+
copyFileSync(plan.source, stagedPath);
179+
const digest = createHash("sha256").update(readFileSync(stagedPath)).digest("hex");
180+
if (digest !== plan.digest) {
181+
unlinkSync(stagedPath);
182+
throw new Error(`Asset changed while staging: ${plan.source}. No upload was attempted.`);
183+
}
160184
}
161185
return seen.size;
162186
}
@@ -186,19 +210,10 @@ function upload(): void {
186210
);
187211
}
188212

189-
/**
190-
* Report what would move, changing nothing.
191-
*
192-
* Planning writes the manifest, so a dry run restores each one after asking
193-
* it. Cheaper than a second code path that could disagree with the real one
194-
* about what counts as hostable.
195-
*/
213+
/** Report what would move, changing nothing. */
196214
function reportDryRun(itemDirs: string[]): void {
197215
for (const dir of itemDirs) {
198-
const manifestPath = join(dir, "registry-item.json");
199-
const before = readFileSync(manifestPath, "utf-8");
200-
const plans = planItem(dir);
201-
writeFileSync(manifestPath, before, "utf-8");
216+
const { plans } = planItem(dir);
202217
if (plans.length > 0) {
203218
console.log(`${dir.replace(`${repoRoot}/`, "")}: ${plans.length} file(s) would be hosted`);
204219
}
@@ -228,13 +243,15 @@ export async function main(argv: string[]): Promise<void> {
228243
return;
229244
}
230245

231-
const plans = itemDirs.flatMap(planItem);
246+
const items = itemDirs.map(planItem).filter((item) => item.plans.length > 0);
247+
const plans = items.flatMap((item) => item.plans);
232248
if (plans.length === 0) {
233249
console.log("No unhosted binary assets found — nothing to do.");
234250
return;
235251
}
236252

237253
const objects = stage(plans);
254+
writeManifests(items);
238255
console.log(
239256
`Staged ${objects} object(s) from ${plans.length} manifest entr(ies) → ${STAGING_DIR}`,
240257
);

0 commit comments

Comments
 (0)