Skip to content

Commit 57df7e8

Browse files
committed
fix(producer): cloud site uploads pack the files their id hashes, symlinks included
1 parent cc9e1bb commit 57df7e8

11 files changed

Lines changed: 224 additions & 52 deletions

File tree

‎packages/aws-lambda/src/s3Transport.ts‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -267,12 +267,20 @@ function isRecord(value: unknown): value is Record<string, unknown> {
267267
* binary — the AWS Lambda Node 22 base image ships a minimal set of
268268
* userland tools and does NOT include `tar` in `/usr/bin`.
269269
*/
270-
export async function tarDirectory(sourceDir: string, destTarball: string): Promise<void> {
270+
export async function tarDirectory(
271+
sourceDir: string,
272+
destTarball: string,
273+
files?: string[],
274+
): Promise<void> {
271275
if (!existsSync(sourceDir) || !statSync(sourceDir).isDirectory()) {
272276
throw new Error(`[s3Transport] tar source must be an existing directory: ${sourceDir}`);
273277
}
274278
mkdirSync(dirname(destTarball), { recursive: true });
275-
await tar.create({ gzip: true, file: destTarball, cwd: sourceDir }, ["."]);
279+
// `files` (relative to `sourceDir`) packs exactly those paths, with symlinks followed.
280+
await tar.create(
281+
{ gzip: true, file: destTarball, cwd: sourceDir, follow: files !== undefined },
282+
files ?? ["."],
283+
);
276284
}
277285

278286
/**

‎packages/aws-lambda/src/sdk/__fixtures__/fakeS3.ts‎

Lines changed: 10 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
* `deploySite`'s `finally` block has rmSync'd it.
99
*/
1010

11+
import { buffer } from "node:stream/consumers";
1112
import type { S3Client } from "@aws-sdk/client-s3";
1213

1314
export interface FakeS3Op {
@@ -19,6 +20,7 @@ export interface FakeS3Op {
1920
export class FakeS3 {
2021
ops: FakeS3Op[] = [];
2122
existing = new Set<string>();
23+
bodies = new Map<string, Buffer>();
2224

2325
async send(command: unknown): Promise<unknown> {
2426
const cmdName = (command as { constructor: { name: string } }).constructor.name;
@@ -39,7 +41,12 @@ export class FakeS3 {
3941
if (cmdName === "PutObjectCommand") {
4042
this.ops.push({ kind: "put", bucket: input.Bucket, key: input.Key });
4143
this.existing.add(`${input.Bucket}/${input.Key}`);
42-
await drainBody((command as { input: { Body: NodeJS.ReadableStream | Buffer } }).input.Body);
44+
this.bodies.set(
45+
`${input.Bucket}/${input.Key}`,
46+
await drainBody(
47+
(command as { input: { Body: NodeJS.ReadableStream | Buffer } }).input.Body,
48+
),
49+
);
4350
return {};
4451
}
4552
throw new Error(`FakeS3: unexpected command ${cmdName}`);
@@ -58,12 +65,6 @@ export function asS3Client(fake: FakeS3): S3Client {
5865
* the workdir — surfacing as an "Unhandled error between tests"
5966
* teardown noise that bun's runner attributes to the next test.
6067
*/
61-
async function drainBody(body: NodeJS.ReadableStream | Buffer): Promise<void> {
62-
if (Buffer.isBuffer(body)) return;
63-
await new Promise<void>((resolve, reject) => {
64-
body.on("data", () => {});
65-
body.on("end", () => resolve());
66-
body.on("close", () => resolve());
67-
body.on("error", reject);
68-
});
68+
async function drainBody(body: NodeJS.ReadableStream | Buffer): Promise<Buffer> {
69+
return Buffer.isBuffer(body) ? body : buffer(body);
6970
}

‎packages/aws-lambda/src/sdk/deploySite.test.ts‎

Lines changed: 34 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,24 +1,36 @@
11
import { afterEach, beforeEach, describe, expect, it } from "bun:test";
2-
import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
2+
import { mkdirSync, mkdtempSync, rmSync, symlinkSync, writeFileSync } from "node:fs";
33
import { tmpdir } from "node:os";
44
import { join } from "node:path";
55
import type { S3Client } from "@aws-sdk/client-s3";
6+
import * as tar from "tar";
67
import { asS3Client, FakeS3 } from "./__fixtures__/fakeS3.js";
78
import { deploySite } from "./deploySite.js";
89

910
let projectDir: string;
11+
let scratch: string;
1012

1113
beforeEach(() => {
1214
projectDir = mkdtempSync(join(tmpdir(), "hf-deploy-site-test-"));
15+
scratch = mkdtempSync(join(tmpdir(), "hf-deploy-site-scratch-"));
1316
mkdirSync(join(projectDir, "assets"));
1417
writeFileSync(join(projectDir, "index.html"), "<html><body>hi</body></html>");
1518
writeFileSync(join(projectDir, "assets", "style.css"), "body { color: red; }");
1619
});
1720

1821
afterEach(() => {
1922
rmSync(projectDir, { recursive: true, force: true });
23+
rmSync(scratch, { recursive: true, force: true });
2024
});
2125

26+
async function uploadedEntries(s3: FakeS3, siteId: string): Promise<string[]> {
27+
const file = join(scratch, "uploaded.tar.gz");
28+
writeFileSync(file, s3.bodies.get(`test-bucket/sites/${siteId}/project.tar.gz`) ?? "");
29+
const entries: string[] = [];
30+
await tar.list({ file, onReadEntry: (entry) => entries.push(`${entry.type} ${entry.path}`) });
31+
return entries;
32+
}
33+
2234
describe("deploySite", () => {
2335
it("uploads the tarball when no matching object exists", async () => {
2436
const s3 = new FakeS3();
@@ -145,4 +157,25 @@ describe("deploySite", () => {
145157
// node_modules contents shouldn't move the hash.
146158
expect(after.siteId).toBe(before.siteId);
147159
});
160+
161+
it("packs a symlinked project and its symlinked files and folders as content", async () => {
162+
mkdirSync(join(scratch, "media"));
163+
writeFileSync(join(scratch, "media", "clip.txt"), "clip");
164+
writeFileSync(join(scratch, "logo.txt"), "logo");
165+
symlinkSync(join(scratch, "media"), join(projectDir, "media"));
166+
symlinkSync(join(scratch, "logo.txt"), join(projectDir, "logo.txt"));
167+
symlinkSync(projectDir, join(scratch, "fx"));
168+
const s3 = new FakeS3();
169+
const result = await deploySite({
170+
projectDir: join(scratch, "fx"),
171+
bucketName: "test-bucket",
172+
s3: asS3Client(s3),
173+
});
174+
expect(await uploadedEntries(s3, result.siteId)).toEqual([
175+
"File assets/style.css",
176+
"File index.html",
177+
"File logo.txt",
178+
"File media/clip.txt",
179+
]);
180+
});
148181
});

‎packages/aws-lambda/src/sdk/deploySite.ts‎

Lines changed: 4 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ import { mkdtempSync, rmSync, statSync } from "node:fs";
1616
import { tmpdir } from "node:os";
1717
import { join } from "node:path";
1818
import { HeadObjectCommand, S3Client } from "@aws-sdk/client-s3";
19-
import { hashProjectDir } from "@hyperframes/producer/distributed";
19+
import { hashProjectDir, listSiteFiles } from "@hyperframes/producer/distributed";
2020
import { formatS3Uri, tarDirectory, uploadFileToS3 } from "../s3Transport.js";
2121

2222
/** Options for {@link deploySite}. */
@@ -65,7 +65,8 @@ export async function deploySite(opts: DeploySiteOptions): Promise<SiteHandle> {
6565
throw new Error(`[deploySite] projectDir is not a directory: ${opts.projectDir}`);
6666
}
6767

68-
const siteId = opts.siteId ?? hashProjectDir(opts.projectDir);
68+
const files = listSiteFiles(opts.projectDir);
69+
const siteId = opts.siteId ?? hashProjectDir(opts.projectDir, files);
6970
const key = `sites/${siteId}/project.tar.gz`;
7071
const projectS3Uri = formatS3Uri({ bucket: opts.bucketName, key });
7172
const s3 = opts.s3 ?? new S3Client({ region: opts.region });
@@ -88,12 +89,7 @@ export async function deploySite(opts: DeploySiteOptions): Promise<SiteHandle> {
8889
const workdir = mkdtempSync(join(tmpdir(), "hf-deploy-site-"));
8990
try {
9091
const tarball = join(workdir, "project.tar.gz");
91-
await tarDirectory(opts.projectDir, tarball);
92-
// Note: tarDirectory packs *everything* under `cwd`. We don't need to
93-
// re-implement the skip list inside the tar pack because the
94-
// producer's plan stage applies the same skip during its copy; the
95-
// archive is slightly bigger than the planDir's compiled/ subtree
96-
// but the cost is bounded by the project's user-authored content.
92+
await tarDirectory(opts.projectDir, tarball, files);
9793
const size = statSync(tarball).size;
9894
await uploadFileToS3(s3, tarball, projectS3Uri, "application/gzip");
9995
return {

‎packages/gcp-cloud-run/src/gcsTransport.ts‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -241,12 +241,20 @@ function isGcsPreconditionFailed(error: unknown): boolean {
241241
* package (pure JS over `node:zlib`) rather than spawning a system tar
242242
* binary so the archive format is independent of the container's userland.
243243
*/
244-
export async function tarDirectory(sourceDir: string, destTarball: string): Promise<void> {
244+
export async function tarDirectory(
245+
sourceDir: string,
246+
destTarball: string,
247+
files?: string[],
248+
): Promise<void> {
245249
if (!existsSync(sourceDir) || !statSync(sourceDir).isDirectory()) {
246250
throw new Error(`[gcsTransport] tar source must be an existing directory: ${sourceDir}`);
247251
}
248252
mkdirSync(dirname(destTarball), { recursive: true });
249-
await tar.create({ gzip: true, file: destTarball, cwd: sourceDir }, ["."]);
253+
// `files` (relative to `sourceDir`) packs exactly those paths, with symlinks followed.
254+
await tar.create(
255+
{ gzip: true, file: destTarball, cwd: sourceDir, follow: files !== undefined },
256+
files ?? ["."],
257+
);
250258
}
251259

252260
/**

‎packages/gcp-cloud-run/src/sdk/deploySite.test.ts‎

Lines changed: 22 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,9 +4,10 @@
44
*/
55

66
import { afterEach, describe, expect, it } from "bun:test";
7-
import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
7+
import { mkdtempSync, rmSync, symlinkSync, writeFileSync } from "node:fs";
88
import { tmpdir } from "node:os";
99
import { join } from "node:path";
10+
import * as tar from "tar";
1011
import { asStorage, FakeGcs } from "../__fixtures__/fakeGcs.js";
1112
import { deploySite } from "./deploySite.js";
1213

@@ -83,4 +84,24 @@ describe("deploySite", () => {
8384
expect(handle.siteId).toBe("my-git-sha");
8485
expect(handle.projectGcsUri).toBe("gs://b/sites/my-git-sha/project.tar.gz");
8586
});
87+
88+
it("packs a symlinked project and its symlinked files as content", async () => {
89+
const gcs = new FakeGcs();
90+
const scratch = mkdtempSync(join(tmpdir(), "hf-site-scratch-"));
91+
tmpDirs.push(scratch);
92+
const real = mkProject("<html>linked</html>");
93+
writeFileSync(join(scratch, "logo.txt"), "logo");
94+
symlinkSync(join(scratch, "logo.txt"), join(real, "logo.txt"));
95+
symlinkSync(real, join(scratch, "fx"));
96+
const handle = await deploySite({
97+
projectDir: join(scratch, "fx"),
98+
bucketName: "b",
99+
storage: asStorage(gcs),
100+
});
101+
const file = join(scratch, "uploaded.tar.gz");
102+
writeFileSync(file, gcs.objects.get(handle.projectGcsUri) ?? "");
103+
const entries: string[] = [];
104+
await tar.list({ file, onReadEntry: (entry) => entries.push(`${entry.type} ${entry.path}`) });
105+
expect(entries).toEqual(["File index.html", "File logo.txt"]);
106+
});
86107
});

‎packages/gcp-cloud-run/src/sdk/deploySite.ts‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ import { mkdtempSync, rmSync, statSync } from "node:fs";
1717
import { tmpdir } from "node:os";
1818
import { join } from "node:path";
1919
import { Storage } from "@google-cloud/storage";
20-
import { hashProjectDir } from "@hyperframes/producer/distributed";
20+
import { hashProjectDir, listSiteFiles } from "@hyperframes/producer/distributed";
2121
import { formatGcsUri, tarDirectory, uploadFileToGcs } from "../gcsTransport.js";
2222

2323
/** Options for {@link deploySite}. */
@@ -65,7 +65,8 @@ export async function deploySite(opts: DeploySiteOptions): Promise<SiteHandle> {
6565
throw new Error(`[deploySite] projectDir is not a directory: ${opts.projectDir}`);
6666
}
6767

68-
const siteId = opts.siteId ?? hashProjectDir(opts.projectDir);
68+
const files = listSiteFiles(opts.projectDir);
69+
const siteId = opts.siteId ?? hashProjectDir(opts.projectDir, files);
6970
const key = `sites/${siteId}/project.tar.gz`;
7071
const projectGcsUri = formatGcsUri({ bucket: opts.bucketName, key });
7172
const storage = opts.storage ?? new Storage();
@@ -89,7 +90,7 @@ export async function deploySite(opts: DeploySiteOptions): Promise<SiteHandle> {
8990
const workdir = mkdtempSync(join(tmpdir(), "hf-deploy-site-"));
9091
try {
9192
const tarball = join(workdir, "project.tar.gz");
92-
await tarDirectory(opts.projectDir, tarball);
93+
await tarDirectory(opts.projectDir, tarball, files);
9394
const size = statSync(tarball).size;
9495
await uploadFileToGcs(storage, tarball, projectGcsUri, "application/gzip");
9596
return {

‎packages/producer/src/distributed.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -115,7 +115,7 @@ export {
115115
validateDistributedRenderConfig,
116116
validateVariablesPayload,
117117
} from "./services/distributed/renderConfigValidation.js";
118-
export { hashProjectDir } from "./services/distributed/projectHash.js";
118+
export { hashProjectDir, listSiteFiles } from "./services/distributed/projectHash.js";
119119

120120
// ── Plan protocol compatibility ────────────────────────────────────────────
121121
// Workers validate this descriptor before consuming layout-specific

‎packages/producer/src/regression-harness-lambda-local.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@ import type {
4545
RenderChunkLambdaResult,
4646
SerializableDistributedRenderConfig,
4747
} from "@hyperframes/aws-lambda";
48+
import { listSiteFiles } from "./services/distributed/projectHash.js";
4849

4950
export type {
5051
LambdaLocalRenderResult,
@@ -78,7 +79,7 @@ export async function runLambdaLocalRender(
7879
const projectKey = `sites/harness/${Date.now()}/project.tar.gz`;
7980
const projectS3Path = join(s3Root, projectKey);
8081
mkdirSync(dirname(projectS3Path), { recursive: true });
81-
await tarDirectory(input.projectDir, projectS3Path);
82+
await tarDirectory(input.projectDir, projectS3Path, listSiteFiles(input.projectDir));
8283

8384
const fakeS3 = new FilesystemBackedFakeS3(s3Root);
8485
const deps: HandlerDeps = {
Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
import { afterEach, describe, expect, it } from "bun:test";
2+
import { createHash } from "node:crypto";
3+
import { mkdirSync, mkdtempSync, rmSync, symlinkSync, writeFileSync } from "node:fs";
4+
import { tmpdir } from "node:os";
5+
import { join } from "node:path";
6+
import { hashProjectDir, listSiteFiles } from "./projectHash.js";
7+
8+
const dirs: string[] = [];
9+
function tempDir(): string {
10+
const dir = mkdtempSync(join(tmpdir(), "hf-project-hash-"));
11+
dirs.push(dir);
12+
return dir;
13+
}
14+
function project(): string {
15+
const dir = tempDir();
16+
writeFileSync(join(dir, "index.html"), "<html>hi</html>");
17+
return dir;
18+
}
19+
afterEach(() => {
20+
for (const dir of dirs.splice(0)) rmSync(dir, { recursive: true, force: true });
21+
});
22+
23+
describe("listSiteFiles / hashProjectDir", () => {
24+
it("follows a symlinked project root", () => {
25+
const real = project();
26+
const link = join(tempDir(), "fx");
27+
symlinkSync(real, link);
28+
expect(listSiteFiles(link)).toEqual(["index.html"]);
29+
expect(hashProjectDir(link)).toBe(hashProjectDir(real));
30+
});
31+
32+
it("lists symlinked files and folders, and hashes their targets' bytes", () => {
33+
const dir = project();
34+
const outside = tempDir();
35+
mkdirSync(join(outside, "media"));
36+
writeFileSync(join(outside, "shared.css"), "a{}");
37+
writeFileSync(join(outside, "media", "clip.txt"), "v1");
38+
symlinkSync(join(outside, "shared.css"), join(dir, "shared.css"));
39+
symlinkSync(join(outside, "media"), join(dir, "media"));
40+
expect(listSiteFiles(dir)).toEqual(["index.html", "media/clip.txt", "shared.css"]);
41+
42+
const before = hashProjectDir(dir);
43+
writeFileSync(join(outside, "media", "clip.txt"), "v2");
44+
expect(hashProjectDir(dir)).not.toBe(before);
45+
});
46+
47+
it("skips top-level skip-list folders", () => {
48+
const dir = project();
49+
mkdirSync(join(dir, "node_modules"));
50+
writeFileSync(join(dir, "node_modules", "junk.js"), "x");
51+
expect(listSiteFiles(dir)).toEqual(["index.html"]);
52+
});
53+
54+
it("does not loop on a folder that links back to an ancestor", () => {
55+
const dir = project();
56+
symlinkSync(dir, join(dir, "self"));
57+
expect(listSiteFiles(dir)).toEqual(["index.html"]);
58+
});
59+
60+
it("throws on a dangling symlink, naming it", () => {
61+
const dir = project();
62+
symlinkSync(join(dir, "missing.png"), join(dir, "broken.png"));
63+
expect(() => listSiteFiles(dir)).toThrow(/cannot read broken\.png/);
64+
});
65+
66+
it("never reuses an id from the unversioned scheme", () => {
67+
const dir = project();
68+
const unversioned = createHash("sha256")
69+
.update("index.html\0<html>hi</html>")
70+
.digest("hex")
71+
.slice(0, 16);
72+
expect(hashProjectDir(dir)).not.toBe(unversioned);
73+
});
74+
});

0 commit comments

Comments
 (0)