Skip to content

Commit 0999878

Browse files
committed
feat(aws-lambda): publish plan v2 directly to S3
1 parent 07f9a3d commit 0999878

7 files changed

Lines changed: 547 additions & 87 deletions

File tree

‎packages/aws-lambda/src/handler.test.ts‎

Lines changed: 25 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
// fallow-ignore-file code-duplication complexity
12
/**
23
* Handler dispatch unit tests.
34
*
@@ -18,14 +19,15 @@ import { afterEach, beforeEach, describe, expect, it, mock } from "bun:test";
1819
import { createHash } from "node:crypto";
1920
import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
2021
import { tmpdir } from "node:os";
21-
import { join } from "node:path";
22+
import { dirname, join } from "node:path";
2223
import {
2324
CURRENT_PLAN_PROTOCOL,
24-
createPlanV2FromV1,
2525
type AssembleResult,
2626
type ChunkResult,
2727
type PlanResult,
28-
type PlanV2Result,
28+
type PlanV2ArtifactPublisher,
29+
type PlanV2Manifest,
30+
publishPlanV2FromV1,
2931
} from "@hyperframes/producer/distributed";
3032
import { recomputePlanHashFromPlanDir } from "../../producer/src/services/render/stages/freezePlan.js";
3133
import type { AssembleEvent, LambdaEvent, PlanEvent, RenderChunkEvent } from "./events.js";
@@ -528,11 +530,19 @@ describe("handler dispatch", () => {
528530
const s3 = new FakeS3Client();
529531
s3.objects.set("s3://bucket/project.tar.gz", await makeMinimalProjectTar());
530532

531-
const planV2Mock = mock(
532-
async (_projectDir: string, _config: unknown, planV2Dir: string): Promise<PlanV2Result> => {
533+
const planV2WithPublisherMock = mock(
534+
async (
535+
projectDir: string,
536+
_config: unknown,
537+
publisher: PlanV2ArtifactPublisher,
538+
options: Readonly<{ stagingParentDir?: string }>,
539+
): Promise<PlanV2Manifest> => {
533540
const v1Dir = join(tmpRoot, `v1-${Date.now()}`);
534541
makeMinimalV1PlanDir(v1Dir, true);
535-
return createPlanV2FromV1(v1Dir, planV2Dir);
542+
const manifest = await publishPlanV2FromV1(v1Dir, publisher);
543+
expect(options.stagingParentDir).toBe(dirname(projectDir));
544+
expect(existsSync(join(dirname(projectDir), "plan-v2"))).toBe(false);
545+
return manifest;
536546
},
537547
);
538548
const renderChunkMock = mock(
@@ -568,7 +578,8 @@ describe("handler dispatch", () => {
568578
plan: mock(async () => {
569579
throw new Error("v1 plan should not be called");
570580
}) as unknown as typeof import("@hyperframes/producer/distributed").plan,
571-
planV2: planV2Mock as unknown as typeof import("@hyperframes/producer/distributed").planV2,
581+
planV2WithPublisher:
582+
planV2WithPublisherMock as unknown as typeof import("@hyperframes/producer/distributed").planV2WithPublisher,
572583
renderChunk:
573584
renderChunkMock as unknown as typeof import("@hyperframes/producer/distributed").renderChunk,
574585
assemble:
@@ -597,6 +608,13 @@ describe("handler dispatch", () => {
597608
if (!("PlanProtocol" in planned) || planned.PlanProtocol !== "v2") {
598609
throw new Error("expected v2 plan result");
599610
}
611+
const planUploads = s3.ops.filter((operation) => operation.kind === "upload");
612+
expect(planUploads.at(-1)?.uri).toBe(planned.PlanV2ManifestS3Uri);
613+
expect(
614+
planUploads
615+
.slice(0, -1)
616+
.every((operation) => operation.uri.startsWith(`${planned.PlanV2ArtifactS3Prefix}/`)),
617+
).toBe(true);
600618

601619
const beforeChunk = s3.ops.length;
602620
const chunk = await handler(

‎packages/aws-lambda/src/handler.ts‎

Lines changed: 35 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -23,11 +23,11 @@ import {
2323
listPlanV2ArtifactsForTarget,
2424
materializePlanV2Target,
2525
plan,
26-
planV2,
26+
planV2WithPublisher,
2727
type PlanResult,
2828
type PlanV2Artifact,
29+
type PlanV2Manifest,
2930
type PlanV2MaterializationTarget,
30-
type PlanV2Result,
3131
readPlanV2Manifest,
3232
renderChunk,
3333
} from "@hyperframes/producer/distributed";
@@ -48,12 +48,11 @@ import {
4848
downloadS3ObjectToFile,
4949
downloadS3ObjectToFileVerified,
5050
parseS3Uri,
51-
sha256File,
5251
tarDirectory,
5352
untarDirectory,
54-
uploadContentAddressedFileToS3,
5553
uploadFileToS3,
5654
} from "./s3Transport.js";
55+
import { S3PlanV2ArtifactPublisher } from "./s3PlanV2Publisher.js";
5756

5857
/**
5958
* Lazily-constructed S3 client. Cached at module scope so warm Lambda
@@ -77,7 +76,7 @@ export interface HandlerDeps {
7776
s3?: S3Client;
7877
primitives?: {
7978
plan: typeof plan;
80-
planV2?: typeof planV2;
79+
planV2WithPublisher?: typeof planV2WithPublisher;
8180
renderChunk: typeof renderChunk;
8281
assemble: typeof assemble;
8382
};
@@ -266,6 +265,8 @@ function primeRuntimeEnv(): void {
266265

267266
// ── Plan ────────────────────────────────────────────────────────────────────
268267

268+
// The v1 handler owns one transactional download, plan, archive, upload, and cleanup lifecycle.
269+
// fallow-ignore-next-line complexity
269270
async function handlePlan(event: PlanEvent, deps?: HandlerDeps): Promise<PlanLambdaResult> {
270271
if (event.PlanProtocol === "v2") {
271272
return handlePlanV2(event, deps);
@@ -350,65 +351,42 @@ async function handlePlanV2(
350351
): Promise<Extract<PlanLambdaResult, { PlanProtocol: "v2" }>> {
351352
const started = Date.now();
352353
const s3 = deps?.s3 ?? getS3Client();
353-
const primitive = deps?.primitives?.planV2 ?? planV2;
354+
const primitive = deps?.primitives?.planV2WithPublisher ?? planV2WithPublisher;
354355
if (!deps?.skipChromeResolution && !process.env.PRODUCER_HEADLESS_SHELL_PATH) {
355356
process.env.PRODUCER_HEADLESS_SHELL_PATH = await resolveChromeExecutablePath();
356357
}
357358

358359
const work = mkdtempSync(join(deps?.tmpRoot ?? tmpdir(), "hf-lambda-plan-v2-"));
359360
const projectArchive = join(work, "project.tar.gz");
360361
const projectDir = join(work, "project");
361-
const planV2Dir = join(work, "plan-v2");
362362
try {
363363
await downloadS3ObjectToFile(s3, event.ProjectS3Uri, projectArchive);
364364
await untarDirectory(projectArchive, projectDir);
365-
const result: PlanV2Result = await primitive(projectDir, { ...event.Config }, planV2Dir);
366-
const manifest = readPlanV2Manifest(planV2Dir);
367-
if (manifest.planHash !== result.planHash) {
368-
throwPlanHashMismatch(result.planHash, manifest.planHash);
369-
}
370-
371-
const outputPrefix = `${trimTrailingSlash(event.PlanOutputS3Prefix)}/v2`;
372-
const artifactPrefix = `${outputPrefix}/artifacts/sha256`;
373-
const uniqueArtifacts = [
374-
...new Map(manifest.artifacts.map((artifact) => [artifact.sha256, artifact])).values(),
375-
];
376-
await mapConcurrent(uniqueArtifacts, 16, async (artifact) => {
377-
const localPath = planV2BlobPath(planV2Dir, artifact.sha256);
378-
await uploadContentAddressedFileToS3(
379-
s3,
380-
localPath,
381-
planV2BlobUri(artifactPrefix, artifact.sha256),
382-
artifact.sha256,
383-
);
384-
});
385-
386-
// Publish the manifest only after every referenced blob is durable.
387-
const manifestUri = `${outputPrefix}/manifest.json`;
388-
await uploadContentAddressedFileToS3(
365+
const publisher = new S3PlanV2ArtifactPublisher({
389366
s3,
390-
result.manifestPath,
391-
manifestUri,
392-
await sha256File(result.manifestPath),
393-
"application/json",
394-
);
367+
planOutputS3Prefix: event.PlanOutputS3Prefix,
368+
temporaryRoot: work,
369+
});
370+
const manifest: PlanV2Manifest = await primitive(projectDir, { ...event.Config }, publisher, {
371+
stagingParentDir: work,
372+
});
395373

396374
return {
397375
Action: "plan",
398376
PlanProtocol: "v2",
399-
PlanV2ManifestS3Uri: manifestUri,
400-
PlanV2ArtifactS3Prefix: artifactPrefix,
401-
PlanHash: result.planHash,
402-
ChunkCount: result.chunkCount,
403-
TotalFrames: result.totalFrames,
404-
Fps: result.fps,
405-
Width: result.width,
406-
Height: result.height,
407-
Format: result.format,
377+
PlanV2ManifestS3Uri: publisher.manifestUri,
378+
PlanV2ArtifactS3Prefix: publisher.artifactPrefix,
379+
PlanHash: manifest.planHash,
380+
ChunkCount: manifest.chunkCount,
381+
TotalFrames: manifest.totalFrames,
382+
Fps: manifest.fps,
383+
Width: manifest.width,
384+
Height: manifest.height,
385+
Format: manifest.format,
408386
HasAudio: manifest.artifacts.some((artifact) => artifact.path === "audio.aac"),
409387
AudioS3Uri: null,
410-
FfmpegVersion: result.ffmpegVersion,
411-
ProducerVersion: result.producerVersion,
388+
FfmpegVersion: manifest.ffmpegVersion,
389+
ProducerVersion: manifest.producerVersion,
412390
DurationMs: Date.now() - started,
413391
};
414392
} finally {
@@ -728,7 +706,16 @@ async function mapConcurrent<T>(
728706
await fn(values[index]!);
729707
}
730708
}
731-
await Promise.all(Array.from({ length: Math.min(concurrency, values.length) }, () => worker()));
709+
const results = await Promise.allSettled(
710+
Array.from({ length: Math.min(concurrency, values.length) }, () => worker()),
711+
);
712+
const failure = results.find(
713+
(result): result is PromiseRejectedResult => result.status === "rejected",
714+
);
715+
// Do not reject while sibling workers may still be writing into invocation
716+
// scratch. The caller removes that directory in `finally`; draining the pool
717+
// first prevents late S3 streams from racing cleanup after another GET fails.
718+
if (failure) throw failure.reason;
732719
}
733720

734721
async function downloadChunkObjects(

‎packages/aws-lambda/src/index.ts‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,10 @@ export {
6565
uploadContentAddressedFileToS3,
6666
uploadFileToS3,
6767
} from "./s3Transport.js";
68+
export {
69+
S3PlanV2ArtifactPublisher,
70+
type S3PlanV2ArtifactPublisherOptions,
71+
} from "./s3PlanV2Publisher.js";
6872

6973
// ── Client-side SDK ─────────────────────────────────────────────────────────
7074
export { deploySite, type DeploySiteOptions, type SiteHandle } from "./sdk/deploySite.js";

0 commit comments

Comments
 (0)