Skip to content

Commit 0a60dee

Browse files
committed
perf(aws-lambda): build the final mp4 inside S3 with its index up front
The Lambda plan asks chunks to carry their audio. Assemble writes the faststart header as part 1 of a multipart upload and copies each chunk's mdat payload with UploadPartCopy, so the video bytes never pass through the function; any failure aborts the upload.
1 parent a960f8a commit 0a60dee

9 files changed

Lines changed: 676 additions & 19 deletions

File tree

‎examples/aws-lambda/template.yaml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -162,6 +162,10 @@ Resources:
162162
Policies:
163163
- S3CrudPolicy:
164164
BucketName: !Ref RenderBucket
165+
- Statement:
166+
- Effect: Allow
167+
Action: s3:AbortMultipartUpload
168+
Resource: !Sub "arn:${AWS::Partition}:s3:::${RenderBucket}/*"
165169
# CloudWatch Logs perms are covered by SAM's default
166170
# AWSLambdaBasicExecutionRole — explicit `CloudWatchLogsFullAccess`
167171
# would be overscope (`logs:*` on `*`, including DeleteLogGroup +

‎packages/aws-lambda/src/cdk/HyperframesRenderStack.snapshot.test.ts‎

Lines changed: 18 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -279,6 +279,18 @@ describe("HyperframesRenderStack — snapshot", () => {
279279
}
280280
});
281281

282+
it("lets the render function abort the multipart uploads its S3 stitch starts", () => {
283+
const cdkPolicies = JSON.stringify(SYNTHED.template.findResources("AWS::IAM::Policy"));
284+
expect(cdkPolicies).toContain("s3:Abort*");
285+
const samFunction = requireRecordProperty(readSamResources(), "RenderFunction", "SAM function");
286+
const samProperties = requireRecordProperty(
287+
samFunction,
288+
"Properties",
289+
"SAM function properties",
290+
);
291+
expect(JSON.stringify(samProperties.Policies)).toContain("s3:AbortMultipartUpload");
292+
});
293+
282294
it("keeps same-named Lambda tasks in different branches apart", () => {
283295
const task = (format: string) => ({
284296
Type: "Task",
@@ -433,7 +445,7 @@ function getV1TaskStates(definition: {
433445
};
434446
}
435447

436-
function readSamDefinition(): { States: Record<string, unknown> } {
448+
function readSamResources(): Record<string, unknown> {
437449
const source = readFileSync(
438450
new URL("../../../../examples/aws-lambda/template.yaml", import.meta.url),
439451
"utf8",
@@ -443,7 +455,11 @@ function readSamDefinition(): { States: Record<string, unknown> } {
443455
// warnings for the intentionally unresolved `!Ref`/`!GetAtt` tags.
444456
const parsed: unknown = parseYaml(source, { logLevel: "silent" });
445457
const root = requireRecord(parsed, "SAM template");
446-
const resources = requireRecordProperty(root, "Resources", "SAM resources");
458+
return requireRecordProperty(root, "Resources", "SAM resources");
459+
}
460+
461+
function readSamDefinition(): { States: Record<string, unknown> } {
462+
const resources = readSamResources();
447463
const stateMachine = requireRecordProperty(
448464
resources,
449465
"RenderStateMachine",

‎packages/aws-lambda/src/handler.test.ts‎

Lines changed: 143 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
*/
1717

1818
import { afterEach, beforeEach, describe, expect, it, mock, spyOn } from "bun:test";
19+
import { spawnSync } from "node:child_process";
1920
import { createHash } from "node:crypto";
2021
import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
2122
import { tmpdir } from "node:os";
@@ -35,7 +36,7 @@ import type { AssembleEvent, LambdaEvent, PlanEvent, RenderChunkEvent } from "./
3536
import { handler, unwrapEvent } from "./handler.js";
3637

3738
interface FakeS3Op {
38-
kind: "download" | "upload";
39+
kind: "download" | "upload" | "copy";
3940
uri: string;
4041
bytes?: number;
4142
}
@@ -63,12 +64,19 @@ class FakeS3Client {
6364
const cmdName = op.constructor?.name ?? "";
6465
const uri = `s3://${op.input.Bucket}/${op.input.Key}`;
6566
if (cmdName === "GetObjectCommand") {
66-
const bytes = this.objects.get(uri) ?? Buffer.alloc(0);
67+
const bytes = this.sliceRange(
68+
this.objects.get(uri) ?? Buffer.alloc(0),
69+
(command as { input: { Range?: string } }).input.Range,
70+
);
6771
this.ops.push({ kind: "download", uri, bytes: bytes.length });
6872
// Mock the AWS SDK stream contract just enough for pipeline() to
6973
// pump bytes into a write stream.
7074
const { Readable } = await import("node:stream");
71-
return { Body: Readable.from([bytes]) };
75+
return {
76+
Body: Object.assign(Readable.from([bytes]), {
77+
transformToByteArray: async () => new Uint8Array(bytes),
78+
}),
79+
};
7280
}
7381
if (cmdName === "HeadObjectCommand") {
7482
const bytes = this.objects.get(uri);
@@ -104,8 +112,37 @@ class FakeS3Client {
104112
if (metadata) this.metadata.set(uri, metadata);
105113
return {};
106114
}
115+
if (cmdName === "CreateMultipartUploadCommand") return { UploadId: "upload-1" };
116+
if (cmdName === "UploadPartCommand") {
117+
const input = (command as { input: { Body: Buffer; PartNumber: number } }).input;
118+
this.parts.set(input.PartNumber, input.Body);
119+
return { ETag: `"part-${input.PartNumber}"` };
120+
}
121+
if (cmdName === "UploadPartCopyCommand") {
122+
const input = (
123+
command as { input: { CopySource: string; CopySourceRange?: string; PartNumber: number } }
124+
).input;
125+
const source = `s3://${decodeURI(input.CopySource)}`;
126+
this.ops.push({ kind: "copy", uri: source });
127+
const bytes = this.objects.get(source) ?? Buffer.alloc(0);
128+
this.parts.set(input.PartNumber, this.sliceRange(bytes, input.CopySourceRange));
129+
return { CopyPartResult: { ETag: `"part-${input.PartNumber}"` } };
130+
}
131+
if (cmdName === "CompleteMultipartUploadCommand") {
132+
const parts = (command as { input: { MultipartUpload: { Parts: { PartNumber: number }[] } } })
133+
.input.MultipartUpload.Parts;
134+
this.objects.set(uri, Buffer.concat(parts.map((part) => this.parts.get(part.PartNumber)!)));
135+
return {};
136+
}
107137
return {};
108138
}
139+
140+
parts = new Map<number, Buffer>();
141+
142+
private sliceRange(bytes: Buffer, range?: string): Buffer {
143+
const match = /^bytes=(\d+)-(\d+)$/.exec(range ?? "");
144+
return match ? bytes.subarray(Number(match[1]), Number(match[2]) + 1) : bytes;
145+
}
109146
}
110147

111148
const tmpDirs: string[] = [];
@@ -730,6 +767,90 @@ describe("handler dispatch", () => {
730767
).toBe(true);
731768
});
732769

770+
async function planStitched(chunkCount: number) {
771+
const tmpRoot = makeTmpRoot();
772+
const s3 = new FakeS3Client();
773+
s3.objects.set("s3://bucket/project.tar.gz", await makeMinimalProjectTar());
774+
const assembleMock = mock(
775+
async (_p: string, _c: readonly string[], _a: string | null, outputPath: string) => {
776+
writeFileSync(outputPath, "LOCAL-JOIN");
777+
return { outputPath, durationMs: 1, framesEncoded: 30 * chunkCount, fileSize: 10 };
778+
},
779+
);
780+
const planV2WithPublisherMock = mock(
781+
async (_dir: string, _config: unknown, publisher: PlanV2ArtifactPublisher) => {
782+
const v1Dir = join(tmpRoot, "v1");
783+
makeMinimalV1PlanDir(v1Dir, true, { chunkCount, chunksCarryAudio: true });
784+
return publishPlanV2FromExecutionPlan(v1Dir, publisher);
785+
},
786+
);
787+
const deps = {
788+
s3: s3 as unknown as import("@aws-sdk/client-s3").S3Client,
789+
primitives: {
790+
planV2WithPublisher:
791+
planV2WithPublisherMock as unknown as typeof import("@hyperframes/producer/distributed").planV2WithPublisher,
792+
assemble:
793+
assembleMock as unknown as typeof import("@hyperframes/producer/distributed").assemble,
794+
},
795+
tmpRoot,
796+
skipChromeResolution: true,
797+
};
798+
const planned = await handler(
799+
{
800+
Action: "plan",
801+
ProjectS3Uri: "s3://bucket/project.tar.gz",
802+
PlanOutputS3Prefix: "s3://bucket/renders/f/",
803+
Config: { fps: 30, width: 640, height: 360, format: "mp4" },
804+
},
805+
deps,
806+
);
807+
if (!("PlanProtocol" in planned) || planned.PlanProtocol !== "v2")
808+
throw new Error("expected v2");
809+
const assemble = (chunkS3Uris: string[]) =>
810+
handler(
811+
{
812+
Action: "assemble",
813+
PlanV2ManifestS3Uri: planned.PlanV2ManifestS3Uri,
814+
PlanV2ArtifactS3Prefix: planned.PlanV2ArtifactS3Prefix,
815+
PlanHash: planned.PlanHash,
816+
ChunkS3Uris: chunkS3Uris,
817+
AudioS3Uri: null,
818+
OutputS3Uri: "s3://bucket/renders/f/output.mp4",
819+
Format: "mp4",
820+
},
821+
deps,
822+
);
823+
return { s3, assembleMock, assemble };
824+
}
825+
826+
it.skipIf(spawnSync("ffmpeg", ["-version"]).status !== 0)(
827+
"stitches mp4 chunks inside S3 instead of joining them locally",
828+
async () => {
829+
const { s3, assembleMock, assemble } = await planStitched(2);
830+
const chunks = [0, 1].map((index) => {
831+
const path = join(makeTmpRoot(), "chunk.mp4");
832+
const lavfi = ["testsrc2=size=64x48:rate=30:duration=1", "sine=duration=1"];
833+
const muxed = spawnSync("ffmpeg", [
834+
...["-nostdin", "-loglevel", "error", "-f", "lavfi", "-i", lavfi[0]!],
835+
...["-f", "lavfi", "-i", lavfi[1]!, "-c:v", "libx264", "-bf", "0", "-c:a", "aac"],
836+
...["-shortest", "-y", path],
837+
]);
838+
if (muxed.status !== 0) throw new Error(String(muxed.stderr));
839+
const uri = `s3://bucket/renders/f/chunks/000${index}.mp4`;
840+
s3.objects.set(uri, readFileSync(path));
841+
return uri;
842+
});
843+
const result = await assemble(chunks);
844+
const output = s3.objects.get("s3://bucket/renders/f/output.mp4")!;
845+
expect(output.toString("latin1", 4, 8)).toBe("ftyp");
846+
expect(
847+
output.toString("latin1", output.readUInt32BE(0) + 4, output.readUInt32BE(0) + 8),
848+
).toBe("moov");
849+
expect(result).toMatchObject({ FramesEncoded: 60, FileSize: output.length });
850+
expect(assembleMock).not.toHaveBeenCalled();
851+
},
852+
);
853+
733854
it("rejects unknown plan protocol values", async () => {
734855
const tmpRoot = makeTmpRoot();
735856
const s3 = new FakeS3Client();
@@ -903,14 +1024,19 @@ async function makeMinimalPlanTar(): Promise<Buffer> {
9031024
return rf(tarPath);
9041025
}
9051026

906-
function makeMinimalV1PlanDir(dir: string, withAudio: boolean): void {
1027+
function makeMinimalV1PlanDir(
1028+
dir: string,
1029+
withAudio: boolean,
1030+
options: { chunkCount?: number; chunksCarryAudio?: boolean } = {},
1031+
): void {
1032+
const chunkCount = options.chunkCount ?? 1;
9071033
mkdirSync(join(dir, "meta"), { recursive: true });
9081034
mkdirSync(join(dir, "compiled"), { recursive: true });
9091035
writeFileSync(join(dir, "compiled", "index.html"), "<html>aws v2 fixture</html>");
9101036
const planJson = {
9111037
planHash: "a".repeat(64),
912-
chunkCount: 1,
913-
totalFrames: 30,
1038+
chunkCount,
1039+
totalFrames: 30 * chunkCount,
9141040
dimensions: { fpsNum: 30, fpsDen: 1, width: 640, height: 360, format: "mp4" },
9151041
ffmpegVersion: "6.0",
9161042
producerVersion: "test",
@@ -919,9 +1045,18 @@ function makeMinimalV1PlanDir(dir: string, withAudio: boolean): void {
9191045
writeFileSync(join(dir, "plan.json"), JSON.stringify(planJson));
9201046
writeFileSync(
9211047
join(dir, "meta", "chunks.json"),
922-
JSON.stringify([{ index: 0, startFrame: 0, endFrame: 30 }]),
1048+
JSON.stringify(
1049+
Array.from({ length: chunkCount }, (_, index) => ({
1050+
index,
1051+
startFrame: index * 30,
1052+
endFrame: (index + 1) * 30,
1053+
})),
1054+
),
1055+
);
1056+
writeFileSync(
1057+
join(dir, "meta", "encoder.json"),
1058+
options.chunksCarryAudio ? JSON.stringify({ chunksCarryAudio: true }) : "{}",
9231059
);
924-
writeFileSync(join(dir, "meta", "encoder.json"), "{}");
9251060
if (withAudio) writeFileSync(join(dir, "audio.m4a"), "AAC");
9261061
planJson.planHash = recomputePlanHashFromPlanDir(dir);
9271062
writeFileSync(join(dir, "plan.json"), JSON.stringify(planJson));

‎packages/aws-lambda/src/handler.ts‎

Lines changed: 45 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ import { basename, join } from "node:path";
1717
import { S3Client } from "@aws-sdk/client-s3";
1818
import {
1919
assemble,
20+
chunksAreStitchedMp4,
2021
type AssembleResult,
2122
type ChunkRenderer,
2223
type ChunkResult,
@@ -36,6 +37,7 @@ import {
3637
renderChunk,
3738
} from "@hyperframes/producer/distributed";
3839
import { resolveChromeExecutablePath } from "./chromium.js";
40+
import { stitchMp4InS3 } from "./s3Stitch.js";
3941
import { type DistributedFormat, formatExtension } from "./formatExtension.js";
4042
import type {
4143
AssembleEvent,
@@ -440,6 +442,7 @@ async function handlePlanV2(
440442
});
441443
const manifest: PlanV2Manifest = await primitive(projectDir, config, publisher, {
442444
stagingParentDir: work,
445+
chunksCarryAudio: true,
443446
});
444447

445448
return {
@@ -551,7 +554,7 @@ async function handleRenderChunkV2(
551554
}
552555
const work = mkdtempSync(join(deps?.tmpRoot ?? tmpdir(), "hf-lambda-chunk-v2-"));
553556
try {
554-
const planDir = await downloadAndMaterializePlanV2(
557+
const { planDir } = await downloadAndMaterializePlanV2(
555558
s3,
556559
event,
557560
{ role: "chunk", chunkIndex: event.ChunkIndex },
@@ -677,7 +680,15 @@ async function handleAssembleV2(
677680
const primitive = deps?.primitives?.assemble ?? assemble;
678681
const work = mkdtempSync(join(deps?.tmpRoot ?? tmpdir(), "hf-lambda-assemble-v2-"));
679682
try {
680-
const planDir = await downloadAndMaterializePlanV2(s3, event, { role: "assembler" }, work);
683+
const { planDir, manifest } = await downloadAndMaterializePlanV2(
684+
s3,
685+
event,
686+
{ role: "assembler" },
687+
work,
688+
);
689+
if (chunksAreStitchedMp4(planDir, event.Cfr === true)) {
690+
return await stitchChunksInS3(s3, event, manifest, started);
691+
}
681692
// `downloadAndMaterializePlanV2` materializes atomically. Audio is
682693
// assembler-only and lives at the familiar v1-compatible location.
683694
const audioPath = resolvePlanAudioPath(planDir);
@@ -718,6 +729,36 @@ async function handleAssembleV2(
718729
}
719730
}
720731

732+
async function stitchChunksInS3(
733+
s3: S3Client,
734+
event: AssembleV2Event,
735+
plan: PlanV2Manifest,
736+
started: number,
737+
): Promise<AssembleLambdaResult> {
738+
if (event.ChunkS3Uris.length !== plan.chunkCount) {
739+
throw new Error(
740+
`[handler] ${event.ChunkS3Uris.length} chunk URIs for a plan of ${plan.chunkCount} chunks`,
741+
);
742+
}
743+
const stitchStarted = Date.now();
744+
const stitched = await stitchMp4InS3(s3, event.ChunkS3Uris, event.OutputS3Uri);
745+
logEvent({
746+
event: "assemble_stages",
747+
mode: "s3-stitch",
748+
planMs: stitchStarted - started,
749+
stitchMs: Date.now() - stitchStarted,
750+
parts: stitched.parts,
751+
downloadedBytes: stitched.downloadedBytes,
752+
});
753+
return {
754+
Action: "assemble",
755+
OutputS3Uri: event.OutputS3Uri,
756+
FramesEncoded: plan.totalFrames,
757+
FileSize: stitched.fileSize,
758+
DurationMs: Date.now() - started,
759+
};
760+
}
761+
721762
async function downloadAndMaterializePlanV2(
722763
s3: S3Client,
723764
event: {
@@ -727,7 +768,7 @@ async function downloadAndMaterializePlanV2(
727768
},
728769
target: PlanV2MaterializationTarget,
729770
work: string,
730-
): Promise<string> {
771+
): Promise<{ planDir: string; manifest: PlanV2Manifest }> {
731772
const transportDir = join(work, "plan-v2");
732773
mkdirSync(transportDir, { recursive: true });
733774
await downloadS3ObjectToFile(s3, event.PlanV2ManifestS3Uri, join(transportDir, "plan.json"));
@@ -744,7 +785,7 @@ async function downloadAndMaterializePlanV2(
744785
});
745786
const planDir = join(work, "plan");
746787
materializePlanV2Target(transportDir, target, planDir);
747-
return planDir;
788+
return { planDir, manifest };
748789
}
749790

750791
async function downloadPlanV2Artifact(

0 commit comments

Comments
 (0)