Skip to content

Commit 6d3ed92

Browse files
committed
perf(aws-lambda): the plan step probes project videos in place instead of downloading them
1 parent 62eb3c8 commit 6d3ed92

24 files changed

Lines changed: 434 additions & 80 deletions

‎bun.lock‎

Lines changed: 3 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎packages/aws-lambda/package.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,7 @@
7171
"dependencies": {
7272
"@aws-sdk/client-s3": "^3.1146.0",
7373
"@aws-sdk/client-sfn": "^3.1146.0",
74+
"@aws-sdk/s3-request-presigner": "3.1146.0",
7475
"@hyperframes/producer": "workspace:^",
7576
"@sparticuz/chromium": "148.0.0",
7677
"ffmpeg-static": "^5.2.0",

‎packages/aws-lambda/src/handler.test.ts‎

Lines changed: 28 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ import { createHash } from "node:crypto";
2020
import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
2121
import { tmpdir } from "node:os";
2222
import { dirname, join } from "node:path";
23+
import { S3Client } from "@aws-sdk/client-s3";
2324
import {
2425
CURRENT_PLAN_PROTOCOL,
2526
PlanVideosMetadataError,
@@ -28,6 +29,7 @@ import {
2829
type PlanResult,
2930
type PlanV2ArtifactPublisher,
3031
type PlanV2Manifest,
32+
type PlanV2WithPublisherOptions,
3133
publishPlanV2FromExecutionPlan,
3234
} from "@hyperframes/producer/distributed";
3335
import { recomputePlanHashFromPlanDir } from "../../producer/src/services/render/stages/freezePlan.js";
@@ -744,6 +746,16 @@ describe("handler dispatch", () => {
744746
});
745747
});
746748

749+
/** A real client (so it can pre-sign offline) whose requests go to the fake. */
750+
function presigningClient(fake: FakeS3Client): import("@aws-sdk/client-s3").S3Client {
751+
const client = new S3Client({
752+
region: "us-east-2",
753+
credentials: { accessKeyId: "AKIDTEST", secretAccessKey: "test-secret" },
754+
});
755+
client.send = fake.send.bind(fake) as S3Client["send"];
756+
return client;
757+
}
758+
747759
describe("handler — plan from a deployed site", () => {
748760
function storeSiteFile(s3: FakeS3Client, bytes: string): string {
749761
const sha256 = createHash("sha256").update(bytes).digest("hex");
@@ -762,7 +774,7 @@ describe("handler — plan from a deployed site", () => {
762774
};
763775
}
764776

765-
it("rebuilds the project from the content store and reuses its media without uploading it", async () => {
777+
it("plans a site's video in place: never downloads, never uploads its bytes", async () => {
766778
const s3 = new FakeS3Client();
767779
const html = storeSiteFile(s3, "<html></html>");
768780
const media = storeSiteFile(s3, "VIDEO-BYTES");
@@ -778,10 +790,19 @@ describe("handler — plan from a deployed site", () => {
778790
),
779791
);
780792
const planV2WithPublisher = mock(
781-
async (projectDir: string, _config: unknown, publisher: PlanV2ArtifactPublisher) => {
793+
async (
794+
projectDir: string,
795+
_config: unknown,
796+
publisher: PlanV2ArtifactPublisher,
797+
options: PlanV2WithPublisherOptions,
798+
) => {
782799
expect(readFileSync(join(projectDir, "index.html"), "utf-8")).toBe("<html></html>");
783-
const mediaPath = join(projectDir, "assets", "bg.mp4");
784-
await publisher.putBlob({ sourcePath: mediaPath, sha256: media, sizeBytes: 11 });
800+
expect(existsSync(join(projectDir, "assets", "bg.mp4"))).toBe(false);
801+
const remote = options.remoteFiles?.["assets/bg.mp4"];
802+
expect(remote).toMatchObject({ sha256: media, sizeBytes: 11 });
803+
expect(remote?.url).toContain(`/cas/sha256/${media.slice(0, 2)}/${media}?`);
804+
expect(remote?.url).toContain("X-Amz-Signature=");
805+
await publisher.referenceStoredBlob!({ sha256: media, sizeBytes: 11 });
785806
const manifest = {
786807
planHash: "c".repeat(64),
787808
chunkCount: 1,
@@ -800,7 +821,7 @@ describe("handler — plan from a deployed site", () => {
800821
);
801822

802823
const result = await handler(planEvent(), {
803-
s3: s3 as unknown as import("@aws-sdk/client-s3").S3Client,
824+
s3: presigningClient(s3),
804825
primitives: {
805826
plan: mock() as unknown as typeof import("@hyperframes/producer/distributed").plan,
806827
planV2WithPublisher:
@@ -819,6 +840,8 @@ describe("handler — plan from a deployed site", () => {
819840
"s3://bucket/renders/r1/v2/manifest.json",
820841
]);
821842
expect(s3.ops.some((op) => op.uri.endsWith(".tar.gz"))).toBe(false);
843+
const mediaKey = `s3://bucket/cas/sha256/${media.slice(0, 2)}/${media}`;
844+
expect(s3.ops.filter((op) => op.uri === mediaKey && op.kind === "download")).toEqual([]);
822845
});
823846

824847
it("refuses a site manifest path that escapes the project", async () => {

‎packages/aws-lambda/src/handler.ts‎

Lines changed: 16 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ import {
3333
type PlanV2Manifest,
3434
type PlanV2MaterializationTarget,
3535
readPlanV2Manifest,
36+
type RemoteProjectFile,
3637
renderChunk,
3738
} from "@hyperframes/producer/distributed";
3839
import { resolveChromeExecutablePath } from "./chromium.js";
@@ -342,7 +343,7 @@ async function handlePlan(event: PlanEvent, deps?: HandlerDeps): Promise<PlanLam
342343
const planDir = join(work, "plan");
343344

344345
try {
345-
const projectDir = await fetchProject(s3, event.ProjectS3Uri, work);
346+
const { projectDir } = await fetchProject(s3, event.ProjectS3Uri, work);
346347

347348
const config: DistributedRenderConfig = {
348349
...event.Config,
@@ -394,16 +395,21 @@ async function handlePlan(event: PlanEvent, deps?: HandlerDeps): Promise<PlanLam
394395
* `ProjectS3Uri` is a site manifest from `deploySite`, or a `.tar.gz` of the project
395396
* (hand-built uploads such as `examples/aws-lambda/scripts/smoke.sh`).
396397
*/
397-
async function fetchProject(s3: S3Client, projectS3Uri: string, work: string): Promise<string> {
398+
async function fetchProject(
399+
s3: S3Client,
400+
projectS3Uri: string,
401+
work: string,
402+
leaveVideosRemote = false,
403+
): Promise<{ projectDir: string; remoteFiles: Record<string, RemoteProjectFile> }> {
398404
const projectDir = join(work, "project");
399405
if (isSiteManifestUri(projectS3Uri)) {
400-
await downloadSiteFiles(s3, projectS3Uri, projectDir);
401-
} else {
402-
const projectArchive = join(work, "project.tar.gz");
403-
await downloadS3ObjectToFile(s3, projectS3Uri, projectArchive);
404-
await untarDirectory(projectArchive, projectDir);
406+
const remoteFiles = await downloadSiteFiles(s3, projectS3Uri, projectDir, leaveVideosRemote);
407+
return { projectDir, remoteFiles };
405408
}
406-
return projectDir;
409+
const projectArchive = join(work, "project.tar.gz");
410+
await downloadS3ObjectToFile(s3, projectS3Uri, projectArchive);
411+
await untarDirectory(projectArchive, projectDir);
412+
return { projectDir, remoteFiles: {} };
407413
}
408414

409415
// Plan v2 orchestration is kept as one transactional boundary: stage, CAS upload,
@@ -422,14 +428,15 @@ async function handlePlanV2(
422428

423429
const work = mkdtempSync(join(deps?.tmpRoot ?? tmpdir(), "hf-lambda-plan-v2-"));
424430
try {
425-
const projectDir = await fetchProject(s3, event.ProjectS3Uri, work);
431+
const { projectDir, remoteFiles } = await fetchProject(s3, event.ProjectS3Uri, work, true);
426432
const publisher = new S3PlanV2ArtifactPublisher({
427433
s3,
428434
planOutputS3Prefix: event.PlanOutputS3Prefix,
429435
temporaryRoot: work,
430436
});
431437
const manifest: PlanV2Manifest = await primitive(projectDir, { ...event.Config }, publisher, {
432438
stagingParentDir: work,
439+
remoteFiles,
433440
});
434441

435442
return {

‎packages/aws-lambda/src/s3PlanV2Publisher.test.ts‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -114,6 +114,30 @@ describe("S3PlanV2ArtifactPublisher", () => {
114114
expect(publisher.manifestUri).toBe("s3://bucket/render/v2/manifest.json");
115115
});
116116

117+
it("references a remote project file already in the store, and refuses one that is not", async () => {
118+
const digest = createHash("sha256").update("remote video").digest("hex");
119+
const s3 = new FakeS3();
120+
s3.objects.set(`s3://bucket/cas/sha256/${digest.slice(0, 2)}/${digest}`, {
121+
bytes: Buffer.from("remote video"),
122+
sha256: digest,
123+
});
124+
const publisher = new S3PlanV2ArtifactPublisher({
125+
s3: s3.asClient(),
126+
planOutputS3Prefix: "s3://bucket/renders/remote",
127+
temporaryRoot: makeSource("x").root,
128+
});
129+
130+
await publisher.referenceStoredBlob({ sha256: digest, sizeBytes: 12 });
131+
await expect(
132+
publisher.referenceStoredBlob({ sha256: "e".repeat(64), sizeBytes: 12 }),
133+
).rejects.toMatchObject({ name: "PLAN_ARTIFACT_MISSING" });
134+
await publisher.commitManifest(manifestFor(digest));
135+
136+
expect(s3.puts.map((operation) => operation.uri)).toEqual([
137+
"s3://bucket/renders/remote/v2/manifest.json",
138+
]);
139+
});
140+
117141
it("reuses a blob the SDK or another render already stored instead of uploading it", async () => {
118142
const source = makeSource("shared media");
119143
const s3 = new FakeS3();

‎packages/aws-lambda/src/s3PlanV2Publisher.ts‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import {
1010
} from "@hyperframes/producer/distributed";
1111
import { CONTENT_STORE_PREFIX } from "./contentStore.js";
1212
import {
13+
ensureContentStoreObject,
1314
formatS3Uri,
1415
parseS3Uri,
1516
putContentStoreFile,
@@ -102,6 +103,13 @@ export class S3PlanV2ArtifactPublisher implements PlanV2ArtifactPublisher {
102103
this.#publishedDigests.add(digest);
103104
}
104105

106+
async referenceStoredBlob(blob: Readonly<Omit<PlanV2PublishBlob, "sourcePath">>): Promise<void> {
107+
this.#assertOpen("reference a stored blob");
108+
const digest = assertSha256(blob.sha256, "S3 referenced blob sha256");
109+
await ensureContentStoreObject(this.#s3, this.#bucket, digest, blob.sizeBytes);
110+
this.#publishedDigests.add(digest);
111+
}
112+
105113
async commitManifest(manifestBytes: string): Promise<void> {
106114
this.#assertOpen("commit a manifest");
107115
for (const digest of manifestDigests(manifestBytes)) {

‎packages/aws-lambda/src/s3Transport.ts‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ import {
3535
PutObjectCommand,
3636
type S3Client,
3737
} from "@aws-sdk/client-s3";
38+
import { getSignedUrl } from "@aws-sdk/s3-request-presigner";
3839
import * as tar from "tar";
3940
import { CONTENT_STORE_REFRESH_AFTER_DAYS, contentStoreUri } from "./contentStore.js";
4041

@@ -309,6 +310,17 @@ export async function ensureContentStoreObject(
309310
throw error;
310311
}
311312

313+
export async function presignS3GetUrl(
314+
client: S3Client,
315+
uri: string,
316+
expiresInSeconds: number,
317+
): Promise<string> {
318+
const { bucket, key } = parseS3Uri(uri);
319+
return getSignedUrl(client, new GetObjectCommand({ Bucket: bucket, Key: key }), {
320+
expiresIn: expiresInSeconds,
321+
});
322+
}
323+
312324
// S3 lifecycle expiry counts from the last write, not the last read, so reuse has to write.
313325
async function refreshAgingObject(
314326
client: S3Client,

‎packages/aws-lambda/src/siteManifest.ts‎

Lines changed: 24 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1,21 +1,23 @@
1-
/**
2-
* A deployed site is `sites/<siteId>/files.json`: the project's files by path, each stored once
3-
* in the shared content store under its SHA-256 (see {@link contentStoreUri}).
4-
*/
1+
// A deployed site is `sites/<siteId>/files.json`: the project's files by path, each stored once
2+
// in the shared content store under its SHA-256 (see `contentStoreUri`).
53

64
import { readFileSync } from "node:fs";
7-
import { join } from "node:path";
5+
import { extname, join } from "node:path";
86
import type { S3Client } from "@aws-sdk/client-s3";
7+
import type { RemoteProjectFile } from "@hyperframes/producer/distributed";
98
import { contentStoreUri } from "./contentStore.js";
109
import {
1110
downloadS3ObjectToFile,
1211
downloadS3ObjectToFileVerified,
1312
formatS3Uri,
1413
mapConcurrent,
1514
parseS3Uri,
15+
presignS3GetUrl,
1616
} from "./s3Transport.js";
1717

1818
const SITE_MANIFEST_NAME = "files.json";
19+
const VIDEO_EXTENSIONS_PLANNED_IN_PLACE = new Set([".mp4", ".mov", ".m4v", ".webm", ".mkv"]);
20+
const REMOTE_URL_TTL_SECONDS = 3600;
1921

2022
export interface SiteFile {
2123
/** `/`-separated, relative to the project root. */
@@ -56,24 +58,31 @@ export function parseSiteManifest(text: string): SiteFile[] {
5658
});
5759
}
5860

59-
/** Rebuild the project under `projectDir`, verifying every file's SHA-256 on the way. */
61+
/** Rebuild the project under `projectDir`, SHA-256 verified; videos can stay remote, as URLs. */
6062
export async function downloadSiteFiles(
6163
s3: S3Client,
6264
manifestUri: string,
6365
projectDir: string,
64-
): Promise<void> {
66+
leaveVideosRemote: boolean,
67+
): Promise<Record<string, RemoteProjectFile>> {
6568
const manifestPath = `${projectDir}.${SITE_MANIFEST_NAME}`;
6669
await downloadS3ObjectToFile(s3, manifestUri, manifestPath);
6770
const files = parseSiteManifest(readFileSync(manifestPath, "utf-8"));
6871
const { bucket } = parseS3Uri(manifestUri);
69-
await mapConcurrent(files, 16, (file) =>
70-
downloadS3ObjectToFileVerified(
71-
s3,
72-
contentStoreUri(bucket, file.sha256),
73-
join(projectDir, ...file.path.split("/")),
74-
file.sha256,
75-
),
76-
);
72+
const remoteFiles: Record<string, RemoteProjectFile> = {};
73+
await mapConcurrent(files, 16, async ({ path, sha256, sizeBytes }) => {
74+
const uri = contentStoreUri(bucket, sha256);
75+
if (leaveVideosRemote && VIDEO_EXTENSIONS_PLANNED_IN_PLACE.has(extname(path).toLowerCase())) {
76+
remoteFiles[path] = {
77+
url: await presignS3GetUrl(s3, uri, REMOTE_URL_TTL_SECONDS),
78+
sha256,
79+
sizeBytes,
80+
};
81+
return;
82+
}
83+
await downloadS3ObjectToFileVerified(s3, uri, join(projectDir, ...path.split("/")), sha256);
84+
});
85+
return remoteFiles;
7786
}
7887

7988
function isSafeRelativePath(path: string): boolean {

‎packages/engine/src/services/audioMixer.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1118,6 +1118,7 @@ export async function processCompositionAudio(
11181118
config?: Partial<Pick<EngineConfig, "ffmpegProcessTimeout" | "audioGain">> & {
11191119
/** Serialised master bus chain, from `parseMasterFxChain`. */
11201120
masterFxChain?: string;
1121+
urlByAbsentLocalPath?: ReadonlyMap<string, string>;
11211122
},
11221123
compiledDir?: string,
11231124
): Promise<MixResult> {
@@ -1172,6 +1173,7 @@ export async function processCompositionAudio(
11721173
if (!isHttpUrl(srcPath)) {
11731174
// Same browser-URL path semantics as videos.
11741175
srcPath = resolveProjectRelativeSrc(element.src, baseDir, compiledDir);
1176+
srcPath = config?.urlByAbsentLocalPath?.get(srcPath) ?? srcPath;
11751177
}
11761178

11771179
if (isHttpUrl(srcPath)) {

‎packages/engine/src/services/videoFrameExtractor.ts‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -247,6 +247,7 @@ export interface ExtractionOptions {
247247
deferRangeExtraction?: boolean;
248248
/** Directories the caller rewrites on every render (downloaded media); their sources are cached by content. */
249249
contentKeyedDirs?: readonly string[];
250+
urlByAbsentLocalPath?: ReadonlyMap<string, string>;
250251
}
251252

252253
const DOWNLOAD_SUBDIR = "_downloads";
@@ -1798,6 +1799,11 @@ export async function extractAllVideoFrames(
17981799
if (!isHttpUrl(videoPath)) {
17991800
videoPath = resolveProjectRelativeSrc(video.src, baseDir, compiledDir);
18001801
}
1802+
const remoteUrl = options.urlByAbsentLocalPath?.get(videoPath);
1803+
if (remoteUrl) {
1804+
resolvedVideos.push({ video, videoPath: remoteUrl });
1805+
continue;
1806+
}
18011807

18021808
if (isHttpUrl(videoPath)) {
18031809
const downloadDir = join(options.outputDir, DOWNLOAD_SUBDIR);

0 commit comments

Comments
 (0)