Skip to content

Commit 5a3e258

Browse files
committed
fix(producer): a cloud chunk keeps the missing image's error name so retries stop
1 parent ec7eebf commit 5a3e258

3 files changed

Lines changed: 90 additions & 13 deletions

File tree

‎packages/gcp-cloud-run/src/server.test.ts‎

Lines changed: 27 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -429,22 +429,39 @@ describe("createApp HTTP mapping", () => {
429429
expect(await res.json()).toMatchObject({ error: "PLAN_PROTOCOL_UNSUPPORTED" });
430430
});
431431

432-
it("does not retry an image a frame shows but never got", async () => {
432+
it("does not retry a chunk whose frame shows an image that never arrived", async () => {
433433
const gcs = new FakeGcs();
434434
await seedProjectTar(gcs, planBody.ProjectGcsUri);
435-
const app = createApp(
436-
depsWith(gcs, {
437-
planV2WithPublisher: async () => {
438-
throw Object.assign(new Error("test"), { name: "ImageMissingFromFrameError" });
439-
},
440-
}),
441-
);
442-
const res = await app.request("/", {
435+
const root = mkTmp("hf-image-missing-");
436+
const deps = depsWith(gcs, {
437+
planV2WithPublisher: async (_projectDir, _config, publisher) => {
438+
const v1Dir = join(root, "v1");
439+
makeMinimalV1PlanDir(v1Dir, false);
440+
return publishPlanV2FromExecutionPlan(v1Dir, publisher);
441+
},
442+
renderChunk: async () => {
443+
throw Object.assign(new Error("test"), { name: "ImageMissingFromFrameError" });
444+
},
445+
});
446+
const planned = await dispatch(planBody, deps);
447+
if (!("PlanProtocol" in planned) || planned.PlanProtocol !== "v2") {
448+
throw new Error("expected v2 plan result");
449+
}
450+
const res = await createApp(deps).request("/", {
443451
method: "POST",
444452
headers: { "content-type": "application/json" },
445-
body: JSON.stringify(planBody),
453+
body: JSON.stringify({
454+
Action: "renderChunk",
455+
PlanV2ManifestGcsUri: planned.PlanV2ManifestGcsUri,
456+
PlanV2ArtifactGcsPrefix: planned.PlanV2ArtifactGcsPrefix,
457+
PlanHash: planned.PlanHash,
458+
ChunkIndex: 0,
459+
ChunkOutputGcsPrefix: "gs://b/renders/image-missing/",
460+
Format: "mp4",
461+
}),
446462
});
447463
expect(res.status).toBe(400);
464+
expect(await res.json()).toMatchObject({ error: "ImageMissingFromFrameError" });
448465
});
449466

450467
it.each([

‎packages/producer/src/services/distributed/renderChunk.ts‎

Lines changed: 15 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -383,12 +383,24 @@ export async function runCaptureWithScreenshotFallback<T>(input: {
383383
try {
384384
return await input.run(input.forceScreenshot);
385385
} catch (error) {
386-
if (input.forceScreenshot || (await input.canRetry?.()) === false) throw error;
387-
if (!shouldRetryChunkCaptureWithScreenshot(error)) throw error;
386+
if (input.forceScreenshot || (await input.canRetry?.()) === false)
387+
throw imageMissingIn(error) ?? error;
388+
if (!shouldRetryChunkCaptureWithScreenshot(error)) throw imageMissingIn(error) ?? error;
388389
input.onFallback?.(error);
389390
await input.resetForScreenshotRetry();
390-
return await input.run(true);
391+
return await input.run(true).catch((retryError: unknown) => {
392+
throw imageMissingIn(retryError) ?? retryError;
393+
});
394+
}
395+
}
396+
397+
function imageMissingIn(error: unknown): unknown {
398+
let e = error;
399+
for (let depth = 0; depth < 5 && typeof e === "object" && e !== null; depth++) {
400+
if ((e as { name?: unknown }).name === "ImageMissingFromFrameError") return e;
401+
e = (e as { cause?: unknown }).cause;
391402
}
403+
return undefined;
392404
}
393405

394406
/**

‎packages/producer/src/services/distributed/renderChunkFallback.test.ts‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
import { afterEach, describe, expect, it } from "bun:test";
22
import {
3+
CaptureFailure,
34
type CaptureSession,
5+
ImageMissingFromFrameError,
46
createFrameLookupTable,
57
readWebGlVendorInfoFromCanvas,
68
resolveConfig,
@@ -15,6 +17,7 @@ import {
1517
runCaptureWithScreenshotFallback,
1618
shouldRetryChunkCaptureWithScreenshot,
1719
} from "./renderChunk.js";
20+
import { CaptureStageError } from "../render/captureStageError.js";
1821

1922
const originalProbeTimeout = process.env.PRODUCER_BEGINFRAME_PROBE_TIMEOUT_MS;
2023

@@ -306,3 +309,48 @@ describe("a chunk whose workers launched in different capture modes", () => {
306309
expect(() => observedChunkCaptureMode([{ captureMode: "gpu" }])).toThrow(/invalid mode/);
307310
});
308311
});
312+
313+
describe("a chunk whose frame shows an image that never arrived", () => {
314+
const stopped = () => new ImageMissingFromFrameError(0, ["photo.png"], false);
315+
const throughMultiWorkerCapture = () =>
316+
new CaptureStageError({
317+
cause: new CaptureFailure({
318+
kind: "authoring",
319+
message: "[Parallel] Capture failed: Worker 0",
320+
cause: new CaptureFailure({ kind: "authoring", message: "worker", cause: stopped() }),
321+
}),
322+
browserConsole: [],
323+
});
324+
const run = (first: unknown, second?: unknown) =>
325+
runCaptureWithScreenshotFallback({
326+
forceScreenshot: false,
327+
run: async (forceScreenshot) => {
328+
throw forceScreenshot ? second : first;
329+
},
330+
resetForScreenshotRetry: () => {},
331+
});
332+
333+
it("throws the image's own error from a multi-worker capture, so cloud retry lists see its name", async () => {
334+
await expect(run(throughMultiWorkerCapture())).rejects.toMatchObject({
335+
name: "ImageMissingFromFrameError",
336+
sources: ["photo.png"],
337+
});
338+
});
339+
340+
it("throws it from a single-worker capture too", async () => {
341+
const wrapped = new CaptureStageError({ cause: stopped(), browserConsole: [] });
342+
await expect(run(wrapped)).rejects.toMatchObject({ name: "ImageMissingFromFrameError" });
343+
});
344+
345+
it("throws it when it comes from the screenshot retry", async () => {
346+
const beginFrameFailed = new Error("HeadlessExperimental.beginFrame timed out");
347+
await expect(run(beginFrameFailed, throughMultiWorkerCapture())).rejects.toMatchObject({
348+
name: "ImageMissingFromFrameError",
349+
});
350+
});
351+
352+
it("leaves any other capture failure as it was", async () => {
353+
const other = new CaptureStageError({ cause: new Error("boom"), browserConsole: [] });
354+
await expect(run(other)).rejects.toBe(other);
355+
});
356+
});

0 commit comments

Comments
 (0)