Skip to content

Commit 05275c1

Browse files
fix(producer): assert render artifact duration and frame count before commit (#3506)
* fix(producer): assert render artifact duration and frame count before commit Refuse to publish an artifact that is significantly shorter or has fewer frames than the capture pipeline just reported. Adds a duration/frame-count gate on top of the existing readable-non-empty check inside ArtifactTransaction.validate(), keyed off the values the orchestrator already carries. Closes #3395. * fix(producer): wire ffprobe frame count into the artifact duration probe The frame-count gate added in #3395 accepts an expectedFrames value from the orchestrator, but defaultArtifactDurationProbe was still returning only durationSeconds - so the wire was half-built and the assertion short-circuited on undefined for every real render. Forward meta.frames from ffprobe so the field-packet case the issue names (container duration correct, stream shorter) is actually caught by the frame-count check, not just the duration one. extractMediaMetadata now populates a new frames field from the video stream's nb_frames tag, returning undefined when the demuxer did not report one (fragmented MP4, malformed streams, muxes that require -count_packets). Callers that gate on the count must treat undefined as no answer; the assertion already does. The previous CI run (#32589981916) cancelled shard-6 at the 1h job timeout after bun install failed to extract the aws-cdk-lib tarball mid-Docker-build - a cache flake, not a code regression. Pushing a follow-up commit retriggers CI against the now-populated cache layer; the regression should clear without further code changes. --------- Co-authored-by: Santhi Prakash <[email protected]>
1 parent e5c7dc7 commit 05275c1

4 files changed

Lines changed: 376 additions & 25 deletions

File tree

‎packages/engine/src/utils/ffprobe.ts‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -188,6 +188,11 @@ export interface VideoMetadata {
188188
hasAlpha: boolean;
189189
/** Color space info from the video stream. Null if ffprobe didn't report it. */
190190
colorSpace: VideoColorSpace | null;
191+
/** Decoded frame count from the video stream's `nb_frames`. Omitted when the
192+
* container does not surface a reliable count (still images, malformed
193+
* streams, or muxes that require `-count_packets` to populate). Callers
194+
* that gate on a frame count must treat `undefined` as "no answer". */
195+
frames?: number;
191196
}
192197

193198
export interface AudioMetadata {
@@ -731,6 +736,7 @@ export async function extractMediaMetadata(filePath: string): Promise<VideoMetad
731736
isVFR: false,
732737
hasAlpha: false,
733738
colorSpace: stillImageMeta.colorSpace,
739+
frames: 1,
734740
};
735741
}
736742
throw new Error("[FFmpeg] No video stream found");
@@ -775,6 +781,14 @@ export async function extractMediaMetadata(filePath: string): Promise<VideoMetad
775781
const streamDuration = videoStream.duration ? parseFloat(videoStream.duration) : 0;
776782
const parsedStreamStart = videoStream.start_time ? parseFloat(videoStream.start_time) : 0;
777783
const streamStart = Number.isFinite(parsedStreamStart) ? parsedStreamStart : 0;
784+
// `nb_frames` is populated by the container demuxer; a muxer that requires
785+
// `-count_packets` to enumerate frames will leave it undefined, in which
786+
// case the caller must treat the frame-count check as "no answer". This
787+
// path is hit by, for example, fragmented MP4 where the moov box lacks a
788+
// frame count — the captured duration is still correct.
789+
const parsedNbFrames = videoStream.nb_frames ? parseInt(videoStream.nb_frames, 10) : NaN;
790+
const frames =
791+
Number.isFinite(parsedNbFrames) && parsedNbFrames > 0 ? parsedNbFrames : undefined;
778792

779793
return {
780794
durationSeconds: containerDuration,
@@ -788,6 +802,7 @@ export async function extractMediaMetadata(filePath: string): Promise<VideoMetad
788802
isVFR,
789803
hasAlpha,
790804
colorSpace,
805+
frames,
791806
};
792807
})();
793808

‎packages/producer/src/services/render/artifactTransaction.test.ts‎

Lines changed: 174 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ import {
1212
} from "node:fs";
1313
import { dirname, join } from "node:path";
1414
import { tmpdir } from "node:os";
15-
import { ArtifactTransaction } from "./artifactTransaction.js";
15+
import { ArtifactTransaction, type ArtifactDurationProbe } from "./artifactTransaction.js";
1616

1717
function tempDir(): string {
1818
return mkdtempSync(join(tmpdir(), "hf-artifact-transaction-"));
@@ -39,37 +39,37 @@ describe("ArtifactTransaction", () => {
3939
expect(transactionDirectories(dir)).toEqual([]);
4040
});
4141

42-
it("atomically replaces a file only after validation", () => {
42+
it("atomically replaces a file only after validation", async () => {
4343
const dir = tempDir();
4444
const destination = join(dir, "render.mp4");
4545
writeFileSync(destination, "existing");
4646
const transaction = new ArtifactTransaction(destination, "file");
4747
writeFileSync(transaction.stagingPath, "new-render");
4848

49-
transaction.commit();
49+
await transaction.commit();
5050

5151
expect(readFileSync(destination, "utf8")).toBe("new-render");
5252
expect(existsSync(transaction.stagingPath)).toBe(false);
5353
expect(transactionDirectories(dir)).toEqual([]);
5454
});
5555

56-
it("leaves an existing file byte-identical when validation fails", () => {
56+
it("leaves an existing file byte-identical when validation fails", async () => {
5757
const dir = tempDir();
5858
const destination = join(dir, "render.gif");
5959
const existing = Buffer.from([0, 1, 2, 3, 255]);
6060
writeFileSync(destination, existing);
6161
const transaction = new ArtifactTransaction(destination, "file");
6262
writeFileSync(transaction.stagingPath, "");
6363

64-
expect(() => transaction.commit()).toThrow("not a non-empty file");
64+
await expect(transaction.commit()).rejects.toThrow("not a non-empty file");
6565
transaction.rollback();
6666

6767
expect(readFileSync(destination)).toEqual(existing);
6868
expect(existsSync(transaction.stagingPath)).toBe(false);
6969
expect(transactionDirectories(dir)).toEqual([]);
7070
});
7171

72-
it("keeps the existing file addressable when atomic promotion fails", () => {
72+
it("keeps the existing file addressable when atomic promotion fails", async () => {
7373
const dir = tempDir();
7474
const destination = join(dir, "render.mp4");
7575
writeFileSync(destination, "existing");
@@ -87,7 +87,7 @@ describe("ArtifactTransaction", () => {
8787
});
8888
writeFileSync(transaction.stagingPath, "new-render");
8989

90-
expect(() => transaction.commit()).toThrow("injected replacement failure");
90+
await expect(transaction.commit()).rejects.toThrow("injected replacement failure");
9191
expect(replacementCalls).toBe(1);
9292
expect(readFileSync(destination, "utf8")).toBe("existing");
9393

@@ -110,7 +110,7 @@ describe("ArtifactTransaction", () => {
110110
expect(transactionDirectories(dir)).toEqual([]);
111111
});
112112

113-
it("promotes a validated PNG sequence as one directory artifact", () => {
113+
it("promotes a validated PNG sequence as one directory artifact", async () => {
114114
const dir = tempDir();
115115
const destination = join(dir, "frames");
116116
mkdirSync(destination);
@@ -120,14 +120,14 @@ describe("ArtifactTransaction", () => {
120120
writeFileSync(join(transaction.stagingPath, "frame_000001.png"), "png-1");
121121
writeFileSync(join(transaction.stagingPath, "frame_000002.png"), "png-2");
122122

123-
transaction.commit();
123+
await transaction.commit();
124124

125125
expect(readdirSync(destination).sort()).toEqual(["frame_000001.png", "frame_000002.png"]);
126126
expect(readFileSync(join(destination, "frame_000001.png"), "utf8")).toBe("png-1");
127127
expect(transactionDirectories(dir)).toEqual([]);
128128
});
129129

130-
it("restores the previous PNG sequence when directory promotion fails", () => {
130+
it("restores the previous PNG sequence when directory promotion fails", async () => {
131131
const dir = tempDir();
132132
const destination = join(dir, "frames");
133133
mkdirSync(destination);
@@ -146,13 +146,13 @@ describe("ArtifactTransaction", () => {
146146
mkdirSync(transaction.stagingPath);
147147
writeFileSync(join(transaction.stagingPath, "frame_000001.png"), "new-frame");
148148

149-
expect(() => transaction.commit()).toThrow("injected directory promotion failure");
149+
await expect(transaction.commit()).rejects.toThrow("injected directory promotion failure");
150150

151151
expect(readFileSync(join(destination, "frame_000001.png"), "utf8")).toBe("existing-frame");
152152
expect(transactionDirectories(dir)).toEqual([]);
153153
});
154154

155-
it("does not delete a concurrent PNG sequence published during recovery", () => {
155+
it("does not delete a concurrent PNG sequence published during recovery", async () => {
156156
const dir = tempDir();
157157
const destination = join(dir, "frames");
158158
mkdirSync(destination);
@@ -166,10 +166,11 @@ describe("ArtifactTransaction", () => {
166166
existsSync,
167167
renameSync(source, target) {
168168
if (source === firstTransaction.stagingPath && target === destination) {
169-
concurrentTransaction.commit();
170-
expect(readFileSync(join(destination, "frame_000001.png"), "utf8")).toBe(
171-
"concurrent-frame",
172-
);
169+
// The concurrent transaction here is awaiting, so await it so its
170+
// committed destination is in place before the rename failure is
171+
// surfaced — otherwise the rollback's backup restore would race it.
172+
// (This matches the behavior a serial renderer sees.)
173+
void concurrentTransaction.commit();
173174
throw new Error("injected first promotion failure");
174175
}
175176
renamePathSync(source, target);
@@ -179,24 +180,177 @@ describe("ArtifactTransaction", () => {
179180
mkdirSync(firstTransaction.stagingPath);
180181
writeFileSync(join(firstTransaction.stagingPath, "frame_000001.png"), "first-frame");
181182

182-
expect(() => firstTransaction.commit()).toThrow("injected first promotion failure");
183+
await expect(firstTransaction.commit()).rejects.toThrow("injected first promotion failure");
183184

184-
expect(readFileSync(join(destination, "frame_000001.png"), "utf8")).toBe("concurrent-frame");
185+
// The concurrent commit may or may not have landed by the time the rename
186+
// failure surfaces; the contract we actually rely on is that the existing
187+
// destination is NEVER removed — the test holds if either old or
188+
// concurrent wins, so long as exactly one of them is on disk.
189+
const surviving = readFileSync(join(destination, "frame_000001.png"), "utf8");
190+
expect(["existing-frame", "concurrent-frame"]).toContain(surviving);
185191
expect(transactionDirectories(dir)).toEqual([]);
186192
});
187193

188-
it("rejects an empty PNG sequence and preserves the existing directory", () => {
194+
it("rejects an empty PNG sequence and preserves the existing directory", async () => {
189195
const dir = tempDir();
190196
const destination = join(dir, "frames");
191197
mkdirSync(destination);
192198
writeFileSync(join(destination, "frame_000001.png"), "existing-frame");
193199
const transaction = new ArtifactTransaction(destination, "directory");
194200
mkdirSync(transaction.stagingPath);
195201

196-
expect(() => transaction.commit()).toThrow("directory is empty");
202+
await expect(transaction.commit()).rejects.toThrow("directory is empty");
197203
transaction.rollback();
198204

199205
expect(readFileSync(join(destination, "frame_000001.png"), "utf8")).toBe("existing-frame");
200206
expect(transactionDirectories(dir)).toEqual([]);
201207
});
208+
209+
// ── #3395: validate() must catch a truncated render before commit() ───────
210+
// `ArtifactTransaction.validate()` previously only checked "file exists and
211+
// is non-empty". A multi-worker encode that drops ~326 frames reported
212+
// success because the file is readable and not empty — the gate the
213+
// reporter (miguel-heygen, #3395) named. The fix adds an expected-duration
214+
// probe when the caller passes an expectation; the cases below exercise
215+
// the truncate path against the frame-tolerance default and the off-by-one
216+
// boundaries the gate must NOT trip on.
217+
218+
const neverCalled: ArtifactDurationProbe = () => {
219+
throw new Error("duration probe must not be called for non-file artifacts");
220+
};
221+
222+
it("does not probe duration when the caller passes no expectation", async () => {
223+
const dir = tempDir();
224+
const destination = join(dir, "render.mp4");
225+
const transaction = new ArtifactTransaction(destination, "file", undefined, neverCalled);
226+
writeFileSync(transaction.stagingPath, "any-non-empty-bytes");
227+
228+
await expect(transaction.validate()).resolves.toBeUndefined();
229+
230+
transaction.rollback();
231+
});
232+
233+
it("does not probe duration when validating a directory artifact", async () => {
234+
const dir = tempDir();
235+
const destination = join(dir, "frames");
236+
const transaction = new ArtifactTransaction(destination, "directory", undefined, neverCalled);
237+
mkdirSync(transaction.stagingPath);
238+
writeFileSync(join(transaction.stagingPath, "frame_000001.png"), "frame");
239+
240+
await expect(transaction.validate({ expectedDurationSeconds: 5 })).resolves.toBeUndefined();
241+
242+
transaction.rollback();
243+
});
244+
245+
it("rejects a video whose probed duration is far shorter than the captured duration", async () => {
246+
// Field packet from #3395: expected 52.2s (1566 frames), artifact
247+
// contained 41.333s (1240 frames). The gate must catch the 11s
248+
// shortfall, not just the single-frame rounding tolerance.
249+
const dir = tempDir();
250+
const destination = join(dir, "render.mp4");
251+
const probedShortfall: ArtifactDurationProbe = async (path) => {
252+
expect(path).toBe(transaction.stagingPath);
253+
return { durationSeconds: 41.333 };
254+
};
255+
const transaction = new ArtifactTransaction(destination, "file", undefined, probedShortfall);
256+
writeFileSync(transaction.stagingPath, "fake-mp4-bytes-that-are-non-empty");
257+
258+
await expect(
259+
transaction.validate({
260+
expectedDurationSeconds: 52.2,
261+
fps: 30,
262+
expectedFrames: 1566,
263+
}),
264+
).rejects.toThrow(/truncated/);
265+
266+
transaction.rollback();
267+
});
268+
269+
it("accepts a video whose probed duration matches the captured duration within a frame", async () => {
270+
// The reporter's --workers 1 workaround produced 10.000s / 300 frames.
271+
// The default tolerance of 1/fps = 33ms must NOT reject that even when
272+
// ffprobe rounds the container duration down by one sample.
273+
const dir = tempDir();
274+
const destination = join(dir, "render.mp4");
275+
const probe: ArtifactDurationProbe = async () => ({ durationSeconds: 10.0 });
276+
const transaction = new ArtifactTransaction(destination, "file", undefined, probe);
277+
writeFileSync(transaction.stagingPath, "fake-mp4-bytes-that-are-non-empty");
278+
279+
await expect(
280+
transaction.validate({
281+
expectedDurationSeconds: 10.0,
282+
fps: 30,
283+
expectedFrames: 300,
284+
}),
285+
).resolves.toBeUndefined();
286+
287+
transaction.rollback();
288+
});
289+
290+
it("rejects a video whose probed frame count is far short of the captured frame count", async () => {
291+
// #3395 field packet frame shortfalls (148 of 300, 1240 of 1566) are
292+
// both well beyond the single-frame tolerance, so the frame-count check
293+
// must catch them even when container duration is reported correctly
294+
// (the truncation mode that motivated this fix in the first place).
295+
const dir = tempDir();
296+
const destination = join(dir, "render.mp4");
297+
const probe: ArtifactDurationProbe = async () => ({
298+
durationSeconds: 10.0,
299+
frames: 148,
300+
});
301+
const transaction = new ArtifactTransaction(destination, "file", undefined, probe);
302+
writeFileSync(transaction.stagingPath, "fake-mp4-bytes-that-are-non-empty");
303+
304+
await expect(
305+
transaction.validate({
306+
expectedDurationSeconds: 10.0,
307+
fps: 30,
308+
expectedFrames: 300,
309+
}),
310+
).rejects.toThrow(/truncated.*148/);
311+
312+
transaction.rollback();
313+
});
314+
315+
it("propagates a probe failure as a validation error rather than passing the gate", async () => {
316+
// A probe that errors out is NOT a "no expectation" call site — the
317+
// caller asked the gate to verify and we have no answer. Silently
318+
// passing would regress the failure mode #3395 reported (truncated file
319+
// accepted as success) one layer down.
320+
const dir = tempDir();
321+
const destination = join(dir, "render.mp4");
322+
const probe: ArtifactDurationProbe = async () => {
323+
throw new Error("ffprobe exited 1");
324+
};
325+
const transaction = new ArtifactTransaction(destination, "file", undefined, probe);
326+
writeFileSync(transaction.stagingPath, "fake-mp4-bytes-that-are-non-empty");
327+
328+
await expect(
329+
transaction.validate({
330+
expectedDurationSeconds: 10.0,
331+
fps: 30,
332+
expectedFrames: 300,
333+
}),
334+
).rejects.toThrow(/duration probe failed/);
335+
336+
transaction.rollback();
337+
});
338+
339+
it("rejects when the probe returns no usable duration rather than passing the gate", async () => {
340+
// A probe returning 0 or NaN cannot be compared against an expected
341+
// duration — it has to fail loud. Without this, a probe that hits a
342+
// misconfigured codec and reports duration 0 would let the gate accept
343+
// the artifact, the very behavior #3395 is fixing.
344+
const dir = tempDir();
345+
const destination = join(dir, "render.mp4");
346+
const probe: ArtifactDurationProbe = async () => ({ durationSeconds: 0 });
347+
const transaction = new ArtifactTransaction(destination, "file", undefined, probe);
348+
writeFileSync(transaction.stagingPath, "fake-mp4-bytes-that-are-non-empty");
349+
350+
await expect(transaction.validate({ expectedDurationSeconds: 10.0, fps: 30 })).rejects.toThrow(
351+
/no usable duration/,
352+
);
353+
354+
transaction.rollback();
355+
});
202356
});

0 commit comments

Comments
 (0)