Skip to content

Commit 6afcb48

Browse files
jrusso1020claude
andauthored
fix(engine): sample VFR video by timestamp so it does not freeze on FFmpeg 8 (#5572)
* fix(engine): sample VFR video by timestamp so it does not freeze on FFmpeg 8 VFR sources were resampled with `-fps_mode cfr -r`, which on FFmpeg 8 honours each frame's duration. In mp4 files with B-frames those durations are decode-order gaps, so a frame can claim 0.4s while the next one starts 17ms later, and the extracted video froze on it. VFR now uses the same fps filter sampling as CFR (the frame on screen at each output tick), bounded by -frames:v so a window that ends on a still keeps its rounded-up frame count. The two-process raw pipe that worked around FFmpeg 5.1 dropping durations under -vf is no longer needed and is removed. The vfr-screen-recording golden is regenerated: FFmpeg 5.1's cfr mapping it was rendered with ran one to two source frames late. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]> * fix(engine): move the extraction cache to v7 so v6 VFR frames are not reused v6 entries for VFR sources hold frames sampled by packet duration, which FFmpeg 8 freezes and FFmpeg 5.1 lags. The key is unchanged by the fix, so a warm cache served them after the upgrade. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]> --------- Co-authored-by: Claude Opus 5.5 (1M context) <[email protected]>
1 parent 9a2c1c6 commit 6afcb48

9 files changed

Lines changed: 122 additions & 305 deletions

File tree

‎packages/engine/src/services/extractionCache.test.ts‎

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -67,8 +67,8 @@ function seedPartialDir(entry: { dir: string; keyHash: string }, frameContent: s
6767
}
6868

6969
describe("extractionCache constants", () => {
70-
it("exposes the v6 schema prefix", () => {
71-
expect(SCHEMA_PREFIX).toBe("hfcache-v6-");
70+
it("exposes the v7 schema prefix", () => {
71+
expect(SCHEMA_PREFIX).toBe("hfcache-v7-");
7272
});
7373

7474
it("exposes the frame filename prefix shared with the extractor", () => {
@@ -286,6 +286,19 @@ describe("lookupCacheEntry / markCacheEntryComplete", () => {
286286
expect(lookup.entry.dir).toBe(join(tmpRoot, `${SCHEMA_PREFIX}${keyHash.slice(0, 16)}`));
287287
});
288288

289+
it("does not reuse a complete v6 entry, whose VFR frames were sampled by packet duration", () => {
290+
const input = base(sourceFile);
291+
const keyHash = computeCacheKey(input);
292+
const staleV6Dir = join(tmpRoot, `hfcache-v6-${keyHash.slice(0, 16)}`);
293+
mkdirSync(staleV6Dir, { recursive: true });
294+
writeFileSync(join(staleV6Dir, "frame_00001.jpg"), "old", "utf-8");
295+
writeFileSync(join(staleV6Dir, COMPLETE_SENTINEL), "", "utf-8");
296+
297+
const lookup = lookupCacheEntry(tmpRoot, input);
298+
expect(lookup.hit).toBe(false);
299+
expect(lookup.entry.dir).toBe(join(tmpRoot, `${SCHEMA_PREFIX}${keyHash.slice(0, 16)}`));
300+
});
301+
289302
it("misses on an empty cache root", () => {
290303
const lookup = lookupCacheEntry(tmpRoot, base(sourceFile));
291304
expect(lookup.hit).toBe(false);

‎packages/engine/src/services/extractionCache.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,7 @@ export const GC_MARKER = ".hf-last-gc";
6262
* Current schema version. Bump it whenever extraction writes different frames for the same key,
6363
* or warm entries keep serving the old frames across a deploy. Each bump's commit says why.
6464
*/
65-
export const SCHEMA_PREFIX = "hfcache-v6-";
65+
export const SCHEMA_PREFIX = "hfcache-v7-";
6666

6767
/** Truncated hex chars of SHA-256 used for the entry directory name. */
6868
const KEY_HEX_CHARS = 16;

‎packages/engine/src/services/videoFrameExtractor.concurrency.test.ts‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -91,7 +91,7 @@ describe.skipIf(process.platform === "win32")("extractAllVideoFrames ffmpeg conc
9191
expect(Math.max(...alive)).toBe(1);
9292
}, 120_000);
9393

94-
it("gives a variable-frame-rate clip's two-process pipeline one slot", async () => {
94+
it("runs a variable-frame-rate clip's ffmpeg in the one slot", async () => {
9595
const clips = await Promise.all(
9696
[0, 1].map((i) =>
9797
synth(`vfr-${i}`, `color=c=0x${i ? "28C83C" : "C83C28"}:s=32x32:d=2:r=60`, [
@@ -100,8 +100,8 @@ describe.skipIf(process.platform === "win32")("extractAllVideoFrames ffmpeg conc
100100
),
101101
);
102102
const alive = await aliveCounts("vfr", clips, 1, 30);
103-
expect(alive).toHaveLength(4);
104-
expect(Math.max(...alive)).toBeLessThanOrEqual(2);
103+
expect(alive).toHaveLength(2);
104+
expect(Math.max(...alive)).toBe(1);
105105
}, 60_000);
106106

107107
it("lets a cancelled extraction leave the queue while another holds the slot", async () => {

‎packages/engine/src/services/videoFrameExtractor.test.ts‎

Lines changed: 56 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2444,6 +2444,59 @@ describe.skipIf(!HAS_FFMPEG)("extractAllVideoFrames on a VFR source", () => {
24442444
expect(result.totalFrames).toBe(3);
24452445
}, 60_000);
24462446

2447+
// With B-frames, mp4 packet durations are decode-order gaps, so a frame can claim 0.5s while
2448+
// the next one starts 17ms later. Each slot must still show the frame on screen at its time:
2449+
// `-fps_mode cfr` on ffmpeg 8 held such a frame for 14 slots and cut the still before a gap short.
2450+
it("samples a B-frame VFR source by timestamp, not by packet duration", async () => {
2451+
const source = join(FIXTURE_DIR, "vfr_bframes.mp4");
2452+
const synth = await runFfmpeg([
2453+
"-y",
2454+
"-v",
2455+
"error",
2456+
"-f",
2457+
"lavfi",
2458+
"-i",
2459+
"testsrc2=s=320x180:d=4:rate=60",
2460+
"-vf",
2461+
"select='not(between(n\\,30\\,89))*not(between(n\\,150\\,179))'",
2462+
"-fps_mode",
2463+
"vfr",
2464+
"-c:v",
2465+
"libx264",
2466+
"-preset",
2467+
"medium",
2468+
"-bf",
2469+
"3",
2470+
"-g",
2471+
"600",
2472+
"-pix_fmt",
2473+
"yuv420p",
2474+
source,
2475+
]);
2476+
if (!synth.success) throw new Error(`fixture synthesis failed: ${synth.stderr.slice(-400)}`);
2477+
const distinctFrames = (paths: Iterable<string>) =>
2478+
new Set([...paths].map((path) => readFileSync(path).toString("base64"))).size;
2479+
2480+
// 2.5s to 3s is a still; every 30fps slot after it lands on its own 60fps source frame.
2481+
const afterGap = await extractVideoFramesRange(source, "vfr-bframes-after-gap", 2.5, 1.5, {
2482+
fps: 30,
2483+
outputDir: join(FIXTURE_DIR, "out-vfr-bframes-after-gap"),
2484+
format: "png",
2485+
});
2486+
expect(afterGap.metadata.isVFR).toBe(true);
2487+
expect(afterGap.totalFrames).toBe(45);
2488+
const slots = [...afterGap.framePaths.entries()].sort(([a], [b]) => a - b);
2489+
expect(distinctFrames(slots.slice(0, 15).map(([, path]) => path))).toBe(1);
2490+
expect(distinctFrames(slots.slice(15).map(([, path]) => path))).toBe(30);
2491+
2492+
const endsOnStill = await extractVideoFramesRange(source, "vfr-bframes-still", 0, 3, {
2493+
fps: 30,
2494+
outputDir: join(FIXTURE_DIR, "out-vfr-bframes-still"),
2495+
format: "png",
2496+
});
2497+
expect(endsOnStill.totalFrames).toBe(90);
2498+
}, 60_000);
2499+
24472500
it("produces the expected frame count for a mid-file segment", async () => {
24482501
const outputDir = join(FIXTURE_DIR, "out-mid-segment");
24492502
mkdirSync(outputDir, { recursive: true });
@@ -3751,7 +3804,7 @@ describe.skipIf(!HAS_ZSCALE)("forced-SDR HDR extraction", () => {
37513804
}, 60_000);
37523805

37533806
// A second process would read the frames back without the HDR10 light-level metadata the tone map uses.
3754-
it("tone-maps a VFR HDR10 window in one process, exactly as the unfiltered resample would", async () => {
3807+
it("tone-maps a VFR HDR10 window in one process, as the reference tone map would", async () => {
37553808
const source = join(fixtureDir, "pq-vfr-4000nit.mp4");
37563809
const synthesized = await runFfmpeg([
37573810
"-y",
@@ -3807,7 +3860,8 @@ describe.skipIf(!HAS_ZSCALE)("forced-SDR HDR extraction", () => {
38073860

38083861
expect(result.metadata.isVFR).toBe(true);
38093862
const referenceFrames = readdirSync(referenceDir).sort();
3810-
expect(result.totalFrames).toBe(referenceFrames.length);
3863+
// One slot per 30fps tick before 0.616666s; x265 packet durations cut the reference's still short.
3864+
expect(result.totalFrames).toBe(19);
38113865
expect(readFileSync(result.framePaths.get(0)!)).toEqual(
38123866
readFileSync(join(referenceDir, referenceFrames[0]!)),
38133867
);

‎packages/engine/src/services/videoFrameExtractor.ts‎

Lines changed: 36 additions & 122 deletions
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,7 @@ import {
5757
UrlDownloadError,
5858
writeUrlDownloadTelemetry,
5959
} from "../utils/urlDownloader.js";
60-
import { runFfmpeg, runFfmpegPipeline, type RunFfmpegResult } from "../utils/runFfmpeg.js";
60+
import { runFfmpeg, type RunFfmpegResult } from "../utils/runFfmpeg.js";
6161
import { isFfmpegFilterAvailable } from "../utils/psnrFilterAvailability.js";
6262
import { DEFAULT_CONFIG, type EngineConfig } from "../config.js";
6363
import { unwrapTemplate } from "../utils/htmlTemplate.js";
@@ -127,9 +127,9 @@ export function isVideoFrameFormat(value: unknown): value is VideoFrameFormat {
127127
/**
128128
* Resolve the frame count produced for a requested extraction duration.
129129
*
130-
* CFR extraction uses FFmpeg's fps filter, whose end boundary rounds to the
131-
* nearest frame. The VFR path normalizes with `-fps_mode cfr -r`, whose end
132-
* boundary rounds up. Keep this calculation shared by superset slicing and
130+
* Both paths sample with FFmpeg's fps filter. A CFR window ends at the nearest
131+
* frame boundary; a VFR window rounds up, keeping the slot a trailing still
132+
* covers. Keep this calculation shared by superset slicing and
133133
* producer coverage accounting so a complete VFR extraction cannot be
134134
* rejected because the two paths disagree by one frame.
135135
*/
@@ -297,8 +297,8 @@ function sdrToHdrTransformKey(transfer: HdrTransfer): string {
297297
* reflects max-across-parallel-workers, not sum.
298298
* - hdrPreflightMs includes its probe-time sibling (hdrProbeMs); the
299299
* probe-only field is a finer decomposition, not a separate carve-out.
300-
* - vfrPreflightCount reports sources classified as VFR and routed through
301-
* the one-pass `-fps_mode cfr -r` extraction path. DEFINITION CHANGE:
300+
* - vfrPreflightCount reports sources classified as VFR and extracted with
301+
* a rounded-up frame count. DEFINITION CHANGE:
302302
* before the one-pass refactor, vfrPreflightMs timed a per-source
303303
* VFR-to-CFR re-encode and could reach seconds; it now times only the
304304
* (promise-cached) classification probe and is expected to be ~0.
@@ -829,61 +829,6 @@ function chromeGuessForUntaggedMatrix(metadata: VideoMetadata): string[] {
829829
return [`setparams=colorspace=${hd ? "bt709" : "smpte170m"}`];
830830
}
831831

832-
/** Pixel formats nut carries as raw video unchanged on ffmpeg 5.1 to 8.1 (yuvj only loses its range). */
833-
const NUT_RAW_PIXEL_FORMATS = new Set([
834-
"yuv420p",
835-
"yuvj420p",
836-
"yuv422p",
837-
"yuvj422p",
838-
"yuv444p",
839-
"yuvj444p",
840-
"yuva420p",
841-
"yuva444p",
842-
"yuv440p",
843-
"yuv411p",
844-
"yuv410p",
845-
"yuva422p",
846-
"yuv420p10le",
847-
"yuv422p10le",
848-
"yuv444p10le",
849-
"yuva420p10le",
850-
"yuva444p10le",
851-
"yuv420p12le",
852-
"nv12",
853-
"nv21",
854-
"yuyv422",
855-
"uyvy422",
856-
"gray",
857-
"gray10le",
858-
"ya8",
859-
"gbrp",
860-
"gbrap",
861-
"gbrp10le",
862-
"gbrp12le",
863-
"gbrap10le",
864-
"rgb24",
865-
"bgr24",
866-
"rgba",
867-
"bgra",
868-
"argb",
869-
"abgr",
870-
"rgb48le",
871-
"rgba64le",
872-
]);
873-
874-
/** nut drops colour tags, so raw frames get the source's back before any other filter reads them. */
875-
function restoreSourceColourFilter(metadata: VideoMetadata): string[] {
876-
const tags = [
877-
["range", metadata.colorRange],
878-
["colorspace", metadata.colorSpace?.colorSpace],
879-
["color_primaries", metadata.colorSpace?.colorPrimaries],
880-
["color_trc", metadata.colorSpace?.colorTransfer],
881-
].filter(([, value]) => value && value !== "unknown" && value !== "reserved");
882-
return tags.length > 0
883-
? [`setparams=${tags.map(([key, value]) => `${key}=${value}`).join(":")}`]
884-
: [];
885-
}
886-
887832
/**
888833
* Longest stretch of output one ffmpeg process extracts. `ffmpegProcessTimeout` bounds a
889834
* single process, so one process over a long clip timed out however much time the render
@@ -1036,8 +981,8 @@ export async function extractVideoFramesRange(
1036981
const toneMappedToSdr = options.toneMapHdrToSdr === true && isHdr && (await toneMapsWithZscale());
1037982
const decodeWithVideoToolbox = isHdr && isMacOS && !toneMappedToSdr;
1038983

1039-
const sampleCfrAtOutputFps = !options.finalFrameOnly && !metadata.isVFR;
1040-
if (options.frameRanges && !sampleCfrAtOutputFps) {
984+
const sampleAtOutputFps = !options.finalFrameOnly;
985+
if (options.frameRanges && (!sampleAtOutputFps || metadata.isVFR)) {
1041986
throw new Error("frameRanges extraction needs a constant-frame-rate source");
1042987
}
1043988

@@ -1066,15 +1011,15 @@ export async function extractVideoFramesRange(
10661011
args.push("-i", videoPath, "-ss", String(startTime), "-frames:v", "1");
10671012
} else {
10681013
// An accurate seek drops the frame on screen at startTime; the fps sampling below needs it.
1069-
if (sampleCfrAtOutputFps) args.push("-noaccurate_seek");
1014+
if (sampleAtOutputFps) args.push("-noaccurate_seek");
10701015
args.push("-ss", String(startTime), "-i", videoPath, "-t", String(duration));
10711016
}
10721017

10731018
const vfFilters: string[] = [];
10741019
if (decodeWithVideoToolbox) {
10751020
vfFilters.push("format=nv12");
10761021
}
1077-
if (sampleCfrAtOutputFps) {
1022+
if (sampleAtOutputFps) {
10781023
// Each slot takes the last frame starting at or before its time. Ticks of 2 ms or finer
10791024
// first shift half a tick, absorbing container rounding (WebM/MKV use whole ms).
10801025
vfFilters.push(
@@ -1110,59 +1055,31 @@ export async function extractVideoFramesRange(
11101055
encodeArgs.push("-y", outputPattern);
11111056

11121057
const runOptions = { signal, timeout: ffmpegProcessTimeout };
1113-
const resampleVfrToCfr = !options.finalFrameOnly && metadata.isVFR;
1114-
if (resampleVfrToCfr) args.push("-fps_mode", "cfr", "-r", ffmpegFps);
1115-
let processResult: RunFfmpegResult;
1116-
if (
1117-
resampleVfrToCfr &&
1118-
vfFilters.length > 0 &&
1119-
!isHdr &&
1120-
NUT_RAW_PIXEL_FORMATS.has(metadata.pixelFormat ?? "")
1121-
) {
1122-
// ffmpeg <6.1 ignores frame durations in CFR resampling once any -vf is set,
1123-
// cutting a trailing still short, so the SDR filters run in a second process.
1124-
processResult = await inExtractionSlot(
1125-
() =>
1126-
runFfmpegPipeline(
1127-
[...args, "-an", "-sn", "-dn", "-c:v", "rawvideo", "-f", "nut", "pipe:1"],
1128-
[
1129-
"-f",
1130-
"nut",
1131-
"-i",
1132-
"pipe:0",
1133-
"-vf",
1134-
[...restoreSourceColourFilter(metadata), ...vfFilters].join(","),
1135-
"-fps_mode",
1136-
"passthrough",
1137-
...encodeArgs,
1138-
],
1058+
const filterArgs = vfFilters.length > 0 ? ["-vf", vfFilters.join(",")] : [];
1059+
const segmentFrames = Math.round(EXTRACTION_SEGMENT_SECONDS * fps);
1060+
const totalFrames = extractionFrameCountForDuration(duration, normalizedFps, metadata.isVFR);
1061+
// A VFR window rounds its frame count up, and only the segments' `-frames:v` bound
1062+
// emits that last slot: the fps filter ends a `-t` window at the nearest slot. A
1063+
// decimal rate counts zero frames here and keeps the `-t` window.
1064+
const boundVfrFrameCount = metadata.isVFR && totalFrames > 0;
1065+
const processResult =
1066+
sampleAtOutputFps && (options.frameRanges || boundVfrFrameCount || totalFrames > segmentFrames)
1067+
? await runSegmentedExtraction({
1068+
decodeArgs,
1069+
filterAndEncodeArgs: [...filterArgs, ...encodeArgs],
1070+
videoPath,
1071+
startTime,
1072+
fps,
1073+
segments: splitFrameRanges(
1074+
options.frameRanges ?? [{ firstFrame: 0, frames: totalFrames }],
1075+
segmentFrames,
1076+
),
11391077
runOptions,
1140-
),
1141-
signal,
1142-
);
1143-
} else {
1144-
const filterArgs = vfFilters.length > 0 ? ["-vf", vfFilters.join(",")] : [];
1145-
const segmentFrames = Math.round(EXTRACTION_SEGMENT_SECONDS * fps);
1146-
const totalFrames = extractionFrameCountForDuration(duration, normalizedFps, false);
1147-
processResult =
1148-
sampleCfrAtOutputFps && (options.frameRanges || totalFrames > segmentFrames)
1149-
? await runSegmentedExtraction({
1150-
decodeArgs,
1151-
filterAndEncodeArgs: [...filterArgs, ...encodeArgs],
1152-
videoPath,
1153-
startTime,
1154-
fps,
1155-
segments: splitFrameRanges(
1156-
options.frameRanges ?? [{ firstFrame: 0, frames: totalFrames }],
1157-
segmentFrames,
1158-
),
1159-
runOptions,
1160-
})
1161-
: await inExtractionSlot(
1162-
() => runFfmpeg([...args, ...filterArgs, ...encodeArgs], runOptions),
1163-
signal,
1164-
);
1165-
}
1078+
})
1079+
: await inExtractionSlot(
1080+
() => runFfmpeg([...args, ...filterArgs, ...encodeArgs], runOptions),
1081+
signal,
1082+
);
11661083
if (processResult.failureReason === "external_interruption") {
11671084
throw new VideoSourceExtractionError(
11681085
"external_interruption",
@@ -1714,11 +1631,8 @@ function buildSupersetGroup(
17141631
): SupersetGroupPlan | null {
17151632
if (misses.length < 2) return null;
17161633
if (misses.some(({ work }) => work.finalFrameOnly)) return null;
1717-
// VFR normalization (`-fps_mode cfr -r`) establishes its duplicate/drop
1718-
// phase relative to each seek. A union extraction therefore cannot be
1719-
// sliced into the same frames as independently sought member ranges, even
1720-
// when their offsets land on an integral output-frame boundary. Keep VFR
1721-
// ranges direct until the extractor has a proven absolute timestamp phase.
1634+
// VFR ranges round their frame counts up, so a union extraction's slices
1635+
// are not proven to match independently sought member ranges. Keep them direct.
17221636
if (misses.some(({ work }) => work.metadata.isVFR)) return null;
17231637
const baseStart = Math.min(...misses.map(({ work }) => work.video.mediaStart));
17241638
if (!misses.every(({ work }) => isIntegralFrameOffset(work.video.mediaStart - baseStart, fps))) {

0 commit comments

Comments
 (0)