Skip to content

Commit 3a64ea8

Browse files
fix(engine): forced-SDR renders show tone-mapped HDR footage as Studio does (#4600)
* fix(engine): forced-SDR renders show tone-mapped HDR footage as Studio does Forced-SDR extraction tone-maps HDR frames with zscale, but the frames kept the BT.709 transfer tag, so Chrome re-curved them (a tone-mapped 388d96 painted as 2d8790), and JPEG frames kept the BT.709 matrix every JPEG reader decodes as BT.601. Tone-mapped frames now get the same sRGB tag and BT.601 JPEG conversion as other SDR frames. macOS used VideoToolbox, which does not tone-map: its frames stay PQ and Chrome tone-maps them itself, darker than the hable curve Linux and Studio use. macOS now takes the zscale path whenever ffmpeg has zscale, found through the shared ffmpeg filter probe; without it the old path stays and the render warns once. Co-authored-by: Kenneth Shortrede <[email protected]> * fix(engine): forced-SDR renders tone-map HDR footage that lacks matrix or primaries tags zscale finds no path between colourspaces without them, so the extraction failed. Untagged HDR footage is BT.2020 in practice, and now gets those tags. Co-authored-by: Kenneth Shortrede <[email protected]> * fix(engine): forced-SDR tone map also tags the transfer ffprobe read from the container From ffmpeg 8.1 ffprobe reports a container-only PQ or HLG tag; the frames still lack it, so zscale failed where macOS VideoToolbox used to render. Co-authored-by: Kenneth Shortrede <[email protected]> * test(engine): gate the container-only transfer case on the ffprobe that reads it * docs(engine): shorten the filter-probe comment to the comment limit * fix(engine): keep valid per-frame colour tags when tone-mapping HDR footage * test(engine): build the mixed HLG and PQ fixture without B-frames so its timestamps are valid * test(engine): build the trimmed mixed fixture without B-frames too --------- Co-authored-by: Kenneth Shortrede <[email protected]>
1 parent 0744667 commit 3a64ea8

4 files changed

Lines changed: 420 additions & 42 deletions

File tree

‎packages/engine/src/services/videoFrameExtractor.test.ts‎

Lines changed: 304 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,9 @@ import { RATE_RANGE } from "@hyperframes/core/audio-automation";
6868
// below run too — they exercise the extractor in isolation against a
6969
// synthesized VFR fixture.
7070
const HAS_FFMPEG = spawnSync("ffmpeg", ["-version"]).status === 0;
71+
const HAS_ZSCALE =
72+
HAS_FFMPEG &&
73+
/\szscale\s/.test(spawnSync("ffmpeg", ["-hide_banner", "-filters"]).stdout.toString());
7174

7275
describe("resolveVideoExtractionDuration", () => {
7376
const metadata = (
@@ -3140,7 +3143,17 @@ describe.skipIf(!HAS_FFMPEG)("extractAllVideoFrames on a VFR source", () => {
31403143
}, 60_000);
31413144
});
31423145

3143-
describe.skipIf(!HAS_FFMPEG || process.platform === "darwin")("forced-SDR HDR extraction", () => {
3146+
// Release builds print "version 8.1.3" or "n8.1.3"; master builds print "N-<number>" and are newer.
3147+
const FFPROBE_VERSION = spawnSync("ffprobe", ["-version"]).stdout?.toString() ?? "";
3148+
const FFPROBE_RELEASE = FFPROBE_VERSION.match(/version n?(\d+)\.(\d+)/)
3149+
?.slice(1)
3150+
.map(Number);
3151+
const FFPROBE_READS_CONTAINER_TRANSFER =
3152+
/version N-/.test(FFPROBE_VERSION) ||
3153+
(FFPROBE_RELEASE !== undefined &&
3154+
(FFPROBE_RELEASE[0]! > 8 || (FFPROBE_RELEASE[0] === 8 && FFPROBE_RELEASE[1]! >= 1)));
3155+
3156+
describe.skipIf(!HAS_ZSCALE)("forced-SDR HDR extraction", () => {
31443157
let fixtureDir = "";
31453158

31463159
beforeAll(() => {
@@ -3151,8 +3164,11 @@ describe.skipIf(!HAS_FFMPEG || process.platform === "darwin")("forced-SDR HDR ex
31513164
rmSync(fixtureDir, { recursive: true, force: true });
31523165
});
31533166

3154-
it("matches Studio's HLG tone map and isolates transformed cache entries", async () => {
3155-
const source = join(fixtureDir, "hlg-warm.mp4");
3167+
async function synthesizeHlgClip(
3168+
path: string,
3169+
vui = "colour_primaries=9:transfer_characteristics=18:matrix_coefficients=9",
3170+
transfer = "arib-std-b67",
3171+
): Promise<void> {
31563172
const synthesized = await runFfmpeg([
31573173
"-y",
31583174
"-hide_banner",
@@ -3163,24 +3179,205 @@ describe.skipIf(!HAS_FFMPEG || process.platform === "darwin")("forced-SDR HDR ex
31633179
"-i",
31643180
"color=c=0xe0b080:s=64x64:r=1:d=1",
31653181
"-vf",
3166-
"zscale=pin=bt709:tin=bt709:min=bt709:p=bt2020:t=arib-std-b67:m=bt2020nc:r=tv,format=yuv420p",
3182+
`zscale=pin=bt709:tin=bt709:min=bt709:p=bt2020:t=${transfer}:m=bt2020nc:r=tv,format=yuv420p`,
31673183
"-c:v",
31683184
"libx264",
31693185
"-preset",
31703186
"ultrafast",
31713187
"-color_primaries",
31723188
"bt2020",
31733189
"-color_trc",
3174-
"arib-std-b67",
3190+
transfer,
31753191
"-colorspace",
31763192
"bt2020nc",
31773193
"-bsf:v",
3178-
"h264_metadata=colour_primaries=9:transfer_characteristics=18:matrix_coefficients=9",
3179-
source,
3194+
`h264_metadata=${vui}`,
3195+
path,
31803196
]);
31813197
if (!synthesized.success) {
31823198
throw new Error(`HLG fixture synthesis failed: ${synthesized.stderr.slice(-400)}`);
31833199
}
3200+
}
3201+
3202+
it.each([
3203+
[
3204+
"HLG",
3205+
"matrix",
3206+
"arib-std-b67",
3207+
"colour_primaries=9:transfer_characteristics=18:matrix_coefficients=2",
3208+
],
3209+
[
3210+
"HLG",
3211+
"primaries",
3212+
"arib-std-b67",
3213+
"colour_primaries=2:transfer_characteristics=18:matrix_coefficients=9",
3214+
],
3215+
[
3216+
"HLG",
3217+
"matrix and primaries",
3218+
"arib-std-b67",
3219+
"colour_primaries=2:transfer_characteristics=18:matrix_coefficients=2",
3220+
],
3221+
[
3222+
"PQ",
3223+
"matrix and primaries",
3224+
"smpte2084",
3225+
"colour_primaries=2:transfer_characteristics=16:matrix_coefficients=2",
3226+
],
3227+
[
3228+
"HLG",
3229+
"transfer",
3230+
"arib-std-b67",
3231+
"colour_primaries=9:transfer_characteristics=2:matrix_coefficients=9",
3232+
],
3233+
])(
3234+
"tone-maps %s footage whose video stream has no %s tag like the fully tagged clip",
3235+
async (name, missing, transfer, vui) => {
3236+
const slug = `${name}-no-${missing}`.replace(/\W+/g, "-");
3237+
const tagged = join(fixtureDir, `${name}-fully-tagged.mp4`);
3238+
const untagged = join(fixtureDir, `${slug}.mp4`);
3239+
const vuiTransfer = transfer === "smpte2084" ? 16 : 18;
3240+
await synthesizeHlgClip(
3241+
tagged,
3242+
`colour_primaries=9:transfer_characteristics=${vuiTransfer}:matrix_coefficients=9`,
3243+
transfer,
3244+
);
3245+
await synthesizeHlgClip(untagged, vui, transfer);
3246+
const extract = (source: string, id: string) =>
3247+
extractVideoFramesRange(source, id, 0, 1, {
3248+
fps: 1,
3249+
outputDir: join(fixtureDir, `out-${id}`),
3250+
format: "png",
3251+
toneMapHdrToSdr: true,
3252+
});
3253+
3254+
if (missing === "transfer") {
3255+
// ffprobe before 8.1 reads the stream's unspecified transfer, so the clip is SDR there, as on main.
3256+
if (!FFPROBE_READS_CONTAINER_TRANSFER) {
3257+
await expect(extract(untagged, slug)).resolves.toBeDefined();
3258+
return;
3259+
}
3260+
expect((await extractVideoMetadata(untagged)).colorSpace?.colorTransfer).toBe(transfer);
3261+
}
3262+
const reference = await extract(tagged, `ref-${slug}`);
3263+
const result = await extract(untagged, slug);
3264+
3265+
expect(readFileSync(result.framePaths.get(0)!)).toEqual(
3266+
readFileSync(reference.framePaths.get(0)!),
3267+
);
3268+
},
3269+
60_000,
3270+
);
3271+
3272+
// One stream with HLG frames, then PQ frames: each is tone-mapped with its own transfer.
3273+
it("tone-maps each frame of a mixed HLG and PQ stream with its own transfer", async () => {
3274+
const segment = async (name: string, transfer: number) => {
3275+
const path = join(fixtureDir, `mixed-${name}.h264`);
3276+
const result = await runFfmpeg([
3277+
"-y",
3278+
"-v",
3279+
"error",
3280+
"-f",
3281+
"lavfi",
3282+
"-i",
3283+
"testsrc2=s=160x90:r=25:d=0.2",
3284+
"-c:v",
3285+
"libx264",
3286+
// No B-frames: genpts stamps a raw stream in decode order, so reordered frames get wrong times.
3287+
"-bf",
3288+
"0",
3289+
"-pix_fmt",
3290+
"yuv420p",
3291+
"-bsf:v",
3292+
`h264_metadata=colour_primaries=9:transfer_characteristics=${transfer}:matrix_coefficients=9`,
3293+
"-f",
3294+
"h264",
3295+
path,
3296+
]);
3297+
if (!result.success)
3298+
throw new Error(`segment synthesis failed: ${result.stderr.slice(-400)}`);
3299+
return readFileSync(path);
3300+
};
3301+
const mux = async (name: string, stream: Buffer) => {
3302+
const raw = join(fixtureDir, `${name}.h264`);
3303+
const path = join(fixtureDir, `${name}.mp4`);
3304+
writeFileSync(raw, stream);
3305+
const result = await runFfmpeg([
3306+
...["-y", "-v", "error", "-fflags", "+genpts", "-r", "25", "-f", "h264", "-i", raw],
3307+
...["-c", "copy", path],
3308+
]);
3309+
if (!result.success) throw new Error(`mux failed: ${result.stderr.slice(-400)}`);
3310+
return path;
3311+
};
3312+
const hlg = await segment("hlg", 18);
3313+
const pq = await segment("pq", 16);
3314+
const mixed = await mux("mixed-hlg-pq", Buffer.concat([hlg, pq]));
3315+
const pqOnly = await mux("mixed-pq-only", pq);
3316+
const extract = (source: string, id: string, duration: number) =>
3317+
extractVideoFramesRange(source, id, 0, duration, {
3318+
fps: 25,
3319+
outputDir: join(fixtureDir, `out-${id}`),
3320+
format: "png",
3321+
toneMapHdrToSdr: true,
3322+
});
3323+
3324+
const mixedFrames = await extract(mixed, "mixed-hlg-pq", 0.4);
3325+
const pqFrames = await extract(pqOnly, "mixed-pq-only", 0.2);
3326+
3327+
expect(readFileSync(mixedFrames.framePaths.get(5)!)).toEqual(
3328+
readFileSync(pqFrames.framePaths.get(0)!),
3329+
);
3330+
}, 60_000);
3331+
3332+
// A stream-copy trim opens on edit-list pre-roll, which ffprobe counts as packets without frames.
3333+
it("keeps per-frame tags in a mixed HLG and PQ file trimmed with a stream copy", async () => {
3334+
const run = async (args: string[]) => {
3335+
const result = await runFfmpeg(["-y", "-v", "error", ...args]);
3336+
if (!result.success) throw new Error(`fixture failed: ${result.stderr.slice(-400)}`);
3337+
};
3338+
const segment = async (name: string, transfer: number) => {
3339+
const path = join(fixtureDir, `trim-${name}.h264`);
3340+
await run([
3341+
...["-f", "lavfi", "-i", "testsrc2=s=160x90:r=25:d=2"],
3342+
...["-c:v", "libx264", "-g", "50", "-bf", "0"],
3343+
...["-pix_fmt", "yuv420p", "-bsf:v"],
3344+
`h264_metadata=colour_primaries=9:transfer_characteristics=${transfer}:matrix_coefficients=9`,
3345+
...["-f", "h264", path],
3346+
]);
3347+
return readFileSync(path);
3348+
};
3349+
const mux = async (name: string, stream: Buffer) => {
3350+
const raw = join(fixtureDir, `${name}.h264`);
3351+
const path = join(fixtureDir, `${name}.mp4`);
3352+
writeFileSync(raw, stream);
3353+
await run(["-fflags", "+genpts", "-r", "25", "-f", "h264", "-i", raw, "-c", "copy", path]);
3354+
return path;
3355+
};
3356+
const pq = await segment("pq", 16);
3357+
const mixed = await mux("trim-hlg-then-pq", Buffer.concat([await segment("hlg", 18), pq]));
3358+
const pqOnly = await mux("trim-pq-only", pq);
3359+
const trimmed = join(fixtureDir, "trim-from-hlg.mp4");
3360+
await run(["-ss", "1", "-i", mixed, "-c", "copy", trimmed]);
3361+
const extract = (source: string, id: string, start: number) =>
3362+
extractVideoFramesRange(source, id, start, 0.04, {
3363+
fps: 25,
3364+
outputDir: join(fixtureDir, `out-${id}`),
3365+
format: "png",
3366+
toneMapHdrToSdr: true,
3367+
});
3368+
3369+
// Trimmed 1.6 s is the mixed file's 2.6 s: the PQ segment's 0.6 s.
3370+
const fromTrim = await extract(trimmed, "trim-from-hlg", 1.6);
3371+
const fromPq = await extract(pqOnly, "trim-pq-only", 0.6);
3372+
3373+
const shown = readFirstFramePixel(fromTrim.framePaths.get(0)!, 40, 40);
3374+
const expected = readFirstFramePixel(fromPq.framePaths.get(0)!, 40, 40);
3375+
expect(Math.max(...shown.map((v, i) => Math.abs(v - expected[i]!)))).toBeLessThanOrEqual(2);
3376+
}, 60_000);
3377+
3378+
it("matches Studio's HLG tone map and isolates transformed cache entries", async () => {
3379+
const source = join(fixtureDir, "hlg-warm.mp4");
3380+
await synthesizeHlgClip(source);
31843381

31853382
const reference = join(fixtureDir, "studio-reference.png");
31863383
const referenceResult = await runFfmpeg([
@@ -3243,9 +3440,107 @@ describe.skipIf(!HAS_FFMPEG || process.platform === "darwin")("forced-SDR HDR ex
32433440
if (!path) throw new Error("expected extracted frame");
32443441
return readFileSync(path);
32453442
};
3246-
expect(frame(toneMapped)).toEqual(readFileSync(reference));
3443+
// Same pixels as Studio's tone map, declared as sRGB so Chrome shows them unconverted.
3444+
const rgb = (path: string): Buffer =>
3445+
spawnSync("ffmpeg", ["-v", "error", "-i", path, "-f", "rawvideo", "-pix_fmt", "rgb24", "-"])
3446+
.stdout;
3447+
const toneMappedPath = toneMapped.extracted[0]!.framePaths.get(0)!;
3448+
expect(rgb(toneMappedPath)).toEqual(rgb(reference));
3449+
expect(pngChunkTypes(toneMappedPath)).toContain("sRGB");
3450+
expect(pngChunkTypes(toneMappedPath)).not.toContain("cICP");
32473451
expect(frame(plain)).not.toEqual(frame(toneMapped));
3452+
3453+
const toneMappedJpg = await extractAllVideoFrames([video("tone-mapped-jpg")], fixtureDir, {
3454+
fps: 1,
3455+
outputDir: join(fixtureDir, "tone-mapped-jpg"),
3456+
format: "jpg",
3457+
toneMapHdrToSdr: true,
3458+
});
3459+
expect(toneMappedJpg.errors).toEqual([]);
3460+
const jpgPixels = rgb(toneMappedJpg.extracted[0]!.framePaths.get(0)!);
3461+
const referencePixels = rgb(reference);
3462+
const worst = Math.max(...[...jpgPixels].map((v, i) => Math.abs(v - referencePixels[i]!)));
3463+
expect(worst, "tone-mapped jpg against Studio's tone map").toBeLessThanOrEqual(3);
32483464
expect(frame(toneMappedAgain)).toEqual(frame(toneMapped));
3465+
3466+
// A macOS ffmpeg that has zscale uses the same tone map, not VideoToolbox.
3467+
const platform = Object.getOwnPropertyDescriptor(process, "platform")!;
3468+
Object.defineProperty(process, "platform", { value: "darwin" });
3469+
try {
3470+
const mac = await extractAllVideoFrames([video("tone-mapped-mac")], fixtureDir, {
3471+
fps: 1,
3472+
outputDir: join(fixtureDir, "tone-mapped-mac"),
3473+
format: "png",
3474+
toneMapHdrToSdr: true,
3475+
});
3476+
expect(mac.errors).toEqual([]);
3477+
expect(rgb(mac.extracted[0]!.framePaths.get(0)!)).toEqual(rgb(reference));
3478+
} finally {
3479+
Object.defineProperty(process, "platform", platform);
3480+
}
3481+
}, 60_000);
3482+
3483+
it("warns once and keeps VideoToolbox, under its own cache key, when a macOS ffmpeg has no zscale", async () => {
3484+
const source = join(fixtureDir, "hlg-no-zscale.mp4");
3485+
await synthesizeHlgClip(source);
3486+
const cacheDir = join(fixtureDir, "no-zscale-cache");
3487+
const clip = (id: string): VideoElement => ({
3488+
id,
3489+
src: source,
3490+
start: 0,
3491+
end: 1,
3492+
mediaStart: 0,
3493+
loop: false,
3494+
hasAudio: false,
3495+
});
3496+
const options = (id: string) => ({
3497+
fps: 1,
3498+
outputDir: join(fixtureDir, id),
3499+
format: "png" as const,
3500+
toneMapHdrToSdr: true,
3501+
});
3502+
const zscale = await extractAllVideoFrames(
3503+
[clip("zscale")],
3504+
fixtureDir,
3505+
options("zscale"),
3506+
undefined,
3507+
{
3508+
extractCacheDir: cacheDir,
3509+
},
3510+
);
3511+
expect(zscale.errors).toEqual([]);
3512+
3513+
const realPlatform = process.platform;
3514+
vi.resetModules();
3515+
vi.doMock("../utils/psnrFilterAvailability.js", () => ({
3516+
isFfmpegFilterAvailable: async () => false,
3517+
isPsnrFilterAvailable: async () => false,
3518+
}));
3519+
const platform = Object.getOwnPropertyDescriptor(process, "platform")!;
3520+
Object.defineProperty(process, "platform", { value: "darwin" });
3521+
const stderr = vi.spyOn(process.stderr, "write").mockImplementation(() => true);
3522+
try {
3523+
const { extractAllVideoFrames: extractOnMac } = await import("./videoFrameExtractor.js");
3524+
for (const [i, id] of ["no-zscale-1", "no-zscale-2"].entries()) {
3525+
const result = await extractOnMac([clip(id)], fixtureDir, options(id), undefined, {
3526+
extractCacheDir: cacheDir,
3527+
});
3528+
// The zscale frames above must not be served for the VideoToolbox path.
3529+
if (i === 0) expect(result.phaseBreakdown.cacheHits).toBe(0);
3530+
// Off macOS the VideoToolbox decode itself fails, which proves it was attempted.
3531+
if (realPlatform !== "darwin")
3532+
expect(JSON.stringify(result.errors)).toMatch(/videotoolbox/i);
3533+
}
3534+
const zscaleWarnings = stderr.mock.calls.filter(([message]) =>
3535+
String(message).includes("no zscale filter"),
3536+
);
3537+
expect(zscaleWarnings).toHaveLength(1);
3538+
} finally {
3539+
stderr.mockRestore();
3540+
Object.defineProperty(process, "platform", platform);
3541+
vi.doUnmock("../utils/psnrFilterAvailability.js");
3542+
vi.resetModules();
3543+
}
32493544
}, 60_000);
32503545

32513546
// A second process would read the frames back without the HDR10 light-level metadata the tone map uses.
@@ -3283,7 +3578,7 @@ describe.skipIf(!HAS_FFMPEG || process.platform === "darwin")("forced-SDR HDR ex
32833578
"-t",
32843579
"0.616666",
32853580
"-vf",
3286-
"zscale=t=linear:npl=100,tonemap=hable:desat=0,zscale=p=bt709:t=bt709:m=bt709:r=tv",
3581+
"zscale=t=linear:npl=100,tonemap=hable:desat=0,zscale=p=bt709:t=bt709:m=bt709:r=tv,setparams=color_primaries=bt709:color_trc=iec61966-2-1",
32873582
"-fps_mode",
32883583
"cfr",
32893584
"-r",

0 commit comments

Comments
 (0)