Skip to content

Commit b31dc81

Browse files
committed
fix(engine): keep valid per-frame colour tags when tone-mapping HDR footage
1 parent 225d2db commit b31dc81

2 files changed

Lines changed: 134 additions & 7 deletions

File tree

‎packages/engine/src/services/videoFrameExtractor.test.ts‎

Lines changed: 102 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3177,6 +3177,108 @@ describe.skipIf(!HAS_ZSCALE)("forced-SDR HDR extraction", () => {
31773177
60_000,
31783178
);
31793179

3180+
// One stream with HLG frames, then PQ frames: each is tone-mapped with its own transfer.
3181+
it("tone-maps each frame of a mixed HLG and PQ stream with its own transfer", async () => {
3182+
const segment = async (name: string, transfer: number) => {
3183+
const path = join(fixtureDir, `mixed-${name}.h264`);
3184+
const result = await runFfmpeg([
3185+
"-y",
3186+
"-v",
3187+
"error",
3188+
"-f",
3189+
"lavfi",
3190+
"-i",
3191+
"testsrc2=s=160x90:r=25:d=0.2",
3192+
"-c:v",
3193+
"libx264",
3194+
"-pix_fmt",
3195+
"yuv420p",
3196+
"-bsf:v",
3197+
`h264_metadata=colour_primaries=9:transfer_characteristics=${transfer}:matrix_coefficients=9`,
3198+
"-f",
3199+
"h264",
3200+
path,
3201+
]);
3202+
if (!result.success)
3203+
throw new Error(`segment synthesis failed: ${result.stderr.slice(-400)}`);
3204+
return readFileSync(path);
3205+
};
3206+
const mux = async (name: string, stream: Buffer) => {
3207+
const raw = join(fixtureDir, `${name}.h264`);
3208+
const path = join(fixtureDir, `${name}.mp4`);
3209+
writeFileSync(raw, stream);
3210+
const result = await runFfmpeg([
3211+
...["-y", "-v", "error", "-fflags", "+genpts", "-r", "25", "-f", "h264", "-i", raw],
3212+
...["-c", "copy", path],
3213+
]);
3214+
if (!result.success) throw new Error(`mux failed: ${result.stderr.slice(-400)}`);
3215+
return path;
3216+
};
3217+
const hlg = await segment("hlg", 18);
3218+
const pq = await segment("pq", 16);
3219+
const mixed = await mux("mixed-hlg-pq", Buffer.concat([hlg, pq]));
3220+
const pqOnly = await mux("mixed-pq-only", pq);
3221+
const extract = (source: string, id: string, duration: number) =>
3222+
extractVideoFramesRange(source, id, 0, duration, {
3223+
fps: 25,
3224+
outputDir: join(fixtureDir, `out-${id}`),
3225+
format: "png",
3226+
toneMapHdrToSdr: true,
3227+
});
3228+
3229+
const mixedFrames = await extract(mixed, "mixed-hlg-pq", 0.4);
3230+
const pqFrames = await extract(pqOnly, "mixed-pq-only", 0.2);
3231+
3232+
expect(readFileSync(mixedFrames.framePaths.get(5)!)).toEqual(
3233+
readFileSync(pqFrames.framePaths.get(0)!),
3234+
);
3235+
}, 60_000);
3236+
3237+
// A stream-copy trim opens on edit-list pre-roll, which ffprobe counts as packets without frames.
3238+
it("keeps per-frame tags in a mixed HLG and PQ file trimmed with a stream copy", async () => {
3239+
const run = async (args: string[]) => {
3240+
const result = await runFfmpeg(["-y", "-v", "error", ...args]);
3241+
if (!result.success) throw new Error(`fixture failed: ${result.stderr.slice(-400)}`);
3242+
};
3243+
const segment = async (name: string, transfer: number) => {
3244+
const path = join(fixtureDir, `trim-${name}.h264`);
3245+
await run([
3246+
...["-f", "lavfi", "-i", "testsrc2=s=160x90:r=25:d=2", "-c:v", "libx264", "-g", "50"],
3247+
...["-pix_fmt", "yuv420p", "-bsf:v"],
3248+
`h264_metadata=colour_primaries=9:transfer_characteristics=${transfer}:matrix_coefficients=9`,
3249+
...["-f", "h264", path],
3250+
]);
3251+
return readFileSync(path);
3252+
};
3253+
const mux = async (name: string, stream: Buffer) => {
3254+
const raw = join(fixtureDir, `${name}.h264`);
3255+
const path = join(fixtureDir, `${name}.mp4`);
3256+
writeFileSync(raw, stream);
3257+
await run(["-fflags", "+genpts", "-r", "25", "-f", "h264", "-i", raw, "-c", "copy", path]);
3258+
return path;
3259+
};
3260+
const pq = await segment("pq", 16);
3261+
const mixed = await mux("trim-hlg-then-pq", Buffer.concat([await segment("hlg", 18), pq]));
3262+
const pqOnly = await mux("trim-pq-only", pq);
3263+
const trimmed = join(fixtureDir, "trim-from-hlg.mp4");
3264+
await run(["-ss", "1", "-i", mixed, "-c", "copy", trimmed]);
3265+
const extract = (source: string, id: string, start: number) =>
3266+
extractVideoFramesRange(source, id, start, 0.04, {
3267+
fps: 25,
3268+
outputDir: join(fixtureDir, `out-${id}`),
3269+
format: "png",
3270+
toneMapHdrToSdr: true,
3271+
});
3272+
3273+
// Trimmed 1.6 s is the mixed file's 2.6 s: the PQ segment's 0.6 s.
3274+
const fromTrim = await extract(trimmed, "trim-from-hlg", 1.6);
3275+
const fromPq = await extract(pqOnly, "trim-pq-only", 0.6);
3276+
3277+
const shown = readFirstFramePixel(fromTrim.framePaths.get(0)!, 40, 40);
3278+
const expected = readFirstFramePixel(fromPq.framePaths.get(0)!, 40, 40);
3279+
expect(Math.max(...shown.map((v, i) => Math.abs(v - expected[i]!)))).toBeLessThanOrEqual(2);
3280+
}, 60_000);
3281+
31803282
it("matches Studio's HLG tone map and isolates transformed cache entries", async () => {
31813283
const source = join(fixtureDir, "hlg-warm.mp4");
31823284
await synthesizeHlgClip(source);

‎packages/engine/src/services/videoFrameExtractor.ts‎

Lines changed: 32 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@ import { isKnownInactiveTimelineWindow } from "./mediaTimelineWindow.js";
3434
import {
3535
extractFinalVideoFrameTimestamp,
3636
extractMediaMetadata,
37+
type VideoColorSpace,
3738
type VideoMetadata,
3839
} from "../utils/ffprobe.js";
3940
import {
@@ -775,14 +776,37 @@ export function parseImageElements(html: string): ImageElement[] {
775776
return images;
776777
}
777778

778-
/** zscale reads per-frame tags that ffprobe may not see; untagged HDR footage is BT.2020. */
779-
function hdrFrameColourTags(metadata: VideoMetadata): string {
779+
/** zscale reads each frame's own tags; set only those the first frame lacks, from ffprobe or BT.2020. */
780+
function hdrFrameColourTags(metadata: VideoMetadata, frame: Partial<VideoColorSpace>): string[] {
780781
const known = (value: string | undefined) =>
781782
value && value !== "unknown" && value !== "reserved" ? value : undefined;
782-
const matrix = known(metadata.colorSpace?.colorSpace) ?? "bt2020nc";
783-
const primaries = known(metadata.colorSpace?.colorPrimaries) ?? "bt2020";
784-
const transfer = metadata.colorSpace?.colorTransfer;
785-
return `setparams=colorspace=${matrix}:color_primaries=${primaries}:color_trc=${transfer}`;
783+
const probed = metadata.colorSpace;
784+
const fill = (key: keyof VideoColorSpace, fallback?: string) =>
785+
known(frame[key]) ? undefined : (known(probed?.[key]) ?? fallback);
786+
const tags = [
787+
["colorspace", fill("colorSpace", "bt2020nc")],
788+
["color_primaries", fill("colorPrimaries", "bt2020")],
789+
["color_trc", fill("colorTransfer")],
790+
].filter(([, value]) => value);
791+
return tags.length
792+
? [`setparams=${tags.map(([key, value]) => `${key}=${value}`).join(":")}`]
793+
: [];
794+
}
795+
796+
/** The first shown frame's tags, read by ffmpeg, which decodes past the edit-list pre-roll ffprobe counts. */
797+
async function readFirstFrameColour(
798+
videoPath: string,
799+
signal?: AbortSignal,
800+
): Promise<Partial<VideoColorSpace>> {
801+
const result = await runFfmpeg(
802+
[
803+
...["-hide_banner", "-nostats", "-i", videoPath, "-map", "0:v:0", "-frames:v", "1"],
804+
...["-vf", "showinfo", "-f", "null", "-"],
805+
],
806+
{ signal },
807+
);
808+
const tags = / color_space:(\S+) color_primaries:(\S+) color_trc:(\S+)/.exec(result.stderr);
809+
return tags ? { colorSpace: tags[1], colorPrimaries: tags[2], colorTransfer: tags[3] } : {};
786810
}
787811

788812
/** Chrome plays untagged VP9 and AV1 as BT.601, H.264 and VP8 as BT.709 from 720 coded lines (assumed for the rest). */
@@ -954,7 +978,8 @@ export async function extractVideoFramesRange(
954978
vfFilters.push(SDR_TO_HDR_COLORSPACE_FILTER);
955979
}
956980
if (toneMappedToSdr) {
957-
vfFilters.push(hdrFrameColourTags(metadata), HDR_TO_SDR_TONEMAP_FILTER);
981+
const frameColour = await readFirstFrameColour(videoPath, signal);
982+
vfFilters.push(...hdrFrameColourTags(metadata, frameColour), HDR_TO_SDR_TONEMAP_FILTER);
958983
}
959984
if (toneMappedToSdr || (!isHdr && !options.sdrToHdrTransfer)) {
960985
vfFilters.push(

0 commit comments

Comments
 (0)