Skip to content

Commit 5ee9e92

Browse files
fix(core): hlg footage tone-maps to sdr at the bt.2408 reference white (#7234)
* fix(core): hlg footage tone-maps to sdr at the bt.2408 reference white * fix(core): an HLG clip whose stream leaves its transfer unset keeps HLG's white An SPS with an unspecified transfer takes the stream's tag, as its frames do; a failed keyframe read is not cached; the grey test needs ffmpeg 8.
1 parent 45705fa commit 5ee9e92

12 files changed

Lines changed: 270 additions & 47 deletions

File tree

‎packages/cli/src/commands/measure/probe.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ import {
99
runFfmpeg,
1010
type VideoInfo,
1111
} from "../../measure/media.js";
12+
import { toneMapReferenceNits } from "@hyperframes/core";
1213
import { execFileSync } from "node:child_process";
1314
import { dirname, resolve } from "node:path";
1415
import { makeOutputFolder } from "../../utils/outputFolder.js";
@@ -54,7 +55,7 @@ function normalize(info: VideoInfo, out: string): { path: string; fps: string }
5455
"install an ffmpeg built with libzimg, e.g. brew install ffmpeg",
5556
);
5657
filters.push(
57-
"zscale=t=linear:npl=100,format=gbrpf32le,zscale=p=bt709,tonemap=hable:desat=0,zscale=t=bt709:m=bt709:r=tv",
58+
`zscale=t=linear:npl=${toneMapReferenceNits(info.transfer ? [info.transfer] : [])},format=gbrpf32le,zscale=p=bt709,tonemap=hable:desat=0,zscale=t=bt709:m=bt709:r=tv`,
5859
);
5960
}
6061
makeOutputFolder(dirname(out));

‎packages/cli/src/commands/snapshot.ts‎

Lines changed: 19 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -71,10 +71,14 @@ import {
7171
import {
7272
firstFrameColourArgs,
7373
hdrToSdrToneMapFilter,
74+
keyframeTransferArgs,
75+
parseKeyframeTransfers,
76+
type ToneMapFrameColour,
7477
isClipVisibleAt,
7578
isInClipWindow,
7679
MEDIA_RENDER_ID_ATTR,
7780
parseFirstFrameColour,
81+
readsKeyframeTransfers,
7882
readElementRateSpec,
7983
sourceTimeAt,
8084
timeAtSourceTime,
@@ -620,20 +624,26 @@ async function videoColour(ffmpegPath: string, videoPath: string) {
620624
metadata && isHdrColorSpace(metadata.colorSpace)
621625
? hdrToSdrToneMapFilter(
622626
metadata.colorSpace ?? {},
623-
parseFirstFrameColour(
624-
(
625-
await runFfmpegOnce(
626-
ffmpegPath,
627-
firstFrameColourArgs(videoPath),
628-
FFMPEG_EXTRACT_TIMEOUT_MS,
629-
)
630-
).stderr,
631-
),
627+
await frameColour(ffmpegPath, videoPath, metadata.videoCodec),
632628
)
633629
: null;
634630
return { metadata, toneMap };
635631
}
636632

633+
async function frameColour(
634+
ffmpegPath: string,
635+
videoPath: string,
636+
codec: string,
637+
): Promise<ToneMapFrameColour> {
638+
const read = (args: string[]) => runFfmpegOnce(ffmpegPath, args, FFMPEG_EXTRACT_TIMEOUT_MS);
639+
const probe = await read(firstFrameColourArgs(videoPath));
640+
const first = parseFirstFrameColour(probe.stderr);
641+
if (probe.code !== 0 || !readsKeyframeTransfers(first)) return first;
642+
const keyframes = await read(keyframeTransferArgs(videoPath, codec));
643+
const keyframeTransfers = keyframes.code === 0 ? parseKeyframeTransfers(keyframes.stderr) : [];
644+
return { ...first, keyframeTransfers };
645+
}
646+
637647
function frameFilters(
638648
select: number | undefined,
639649
displayedPixels: number,

‎packages/cli/src/measure/media.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,7 @@ export interface VideoInfo {
7272
/** The container's frame rate and the stream's average differ: frames are not evenly spaced. */
7373
vfr: boolean;
7474
hdr: boolean;
75+
transfer?: string;
7576
codec: string;
7677
}
7778

@@ -160,6 +161,7 @@ export function probeVideo(path: string): VideoInfo {
160161
avgFps: avg || fps,
161162
vfr: avg > 0 && Math.abs(avg - fps) / fps > 0.01,
162163
hdr: /smpte2084|arib-std-b67/.test(video.color_transfer ?? ""),
164+
transfer: video.color_transfer,
163165
codec: video.codec_name ?? "",
164166
};
165167
}
Lines changed: 87 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,19 @@
1+
import { spawnSync } from "node:child_process";
12
import { describe, expect, it } from "vitest";
2-
import { hdrToSdrToneMapFilter } from "./hdrToneMap";
3+
import {
4+
hdrToSdrToneMapFilter,
5+
keyframeTransferArgs,
6+
parseKeyframeTransfers,
7+
toneMapReferenceNits,
8+
} from "./hdrToneMap";
39

410
const PQ = { colorSpace: "bt2020nc", colorPrimaries: "bt2020", colorTransfer: "smpte2084" };
5-
const TONE_MAP =
6-
"zscale=t=linear:npl=100,tonemap=hable:desat=0,zscale=p=bt709:t=bt709:m=bt709:r=tv";
11+
const toneMap = (npl: number) =>
12+
`zscale=t=linear:npl=${npl},tonemap=hable:desat=0,zscale=p=bt709:t=bt709:m=bt709:r=tv`;
713

814
describe("hdrToSdrToneMapFilter", () => {
915
it("leaves every frame its own tags when the first frame carries them all", () => {
10-
expect(hdrToSdrToneMapFilter(PQ, PQ)).toBe(TONE_MAP);
16+
expect(hdrToSdrToneMapFilter(PQ, PQ)).toBe(toneMap(100));
1117
});
1218

1319
it("resizes in the first zscale, before the tone map, when it is given a size", () => {
@@ -18,10 +24,10 @@ describe("hdrToSdrToneMapFilter", () => {
1824

1925
it("sets the probed tags the first frame lacks", () => {
2026
expect(hdrToSdrToneMapFilter(PQ, {})).toBe(
21-
`setparams=colorspace=bt2020nc:color_primaries=bt2020:color_trc=smpte2084,${TONE_MAP}`,
27+
`setparams=colorspace=bt2020nc:color_primaries=bt2020:color_trc=smpte2084,${toneMap(100)}`,
2228
);
2329
expect(hdrToSdrToneMapFilter(PQ, { colorTransfer: "arib-std-b67" })).toBe(
24-
`setparams=colorspace=bt2020nc:color_primaries=bt2020,${TONE_MAP}`,
30+
`setparams=colorspace=bt2020nc:color_primaries=bt2020,${toneMap(100)}`,
2531
);
2632
});
2733

@@ -31,9 +37,82 @@ describe("hdrToSdrToneMapFilter", () => {
3137
{ colorSpace: "unknown", colorTransfer: "arib-std-b67" },
3238
{ colorSpace: "unknown", colorPrimaries: "unknown", colorTransfer: "arib-std-b67" },
3339
),
34-
).toBe(`setparams=colorspace=bt2020nc:color_primaries=bt2020,${TONE_MAP}`);
40+
).toBe(`setparams=colorspace=bt2020nc:color_primaries=bt2020,${toneMap(100)}`);
3541
expect(hdrToSdrToneMapFilter({ colorPrimaries: "reserved" }, {})).toBe(
36-
`setparams=colorspace=bt2020nc:color_primaries=bt2020,${TONE_MAP}`,
42+
`setparams=colorspace=bt2020nc:color_primaries=bt2020,${toneMap(100)}`,
3743
);
3844
});
45+
46+
it("maps a stream at BT.2408's 203-nit HLG reference white only when every keyframe is HLG", () => {
47+
const hlg = { ...PQ, colorTransfer: "arib-std-b67" };
48+
const keyframes = (...keyframeTransfers: string[]) => ({ ...hlg, keyframeTransfers });
49+
expect(hdrToSdrToneMapFilter(hlg, keyframes("arib-std-b67", "arib-std-b67"))).toBe(
50+
toneMap(203),
51+
);
52+
expect(hdrToSdrToneMapFilter(hlg, keyframes("arib-std-b67", "smpte2084"))).toBe(toneMap(100));
53+
expect(hdrToSdrToneMapFilter(hlg, keyframes("arib-std-b67", "bt709"))).toBe(toneMap(100));
54+
expect(hdrToSdrToneMapFilter(hlg, keyframes())).toBe(toneMap(100));
55+
const untagged = (keyframeTransfers: string[]) => ({
56+
colorTransfer: "unknown",
57+
keyframeTransfers,
58+
});
59+
expect(hdrToSdrToneMapFilter(hlg, untagged(["unspecified", "arib-std-b67"]))).toContain(
60+
"npl=203",
61+
);
62+
expect(hdrToSdrToneMapFilter(PQ, untagged(["unspecified"]))).toContain("npl=100");
63+
expect(hdrToSdrToneMapFilter(PQ, { ...PQ, keyframeTransfers: ["smpte2084"] })).toBe(
64+
toneMap(100),
65+
);
66+
expect(toneMapReferenceNits(undefined)).toBe(100);
67+
});
68+
69+
it("names each SPS transfer trace_headers prints, and reads only keyframes' headers", () => {
70+
const line = (code: number) =>
71+
`[trace_headers @ 0x1] 287 transfer_characteristics 00010010 = ${code}`;
72+
expect(parseKeyframeTransfers([18, 16, 1, 2, 4].map(line).join("\n"))).toEqual([
73+
"arib-std-b67",
74+
"smpte2084",
75+
"bt709",
76+
"unspecified",
77+
"code 4",
78+
]);
79+
expect(keyframeTransferArgs("clip.mov", "hevc")).toContain(
80+
"filter_units=pass_types=33,trace_headers",
81+
);
82+
expect(keyframeTransferArgs("clip.mov")).toEqual(expect.arrayContaining(["-discard", "nokey"]));
83+
});
84+
});
85+
86+
const FFMPEG_MAJOR = Number(
87+
/ffmpeg version n?(\d+)/.exec(spawnSync("ffmpeg", ["-version"]).stdout?.toString() ?? "")?.[1] ??
88+
0,
89+
);
90+
const HAS_ZSCALE = /\szscale\s/.test(
91+
spawnSync("ffmpeg", ["-hide_banner", "-filters"]).stdout?.toString() ?? "",
92+
);
93+
94+
/** The 8-bit full-range grey the chain makes of one 10-bit HLG code value. */
95+
function toneMappedGrey(code: number): number {
96+
const hlg = { colorSpace: "bt2020nc", colorPrimaries: "bt2020", colorTransfer: "arib-std-b67" };
97+
const tags =
98+
"setparams=color_primaries=bt2020:color_trc=arib-std-b67:colorspace=bt2020nc:range=tv";
99+
const run = spawnSync("ffmpeg", [
100+
...["-v", "error", "-f", "lavfi", "-i", "color=c=black:s=16x16:d=0.04"],
101+
...[
102+
"-vf",
103+
`format=yuv420p10le,geq=lum=${code}:cb=512:cr=512,${tags},${hdrToSdrToneMapFilter(hlg, { ...hlg, keyframeTransfers: ["arib-std-b67"] })}`,
104+
],
105+
...["-frames:v", "1", "-pix_fmt", "gray", "-f", "rawvideo", "pipe:1"],
106+
]);
107+
const luma = run.stdout[0] ?? Number.NaN;
108+
return Math.round(((luma - 16) * 255) / 219);
109+
}
110+
111+
// ffmpeg before 8 bundles an older zimg that maps HLG white near 174, outside the tolerance.
112+
describe.skipIf(!HAS_ZSCALE || FFMPEG_MAJOR < 8)("hdrToSdrToneMapFilter on HLG grey", () => {
113+
// Apple's own SDR conversion (avconvert) of these 10-bit steps gives 105 and 161; at 100 nits ours gave 142 and 216.
114+
it("puts mid grey and reference white where Apple's SDR conversion does", () => {
115+
expect(Math.abs(toneMappedGrey(551) - 105)).toBeLessThanOrEqual(8);
116+
expect(Math.abs(toneMappedGrey(745) - 161)).toBeLessThanOrEqual(8);
117+
});
39118
});

‎packages/core/src/hdrToneMap.ts‎

Lines changed: 47 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,22 @@ export interface ToneMapSourceColour {
44
colorTransfer?: string;
55
}
66

7+
export interface ToneMapFrameColour extends ToneMapSourceColour {
8+
keyframeTransfers?: readonly string[];
9+
}
10+
11+
const BT2408_HLG_REFERENCE_WHITE_NITS = 203;
12+
const SDR_NOMINAL_PEAK_NITS = 100;
13+
14+
/** One zscale serves a stream: only an all-HLG one takes HLG's white, so a mixed one's PQ stays as is. */
15+
export const toneMapReferenceNits = (keyframeTransfers: readonly string[] | undefined): number =>
16+
keyframeTransfers?.length && keyframeTransfers.every((transfer) => transfer === "arib-std-b67")
17+
? BT2408_HLG_REFERENCE_WHITE_NITS
18+
: SDR_NOMINAL_PEAK_NITS;
19+
20+
export const readsKeyframeTransfers = ({ colorTransfer }: ToneMapSourceColour): boolean =>
21+
colorTransfer === "arib-std-b67" || !known(colorTransfer);
22+
723
function known(value: string | undefined): string | undefined {
824
return value && value !== "unknown" && value !== "reserved" ? value : undefined;
925
}
@@ -14,7 +30,7 @@ function known(value: string | undefined): string | undefined {
1430
*/
1531
export function hdrToSdrToneMapFilter(
1632
probed: ToneMapSourceColour,
17-
firstFrame: ToneMapSourceColour,
33+
firstFrame: ToneMapFrameColour,
1834
size?: { width: string; height: string },
1935
): string {
2036
const fill = (key: keyof ToneMapSourceColour, fallback?: string) =>
@@ -26,7 +42,13 @@ export function hdrToSdrToneMapFilter(
2642
].filter(([, value]) => value);
2743
const setTags = tags.length ? `setparams=${tags.map((tag) => tag.join("=")).join(":")},` : "";
2844
const resizeBeforeToneMap = size ? `w=${size.width}:h=${size.height}:` : "";
29-
return `${setTags}zscale=${resizeBeforeToneMap}t=linear:npl=100,tonemap=hable:desat=0,zscale=p=bt709:t=bt709:m=bt709:r=tv`;
45+
const stream = known(firstFrame.colorTransfer) ?? known(probed.colorTransfer);
46+
const npl = toneMapReferenceNits(
47+
firstFrame.keyframeTransfers?.map((transfer) =>
48+
transfer === "unspecified" ? (stream ?? transfer) : transfer,
49+
),
50+
);
51+
return `${setTags}zscale=${resizeBeforeToneMap}t=linear:npl=${npl},tonemap=hable:desat=0,zscale=p=bt709:t=bt709:m=bt709:r=tv`;
3052
}
3153

3254
/** ffmpeg args printing the first shown frame's tags; ffprobe's packet-bounded reads miss edit-list pre-roll. */
@@ -37,6 +59,29 @@ export function firstFrameColourArgs(videoPath: string): string[] {
3759
];
3860
}
3961

62+
const SPS_UNIT_TYPES: Record<string, string> = { h264: "7", hevc: "33" };
63+
const TRANSFER_NAMES: Record<string, string> = {
64+
"1": "bt709",
65+
"2": "unspecified",
66+
"16": "smpte2084",
67+
"18": "arib-std-b67",
68+
};
69+
70+
/** ffmpeg args tracing the transfer of each SPS, in the header and in keyframes, without decoding a picture. */
71+
export function keyframeTransferArgs(videoPath: string, codec?: string): string[] {
72+
const sps = codec ? SPS_UNIT_TYPES[codec] : undefined;
73+
const trace = sps ? `filter_units=pass_types=${sps},trace_headers` : "trace_headers";
74+
return [
75+
...["-hide_banner", "-nostats", "-discard", "nokey", "-i", videoPath, "-map", "0:v:0"],
76+
...["-c", "copy", "-bsf:v", trace, "-f", "null", "-"],
77+
];
78+
}
79+
80+
export const parseKeyframeTransfers = (stderr: string): string[] =>
81+
[...stderr.matchAll(/transfer_characteristics +[01]+ = (\d+)/g)].map(
82+
([, code = ""]) => TRANSFER_NAMES[code] ?? `code ${code}`,
83+
);
84+
4085
export function parseFirstFrameColour(stderr: string): ToneMapSourceColour {
4186
const tags = / color_space:(\S+) color_primaries:(\S+) color_trc:(\S+)/.exec(stderr);
4287
return tags ? { colorSpace: tags[1], colorPrimaries: tags[2], colorTransfer: tags[3] } : {};

‎packages/core/src/index.ts‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -280,7 +280,12 @@ export { bakeHfColorGradingCube, type BakeHfColorGradingCubeOptions } from "./co
280280
export {
281281
firstFrameColourArgs,
282282
hdrToSdrToneMapFilter,
283+
keyframeTransferArgs,
284+
parseKeyframeTransfers,
285+
readsKeyframeTransfers,
286+
toneMapReferenceNits,
283287
parseFirstFrameColour,
288+
type ToneMapFrameColour,
284289
type ToneMapSourceColour,
285290
} from "./hdrToneMap";
286291
export {

‎packages/engine/src/services/videoFrameExtractor.concurrency.test.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -146,7 +146,7 @@ describe.skipIf(process.platform === "win32")("extractAllVideoFrames ffmpeg conc
146146
delete process.env.HF_TEST_SHIM_DIR;
147147
}
148148
const alive = readFileSync(join(run, "alive"), "utf8").trim().split(/\s+/).map(Number);
149-
expect(alive).toHaveLength(8 + 2);
149+
expect(alive).toHaveLength(8 + 2 * 2);
150150
expect(Math.max(...alive)).toBe(1);
151151
},
152152
60_000,

0 commit comments

Comments
 (0)