Skip to content

Commit ea742c5

Browse files
fix(engine): exports carry full colour tags with every encoder, SDR and HDR (#5742)
* fix(engine): SDR exports carry BT.709 transfer and primaries with every encoder The SDR conversion filter set the BT.709 matrix on its frames but not the transfer or primaries. Encoders that take colour from the frames rather than from -color_* options (VideoToolbox on macOS) wrote color_space=bt709 with color_transfer and color_primaries unknown. The capture and raw-input SDR filters now end in the tagged BT.709 filter ProRes already uses. * fix(engine): HDR exports tag BT.2020 with PQ or HLG on the frames too VideoToolbox HDR encodes had the same gap as SDR: given only -color_* options it wrote bt2020nc with transfer and primaries unknown, so players read the export as SDR. Every HDR branch of the streamed and disk encoders now tags the frames through one hdrFrameTagsFilter built from the encoder colour params. * test(engine): colour-tag probes pass -- before the output path
1 parent 26d7dbc commit ea742c5

6 files changed

Lines changed: 145 additions & 18 deletions

File tree

‎packages/engine/src/services/chunkEncoder.test.ts‎

Lines changed: 50 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1226,7 +1226,9 @@ describe("buildEncoderArgs color space", () => {
12261226
"out.mp4",
12271227
"nvenc",
12281228
);
1229-
expect(args[args.indexOf("-vf") + 1]).toBe("pad=ceil(iw/2)*2:ceil(ih/2)*2");
1229+
expect(args[args.indexOf("-vf") + 1]).toBe(
1230+
"setparams=color_primaries=bt2020:color_trc=smpte2084:colorspace=bt2020nc:range=tv,pad=ceil(iw/2)*2:ceil(ih/2)*2",
1231+
);
12301232
});
12311233

12321234
it("converts alpha ProRes to BT.709 without an even-dim pad", () => {
@@ -1897,6 +1899,53 @@ describe.skipIf(!HAS_FFMPEG)("buildEncoderArgs SDR colour", () => {
18971899
}
18981900
}, 30_000);
18991901

1902+
// VideoToolbox, like a libx264/libx265 given no colour options, writes only the colour its frames carry.
1903+
const streamTagsThrough = (args: string[], encoder: string[]): string => {
1904+
const dir = mkdtempSync(join(tmpdir(), "hf-frame-tags-"));
1905+
try {
1906+
const out = join(dir, "out.mp4");
1907+
const vfAt = args.indexOf("-vf");
1908+
const vf = vfAt === -1 ? [] : ["-vf", args[vfAt + 1]!];
1909+
const source = ["-f", "lavfi", "-i", "color=c=0x808080:s=64x16:d=0.1"];
1910+
const encode = spawnSync(ffmpeg, ["-y", "-v", "error", ...source, ...vf, ...encoder, out]);
1911+
expect(encode.status).toBe(0);
1912+
const entries = "stream=color_space,color_transfer,color_primaries";
1913+
const probe = ["-v", "error", "-show_entries", entries, "-of", "csv=p=0", "--", out];
1914+
return spawnSync(getFfprobeBinary(), probe).stdout.toString().trim();
1915+
} finally {
1916+
rmSync(dir, { recursive: true, force: true });
1917+
}
1918+
};
1919+
const tinyFrame = { fps: { num: 30, den: 1 }, width: 64, height: 16, preset: "ultrafast" };
1920+
1921+
it("tags BT.709 on the frames, so an encoder that reads only them tags the stream", () => {
1922+
const args = buildEncoderArgs(
1923+
{ ...tinyFrame, codec: "h264", quality: 23 },
1924+
["-i", "in"],
1925+
"o.mp4",
1926+
);
1927+
1928+
expect(streamTagsThrough(args, ["-c:v", "libx264", "-preset", "ultrafast"])).toBe(
1929+
"bt709,bt709,bt709",
1930+
);
1931+
});
1932+
1933+
it.each([
1934+
["pq", "smpte2084", null],
1935+
["hlg", "arib-std-b67", null],
1936+
["pq", "smpte2084", "videotoolbox"],
1937+
] as const)("tags HDR %s (%s) on the frames for encoder %s", (transfer, trc, gpu) => {
1938+
const args = buildEncoderArgs(
1939+
{ ...tinyFrame, codec: "h265", quality: 23, hdr: { transfer }, useGpu: gpu !== null },
1940+
["-i", "in"],
1941+
"o.mp4",
1942+
gpu,
1943+
);
1944+
const x265 = ["-c:v", "libx265", "-pix_fmt", "yuv420p10le", "-x265-params", "log-level=error"];
1945+
1946+
expect(streamTagsThrough(args, x265)).toBe(`bt2020nc,${trc},bt2020`);
1947+
});
1948+
19001949
it("keeps the rightmost 8 columns of a 1080-wide (16n + 8) mp4", () => {
19011950
const dir = mkdtempSync(join(tmpdir(), "hf-sdr-right-edge-"));
19021951
try {

‎packages/engine/src/services/chunkEncoder.ts‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ import {
2929
buildVideoToolboxRateControlArgs,
3030
mapPresetForGpuEncoder,
3131
} from "../utils/gpuEncoder.js";
32-
import { type HdrTransfer, getHdrEncoderColorParams } from "../utils/hdr.js";
32+
import { type HdrTransfer, getHdrEncoderColorParams, hdrFrameTagsFilter } from "../utils/hdr.js";
3333
import { withEvenDimensionPad } from "../utils/evenDimensions.js";
3434
import {
3535
SDR_CAPTURE_TO_BT709_FILTER,
@@ -460,8 +460,9 @@ export function buildEncoderArgs(
460460

461461
// Range conversion: Chrome's full-range capture → limited/TV range; SDR also
462462
// converts to the BT.709 matrix it is tagged with.
463+
const hdrTags = options.hdr ? hdrFrameTagsFilter(options.hdr.transfer) : undefined;
463464
const sdrFilter = options.hdr ? undefined : SDR_CAPTURE_TO_BT709_FILTER;
464-
const captureFilter = sdrFilter ?? "scale=in_range=pc:out_range=tv";
465+
const captureFilter = sdrFilter ?? `scale=in_range=pc:out_range=tv,${hdrTags}`;
465466
if (gpuEncoder === "vaapi") {
466467
// vaapi already runs `format=nv12,hwupload`; the nv12 conversion aligns
467468
// odd dimensions before upload, so only prepend the colour conversion.
@@ -474,7 +475,12 @@ export function buildEncoderArgs(
474475
// encoder. They hit the same "height not divisible by 2" abort as
475476
// libx264 on an odd-sized 4:2:0 canvas, so pad odd dimensions up to even
476477
// on the software side before the encode.
477-
const vf = withEvenDimensionPad(sdrFilter ?? "", pixelFormat, options.width, options.height);
478+
const vf = withEvenDimensionPad(
479+
sdrFilter ?? hdrTags ?? "",
480+
pixelFormat,
481+
options.width,
482+
options.height,
483+
);
478484
if (vf) args.push("-vf", vf);
479485
} else {
480486
// The scale filter handles both 8-bit and 10-bit correctly. Pad odd

‎packages/engine/src/services/streamingEncoder.test.ts‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1228,6 +1228,70 @@ describe("createFrameReorderBuffer abort (interleaved parallel drain)", () => {
12281228
});
12291229
});
12301230

1231+
describe.skipIf(spawnSync("ffmpeg", ["-version"]).status !== 0)(
1232+
"buildStreamingArgs frame tags",
1233+
() => {
1234+
// VideoToolbox, like a libx264/libx265 given no colour options, writes only the colour its frames carry.
1235+
const streamTagsThrough = (args: string[], encoder: string[]): string => {
1236+
const dir = mkdtempSync(join(tmpdir(), "se-frame-tags-"));
1237+
try {
1238+
const out = join(dir, "out.mp4");
1239+
const vfAt = args.indexOf("-vf");
1240+
const vf = vfAt === -1 ? [] : ["-vf", args[vfAt + 1]!];
1241+
const source = ["-f", "lavfi", "-i", "color=c=0x808080:s=64x16:d=0.1"];
1242+
const encode = spawnSync("ffmpeg", [
1243+
"-y",
1244+
"-v",
1245+
"error",
1246+
...source,
1247+
...vf,
1248+
...encoder,
1249+
out,
1250+
]);
1251+
expect(encode.status).toBe(0);
1252+
const entries = "stream=color_space,color_transfer,color_primaries";
1253+
const probe = ["-v", "error", "-show_entries", entries, "-of", "csv=p=0", "--", out];
1254+
return spawnSync("ffprobe", probe).stdout.toString().trim();
1255+
} finally {
1256+
rmSync(dir, { recursive: true, force: true });
1257+
}
1258+
};
1259+
const tiny = { width: 64, height: 16 };
1260+
1261+
it.each([
1262+
["captured frames", {}],
1263+
["raw RGB frames", { rawInputFormat: "rgb48le" as const }],
1264+
])("tags BT.709 on %s, so an encoder that reads only them tags the stream", (_, extra) => {
1265+
const args = buildStreamingArgs({ ...baseSdr, ...tiny, ...extra }, "o.mp4");
1266+
1267+
expect(streamTagsThrough(args, ["-c:v", "libx264", "-preset", "ultrafast"])).toBe(
1268+
"bt709,bt709,bt709",
1269+
);
1270+
});
1271+
1272+
it.each([
1273+
[baseHdrPq, "smpte2084", null],
1274+
[baseHdrHlg, "arib-std-b67", null],
1275+
[baseHdrPq, "smpte2084", "videotoolbox"],
1276+
] as const)(
1277+
"tags raw HDR frames %#, so an encoder that reads only them tags the stream",
1278+
(base, trc, gpu) => {
1279+
const args = buildStreamingArgs({ ...base, ...tiny, useGpu: gpu !== null }, "o.mp4", gpu);
1280+
const x265 = [
1281+
"-c:v",
1282+
"libx265",
1283+
"-pix_fmt",
1284+
"yuv420p10le",
1285+
"-x265-params",
1286+
"log-level=error",
1287+
];
1288+
1289+
expect(streamTagsThrough(args, x265)).toBe(`bt2020nc,${trc},bt2020`);
1290+
},
1291+
);
1292+
},
1293+
);
1294+
12311295
describe.skipIf(spawnSync("ffmpeg", ["-version"]).status !== 0)(
12321296
"buildStreamingArgs raw SDR colour",
12331297
() => {

‎packages/engine/src/services/streamingEncoder.ts‎

Lines changed: 15 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -37,11 +37,10 @@ import {
3737
isExternalFfmpegInterruption,
3838
} from "../utils/runFfmpeg.js";
3939
import { getFfmpegBinary } from "../utils/ffmpegBinaries.js";
40-
import { getHdrEncoderColorParams } from "../utils/hdr.js";
40+
import { getHdrEncoderColorParams, hdrFrameTagsFilter } from "../utils/hdr.js";
4141
import { withEvenDimensionPad } from "../utils/evenDimensions.js";
4242
import {
4343
SDR_CAPTURE_TO_BT709_FILTER,
44-
SDR_RGB_TO_BT709_FILTER,
4544
SDR_RGB_TO_TAGGED_BT709_FILTER,
4645
} from "../utils/sdrCaptureColor.js";
4746
import { DEFAULT_CONFIG, type EngineConfig } from "../config.js";
@@ -424,24 +423,28 @@ export function buildStreamingArgs(
424423
);
425424
}
426425

427-
// Raw HDR input (from WebGPU pipeline) is already PQ-encoded, so only SDR gets a
428-
// conversion: raw sRGB or Chrome's BT.601 JPEG screenshots to BT.709 limited range.
429-
const sdrFilter = options.rawInputFormat
430-
? SDR_RGB_TO_BT709_FILTER
431-
: SDR_CAPTURE_TO_BT709_FILTER;
432-
if (options.rawInputFormat && options.hdr) {
433-
// No filter needed — PQ data goes straight to encoder
434-
} else if (gpuEncoder === "vaapi") {
426+
// Raw HDR input (from WebGPU pipeline) is already PQ-encoded and is only tagged; SDR is
427+
// converted: raw sRGB or Chrome's BT.601 JPEG screenshots to tagged BT.709 limited range.
428+
const frameFilter =
429+
options.rawInputFormat && options.hdr
430+
? hdrFrameTagsFilter(options.hdr.transfer)
431+
: options.rawInputFormat
432+
? SDR_RGB_TO_TAGGED_BT709_FILTER
433+
: SDR_CAPTURE_TO_BT709_FILTER;
434+
if (gpuEncoder === "vaapi") {
435435
// vaapi already runs `format=nv12,hwupload`; the nv12 conversion aligns
436436
// odd dimensions before upload, so only prepend the colour conversion.
437437
const vfIdx = args.indexOf("-vf");
438438
if (vfIdx !== -1) {
439-
args[vfIdx + 1] = `${sdrFilter},${args[vfIdx + 1]}`;
439+
args[vfIdx + 1] = `${frameFilter},${args[vfIdx + 1]}`;
440440
}
441441
} else {
442442
// Pad odd dimensions up to even so 4:2:0 encoders (software and
443443
// nvenc/videotoolbox/qsv/amf) don't abort with "height not divisible by 2".
444-
args.push("-vf", withEvenDimensionPad(sdrFilter, pixelFormat, options.width, options.height));
444+
args.push(
445+
"-vf",
446+
withEvenDimensionPad(frameFilter, pixelFormat, options.width, options.height),
447+
);
445448
}
446449

447450
// Fixed timescale for consistent A/V timing across platforms.

‎packages/engine/src/utils/hdr.ts‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -106,6 +106,11 @@ export function getHdrEncoderColorParams(
106106
};
107107
}
108108

109+
export function hdrFrameTagsFilter(transfer: HdrTransfer): string {
110+
const { colorPrimaries, colorTrc, colorspace } = getHdrEncoderColorParams(transfer);
111+
return `setparams=color_primaries=${colorPrimaries}:color_trc=${colorTrc}:colorspace=${colorspace}:range=tv`;
112+
}
113+
109114
export interface CompositionHdrInfo {
110115
hasHdr: boolean;
111116
dominantTransfer: HdrTransfer | null;
Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
export const SDR_RGB_TO_BT709_FILTER = "scale=out_color_matrix=bt709:out_range=tv:flags=neighbor";
22

3-
export const SDR_CAPTURE_TO_BT709_FILTER = `scale=in_color_matrix=bt601:in_range=pc:flags=neighbor+accurate_rnd,format=gbrp,${SDR_RGB_TO_BT709_FILTER}`;
4-
53
export const SDR_RGB_TO_TAGGED_BT709_FILTER = `${SDR_RGB_TO_BT709_FILTER},setparams=color_primaries=bt709:color_trc=bt709:colorspace=bt709:range=tv`;
4+
5+
export const SDR_CAPTURE_TO_BT709_FILTER = `scale=in_color_matrix=bt601:in_range=pc:flags=neighbor+accurate_rnd,format=gbrp,${SDR_RGB_TO_TAGGED_BT709_FILTER}`;

0 commit comments

Comments
 (0)