Skip to content

Commit c8e105d

Browse files
authored
fix(engine): encode and tag ProRes mov output as BT.709 (#5382)
* fix(engine): encode and tag ProRes mov output as BT.709 * test(engine): read the ProRes colour tags from the first frame * test(engine): end the ProRes tag probe's options with --
1 parent b100ff1 commit c8e105d

9 files changed

Lines changed: 136 additions & 28 deletions

File tree

‎packages/cli/src/background-removal/pipeline.test.ts‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -79,8 +79,15 @@ describe("background-removal/pipeline — buildEncoderArgs", () => {
7979
},
8080
);
8181

82-
it.each(["mov", "png"] as const)("keeps the %s output color path", (format) => {
83-
const args = buildEncoderArgs(format, 64, 64, 30, `/tmp/cutout.${format}`);
82+
it("converts raw RGBA to the tagged BT.709 matrix for the ProRes mov", () => {
83+
const args = buildEncoderArgs("mov", 64, 64, 30, "/tmp/cutout.mov");
84+
const filter = args[args.indexOf("-vf") + 1];
85+
expect(filter).toContain("out_color_matrix=bt709");
86+
expect(filter).toContain("setparams=color_primaries=bt709:color_trc=bt709:colorspace=bt709");
87+
});
88+
89+
it("keeps the png output color path", () => {
90+
const args = buildEncoderArgs("png", 64, 64, 30, "/tmp/cutout.png");
8491
expect(args).not.toContain("-vf");
8592
});
8693

‎packages/cli/src/background-removal/pipeline.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ import { type Device, type ModelId } from "./manager.js";
2323
import {
2424
DEFAULT_VP9_CPU_USED,
2525
SDR_RGB_TO_BT709_FILTER,
26+
SDR_RGB_TO_TAGGED_BT709_FILTER,
2627
renderProvenanceArgs,
2728
} from "@hyperframes/engine";
2829

@@ -192,6 +193,8 @@ export function buildEncoderArgs(
192193
if (format === "mov") {
193194
return [
194195
...base,
196+
"-vf",
197+
SDR_RGB_TO_TAGGED_BT709_FILTER,
195198
"-c:v",
196199
"prores_ks",
197200
"-profile:v",

‎packages/cli/src/background-removal/pipeline.windowsHide.test.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,8 @@ vi.mock("./inference.js", () => ({
3636
vi.mock("@hyperframes/engine", () => ({
3737
DEFAULT_VP9_CPU_USED: 4,
3838
SDR_RGB_TO_BT709_FILTER: "scale=out_color_matrix=bt709:out_range=tv:flags=neighbor",
39+
SDR_RGB_TO_TAGGED_BT709_FILTER:
40+
"scale=out_color_matrix=bt709:out_range=tv:flags=neighbor,setparams=color_primaries=bt709:color_trc=bt709:colorspace=bt709:range=tv",
3941
renderProvenanceArgs: () => [],
4042
extractMediaMetadata: async () => ({ width: 1, height: 1, fps: 1, durationSeconds: 1 }),
4143
}));

‎packages/engine/src/index.ts‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,10 @@ export {
6161
type EngineConfig,
6262
type ExtractCacheDirResolution,
6363
} from "./config.js";
64-
export { SDR_RGB_TO_BT709_FILTER } from "./utils/sdrCaptureColor.js";
64+
export {
65+
SDR_RGB_TO_BT709_FILTER,
66+
SDR_RGB_TO_TAGGED_BT709_FILTER,
67+
} from "./utils/sdrCaptureColor.js";
6568
export {
6669
DEFAULT_VP9_CPU_USED,
6770
MAX_VP9_CPU_USED,

‎packages/engine/src/services/chunkEncoder.test.ts‎

Lines changed: 92 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -14,8 +14,11 @@ import {
1414
resolveLockedGopSize,
1515
} from "./chunkEncoder.js";
1616
import { renderProvenanceArgs } from "../utils/renderProvenance.js";
17-
import { SDR_CAPTURE_TO_BT709_FILTER } from "../utils/sdrCaptureColor.js";
18-
import { getFfmpegBinary } from "../utils/ffmpegBinaries.js";
17+
import {
18+
SDR_CAPTURE_TO_BT709_FILTER,
19+
SDR_RGB_TO_TAGGED_BT709_FILTER,
20+
} from "../utils/sdrCaptureColor.js";
21+
import { getFfmpegBinary, getFfprobeBinary } from "../utils/ffmpegBinaries.js";
1922

2023
const HAS_FFMPEG = spawnSync(getFfmpegBinary(), ["-version"]).status === 0;
2124

@@ -1226,7 +1229,7 @@ describe("buildEncoderArgs color space", () => {
12261229
expect(args[args.indexOf("-vf") + 1]).toBe("pad=ceil(iw/2)*2:ceil(ih/2)*2");
12271230
});
12281231

1229-
it("leaves alpha ProRes untouched (no even-dim pad)", () => {
1232+
it("converts alpha ProRes to BT.709 without an even-dim pad", () => {
12301233
const args = buildEncoderArgs(
12311234
{
12321235
...baseOptions,
@@ -1238,7 +1241,7 @@ describe("buildEncoderArgs color space", () => {
12381241
inputArgs,
12391242
"out.mov",
12401243
);
1241-
expect(args.indexOf("-vf")).toBe(-1);
1244+
expect(args[args.indexOf("-vf") + 1]).toBe(SDR_RGB_TO_TAGGED_BT709_FILTER);
12421245
expect(args.join(" ")).not.toContain("pad=");
12431246
});
12441247

@@ -1820,26 +1823,27 @@ describe("buildConcatArgs", () => {
18201823
});
18211824

18221825
describe.skipIf(!HAS_FFMPEG)("buildEncoderArgs SDR colour", () => {
1826+
const ffmpeg = getFfmpegBinary();
1827+
const pixelAt = (file: string, decode: string, x: number, format = "rgb24"): number[] => [
1828+
...spawnSync(ffmpeg, [
1829+
"-v",
1830+
"error",
1831+
"-i",
1832+
file,
1833+
"-vf",
1834+
`${decode}format=${format},crop=1:1:${x}:8`,
1835+
"-frames:v",
1836+
"1",
1837+
"-f",
1838+
"rawvideo",
1839+
"-",
1840+
]).stdout,
1841+
];
1842+
18231843
// Chrome captures are BT.601 JPEGs. A direct YUV-to-YUV scale keeps that matrix under the BT.709
18241844
// tag (ffmpeg 7 and older) or tints greys (8 and newer), so the encode goes through RGB.
18251845
it("delivers Chrome's JPEG colours in the BT.709 the mp4 is tagged with", () => {
1826-
const ffmpeg = getFfmpegBinary();
18271846
const dir = mkdtempSync(join(tmpdir(), "hf-sdr-colour-"));
1828-
const rgbAt = (file: string, decode: string, x: number): number[] => [
1829-
...spawnSync(ffmpeg, [
1830-
"-v",
1831-
"error",
1832-
"-i",
1833-
file,
1834-
"-vf",
1835-
`${decode}format=rgb24,crop=1:1:${x}:8`,
1836-
"-frames:v",
1837-
"1",
1838-
"-f",
1839-
"rawvideo",
1840-
"-",
1841-
]).stdout,
1842-
];
18431847
try {
18441848
for (const color of ["0xC83C28", "0xFE0000", "0x101010", "0x2050E0"]) {
18451849
const jpg = join(dir, "frame.jpg");
@@ -1879,8 +1883,8 @@ describe.skipIf(!HAS_FFMPEG)("buildEncoderArgs SDR colour", () => {
18791883
[8, 2],
18801884
[32, 2],
18811885
] as const) {
1882-
const captured = rgbAt(jpg, "", x);
1883-
const delivered = rgbAt(out, "scale=in_color_matrix=bt709:in_range=tv,", x);
1886+
const captured = pixelAt(jpg, "", x);
1887+
const delivered = pixelAt(out, "scale=in_color_matrix=bt709:in_range=tv,", x);
18841888
const worst = Math.max(...delivered.map((v, i) => Math.abs(v - captured[i]!)));
18851889
expect(
18861890
worst,
@@ -1892,4 +1896,70 @@ describe.skipIf(!HAS_FFMPEG)("buildEncoderArgs SDR colour", () => {
18921896
rmSync(dir, { recursive: true, force: true });
18931897
}
18941898
}, 30_000);
1899+
1900+
// Untagged HD reads as BT.709 in QuickTime, so ProRes has to be BT.709 too. MOV captures are
1901+
// RGBA PNGs, and the conversion must leave their alpha plane alone. The tags come from
1902+
// setparams because prores_ks ignores -color_* (ffmpeg 5.1 writes none, 8.1 only the matrix).
1903+
it("delivers PNG colours and alpha in the BT.709 the ProRes mov is tagged with", () => {
1904+
const dir = mkdtempSync(join(tmpdir(), "hf-prores-colour-"));
1905+
const png = join(dir, "frame.png");
1906+
const out = join(dir, "out.mov");
1907+
try {
1908+
for (const color of ["0xC83C28", "0x00FF00", "0x101010"]) {
1909+
const synth = spawnSync(ffmpeg, [
1910+
"-v",
1911+
"error",
1912+
"-y",
1913+
"-f",
1914+
"lavfi",
1915+
"-i",
1916+
`color=c=${color}@0.5:s=64x16,format=rgba`,
1917+
"-frames:v",
1918+
"1",
1919+
png,
1920+
]);
1921+
expect(synth.status).toBe(0);
1922+
const args = buildEncoderArgs(
1923+
{
1924+
fps: { num: 30, den: 1 },
1925+
width: 64,
1926+
height: 16,
1927+
codec: "prores",
1928+
preset: "4444",
1929+
quality: 23,
1930+
pixelFormat: "yuva444p10le",
1931+
},
1932+
["-i", png],
1933+
out,
1934+
);
1935+
expect(spawnSync(ffmpeg, args).status).toBe(0);
1936+
1937+
const captured = pixelAt(png, "", 8, "rgba");
1938+
const delivered = pixelAt(out, "scale=in_color_matrix=bt709:in_range=tv,", 8, "rgba");
1939+
const worst = Math.max(...delivered.map((v, i) => Math.abs(v - captured[i]!)));
1940+
expect(worst, `${color}: capture ${captured} delivered ${delivered}`).toBeLessThanOrEqual(
1941+
2,
1942+
);
1943+
}
1944+
// Read the first frame, not the stream: a prores_ks mov's stream-level range depends on
1945+
// the ffprobe version (7.1 and 8.0 report unknown), while every version tags the frames.
1946+
const tags = spawnSync(getFfprobeBinary(), [
1947+
"-v",
1948+
"error",
1949+
"-select_streams",
1950+
"v:0",
1951+
"-read_intervals",
1952+
"%+#1",
1953+
"-show_entries",
1954+
"frame=color_range,color_space,color_primaries,color_transfer",
1955+
"-of",
1956+
"csv=p=0",
1957+
"--",
1958+
out,
1959+
]);
1960+
expect(tags.stdout.toString().trim()).toBe("tv,bt709,bt709,bt709");
1961+
} finally {
1962+
rmSync(dir, { recursive: true, force: true });
1963+
}
1964+
}, 30_000);
18951965
});

‎packages/engine/src/services/chunkEncoder.ts‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,10 @@ import {
3131
} from "../utils/gpuEncoder.js";
3232
import { type HdrTransfer, getHdrEncoderColorParams } from "../utils/hdr.js";
3333
import { withEvenDimensionPad } from "../utils/evenDimensions.js";
34-
import { SDR_CAPTURE_TO_BT709_FILTER } from "../utils/sdrCaptureColor.js";
34+
import {
35+
SDR_CAPTURE_TO_BT709_FILTER,
36+
SDR_RGB_TO_TAGGED_BT709_FILTER,
37+
} from "../utils/sdrCaptureColor.js";
3538
import {
3639
describeFfmpegFailure,
3740
ffmpegStatsReader,
@@ -408,6 +411,7 @@ export function buildEncoderArgs(
408411
}
409412
} else if (codec === "prores") {
410413
args.push("-c:v", "prores_ks", "-profile:v", preset, "-vendor", "apl0");
414+
args.push("-vf", SDR_RGB_TO_TAGGED_BT709_FILTER);
411415
args.push("-pix_fmt", pixelFormat);
412416
appendRenderProvenanceArgs(args, outputPath);
413417
return [...args, "-y", outputPath];

‎packages/engine/src/services/streamingEncoder.test.ts‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,10 @@ import {
2626
} from "./streamingEncoder.js";
2727
import { DEFAULT_HDR10_MASTERING } from "../utils/hdr.js";
2828
import { type GpuEncoder } from "../utils/gpuEncoder.js";
29-
import { SDR_CAPTURE_TO_BT709_FILTER } from "../utils/sdrCaptureColor.js";
29+
import {
30+
SDR_CAPTURE_TO_BT709_FILTER,
31+
SDR_RGB_TO_TAGGED_BT709_FILTER,
32+
} from "../utils/sdrCaptureColor.js";
3033

3134
const baseHdrPq: StreamingEncoderOptions = {
3235
fps: { num: 30, den: 1 },
@@ -241,6 +244,15 @@ describe("buildStreamingArgs", () => {
241244
`${SDR_CAPTURE_TO_BT709_FILTER},pad=ceil(iw/2)*2:ceil(ih/2)*2`,
242245
);
243246
});
247+
248+
it("converts ProRes to the BT.709 it is tagged with and keeps the alpha format", () => {
249+
const args = buildStreamingArgs(
250+
{ ...baseVp9, codec: "prores", preset: "4444", pixelFormat: "yuva444p10le" },
251+
"/tmp/out.mov",
252+
);
253+
expect(args[args.indexOf("-vf") + 1]).toBe(SDR_RGB_TO_TAGGED_BT709_FILTER);
254+
expect(args[args.indexOf("-pix_fmt") + 1]).toBe("yuva444p10le");
255+
});
244256
});
245257

246258
describe("output path", () => {

‎packages/engine/src/services/streamingEncoder.ts‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,11 @@ import {
3939
import { getFfmpegBinary } from "../utils/ffmpegBinaries.js";
4040
import { getHdrEncoderColorParams } from "../utils/hdr.js";
4141
import { withEvenDimensionPad } from "../utils/evenDimensions.js";
42-
import { SDR_CAPTURE_TO_BT709_FILTER, SDR_RGB_TO_BT709_FILTER } from "../utils/sdrCaptureColor.js";
42+
import {
43+
SDR_CAPTURE_TO_BT709_FILTER,
44+
SDR_RGB_TO_BT709_FILTER,
45+
SDR_RGB_TO_TAGGED_BT709_FILTER,
46+
} from "../utils/sdrCaptureColor.js";
4347
import { DEFAULT_CONFIG, type EngineConfig } from "../config.js";
4448
import { fpsToFfmpegArg, fpsToNumber, type Fps } from "@hyperframes/core";
4549
import { appendVp9CpuUsedArg } from "./vp9Options.js";
@@ -385,6 +389,7 @@ export function buildStreamingArgs(
385389
}
386390
} else if (codec === "prores") {
387391
args.push("-c:v", "prores_ks", "-profile:v", preset, "-vendor", "apl0");
392+
args.push("-vf", SDR_RGB_TO_TAGGED_BT709_FILTER);
388393
args.push("-pix_fmt", pixelFormat);
389394
appendRenderProvenanceArgs(args, outputPath);
390395
return [...args, "-y", outputPath];
Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
11
export const SDR_RGB_TO_BT709_FILTER = "scale=out_color_matrix=bt709:out_range=tv:flags=neighbor";
22

33
export const SDR_CAPTURE_TO_BT709_FILTER = `scale=in_color_matrix=bt601:in_range=pc:flags=neighbor,format=gbrp,${SDR_RGB_TO_BT709_FILTER}`;
4+
5+
export const SDR_RGB_TO_TAGGED_BT709_FILTER = `${SDR_RGB_TO_BT709_FILTER},setparams=color_primaries=bt709:color_trc=bt709:colorspace=bt709:range=tv`;

0 commit comments

Comments
 (0)