Skip to content

Commit e51ec11

Browse files
perf(engine): on a Mac, video frames for a render decode on the media engine (#5878)
Frame extraction decoded every source in software, so a long 4K60 HEVC clip spent much of its CPU on decoding. On macOS, extraction now asks ffmpeg for VideoToolbox when this ffmpeg has it and the source cannot carry alpha; ffmpeg falls back to software for a stream VideoToolbox cannot decode, and the frames written are byte-identical.
1 parent 924da50 commit e51ec11

4 files changed

Lines changed: 163 additions & 12 deletions

File tree

Lines changed: 105 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,105 @@
1+
import { mkdtempSync, readFileSync, readdirSync, rmSync } from "node:fs";
2+
import { tmpdir } from "node:os";
3+
import { join } from "node:path";
4+
import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
5+
6+
vi.mock("../utils/runFfmpeg.js", async (importOriginal) => {
7+
const real = await importOriginal<typeof import("../utils/runFfmpeg.js")>();
8+
return { ...real, runFfmpeg: vi.fn(real.runFfmpeg) };
9+
});
10+
const probe = vi.hoisted(() => ({ hasVideoToolbox: true }));
11+
vi.mock("../utils/psnrFilterAvailability.js", async (importOriginal) => ({
12+
...(await importOriginal<typeof import("../utils/psnrFilterAvailability.js")>()),
13+
isFfmpegHwaccelAvailable: async () => probe.hasVideoToolbox,
14+
}));
15+
16+
const { runFfmpeg } = await import("../utils/runFfmpeg.js");
17+
const { extractVideoFramesRange } = await import("./videoFrameExtractor.js");
18+
19+
describe.skipIf(process.platform === "win32")("extraction decode on the Mac's media engine", () => {
20+
const dir = mkdtempSync(join(tmpdir(), "hf-extract-media-engine-"));
21+
const realPlatform = process.platform;
22+
const platform = Object.getOwnPropertyDescriptor(process, "platform")!;
23+
const clips = { h264: join(dir, "h264.mp4"), vp9: join(dir, "vp9.webm") };
24+
25+
beforeAll(async () => {
26+
for (const [clip, codec] of [
27+
[clips.h264, ["-c:v", "libx264", "-preset", "ultrafast", "-pix_fmt", "yuv420p"]],
28+
[clips.vp9, ["-c:v", "libvpx-vp9", "-deadline", "realtime", "-pix_fmt", "yuv420p"]],
29+
] as const) {
30+
const made = await runFfmpeg([
31+
...["-y", "-v", "error", "-f", "lavfi", "-i", "testsrc2=s=64x64:d=1:r=10", ...codec, clip],
32+
]);
33+
expect(made.success, made.stderr).toBe(true);
34+
}
35+
});
36+
37+
beforeEach(() => {
38+
vi.mocked(runFfmpeg).mockClear();
39+
probe.hasVideoToolbox = true;
40+
});
41+
42+
afterAll(() => rmSync(dir, { recursive: true, force: true }));
43+
44+
/** Extracts on `os`, returning the extraction's ffmpeg args and frames. */
45+
async function extractOn(os: NodeJS.Platform, clip: string, id: string) {
46+
Object.defineProperty(process, "platform", { ...platform, value: os });
47+
let frames: string[] = [];
48+
try {
49+
const out = join(dir, id);
50+
await extractVideoFramesRange(clip, id, 0, 1, { fps: 10, outputDir: dir }, undefined, {}, out)
51+
.then(
52+
() =>
53+
(frames = readdirSync(out)
54+
.sort()
55+
.map((name) => join(out, name))),
56+
)
57+
.catch((error: unknown) => {
58+
// Off a Mac, an ffmpeg asked for VideoToolbox fails; the args are what is checked there.
59+
if (realPlatform === "darwin" || os !== "darwin") throw error;
60+
});
61+
} finally {
62+
Object.defineProperty(process, "platform", platform);
63+
}
64+
const args = vi
65+
.mocked(runFfmpeg)
66+
.mock.calls.map(([callArgs]) => callArgs)
67+
.find((callArgs) => callArgs.includes(clip) && callArgs.includes("-vf"))!;
68+
return { args, frames };
69+
}
70+
71+
const hwaccel = (args: string[]) =>
72+
args.includes("-hwaccel") ? args[args.indexOf("-hwaccel") + 1] : null;
73+
74+
it("decodes on the media engine on macOS when this ffmpeg has VideoToolbox", async () => {
75+
const { args } = await extractOn("darwin", clips.h264, "mac");
76+
expect(hwaccel(args)).toBe("videotoolbox");
77+
});
78+
79+
it("decodes in software when this ffmpeg has no VideoToolbox", async () => {
80+
probe.hasVideoToolbox = false;
81+
const { args } = await extractOn("darwin", clips.h264, "mac-no-vt");
82+
expect(hwaccel(args)).toBeNull();
83+
});
84+
85+
it("keeps the software decoder for a codec that may carry alpha", async () => {
86+
const { args } = await extractOn("darwin", clips.vp9, "mac-vp9");
87+
expect(hwaccel(args)).toBeNull();
88+
});
89+
90+
it("never asks for VideoToolbox off macOS", async () => {
91+
const { args } = await extractOn("linux", clips.h264, "linux");
92+
expect(hwaccel(args)).toBeNull();
93+
});
94+
95+
it.runIf(realPlatform === "darwin")("writes the same frames as software decode", async () => {
96+
const engine = await extractOn("darwin", clips.h264, "same-engine");
97+
probe.hasVideoToolbox = false;
98+
const software = await extractOn("darwin", clips.h264, "same-software");
99+
expect(hwaccel(engine.args)).toBe("videotoolbox");
100+
expect(engine.frames).toHaveLength(10);
101+
expect(engine.frames.map((f) => readFileSync(f))).toEqual(
102+
software.frames.map((f) => readFileSync(f)),
103+
);
104+
});
105+
});

‎packages/engine/src/services/videoFrameExtractor.ts‎

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,10 @@ import {
6666
writeUrlDownloadTelemetry,
6767
} from "../utils/urlDownloader.js";
6868
import { runFfmpeg, type RunFfmpegResult } from "../utils/runFfmpeg.js";
69-
import { isFfmpegFilterAvailable } from "../utils/psnrFilterAvailability.js";
69+
import {
70+
isFfmpegFilterAvailable,
71+
isFfmpegHwaccelAvailable,
72+
} from "../utils/psnrFilterAvailability.js";
7073
import { DEFAULT_CONFIG, type EngineConfig } from "../config.js";
7174
import { unwrapTemplate } from "../utils/htmlTemplate.js";
7275
import {
@@ -1075,7 +1078,13 @@ async function extractVideoFramesRangeUnredacted(
10751078
const isHdr = isHdrColorSpaceUtil(metadata.colorSpace);
10761079
const isMacOS = process.platform === "darwin";
10771080
const toneMappedToSdr = options.toneMapHdrToSdr === true && isHdr && (await toneMapsWithZscale());
1078-
const decodeWithVideoToolbox = isHdr && isMacOS && !toneMappedToSdr;
1081+
const keepsHdrForChrome = isHdr && isMacOS && !toneMappedToSdr;
1082+
const decodeWithVideoToolbox =
1083+
keepsHdrForChrome ||
1084+
(isMacOS &&
1085+
!metadata.hasAlpha &&
1086+
!codecMayHaveAlpha(metadata.videoCodec) &&
1087+
(await isFfmpegHwaccelAvailable("videotoolbox")));
10791088

10801089
const sampleAtOutputFps = !options.finalFrameOnly;
10811090
if (options.frameRanges && (!sampleAtOutputFps || metadata.isVFR)) {
@@ -1112,7 +1121,7 @@ async function extractVideoFramesRangeUnredacted(
11121121
}
11131122

11141123
const vfFilters: string[] = [];
1115-
if (decodeWithVideoToolbox) {
1124+
if (keepsHdrForChrome) {
11161125
vfFilters.push("format=nv12");
11171126
}
11181127
const samplingAt = vfFilters.length;

‎packages/engine/src/utils/psnrFilterAvailability.test.ts‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -189,3 +189,29 @@ describe("isPsnrFilterAvailable", () => {
189189
await expect(isPsnrFilterAvailable()).resolves.toBe(false);
190190
});
191191
});
192+
193+
describe("isFfmpegHwaccelAvailable", () => {
194+
it("reads `ffmpeg -hwaccels`", async () => {
195+
const { execFile, calls } = createExecFileSpy({
196+
kind: "ok",
197+
stdout: "Hardware acceleration methods:\nvideotoolbox\n",
198+
});
199+
vi.doMock("node:child_process", () => ({ execFile }));
200+
201+
const { isFfmpegHwaccelAvailable } = await import("./psnrFilterAvailability.js");
202+
await expect(isFfmpegHwaccelAvailable("videotoolbox")).resolves.toBe(true);
203+
await expect(isFfmpegHwaccelAvailable("cuda")).resolves.toBe(false);
204+
expect(calls.map((call) => call.args)).toEqual([["-hide_banner", "-hwaccels"]]);
205+
});
206+
207+
it("does not answer from the filter list, or the filter question from the hwaccel list", async () => {
208+
const { execFile, calls } = createExecFileSpy({ kind: "ok", stdout: "videotoolbox\n" });
209+
vi.doMock("node:child_process", () => ({ execFile }));
210+
211+
const { isFfmpegFilterAvailable, isFfmpegHwaccelAvailable } =
212+
await import("./psnrFilterAvailability.js");
213+
await isFfmpegFilterAvailable("zscale");
214+
await isFfmpegHwaccelAvailable("videotoolbox");
215+
expect(calls.map((call) => call.args[1])).toEqual(["-filters", "-hwaccels"]);
216+
});
217+
});

‎packages/engine/src/utils/psnrFilterAvailability.ts‎

Lines changed: 20 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,16 @@ import { getFfmpegBinary } from "./ffmpegBinaries.js";
1515
* not change across renders within the same CLI invocation, and re-probing
1616
* per session would burn ~50-100ms of subprocess spawn per capture worker.
1717
*/
18-
let cachedFilterList: Promise<string | null> | null = null;
18+
const cachedLists = new Map<string, Promise<string | null>>();
19+
20+
function listed(flag: "-filters" | "-hwaccels"): Promise<string | null> {
21+
let list = cachedLists.get(flag);
22+
if (!list) cachedLists.set(flag, (list = listFfmpeg(flag)));
23+
return list;
24+
}
25+
26+
const listsWord = (stdout: string | null, name: string) =>
27+
stdout !== null && new RegExp(`(^|\\s)${name}(\\s|$)`, "m").test(stdout);
1928

2029
/**
2130
* Returns true when the resident ffmpeg exposes the `psnr` filter. False on
@@ -30,26 +39,28 @@ export function isPsnrFilterAvailable(): Promise<boolean> {
3039
return isFfmpegFilterAvailable("psnr");
3140
}
3241

42+
// `-filters` and `-hwaccels` list one name per line; a whole-word match ignores longer names
43+
// and banner prose that mentions it.
3344
export async function isFfmpegFilterAvailable(name: string): Promise<boolean> {
34-
if (cachedFilterList === null) cachedFilterList = listFilters();
35-
const stdout = await cachedFilterList;
36-
// `-filters` lists one filter per line; a whole-word match ignores longer names
37-
// and banner prose that mentions the filter.
38-
return stdout !== null && new RegExp(`(^|\\s)${name}(\\s|$)`, "m").test(stdout);
45+
return listsWord(await listed("-filters"), name);
46+
}
47+
48+
export async function isFfmpegHwaccelAvailable(name: string): Promise<boolean> {
49+
return listsWord(await listed("-hwaccels"), name);
3950
}
4051

4152
/** Test-only: drop the memoized probe result. */
4253
export function resetPsnrFilterAvailabilityCache(): void {
43-
cachedFilterList = null;
54+
cachedLists.clear();
4455
}
4556

46-
async function listFilters(): Promise<string | null> {
57+
async function listFfmpeg(flag: "-filters" | "-hwaccels"): Promise<string | null> {
4758
// Match `psnr.ts`: promisify lazily so a partial `node:child_process` mock
4859
// (test that omits `execFile`) doesn't crash at module load — it fails at
4960
// call time instead, and the try/catch below reports no filters at all.
5061
const execFileP = promisify(execFile);
5162
try {
52-
const { stdout } = await execFileP(getFfmpegBinary(), ["-hide_banner", "-filters"], {
63+
const { stdout } = await execFileP(getFfmpegBinary(), ["-hide_banner", flag], {
5364
maxBuffer: 4 * 1024 * 1024,
5465
timeout: 5_000,
5566
windowsHide: true,

0 commit comments

Comments
 (0)