Skip to content

Commit 0abfa49

Browse files
committed
perf(engine): vp9 and vp8 sources without an alpha plane extract as jpeg
A VP8/VP9 source with no alpha tag has one frame decoded with the alpha-aware decoder when its metadata is read; if that frame has no alpha plane, its frames extract as jpeg instead of png. Tagged alpha, a decoded alpha plane, a failed probe, or a plan written without the field keep png.
1 parent 221f367 commit 0abfa49

6 files changed

Lines changed: 239 additions & 8 deletions

File tree

‎packages/engine/src/services/videoFrameExtractor.test.ts‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -700,6 +700,41 @@ describe("resolveFrameFormat", () => {
700700
"png",
701701
);
702702
});
703+
704+
type FormatCase = Pick<VideoMetadata, "videoCodec" | "hasAlpha" | "alphaPlaneDecoded"> & {
705+
requested: "auto" | "jpg" | "png";
706+
};
707+
const formatCases: FormatCase[] = ["vp9", "vp8", "h264", "prores"].flatMap((videoCodec) =>
708+
[true, false].flatMap((hasAlpha) =>
709+
[true, false, undefined].flatMap((alphaPlaneDecoded) =>
710+
(["auto", "jpg", "png"] as const).map((requested) => ({
711+
videoCodec,
712+
hasAlpha,
713+
alphaPlaneDecoded,
714+
requested,
715+
})),
716+
),
717+
),
718+
);
719+
function expectedFormat({ videoCodec, hasAlpha, alphaPlaneDecoded, requested }: FormatCase) {
720+
const sidecar = videoCodec === "vp9" || videoCodec === "vp8";
721+
const keepsPng = hasAlpha || (sidecar && alphaPlaneDecoded !== false);
722+
return keepsPng || requested === "png" ? "png" : "jpg";
723+
}
724+
725+
it("lets VP8/VP9 leave png only when a decode probe found no alpha plane", () => {
726+
expect(formatCases).toHaveLength(72);
727+
for (const { requested, ...fields } of formatCases) {
728+
const actual = resolveFrameFormat(metadata(fields), requested);
729+
expect(actual, JSON.stringify(fields)).toBe(expectedFormat({ requested, ...fields }));
730+
}
731+
expect(resolveFrameFormat(metadata({ videoCodec: "vp9", alphaPlaneDecoded: false }))).toBe(
732+
"jpg",
733+
);
734+
expect(resolveFrameFormat(metadata({ videoCodec: "vp8", alphaPlaneDecoded: false }))).toBe(
735+
"jpg",
736+
);
737+
});
703738
});
704739

705740
describe("parseVideoElements", () => {

‎packages/engine/src/services/videoFrameExtractor.ts‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1608,7 +1608,8 @@ export function resolveVideoExtractionDuration(
16081608

16091609
/** Codecs whose bitstream can carry alpha: decoded with the alpha-aware decoder whether or not a tag says so. */
16101610
const ALPHA_CAPABLE_CODECS = new Set(["vp9", "vp8", "prores"]);
1611-
/** Their alpha rides a side channel ffprobe can miss, so frames stay PNG; ProRes alpha shows in its pixel format. */
1611+
/** Their alpha rides a side channel ffprobe can miss, so frames stay PNG unless a decoded frame showed no
1612+
* alpha plane; ProRes alpha shows in its pixel format. */
16121613
const SIDECAR_ALPHA_CODECS = new Set(["vp9", "vp8"]);
16131614

16141615
export function codecMayHaveAlpha(codec: string | undefined): boolean {
@@ -1623,8 +1624,8 @@ export function resolveFrameFormat(
16231624
metadata: VideoMetadata,
16241625
requested?: VideoFrameFormat,
16251626
): CacheFrameFormat {
1626-
if (metadata.hasAlpha || SIDECAR_ALPHA_CODECS.has((metadata.videoCodec ?? "").toLowerCase()))
1627-
return "png";
1627+
const sidecar = SIDECAR_ALPHA_CODECS.has((metadata.videoCodec ?? "").toLowerCase());
1628+
if (metadata.hasAlpha || (sidecar && metadata.alphaPlaneDecoded !== false)) return "png";
16281629
if (requested === "png" || requested === "jpg") return requested;
16291630
return "jpg";
16301631
}

‎packages/engine/src/utils/ffprobe.test.ts‎

Lines changed: 101 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,8 @@ import { EventEmitter } from "events";
33
import { spawnSync } from "child_process";
44
import { mkdtempSync, readFileSync, renameSync, rmSync, writeFileSync } from "fs";
55
import { tmpdir } from "os";
6-
import { basename, resolve } from "path";
7-
import { afterEach, describe, expect, it, vi } from "vitest";
6+
import { basename, join, resolve } from "path";
7+
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest";
88
import { extractMediaMetadata, extractPngMetadataFromBuffer, parseFrameRate } from "./ffprobe.js";
99

1010
function crc32(buf: Buffer): number {
@@ -1203,6 +1203,39 @@ describe("ffprobe missing-binary fallback", () => {
12031203
expect(meta.hasAlpha).toBe(true);
12041204
});
12051205

1206+
it("leaves alphaPlaneDecoded absent when the untagged VP9 decode probe fails", async () => {
1207+
const { spawn, calls } = createSpawnSpy([
1208+
{
1209+
kind: "exit",
1210+
code: 0,
1211+
stdout: JSON.stringify({
1212+
streams: [
1213+
{
1214+
codec_type: "video",
1215+
codec_name: "vp9",
1216+
width: 320,
1217+
height: 180,
1218+
r_frame_rate: "30/1",
1219+
avg_frame_rate: "30/1",
1220+
pix_fmt: "yuv420p",
1221+
},
1222+
],
1223+
format: { duration: "1.5" },
1224+
}),
1225+
},
1226+
{ kind: "exit", code: 1, stderr: "decoder failed" },
1227+
]);
1228+
vi.resetModules();
1229+
vi.doMock("child_process", () => ({ spawn }));
1230+
1231+
const { extractMediaMetadata: extractMediaMetadataMocked } = await import("./ffprobe.js");
1232+
const meta = await extractMediaMetadataMocked("/tmp/probe-fails.webm");
1233+
1234+
expect(calls[1]?.args).toEqual(expect.arrayContaining(["-c:v", "libvpx-vp9"]));
1235+
expect(meta.hasAlpha).toBe(false);
1236+
expect("alphaPlaneDecoded" in meta).toBe(false);
1237+
});
1238+
12061239
it("extractMediaMetadata rethrows ffprobe-missing error for non-image files without fallback", async () => {
12071240
const { spawn } = createSpawnSpy([{ kind: "missing" }]);
12081241
hidePathBinaries();
@@ -2136,3 +2169,69 @@ describe("runFfprobe process and stream handling", () => {
21362169
);
21372170
});
21382171
});
2172+
2173+
describe.skipIf(!HAS_FFPROBE)("VP8/VP9 decoded alpha plane probe on real files", () => {
2174+
let dir: string;
2175+
const encode = (name: string, args: string[]): string => {
2176+
const out = join(dir, name);
2177+
const res = spawnSync("ffmpeg", [
2178+
"-v",
2179+
"error",
2180+
"-f",
2181+
"lavfi",
2182+
"-i",
2183+
"[email protected]:s=64x64:d=1,format=rgba",
2184+
...args,
2185+
"-y",
2186+
out,
2187+
]);
2188+
expect(res.status).toBe(0);
2189+
return out;
2190+
};
2191+
let alpha: string;
2192+
let alphaVp8: string;
2193+
let opaqueVp9: string;
2194+
let opaqueVp8: string;
2195+
let mpeg4: string;
2196+
beforeAll(() => {
2197+
dir = mkdtempSync(join(tmpdir(), "hf-decoded-alpha-"));
2198+
alpha = encode("alpha.webm", ["-c:v", "libvpx-vp9", "-pix_fmt", "yuva420p"]);
2199+
// libvpx refuses transparency with its default alt-ref frames.
2200+
alphaVp8 = encode("alpha-vp8.webm", [
2201+
"-c:v",
2202+
"libvpx",
2203+
"-auto-alt-ref",
2204+
"0",
2205+
"-pix_fmt",
2206+
"yuva420p",
2207+
]);
2208+
opaqueVp9 = encode("opaque-vp9.webm", ["-c:v", "libvpx-vp9", "-pix_fmt", "yuv420p"]);
2209+
opaqueVp8 = encode("opaque-vp8.webm", ["-c:v", "libvpx", "-pix_fmt", "yuv420p"]);
2210+
mpeg4 = encode("opaque.mp4", ["-c:v", "mpeg4", "-pix_fmt", "yuv420p"]);
2211+
});
2212+
afterAll(() => rmSync(dir, { recursive: true, force: true }));
2213+
2214+
it("probeDecodedAlphaPlane sees the alpha plane only in the alpha stream", async () => {
2215+
const { probeDecodedAlphaPlane } = await import("./ffprobe.js");
2216+
expect(await probeDecodedAlphaPlane(alpha, "libvpx-vp9")).toBe(true);
2217+
expect(await probeDecodedAlphaPlane(alphaVp8, "libvpx")).toBe(true);
2218+
expect(await probeDecodedAlphaPlane(opaqueVp9, "libvpx-vp9")).toBe(false);
2219+
expect(await probeDecodedAlphaPlane(opaqueVp8, "libvpx")).toBe(false);
2220+
expect(await probeDecodedAlphaPlane(join(dir, "missing.webm"), "libvpx-vp9")).toBeUndefined();
2221+
});
2222+
2223+
it("extractMediaMetadata records alphaPlaneDecoded=false only for untagged VP8/VP9", async () => {
2224+
const { extractMediaMetadata } = await import("./ffprobe.js");
2225+
const vp9 = await extractMediaMetadata(opaqueVp9);
2226+
expect(vp9).toMatchObject({ videoCodec: "vp9", hasAlpha: false, alphaPlaneDecoded: false });
2227+
const vp8 = await extractMediaMetadata(opaqueVp8);
2228+
expect(vp8).toMatchObject({ videoCodec: "vp8", hasAlpha: false, alphaPlaneDecoded: false });
2229+
// Tagged alpha is already decided by hasAlpha, so it is not probed.
2230+
const tagged = await extractMediaMetadata(alpha);
2231+
expect(tagged.hasAlpha).toBe(true);
2232+
expect("alphaPlaneDecoded" in tagged).toBe(false);
2233+
const other = await extractMediaMetadata(mpeg4);
2234+
expect(other.videoCodec).toBe("mpeg4");
2235+
expect("alphaPlaneDecoded" in other).toBe(false);
2236+
});
2237+
});

‎packages/engine/src/utils/ffprobe.ts‎

Lines changed: 68 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,18 @@
11
// fallow-ignore-file code-duplication complexity
22
import { spawn } from "child_process";
3-
import { createReadStream, readFileSync, statSync } from "fs";
3+
import { createReadStream, mkdtempSync, readFileSync, rmSync, statSync } from "fs";
44
import { chunkCrc32 } from "./crc32.js";
55
import { StringDecoder } from "node:string_decoder";
6-
import { basename } from "path";
6+
import { tmpdir } from "os";
7+
import { basename, join } from "path";
78
import { redactTelemetryString } from "@hyperframes/core";
8-
import { videoStreamHasAlpha } from "@hyperframes/parsers/media-alpha";
9+
import {
10+
alphaDecoderFor,
11+
pixelFormatHasAlpha,
12+
videoStreamHasAlpha,
13+
} from "@hyperframes/parsers/media-alpha";
914
import { urlInputArgs } from "./urlInput.js";
15+
import { runFfmpeg } from "./runFfmpeg.js";
1016
import { FFPROBE_PATH_ENV, getFfprobeBinary } from "./ffmpegBinaries.js";
1117
import { ManagedChildProcess } from "./managedChildProcess.js";
1218
import { trackChildProcess } from "./processTracker.js";
@@ -209,6 +215,9 @@ export interface VideoMetadata {
209215
isVFR: boolean;
210216
/** True when the stream carries an alpha channel. */
211217
hasAlpha: boolean;
218+
/** VP8/VP9 without an alpha tag: whether a frame decoded with the alpha-aware decoder had an alpha plane.
219+
* Absent when not probed or the probe failed. */
220+
alphaPlaneDecoded?: boolean;
212221
/** Color space info from the video stream. Null if ffprobe didn't report it. */
213222
colorSpace: VideoColorSpace | null;
214223
colorRange?: string;
@@ -813,6 +822,57 @@ export function parseFrameRate(frameRateStr: string | undefined): number {
813822
return rounded > 0 ? rounded : 0.01;
814823
}
815824

825+
/**
826+
* Whether the first frame decoded with `decoder` (alpha-aware) has an alpha pixel format; undefined on any
827+
* failure. ffprobe cannot pick a decoder, so ffmpeg writes the frame as rawvideo, which keeps its pix_fmt.
828+
*/
829+
export async function probeDecodedAlphaPlane(
830+
filePath: string,
831+
decoder: string,
832+
): Promise<boolean | undefined> {
833+
let probeDir: string | undefined;
834+
try {
835+
probeDir = mkdtempSync(join(tmpdir(), "hf-decoded-alpha-"));
836+
const framePath = join(probeDir, "frame.nut");
837+
const decoded = await runFfmpeg(
838+
[
839+
"-v",
840+
"error",
841+
...urlInputArgs(filePath),
842+
"-c:v",
843+
decoder,
844+
"-i",
845+
filePath,
846+
"-frames:v",
847+
"1",
848+
"-vf",
849+
"scale=8:8",
850+
"-c:v",
851+
"rawvideo",
852+
"-f",
853+
"nut",
854+
framePath,
855+
],
856+
{ timeout: 30_000 },
857+
);
858+
if (!decoded.success) return undefined;
859+
const stdout = await runFfprobe(framePath, [
860+
"-select_streams",
861+
"v:0",
862+
"-show_entries",
863+
"stream=pix_fmt",
864+
"-of",
865+
"json",
866+
]);
867+
const pixFmt = (JSON.parse(stdout) as FFProbeOutput).streams?.[0]?.pix_fmt;
868+
return pixFmt ? pixelFormatHasAlpha(pixFmt) : undefined;
869+
} catch {
870+
return undefined;
871+
} finally {
872+
if (probeDir) rmSync(probeDir, { recursive: true, force: true });
873+
}
874+
}
875+
816876
/**
817877
* Probe a media file (video, image, or container) and return normalized metadata.
818878
*
@@ -897,6 +957,10 @@ export async function extractMediaMetadata(filePath: string): Promise<VideoMetad
897957
merged.colorTransfer || merged.colorPrimaries || merged.colorSpace ? merged : null;
898958
const pixelFormat = videoStream.pix_fmt || "";
899959
const hasAlpha = videoStreamHasAlpha(videoStream);
960+
const alphaDecoder = hasAlpha ? null : alphaDecoderFor(videoStream.codec_name);
961+
const alphaPlaneDecoded = alphaDecoder
962+
? await probeDecodedAlphaPlane(filePath, alphaDecoder)
963+
: undefined;
900964

901965
const containerDuration = output?.format.duration ? parseFloat(output.format.duration) : 0;
902966
const streamDuration = videoStream.duration ? parseFloat(videoStream.duration) : 0;
@@ -941,6 +1005,7 @@ export async function extractMediaMetadata(filePath: string): Promise<VideoMetad
9411005
hasAudio: output?.streams.some((s) => s.codec_type === "audio") ?? false,
9421006
isVFR,
9431007
hasAlpha,
1008+
...(alphaPlaneDecoded === undefined ? {} : { alphaPlaneDecoded }),
9441009
colorSpace,
9451010
colorRange: videoStream.color_range,
9461011
pixelFormat,

‎packages/producer/src/services/distributed/shared.ts‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -250,6 +250,12 @@ function readVideoMetadata(
250250
hasAudio: readBoolean(record.hasAudio, `${field}.hasAudio`),
251251
isVFR: readBoolean(record.isVFR, `${field}.isVFR`),
252252
hasAlpha: readBoolean(record.hasAlpha, `${field}.hasAlpha`),
253+
// Optional: plans from producers without the decode probe keep today's png for vp8/vp9.
254+
...(record.alphaPlaneDecoded === undefined
255+
? {}
256+
: {
257+
alphaPlaneDecoded: readBoolean(record.alphaPlaneDecoded, `${field}.alphaPlaneDecoded`),
258+
}),
253259
colorSpace,
254260
};
255261
}

‎packages/producer/src/services/distributed/videoMetadata.test.ts‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -120,6 +120,31 @@ describe("distributed video metadata", () => {
120120
expect(legacyParsed?.videoStreamLeadSeconds).toBe(0);
121121
});
122122

123+
it("keeps alphaPlaneDecoded when present and accepts plans without it", () => {
124+
const withField = (alphaPlaneDecoded: unknown) =>
125+
buildPlanVideosJson({
126+
videos: [video()],
127+
extracted: [
128+
extractedMetadata({
129+
metadata: {
130+
...extractedMetadata().metadata,
131+
videoCodec: "vp9",
132+
alphaPlaneDecoded,
133+
} as PlanVideosJson["extracted"][number]["metadata"],
134+
}),
135+
],
136+
compositionEnd: 8,
137+
});
138+
const roundTrip = (plan: PlanVideosJson) =>
139+
parsePlanVideosJson(JSON.parse(JSON.stringify(plan))).extracted[0]?.metadata;
140+
141+
expect(roundTrip(withField(false))?.alphaPlaneDecoded).toBe(false);
142+
expect(roundTrip(withField(true))?.alphaPlaneDecoded).toBe(true);
143+
const legacy = roundTrip(withField(undefined));
144+
expect(legacy && "alphaPlaneDecoded" in legacy).toBe(false);
145+
expect(() => withField("no")).toThrow(PlanVideosMetadataError);
146+
});
147+
123148
it.each([Number.NaN, Number.POSITIVE_INFINITY, 0, 2])(
124149
"fails closed when no safe composition boundary can be derived (%s)",
125150
(compositionEnd) => {

0 commit comments

Comments
 (0)