Skip to content

Commit cad7dbe

Browse files
fix(engine): warn when a video input declares alpha but decodes opaque (#3771)
* fix(engine): warn when a video input declares alpha but decodes opaque - Problem: a video input whose alpha_mode=1 tag outlives its alpha plane (a remux drops the BlockAdditional sidecar while keeping the tag) composites as a solid rectangle, and nothing tells the user their file, not the renderer, is the problem (#3220 / #3226). - Fix: during extraction preflight, when an input declares alpha (hasAlpha) and its codec can carry alpha, sample the decoded alpha plane (up to 3 frames at 8x8 rgba, bounded 30s, ~768-byte ceiling). If uniformly opaque, emit a non-blocking stderr warning naming the file with the re-export remedy. Inconclusive probes stay silent; opaque full-frame backgrounds are legitimate and never fail the render. - Verification: 8 new unit tests (byte logic + message); engine suite 1483/1487 with 4 pre-existing failures on clean main (missing hdr-regression PNG fixture in this clone); typecheck/lint/format clean; real-fixture checks: lying-tag WebM warns, genuine-alpha WebM stays silent, both agree with ffmpeg alphaextract. Signed-off-by: Santhi Prakash <[email protected]> * fix(engine): probe input alpha with the decoder that matches its codec * style(engine): format the alpha probe test * fix(engine): keep the alpha probe from failing or racing a render --------- Signed-off-by: Santhi Prakash <[email protected]> Co-authored-by: Miguel Ángel <[email protected]>
1 parent 0eba8d6 commit cad7dbe

3 files changed

Lines changed: 207 additions & 0 deletions

File tree

‎packages/engine/src/services/videoFrameExtractor.ts‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@ import {
4141
extractMediaMetadata,
4242
type VideoMetadata,
4343
} from "../utils/ffprobe.js";
44+
import { inputAlphaOpaqueWarning, probeInputAlphaPlane } from "../utils/alphaPlaneProbe.js";
4445
import {
4546
analyzeCompositionHdr,
4647
isHdrColorSpace as isHdrColorSpaceUtil,
@@ -1844,6 +1845,23 @@ export async function extractAllVideoFrames(
18441845
const sdrToHdrTransfers: Array<HdrTransfer | undefined> = resolvedVideos.map(() => undefined);
18451846
breakdown.hdrProbeMs = Date.now() - phase2ProbeStart;
18461847

1848+
// Warning only: an opaque video used as a full-frame background is legitimate.
1849+
const alphaWarnedSrcs = new Set<string>();
1850+
if (resolvedVideos.length > 0) {
1851+
await Promise.all(
1852+
resolvedVideos.map(async ({ video, videoPath }, index) => {
1853+
if (signal?.aborted) return;
1854+
const metadata = videoMetadata[index];
1855+
if (!metadata?.hasAlpha || !codecMayHaveAlpha(metadata.videoCodec)) return;
1856+
if (alphaWarnedSrcs.has(video.src)) return;
1857+
alphaWarnedSrcs.add(video.src);
1858+
const decoder = decoderForCodec(metadata.videoCodec);
1859+
if ((await probeInputAlphaPlane(videoPath, decoder, signal)) !== true) return;
1860+
process.stderr.write(inputAlphaOpaqueWarning(video.src));
1861+
}),
1862+
);
1863+
}
1864+
18471865
const hdrPreflightStart = Date.now();
18481866
const hdrInfo = analyzeCompositionHdr(videoColorSpaces);
18491867
// Track entries the HDR preflight validated as non-extractable so they can
Lines changed: 115 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,115 @@
1+
import { spawnSync } from "node:child_process";
2+
import { mkdtempSync, rmSync } from "node:fs";
3+
import { tmpdir } from "node:os";
4+
import { join } from "node:path";
5+
import { afterAll, beforeAll, describe, expect, it } from "vitest";
6+
import { decoderForCodec } from "../services/videoFrameExtractor.js";
7+
import {
8+
inputAlphaOpaqueWarning,
9+
probeInputAlphaPlane,
10+
sampledRgbaAlphaIsFullyOpaque,
11+
} from "./alphaPlaneProbe.js";
12+
import { getFfmpegBinary } from "./ffmpegBinaries.js";
13+
14+
const BYTES_PER_FRAME = 8 * 8 * 4; // 256 — one 8x8 rgba frame
15+
16+
/** Build a raw rgba sample where the alpha byte of every pixel is `alpha`. */
17+
function sample(alpha: number, frameCount: number): Buffer {
18+
const buf = Buffer.alloc(BYTES_PER_FRAME * frameCount, 0);
19+
for (let i = 3; i < buf.length; i += 4) {
20+
buf[i] = alpha;
21+
}
22+
return buf;
23+
}
24+
25+
describe("sampledRgbaAlphaIsFullyOpaque", () => {
26+
it("returns true when every alpha byte in a single frame is 255", () => {
27+
expect(sampledRgbaAlphaIsFullyOpaque(sample(255, 1))).toBe(true);
28+
});
29+
30+
it("returns true across a multi-frame sample (2 and 3 frames)", () => {
31+
expect(sampledRgbaAlphaIsFullyOpaque(sample(255, 2))).toBe(true);
32+
expect(sampledRgbaAlphaIsFullyOpaque(sample(255, 3))).toBe(true);
33+
});
34+
35+
it("returns false when any pixel shows full transparency", () => {
36+
const buf = sample(255, 1);
37+
buf[3] = 0; // first pixel's alpha
38+
expect(sampledRgbaAlphaIsFullyOpaque(buf)).toBe(false);
39+
});
40+
41+
it("returns false when any pixel shows partial transparency", () => {
42+
const buf = sample(255, 2);
43+
buf[3 + 4 * 10] = 254; // partial alpha on the 11th pixel
44+
expect(sampledRgbaAlphaIsFullyOpaque(buf)).toBe(false);
45+
});
46+
47+
it("returns undefined for an empty sample (inconclusive, not a warning)", () => {
48+
expect(sampledRgbaAlphaIsFullyOpaque(Buffer.alloc(0))).toBeUndefined();
49+
});
50+
51+
it("returns undefined for a byte count that is not a whole-frame multiple", () => {
52+
expect(sampledRgbaAlphaIsFullyOpaque(Buffer.alloc(BYTES_PER_FRAME - 1))).toBeUndefined();
53+
});
54+
55+
it("returns undefined for an oversized sample beyond the 3-frame ceiling", () => {
56+
expect(sampledRgbaAlphaIsFullyOpaque(Buffer.alloc(BYTES_PER_FRAME * 4))).toBeUndefined();
57+
});
58+
});
59+
60+
describe("inputAlphaOpaqueWarning", () => {
61+
it("names the offending file and carries the re-export remedy", () => {
62+
const line = inputAlphaOpaqueWarning("avatar.webm");
63+
expect(line).toContain('src="avatar.webm"');
64+
expect(line).toContain("declares an alpha channel");
65+
expect(line).toContain("first frames decode fully opaque");
66+
expect(line).toContain("alpha pixel format");
67+
expect(line).not.toContain("yuva420p");
68+
expect(line.endsWith("\n")).toBe(true);
69+
});
70+
});
71+
72+
const HAS_FFMPEG = spawnSync(getFfmpegBinary(), ["-version"], { encoding: "utf-8" }).status === 0;
73+
74+
describe.skipIf(!HAS_FFMPEG)("probeInputAlphaPlane on real files", () => {
75+
let dir: string;
76+
beforeAll(() => {
77+
dir = mkdtempSync(join(tmpdir(), "hf-alpha-probe-test-"));
78+
});
79+
afterAll(() => {
80+
rmSync(dir, { recursive: true, force: true });
81+
});
82+
83+
const encode = (name: string, color: string, codecArgs: string[]): string => {
84+
const out = join(dir, name);
85+
const args = [
86+
"-v",
87+
"error",
88+
"-f",
89+
"lavfi",
90+
"-i",
91+
`color=c=${color}:s=32x32:d=0.2,format=rgba`,
92+
...codecArgs,
93+
];
94+
const res = spawnSync(getFfmpegBinary(), [...args, "-y", out]);
95+
expect(res.status).toBe(0);
96+
return out;
97+
};
98+
const vp9 = ["-c:v", "libvpx-vp9", "-pix_fmt", "yuva420p"];
99+
const prores = ["-c:v", "prores_ks", "-profile:v", "4", "-pix_fmt", "yuva444p10le"];
100+
101+
it("reports a VP9 input whose alpha is all opaque", async () => {
102+
const file = encode("opaque.webm", "red", vp9);
103+
expect(await probeInputAlphaPlane(file, decoderForCodec("vp9"))).toBe(true);
104+
});
105+
106+
it("does not flag a VP9 input that is really transparent", async () => {
107+
const file = encode("clear.webm", "[email protected]", vp9);
108+
expect(await probeInputAlphaPlane(file, decoderForCodec("vp9"))).toBe(false);
109+
});
110+
111+
it("probes ProRes 4444 with its own decoder, not the VP9 one", async () => {
112+
const file = encode("opaque.mov", "red", prores);
113+
expect(await probeInputAlphaPlane(file, decoderForCodec("prores"))).toBe(true);
114+
});
115+
});
Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
import { mkdtempSync, readFileSync, rmSync } from "node:fs";
2+
import { tmpdir } from "node:os";
3+
import { join } from "node:path";
4+
import { runFfmpeg } from "./runFfmpeg.js";
5+
6+
// Up to 3 frames at 8x8 rgba; a 1- or 2-frame clip is still a whole sample.
7+
const BYTES_PER_SAMPLE_FRAME = 8 * 8 * 4;
8+
const MAX_SAMPLE_BYTES = BYTES_PER_SAMPLE_FRAME * 3;
9+
10+
/** true: every sampled alpha byte is 255; false: some transparency; undefined: malformed sample. */
11+
export function sampledRgbaAlphaIsFullyOpaque(buf: Buffer): boolean | undefined {
12+
if (
13+
buf.length === 0 ||
14+
buf.length > MAX_SAMPLE_BYTES ||
15+
buf.length % BYTES_PER_SAMPLE_FRAME !== 0
16+
) {
17+
return undefined;
18+
}
19+
for (let i = 3; i < buf.length; i += 4) {
20+
if (buf[i] !== 255) return false;
21+
}
22+
return true;
23+
}
24+
25+
/**
26+
* Whether a video input that declares alpha decodes fully opaque (a remux can keep `alpha_mode=1`
27+
* while dropping the alpha sidecar). `decoder` must keep alpha. Any failure returns undefined.
28+
*/
29+
export async function probeInputAlphaPlane(
30+
videoPath: string,
31+
decoder: string,
32+
signal?: AbortSignal,
33+
): Promise<boolean | undefined> {
34+
let probeDir: string | undefined;
35+
try {
36+
probeDir = mkdtempSync(join(tmpdir(), "hf-alpha-probe-"));
37+
const samplePath = join(probeDir, "alpha.raw");
38+
const result = await runFfmpeg(
39+
[
40+
"-v",
41+
"error",
42+
"-c:v",
43+
decoder,
44+
"-i",
45+
videoPath,
46+
"-frames:v",
47+
"3",
48+
"-vf",
49+
"scale=8:8",
50+
"-pix_fmt",
51+
"rgba",
52+
"-f",
53+
"rawvideo",
54+
samplePath,
55+
],
56+
{ timeout: 30_000, signal },
57+
);
58+
if (!result.success) return undefined;
59+
return sampledRgbaAlphaIsFullyOpaque(readFileSync(samplePath));
60+
} catch {
61+
return undefined;
62+
} finally {
63+
if (probeDir) rmSync(probeDir, { recursive: true, force: true });
64+
}
65+
}
66+
67+
export function inputAlphaOpaqueWarning(src: string): string {
68+
return (
69+
`[hyperframes:render] WARNING: video src="${src}" declares an alpha channel ` +
70+
"but its first frames decode fully opaque. If it should be transparent, re-export " +
71+
"it with an alpha pixel format and avoid remuxing afterward, which can drop the " +
72+
"alpha while keeping the tag.\n"
73+
);
74+
}

0 commit comments

Comments
 (0)