Skip to content

Commit 23c920f

Browse files
fix(cli): check finds words under narrow, mirrored or backed cutouts
Review follow-ups on #5657: - A matte the probe walk passes over, even at clear-alpha points, now triggers the per-word check, so a feature between two probe columns (an arm, a mic stand) still fails the word it hides. - Mirrored videos and images map viewport points through the flip; a rotated, skewed or 3D-transformed video falls back to opaque with video_alpha_unread. - A video with an opaque CSS background keeps its whole box occluding. - cli.mdx lists every unread cause and why a fully transparent frame is one. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
1 parent d62bc0c commit 23c920f

4 files changed

Lines changed: 219 additions & 36 deletions

File tree

‎docs/packages/cli.mdx‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -862,9 +862,15 @@ An opaque video covers its whole box, as an opaque image does. A video with
862862
transparency, such as a `remove-background` cutout, covers only where its
863863
alpha is: a word partly behind the subject is an info finding whose
864864
`coveredFraction` is the most covered word's share, and a word 80% hidden or two
865-
neighbouring words each half hidden are an error. When a video's frame cannot
866-
be read (not decoded yet, or cross-origin), the audit treats the video as
867-
opaque and adds a `video_alpha_unread` info finding saying why.
865+
neighbouring words each half hidden are an error. A CSS background on the video
866+
still covers its whole box, and a mirrored video (`scaleX(-1)`) is read on the
867+
side it is painted. When a video's frame cannot be read, the audit treats the
868+
video as opaque and adds a `video_alpha_unread` info finding saying why: no frame
869+
decoded yet, the seek still landing, a cross-origin source, no frame size, a
870+
rotated, skewed or 3D-transformed video, or a frame that reads back fully
871+
transparent. A fully transparent frame is treated as unread on purpose, since a
872+
failed draw looks the same; mark text that sits under a cutout before its subject
873+
enters with `data-layout-allow-occlusion`.
868874

869875
Contrast failures are errors, and each one reports the sampled foreground and
870876
background colours, the measured ratio against the required one, and a

‎packages/cli/src/commands/layout-audit.browser.js‎

Lines changed: 60 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1026,9 +1026,46 @@
10261026
return { x: objectPositionOffset(x, freeX), y: objectPositionOffset(y, freeY) };
10271027
}
10281028

1029+
const IDENTITY_FLIP = { flipX: false, flipY: false };
1030+
1031+
// 2D CSS matrix components [a, b, c, d], or null when the transform is 3D or unparsed.
1032+
function transformMatrix(value) {
1033+
if (!value || value === "none") return [1, 0, 0, 1];
1034+
const match = /^matrix\(([^)]*)\)$/.exec(value.trim());
1035+
if (!match) return null;
1036+
const parts = match[1].split(",").map(Number);
1037+
return parts.length === 6 && parts.every(Number.isFinite) ? parts.slice(0, 4) : null;
1038+
}
1039+
1040+
// Which axes the element's paint is mirrored on, or null when a rotation, skew or 3D transform
1041+
// (on it or an ancestor) means its bounding box no longer maps axis-aligned onto its pixels.
1042+
function paintFlips(element) {
1043+
let flipX = false;
1044+
let flipY = false;
1045+
for (let current = element; current; current = current.parentElement) {
1046+
const style = getComputedStyle(current);
1047+
const matrix = transformMatrix(style.transform);
1048+
if (!matrix) return null;
1049+
const [a, b, c, d] = matrix;
1050+
if (Math.abs(b) > 1e-6 || Math.abs(c) > 1e-6) return null;
1051+
if (style.rotate && style.rotate !== "none" && parseFloat(style.rotate) !== 0) return null;
1052+
const scale = String(style.scale || "none")
1053+
.trim()
1054+
.split(/\s+/)
1055+
.map(Number);
1056+
const scaleX = Number.isFinite(scale[0]) ? scale[0] : 1;
1057+
const scaleY = Number.isFinite(scale[1]) ? scale[1] : scaleX;
1058+
if (a * scaleX < 0) flipX = !flipX;
1059+
if (d * scaleY < 0) flipY = !flipY;
1060+
}
1061+
return flipX || flipY ? { flipX, flipY } : IDENTITY_FLIP;
1062+
}
1063+
10291064
function paintedSourceMapping(element, sourceWidth, sourceHeight) {
10301065
const rect = element.getBoundingClientRect();
10311066
if (!sourceWidth || !sourceHeight || !rect.width || !rect.height) return null;
1067+
const flips = paintFlips(element);
1068+
if (!flips) return null;
10321069

10331070
const style = getComputedStyle(element);
10341071
const fit = style.objectFit || "fill";
@@ -1057,8 +1094,9 @@
10571094
rect.height - paintedHeight,
10581095
);
10591096
return {
1060-
left: rect.left + offsets.x,
1061-
top: rect.top + offsets.y,
1097+
rect: { left: rect.left, top: rect.top, right: rect.right, bottom: rect.bottom },
1098+
offsets,
1099+
...flips,
10621100
paintedWidth,
10631101
paintedHeight,
10641102
scaleX,
@@ -1070,8 +1108,9 @@
10701108

10711109
// The source pixel painted at a viewport point, or null outside the painted content box.
10721110
function sourcePixelAt(mapping, x, y) {
1073-
const localX = x - mapping.left;
1074-
const localY = y - mapping.top;
1111+
const { rect, offsets } = mapping;
1112+
const localX = (mapping.flipX ? rect.right - x : x - rect.left) - offsets.x;
1113+
const localY = (mapping.flipY ? rect.bottom - y : y - rect.top) - offsets.y;
10751114
if (
10761115
localX < 0 ||
10771116
localY < 0 ||
@@ -1157,8 +1196,10 @@
11571196
function readVideoAlphaFrame(video) {
11581197
if (video.seeking) return unreadVideoFrame("the seek had not landed");
11591198
if (video.readyState < 2) return unreadVideoFrame("no frame was decoded at this time");
1199+
if (!video.videoWidth || !video.videoHeight)
1200+
return unreadVideoFrame("the video has no frame size");
11601201
const mapping = paintedSourceMapping(video, video.videoWidth, video.videoHeight);
1161-
if (!mapping) return unreadVideoFrame("the video has no frame size");
1202+
if (!mapping) return unreadVideoFrame("it is rotated, skewed or transformed in 3D");
11621203
const downscale = Math.min(
11631204
1,
11641205
VIDEO_ALPHA_MAX_SIDE / Math.max(mapping.sourceWidth, mapping.sourceHeight),
@@ -1215,8 +1256,10 @@
12151256
return (frame.alpha[pixel.y * frame.width + pixel.x] / 255) * frame.opacity >= OCCLUDING_ALPHA;
12161257
}
12171258

1259+
// A video with an opaque CSS background paints that background behind its clear pixels.
12181260
function matteFrameFor(element) {
12191261
if (element.tagName !== "VIDEO") return null;
1262+
if (hasOpaqueBackground(getComputedStyle(element))) return null;
12201263
const frame = videoAlphaFrame(element);
12211264
return frame.kind === "matte" ? frame : null;
12221265
}
@@ -1282,7 +1325,7 @@
12821325
// there is related to the text, non-opaque, sharing a 3D context with it, or
12831326
// part of a transient crossfade overlap.
12841327
// fallow-ignore-next-line complexity
1285-
function occluderAt(element, x, y) {
1328+
function occluderAt(element, x, y, seenMattes) {
12861329
// Walk the paint-ordered stack: a transparent layer on top must not mask an opaque one below it.
12871330
const stack =
12881331
typeof document.elementsFromPoint === "function"
@@ -1295,6 +1338,8 @@
12951338
// Pair-specific exemptions excuse this hit only; keep walking for deeper occluders.
12961339
if (sharedPreserve3d(element, hit)) continue;
12971340
if (isCrossSceneTransitionOverlap(element, hit)) continue;
1341+
const matte = seenMattes && matteFrameFor(hit);
1342+
if (matte && !seenMattes.has(matte)) seenMattes.set(matte, hit);
12981343
if (isOpaqueOccluder(hit, x, y)) return hit;
12991344
}
13001345
return null;
@@ -1329,25 +1374,27 @@
13291374
let occluder = null;
13301375
let hits = 0;
13311376
let opaqueHits = 0;
1332-
const mattes = new Set();
1377+
// Every matte above the text, including where its alpha is clear: a feature narrower than the
1378+
// probe grid (an arm, a mic stand) can hide a word between two probe columns.
1379+
const seenMattes = new Map();
13331380
for (const textRect of textRects) {
13341381
for (const yFraction of OCCLUSION_PROBE_Y_FRACTIONS) {
13351382
const y = textRect.top + textRect.height * yFraction;
13361383
for (const xFraction of OCCLUSION_PROBE_X_FRACTIONS) {
1337-
const hit = occluderAt(element, textRect.left + textRect.width * xFraction, y);
1384+
const x = textRect.left + textRect.width * xFraction;
1385+
const hit = occluderAt(element, x, y, seenMattes);
13381386
if (!hit) continue;
13391387
hits += 1;
13401388
if (!occluder) occluder = hit;
1341-
const matte = matteFrameFor(hit);
1342-
if (matte) mattes.add(matte);
1343-
else opaqueHits += 1;
1389+
if (!matteFrameFor(hit)) opaqueHits += 1;
13441390
}
13451391
}
13461392
}
1393+
const judgeByMatte = opaqueHits === 0 && seenMattes.size > 0;
13471394
return {
1348-
occluder,
1395+
occluder: occluder || (judgeByMatte ? seenMattes.values().next().value : null),
13491396
coveredFraction: round(hits / (OCCLUSION_GRID_POINTS * textRects.length)),
1350-
mattes: opaqueHits === 0 ? [...mattes] : [],
1397+
mattes: judgeByMatte ? [...seenMattes.keys()] : [],
13511398
};
13521399
}
13531400

‎packages/cli/src/commands/layout-audit.video-alpha.browser.test.ts‎

Lines changed: 86 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,10 @@ interface VideoScene {
3838
/** Alpha (0-255) the decoded frame paints at a composition point. */
3939
alphaAt?: (x: number, y: number) => number;
4040
video?: { readyState?: number; seeking?: boolean; readError?: Error; overlapsText?: boolean };
41+
/** Computed style the <video> adds to the base, e.g. a transform or a background. */
42+
videoStyle?: Record<string, string>;
43+
/** Report each element's text as one line box, as Chromium does, rather than one per word. */
44+
wholeLine?: boolean;
4145
textAttrs?: string;
4246
}
4347

@@ -97,7 +101,11 @@ function installVideoScene(scene: VideoScene): { getImageData: ReturnType<typeof
97101
});
98102

99103
vi.spyOn(window, "getComputedStyle").mockImplementation(
100-
() => ({ ...BASE_STYLE }) as unknown as CSSStyleDeclaration,
104+
(element) =>
105+
({
106+
...BASE_STYLE,
107+
...(element === video ? scene.videoStyle : {}),
108+
}) as unknown as CSSStyleDeclaration,
101109
);
102110
for (const element of Array.from(document.querySelectorAll("*"))) {
103111
vi.spyOn(element, "getBoundingClientRect").mockReturnValue(domRect(FRAME));
@@ -127,9 +135,13 @@ function installVideoScene(scene: VideoScene): { getImageData: ReturnType<typeof
127135
getClientRects() {
128136
const owner = node?.parentElement?.id ?? "";
129137
if (end === null) {
130-
return (scene.text[owner] ?? []).map(({ left, width }) =>
131-
domRect(wordBox(left, width)),
132-
) as unknown as DOMRectList;
138+
const boxes = (scene.text[owner] ?? []).map(({ left, width }) => wordBox(left, width));
139+
if (scene.wholeLine && boxes.length > 0) {
140+
const left = Math.min(...boxes.map((box) => box.left));
141+
const right = Math.max(...boxes.map((box) => box.left + box.width));
142+
return [domRect(wordBox(left, right - left))] as unknown as DOMRectList;
143+
}
144+
return boxes.map(domRect) as unknown as DOMRectList;
133145
}
134146
return wordRect(node, start, end) as unknown as DOMRectList;
135147
},
@@ -295,6 +307,53 @@ describe("layout-audit.browser text behind a <video>", () => {
295307
expect(occlusionOf(issues, "second")).toBeUndefined();
296308
});
297309

310+
it("fails a word hidden by a matte feature that falls between two probe columns", () => {
311+
// One 1800px line box: its 20% and 35% probe columns sit at x 360 and 630, the strip at 440-540.
312+
installVideoScene({
313+
text: {
314+
caption: [
315+
{ word: "a", left: 0, width: 40 },
316+
{ word: "gh", left: 460, width: 60 },
317+
{ word: "end", left: 1700, width: 100 },
318+
],
319+
},
320+
wholeLine: true,
321+
alphaAt: (x) => (x >= 440 && x < 540 ? 255 : 0),
322+
});
323+
expect(occlusionOf(runAudit(), "caption")).toMatchObject({
324+
severity: "error",
325+
containerSelector: "#clip",
326+
message: `"gh" is 100% behind a transparent video's matte.`,
327+
});
328+
});
329+
330+
it("reads a mirrored video's alpha on the side it is painted", () => {
331+
installVideoScene({
332+
text: {
333+
shown: [{ word: "SHIP", left: 200, width: 200 }],
334+
hidden: [{ word: "SHIP", left: 1500, width: 200 }],
335+
},
336+
videoStyle: { transform: "matrix(-1, 0, 0, 1, 0, 0)" },
337+
// Source alpha: the subject fills the source's left half, so it paints on the right.
338+
alphaAt: subjectLeftOf(960),
339+
});
340+
const issues = runAudit();
341+
expect(occlusionOf(issues, "shown")).toBeUndefined();
342+
expect(occlusionOf(issues, "hidden")).toMatchObject({ severity: "error", coveredFraction: 1 });
343+
});
344+
345+
it("keeps a video's opaque CSS background occluding behind its clear pixels", () => {
346+
installVideoScene({
347+
text: SHIP,
348+
videoStyle: { backgroundColor: "rgb(18, 52, 86)" },
349+
alphaAt: subjectLeftOf(400),
350+
});
351+
expect(occlusionOf(runAudit(), "ship")).toMatchObject({
352+
severity: "error",
353+
message: "Text is hidden beneath an opaque element.",
354+
});
355+
});
356+
298357
it("respects data-layout-allow-occlusion", () => {
299358
installVideoScene({
300359
text: SHIP,
@@ -336,23 +395,33 @@ describe("layout-audit.browser text behind a <video>", () => {
336395
alphaAt: undefined,
337396
reason: "its source is cross-origin",
338397
},
398+
{
399+
name: "a rotated video",
400+
video: {},
401+
videoStyle: { transform: "matrix(0, 1, -1, 0, 0, 0)" },
402+
alphaAt: subjectLeftOf(400),
403+
reason: "it is rotated, skewed or transformed in 3D",
404+
},
339405
{
340406
name: "a frame that reads back fully transparent",
341407
video: {},
342408
alphaAt: () => 0,
343409
reason: "its frame read back fully transparent",
344410
},
345-
])("falls back to opaque with an info note for $name", ({ video, alphaAt, reason }) => {
346-
installVideoScene({ text: SHIP, video, alphaAt });
347-
const issues = runAudit();
348-
expect(occlusionOf(issues, "ship")).toMatchObject({ severity: "error", coveredFraction: 1 });
349-
expect(issues).toContainEqual(
350-
expect.objectContaining({
351-
code: "video_alpha_unread",
352-
severity: "info",
353-
selector: "#clip",
354-
message: `Could not read this video's frame alpha (${reason}), so text under it was judged as if the video were opaque.`,
355-
}),
356-
);
357-
});
411+
])(
412+
"falls back to opaque with an info note for $name",
413+
({ video, videoStyle, alphaAt, reason }) => {
414+
installVideoScene({ text: SHIP, video, videoStyle, alphaAt });
415+
const issues = runAudit();
416+
expect(occlusionOf(issues, "ship")).toMatchObject({ severity: "error", coveredFraction: 1 });
417+
expect(issues).toContainEqual(
418+
expect.objectContaining({
419+
code: "video_alpha_unread",
420+
severity: "info",
421+
selector: "#clip",
422+
message: `Could not read this video's frame alpha (${reason}), so text under it was judged as if the video were opaque.`,
423+
}),
424+
);
425+
},
426+
);
358427
});

0 commit comments

Comments
 (0)