Skip to content

Commit 5be8ba0

Browse files
committed
fix(android): count what a covered sibling shows, not only what it paints
Fuzzing random sibling trees old-vs-new surfaced the one direction the footprint model could still regress: a container whose only painted content is small (one corner icon) but which also carries labelled containers or testID-only markers was condemned as soon as a touch surface covered that icon, since markers and container labels are not paint and never entered the footprint. Footprints now carry two rect sets. `paints` (touch targets, scrollables, labelled leaves) is what a candidate can cover with; it still excludes identifiers and container labels, or the DoKit fix would unwind. `shows` adds every labelled or identified node and is what a covered sibling must lose in full. Focusable-only nodes no longer paint their box either, matching #1733 for descendants as well as siblings. Adds the marker regression. Re-fuzzed 20k trees: new-prunes-more is down to 0.14 %, all of the class where everything the target shows lies under a higher touch/scroll surface. Live captures unchanged.
1 parent e1e706d commit 5be8ba0

2 files changed

Lines changed: 77 additions & 27 deletions

File tree

‎src/platforms/android/__tests__/ui-hierarchy.test.ts‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -426,6 +426,29 @@ test('parseUiHierarchy keeps app content under an overlay whose only controls si
426426
);
427427
});
428428

429+
test('parseUiHierarchy counts identifier-only markers toward what a covered sibling shows', () => {
430+
// A screen container carrying testID markers whose only painted content is one corner icon,
431+
// under a clickable header bar. The bar covers the icon, but the agent would also lose the
432+
// markers, which sit well outside the bar — so the container is not covered.
433+
const xml = `<hierarchy>
434+
<node class="android.widget.FrameLayout" bounds="[0,0][390,844]" visible-to-user="true" drawing-order="0">
435+
<node class="android.view.ViewGroup" resource-id="home-screen" bounds="[0,0][390,844]" visible-to-user="true" drawing-order="1">
436+
<node class="android.widget.ImageView" content-desc="Menu" bounds="[8,8][56,56]" clickable="true" enabled="true" visible-to-user="true" drawing-order="1"/>
437+
<node class="android.view.ViewGroup" resource-id="home-body" bounds="[0,120][390,844]" visible-to-user="true" drawing-order="2"/>
438+
</node>
439+
<node class="android.view.ViewGroup" bounds="[0,0][390,120]" clickable="true" enabled="true" visible-to-user="true" drawing-order="2">
440+
<node class="android.widget.TextView" text="Header" bounds="[72,40][300,80]" enabled="true" visible-to-user="true" drawing-order="1"/>
441+
</node>
442+
</node>
443+
</hierarchy>`;
444+
445+
const result = parseUiHierarchy(xml, 800, { raw: true });
446+
assert.equal(
447+
result.nodes.some((node) => node.identifier === 'home-body'),
448+
true,
449+
);
450+
});
451+
429452
test('parseUiHierarchy compares presented footprints so a sparse overlay never condemns a rich sibling', () => {
430453
// The overlay's only content is a corner badge; the sibling's content spans the screen. Box
431454
// geometry alone (overlay box ⊇ sibling box) would call this covered.

‎src/platforms/android/ui-hierarchy.ts‎

Lines changed: 54 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -452,8 +452,10 @@ type AndroidNodeInclusionInfo = {
452452
};
453453

454454
type AndroidFootprint = {
455-
/** Boxes of what the subtree presents: touch/focus targets, scrollables and labelled leaves. */
456-
rects: Rect[];
455+
/** Boxes of what the subtree paints: touch targets, scrollables and labelled leaves. */
456+
paints: Rect[];
457+
/** Boxes of what an agent would see of the subtree: `paints` plus labelled/identified nodes. */
458+
shows: Rect[];
457459
hasAgentTarget: boolean;
458460
};
459461

@@ -573,35 +575,60 @@ function hasDescendantOcclusionEvidence(node: AndroidNode, state: AndroidTreePru
573575
}
574576

575577
/**
576-
* Where a subtree visibly presents something: the boxes of its touch-consuming surfaces (touch
577-
* targets, scrollables), focus targets and labelled leaves. A full-screen debug overlay holding one
578-
* floating icon presents only that icon, so it can only hide what sits under the icon, never the
579-
* whole app behind it (#1806). The rects are kept apart rather than merged into one bounding box:
580-
* two controls in opposite corners present two corners, not the screen between them.
578+
* What a subtree paints and what it shows. Paint is the boxes of its touch-consuming surfaces
579+
* (touch targets, scrollables) and labelled leaves: a full-screen debug overlay
580+
* holding one floating icon paints only that icon, so it can only hide what sits under the icon,
581+
* never the whole app behind it (#1806). Rects are kept apart rather than merged into one bounding
582+
* box: two controls in opposite corners paint two corners, not the screen between them.
583+
*
584+
* Shows adds every labelled or identified node — a testID marker or a described container paints
585+
* nothing, so it never helps a candidate cover, but an agent would still lose it, so it always
586+
* counts toward what a covered sibling has.
581587
*/
582588
function subtreeFootprint(node: AndroidNode, state: AndroidTreePruneState): AndroidFootprint {
583589
const cached = state.footprintMemo.get(node);
584590
if (cached !== undefined) return cached;
585-
let footprint: AndroidFootprint;
586-
if (presentsOwnBox(node) && hasPositiveRect(node)) {
587-
// Its box is presented as a whole; whatever it contains lies inside that box.
588-
footprint = { rects: [node.rect], hasAgentTarget: isAgentTarget(node) };
589-
} else {
590-
footprint = { rects: [], hasAgentTarget: isAgentTarget(node) };
591-
for (const child of node.children) {
592-
if (child.visibleToUser === false) continue;
593-
const childFootprint = subtreeFootprint(child, state);
594-
footprint.hasAgentTarget ||= childFootprint.hasAgentTarget;
595-
footprint.rects.push(...childFootprint.rects);
596-
}
597-
}
591+
const footprint = hasPositiveRect(node)
592+
? footprintWithinBox(node, node.rect, state)
593+
: childrenFootprint(node, state);
598594
state.footprintMemo.set(node, footprint);
599595
return footprint;
600596
}
601597

602-
function presentsOwnBox(node: AndroidNode): boolean {
598+
function footprintWithinBox(
599+
node: AndroidNode,
600+
ownBox: Rect,
601+
state: AndroidTreePruneState,
602+
): AndroidFootprint {
603+
if (paintsOwnBox(node)) {
604+
// The whole box is painted; whatever it contains lies inside that box.
605+
return { paints: [ownBox], shows: [ownBox], hasAgentTarget: isAgentTarget(node) };
606+
}
607+
const footprint = childrenFootprint(node, state);
608+
if (hasSemanticContent(node)) footprint.shows.push(ownBox);
609+
return footprint;
610+
}
611+
612+
function childrenFootprint(node: AndroidNode, state: AndroidTreePruneState): AndroidFootprint {
613+
const footprint: AndroidFootprint = {
614+
paints: [],
615+
shows: [],
616+
hasAgentTarget: isAgentTarget(node),
617+
};
618+
for (const child of node.children) {
619+
if (child.visibleToUser === false) continue;
620+
const childFootprint = subtreeFootprint(child, state);
621+
footprint.hasAgentTarget ||= childFootprint.hasAgentTarget;
622+
footprint.paints.push(...childFootprint.paints);
623+
footprint.shows.push(...childFootprint.shows);
624+
}
625+
return footprint;
626+
}
627+
628+
/** Focusability is traversal, not paint (#1733); a container's label describes its children. */
629+
function paintsOwnBox(node: AndroidNode): boolean {
603630
return (
604-
isAgentTarget(node) ||
631+
isTouchTarget(node) ||
605632
node.scrollable === true ||
606633
(node.children.length === 0 && hasMeaningfulLabel(node))
607634
);
@@ -704,8 +731,8 @@ function shouldKeepAndroidSibling(
704731
}
705732

706733
/**
707-
* Covered means the sibling's presented content lies under the candidate's presented content, by
708-
* actual overlapped area. Comparing footprints rather than boxes lets two stacked screens with the
734+
* Covered means everything an agent would see of the sibling lies under what the candidate paints,
735+
* by actual overlapped area. Comparing footprints rather than boxes lets two stacked screens with the
709736
* same layout margins still register as covered, while a sparse overlay never condemns a rich
710737
* screen however far apart its controls sit.
711738
*/
@@ -717,8 +744,8 @@ function isCoveredByHigherDrawingOrderSibling(
717744
if (node.visibleToUser === false || node.drawingOrder === undefined || !hasPositiveRect(node)) {
718745
return false;
719746
}
720-
const footprint = subtreeFootprint(node, state).rects;
721-
const coveredRects = footprint.length > 0 ? footprint : [node.rect];
747+
const shows = subtreeFootprint(node, state).shows;
748+
const coveredRects = shows.length > 0 ? shows : [node.rect];
722749
for (const candidate of coveringCandidates) {
723750
if (candidate.node === node || candidate.drawingOrder <= node.drawingOrder) {
724751
continue;
@@ -747,7 +774,7 @@ function coveringCandidateOf(
747774
if (!hasDirectOcclusionEvidence(node) && !hasDescendantOcclusionEvidence(node, state)) {
748775
return null;
749776
}
750-
const footprint = subtreeFootprint(node, state).rects;
777+
const footprint = subtreeFootprint(node, state).paints;
751778
return footprint.length > 0 ? { node, drawingOrder, footprint } : null;
752779
}
753780

0 commit comments

Comments
 (0)