Skip to content

Commit f1ec9b1

Browse files
committed
fix(android): let a sibling hide only what its content covers (#1806)
pruneAndroidCoveredSubtrees credited a higher drawing-order sibling with painting its whole box as soon as it had any content anywhere inside it, or a label of its own. A full-screen DoraemonKit drag surface holding one 189px floating icon therefore condemned the entire app subtree, and an empty labelled match_parent placeholder did the same. Occlusion is now spatial. A subtree's footprint is the bounding box of what it presents (agent targets and labelled leaves); a sibling is covered when its footprint lies under a candidate's footprint. A node's own label is no longer paint evidence: a container's content-desc describes its children and an empty labelled View draws nothing. Only a touch target still hides its full box (scrims). Comparing footprint to footprint keeps stacked screens with matching margins registering as covered. Live on a Pixel 9 Pro XL API 37 emulator with a DoKit-shaped overlay added to the test app: snapshot -i went from 2 nodes + the sparse hint to the full app; helper-XML A/B across home/catalog/form/product-detail recovered every label with none lost, and non-overlay screens are byte-identical.
1 parent 2b6d04a commit f1ec9b1

2 files changed

Lines changed: 161 additions & 36 deletions

File tree

‎src/platforms/android/__tests__/ui-hierarchy.test.ts‎

Lines changed: 72 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -269,7 +269,7 @@ test('parseUiHierarchy reads an omitted clickable attribute the same as clickabl
269269
const tree = (clickable: string) => `<hierarchy>
270270
<node class="android.widget.FrameLayout" bounds="[0,0][390,844]" visible-to-user="true" drawing-order="0">
271271
<node class="android.view.ViewGroup" bounds="[0,0][390,844]" visible-to-user="true" drawing-order="2">
272-
<node class="android.widget.TextView" text="Foreground tile" bounds="[24,120][366,200]" enabled="true" visible-to-user="true" focusable="true" ${clickable} drawing-order="1"/>
272+
<node class="android.widget.TextView" text="Foreground tile" bounds="[0,120][390,844]" enabled="true" visible-to-user="true" focusable="true" ${clickable} drawing-order="1"/>
273273
</node>
274274
<node class="android.view.ViewGroup" bounds="[0,0][390,844]" visible-to-user="true" drawing-order="1">
275275
<node class="android.widget.TextView" text="Background row" bounds="[0,220][280,280]" enabled="true" visible-to-user="true" focusable="true" ${clickable} drawing-order="1"/>
@@ -329,10 +329,15 @@ test('parseUiHierarchy prunes descendants of Android nodes that are not visible
329329
});
330330

331331
test('parseUiHierarchy prunes lower drawing-order subtrees covered by a foreground sibling', () => {
332+
// A pushed screen (header + full-width rows) drawn above a still-attached drawer surface. The
333+
// pushed screen's presented content lies over the drawer's content, so the drawer is covered.
332334
const xml = `<hierarchy>
333335
<node class="android.widget.FrameLayout" bounds="[0,0][390,844]" visible-to-user="true" drawing-order="0">
334336
<node class="android.view.ViewGroup" bounds="[0,0][390,844]" visible-to-user="true" drawing-order="2">
335-
<node class="android.widget.Button" text="Foreground action" bounds="[24,420][366,480]" clickable="true" enabled="true" visible-to-user="true" drawing-order="1"/>
337+
<node class="android.widget.Button" content-desc="Back" bounds="[8,40][56,88]" clickable="true" enabled="true" visible-to-user="true" drawing-order="1"/>
338+
<node class="android.widget.TextView" text="Foreground screen" bounds="[72,44][300,84]" enabled="true" visible-to-user="true" drawing-order="2"/>
339+
<node class="android.widget.Button" text="Foreground action" bounds="[0,120][390,180]" clickable="true" enabled="true" visible-to-user="true" drawing-order="3"/>
340+
<node class="android.widget.Button" text="Foreground footer" bounds="[0,780][390,844]" clickable="true" enabled="true" visible-to-user="true" drawing-order="4"/>
336341
</node>
337342
<node class="android.view.ViewGroup" bounds="[0,0][390,844]" visible-to-user="true" drawing-order="1">
338343
<node class="android.widget.ScrollView" bounds="[0,120][300,844]" scrollable="true" clickable="true" enabled="true" visible-to-user="true" drawing-order="1">
@@ -353,6 +358,69 @@ test('parseUiHierarchy prunes lower drawing-order subtrees covered by a foregrou
353358
);
354359
});
355360

361+
test('parseUiHierarchy keeps app content under a full-screen overlay holding one floating icon (#1806)', () => {
362+
// DoraemonKit injects a full-screen FrameLayout above the whole Activity purely as a drag surface
363+
// for a small floating icon. It presents only the icon, so it can only hide what sits under it.
364+
const xml = `<hierarchy>
365+
<node class="android.widget.FrameLayout" bounds="[0,0][1224,2860]" visible-to-user="true" drawing-order="0">
366+
<node class="android.widget.LinearLayout" bounds="[0,0][1224,2860]" visible-to-user="true" drawing-order="1">
367+
<node class="android.widget.TextView" text="Editor" bounds="[48,160][600,240]" enabled="true" visible-to-user="true" drawing-order="1"/>
368+
<node class="android.widget.Button" text="Save" bounds="[48,2600][1176,2760]" clickable="true" enabled="true" visible-to-user="true" drawing-order="2"/>
369+
</node>
370+
<node class="android.widget.FrameLayout" resource-id="com.example.app:id/dokit_contentview_id" content-desc="dokit_contentview_id_DokitFrameLayout[1]" bounds="[0,146][1224,2912]" visible-to-user="true" drawing-order="2">
371+
<node class="android.widget.ImageView" content-desc="DoKit" bounds="[1000,1200][1189,1389]" clickable="true" enabled="true" visible-to-user="true" drawing-order="1"/>
372+
</node>
373+
</node>
374+
</hierarchy>`;
375+
376+
const result = parseUiHierarchy(xml, 800, { raw: true });
377+
assert.deepEqual(
378+
result.nodes.filter((node) => node.label).map((node) => node.label),
379+
['Editor', 'Save', 'dokit_contentview_id_DokitFrameLayout[1]', 'DoKit'],
380+
);
381+
});
382+
383+
test('parseUiHierarchy keeps app content beside an empty labelled full-screen placeholder (#1806)', () => {
384+
// A match_parent placeholder container that stays VISIBLE and only receives a fragment later.
385+
// Its content-desc is an announcement, not paint: it draws nothing over the app.
386+
const xml = `<hierarchy>
387+
<node class="android.widget.FrameLayout" bounds="[0,0][390,844]" visible-to-user="true" drawing-order="0">
388+
<node class="android.widget.LinearLayout" bounds="[0,0][390,844]" visible-to-user="true" drawing-order="1">
389+
<node class="android.widget.Button" text="Toolbar action" bounds="[0,40][390,100]" clickable="true" enabled="true" visible-to-user="true" drawing-order="1"/>
390+
</node>
391+
<node class="android.widget.FrameLayout" resource-id="com.example.app:id/panel_host" content-desc="View[1]" bounds="[0,0][390,844]" visible-to-user="true" drawing-order="2"/>
392+
</node>
393+
</hierarchy>`;
394+
395+
const result = parseUiHierarchy(xml, 800, { raw: true });
396+
assert.equal(
397+
result.nodes.some((node) => node.label === 'Toolbar action'),
398+
true,
399+
);
400+
});
401+
402+
test('parseUiHierarchy compares presented footprints so a sparse overlay never condemns a rich sibling', () => {
403+
// The overlay's only content is a corner badge; the sibling's content spans the screen. Box
404+
// geometry alone (overlay box ⊇ sibling box) would call this covered.
405+
const xml = `<hierarchy>
406+
<node class="android.widget.FrameLayout" bounds="[0,0][390,844]" visible-to-user="true" drawing-order="0">
407+
<node class="android.view.ViewGroup" bounds="[0,0][390,844]" visible-to-user="true" drawing-order="1">
408+
<node class="android.widget.Button" text="Top action" bounds="[24,60][366,120]" clickable="true" enabled="true" visible-to-user="true" drawing-order="1"/>
409+
<node class="android.widget.Button" text="Bottom action" bounds="[24,760][366,820]" clickable="true" enabled="true" visible-to-user="true" drawing-order="2"/>
410+
</node>
411+
<node class="android.view.ViewGroup" bounds="[0,0][390,844]" visible-to-user="true" drawing-order="2">
412+
<node class="android.widget.Button" text="Badge" bounds="[330,20][380,70]" clickable="true" enabled="true" visible-to-user="true" drawing-order="1"/>
413+
</node>
414+
</node>
415+
</hierarchy>`;
416+
417+
const result = parseUiHierarchy(xml, 800, { raw: true });
418+
assert.deepEqual(
419+
result.nodes.filter((node) => node.label).map((node) => node.label),
420+
['Top action', 'Bottom action', 'Badge'],
421+
);
422+
});
423+
356424
test('parseUiHierarchy keeps visible identifier-only markers beside covering content', () => {
357425
const xml = `<hierarchy>
358426
<node class="android.widget.FrameLayout" bounds="[0,0][390,844]" visible-to-user="true" drawing-order="0">
@@ -556,9 +624,10 @@ test('parseUiHierarchy keeps an overlapped text leaf drawn inside a composite wi
556624
});
557625

558626
test('parseUiHierarchy still condemns a clickable leaf covered by a foreground sibling', () => {
627+
// A tap-to-dismiss scrim is itself a touch target, so it hides its whole box.
559628
const xml = `<hierarchy>
560629
<node class="android.widget.FrameLayout" bounds="[0,0][390,844]" visible-to-user="true" drawing-order="0">
561-
<node class="android.view.ViewGroup" bounds="[0,0][390,844]" visible-to-user="true" drawing-order="2">
630+
<node class="android.view.ViewGroup" bounds="[0,0][390,844]" clickable="true" enabled="true" visible-to-user="true" drawing-order="2">
562631
<node class="android.widget.Button" text="Modal action" bounds="[24,420][366,480]" clickable="true" enabled="true" visible-to-user="true" drawing-order="1"/>
563632
</node>
564633
<node class="android.widget.Button" text="Behind the modal" bounds="[0,220][280,280]" clickable="true" enabled="true" visible-to-user="true" drawing-order="1"/>

‎src/platforms/android/ui-hierarchy.ts‎

Lines changed: 89 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -452,13 +452,20 @@ type AndroidNodeInclusionInfo = {
452452
isVisual: boolean;
453453
};
454454

455+
type AndroidFootprint = {
456+
/** Bounding box of what the subtree presents (agent targets and labelled leaves), if anything. */
457+
rect: Rect | null;
458+
hasAgentTarget: boolean;
459+
};
460+
455461
type AndroidTreePruneState = {
456-
actionableContentMemo: WeakMap<AndroidNode, boolean>;
462+
footprintMemo: WeakMap<AndroidNode, AndroidFootprint>;
457463
};
458464

459-
type AndroidCoveringCandidate = AndroidNode & {
460-
rect: Rect;
465+
type AndroidCoveringCandidate = {
466+
node: AndroidNode;
461467
drawingOrder: number;
468+
footprint: Rect;
462469
};
463470

464471
const ANDROID_WINDOW_TYPE_APPLICATION = 1;
@@ -525,7 +532,7 @@ export function parseUiHierarchyTree(xml: string): AndroidUiHierarchy {
525532
discardInactiveAndroidApplicationWindows(root);
526533
// UiAutomation can expose covered React Native navigation surfaces in the same accessibility
527534
// window. If a higher drawing-order sibling covers them, agents should see the foreground surface.
528-
pruneAndroidCoveredSubtrees(root, { actionableContentMemo: new WeakMap() });
535+
pruneAndroidCoveredSubtrees(root, { footprintMemo: new WeakMap() });
529536
applyAndroidScrollActionHints(root);
530537
return root;
531538
}
@@ -550,22 +557,59 @@ function hasSemanticContent(node: AndroidNode): boolean {
550557
return hasMeaningfulLabel(node) || hasMeaningfulIdentifier(node);
551558
}
552559

553-
/** Focusability is traversal, not paint, so it is not evidence a node hides anything (#1733). */
560+
/**
561+
* Focusability is traversal, not paint (#1733), and a label is an announcement, not paint (#1806):
562+
* a container's content-desc describes its children and an empty labelled View draws nothing. Only
563+
* a touch target is direct evidence that a node hides what lies under its box.
564+
*/
554565
function hasDirectOcclusionEvidence(node: AndroidNode): boolean {
555-
return node.visibleToUser !== false && (isTouchTarget(node) || hasSemanticContent(node));
566+
return node.visibleToUser !== false && isTouchTarget(node);
556567
}
557568

558569
/** Evidence the node is a real surface because it contains something an agent could drive. */
559570
function hasDescendantOcclusionEvidence(node: AndroidNode, state: AndroidTreePruneState): boolean {
560-
const cached = state.actionableContentMemo.get(node);
561-
if (cached !== undefined) return cached;
562-
const result = node.children.some(
563-
(child) =>
564-
child.visibleToUser !== false &&
565-
(isAgentTarget(child) || hasDescendantOcclusionEvidence(child, state)),
571+
return node.children.some(
572+
(child) => child.visibleToUser !== false && subtreeFootprint(child, state).hasAgentTarget,
566573
);
567-
state.actionableContentMemo.set(node, result);
568-
return result;
574+
}
575+
576+
/**
577+
* Where a subtree visibly presents something: the bounding box of its agent targets and labelled
578+
* leaves. A full-screen debug overlay holding one floating icon presents only that icon, so it can
579+
* only hide what sits under the icon, never the whole app behind it (#1806).
580+
*/
581+
function subtreeFootprint(node: AndroidNode, state: AndroidTreePruneState): AndroidFootprint {
582+
const cached = state.footprintMemo.get(node);
583+
if (cached !== undefined) return cached;
584+
let hasAgentTarget = isAgentTarget(node);
585+
let rect: Rect | null =
586+
(isAgentTarget(node) || isLabelledLeaf(node)) && hasPositiveRect(node) ? node.rect : null;
587+
for (const child of node.children) {
588+
if (child.visibleToUser === false) continue;
589+
const childFootprint = subtreeFootprint(child, state);
590+
hasAgentTarget ||= childFootprint.hasAgentTarget;
591+
rect = unionRect(rect, childFootprint.rect);
592+
}
593+
const footprint = { rect, hasAgentTarget };
594+
state.footprintMemo.set(node, footprint);
595+
return footprint;
596+
}
597+
598+
function isLabelledLeaf(node: AndroidNode): boolean {
599+
return node.children.length === 0 && hasMeaningfulLabel(node);
600+
}
601+
602+
function unionRect(left: Rect | null, right: Rect | null): Rect | null {
603+
if (!left) return right;
604+
if (!right) return left;
605+
const x = Math.min(left.x, right.x);
606+
const y = Math.min(left.y, right.y);
607+
return {
608+
x,
609+
y,
610+
width: Math.max(left.x + left.width, right.x + right.width) - x,
611+
height: Math.max(left.y + left.height, right.y + right.height) - y,
612+
};
569613
}
570614

571615
/**
@@ -598,32 +642,45 @@ function pruneAndroidCoveredSubtrees(node: AndroidNode, state: AndroidTreePruneS
598642
return;
599643
}
600644
const siblings = node.children;
601-
const coveringCandidates = siblings.filter((sibling) => canCoverSibling(sibling, state));
645+
const coveringCandidates = siblings
646+
.map((sibling) => coveringCandidateOf(sibling, state))
647+
.filter((candidate) => candidate !== null);
602648
if (coveringCandidates.length === 0) return;
603-
node.children = siblings.filter((child) => shouldKeepAndroidSibling(child, coveringCandidates));
649+
node.children = siblings.filter((child) =>
650+
shouldKeepAndroidSibling(child, coveringCandidates, state),
651+
);
604652
}
605653

606654
function shouldKeepAndroidSibling(
607655
node: AndroidNode,
608656
coveringCandidates: AndroidCoveringCandidate[],
657+
state: AndroidTreePruneState,
609658
): boolean {
610659
return (
611-
isPresentationLeaf(node) || !isCoveredByHigherDrawingOrderSibling(node, coveringCandidates)
660+
isPresentationLeaf(node) ||
661+
!isCoveredByHigherDrawingOrderSibling(node, coveringCandidates, state)
612662
);
613663
}
614664

665+
/**
666+
* Covered means the sibling's presented content lies under the candidate's presented content.
667+
* Comparing footprints rather than boxes lets two stacked screens with the same layout margins
668+
* still register as covered, while a sparse overlay never condemns a rich screen.
669+
*/
615670
function isCoveredByHigherDrawingOrderSibling(
616671
node: AndroidNode,
617672
coveringCandidates: AndroidCoveringCandidate[],
673+
state: AndroidTreePruneState,
618674
): boolean {
619675
if (node.visibleToUser === false || node.drawingOrder === undefined || !hasPositiveRect(node)) {
620676
return false;
621677
}
622-
for (const sibling of coveringCandidates) {
623-
if (sibling === node || sibling.drawingOrder <= node.drawingOrder) {
678+
const coveredRect = subtreeFootprint(node, state).rect ?? node.rect;
679+
for (const candidate of coveringCandidates) {
680+
if (candidate.node === node || candidate.drawingOrder <= node.drawingOrder) {
624681
continue;
625682
}
626-
if (rectCoverage(sibling.rect, node.rect) >= 0.9) {
683+
if (rectCoverage(candidate.footprint, coveredRect) >= 0.9) {
627684
return true;
628685
}
629686
}
@@ -635,21 +692,20 @@ function hasMeaningfulIdentifier(node: AndroidNode): boolean {
635692
return Boolean(identifier && !isGenericAndroidId(identifier));
636693
}
637694

638-
function canCoverSibling(
695+
/** The single occlusion classification. Covering is never re-derived from a raw attribute. */
696+
function coveringCandidateOf(
639697
node: AndroidNode,
640698
state: AndroidTreePruneState,
641-
): node is AndroidCoveringCandidate {
642-
return (
643-
node.visibleToUser !== false &&
644-
node.drawingOrder !== undefined &&
645-
hasPositiveRect(node) &&
646-
hasOcclusionEvidence(node, state)
647-
);
648-
}
649-
650-
/** The single occlusion classification. Covering is never re-derived from a raw attribute. */
651-
function hasOcclusionEvidence(node: AndroidNode, state: AndroidTreePruneState): boolean {
652-
return hasDirectOcclusionEvidence(node) || hasDescendantOcclusionEvidence(node, state);
699+
): AndroidCoveringCandidate | null {
700+
const { drawingOrder } = node;
701+
if (node.visibleToUser === false || drawingOrder === undefined || !hasPositiveRect(node)) {
702+
return null;
703+
}
704+
if (!hasDirectOcclusionEvidence(node) && !hasDescendantOcclusionEvidence(node, state)) {
705+
return null;
706+
}
707+
const footprint = subtreeFootprint(node, state).rect;
708+
return footprint ? { node, drawingOrder, footprint } : null;
653709
}
654710

655711
function hasMeaningfulLabel(node: AndroidNode): boolean {

0 commit comments

Comments
 (0)