Skip to content

Commit 39b9352

Browse files
committed
fix(android): stop empty focusable overlays from hiding app content (#1733)
The covered-subtree pruner let any `hittable` sibling condemn a lower drawing-order sibling it geometrically covers. `hittable` is `clickable ?? focusable`, so a childless, textless, id-less full-screen focusable View qualified as "agent-visible content" and dropped the sibling holding the real UI. Telegram wraps every screen in exactly such a View, which is why `snapshot` returned 1 node on every Telegram screen while the helper and stock `uiautomator dump` both saw the full tree — the loss was in the TS parser, not the Android helper. Key covering candidacy on `clickable` instead: focusability is an accessibility-traversal property and says nothing about painting over a sibling. Also generalise the existing "never condemn a marker leaf" exemption to any childless, non-clickable sibling that carries its own text or identifier, since drawing order plus geometry cannot distinguish a transparent overlay from an opaque one.
1 parent 167f93b commit 39b9352

2 files changed

Lines changed: 112 additions & 11 deletions

File tree

‎src/platforms/android/__tests__/ui-hierarchy.test.ts‎

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -460,6 +460,90 @@ test('parseUiHierarchy keeps React Native content under a transparent Expo tools
460460
);
461461
});
462462

463+
test('parseUiHierarchy keeps app content under a childless focusable full-screen overlay', () => {
464+
// Telegram wraps every screen in a childless full-screen focusable View drawn above the content
465+
// container. Focusability is accessibility traversal, not paint (#1733).
466+
const xml = `<hierarchy>
467+
<node class="android.widget.FrameLayout" bounds="[0,0][390,844]" visible-to-user="true" drawing-order="0">
468+
<node class="android.view.View" bounds="[0,0][390,844]" enabled="true" visible-to-user="true" focusable="true" drawing-order="3"/>
469+
<node class="android.widget.FrameLayout" bounds="[0,0][390,844]" visible-to-user="true" drawing-order="1">
470+
<node class="android.widget.TextView" text="Your phone number" bounds="[24,200][366,260]" enabled="true" visible-to-user="true" drawing-order="1"/>
471+
<node class="android.widget.EditText" text="208 379 7171" bounds="[24,320][366,380]" clickable="true" focusable="true" enabled="true" visible-to-user="true" drawing-order="2"/>
472+
</node>
473+
</node>
474+
</hierarchy>`;
475+
476+
const result = parseUiHierarchy(xml, 800, { raw: true });
477+
assert.equal(
478+
result.nodes.some((node) => node.label === 'Your phone number'),
479+
true,
480+
);
481+
assert.equal(
482+
result.nodes.some((node) => node.label === '208 379 7171'),
483+
true,
484+
);
485+
});
486+
487+
test('parseUiHierarchy keeps content under an overlay whose descendants are only focusable', () => {
488+
const xml = `<hierarchy>
489+
<node class="android.widget.FrameLayout" bounds="[0,0][390,844]" visible-to-user="true" drawing-order="0">
490+
<node class="android.view.ViewGroup" bounds="[0,0][390,844]" visible-to-user="true" drawing-order="2">
491+
<node class="android.view.View" bounds="[0,0][390,844]" enabled="true" visible-to-user="true" focusable="true" drawing-order="1"/>
492+
</node>
493+
<node class="android.view.ViewGroup" bounds="[0,0][390,844]" visible-to-user="true" drawing-order="1">
494+
<node class="android.widget.Button" text="Still visible action" bounds="[0,220][280,280]" clickable="true" enabled="true" visible-to-user="true" drawing-order="1"/>
495+
</node>
496+
</node>
497+
</hierarchy>`;
498+
499+
const result = parseUiHierarchy(xml, 800, { raw: true });
500+
assert.equal(
501+
result.nodes.some((node) => node.label === 'Still visible action'),
502+
true,
503+
);
504+
});
505+
506+
test('parseUiHierarchy keeps an overlapped text leaf drawn inside a composite widget sibling', () => {
507+
// Telegram renders the "+" of "+1" as a TextView sitting inside the country-code EditText's box.
508+
const xml = `<hierarchy>
509+
<node class="android.widget.LinearLayout" bounds="[0,0][390,844]" visible-to-user="true" drawing-order="0">
510+
<node class="android.widget.EditText" text="1" content-desc="Country code" bounds="[40,300][130,380]" clickable="true" focusable="true" enabled="true" visible-to-user="true" drawing-order="2"/>
511+
<node class="android.widget.TextView" text="+" bounds="[40,310][55,370]" enabled="true" visible-to-user="true" drawing-order="1"/>
512+
</node>
513+
</hierarchy>`;
514+
515+
const result = parseUiHierarchy(xml, 800, { raw: true });
516+
assert.equal(
517+
result.nodes.some((node) => node.label === '+'),
518+
true,
519+
);
520+
assert.equal(
521+
result.nodes.some((node) => node.label === '1'),
522+
true,
523+
);
524+
});
525+
526+
test('parseUiHierarchy still condemns a clickable leaf covered by a foreground sibling', () => {
527+
const xml = `<hierarchy>
528+
<node class="android.widget.FrameLayout" bounds="[0,0][390,844]" visible-to-user="true" drawing-order="0">
529+
<node class="android.view.ViewGroup" bounds="[0,0][390,844]" visible-to-user="true" drawing-order="2">
530+
<node class="android.widget.Button" text="Modal action" bounds="[24,420][366,480]" clickable="true" enabled="true" visible-to-user="true" drawing-order="1"/>
531+
</node>
532+
<node class="android.widget.Button" text="Behind the modal" bounds="[0,220][280,280]" clickable="true" enabled="true" visible-to-user="true" drawing-order="1"/>
533+
</node>
534+
</hierarchy>`;
535+
536+
const result = parseUiHierarchy(xml, 800, { raw: true });
537+
assert.equal(
538+
result.nodes.some((node) => node.label === 'Modal action'),
539+
true,
540+
);
541+
assert.equal(
542+
result.nodes.some((node) => node.label === 'Behind the modal'),
543+
false,
544+
);
545+
});
546+
463547
test('parseUiHierarchy ignores attribute-name prefix spoofing', () => {
464548
const xml =
465549
"<hierarchy><node class='android.widget.TextView' hint-text='Spoofed' text='Actual' bounds='[10,20][110,60]'/></hierarchy>";

‎src/platforms/android/ui-hierarchy.ts‎

Lines changed: 28 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -422,6 +422,7 @@ export type AndroidUiHierarchy = {
422422
drawingOrder?: number;
423423
focused?: boolean;
424424
hittable?: boolean;
425+
clickable?: boolean;
425426
depth: number;
426427
parentIndex?: number;
427428
hiddenContentAbove?: boolean;
@@ -493,6 +494,7 @@ export function parseUiHierarchyTree(xml: string): AndroidUiHierarchy {
493494
visibleToUser: attrs.visibleToUser,
494495
drawingOrder: attrs.drawingOrder,
495496
hittable: attrs.clickable ?? attrs.focusable,
497+
clickable: attrs.clickable,
496498
scrollable: attrs.scrollable,
497499
canScrollForward: attrs.canScrollForward,
498500
canScrollBackward: attrs.canScrollBackward,
@@ -554,14 +556,21 @@ function shouldKeepAndroidSibling(
554556
coveringCandidates: AndroidCoveringCandidate[],
555557
): boolean {
556558
return (
557-
isSemanticIdentifierMarker(node) ||
558-
!isCoveredByHigherDrawingOrderSibling(node, coveringCandidates)
559+
isUncondemnableLeaf(node) || !isCoveredByHigherDrawingOrderSibling(node, coveringCandidates)
559560
);
560561
}
561562

562-
function isSemanticIdentifierMarker(node: AndroidNode): boolean {
563-
// RN can emit a screen-level testID as an empty sibling beside its rendered navigator.
564-
return hasMeaningfulIdentifier(node) && !node.hittable && node.children.length === 0;
563+
/**
564+
* A childless sibling that announces something of its own and offers no affordance of its own:
565+
* the screen-level testID RN emits beside its rendered navigator, or a composite widget's label
566+
* drawn inside a higher sibling's box (Telegram renders the `+` of `+1` as a TextView overlapping
567+
* the country-code EditText). Drawing order and geometry cannot tell a transparent overlay from an
568+
* opaque one, and text an agent never sees is the one loss it cannot recover from. Exempting a leaf
569+
* is bounded — it can never resurrect a covered surface — so leaves are not condemned on geometry.
570+
*/
571+
function isUncondemnableLeaf(node: AndroidNode): boolean {
572+
if (node.children.length > 0 || node.clickable) return false;
573+
return hasMeaningfulIdentifier(node) || hasMeaningfulLabel(node);
565574
}
566575

567576
function isCoveredByHigherDrawingOrderSibling(
@@ -599,14 +608,22 @@ function canCoverSibling(
599608
);
600609
}
601610

611+
/**
612+
* Whether this node presents something on its own: an affordance to act on, something to announce,
613+
* or an address to select by. Deliberately keyed on `clickable` rather than `hittable`: `hittable`
614+
* falls back to `focusable`, and focusability is an accessibility-traversal property that says
615+
* nothing about painting over a sibling. Telegram wraps its screens in a full-screen focusable
616+
* `android.view.View` with no text, id, or children — it announces nothing and does nothing, so it
617+
* must not condemn the sibling that holds the actual UI (#1733).
618+
*/
602619
function hasOwnAgentVisibleContent(node: AndroidNode): boolean {
603620
if (node.visibleToUser === false) return false;
604-
if (node.hittable) return true;
621+
return node.clickable === true || hasMeaningfulLabel(node) || hasMeaningfulIdentifier(node);
622+
}
623+
624+
function hasMeaningfulLabel(node: AndroidNode): boolean {
605625
const label = node.label?.trim() ?? '';
606-
if (label && !isGenericAndroidId(label)) return true;
607-
const identifier = node.identifier?.trim() ?? '';
608-
if (identifier && !isGenericAndroidId(identifier)) return true;
609-
return false;
626+
return Boolean(label && !isGenericAndroidId(label));
610627
}
611628

612629
function hasActionableDescendant(node: AndroidNode, state: AndroidTreePruneState): boolean {
@@ -616,7 +633,7 @@ function hasActionableDescendant(node: AndroidNode, state: AndroidTreePruneState
616633
const result = node.children.some(
617634
(child) =>
618635
child.visibleToUser !== false &&
619-
(Boolean(child.hittable) || hasActionableDescendant(child, state)),
636+
(Boolean(child.clickable) || hasActionableDescendant(child, state)),
620637
);
621638
state.actionableContentMemo.set(node, result);
622639
return result;

0 commit comments

Comments
 (0)