Skip to content

Commit 854191e

Browse files
committed
fix(android): fold Compose role nodes by containment in one pre-pass, and pin each guard in tests
1 parent 37df9c0 commit 854191e

4 files changed

Lines changed: 150 additions & 50 deletions

File tree

‎packages/platform-android/src/__tests__/ui-hierarchy-builder.test.ts‎

Lines changed: 76 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -94,52 +94,93 @@ test('launcher-hosted navigation survives omitted wrappers while taskbar app ico
9494
]);
9595
});
9696

97-
test('publishes a Compose role node as its clickable parent and keeps real child controls', () => {
97+
test('folds a Compose role node into its clickable parent only when every guard holds', () => {
9898
const tree = parseUiHierarchyTree(`<hierarchy>
99-
<node class="android.view.View" bounds="[0,0][1080,2400]" visible-to-user="true">
100-
<node class="android.view.View" resource-id="continue-with-email"
101-
bounds="[53,1856][1027,1982]" clickable="true" focusable="true" visible-to-user="true">
102-
<node class="android.widget.TextView" text="Continue with email"
103-
bounds="[352,1895][728,1943]" visible-to-user="true" />
104-
<node class="android.widget.Button" checked="false" bounds="[53,1858][1027,1980]"
105-
visible-to-user="true" />
99+
<node class="android.widget.FrameLayout" bounds="[0,0][1000,1000]" visible-to-user="true">
100+
<node class="android.view.View" resource-id="compose-container" bounds="[0,0][1000,100]"
101+
clickable="true" visible-to-user="true">
102+
<node class="android.widget.TextView" text="Continue" bounds="[400,0][600,100]" visible-to-user="true" />
103+
<node class="android.widget.Button" checked="false" bounds="[0,0][1000,100]" visible-to-user="true" />
106104
</node>
107-
<node class="android.view.View" resource-id="nested-control-row"
108-
bounds="[53,2000][1027,2110]" clickable="true" visible-to-user="true">
109-
<node class="android.widget.Button" bounds="[53,2000][1027,2110]" clickable="true"
110-
visible-to-user="true" />
105+
<node class="android.view.View" resource-id="compose-checkbox" bounds="[0,100][1000,200]"
106+
clickable="true" checked="true" visible-to-user="true">
107+
<node class="android.widget.CheckBox" bounds="[0,130][1000,190]" visible-to-user="true" />
111108
</node>
112-
<node class="android.view.View" resource-id="legacy-row"
113-
bounds="[53,2120][1027,2230]" clickable="true" visible-to-user="true">
114-
<node class="android.widget.CheckBox" bounds="[900,2140][1000,2210]"
115-
visible-to-user="true" />
109+
<node class="android.view.View" resource-id="labeled-role" bounds="[0,200][1000,300]"
110+
clickable="true" visible-to-user="true">
111+
<node class="android.widget.Button" text="Details" bounds="[0,200][1000,300]" visible-to-user="true" />
116112
</node>
117-
<node class="android.widget.Button" bounds="[53,2250][1027,2370]" visible-to-user="true" />
118-
<node class="android.view.View" resource-id="hidden-role-row"
119-
bounds="[53,2380][1027,2390]" clickable="true" visible-to-user="true">
120-
<node class="android.widget.Button" bounds="[53,2380][1027,2390]"
121-
visible-to-user="false" />
113+
<node class="android.view.View" resource-id="identified-role" bounds="[0,300][1000,400]"
114+
clickable="true" visible-to-user="true">
115+
<node class="android.widget.Button" resource-id="inner" bounds="[0,300][1000,400]" visible-to-user="true" />
116+
</node>
117+
<node class="android.view.View" resource-id="parent-role" bounds="[0,400][1000,500]"
118+
clickable="true" visible-to-user="true">
119+
<node class="android.widget.Button" bounds="[0,400][1000,500]" visible-to-user="true">
120+
<node class="android.widget.TextView" text="Inside" bounds="[400,400][600,500]" visible-to-user="true" />
121+
</node>
122+
</node>
123+
<node class="android.view.View" resource-id="nested-control" bounds="[0,500][1000,600]"
124+
clickable="true" visible-to-user="true">
125+
<node class="android.widget.Button" clickable="true" bounds="[0,500][1000,600]" visible-to-user="true" />
126+
</node>
127+
<node class="android.view.View" resource-id="hidden-role" bounds="[0,600][1000,700]"
128+
clickable="true" visible-to-user="true">
129+
<node class="android.widget.Button" bounds="[0,600][1000,700]" visible-to-user="false" />
130+
</node>
131+
<node class="android.view.View" resource-id="outside-role" bounds="[0,700][1000,800]"
132+
clickable="true" visible-to-user="true">
133+
<node class="android.widget.CheckBox" bounds="[900,700][1100,800]" visible-to-user="true" />
134+
</node>
135+
<node class="android.view.View" resource-id="passive-parent" bounds="[0,800][1000,900]"
136+
visible-to-user="true">
137+
<node class="android.widget.Button" bounds="[0,800][1000,900]" visible-to-user="true" />
138+
</node>
139+
<node class="android.widget.LinearLayout" resource-id="native-parent" bounds="[0,900][1000,950]"
140+
clickable="true" visible-to-user="true">
141+
<node class="android.widget.Button" bounds="[0,900][1000,950]" visible-to-user="true" />
142+
</node>
143+
<node class="android.view.View" resource-id="non-role-child" bounds="[0,950][1000,1000]"
144+
clickable="true" visible-to-user="true">
145+
<node class="android.widget.ImageView" bounds="[0,950][1000,1000]" visible-to-user="true" />
122146
</node>
123147
</node>
124148
</hierarchy>`);
125149

126-
const { nodes } = buildUiHierarchySnapshot(tree, undefined, {});
150+
const describe = (nodes: ReturnType<typeof buildUiHierarchySnapshot>['nodes']) =>
151+
nodes
152+
.filter((node) => node.type !== 'android.widget.FrameLayout')
153+
.map((node) =>
154+
[node.type?.split('.').pop(), node.identifier ?? node.label ?? ''].join(' ').trim(),
155+
);
156+
const regular = buildUiHierarchySnapshot(tree, undefined, {}).nodes;
127157
const interactive = buildUiHierarchySnapshot(tree, undefined, { interactiveOnly: true }).nodes;
128-
const published = nodes
129-
.filter((node) => node.parentIndex !== undefined)
130-
.map((node) => `${node.type} ${node.identifier ?? ''}`.trim());
131158

132-
expect(published).toEqual([
133-
'android.widget.Button continue-with-email',
134-
'android.widget.TextView',
135-
'android.view.View nested-control-row',
136-
'android.widget.Button',
137-
'android.view.View legacy-row',
138-
'android.widget.CheckBox',
139-
'android.widget.Button',
140-
'android.view.View hidden-role-row',
159+
expect(describe(regular)).toEqual([
160+
'Button compose-container',
161+
'TextView Continue',
162+
'CheckBox compose-checkbox',
163+
'View labeled-role',
164+
'Button Details',
165+
'View identified-role',
166+
'Button inner',
167+
'View parent-role',
168+
'Button',
169+
'TextView Inside',
170+
'View nested-control',
171+
'Button',
172+
'View hidden-role',
173+
'View outside-role',
174+
'CheckBox',
175+
'View passive-parent',
176+
'Button',
177+
'LinearLayout native-parent',
178+
'Button',
179+
'View non-role-child',
180+
'ImageView',
141181
]);
142-
expect(interactive.find((node) => node.identifier === 'continue-with-email')?.type).toBe(
182+
expect(regular.find((node) => node.identifier === 'compose-checkbox')?.checked).toBe(true);
183+
expect(interactive.find((node) => node.identifier === 'compose-container')?.type).toBe(
143184
'android.widget.Button',
144185
);
145186
});

‎packages/platform-android/src/ui-hierarchy-builder.ts‎

Lines changed: 12 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ import {
2525
import { collectAndroidHiddenNodes } from './ui-hierarchy-visibility.ts';
2626
import { scopePresentedAndroidSnapshot } from './ui-hierarchy-scope.ts';
2727
import { isCollectionContainerType, shouldIncludeAndroidNode } from './ui-hierarchy-inclusion.ts';
28-
import { composeRoleNodeOf } from './ui-hierarchy-compose-role.ts';
28+
import { collectComposeRoleFolds } from './ui-hierarchy-compose-role.ts';
2929
import {
3030
createAndroidSnapshotPresentationNode,
3131
isAndroidSnapshotPresentationFailure,
@@ -73,8 +73,10 @@ type AndroidSnapshotBuildState = {
7373
interactiveDescendantMemo: Map<AndroidNode, boolean>;
7474
/** Subtrees the regular projection hides (invisible / stale window). Empty for raw. */
7575
hidden: ReadonlySet<AndroidNode>;
76-
/** Compose role nodes whose class the regular projection publishes on their parent instead. */
77-
foldedComposeRoleNodes: Set<AndroidNode>;
76+
/** Compose containers mapped to the role node whose class they publish. Empty for raw. */
77+
composeRoleFolds: ReadonlyMap<AndroidNode, AndroidNode>;
78+
/** The role nodes those containers absorb, which the regular projection omits. */
79+
foldedComposeRoleNodes: ReadonlySet<AndroidNode>;
7880
/** Largest helper-reported application/window frame, used as the regular viewport fallback. */
7981
viewport?: Rect;
8082
presentationBudget: AndroidSnapshotPresentationBudget;
@@ -109,6 +111,7 @@ export function buildUiHierarchySnapshot(
109111
// C3: raw is the acquired tree (normalization only); regular additionally hides what Android
110112
// marks invisible and stale application windows. Daemon publication alone owns occlusion.
111113
hidden: new Set(),
114+
composeRoleFolds: new Map(),
112115
foldedComposeRoleNodes: new Set(),
113116
presentationBudget,
114117
presentationNodes: [],
@@ -119,6 +122,10 @@ export function buildUiHierarchySnapshot(
119122
presentationBudget.check('work');
120123
state.viewport = resolveAndroidSnapshotViewport(tree, presentationBudget);
121124
state.hidden = options.raw ? new Set() : collectAndroidHiddenNodes(tree, presentationBudget);
125+
state.composeRoleFolds = options.raw
126+
? new Map()
127+
: collectComposeRoleFolds(tree, presentationBudget);
128+
state.foldedComposeRoleNodes = new Set(state.composeRoleFolds.values());
122129
for (const root of tree.children) {
123130
walkUiHierarchyNode(state, root, 0);
124131
if (state.truncated) break;
@@ -291,7 +298,7 @@ function appendAndroidNodeIfPresented(
291298
windowRect: Rect | undefined,
292299
systemBar: RawSnapshotNode['systemBar'],
293300
): number | undefined {
294-
if (!state.options.raw && state.foldedComposeRoleNodes.has(node)) return parentIndex;
301+
if (state.foldedComposeRoleNodes.has(node)) return parentIndex;
295302
if (
296303
!state.options.raw &&
297304
isPositiveFiniteRect(node.rect) &&
@@ -402,11 +409,9 @@ function createAndroidRawSnapshotNode(
402409
systemChrome: boolean,
403410
systemBar: RawSnapshotNode['systemBar'],
404411
): AndroidRawSnapshotNode {
405-
const composeRole = state.options.raw ? undefined : composeRoleNodeOf(node);
406-
if (composeRole) state.foldedComposeRoleNodes.add(composeRole);
407412
return {
408413
index: state.nodes.length,
409-
type: (composeRole ?? node).type ?? undefined,
414+
type: (state.composeRoleFolds.get(node) ?? node).type ?? undefined,
410415
label: node.label ?? undefined,
411416
value: node.value ?? undefined,
412417
...node.passiveFacts,
Lines changed: 25 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,12 @@
11
import { intersectArea } from '@agent-device/kernel/screenshot-geometry';
22
import type { Rect } from '@agent-device/kernel/snapshot';
3-
import { isAgentTarget, type AndroidNode } from './ui-hierarchy-node.ts';
3+
import type { AndroidSnapshotPresentationBudget } from './snapshot-presentation.ts';
4+
import { isAgentTarget, type AndroidNode, type AndroidUiHierarchy } from './ui-hierarchy-node.ts';
45

56
// Compose's `SemanticsNode.emitFakeNodes` gives a merging node that has a role and children an
67
// extra childless child, which alone carries the role's legacy class name. That child takes no
7-
// touch and covers its parent, which stays a generic `android.view.View`.
8+
// touch and lies inside its parent, which stays a generic `android.view.View`. A native
9+
// `android.view.View` is not a `ViewGroup`, so only a virtual hierarchy gives one children.
810
const COMPOSE_ROLE_CLASS_NAMES = new Set([
911
'android.widget.button',
1012
'android.widget.checkbox',
@@ -13,9 +15,24 @@ const COMPOSE_ROLE_CLASS_NAMES = new Set([
1315
'android.widget.numberpicker',
1416
]);
1517
const COMPOSE_CONTAINER_CLASS_NAME = 'android.view.view';
16-
const MIN_ROLE_NODE_COVERAGE = 0.9;
1718

18-
export function composeRoleNodeOf(node: AndroidNode): AndroidNode | undefined {
19+
/** Each Compose container the regular projection publishes with its role node's class. */
20+
export function collectComposeRoleFolds(
21+
root: AndroidUiHierarchy,
22+
presentationBudget?: AndroidSnapshotPresentationBudget,
23+
): ReadonlyMap<AndroidNode, AndroidNode> {
24+
const folds = new Map<AndroidNode, AndroidNode>();
25+
const visit = (node: AndroidNode): void => {
26+
presentationBudget?.check('work');
27+
const roleNode = composeRoleNodeOf(node);
28+
if (roleNode) folds.set(node, roleNode);
29+
for (const child of node.children) visit(child);
30+
};
31+
visit(root);
32+
return folds;
33+
}
34+
35+
function composeRoleNodeOf(node: AndroidNode): AndroidNode | undefined {
1936
if (!isAgentTarget(node) || node.type?.toLowerCase() !== COMPOSE_CONTAINER_CLASS_NAME) {
2037
return undefined;
2138
}
@@ -30,11 +47,11 @@ function isComposeRoleNode(child: AndroidNode, parent: AndroidNode): boolean {
3047
!isAgentTarget(child) &&
3148
!child.label &&
3249
!child.identifier &&
33-
coverage(child.rect, parent.rect) >= MIN_ROLE_NODE_COVERAGE
50+
liesInside(child.rect, parent.rect)
3451
);
3552
}
3653

37-
function coverage(inner: Rect | undefined, outer: Rect | undefined): number {
38-
if (!inner || !outer || outer.width <= 0 || outer.height <= 0) return 0;
39-
return intersectArea(inner, outer) / (outer.width * outer.height);
54+
function liesInside(inner: Rect | undefined, outer: Rect | undefined): boolean {
55+
if (!inner || !outer || inner.width <= 0 || inner.height <= 0) return false;
56+
return intersectArea(inner, outer) === inner.width * inner.height;
4057
}

‎src/commands/interaction/runtime/resolution-touch-point.test.ts‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,10 @@ import assert from 'node:assert/strict';
22
import { test } from 'vitest';
33
import type { Point } from '@agent-device/kernel/snapshot';
44
import { makeSnapshotState } from '@agent-device/selectors/snapshot-geometry-fixtures';
5+
import {
6+
buildUiHierarchySnapshot,
7+
parseUiHierarchyTree,
8+
} from '@agent-device/platform-android/mechanics';
59
import { createInteractionDevice } from './__tests__/test-utils/index.ts';
610
import { ref, selector } from './selector-read-utils.ts';
711

@@ -121,3 +125,36 @@ test('runtime press refuses a selector match with a zero-area rect and dispatche
121125
);
122126
assert.deepEqual(taps, []);
123127
});
128+
129+
test('runtime press on a Compose button dispatches its center only once its role node is folded', async () => {
130+
const tree = parseUiHierarchyTree(`<hierarchy>
131+
<node class="android.view.View" resource-id="compose-container" bounds="[53,1856][1027,1982]"
132+
clickable="true" focusable="true" visible-to-user="true">
133+
<node class="android.widget.TextView" text="Continue with email" bounds="[352,1895][728,1943]"
134+
visible-to-user="true" />
135+
<node class="android.widget.Button" bounds="[53,1858][1027,1980]" visible-to-user="true" />
136+
</node>
137+
</hierarchy>`);
138+
const press = async (raw: boolean) => {
139+
const taps: Point[] = [];
140+
const device = createInteractionDevice(
141+
makeSnapshotState(buildUiHierarchySnapshot(tree, undefined, { raw }).nodes),
142+
{
143+
tap: async (_context, point) => {
144+
taps.push(point);
145+
return { ok: true };
146+
},
147+
},
148+
);
149+
const failure = await device.interactions
150+
.click(selector('id="compose-container"'), { session: 'default' })
151+
.then(
152+
() => undefined,
153+
(error: { details?: { reason?: string } }) => error,
154+
);
155+
return { taps, reason: failure?.details?.reason };
156+
};
157+
158+
assert.deepEqual(await press(false), { taps: [{ x: 540, y: 1919 }], reason: undefined });
159+
assert.deepEqual(await press(true), { taps: [], reason: 'covered_by_interactive_descendants' });
160+
});

0 commit comments

Comments
 (0)