Skip to content

Commit f3a3e26

Browse files
committed
test(android): harden snapshot presentation seams
1 parent f6ae913 commit f3a3e26

3 files changed

Lines changed: 176 additions & 53 deletions

File tree

Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,107 @@
1+
import type { Rect } from '@agent-device/kernel/snapshot';
2+
import fc from 'fast-check';
3+
import { expect, test } from 'vitest';
4+
import { PROPERTY_RUNS } from '../../../__tests__/test-utils/property-arbitraries.ts';
5+
import {
6+
createAndroidSnapshotPresentationNode,
7+
effectiveAndroidRect,
8+
serializeAndroidRegularPresentationNode,
9+
} from '../snapshot-presentation.ts';
10+
import { parseUiHierarchy } from './ui-hierarchy-fixtures.ts';
11+
12+
const rectArb = fc.record({
13+
x: fc.integer({ min: -500, max: 500 }),
14+
y: fc.integer({ min: -500, max: 500 }),
15+
width: fc.integer({ min: -20, max: 500 }),
16+
height: fc.integer({ min: -20, max: 500 }),
17+
});
18+
19+
const positiveRectArb = fc.record({
20+
x: fc.integer({ min: -500, max: 500 }),
21+
y: fc.integer({ min: -500, max: 500 }),
22+
width: fc.integer({ min: 1, max: 500 }),
23+
height: fc.integer({ min: 1, max: 500 }),
24+
});
25+
26+
test('regular Android snapshots publish cumulative effective geometry while raw keeps reported bounds', () => {
27+
const xml = `<hierarchy>
28+
<node class="android.widget.FrameLayout" bounds="[0,0][400,800]" window-bounds="[0,0][400,800]" visible-to-user="true">
29+
<node class="android.widget.ScrollView" bounds="[0,100][300,500]" scrollable="true" visible-to-user="true">
30+
<node class="android.widget.ScrollView" bounds="[50,150][250,350]" scrollable="true" visible-to-user="true">
31+
<node class="android.widget.Button" text="Partially visible" bounds="[200,300][320,380]" clickable="true" visible-to-user="true" />
32+
<node class="android.widget.Button" text="Outside nested scroll" bounds="[260,360][380,420]" clickable="true" visible-to-user="true" />
33+
</node>
34+
</node>
35+
</node>
36+
</hierarchy>`;
37+
38+
const regular = parseUiHierarchy(xml, 800, {});
39+
const raw = parseUiHierarchy(xml, 800, { raw: true });
40+
41+
expect(regular.nodes.find((node) => node.label === 'Partially visible')).toMatchObject({
42+
rect: { x: 200, y: 300, width: 50, height: 50 },
43+
hittable: true,
44+
});
45+
expect(regular.nodes.find((node) => node.label === 'Outside nested scroll')).toMatchObject({
46+
rect: { x: 260, y: 360, width: 0, height: 0 },
47+
hittable: undefined,
48+
});
49+
expect(raw.nodes.find((node) => node.label === 'Partially visible')).toMatchObject({
50+
rect: { x: 200, y: 300, width: 120, height: 80 },
51+
hittable: true,
52+
});
53+
expect(raw.nodes.find((node) => node.label === 'Outside nested scroll')).toMatchObject({
54+
rect: { x: 260, y: 360, width: 120, height: 60 },
55+
hittable: true,
56+
});
57+
});
58+
59+
test('regular Android snapshots clip descendants to their owning window', () => {
60+
const xml = `<hierarchy>
61+
<node class="android.widget.FrameLayout" bounds="[0,0][400,800]" window-bounds="[0,0][400,800]" window-index="0" window-type="1" window-layer="1" window-active="true" window-focused="true" visible-to-user="true">
62+
<node class="android.widget.Button" text="Main action" bounds="[20,20][180,80]" clickable="true" visible-to-user="true" />
63+
</node>
64+
<node class="android.widget.FrameLayout" bounds="[100,100][300,300]" window-bounds="[100,100][300,300]" window-index="1" window-type="2" window-layer="2" window-active="true" window-focused="true" visible-to-user="true">
65+
<node class="android.widget.Button" text="Outside dialog" bounds="[120,320][180,380]" clickable="true" visible-to-user="true" />
66+
</node>
67+
</hierarchy>`;
68+
69+
const regular = parseUiHierarchy(xml, 800, {});
70+
const dialogAction = regular.nodes.find((node) => node.label === 'Outside dialog');
71+
72+
expect(dialogAction).toMatchObject({
73+
rect: { x: 120, y: 320, width: 0, height: 0 },
74+
hittable: undefined,
75+
});
76+
});
77+
78+
test('property: effective Android geometry stays inside every positive clip and never upgrades actionability', () => {
79+
fc.assert(
80+
fc.property(rectArb, positiveRectArb, positiveRectArb, (reported, viewport, ancestorClip) => {
81+
const effective = effectiveAndroidRect(reported, viewport, ancestorClip);
82+
const raw = { index: 0, depth: 0, rect: reported, hittable: true as const };
83+
const carrier = createAndroidSnapshotPresentationNode(raw, effective);
84+
const regular = serializeAndroidRegularPresentationNode(carrier);
85+
86+
expect(carrier.raw.rect).toEqual(reported);
87+
expect(regular.rect).toEqual(effective);
88+
if (effective && effective.width > 0 && effective.height > 0) {
89+
expect(rectContains(viewport, effective)).toBe(true);
90+
expect(rectContains(ancestorClip, effective)).toBe(true);
91+
expect(regular.hittable).toBe(true);
92+
} else {
93+
expect(regular.hittable).toBeUndefined();
94+
}
95+
}),
96+
{ numRuns: PROPERTY_RUNS },
97+
);
98+
});
99+
100+
function rectContains(outer: Rect, inner: Rect): boolean {
101+
return (
102+
inner.x >= outer.x &&
103+
inner.y >= outer.y &&
104+
inner.x + inner.width <= outer.x + Math.max(0, outer.width) &&
105+
inner.y + inner.height <= outer.y + Math.max(0, outer.height)
106+
);
107+
}
Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
import { expect, test } from 'vitest';
2+
import { shouldIncludeAndroidNode } from '../ui-hierarchy-inclusion.ts';
3+
import type { AndroidNode } from '../ui-hierarchy-node.ts';
4+
5+
test('interactive inclusion keeps positive agent targets and rejects degenerate geometry', () => {
6+
expect(
7+
shouldIncludeAndroidNode(
8+
node({ clickable: true }),
9+
{ interactiveOnly: true },
10+
false,
11+
false,
12+
false,
13+
),
14+
).toBe(true);
15+
expect(
16+
shouldIncludeAndroidNode(
17+
node({ clickable: true, rect: { x: 10, y: 10, width: 0, height: 20 } }),
18+
{ interactiveOnly: true },
19+
false,
20+
false,
21+
false,
22+
),
23+
).toBe(false);
24+
});
25+
26+
test('interactive inclusion keeps a semantic Compose proxy only in actionable context', () => {
27+
const semanticView = node({ type: 'android.view.View', label: 'Start a call' });
28+
expect(
29+
shouldIncludeAndroidNode(semanticView, { interactiveOnly: true }, true, false, false),
30+
).toBe(true);
31+
expect(
32+
shouldIncludeAndroidNode(semanticView, { interactiveOnly: true }, false, false, false),
33+
).toBe(false);
34+
});
35+
36+
test('regular inclusion preserves meaningful structural nodes and actionable descendants', () => {
37+
expect(
38+
shouldIncludeAndroidNode(
39+
node({ type: 'android.view.View', label: 'Panel' }),
40+
{},
41+
false,
42+
false,
43+
false,
44+
),
45+
).toBe(true);
46+
expect(
47+
shouldIncludeAndroidNode(node({ type: 'android.view.View' }), {}, false, true, false),
48+
).toBe(true);
49+
expect(
50+
shouldIncludeAndroidNode(node({ type: 'android.view.View' }), {}, false, false, false),
51+
).toBe(false);
52+
});
53+
54+
function node(overrides: Partial<AndroidNode>): AndroidNode {
55+
return {
56+
type: 'android.widget.Button',
57+
label: null,
58+
value: null,
59+
identifier: null,
60+
packageName: null,
61+
rect: { x: 0, y: 0, width: 20, height: 20 },
62+
clickable: false,
63+
focusable: false,
64+
depth: 0,
65+
children: [],
66+
...overrides,
67+
};
68+
}

‎src/platforms/android/__tests__/ui-hierarchy.test.ts‎

Lines changed: 1 addition & 53 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import { expect, test } from 'vitest';
1+
import { test } from 'vitest';
22
import assert from 'node:assert/strict';
33
import { buildSnapshotState } from '../../../daemon/snapshot-state.ts';
44
import { isNodeVisibleOnScreen } from '@agent-device/contracts/snapshot';
@@ -345,58 +345,6 @@ test('raw Android snapshots are the acquired tree: invisible, stale-window and c
345345
assert.deepEqual(labels({ interactiveOnly: true }), ['Foreground action']);
346346
});
347347

348-
test('regular Android snapshots publish cumulative effective geometry while raw keeps reported bounds', () => {
349-
const xml = `<hierarchy>
350-
<node class="android.widget.FrameLayout" bounds="[0,0][400,800]" window-bounds="[0,0][400,800]" visible-to-user="true">
351-
<node class="android.widget.ScrollView" bounds="[0,100][300,500]" scrollable="true" visible-to-user="true">
352-
<node class="android.widget.ScrollView" bounds="[50,150][250,350]" scrollable="true" visible-to-user="true">
353-
<node class="android.widget.Button" text="Partially visible" bounds="[200,300][320,380]" clickable="true" visible-to-user="true" />
354-
<node class="android.widget.Button" text="Outside nested scroll" bounds="[260,360][380,420]" clickable="true" visible-to-user="true" />
355-
</node>
356-
</node>
357-
</node>
358-
</hierarchy>`;
359-
360-
const regular = parseUiHierarchy(xml, 800, {});
361-
const raw = parseUiHierarchy(xml, 800, { raw: true });
362-
363-
expect(regular.nodes.find((node) => node.label === 'Partially visible')).toMatchObject({
364-
rect: { x: 200, y: 300, width: 50, height: 50 },
365-
hittable: true,
366-
});
367-
expect(regular.nodes.find((node) => node.label === 'Outside nested scroll')).toMatchObject({
368-
rect: { x: 260, y: 360, width: 0, height: 0 },
369-
});
370-
expect(
371-
regular.nodes.find((node) => node.label === 'Outside nested scroll')?.hittable,
372-
).toBeUndefined();
373-
expect(raw.nodes.find((node) => node.label === 'Partially visible')).toMatchObject({
374-
rect: { x: 200, y: 300, width: 120, height: 80 },
375-
hittable: true,
376-
});
377-
expect(raw.nodes.find((node) => node.label === 'Outside nested scroll')).toMatchObject({
378-
rect: { x: 260, y: 360, width: 120, height: 60 },
379-
hittable: true,
380-
});
381-
});
382-
383-
test('regular Android snapshots clip descendants to their owning window', () => {
384-
const xml = `<hierarchy>
385-
<node class="android.widget.FrameLayout" bounds="[0,0][400,800]" window-bounds="[0,0][400,800]" window-index="0" window-type="1" window-layer="1" window-active="true" window-focused="true" visible-to-user="true">
386-
<node class="android.widget.Button" text="Main action" bounds="[20,20][180,80]" clickable="true" visible-to-user="true" />
387-
</node>
388-
<node class="android.widget.FrameLayout" bounds="[100,100][300,300]" window-bounds="[100,100][300,300]" window-index="1" window-type="2" window-layer="2" window-active="true" window-focused="true" visible-to-user="true">
389-
<node class="android.widget.Button" text="Outside dialog" bounds="[120,320][180,380]" clickable="true" visible-to-user="true" />
390-
</node>
391-
</hierarchy>`;
392-
393-
const regular = parseUiHierarchy(xml, 800, {});
394-
const dialogAction = regular.nodes.find((node) => node.label === 'Outside dialog');
395-
396-
expect(dialogAction).toMatchObject({ rect: { x: 120, y: 320, width: 0, height: 0 } });
397-
expect(dialogAction?.hittable).toBeUndefined();
398-
});
399-
400348
test('parseUiHierarchy prunes descendants of Android nodes that are not visible to the user', () => {
401349
const xml = `<hierarchy>
402350
<node class="android.widget.FrameLayout" bounds="[0,0][390,844]" enabled="true" visible-to-user="true">

0 commit comments

Comments
 (0)