Skip to content

Commit 2315981

Browse files
committed
fix(android): resolve snapshot scope once and disclose the API 23 occlusion-scan gap (#1832 C1/C2)
- Android resolves --scope inside its projection only, under the shared scope specification (matchesSnapshotScope in @agent-device/contracts/snapshot: first document-order match over label/value/identifier, empty on no match). The daemon post-wire scopeSnapshotNodes pass skips the android backend, so scope has one owner and one no-match semantics instead of BFS+fallback followed by document-order+empty. - Golden table contracts/fixtures/snapshot-scope-policy.json is asserted against the predicate, the Android projection, and the daemon pass; the Swift runner twin (#1797) consumes the same table. - androidSnapshot.occlusionScanUnavailable discloses helper trees without drawing-order (API 23), where the covered-sibling pruner cannot run. Disclosure only; C1 stays open until occlusion moves to the daemon annotator.
1 parent f03c030 commit 2315981

14 files changed

Lines changed: 423 additions & 43 deletions

File tree

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
[
2+
{
3+
"name": "first document-order match wins over a shallower later sibling (breadth-first would pick index 3)",
4+
"scope": "Settings",
5+
"nodes": [
6+
{ "depth": 0, "type": "FrameLayout" },
7+
{ "depth": 1, "type": "ViewGroup", "identifier": "content" },
8+
{ "depth": 2, "type": "TextView", "label": "Settings" },
9+
{ "depth": 1, "type": "ViewGroup", "identifier": "settings-panel" },
10+
{ "depth": 2, "type": "Button", "label": "Save" }
11+
],
12+
"expectedRootIndex": 2,
13+
"expectedSubtreeIndexes": [2]
14+
},
15+
{
16+
"name": "container match keeps its whole subtree",
17+
"scope": "settings-panel",
18+
"nodes": [
19+
{ "depth": 0, "type": "FrameLayout" },
20+
{ "depth": 1, "type": "TextView", "label": "Title" },
21+
{ "depth": 1, "type": "ViewGroup", "identifier": "settings-panel" },
22+
{ "depth": 2, "type": "Button", "label": "Save" },
23+
{ "depth": 3, "type": "TextView", "label": "Save label" },
24+
{ "depth": 1, "type": "Button", "label": "Cancel" }
25+
],
26+
"expectedRootIndex": 2,
27+
"expectedSubtreeIndexes": [2, 3, 4]
28+
},
29+
{
30+
"name": "case-insensitive substring over the label",
31+
"scope": "sett",
32+
"nodes": [
33+
{ "depth": 0, "type": "FrameLayout" },
34+
{ "depth": 1, "type": "Button", "label": "Open SETTINGS" }
35+
],
36+
"expectedRootIndex": 1,
37+
"expectedSubtreeIndexes": [1]
38+
},
39+
{
40+
"name": "value matches (EditText text)",
41+
"scope": "hello",
42+
"nodes": [
43+
{ "depth": 0, "type": "FrameLayout" },
44+
{ "depth": 1, "type": "EditText", "label": "Message", "value": "Hello world" }
45+
],
46+
"expectedRootIndex": 1,
47+
"expectedSubtreeIndexes": [1]
48+
},
49+
{
50+
"name": "identifier matches (resource-id / testID), including generic android ids",
51+
"scope": "id/toolbar",
52+
"nodes": [
53+
{ "depth": 0, "type": "FrameLayout" },
54+
{ "depth": 1, "type": "Toolbar", "identifier": "com.example:id/toolbar" },
55+
{ "depth": 2, "type": "TextView", "label": "Inbox" }
56+
],
57+
"expectedRootIndex": 1,
58+
"expectedSubtreeIndexes": [1, 2]
59+
},
60+
{
61+
"name": "type never matches",
62+
"scope": "Button",
63+
"nodes": [
64+
{ "depth": 0, "type": "FrameLayout" },
65+
{ "depth": 1, "type": "Button", "label": "OK" }
66+
],
67+
"expectedRootIndex": null,
68+
"expectedSubtreeIndexes": []
69+
},
70+
{
71+
"name": "no match yields an empty snapshot, never the full tree",
72+
"scope": "zzzz-no-match-token",
73+
"nodes": [
74+
{ "depth": 0, "type": "FrameLayout", "label": "Root" },
75+
{ "depth": 1, "type": "Button", "label": "Search" }
76+
],
77+
"expectedRootIndex": null,
78+
"expectedSubtreeIndexes": []
79+
},
80+
{
81+
"name": "root itself may be the scope root",
82+
"scope": "Root",
83+
"nodes": [
84+
{ "depth": 0, "type": "FrameLayout", "label": "Root" },
85+
{ "depth": 1, "type": "Button", "label": "Search" }
86+
],
87+
"expectedRootIndex": 0,
88+
"expectedSubtreeIndexes": [0, 1]
89+
}
90+
]

‎packages/contracts/src/facades/snapshot.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
export { isScrollableNodeLike, isScrollableType } from '../snapshot-scroll.ts';
2+
export { matchesSnapshotScope, type SnapshotScopeCandidate } from '../snapshot-scope.ts';
23
export {
34
buildSnapshotNodeMap,
45
findNearestAncestor,
Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
import { test } from 'vitest';
2+
import assert from 'node:assert/strict';
3+
import fs from 'node:fs';
4+
import path from 'node:path';
5+
import { matchesSnapshotScope } from './facades/snapshot.ts';
6+
7+
// Golden scope-policy table (#1797 / #1832 C2): the SAME JSON is asserted against every runtime
8+
// that resolves `--scope` — this predicate, the Android platform projection
9+
// (src/platforms/android/__tests__/ui-hierarchy-scope.test.ts), the daemon's post-wire pass
10+
// (src/snapshot/snapshot-desktop-surface.test.ts) and the Swift runner twin once #1797 lands it.
11+
// This file proves the PREDICATE + first-document-order-match rule; subtree slicing is proved by
12+
// the runtime legs.
13+
14+
type ScopePolicyNode = { depth: number; label?: string; value?: string; identifier?: string };
15+
type ScopePolicyCase = {
16+
name: string;
17+
scope: string;
18+
nodes: ScopePolicyNode[];
19+
expectedRootIndex: number | null;
20+
expectedSubtreeIndexes: number[];
21+
};
22+
23+
const TABLE_PATH = path.resolve(
24+
import.meta.dirname,
25+
'..',
26+
'..',
27+
'..',
28+
'contracts',
29+
'fixtures',
30+
'snapshot-scope-policy.json',
31+
);
32+
33+
test('the shared scope predicate picks the golden table root as the first document-order match', () => {
34+
const cases = JSON.parse(fs.readFileSync(TABLE_PATH, 'utf8')) as ScopePolicyCase[];
35+
assert.ok(cases.length > 0, 'scope policy table must not be empty');
36+
assert.equal(new Set(cases.map((fixture) => fixture.name)).size, cases.length);
37+
for (const fixture of cases) {
38+
const rootIndex = fixture.nodes.findIndex((node) => matchesSnapshotScope(node, fixture.scope));
39+
assert.equal(rootIndex === -1 ? null : rootIndex, fixture.expectedRootIndex, fixture.name);
40+
assert.deepEqual(
41+
subtreeIndexes(fixture.nodes, fixture.expectedRootIndex),
42+
fixture.expectedSubtreeIndexes,
43+
`${fixture.name}: expectedSubtreeIndexes must be the root's document-order subtree`,
44+
);
45+
}
46+
});
47+
48+
/** The table's own consistency: subtree = root + following nodes deeper than the root. */
49+
function subtreeIndexes(nodes: ScopePolicyNode[], rootIndex: number | null): number[] {
50+
if (rootIndex === null) return [];
51+
const rootDepth = nodes[rootIndex]!.depth;
52+
const indexes = [rootIndex];
53+
for (let index = rootIndex + 1; index < nodes.length; index += 1) {
54+
if (nodes[index]!.depth <= rootDepth) break;
55+
indexes.push(index);
56+
}
57+
return indexes;
58+
}
Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
/**
2+
* Snapshot scope specification (`snapshot --scope <text>`), one rule for every presentation
3+
* runtime (#1797 / #1832):
4+
*
5+
* - a node matches when its label, value, or identifier contains the scope text,
6+
* case-insensitively;
7+
* - the scope root is the FIRST match in document (pre-)order;
8+
* - the scoped snapshot is that root's subtree, re-rooted at depth 0;
9+
* - no match yields an EMPTY snapshot — never the full tree.
10+
*
11+
* The golden table `contracts/fixtures/snapshot-scope-policy.json` pins the rule; every runtime
12+
* that resolves scope (Android platform projection, the daemon's post-wire pass, the Swift runner
13+
* twin) is asserted against the same table so drift turns CI red on whichever side changed.
14+
*/
15+
export type SnapshotScopeCandidate = {
16+
label?: string | null;
17+
value?: string | null;
18+
identifier?: string | null;
19+
};
20+
21+
export function matchesSnapshotScope(node: SnapshotScopeCandidate, scope: string): boolean {
22+
const query = scope.toLowerCase();
23+
return [node.label, node.value, node.identifier].some((field) =>
24+
(field ?? '').toLowerCase().includes(query),
25+
);
26+
}

‎packages/contracts/src/snapshot-types.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,5 +59,7 @@ export type AndroidSnapshotBackendMetadata = {
5959
nodeCount?: number;
6060
helperTruncated?: boolean;
6161
elapsedMs?: number;
62+
/** API 23 helper output carries no `drawing-order`; covered same-window surfaces are not pruned. */
63+
occlusionScanUnavailable?: boolean;
6264
};
6365
export type FindLocator = 'any' | 'text' | 'label' | 'value' | 'role' | 'id';

‎src/daemon/handlers/snapshot-capture.ts‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -177,7 +177,7 @@ export function buildSnapshotState(
177177
? presentIosInteractiveSnapshot(normalizedNodes)
178178
: normalizedNodes;
179179
const scopedNodes =
180-
flags?.snapshotScope && data?.backend !== 'macos-helper'
180+
flags?.snapshotScope && backendScopesAfterWire(data?.backend)
181181
? scopeSnapshotNodes(presentableNodes, flags.snapshotScope)
182182
: presentableNodes;
183183
const snapshotQuality = snapshotCaptureAnnotationsFrom(data).quality;
@@ -218,6 +218,16 @@ function snapshotCaptureFlagsForBackend(
218218
return { ...flags, snapshotScope: undefined };
219219
}
220220

221+
/**
222+
* Scope resolves once per snapshot. Android resolves it inside its projection (the platform
223+
* matcher implements the shared scope specification, `@agent-device/contracts/snapshot`), and the
224+
* macOS helper scopes at capture; a second pass here would re-match inside an already-scoped tree
225+
* and hand the two layers different no-match semantics (#1832 C2).
226+
*/
227+
function backendScopesAfterWire(backend: SnapshotBackend | undefined): boolean {
228+
return backend !== 'macos-helper' && backend !== 'android';
229+
}
230+
221231
function shouldPresentIosInteractiveSnapshot(
222232
backend: SnapshotBackend | undefined,
223233
flags:

‎src/platforms/android/__tests__/snapshot.test.ts‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -584,6 +584,30 @@ test('snapshotAndroid reports helper-side truncation on the public snapshot resu
584584
assert.equal(result.androidSnapshot.helperTruncated, true);
585585
});
586586

587+
test('snapshotAndroid discloses an unavailable occlusion scan when the helper tree carries no drawing-order (API 23)', async () => {
588+
const captured: string[] = [];
589+
const helperAdb: AndroidAdbExecutor = async (args) => {
590+
if (args.includes('--show-versioncode')) return installedHelperProbe;
591+
if (args.includes('instrument')) {
592+
const xml = captured.shift();
593+
if (!xml) throw new Error('unexpected extra capture');
594+
return { exitCode: 0, stdout: helperOutput(xml), stderr: '' };
595+
}
596+
throw new Error(`unexpected helper adb args: ${args.join(' ')}`);
597+
};
598+
const button = (drawingOrder: string) =>
599+
`<node class="android.widget.Button" text="Go" bounds="[0,0][100,40]" clickable="true" visible-to-user="true"${drawingOrder} />`;
600+
601+
captured.push(`<hierarchy>${button('')}</hierarchy>`);
602+
const api23 = await snapshotAndroid(device, { helperAdb, helperArtifact });
603+
assert.equal(api23.androidSnapshot.occlusionScanUnavailable, true);
604+
605+
captured.push(`<hierarchy>${button(' drawing-order="1"')}</hierarchy>`);
606+
const api24 = await snapshotAndroid(device, { helperAdb, helperArtifact });
607+
assert.equal(api24.androidSnapshot.occlusionScanUnavailable, undefined);
608+
assert.equal('occlusionScanUnavailable' in api24.androidSnapshot, false);
609+
});
610+
587611
test('snapshotAndroid emits helper phase diagnostics', async () => {
588612
const helperAdb: AndroidAdbExecutor = async (args) => {
589613
if (args.includes('--show-versioncode')) {
Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,100 @@
1+
import { test } from 'vitest';
2+
import assert from 'node:assert/strict';
3+
import fs from 'node:fs';
4+
import path from 'node:path';
5+
import { buildSnapshotState } from '../../../daemon/handlers/snapshot-capture.ts';
6+
import { parseUiHierarchy } from '../ui-hierarchy.ts';
7+
8+
// Android's scope leg of the golden table (#1832 C2). Android resolves `--scope` exactly once,
9+
// inside its projection; the daemon's post-wire pass skips the android backend. Both facts are
10+
// pinned here: the projection agrees with contracts/fixtures/snapshot-scope-policy.json, and a
11+
// scoped Android snapshot survives buildSnapshotState untouched.
12+
13+
type ScopePolicyCase = {
14+
name: string;
15+
scope: string;
16+
nodes: Array<{ depth: number; label?: string; value?: string; identifier?: string }>;
17+
expectedSubtreeIndexes: number[];
18+
};
19+
20+
const TABLE_PATH = path.resolve(
21+
import.meta.dirname,
22+
'../../../../contracts/fixtures/snapshot-scope-policy.json',
23+
);
24+
25+
/**
26+
* Renders the flat golden list as helper XML. Fixture position rides in the bounds' x origin so
27+
* the assertion never depends on which text field carried the match. Android reads
28+
* `label = text || content-desc` and `value = text`, so a fixture value goes to `text` and a
29+
* label with no value to `content-desc`.
30+
*/
31+
function scopePolicyXml(nodes: ScopePolicyCase['nodes']): string {
32+
const lines: string[] = ['<hierarchy>'];
33+
const open: number[] = [];
34+
nodes.forEach((node, index) => {
35+
while (open.length > node.depth) {
36+
open.pop();
37+
lines.push('</node>');
38+
}
39+
const attrs = [
40+
`class="android.view.View"`,
41+
`bounds="[${index},0][${index + 1},1]"`,
42+
`visible-to-user="true"`,
43+
node.value !== undefined ? `text="${node.value}"` : '',
44+
node.label !== undefined && node.value === undefined ? `content-desc="${node.label}"` : '',
45+
node.identifier !== undefined ? `resource-id="${node.identifier}"` : '',
46+
].filter(Boolean);
47+
lines.push(`<node ${attrs.join(' ')}>`);
48+
open.push(index);
49+
});
50+
while (open.length > 0) {
51+
open.pop();
52+
lines.push('</node>');
53+
}
54+
lines.push('</hierarchy>');
55+
return lines.join('\n');
56+
}
57+
58+
test('the Android projection agrees with every golden scope-policy table case', () => {
59+
const cases = JSON.parse(fs.readFileSync(TABLE_PATH, 'utf8')) as ScopePolicyCase[];
60+
assert.ok(cases.length > 0);
61+
for (const fixture of cases) {
62+
const result = parseUiHierarchy(scopePolicyXml(fixture.nodes), undefined, {
63+
raw: true,
64+
scope: fixture.scope,
65+
});
66+
assert.deepEqual(
67+
result.nodes.map((node) => node.rect?.x),
68+
fixture.expectedSubtreeIndexes,
69+
fixture.name,
70+
);
71+
if (result.nodes.length > 0) assert.equal(result.nodes[0]?.depth, 0, fixture.name);
72+
}
73+
});
74+
75+
test('a scoped Android snapshot is not re-scoped after the wire', () => {
76+
// The projection already re-rooted at "Panel"; a post-wire pass with the same scope would look
77+
// for "Panel" again inside a tree whose root is the panel's first CHILD if membership dropped the
78+
// container. Android's presented nodes must pass through buildSnapshotState unchanged.
79+
const nodes = [
80+
{ index: 0, depth: 0, type: 'android.widget.Button', label: 'Save', hittable: true },
81+
{ index: 1, depth: 0, type: 'android.widget.Button', label: 'Cancel', hittable: true },
82+
];
83+
const state = buildSnapshotState({ nodes, backend: 'android' }, { snapshotScope: 'Panel' });
84+
assert.deepEqual(
85+
state.nodes.map((node) => node.label),
86+
['Save', 'Cancel'],
87+
);
88+
});
89+
90+
test('an Android scope with no match yields an empty snapshot in every projection', () => {
91+
const xml = `<hierarchy>
92+
<node class="android.widget.FrameLayout" bounds="[0,0][390,844]" visible-to-user="true">
93+
<node class="android.widget.Button" text="Search" bounds="[0,0][390,60]" clickable="true" visible-to-user="true"/>
94+
</node>
95+
</hierarchy>`;
96+
for (const options of [{}, { raw: true }, { interactiveOnly: true }]) {
97+
const result = parseUiHierarchy(xml, undefined, { ...options, scope: 'zzzz-no-match-token' });
98+
assert.deepEqual(result.nodes, [], JSON.stringify(options));
99+
}
100+
});

‎src/platforms/android/snapshot-types.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,4 +23,6 @@ export type AndroidSnapshotBackendMetadata = {
2323
nodeCount?: number;
2424
helperTruncated?: boolean;
2525
elapsedMs?: number;
26+
/** API 23 helper output carries no `drawing-order`; covered same-window surfaces are not pruned. */
27+
occlusionScanUnavailable?: boolean;
2628
};

0 commit comments

Comments
 (0)