Skip to content

Commit d2d4d35

Browse files
committed
fix(android): carry the checked state of a checkable control on the snapshot node
The snapshot helper never serialized `checked` or `checkable`, and the host reads only the helper's XML, so no later layer could recover it: a switch, checkbox, or radio button looked the same on or off, `get attrs` had no `checked` field, and snapshot text showed the control as plain. The helper now writes `checked` on every node Android reports as checkable, with both answers, so an unchecked switch reads `false`, a node that cannot be checked reads nothing, and so does a helper older than the attribute. The parser, the Android hierarchy node, and the published snapshot node carry it to `get attrs`, the unchanged-snapshot comparison, the selector digest, and the `[checked]` and `[unchecked]` markers in snapshot text. Both answers render because the diff compares the state and a checkable control rendered as plain would hide that it toggles.
1 parent a363a02 commit d2d4d35

17 files changed

Lines changed: 231 additions & 25 deletions

File tree

‎CHANGELOG.md‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,15 @@
22

33
## Unreleased
44

5+
- Fixed (android): snapshot nodes and `get attrs` carry the `checked` state of a checkable control.
6+
The helper never serialized `checked` or `checkable`, and the host reads only the helper's XML,
7+
so a switch, checkbox, or radio button looked the same on or off: `get attrs` had no `checked`
8+
field and snapshot text showed the control as plain. The helper now writes `checked` on every
9+
node Android reports as checkable, with both answers, so an unchecked switch reads `false`, a
10+
node that cannot be checked reads nothing, and so does a helper older than the attribute. The
11+
parser, the Android hierarchy node, and the published snapshot node carry it through to
12+
`get attrs` and to the `[checked]` and `[unchecked]` markers in snapshot text; `--settle` and
13+
`diff` compare it, so a toggle tap reads as a changed pair.
514
- Fixed (ios): `perf cpu profile report --kind xctrace` on Xcode 27 no longer fails with
615
`Apple xctrace CPU report contained no samples` on a trace that holds thousands of samples. Xcode
716
27 exports each `time-profile` sample stack as `<tagged-backtrace>` instead of `<backtrace>`, and

‎android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/AccessibilityTreeXml.java‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,12 @@ static void appendNode(
6464
// unselected control reports selected=false and a helper older than this attribute reports
6565
// nothing at all.
6666
appendAttribute(xml, "selected", Boolean.toString(node.isSelected()));
67+
// Present only on a checkable control, with both answers: an unchecked switch reports
68+
// checked=false, while a node that cannot be checked reports nothing, like a helper older than
69+
// this attribute.
70+
if (node.isCheckable()) {
71+
appendAttribute(xml, "checked", Boolean.toString(node.isChecked()));
72+
}
6773
boolean scrollable = node.isScrollable();
6874
if (scrollable) {
6975
appendAttribute(xml, "scrollable", "true");
@@ -146,9 +152,8 @@ private static void appendTrueAttribute(StringBuilder xml, String name, boolean
146152
}
147153
}
148154

149-
// Declared residue (agent-device #1832): checked / checkable / long-clickable are not serialized,
150-
// so toggle state is invisible to agents. Adding them is a helper protocol change (new attributes
151-
// + host parser + fields on the wire node), tracked there.
155+
// Declared residue (agent-device #1832): long-clickable is not serialized. Adding it is a helper
156+
// protocol change (new attribute + host parser + field on the wire node).
152157
private static void appendDrawingOrderAttribute(StringBuilder xml, AccessibilityNodeInfo node) {
153158
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.N) {
154159
appendAttribute(xml, "drawing-order", Integer.toString(node.getDrawingOrder()));
Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
import assert from 'node:assert/strict';
2+
import { test } from 'vitest';
3+
import type { SnapshotNode } from '@agent-device/kernel/snapshot';
4+
import { buildSnapshotDiff } from '../snapshot-diff.ts';
5+
6+
function toggle(checked?: boolean): SnapshotNode {
7+
return {
8+
ref: 'e12',
9+
index: 0,
10+
depth: 0,
11+
type: 'android.widget.Switch',
12+
label: 'Wi-Fi switch',
13+
enabled: true,
14+
hittable: true,
15+
...(checked === undefined ? {} : { checked }),
16+
};
17+
}
18+
19+
test('a checked-only flip diffs as changed lines that read differently', () => {
20+
const diff = buildSnapshotDiff([toggle(false)], [toggle(true)]);
21+
const changed = diff.lines.filter((line) => line.kind !== 'unchanged');
22+
23+
assert.equal(diff.summary.unchanged, 0);
24+
assert.equal(changed.length > 0, true);
25+
// The comparable key carries the checked state, so the rendered line has to as well, and both
26+
// answers render: the pair reads `[unchecked]` -> `[checked]`, not two identical lines.
27+
assert.match(changed[0]!.text, /\[unchecked\]/);
28+
assert.match(changed.at(-1)!.text, /\[checked\]/);
29+
});
30+
31+
test.each([
32+
['both unchecked', [toggle(false)], [toggle(false)]],
33+
['both unreported', [toggle()], [toggle()]],
34+
])('a still toggle with %s diffs as unchanged', (_label, previous, current) => {
35+
const diff = buildSnapshotDiff(previous, current);
36+
37+
assert.equal(
38+
diff.lines.every((line) => line.kind === 'unchanged'),
39+
true,
40+
);
41+
});
42+
43+
test('a node that cannot be checked never reads as unchecked', () => {
44+
const diff = buildSnapshotDiff([toggle()], [toggle()]);
45+
assert.equal(
46+
diff.lines.every((line) => !line.text.includes('checked')),
47+
true,
48+
);
49+
});

‎packages/capture-kit/src/snapshot/snapshot-diff.ts‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import {
55
displayLabel,
66
formatRole,
77
formatSnapshotLine,
8+
stateMarkers,
89
} from './snapshot-lines.ts';
910

1011
export type SnapshotDiffResult = {
@@ -27,18 +28,17 @@ type SnapshotComparableLine = {
2728
function snapshotNodeToComparableLine(node: SnapshotNode, depthOverride?: number): string {
2829
const role = formatRole(node.type ?? 'Element');
2930
const textPart = displayLabel(node, role);
30-
const enabledPart = node.enabled === false ? 'disabled' : 'enabled';
31-
const selectedPart = node.selected === true ? 'selected' : 'unselected';
31+
// The states the rendered line prints are the states the key compares, from one function, so a
32+
// fact the diff weighs is always visible in the line it prints.
33+
const statePart = stateMarkers(node).join(',');
3234
const hittablePart = node.hittable === true ? 'hittable' : 'not-hittable';
3335
const depthPart = String(depthOverride ?? node.depth ?? 0);
3436
// The rendered line carries the actions list, so the comparable key has to as
3537
// well: otherwise an action-only change diffs as 'unchanged' while its text
3638
// silently differs from the baseline's. JSON-encoded because the names are
3739
// app-authored and may contain the field separator.
3840
const actionsPart = node.actions ? JSON.stringify(node.actions) : '';
39-
return [depthPart, role, textPart, enabledPart, selectedPart, hittablePart, actionsPart].join(
40-
'|',
41-
);
41+
return [depthPart, role, textPart, statePart, hittablePart, actionsPart].join('|');
4242
}
4343

4444
export function buildSnapshotDiff(

‎packages/capture-kit/src/snapshot/snapshot-freshness/android.ts‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -28,11 +28,12 @@ export function isNavigationSensitiveAction(command: string): boolean {
2828

2929
/**
3030
* Route signature of an Android snapshot, from the fields the Android backend actually carries.
31-
* The helper serializes no `role`, `checked` or `long-clickable` (declared residue,
32-
* #1832), so a signature keying on them would compare constants and claim discrimination it does
33-
* not have. `selected` is left out as a judgement call rather than an inability: a tab-bar tap
34-
* flips it on two nodes, which the 90%-identical threshold below absorbs at every tree size this
35-
* check runs on, so keying on it would only add capture retries.
31+
* The helper serializes no `role` or `long-clickable` (declared residue, #1832), so a signature
32+
* keying on them would compare constants and claim discrimination it does not have. `selected`
33+
* and `checked` are left out as a judgement call rather than an inability: a tab-bar tap flips
34+
* selection on two nodes and a toggle tap flips one checked state, which the 90%-identical
35+
* threshold below absorbs at every tree size this check runs on, so keying on them would only add
36+
* capture retries.
3637
*/
3738
export function buildSnapshotSignatures(nodes: SnapshotState['nodes']): string[] {
3839
return nodes.map((node) =>

‎packages/capture-kit/src/snapshot/snapshot-lines.ts‎

Lines changed: 16 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -195,6 +195,21 @@ export function formatRole(type: string): string {
195195
return lookupRoleLabel(normalized) || normalized || 'element';
196196
}
197197

198+
/**
199+
* The state markers every rendering path prints, and the states a snapshot diff compares: the diff
200+
* renders its lines without text-surface summarizing, and a fact it weighs has to be visible in the
201+
* line it prints, or a flip reads as a changed pair whose two lines look identical. Both checked
202+
* answers render, since a checkable control shown plain would hide that it toggles; a node that
203+
* cannot be checked carries neither.
204+
*/
205+
export function stateMarkers(node: SnapshotNode): string[] {
206+
const markers: string[] = [];
207+
if (node.enabled === false) markers.push('disabled');
208+
if (node.selected === true) markers.push('selected');
209+
if (node.checked !== undefined) markers.push(node.checked ? 'checked' : 'unchecked');
210+
return markers;
211+
}
212+
198213
function lookupRoleLabel(normalized: string): string | undefined {
199214
return Object.prototype.hasOwnProperty.call(ROLE_LABELS, normalized)
200215
? ROLE_LABELS[normalized]
@@ -238,13 +253,7 @@ function buildLineMetadata(
238253
options: SnapshotLineFormatOptions,
239254
textSurface: { text: string; isLargeSurface: boolean; shouldSummarize: boolean },
240255
): string[] {
241-
const metadata: string[] = [];
242-
if (node.enabled === false) metadata.push('disabled');
243-
// Selection is a state a snapshot diff can report as changed, and the diff renders its lines
244-
// without text-surface summarizing. A fact the diff compares has to be visible in the line it
245-
// prints, or a selection flip reads as a changed pair whose two lines look identical.
246-
if (node.selected === true) metadata.push('selected');
247-
metadata.push(...(node.presentationHints ?? []));
256+
const metadata = [...stateMarkers(node), ...(node.presentationHints ?? [])];
248257
if (!options.summarizeTextSurfaces) {
249258
return uniqueMetadata(metadata);
250259
}

‎packages/kernel/src/snapshot.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -230,6 +230,8 @@ export type RawSnapshotNode = {
230230
rect?: Rect;
231231
enabled?: boolean;
232232
selected?: boolean;
233+
/** Checked state of a checkable control (switch, checkbox, radio); absent means not checkable or unavailable. */
234+
checked?: boolean;
233235
focused?: boolean;
234236
/** Native accessibility facts; absent means unavailable, not false. */
235237
editable?: boolean;
Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
import { expect, test } from 'vitest';
2+
import { buildUiHierarchySnapshot, parseUiHierarchyTree } from '../ui-hierarchy.ts';
3+
4+
// A settings row: a switch the helper reports as checkable beside a label that is not. The helper
5+
// writes `checked` only on the checkable node, with both answers.
6+
function togglesXml(wifiChecked: boolean): string {
7+
return `<hierarchy>
8+
<node class="android.widget.LinearLayout" resource-id="toggles" bounds="[0,0][400,200]"
9+
window-index="0" window-type="1" window-layer="1" window-active="true" window-focused="true"
10+
window-bounds="[0,0][400,800]" visible-to-user="true" enabled="true">
11+
<node class="android.widget.TextView" resource-id="wifi-label" text="Wi-Fi"
12+
bounds="[0,0][200,100]" enabled="true" visible-to-user="true" />
13+
<node class="android.widget.Switch" resource-id="wifi-switch" content-desc="Wi-Fi switch"
14+
bounds="[200,0][400,100]" clickable="true" enabled="true" visible-to-user="true"
15+
checked="${wifiChecked}" />
16+
<node class="android.widget.RadioButton" resource-id="size-small" text="Small"
17+
bounds="[0,100][400,200]" clickable="true" enabled="true" visible-to-user="true"
18+
checked="false" />
19+
</node>
20+
</hierarchy>`;
21+
}
22+
23+
// A helper older than the `checked` attribute, or a switch it did not report as checkable.
24+
const UNREPORTED_CHECKED_XML =
25+
'<hierarchy><node class="android.widget.Switch" resource-id="legacy-switch" content-desc="Wi-Fi"' +
26+
' bounds="[200,0][400,100]" clickable="true" enabled="true" visible-to-user="true" /></hierarchy>';
27+
28+
function toggleNodes(raw: boolean, interactiveOnly = false, wifiChecked = true) {
29+
const { nodes } = buildUiHierarchySnapshot(
30+
parseUiHierarchyTree(togglesXml(wifiChecked)),
31+
undefined,
32+
{ raw, interactiveOnly },
33+
);
34+
const byId = (identifier: string) => nodes.find((node) => node.identifier === identifier);
35+
return { label: byId('wifi-label'), wifi: byId('wifi-switch'), small: byId('size-small') };
36+
}
37+
38+
test.each([
39+
{ raw: false, interactiveOnly: false },
40+
{ raw: false, interactiveOnly: true },
41+
{ raw: true, interactiveOnly: false },
42+
{ raw: true, interactiveOnly: true },
43+
])(
44+
'checked state reaches snapshot nodes in every projection (raw=$raw, -i=$interactiveOnly)',
45+
({ raw, interactiveOnly }) => {
46+
const { wifi, small } = toggleNodes(raw, interactiveOnly);
47+
expect(wifi?.checked).toBe(true);
48+
expect(small?.checked).toBe(false);
49+
},
50+
);
51+
52+
test.each([true, false])('the switch answers the state the helper observed (%s)', (wifiChecked) => {
53+
expect(toggleNodes(false, false, wifiChecked).wifi?.checked).toBe(wifiChecked);
54+
});
55+
56+
test('attrs answer explicit false where a node that cannot be checked answers nothing', () => {
57+
// Serialized, because that is the answer an agent reads: an unavailable fact drops the key
58+
// while JSON encodes an observed `false`.
59+
const { label, small } = toggleNodes(false);
60+
expect(JSON.parse(JSON.stringify(small)).checked).toBe(false);
61+
expect(JSON.parse(JSON.stringify(label))).not.toHaveProperty('checked');
62+
});
63+
64+
test('an unreported checked state stays unknown instead of becoming false', () => {
65+
const node = buildUiHierarchySnapshot(parseUiHierarchyTree(UNREPORTED_CHECKED_XML), undefined, {
66+
raw: false,
67+
}).nodes.find((node) => node.identifier === 'legacy-switch');
68+
expect(node).toBeDefined();
69+
expect(node?.checked).toBeUndefined();
70+
});

‎packages/platform-android/src/ui-hierarchy-builder.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -353,6 +353,7 @@ function createAndroidRawSnapshotNode(
353353
enabled: node.enabled,
354354
focused: node.focused,
355355
selected: node.selected,
356+
checked: node.checked,
356357
editable: node.editable,
357358
password: node.password,
358359
hintShowing: node.hintShowing,

‎packages/platform-android/src/ui-hierarchy-node.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ export type AndroidUiHierarchy = {
1515
visibleToUser?: boolean;
1616
focused?: boolean;
1717
selected?: boolean;
18+
checked?: boolean;
1819
editable?: boolean;
1920
password?: boolean;
2021
hintShowing?: boolean;

0 commit comments

Comments
 (0)