Skip to content

Commit cfa9653

Browse files
thymikeeclaude
andauthored
feat(ios): interpret React Native accessibility value semantics (#3589)
* feat(ios): interpret React Native accessibility value semantics * chore(gates): export the shared iOS value interpreter * test(snapshot): pin RN value interpretation inputs and move iOS metadata docs Co-Authored-By: Claude Opus 5.5 <[email protected]> * refactor(snapshot): omit derived iOS value comparison * fix(snapshot): compare iOS value interpretation eligibility --------- Co-authored-by: Claude Opus 5.5 <[email protected]>
1 parent 4bdeeb9 commit cfa9653

21 files changed

Lines changed: 940 additions & 36 deletions

File tree

‎contracts/fixtures/ios-react-native-value-semantics.json‎

Lines changed: 554 additions & 0 deletions
Large diffs are not rendered by default.

‎packages/capture-kit/package.json‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -317,6 +317,10 @@
317317
"./durable-capture/session-binding": {
318318
"types": "./src/durable-capture/session-binding.ts",
319319
"default": "./src/durable-capture/session-binding.ts"
320+
},
321+
"./ios-react-native-value": {
322+
"types": "./src/ios-react-native-value.ts",
323+
"default": "./src/ios-react-native-value.ts"
320324
}
321325
},
322326
"devDependencies": {
Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
import assert from 'node:assert/strict';
2+
import { readFileSync } from 'node:fs';
3+
import { test } from 'vitest';
4+
import fc from 'fast-check';
5+
import type { RawSnapshotNode } from '@agent-device/kernel/snapshot';
6+
import { interpretIosReactNativeValue } from './ios-react-native-value.ts';
7+
8+
const fixture = JSON.parse(
9+
readFileSync(
10+
new URL('../../../contracts/fixtures/ios-react-native-value-semantics.json', import.meta.url),
11+
'utf8',
12+
),
13+
) as {
14+
cases: {
15+
name: string;
16+
renderer: string;
17+
locale: string;
18+
node: RawSnapshotNode;
19+
expected: Partial<RawSnapshotNode>;
20+
}[];
21+
};
22+
for (const entry of fixture.cases) {
23+
test(`${entry.renderer}/${entry.locale}: ${entry.name}`, () => {
24+
const before = structuredClone(entry.node);
25+
assert.deepEqual(interpretIosReactNativeValue(entry.node), {
26+
...entry.node,
27+
...entry.expected,
28+
});
29+
assert.deepEqual(entry.node, before);
30+
});
31+
}
32+
33+
test('preserves arbitrary native-control values and identities', () => {
34+
fc.assert(
35+
fc.property(
36+
fc.string(),
37+
fc.constantFrom('TextField', 'Switch', 'Button', 'StaticText'),
38+
(value, type) => {
39+
const node = { index: 0, type, value };
40+
const before = structuredClone(node);
41+
assert.equal(interpretIosReactNativeValue(node), node);
42+
assert.deepEqual(node, before);
43+
},
44+
),
45+
{ numRuns: 100 },
46+
);
47+
});
48+
49+
test('preserves raw checked facts and arbitrary suffixes without mutating or reinterpreting', () => {
50+
fc.assert(
51+
fc.property(fc.boolean(), fc.string(), (checked, suffix) => {
52+
const node = {
53+
index: 0,
54+
type: 'Other',
55+
value: `checkbox, mixed, payload:${suffix}`,
56+
checked,
57+
};
58+
const before = structuredClone(node);
59+
const result = interpretIosReactNativeValue(node);
60+
assert.deepEqual(node, before);
61+
assert.equal(result.value, before.value);
62+
assert.equal(result.checked, before.checked);
63+
assert.equal(result.iosValueSemantics?.checked, 'mixed');
64+
assert.equal(interpretIosReactNativeValue(result), result);
65+
}),
66+
{ numRuns: 100 },
67+
);
68+
});
Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,96 @@
1+
import { normalizeType } from '@agent-device/contracts/snapshot';
2+
import type { IosValueSemantics, RawSnapshotNode } from '@agent-device/kernel/snapshot';
3+
4+
const RN_VIEW_CLASSES = new Set(['RCTView', 'RCTViewComponentView']);
5+
const ENGLISH_ROLES: Readonly<Record<string, string>> = {
6+
alert: 'alert',
7+
checkbox: 'checkbox',
8+
'combo box': 'combobox',
9+
menu: 'menu',
10+
'menu bar': 'menubar',
11+
'menu item': 'menuitem',
12+
'progress bar': 'progressbar',
13+
'radio button': 'radio',
14+
'radio group': 'radiogroup',
15+
'scroll bar': 'scrollbar',
16+
'spin button': 'spinbutton',
17+
switch: 'switch',
18+
tab: 'tab',
19+
'tab list': 'tablist',
20+
timer: 'timer',
21+
'tool bar': 'toolbar',
22+
};
23+
24+
type StateComponent = Pick<IosValueSemantics, 'checked' | 'expanded' | 'busy'>;
25+
const ENGLISH_STATES: Readonly<Record<string, StateComponent>> = {
26+
checked: { checked: true },
27+
unchecked: { checked: false },
28+
mixed: { checked: 'mixed' },
29+
expanded: { expanded: true },
30+
collapsed: { expanded: false },
31+
busy: { busy: true },
32+
};
33+
34+
/** Interprets RN's value grammar without removing value evidence or overriding native facts. */
35+
export function interpretIosReactNativeValue(node: RawSnapshotNode): RawSnapshotNode {
36+
if (node.iosValueSemantics || typeof node.value !== 'string') return node;
37+
if (!isInterpretationCandidate(node)) return node;
38+
const semantics = parseEnglishValue(node.value, node.role);
39+
if (!semantics || !hasSupportedSignature(node, semantics)) return node;
40+
return {
41+
...node,
42+
iosValueSemantics: semantics,
43+
...(node.checked === undefined && typeof semantics.checked === 'boolean'
44+
? { checked: semantics.checked }
45+
: {}),
46+
};
47+
}
48+
49+
function isInterpretationCandidate(node: RawSnapshotNode): boolean {
50+
if (normalizeType(node.type ?? '') !== 'other' || node.editable || node.password) return false;
51+
if (RN_VIEW_CLASSES.has(node.role ?? '')) return true;
52+
return node.role === undefined || normalizeType(node.role) === 'other';
53+
}
54+
55+
function hasSupportedSignature(node: RawSnapshotNode, semantics: IosValueSemantics): boolean {
56+
const checkableRole = semantics.role === 'checkbox' || semantics.role === 'radio';
57+
if (semantics.role !== undefined && !checkableRole && node.role !== 'RCTView') return false;
58+
if (RN_VIEW_CLASSES.has(node.role ?? '')) return true;
59+
return checkableRole && semantics.checked !== undefined;
60+
}
61+
62+
function parseEnglishValue(
63+
value: string,
64+
nativeClass: string | undefined,
65+
): IosValueSemantics | undefined {
66+
const components = value.split(', ');
67+
const first = components[0] ?? '';
68+
const role = Object.hasOwn(ENGLISH_ROLES, first) ? ENGLISH_ROLES[first] : undefined;
69+
const semantics: IosValueSemantics = {
70+
source: 'react-native-ios-value-heuristic',
71+
encoding: 'en',
72+
...(role === undefined ? {} : { role }),
73+
};
74+
let lastStateOrder = 0;
75+
for (const component of components.slice(role === undefined ? 0 : 1)) {
76+
if (!Object.hasOwn(ENGLISH_STATES, component)) break;
77+
const state = ENGLISH_STATES[component]!;
78+
if (Object.keys(state).some((key) => Object.hasOwn(semantics, key))) break;
79+
const stateOrder = fabricStateOrder(state);
80+
if (nativeClass === 'RCTViewComponentView' && stateOrder < lastStateOrder) break;
81+
lastStateOrder = stateOrder;
82+
Object.assign(semantics, state);
83+
}
84+
return semantics.role !== undefined ||
85+
semantics.checked !== undefined ||
86+
semantics.expanded !== undefined ||
87+
semantics.busy !== undefined
88+
? semantics
89+
: undefined;
90+
}
91+
92+
function fabricStateOrder(state: StateComponent): number {
93+
if (state.checked !== undefined) return 1;
94+
if (state.expanded !== undefined) return 2;
95+
return 3;
96+
}

‎packages/daemon/src/server/__tests__/response-views.test.ts‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -642,3 +642,18 @@ test('screenshot digest keeps the capture display rotation', () => {
642642
);
643643
expect(digest.displayRotation).toBe('landscape-left');
644644
});
645+
646+
test('attrs digests preserve interpreted RN mixed, expanded and busy evidence with provenance', () => {
647+
const iosValueSemantics = {
648+
source: 'react-native-ios-value-heuristic',
649+
encoding: 'en',
650+
role: 'checkbox',
651+
checked: 'mixed',
652+
expanded: false,
653+
busy: true,
654+
};
655+
for (const view of [findView, getView]) {
656+
const digest = view!({ ref: 'e7', node: { ...MATCHED_NODE, iosValueSemantics } }, 'digest');
657+
expect(digest.node).toEqual({ ...COMPACT_NODE, iosValueSemantics });
658+
}
659+
});

‎packages/daemon/src/server/response-views.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -119,6 +119,7 @@ const SELECTOR_DIGEST_NODE_FIELDS = [
119119
'enabled',
120120
'selected',
121121
'checked',
122+
'iosValueSemantics',
122123
'focused',
123124
'heading',
124125
'roleDescription',

‎packages/kernel/src/snapshot.ts‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -264,13 +264,25 @@ export type SnapshotCoveringNode = {
264264
/** Upper bound for a covering-node label carried in diagnostics; longer names are clipped. */
265265
export const SNAPSHOT_COVERING_LABEL_MAX_LENGTH = 80;
266266

267+
/** Evidence from an exact English React Native iOS value signature, not proof of its renderer. */
268+
export type IosValueSemantics = {
269+
source: 'react-native-ios-value-heuristic';
270+
encoding: 'en';
271+
role?: string;
272+
checked?: boolean | 'mixed';
273+
expanded?: boolean;
274+
busy?: true;
275+
};
276+
267277
export type RawSnapshotNode = {
268278
index: number;
269279
type?: string;
270280
role?: string;
271281
subrole?: string;
272282
label?: string;
273283
value?: string;
284+
/** Derived RN evidence; value and native type/role remain verbatim. Explicit facts take precedence. */
285+
iosValueSemantics?: IosValueSemantics;
274286
/**
275287
* Android content description when it is not already the `label`. An Android node is
276288
* labelled by its text and falls back to the content description only when it has none,

‎packages/platform-apple/src/__tests__/interactor-runner-provider.test.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -293,7 +293,7 @@ test('a message-less runner capture leaves its disclosures to the verdict', asyn
293293
});
294294

295295
test('macOS app snapshots preserve runner nodes outside the iOS presentation engine', async () => {
296-
const nodes = [{ index: 0, type: 'Application', label: 'System Settings' }];
296+
const nodes = [{ index: 0, type: 'Other', value: 'checkbox, unchecked' }];
297297
const interactor = createAppleInteractor(
298298
MACOS_DEVICE,
299299
{},

‎packages/platform-apple/src/interactor.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -250,6 +250,7 @@ async function captureAppleRunnerSnapshot(
250250
),
251251
{ backend: 'xctest' },
252252
),
253+
device,
253254
);
254255
assertSnapshotObservationResponse(options, result);
255256
assertReportedRunnerSnapshotNodes(device, options, result);

‎packages/platform-apple/src/runner/__tests__/snapshot-presentation.test.ts‎

Lines changed: 65 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
import { IOS_SIMULATOR } from '../../__tests__/device-fixtures.ts';
12
import { AppError } from '@agent-device/kernel/errors';
23
import assert from 'node:assert/strict';
34
import { test } from 'vitest';
@@ -84,17 +85,23 @@ test('a sparse capture of a presented system surface names the surface host', ()
8485
// The registry, not the wire, decides the kind: a payment host stamped with the wrong kind reads
8586
// back as `payment`, and a bundle id the registry does not know is dropped rather than surfaced.
8687
test('the wire reader takes surface kind from the registry and drops unregistered hosts', () => {
87-
const mismatched = readAppleSnapshotResult({
88-
systemSurface: { bundleId: 'com.apple.PassbookUIService', kind: 'web-auth' },
89-
});
88+
const mismatched = readAppleSnapshotResult(
89+
{
90+
systemSurface: { bundleId: 'com.apple.PassbookUIService', kind: 'web-auth' },
91+
},
92+
IOS_SIMULATOR,
93+
);
9094
assert.deepEqual(mismatched.systemSurface, {
9195
bundleId: 'com.apple.PassbookUIService',
9296
kind: 'payment',
9397
});
9498

95-
const unregistered = readAppleSnapshotResult({
96-
systemSurface: { bundleId: 'com.example.notahost', kind: 'payment' },
97-
});
99+
const unregistered = readAppleSnapshotResult(
100+
{
101+
systemSurface: { bundleId: 'com.example.notahost', kind: 'payment' },
102+
},
103+
IOS_SIMULATOR,
104+
);
98105
assert.equal(unregistered.systemSurface, undefined);
99106
});
100107

@@ -171,16 +178,22 @@ test('a presented system surface travels with an undeclared-payload refusal as t
171178
});
172179

173180
test('readAppleSnapshotResult keeps registry-known system surface provenance only', () => {
174-
const known = readAppleSnapshotResult({
175-
nodes: [],
176-
systemSurface: { bundleId: 'com.apple.SafariViewService', kind: 'web-auth' },
177-
});
181+
const known = readAppleSnapshotResult(
182+
{
183+
nodes: [],
184+
systemSurface: { bundleId: 'com.apple.SafariViewService', kind: 'web-auth' },
185+
},
186+
IOS_SIMULATOR,
187+
);
178188
assert.deepEqual(known.systemSurface, SYSTEM_SHEET);
179189

180-
const unknown = readAppleSnapshotResult({
181-
nodes: [],
182-
systemSurface: { bundleId: 'com.example.PhishingService', kind: 'web-auth' },
183-
});
190+
const unknown = readAppleSnapshotResult(
191+
{
192+
nodes: [],
193+
systemSurface: { bundleId: 'com.example.PhishingService', kind: 'web-auth' },
194+
},
195+
IOS_SIMULATOR,
196+
);
184197
assert.equal(unknown.systemSurface, undefined);
185198
});
186199

@@ -246,26 +259,35 @@ test('the reader forwards each published keyboard band shape unchanged', () => {
246259
// The landscape band #2653 confirmed on iPhone 17 Pro: the runner answers `app.keyboards` in the
247260
// app's own orientation space, so the daemon reads these numbers beside node rects unchanged.
248261
assert.deepEqual(
249-
readAppleSnapshotResult({
250-
keyboard: { kind: 'visible', frame: { x: 0, y: 198, width: 874, height: 204 } },
251-
}).keyboard,
262+
readAppleSnapshotResult(
263+
{
264+
keyboard: { kind: 'visible', frame: { x: 0, y: 198, width: 874, height: 204 } },
265+
},
266+
IOS_SIMULATOR,
267+
).keyboard,
252268
{ kind: 'visible', frame: { x: 0, y: 198, width: 874, height: 204 } },
253269
);
254-
assert.deepEqual(readAppleSnapshotResult({ keyboard: { kind: 'absent' } }).keyboard, {
255-
kind: 'absent',
256-
});
257270
assert.deepEqual(
258-
readAppleSnapshotResult({
259-
keyboard: { kind: 'unmeasurable', reason: 'keyboard-frame-query-timeout' },
260-
}).keyboard,
271+
readAppleSnapshotResult({ keyboard: { kind: 'absent' } }, IOS_SIMULATOR).keyboard,
272+
{
273+
kind: 'absent',
274+
},
275+
);
276+
assert.deepEqual(
277+
readAppleSnapshotResult(
278+
{
279+
keyboard: { kind: 'unmeasurable', reason: 'keyboard-frame-query-timeout' },
280+
},
281+
IOS_SIMULATOR,
282+
).keyboard,
261283
{ kind: 'unmeasurable', reason: 'keyboard-frame-query-timeout' },
262284
);
263285
});
264286

265287
test('a capture from a tier that never reads the keyboard publishes no fact at all', () => {
266288
// The query sweep and private-AX tiers answer with no `keyboard` key, which is how the daemon
267289
// learns to keep deriving the band from that tree instead of being told the screen is clear.
268-
assert.equal(readAppleSnapshotResult({ nodes: [] }).keyboard, undefined);
290+
assert.equal(readAppleSnapshotResult({ nodes: [] }, IOS_SIMULATOR).keyboard, undefined);
269291
});
270292

271293
// Routing proof: every reason below is emitted only by the kernel reader, never by a wire producer.
@@ -280,7 +302,7 @@ test('a malformed keyboard payload is restated by the kernel reader, never forwa
280302
];
281303

282304
for (const [payload, reason] of cases) {
283-
const read = readAppleSnapshotResult({ keyboard: payload }).keyboard;
305+
const read = readAppleSnapshotResult({ keyboard: payload }, IOS_SIMULATOR).keyboard;
284306
assert.deepEqual(read, { kind: 'unmeasurable', reason }, `payload ${JSON.stringify(payload)}`);
285307
}
286308
});
@@ -386,3 +408,21 @@ test('a raw projection publishes no viewport, because nothing validated its box
386408

387409
assert.equal('viewport' in presented, false);
388410
});
411+
412+
test('runner presented and quality nodes share RN interpretation without replacing raw evidence', () => {
413+
const node = { index: 0, type: 'Other', value: 'checkbox, unchecked' };
414+
const result = readAppleSnapshotResult(
415+
{
416+
nodes: [node],
417+
qualityPayload: { nodes: [node], truncated: false },
418+
},
419+
IOS_SIMULATOR,
420+
);
421+
for (const interpreted of [result.nodes?.[0], result.qualityPayload?.nodes[0]]) {
422+
assert.equal(interpreted?.checked, false);
423+
assert.equal(interpreted?.value, node.value);
424+
assert.equal(interpreted?.type, node.type);
425+
assert.equal(interpreted?.iosValueSemantics?.role, 'checkbox');
426+
}
427+
assert.equal('checked' in node, false);
428+
});

0 commit comments

Comments
 (0)