Skip to content

Commit ce8afdf

Browse files
fix(apple): type a text-entry-trait element as a text input in the simulator AX tree
A Compose Multiplatform text field reports the automation type of a plain view, so the bridge typed it Other and left it non-editable while the XCTest tree types the same element TextView. Derive the type from the text-entry, secure-text-field, and text-area traits, as XCTest does.
1 parent b396bd0 commit ce8afdf

4 files changed

Lines changed: 74 additions & 2 deletions

File tree

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
---
2+
'agent-device': patch
3+
---
4+
5+
Type a custom-drawn iOS text input, such as a Compose Multiplatform text field, as `TextField`, `SecureTextField`, `TextView`, or `SearchField` in the Simulator AX snapshot, from its text-entry traits, the way the XCTest snapshot does. The input now reports `editable: true` instead of reading as a non-editable `Other`.

‎docs/adr/0004-ios-snapshot-backend-strategy.md‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -745,6 +745,12 @@ XCTest types an element with that trait as `Button`.
745745
The decoder does the same.
746746
Otherwise, a `role=button` tab selector that matches the runner tree does not match the bridge tree.
747747

748+
A custom-drawn text input, such as a Compose Multiplatform text field, reports the automation type `Other`.
749+
It carries the private text-entry trait (bit 18), and also the secure-text-field trait (bit 24) or the text-area trait (bit 47) where they apply.
750+
XCTest types such an element as `TextField`, `SecureTextField`, or `TextView`.
751+
The decoder does the same for an `Other`, and types a search-field-trait (bit 10) input as `SearchField`.
752+
Otherwise, the bridge tree reports the input as not editable, and a `fill` client refuses it.
753+
748754
The snapshot API reports the frames of a windowed app relative to its window, not to the screen.
749755
Windowed Apps and Stage Manager both put an app in such a window.
750756
A tap at a published point then misses the app.

‎packages/platform-apple/src/snapshot-source/tree.test.ts‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -437,6 +437,46 @@ test('the bridge tree types a tab-button-trait element as Button, matching the X
437437
);
438438
});
439439

440+
test('the bridge tree types a text-entry-trait element as a text input, matching the XCTest tree', () => {
441+
// Real guest word from a Compose Multiplatform 1.12 BasicTextField on iOS 27: automation type
442+
// Other, with the text-entry (bit 18) and text-area (bit 47) traits. The XCTest tree types it
443+
// TextView and reports it editable.
444+
const composeFieldTraits = 140746078552064n;
445+
const field = (type: number, word: bigint) =>
446+
decodeSnapshotBridgeTree(
447+
{
448+
[application]: 'Application',
449+
[children]: [
450+
{
451+
[application]: 'AccessibilityElement',
452+
[baseType]: 'UIAccessibilityElement',
453+
[automationType]: type,
454+
[label]: 'Name',
455+
[frame]: { X: 24, Y: 391, Width: 354, Height: 48 },
456+
[traits]: word.toString(),
457+
[children]: [],
458+
},
459+
],
460+
},
461+
{ truncated: false },
462+
limits,
463+
).nodes[1];
464+
465+
assert.deepEqual(
466+
[field(0, composeFieldTraits)?.type, field(0, composeFieldTraits)?.editable],
467+
['TextView', true],
468+
);
469+
assert.equal(field(0, composeFieldTraits & ~(1n << 47n))?.type, 'TextField');
470+
assert.equal(field(0, composeFieldTraits | (1n << 24n))?.type, 'SecureTextField');
471+
assert.equal(field(0, composeFieldTraits | (1n << 10n))?.type, 'SearchField');
472+
assert.equal(
473+
field(0, composeFieldTraits & ~(1n << 18n))?.type,
474+
'Other',
475+
'without the text-entry trait the automation type decides',
476+
);
477+
assert.equal(field(45, composeFieldTraits)?.type, 'SearchField', 'a declared input type is kept');
478+
});
479+
440480
test('the bridge tree reads keyboard focus from the is-editing trait, matching the XCTest tree', () => {
441481
const editor = (word?: unknown) => ({
442482
[automationType]: 57,

‎packages/platform-apple/src/snapshot-source/tree.ts‎

Lines changed: 23 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -148,6 +148,18 @@ const KEYBOARD_FOCUS_TRAIT = 1n << 21n;
148148
*/
149149
const TAB_BUTTON_TRAIT = 1n << 28n;
150150

151+
/**
152+
* The traits UIKit sets on a text input: the private text entry (bit 18), secure text field (bit 24),
153+
* and text area (bit 47), and the public search field (bit 10). A custom-drawn input, such as a
154+
* Compose Multiplatform text field, reports the automation type of a plain view (`Other`), and
155+
* XCTest types it from these bits, so the runner path answers `TextField`, `SecureTextField`,
156+
* `TextView`, or `SearchField` for it. The bridge derives the type from them for an `Other` to match.
157+
*/
158+
const SEARCH_FIELD_TRAIT = 1n << 10n;
159+
const TEXT_ENTRY_TRAIT = 1n << 18n;
160+
const SECURE_TEXT_FIELD_TRAIT = 1n << 24n;
161+
const TEXT_AREA_TRAIT = 1n << 47n;
162+
151163
/**
152164
* Two hosts put another process's UI into this one's tree as an `AXRemoteElement`, with the
153165
* children in that other process:
@@ -312,8 +324,17 @@ function elementTypeName(
312324
if (elementClass !== undefined && ELEMENT_TYPE_NAMES.includes(elementClass)) {
313325
return elementClass;
314326
}
315-
if (automationType === undefined) return undefined;
316-
return ELEMENT_TYPE_NAMES[automationType] ?? 'Other';
327+
const automationTypeName =
328+
automationType === undefined ? undefined : (ELEMENT_TYPE_NAMES[automationType] ?? 'Other');
329+
if (automationTypeName !== undefined && automationTypeName !== 'Other') return automationTypeName;
330+
return textEntryTypeName(traits) ?? automationTypeName;
331+
}
332+
333+
function textEntryTypeName(traits: bigint | undefined): string | undefined {
334+
if (traits === undefined || (traits & TEXT_ENTRY_TRAIT) === 0n) return undefined;
335+
if ((traits & SECURE_TEXT_FIELD_TRAIT) !== 0n) return 'SecureTextField';
336+
if ((traits & SEARCH_FIELD_TRAIT) !== 0n) return 'SearchField';
337+
return (traits & TEXT_AREA_TRAIT) !== 0n ? 'TextView' : 'TextField';
317338
}
318339

319340
function isRemoteContentHost(node: RawSnapshotNode): boolean {

0 commit comments

Comments
 (0)