Skip to content

Commit ee2017d

Browse files
thymikeeclaude
andcommitted
feat(snapshot): acquire native checked, password, and editable states on every mobile producer
Native switches, checkboxes, and secure fields report the same tri-state checked/password/editable facts on the iOS runner, AX bridge, Appium iOS, limrun, and Android Appium; absence never becomes false. Appium adds heading and roleDescription, and Android Appium reads editable/password only as reported instead of inferring editable from enabled. Rules live in contracts/fixtures/native-snapshot-states.json. Closes #3524 Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent 971d24d commit ee2017d

32 files changed

Lines changed: 809 additions & 155 deletions

‎apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerApp.m‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -573,6 +573,24 @@ - (void)viewDidLoad {
573573
self.alertActivationBusyAnswer = busyAnswer;
574574
}
575575

576+
if ([NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-native-states"]) {
577+
for (NSUInteger index = 0; index < 3; index++) {
578+
UITextField *field = [[UITextField alloc] initWithFrame:CGRectMake(40, 120 + index * 60, 260, 44)];
579+
field.accessibilityIdentifier = @[@"native-field", @"native-secure", @"native-disabled"][index];
580+
field.placeholder = @"Native input";
581+
field.secureTextEntry = index == 1;
582+
field.enabled = index != 2;
583+
field.borderStyle = UITextBorderStyleRoundedRect;
584+
[self.view addSubview:field];
585+
}
586+
for (NSUInteger index = 0; index < 2; index++) {
587+
UISwitch *toggle = [[UISwitch alloc] initWithFrame:CGRectMake(40, 340 + index * 60, 60, 40)];
588+
toggle.accessibilityIdentifier = index == 0 ? @"native-off" : @"native-on";
589+
toggle.on = index == 1;
590+
[self.view addSubview:toggle];
591+
}
592+
}
593+
576594
if ([NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-text-entry-regression"]) {
577595
BOOL digitCountValue =
578596
[NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-text-entry-digit-count-value"];

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -270,6 +270,7 @@ extension RunnerTests {
270270
let identifier = element.identifier.trimmingCharacters(in: .whitespacesAndNewlines)
271271
let valueText = String(describing: element.value ?? "")
272272
.trimmingCharacters(in: .whitespacesAndNewlines)
273+
273274
let node = SnapshotPresentation.singleElementRead(
274275
RawAXNode(
275276
index: 0,

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+PrivateAXPresentation.swift‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -67,8 +67,9 @@ extension RunnerTests {
6767
private func privateAXNode(_ fields: PrivateAXFields, index: Int, depth: Int,
6868
parentIndex: Int?) -> RawAXNode
6969
{
70+
let type = fields.elementType.map(elementTypeName) ?? "Element(\(fields.rawType))"
7071
return RawAXNode(index: index,
71-
type: fields.elementType.map(elementTypeName) ?? "Element(\(fields.rawType))",
72+
type: type,
7273
label: fields.label.isEmpty ? nil : fields.label,
7374
identifier: fields.identifier.isEmpty ? nil : fields.identifier,
7475
value: fields.value.isEmpty ? nil : fields.value,

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift‎

Lines changed: 3 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -99,6 +99,8 @@ extension RunnerTests {
9999
XCUIElement.ElementType.activityIndicator.rawValue: "ActivityIndicator",
100100
XCUIElement.ElementType.progressIndicator.rawValue: "ProgressIndicator",
101101
XCUIElement.ElementType.checkBox.rawValue: "CheckBox",
102+
XCUIElement.ElementType.radioButton.rawValue: "RadioButton",
103+
XCUIElement.ElementType.toggle.rawValue: "Toggle",
102104
XCUIElement.ElementType.menuItem.rawValue: "MenuItem",
103105
XCUIElement.ElementType.webView.rawValue: "WebView",
104106
XCUIElement.ElementType.other.rawValue: "Other",
@@ -497,23 +499,7 @@ extension RunnerTests {
497499
nodes[0] = interactiveRootNode(rect: rootRect)
498500
for candidate in candidates {
499501
nodes.append(
500-
RawAXNode(
501-
index: nodes.count,
502-
type: candidate.type,
503-
label: candidate.label,
504-
identifier: candidate.identifier,
505-
value: candidate.value,
506-
placeholder: candidate.placeholder,
507-
rect: candidate.rect,
508-
enabled: candidate.enabled,
509-
focused: candidate.focused,
510-
selected: candidate.selected,
511-
hittable: candidate.hittable,
512-
depth: 1,
513-
parentIndex: 0,
514-
hiddenContentAbove: nil,
515-
hiddenContentBelow: nil
516-
)
502+
candidate.reindexed(index: nodes.count, depth: 1, parentIndex: 0)
517503
)
518504
}
519505
return (

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotAcquisition.swift‎

Lines changed: 1 addition & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -357,23 +357,7 @@ extension RunnerTests {
357357
if uniqueCandidates.count < 2 { return [] }
358358

359359
return uniqueCandidates.enumerated().map { offset, node in
360-
RawAXNode(
361-
index: startingIndex + offset,
362-
type: node.type,
363-
label: node.label,
364-
identifier: node.identifier,
365-
value: node.value,
366-
placeholder: node.placeholder,
367-
rect: node.rect,
368-
enabled: node.enabled,
369-
focused: node.focused,
370-
selected: node.selected,
371-
hittable: node.hittable,
372-
depth: depth,
373-
parentIndex: parentIndex,
374-
hiddenContentAbove: nil,
375-
hiddenContentBelow: nil
376-
)
360+
node.reindexed(index: startingIndex + offset, depth: depth, parentIndex: parentIndex)
377361
}
378362
}
379363

@@ -592,7 +576,6 @@ extension RunnerTests {
592576
let valueText = snapshotValueText(element)
593577
let elementType = element.elementType
594578
let enabled = element.isEnabled
595-
596579
node = RawAXNode(
597580
index: index,
598581
type: elementTypeName(elementType),

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SystemModal.swift‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,14 @@ extension RunnerTests {
1111
guard case .resolved(let resolvedModal) = resolveBlockingSystemModal(deadline: deadline) else {
1212
return nil
1313
}
14+
return snapshotForBlockingSystemModal(resolvedModal, deadline: deadline)
15+
#endif
16+
}
17+
18+
func snapshotForBlockingSystemModal(
19+
_ resolvedModal: ResolvedBlockingSystemModal,
20+
deadline: Date = .distantFuture
21+
) -> DataPayload? {
1422
let modal = resolvedModal.root
1523
let actions = resolvedModal.actions
1624

@@ -62,7 +70,6 @@ extension RunnerTests {
6270
}
6371

6472
return DataPayload(nodes: nodes, truncated: false)
65-
#endif
6673
}
6774

6875
func firstBlockingSystemModal(
@@ -217,13 +224,17 @@ extension RunnerTests {
217224
) -> PresentedNode {
218225
let label = (labelOverride ?? element.label).trimmingCharacters(in: .whitespacesAndNewlines)
219226
let identifier = (identifierOverride ?? element.identifier).trimmingCharacters(in: .whitespacesAndNewlines)
227+
let states = SnapshotNativeStates(type: type, value: element.value.map { String(describing: $0) })
220228
return SnapshotPresentation.singleElementRead(
221229
RawAXNode(
222230
index: index,
223231
type: type,
224232
label: label.isEmpty ? nil : label,
225233
identifier: identifier.isEmpty ? nil : identifier,
226234
value: nil,
235+
checked: states.checked,
236+
password: states.password,
237+
editable: states.editable,
227238
rect: SnapshotRect(element.frame),
228239
enabled: element.isEnabled,
229240
focused: nil,

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+InteractionTests.swift‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,29 @@ import AgentDeviceSnapshotPresentation
33

44
#if AGENT_DEVICE_RUNNER_UNIT_TESTS
55
extension RunnerTests {
6+
#if os(iOS)
7+
@MainActor
8+
func testQueryElementPublishesNativeControlStates() throws {
9+
app.launchArguments = ["--agent-device-native-states"]
10+
app.launch()
11+
defer { app.terminate() }
12+
let expected: [(String, Bool?, Bool?, Bool?)] = [
13+
("native-off", false, nil, nil),
14+
("native-on", true, nil, nil),
15+
("native-secure", nil, true, true),
16+
("native-disabled", nil, false, true),
17+
]
18+
for (identifier, checked, password, editable) in expected {
19+
XCTAssertTrue(app.descendants(matching: .any)[identifier].waitForExistence(timeout: appExistenceTimeout), identifier)
20+
let response = queryElement(app: app, selectorKey: "id", selectorValue: identifier)
21+
let node = try XCTUnwrap(response.data?.nodes?.first)
22+
XCTAssertEqual(node.checked, checked, identifier)
23+
XCTAssertEqual(node.password, password, identifier)
24+
XCTAssertEqual(node.editable, editable, identifier)
25+
}
26+
}
27+
#endif
28+
629
func testDesktopScrollWheelDeltasMapDirections() {
730
XCTAssertEqual(desktopScrollWheelDeltas(direction: .up, pixels: 120).vertical, 120)
831
XCTAssertEqual(desktopScrollWheelDeltas(direction: .down, pixels: 120).vertical, -120)

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+PrivateAXPresentationTests.swift‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,22 @@ import AgentDeviceSnapshotPresentation
33

44
#if AGENT_DEVICE_RUNNER_UNIT_TESTS
55
extension RunnerTests {
6+
func testPrivateAXAcquiresNativeStatesThroughNativeTypeMapping() throws {
7+
let hint = CaptureHint(projection: .raw, depth: nil, regularPresentedDepth: nil, interactiveOnly: false, customActions: false)
8+
for type in [XCUIElement.ElementType.switch, .checkBox, .radioButton, .toggle] {
9+
for value in ["0", "1", "unknown"] {
10+
let node = try XCTUnwrap(privateAXAcquisition(rawRoot: ["type": Int(type.rawValue), "value": value], hint: hint).first)
11+
XCTAssertEqual(node.checked, value == "unknown" ? nil : value == "1", "\(type): \(value)")
12+
}
13+
}
14+
for type in [XCUIElement.ElementType.textField, .secureTextField, .textView, .searchField] {
15+
let node = try XCTUnwrap(privateAXAcquisition(rawRoot: ["type": Int(type.rawValue), "enabled": false], hint: hint).first)
16+
XCTAssertEqual(node.editable, true)
17+
XCTAssertEqual(node.password, type == .secureTextField)
18+
XCTAssertFalse(node.enabled)
19+
}
20+
}
21+
622
fileprivate static func privateAXFrame(
723
_ x: Double, _ y: Double, _ width: Double, _ height: Double
824
) -> [String: Any] {
Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
import XCTest
2+
import AgentDeviceSnapshotPresentation
3+
4+
extension RunnerTests {
5+
#if AGENT_DEVICE_RUNNER_UNIT_TESTS && os(iOS)
6+
@MainActor
7+
func testNativeStatesInTreeAndFlatAcquisition() throws {
8+
app.launchArguments = ["--agent-device-native-states"]
9+
app.launch()
10+
defer { app.terminate() }
11+
let expected: [(String, Bool?, Bool?, Bool?, Bool)] = [
12+
("native-off", false, nil, nil, true),
13+
("native-on", true, nil, nil, true),
14+
("native-field", nil, false, true, true),
15+
("native-secure", nil, true, true, true),
16+
("native-disabled", nil, false, true, false),
17+
]
18+
for (identifier, checked, password, editable, enabled) in expected {
19+
let element = app.descendants(matching: .any)[identifier]
20+
XCTAssertTrue(element.waitForExistence(timeout: appExistenceTimeout), identifier)
21+
let snapshot = try element.snapshot()
22+
let tree = makeSnapshotNode(snapshot: snapshot, evaluation: evaluateSnapshot(snapshot), depth: 1, index: 1, parentIndex: 0)
23+
let flat = try XCTUnwrap(flatSnapshotNode(element: element, index: 1, parentIndex: 0))
24+
for node in [tree, flat] {
25+
XCTAssertEqual(node.checked, checked, identifier)
26+
XCTAssertEqual(node.password, password, identifier)
27+
XCTAssertEqual(node.editable, editable, identifier)
28+
XCTAssertEqual(node.enabled, enabled, identifier)
29+
}
30+
}
31+
}
32+
#endif
33+
}
Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
import XCTest
2+
import AgentDeviceSnapshotPresentation
3+
4+
extension RunnerTests {
5+
#if AGENT_DEVICE_RUNNER_UNIT_TESTS && os(iOS)
6+
@MainActor
7+
func testBlockingModalSnapshotPublishesNativeSwitchStates() throws {
8+
app.launchArguments = ["--agent-device-native-states"]
9+
app.launch()
10+
defer { app.terminate() }
11+
XCTAssertTrue(app.switches["native-off"].waitForExistence(timeout: appExistenceTimeout))
12+
let resolved = ResolvedBlockingSystemModal(root: app, ownerApp: app, actions: actionableElements(in: app))
13+
let payload = try XCTUnwrap(snapshotForBlockingSystemModal(resolved))
14+
let off = try XCTUnwrap(payload.nodes?.first { $0.identifier == "native-off" })
15+
let on = try XCTUnwrap(payload.nodes?.first { $0.identifier == "native-on" })
16+
XCTAssertEqual(off.checked, false)
17+
XCTAssertEqual(on.checked, true)
18+
XCTAssertNil(off.password)
19+
XCTAssertNil(off.editable)
20+
}
21+
#endif
22+
}

0 commit comments

Comments
 (0)