Skip to content

Commit 80aa8df

Browse files
committed
feat(ios): the snapshot node carries the field's placeholder
Android nodes carry the field's hint as placeholder since #2927; iOS nodes carried nothing, so a consumer could neither query a field by its placeholder nor tell an empty field (XCTest reports the placeholder as its value) from one holding that text. XCTest exposes placeholderValue on every element and snapshot, and the AX server exposes XC_kAXXCAttributePlaceholderValue. Every iOS producer now publishes placeholder on RawAXNode and PresentedNode: the XCTest tree and query sweeps read placeholderValue off the snapshot or element, the runner's private-AX reader asks for the keypath beside value, and the Simulator AX bridge requests the attribute (source v1.8.0, one recompile per host). An empty placeholder reads as none. The docs name the fact for both platforms.
1 parent c31cc49 commit 80aa8df

17 files changed

Lines changed: 92 additions & 8 deletions

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAXSnapshotBridge.m‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -365,6 +365,7 @@ + (NSArray *)snapshotAttributes
365365
@"identifier",
366366
@"label",
367367
@"value",
368+
@"placeholderValue",
368369
@"frame",
369370
@"enabled",
370371
@"selected",
@@ -388,8 +389,8 @@ + (NSArray *)snapshotAttributes
388389
// The mapper expands keypaths with extra attributes (automation type, window display
389390
// id, base type) that are disproportionately expensive for the AX server to compute
390391
// on large React Native trees. Keep only the attributes we actually consume.
391-
NSArray *needed = @[ @"ElementType", @"Identifier", @"Label", @"Value", @"Frame",
392-
@"Enabled", @"Selected", @"Focus" ];
392+
NSArray *needed = @[ @"ElementType", @"Identifier", @"Label", @"Value", @"PlaceholderValue",
393+
@"Frame", @"Enabled", @"Selected", @"Focus" ];
393394
NSMutableArray *filtered = [NSMutableArray array];
394395
for (id attribute in (NSArray *)mapped) {
395396
NSString *name = [attribute description];
@@ -764,6 +765,7 @@ + (nullable NSMutableDictionary *)dictionaryForSnapshot:(id)snapshot
764765
result[@"identifier"] = [self stringValueForKey:@"identifier" snapshot:snapshot] ?: @"";
765766
result[@"label"] = [self stringValueForKey:@"label" snapshot:snapshot] ?: @"";
766767
result[@"value"] = [self stringValueForKey:@"value" snapshot:snapshot] ?: @"";
768+
result[@"placeholder"] = [self stringValueForKey:@"placeholderValue" snapshot:snapshot] ?: @"";
767769
result[@"frame"] = [self frameValueForSnapshot:snapshot];
768770
result[@"enabled"] = [self boolNumberForKey:@"enabled" snapshot:snapshot defaultValue:YES];
769771
result[@"selected"] = [self boolNumberForKey:@"selected" snapshot:snapshot defaultValue:NO];

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+PrivateAXPresentation.swift‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ struct PrivateAXFields {
88
let label: String
99
let identifier: String
1010
let value: String
11+
let placeholder: String
1112
let rawType: Int
1213
let elementType: XCUIElement.ElementType?
1314
let enabled: Bool
@@ -52,6 +53,7 @@ extension RunnerTests {
5253
label: privateAXPresentationString(raw["label"]),
5354
identifier: privateAXPresentationString(raw["identifier"]),
5455
value: privateAXPresentationString(raw["value"]),
56+
placeholder: privateAXPresentationString(raw["placeholder"]),
5557
rawType: rawType,
5658
elementType: privateAXElementType(rawElementType: rawType),
5759
enabled: privateAXPresentationBool(raw["enabled"]) ?? true,
@@ -70,6 +72,7 @@ extension RunnerTests {
7072
label: fields.label.isEmpty ? nil : fields.label,
7173
identifier: fields.identifier.isEmpty ? nil : fields.identifier,
7274
value: fields.value.isEmpty ? nil : fields.value,
75+
placeholder: fields.placeholder.isEmpty ? nil : fields.placeholder,
7376
rect: SnapshotRect(fields.rect), enabled: fields.enabled,
7477
focused: fields.focused, selected: fields.selected,
7578
hittable: false,

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ extension RunnerTests {
3535
let label: String
3636
let identifier: String
3737
let valueText: String?
38+
let placeholder: String?
3839
let focused: Bool
3940
let selected: Bool
4041
}
@@ -500,6 +501,7 @@ extension RunnerTests {
500501
label: candidate.label,
501502
identifier: candidate.identifier,
502503
value: candidate.value,
504+
placeholder: candidate.placeholder,
503505
rect: candidate.rect,
504506
enabled: candidate.enabled,
505507
focused: candidate.focused,

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotAcquisition.swift‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -189,6 +189,7 @@ extension RunnerTests {
189189
label: label,
190190
identifier: identifier,
191191
valueText: valueText,
192+
placeholder: placeholderText(snapshot.placeholderValue),
192193
focused: snapshotHasFocus(snapshot),
193194
selected: snapshotIsSelected(snapshot)
194195
)
@@ -209,6 +210,7 @@ extension RunnerTests {
209210
label: evaluation.label.isEmpty ? nil : evaluation.label,
210211
identifier: evaluation.identifier.isEmpty ? nil : evaluation.identifier,
211212
value: evaluation.valueText,
213+
placeholder: evaluation.placeholder,
212214
rect: SnapshotRect(snapshot.frame),
213215
enabled: snapshot.isEnabled,
214216
focused: evaluation.focused ? true : nil,
@@ -235,6 +237,14 @@ extension RunnerTests {
235237
return text.isEmpty ? nil : text
236238
}
237239

240+
/// The placeholder as the node publishes it: XCTest answers `placeholderValue` for a text field
241+
/// whether or not the field is empty, and an empty string for everything else, which reads as
242+
/// no placeholder. The private-AX bridge asks the AX server the same attribute.
243+
func placeholderText(_ placeholderValue: String?) -> String? {
244+
let text = placeholderValue?.trimmingCharacters(in: .whitespacesAndNewlines) ?? ""
245+
return text.isEmpty ? nil : text
246+
}
247+
238248
private func snapshotAppFrame(app: XCUIApplication) -> CGRect {
239249
#if os(iOS)
240250
return onScreenWindowFrame(app: app)
@@ -341,6 +351,7 @@ extension RunnerTests {
341351
label: node.label,
342352
identifier: node.identifier,
343353
value: node.value,
354+
placeholder: node.placeholder,
344355
rect: node.rect,
345356
enabled: node.enabled,
346357
focused: node.focused,
@@ -395,6 +406,7 @@ extension RunnerTests {
395406
label: label.isEmpty ? nil : label,
396407
identifier: identifier.isEmpty ? nil : identifier,
397408
value: valueText,
409+
placeholder: placeholderText(element.placeholderValue),
398410
rect: SnapshotRect(frame),
399411
enabled: element.isEnabled,
400412
focused: elementHasFocus(element) ? true : nil,
@@ -558,6 +570,7 @@ extension RunnerTests {
558570
label: label.isEmpty ? nil : label,
559571
identifier: identifier.isEmpty ? nil : identifier,
560572
value: valueText,
573+
placeholder: placeholderText(element.placeholderValue),
561574
rect: SnapshotRect(frame),
562575
enabled: enabled,
563576
focused: elementHasFocus(element) ? true : nil,

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+PrivateAXPresentationTests.swift‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -117,6 +117,29 @@ extension RunnerTests {
117117
)
118118
}
119119

120+
/// The private-AX reader hands over `placeholderValue` beside `value`; the acquisition publishes
121+
/// it on the node and reads an empty one as no placeholder, the way every other producer does.
122+
func testPrivateAXAcquisitionCarriesTheFieldPlaceholder() {
123+
let frame = Self.privateAXFrame
124+
let nodes = privateAXNormalizedAcquisition(
125+
rawRoot: [
126+
"type": Int(XCUIElement.ElementType.application.rawValue), "frame": frame(0, 0, 402, 874),
127+
"children": [
128+
["type": Int(XCUIElement.ElementType.textField.rawValue), "value": "Type your name",
129+
"placeholder": "Type your name", "frame": frame(16, 200, 370, 44), "children": []],
130+
["type": Int(XCUIElement.ElementType.button.rawValue), "label": "Save",
131+
"placeholder": "", "frame": frame(16, 300, 370, 44), "children": []]
132+
]
133+
],
134+
hint: CaptureHint(
135+
projection: .raw, depth: nil, regularPresentedDepth: nil,
136+
interactiveOnly: false, customActions: false),
137+
viewport: CGRect(x: 0, y: 0, width: 402, height: 874),
138+
interfaceOrientation: RunnerInterfaceOrientation.portrait)
139+
140+
XCTAssertEqual(nodes.map(\.placeholder), [nil, "Type your name", nil])
141+
}
142+
120143
func testPrivateAXRegularPresentationProjectsToViewportAndKeepsScrollHint() throws {
121144
let nodes = try privateAXRegularPresentation(
122145
rawRoot: Self.privateAXScrolledFixture,

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotPresentationTests.swift‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ extension RunnerTests {
1212
label: "Continue",
1313
identifier: "continue-button",
1414
value: "Ready",
15+
placeholder: "Type here",
1516
rect: SnapshotRect(x: 10, y: 20, width: 100, height: 44),
1617
enabled: true,
1718
focused: true,
@@ -54,7 +55,7 @@ extension RunnerTests {
5455

5556
XCTAssertEqual(
5657
String(decoding: encoded, as: UTF8.self),
57-
#"[{"actions":["Open menu"],"depth":2,"enabled":true,"focused":true,"hiddenContentAbove":true,"hiddenContentBelow":true,"hittable":true,"identifier":"continue-button","index":3,"label":"Continue","parentIndex":1,"rect":{"height":44,"width":100,"x":10,"y":20},"selected":true,"type":"Button","value":"Ready"}]"#
58+
#"[{"actions":["Open menu"],"depth":2,"enabled":true,"focused":true,"hiddenContentAbove":true,"hiddenContentBelow":true,"hittable":true,"identifier":"continue-button","index":3,"label":"Continue","parentIndex":1,"placeholder":"Type here","rect":{"height":44,"width":100,"x":10,"y":20},"selected":true,"type":"Button","value":"Ready"}]"#
5859
)
5960
XCTAssertEqual(capture.truncated, true)
6061
XCTAssertEqual(capture.effectiveDepth, 4)

‎apple/snapshot-bridge/SnapshotBridgeRuntime.m‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818
NSString *const kProtocolVersionKey = @"protocolVersion";
1919
NSString *const kSourceVersionKey = @"sourceVersion";
2020
NSString *const kRequestIdKey = @"requestId";
21-
NSString *const kSourceVersion = @"agent-device-simulator-ax-v1.7.0";
21+
NSString *const kSourceVersion = @"agent-device-simulator-ax-v1.8.0";
2222
const NSUInteger kProtocolVersion = 1;
2323
const uint32_t kMaximumFrameBytes = 16 * 1024 * 1024;
2424
const NSUInteger kMaximumDepth = 128;
@@ -29,6 +29,7 @@
2929
static NSString *const kAttributeElementBaseType = @"XC_kAXXCAttributeElementBaseType";
3030
static NSString *const kAttributeLabel = @"XC_kAXXCAttributeLabel";
3131
static NSString *const kAttributeValue = @"XC_kAXXCAttributeValue";
32+
static NSString *const kAttributePlaceholderValue = @"XC_kAXXCAttributePlaceholderValue";
3233
static NSString *const kAttributeIdentifier = @"XC_kAXXCAttributeIdentifier";
3334
static NSString *const kAttributeFrame = @"XC_kAXXCAttributeFrame";
3435
static NSString *const kAttributeAutomationType = @"XC_kAXXCAttributeAutomationType";
@@ -330,6 +331,7 @@ - (nullable NSDictionary *)snapshotForProcess:(pid_t)pid
330331
kAttributeElementBaseType,
331332
kAttributeLabel,
332333
kAttributeValue,
334+
kAttributePlaceholderValue,
333335
kAttributeIdentifier,
334336
kAttributeFrame,
335337
kAttributeAutomationType,

‎apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotModels.swift‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,8 @@ public struct RawAXNode: Equatable {
3434
public let label: String?
3535
public let identifier: String?
3636
public let value: String?
37+
/// The text a text field shows while empty (`placeholderValue`); `nil` for every other node.
38+
public let placeholder: String?
3739
public var rect: SnapshotRect
3840
public let enabled: Bool
3941
public let focused: Bool?
@@ -52,6 +54,7 @@ public struct RawAXNode: Equatable {
5254
label: String?,
5355
identifier: String?,
5456
value: String?,
57+
placeholder: String? = nil,
5558
rect: SnapshotRect,
5659
enabled: Bool,
5760
focused: Bool?,
@@ -68,6 +71,7 @@ public struct RawAXNode: Equatable {
6871
self.label = label
6972
self.identifier = identifier
7073
self.value = value
74+
self.placeholder = placeholder
7175
self.rect = rect
7276
self.enabled = enabled
7377
self.focused = focused
@@ -272,6 +276,7 @@ public struct PresentedNode: Codable, Equatable {
272276
public let label: String?
273277
public let identifier: String?
274278
public let value: String?
279+
public let placeholder: String?
275280
public let rect: SnapshotRect
276281
public let enabled: Bool
277282
public let focused: Bool?
@@ -295,6 +300,7 @@ public struct PresentedNode: Codable, Equatable {
295300
self.label = raw.label
296301
self.identifier = raw.identifier
297302
self.value = raw.value
303+
self.placeholder = raw.placeholder
298304
self.rect = rect ?? raw.rect
299305
self.enabled = raw.enabled
300306
self.focused = raw.focused

‎apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotPresentationProjection.swift‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -140,6 +140,7 @@ extension SnapshotPresentation {
140140
label: raw.label,
141141
identifier: raw.identifier,
142142
value: raw.value,
143+
placeholder: raw.placeholder,
143144
rect: raw.rect,
144145
enabled: raw.enabled,
145146
focused: raw.focused,

‎apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotVisibilityFold.swift‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -199,6 +199,7 @@ public enum SnapshotVisibilityFold {
199199
label: node.label,
200200
identifier: node.identifier,
201201
value: node.value,
202+
placeholder: node.placeholder,
202203
rect: node.rect,
203204
enabled: node.enabled,
204205
focused: node.focused,

0 commit comments

Comments
 (0)