Skip to content

Commit 965e77b

Browse files
committed
fix(ios): preserve normalized press bindings and address readiness review
1 parent f757b60 commit 965e77b

23 files changed

Lines changed: 532 additions & 121 deletions

File tree

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -372,9 +372,9 @@ extension RunnerTests {
372372
/// A mutation-read: count every exact target binding, including blocked duplicates. Unlike
373373
/// public selector reads, a hittable sibling can never stand in for the authorized control.
374374
func queryAuthorizedPressTarget(app: XCUIApplication, target: AuthorizedPressTarget, appBundleId: String? = nil) throws -> Response {
375-
let expected = target.rect.cgRect
376-
let point = CGPoint(x: target.point.x, y: target.point.y)
377-
guard !expected.isEmpty, !expected.isInfinite, expected.contains(point) else {
375+
let expected = target.rect
376+
guard expected.width > 0, expected.height > 0,
377+
[expected.x, expected.y, expected.width, expected.height].allSatisfy({ $0.isFinite }) else {
378378
return Response(ok: true, data: DataPayload(pressReadiness: "missing", pressReadinessAppBundleId: appBundleId))
379379
}
380380
func text(_ value: Any?) -> String {
@@ -389,7 +389,7 @@ extension RunnerTests {
389389
.filter { element in
390390
guard element.exists, element.elementType.rawValue == type else { return false }
391391
let frame = element.frame
392-
guard frame.contains(point) else { return false }
392+
// The canonical point can use a different space; only source identity and the normalized unclipped frame bind this control read.
393393
return target.matches(type: target.type, identifier: element.identifier,
394394
label: element.label, value: target.type == "SecureTextField" ? nil : text(element.value), rect: SnapshotRect(frame))
395395
}

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SelectorMatchPolicyTests.swift‎

Lines changed: 26 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,24 @@ extension RunnerTests {
3535
XCTAssertEqual(fixture.expected.matches(type: candidate.type, identifier: candidate.identifier,
3636
label: candidate.label, value: candidate.value, rect: candidate.rect), fixture.matches, fixture.name)
3737
}
38+
// JSON cannot encode non-finite numbers; both consumers exercise them natively.
39+
let reference = fixtures[0].expected
40+
let r = reference.rect
41+
for invalid in [Double.nan, Double.infinity, -Double.infinity] {
42+
for rect in [
43+
SnapshotRect(x: invalid, y: r.y, width: r.width, height: r.height),
44+
SnapshotRect(x: r.x, y: invalid, width: r.width, height: r.height),
45+
SnapshotRect(x: r.x, y: r.y, width: invalid, height: r.height),
46+
SnapshotRect(x: r.x, y: r.y, width: r.width, height: invalid),
47+
] {
48+
XCTAssertFalse(reference.matches(type: reference.type, identifier: reference.identifier,
49+
label: reference.label, value: reference.value, rect: rect))
50+
let invalidTarget = AuthorizedPressTarget(type: reference.type, identifier: reference.identifier,
51+
label: reference.label, value: reference.value, rect: rect, point: reference.point)
52+
XCTAssertFalse(invalidTarget.matches(type: reference.type, identifier: reference.identifier,
53+
label: reference.label, value: reference.value, rect: rect))
54+
}
55+
}
3856
}
3957

4058
func testDirectSelectorRejectsTwoRawMatchesBeforeHittabilityPreference() {
@@ -156,11 +174,14 @@ extension RunnerTests {
156174
// The same native owner must not use the read preference for a previously resolved target.
157175
for element in matches {
158176
let frame = element.frame
159-
let target = AuthorizedPressTarget(type: elementTypeName(element.elementType),
160-
identifier: element.identifier, label: element.label, value: String(describing: element.value ?? ""),
161-
rect: SnapshotRect(frame), point: PressTargetPoint(x: frame.midX, y: frame.midY))
162-
let readiness = try queryAuthorizedPressTarget(app: app, target: target)
163-
XCTAssertEqual(readiness.data?.pressReadiness, element.isHittable ? "ready" : "blocked")
177+
// Canonical presentation coordinates must not filter the source-native identity/frame.
178+
for point in [PressTargetPoint(x: frame.midX, y: frame.midY), PressTargetPoint(x: -10_000, y: -10_000)] {
179+
let target = AuthorizedPressTarget(type: elementTypeName(element.elementType),
180+
identifier: element.identifier, label: element.label, value: String(describing: element.value ?? ""),
181+
rect: SnapshotRect(frame), point: point)
182+
let readiness = try queryAuthorizedPressTarget(app: app, target: target)
183+
XCTAssertEqual(readiness.data?.pressReadiness, element.isHittable ? "ready" : "blocked")
184+
}
164185
}
165186
}
166187
#endif

‎apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotCoordinateSpace.swift‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -274,7 +274,8 @@ extension SnapshotGeometrySpace {
274274
enabled: node.enabled,
275275
frame: frame,
276276
viewport: viewport
277-
)
277+
),
278+
nativePressTargetRect: SnapshotRect(nodeSpace.orientedFrame(of: node.nativePressTarget.rect.cgRect))
278279
)
279280
)
280281
}

‎apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotModels.swift‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@ public struct NativePressTarget: Codable, Equatable {
4545
}
4646

4747
public struct RawAXNode: Equatable {
48-
public let nativePressTarget: NativePressTarget
48+
public private(set) var nativePressTarget: NativePressTarget
4949
public private(set) var index: Int
5050
public let type: String
5151
public let label: String?
@@ -106,10 +106,13 @@ public struct RawAXNode: Equatable {
106106
self.actions = actions
107107
}
108108

109-
func replacing(rect: SnapshotRect, hittable: Bool?) -> RawAXNode {
109+
func replacing(rect: SnapshotRect, hittable: Bool?, nativePressTargetRect: SnapshotRect) -> RawAXNode {
110110
var updated = self
111111
updated.rect = rect
112112
updated.hittable = hittable
113+
let source = nativePressTarget
114+
updated.nativePressTarget = NativePressTarget(type: source.type, identifier: source.identifier,
115+
label: source.label, value: source.value, rect: nativePressTargetRect)
113116
return updated
114117
}
115118

‎apple/snapshot-presentation/Tests/AgentDeviceSnapshotPresentationTests/CoordinateSpaceTests.swift‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -408,6 +408,7 @@ final class CoordinateSpaceTests: XCTestCase {
408408
)
409409
XCTAssertEqual(normalized.count, 4)
410410
XCTAssertEqual(normalized[3].rect.cgRect, expected.cgRect, testCase.name)
411+
XCTAssertEqual(normalized[3].nativePressTarget.rect.cgRect, expected.cgRect, testCase.name)
411412
}
412413
}
413414

‎contracts/fixtures/authorized-press-target.json‎

Lines changed: 228 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -317,5 +317,233 @@
317317
}
318318
},
319319
"matches": true
320+
},
321+
{
322+
"name": "shared nonempty identifier and value",
323+
"expected": {
324+
"type": "Button",
325+
"label": "Continue",
326+
"rect": {
327+
"x": 40,
328+
"y": 240,
329+
"width": 100,
330+
"height": 50
331+
},
332+
"point": {
333+
"x": 90,
334+
"y": 265
335+
},
336+
"identifier": "continue-control",
337+
"value": "Ready"
338+
},
339+
"candidate": {
340+
"type": "Button",
341+
"label": "Continue",
342+
"rect": {
343+
"x": 40,
344+
"y": 240,
345+
"width": 100,
346+
"height": 50
347+
},
348+
"point": {
349+
"x": 90,
350+
"y": 265
351+
},
352+
"identifier": "continue-control",
353+
"value": "Ready"
354+
},
355+
"matches": true
356+
},
357+
{
358+
"name": "negative width",
359+
"expected": {
360+
"type": "Button",
361+
"label": "Continue",
362+
"rect": {
363+
"x": 40,
364+
"y": 240,
365+
"width": -1,
366+
"height": 50
367+
},
368+
"point": {
369+
"x": 90,
370+
"y": 265
371+
}
372+
},
373+
"candidate": {
374+
"type": "Button",
375+
"label": "Continue",
376+
"rect": {
377+
"x": 40,
378+
"y": 240,
379+
"width": -1,
380+
"height": 50
381+
},
382+
"point": {
383+
"x": 90,
384+
"y": 265
385+
}
386+
},
387+
"matches": false
388+
},
389+
{
390+
"name": "positive y width height rounding",
391+
"expected": {
392+
"type": "Button",
393+
"label": "Continue",
394+
"rect": {
395+
"x": 40,
396+
"y": 240,
397+
"width": 100,
398+
"height": 50
399+
},
400+
"point": {
401+
"x": 90,
402+
"y": 265
403+
}
404+
},
405+
"candidate": {
406+
"type": "Button",
407+
"label": "Continue",
408+
"rect": {
409+
"x": 40,
410+
"y": 241,
411+
"width": 101,
412+
"height": 51
413+
},
414+
"point": {
415+
"x": 90,
416+
"y": 265
417+
}
418+
},
419+
"matches": true
420+
},
421+
{
422+
"name": "negative y width height rounding",
423+
"expected": {
424+
"type": "Button",
425+
"label": "Continue",
426+
"rect": {
427+
"x": 40,
428+
"y": 240,
429+
"width": 100,
430+
"height": 50
431+
},
432+
"point": {
433+
"x": 90,
434+
"y": 265
435+
}
436+
},
437+
"candidate": {
438+
"type": "Button",
439+
"label": "Continue",
440+
"rect": {
441+
"x": 40,
442+
"y": 239,
443+
"width": 99,
444+
"height": 49
445+
},
446+
"point": {
447+
"x": 90,
448+
"y": 265
449+
}
450+
},
451+
"matches": true
452+
},
453+
{
454+
"name": "y outside capture rounding",
455+
"expected": {
456+
"type": "Button",
457+
"label": "Continue",
458+
"rect": {
459+
"x": 40,
460+
"y": 240,
461+
"width": 100,
462+
"height": 50
463+
},
464+
"point": {
465+
"x": 90,
466+
"y": 265
467+
}
468+
},
469+
"candidate": {
470+
"type": "Button",
471+
"label": "Continue",
472+
"rect": {
473+
"x": 40,
474+
"y": 242,
475+
"width": 100,
476+
"height": 50
477+
},
478+
"point": {
479+
"x": 90,
480+
"y": 265
481+
}
482+
},
483+
"matches": false
484+
},
485+
{
486+
"name": "width outside capture rounding",
487+
"expected": {
488+
"type": "Button",
489+
"label": "Continue",
490+
"rect": {
491+
"x": 40,
492+
"y": 240,
493+
"width": 100,
494+
"height": 50
495+
},
496+
"point": {
497+
"x": 90,
498+
"y": 265
499+
}
500+
},
501+
"candidate": {
502+
"type": "Button",
503+
"label": "Continue",
504+
"rect": {
505+
"x": 40,
506+
"y": 240,
507+
"width": 102,
508+
"height": 50
509+
},
510+
"point": {
511+
"x": 90,
512+
"y": 265
513+
}
514+
},
515+
"matches": false
516+
},
517+
{
518+
"name": "height outside capture rounding",
519+
"expected": {
520+
"type": "Button",
521+
"label": "Continue",
522+
"rect": {
523+
"x": 40,
524+
"y": 240,
525+
"width": 100,
526+
"height": 50
527+
},
528+
"point": {
529+
"x": 90,
530+
"y": 265
531+
}
532+
},
533+
"candidate": {
534+
"type": "Button",
535+
"label": "Continue",
536+
"rect": {
537+
"x": 40,
538+
"y": 240,
539+
"width": 100,
540+
"height": 52
541+
},
542+
"point": {
543+
"x": 90,
544+
"y": 265
545+
}
546+
},
547+
"matches": false
320548
}
321549
]

‎contracts/fixtures/runner-requests.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,7 @@
6464
{"name": "ios-simulator.runner-command-recovery.status", "producer": "packages/apple-runner/src/__tests__/index-runner-requests.test.ts", "request": {"command": "status", "statusCommandId": "<commandId>"}},
6565
{"name": "ios-simulator.runner-disposal.shutdown", "producer": "packages/apple-runner/src/__tests__/index-runner-requests.test.ts", "request": {"command": "shutdown", "commandId": "<commandId>"}},
6666
{"name": "ios-simulator.runner-lifecycle-prepare.uptime", "producer": "packages/apple-runner/src/__tests__/index-runner-requests.test.ts", "request": {"command": "uptime", "commandId": "<commandId>"}},
67+
{"name": "ios-simulator.runner-selector-query.authorized-target", "producer": "packages/platform-apple/src/__tests__/index-runner-requests.test.ts", "request": {"command": "querySelector", "authorizedTarget": {"type": "Button", "identifier": "continue-control", "label": "Continue", "value": "Ready", "rect": {"x": 40, "y": 240, "width": 100, "height": 50}, "point": {"x": 90, "y": 265}}, "appBundleId": "com.example.app", "commandId": "<commandId>"}},
6768
{"name": "ios-simulator.runner-selector-query.label", "producer": "packages/platform-apple/src/__tests__/index-runner-requests.test.ts", "request": {"command": "querySelector", "selectorKey": "label", "selectorValue": "Go", "appBundleId": "com.example.app", "commandId": "<commandId>"}},
6869
{"name": "ios-simulator.runner-sequence.bound-orientation", "producer": "packages/platform-apple/src/__tests__/interactions-press-series-runner-requests.test.ts", "request": {"command": "sequence", "steps": [{"kind": "tap", "x": 10, "y": 20, "synthesized": true}], "appBundleId": "com.example.app", "sequenceOrientation": 1, "commandId": "<commandId>"}},
6970
{"name": "ios-simulator.runner-session-readiness.uptime", "producer": "packages/apple-runner/src/__tests__/index-runner-requests.test.ts", "request": {"command": "uptime", "commandId": "<commandId>"}},

0 commit comments

Comments
 (0)