Skip to content

Commit 513d3bf

Browse files
committed
fix: wait for native iOS press readiness before input
Bind the native hittability read to the canonically authorized target, preserve the input point, and poll within the original request budget without a public wait API. Refuse changed or ambiguous targets and expired or canceled requests before dispatch. Fixes callstack#4388
1 parent eec35e9 commit 513d3bf

26 files changed

Lines changed: 919 additions & 11 deletions

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -449,6 +449,9 @@ extension RunnerTests {
449449
let found = try findElement(app: activeApp, text: text) != nil
450450
return Response(ok: true, data: DataPayload(found: found))
451451
case .querySelector:
452+
if let target = command.authorizedTarget {
453+
return try queryAuthorizedPressTarget(app: activeApp, target: target)
454+
}
452455
guard let selectorKey = command.selectorKey, let selectorValue = command.selectorValue else {
453456
return Response(ok: false, error: ErrorPayload(message: "querySelector requires selectorKey and selectorValue"))
454457
}

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -369,6 +369,34 @@ extension RunnerTests {
369369
try resolveRunnerWindow(app: app).frame
370370
}
371371

372+
/// A mutation-read: count every exact target binding, including blocked duplicates. Unlike
373+
/// public selector reads, a hittable sibling can never stand in for the authorized control.
374+
func queryAuthorizedPressTarget(app: XCUIApplication, target: AuthorizedPressTarget) throws -> Response {
375+
let expected = target.rect.cgRect
376+
let point = CGPoint(x: target.point.x, y: target.point.y)
377+
guard !expected.isEmpty, !expected.isInfinite, expected.contains(point) else {
378+
return Response(ok: true, data: DataPayload(pressReadiness: "missing"))
379+
}
380+
func text(_ value: Any?) -> String {
381+
String(describing: value ?? "").trimmingCharacters(in: .whitespacesAndNewlines)
382+
}
383+
guard let type = Self.elementTypeNamesByRawValue.first(where: { $0.value == target.type })?.key else {
384+
return Response(ok: true, data: DataPayload(pressReadiness: "missing"))
385+
}
386+
let identity = NSPredicate(format: "elementType == %@", NSNumber(value: type))
387+
let matches = try elementQueries(app: app, matching: .any)
388+
.flatMap { $0.matching(identity).allElementsBoundByIndex }
389+
.filter { element in
390+
guard element.exists else { return false }
391+
let frame = element.frame
392+
guard frame.contains(point) else { return false }
393+
return target.matches(type: elementTypeName(element.elementType), identifier: element.identifier,
394+
label: element.label, value: text(element.value), rect: SnapshotRect(frame))
395+
}
396+
let state = matches.count > 1 ? "ambiguous" : matches.first.map { $0.isHittable ? "ready" : "blocked" } ?? "missing"
397+
return Response(ok: true, data: DataPayload(pressReadiness: state))
398+
}
399+
372400
func queryElement(app: XCUIApplication, selectorKey: String, selectorValue: String) throws -> Response {
373401
// querySelector is a read — it backs get/is/wait and the offscreen-refusal
374402
// double-check, none of which mutate. The fail-closed raw-match rule exists

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -239,6 +239,29 @@ struct RunnerLitPanelFact: Codable {
239239
let heightPt: Double
240240
}
241241

242+
struct AuthorizedPressTarget: Codable {
243+
let type: String
244+
let identifier: String?
245+
let label: String?
246+
let value: String?
247+
let rect: SnapshotRect
248+
let point: PressTargetPoint
249+
250+
func matches(type: String, identifier: String?, label: String?, value: String?, rect: SnapshotRect) -> Bool {
251+
func text(_ value: String?) -> String { (value ?? "").trimmingCharacters(in: .whitespacesAndNewlines) }
252+
return self.type == type && text(self.identifier) == text(identifier)
253+
&& text(self.label) == text(label) && text(self.value) == text(value)
254+
&& [self.rect.x - rect.x, self.rect.y - rect.y, self.rect.width - rect.width, self.rect.height - rect.height]
255+
.allSatisfy { $0.isFinite && abs($0) <= 1 }
256+
&& self.rect.width >= 0 && self.rect.height >= 0 && rect.width >= 0 && rect.height >= 0
257+
}
258+
}
259+
260+
struct PressTargetPoint: Codable {
261+
let x: Double
262+
let y: Double
263+
}
264+
242265
struct Command: Codable {
243266
let command: CommandType
244267
let commandId: String?
@@ -247,6 +270,7 @@ struct Command: Codable {
247270
let text: String?
248271
let selectorKey: String?
249272
let selectorValue: String?
273+
let authorizedTarget: AuthorizedPressTarget?
250274
let allowNonHittableCoordinateFallback: Bool?
251275
let delayMs: Int?
252276
let textEntryMode: String?
@@ -492,6 +516,7 @@ struct DataPayload: Codable {
492516
var imageBase64: String?
493517
var text: String?
494518
var found: Bool?
519+
var pressReadiness: String?
495520
var items: [String]?
496521
/// Set on an alert read that is SpringBoard's `Open in “<App>”?` launch confirmation, in the
497522
/// device's language; absent otherwise.

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SelectorMatchPolicyTests.swift‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,26 @@ import AgentDeviceSnapshotPresentation
33

44
extension RunnerTests {
55
#if AGENT_DEVICE_RUNNER_UNIT_TESTS
6+
func testAuthorizedPressTargetMatchesGoldenParityTable() throws {
7+
struct Fixture: Decodable {
8+
let name: String
9+
let expected: AuthorizedPressTarget
10+
let candidate: AuthorizedPressTarget
11+
let matches: Bool
12+
}
13+
let root = URL(fileURLWithPath: #filePath)
14+
.deletingLastPathComponent().deletingLastPathComponent().deletingLastPathComponent()
15+
.deletingLastPathComponent().deletingLastPathComponent().deletingLastPathComponent()
16+
let fixtures = try JSONDecoder().decode([Fixture].self, from:
17+
Data(contentsOf: root.appendingPathComponent("contracts/fixtures/authorized-press-target.json")))
18+
XCTAssertFalse(fixtures.isEmpty)
19+
for fixture in fixtures {
20+
let candidate = fixture.candidate
21+
XCTAssertEqual(fixture.expected.matches(type: candidate.type, identifier: candidate.identifier,
22+
label: candidate.label, value: candidate.value, rect: candidate.rect), fixture.matches, fixture.name)
23+
}
24+
}
25+
626
func testDirectSelectorRejectsTwoRawMatchesBeforeHittabilityPreference() {
727
let decision = classifyDirectSelectorCandidates(
828
[
@@ -118,6 +138,16 @@ extension RunnerTests {
118138
XCTAssertEqual(response.data?.nodes?.count, 1)
119139
XCTAssertEqual(response.data?.nodes?.first?.identifier, duplicateIdentifier)
120140
XCTAssertEqual(response.data?.nodes?.first?.hittable, true)
141+
142+
// The same native owner must not use the read preference for a previously resolved target.
143+
for element in matches {
144+
let frame = element.frame
145+
let target = AuthorizedPressTarget(type: elementTypeName(element.elementType),
146+
identifier: element.identifier, label: element.label, value: String(describing: element.value ?? ""),
147+
rect: SnapshotRect(frame), point: PressTargetPoint(x: frame.midX, y: frame.midY))
148+
let readiness = try queryAuthorizedPressTarget(app: app, target: target)
149+
XCTAssertEqual(readiness.data?.pressReadiness, element.isHittable ? "ready" : "blocked")
150+
}
121151
}
122152
#endif
123153
#endif

0 commit comments

Comments
 (0)