Skip to content

Commit fc26237

Browse files
committed
fix(ios-runner): route every synthesized gesture through one fallback helper
A synthesized tap step inside `sequence` resolved its context with the synthesized-drag policy, so it refused the XCTest coordinate fallback when accessibility was unavailable and failed outright when no window frame resolved. A standalone synthesized tap falls back in both cases. `performSynthesizedGesture` now performs the synthesized gesture, decides the XCTest coordinate fallback from the gesture kind's policy, and logs the decision. Selector tap, coordinate tap, sequence tap step, type focus and drag use it; each site keeps its own XCTest gesture and bookkeeping. A sequence tap step takes the coordinate-tap policy, and the fallback decision reads the kind rather than a policy copy on the coordinate context. Remove the unused `whenAccessibilityHealthy` keyboard policy, the drag fallback branch that a nil frame could never reach, and the text-entry focus predicate the helper replaces. Scroll keeps `privateSynthesisRequired`. Fixes #2788
1 parent a363a02 commit fc26237

12 files changed

Lines changed: 193 additions & 175 deletions

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift‎

Lines changed: 16 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -78,20 +78,15 @@ extension RunnerTests {
7878
app: activeApp,
7979
policy: synthesizedGesturePolicy(policyKind)
8080
)
81-
let (timing, outcome) = performGesture(activeApp, idleTimeout: false) {
81+
switch performSynthesizedGesture(activeApp, kind: policyKind, context: context, synthesize: {
8282
synthesizedTapAt(
8383
app: activeApp,
8484
x: touchPoint.x,
8585
y: touchPoint.y,
8686
context: context
8787
)
88-
}
89-
if case .performed = outcome {
90-
logSynthesizedGesturePolicyDecision(
91-
kind: policyKind,
92-
context: context,
93-
fallbackAttempted: false
94-
)
88+
}) {
89+
case .performed(let timing):
9590
if isTextEntry {
9691
waitForTextEntryReadinessAfterTap(app: activeApp, element: element)
9792
}
@@ -107,13 +102,12 @@ extension RunnerTests {
107102
? match.usedNonHittableFallback
108103
: nil
109104
)
105+
case .xctestFallback(let message, let hint):
106+
fallback = GestureFallback(strategy: "xctest-coordinate-tap", message: message, hint: hint)
107+
case .refused(_, let message, let hint):
108+
clearRememberedTextEntryTap()
109+
return unsupportedResponse(message: message, hint: hint)
110110
}
111-
logSynthesizedGesturePolicyDecision(
112-
kind: policyKind,
113-
context: context,
114-
fallbackAttempted: true
115-
)
116-
fallback = gestureFallback(strategy: "xctest-coordinate-tap", from: outcome)
117111
}
118112
let (timing, outcome) = performGesture(activeApp) {
119113
if expectedPoint != nil || match.usedNonHittableFallback {
@@ -172,16 +166,18 @@ extension RunnerTests {
172166
app: activeApp,
173167
policy: synthesizedGesturePolicy(policyKind)
174168
)
175-
let (timing, outcome) = performGesture(activeApp, idleTimeout: false) {
169+
switch performSynthesizedGesture(activeApp, kind: policyKind, context: context, synthesize: {
176170
synthesizedTapAt(app: activeApp, x: x, y: y, context: context)
177-
}
178-
if case .performed = outcome {
179-
logSynthesizedGesturePolicyDecision(kind: policyKind, context: context, fallbackAttempted: false)
171+
}) {
172+
case .performed(let timing):
180173
rememberTextEntryTap(textInput)
181174
return gestureResponse(message: "tapped", timing: timing)
175+
case .xctestFallback(let message, let hint):
176+
fallback = GestureFallback(strategy: "xctest-coordinate-tap", message: message, hint: hint)
177+
case .refused(_, let message, let hint):
178+
clearRememberedTextEntryTap()
179+
return unsupportedResponse(message: message, hint: hint)
182180
}
183-
logSynthesizedGesturePolicyDecision(kind: policyKind, context: context, fallbackAttempted: true)
184-
fallback = gestureFallback(strategy: "xctest-coordinate-tap", from: outcome)
185181
}
186182
let touchFrame = resolvedTouchVisualizationFrame(app: activeApp, x: x, y: y)
187183
let (timing, outcome) = performGesture(activeApp) { tapAt(app: activeApp, x: x, y: y) }

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+GestureExecution.swift‎

Lines changed: 8 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -25,13 +25,17 @@ extension RunnerTests {
2525
case .performed:
2626
return nil
2727
case .unsupported(let message, let hint):
28-
return Response(
29-
ok: false,
30-
error: ErrorPayload(code: "UNSUPPORTED_OPERATION", message: message, hint: hint)
31-
)
28+
return unsupportedResponse(message: message, hint: hint)
3229
}
3330
}
3431

32+
func unsupportedResponse(message: String, hint: String?) -> Response {
33+
Response(
34+
ok: false,
35+
error: ErrorPayload(code: "UNSUPPORTED_OPERATION", message: message, hint: hint)
36+
)
37+
}
38+
3539
/// Optional visualization frame returned with a gesture response.
3640
enum GestureFrame {
3741
case none
@@ -45,15 +49,6 @@ extension RunnerTests {
4549
let hint: String?
4650
}
4751

48-
func gestureFallback(strategy: String, from outcome: RunnerInteractionOutcome) -> GestureFallback? {
49-
switch outcome {
50-
case .performed:
51-
return nil
52-
case .unsupported(let message, let hint):
53-
return GestureFallback(strategy: strategy, message: message, hint: hint)
54-
}
55-
}
56-
5752

5853
/// Runs a gesture action with uniform timing capture. Touch gestures pass `idleTimeout: true`
5954
/// (the default) to run inside the scroll idle-timeout + quiescence-skip wrapper; synthesis

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+ScrollDragExecution.swift‎

Lines changed: 8 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -149,19 +149,6 @@ extension RunnerTests {
149149
context: context
150150
)
151151
else {
152-
if context?.allowsXCTestCoordinateFallback == true {
153-
logSynthesizedGesturePolicyDecision(kind: policyKind, context: context, fallbackAttempted: true)
154-
return executeCoordinateDragFallback(
155-
activeApp: activeApp,
156-
x: x,
157-
y: y,
158-
x2: x2,
159-
y2: y2,
160-
durationMs: durationMs,
161-
message: message,
162-
fallback: nil
163-
)
164-
}
165152
logSynthesizedGesturePolicyDecision(kind: policyKind, context: context, fallbackAttempted: false)
166153
return Response(
167154
ok: false,
@@ -179,7 +166,7 @@ extension RunnerTests {
179166
y2: plan.points.y2,
180167
referenceFrame: plan.referenceFrame
181168
)
182-
let (timing, outcome) = performGesture(activeApp, idleTimeout: false) {
169+
switch performSynthesizedGesture(activeApp, kind: policyKind, context: plan.context, synthesize: {
183170
synthesizedDragAt(
184171
app: activeApp,
185172
x: plan.points.x,
@@ -190,13 +177,10 @@ extension RunnerTests {
190177
profile: profile,
191178
context: plan.context
192179
)
193-
}
194-
if case .performed = outcome {
195-
logSynthesizedGesturePolicyDecision(kind: policyKind, context: plan.context, fallbackAttempted: false)
180+
}) {
181+
case .performed(let timing):
196182
return gestureResponse(message: message, timing: timing, frame: .drag(dragFrame))
197-
}
198-
if plan.context.allowsXCTestCoordinateFallback {
199-
logSynthesizedGesturePolicyDecision(kind: policyKind, context: plan.context, fallbackAttempted: true)
183+
case .xctestFallback(let fallbackMessage, let hint):
200184
return executeCoordinateDragFallback(
201185
activeApp: activeApp,
202186
x: plan.points.x,
@@ -205,11 +189,11 @@ extension RunnerTests {
205189
y2: plan.points.y2,
206190
durationMs: durationMs,
207191
message: message,
208-
fallback: gestureFallback(strategy: "xctest-coordinate-drag", from: outcome)
192+
fallback: GestureFallback(strategy: "xctest-coordinate-drag", message: fallbackMessage, hint: hint)
209193
)
194+
case .refused(_, let refusalMessage, let hint):
195+
return unsupportedResponse(message: refusalMessage, hint: hint)
210196
}
211-
logSynthesizedGesturePolicyDecision(kind: policyKind, context: plan.context, fallbackAttempted: false)
212-
return unsupportedResponse(for: outcome)
213197
#else
214198
return nil
215199
#endif
@@ -223,7 +207,7 @@ extension RunnerTests {
223207
y2: Double,
224208
durationMs: Double,
225209
message: String,
226-
fallback: GestureFallback?
210+
fallback: GestureFallback
227211
) -> Response {
228212
let dragPoints = keyboardAvoidingDragPoints(app: activeApp, x: x, y: y, x2: x2, y2: y2)
229213
let dragFrame = resolvedDragVisualizationFrame(

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SequenceExecution.swift‎

Lines changed: 20 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -143,51 +143,21 @@ extension RunnerTests {
143143
) -> SequenceStepOutcome {
144144
let x = step.x ?? 0
145145
let y = step.y ?? 0
146-
// Synthesized HID tap fast path mirrors the individual `tap` command (idleTimeout:false, with
147-
// a tapAt fallback when synthesis is unsupported), so fusing a jittered tap series does not
148-
// change the touch mechanism for these inputs.
149-
if step.kind == "tap", step.synthesized == true {
150-
let policyKind = SynthesizedGesturePolicyKind.synthesizedDrag
151-
#if os(iOS)
152-
guard let synthesizedContext else {
153-
let nowMs = ProcessInfo.processInfo.systemUptime * 1000
154-
logSynthesizedGesturePolicyDecision(kind: policyKind, context: nil, fallbackAttempted: false)
155-
return SequenceStepOutcome(
156-
outcome: .unsupported(
157-
message: "synthesized coordinate tap could not resolve a finite coordinate frame",
158-
hint: "Retry after the app is foregrounded, or use a plain screenshot to choose coordinates."
159-
),
160-
gestureStartUptimeMs: nowMs,
161-
gestureEndUptimeMs: nowMs
162-
)
163-
}
164-
#endif
165-
let (timing, outcome) = performGesture(activeApp, idleTimeout: false) {
146+
if let policyKind = synthesizedPolicyKind(forSequenceStep: step) {
147+
switch performSynthesizedGesture(activeApp, kind: policyKind, context: synthesizedContext, synthesize: {
166148
synthesizedTapAt(app: activeApp, x: x, y: y, context: synthesizedContext)
167-
}
168-
if case .performed = outcome {
169-
logSynthesizedGesturePolicyDecision(kind: policyKind, context: synthesizedContext, fallbackAttempted: false)
170-
if let pauseMs = step.pauseMs, pauseMs > 0 {
171-
sleepFor(min(max(pauseMs, 0), 10000) / 1000.0)
172-
}
173-
return SequenceStepOutcome(
174-
outcome: outcome,
175-
gestureStartUptimeMs: timing.gestureStartUptimeMs,
176-
gestureEndUptimeMs: timing.gestureEndUptimeMs
177-
)
178-
}
179-
#if os(iOS)
180-
guard synthesizedContext.allowsXCTestCoordinateFallback else {
181-
logSynthesizedGesturePolicyDecision(kind: policyKind, context: synthesizedContext, fallbackAttempted: false)
182-
return SequenceStepOutcome(
183-
outcome: outcome,
184-
gestureStartUptimeMs: timing.gestureStartUptimeMs,
185-
gestureEndUptimeMs: timing.gestureEndUptimeMs
149+
}) {
150+
case .performed(let timing):
151+
return finishedSequenceStep(step, timing: timing, outcome: .performed)
152+
case .refused(let timing, let message, let hint):
153+
return finishedSequenceStep(
154+
step,
155+
timing: timing,
156+
outcome: .unsupported(message: message, hint: hint)
186157
)
158+
case .xctestFallback:
159+
break
187160
}
188-
logSynthesizedGesturePolicyDecision(kind: policyKind, context: synthesizedContext, fallbackAttempted: true)
189-
#endif
190-
// Synthesis unsupported (e.g. macOS) — fall through to the drag-based tapAt below.
191161
}
192162
let (timing, outcome) = performGesture(activeApp) {
193163
switch step.kind {
@@ -201,6 +171,14 @@ extension RunnerTests {
201171
return tapAt(app: activeApp, x: x, y: y)
202172
}
203173
}
174+
return finishedSequenceStep(step, timing: timing, outcome: outcome)
175+
}
176+
177+
private func finishedSequenceStep(
178+
_ step: SequenceStep,
179+
timing: (gestureStartUptimeMs: Double, gestureEndUptimeMs: Double),
180+
outcome: RunnerInteractionOutcome
181+
) -> SequenceStepOutcome {
204182
// Sleep AFTER the step — pauseMs is the inter-step gap — but only when the step performed.
205183
// assembleSequenceExecution stops at the first unsupported outcome, so pausing after a failed
206184
// step would burn up to 10s of watchdog budget with no following step to separate from.

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedGesturePolicy.swift‎

Lines changed: 48 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -14,15 +14,12 @@ enum RunnerAccessibilityHealth: String, Equatable {
1414

1515
enum SynthesizedKeyboardPolicy: String, Equatable, Hashable {
1616
case never
17-
case whenAccessibilityHealthy
1817
case requiredWhenAvailable
1918

2019
func allowsProbe(accessibilityHealth: RunnerAccessibilityHealth) -> Bool {
2120
switch self {
2221
case .never:
2322
return false
24-
case .whenAccessibilityHealthy:
25-
return accessibilityHealth == .healthy
2623
case .requiredWhenAvailable:
2724
return accessibilityHealth != .unavailable
2825
}
@@ -63,23 +60,17 @@ struct SynthesizedCoordinateContext {
6360
/// through this same window so geometry and routing can never name different windows.
6461
let resolvedWindow: XCUIElement
6562
let keyboardPolicy: SynthesizedKeyboardPolicy
66-
let fallbackPolicy: SynthesizedFallbackPolicy
6763
let accessibilityHealth: RunnerAccessibilityHealth
6864

6965
func withReferenceFrame(_ frame: CGRect) -> SynthesizedCoordinateContext {
7066
SynthesizedCoordinateContext(
7167
referenceFrame: frame,
7268
resolvedWindow: resolvedWindow,
7369
keyboardPolicy: keyboardPolicy,
74-
fallbackPolicy: fallbackPolicy,
7570
accessibilityHealth: accessibilityHealth
7671
)
7772
}
7873

79-
var allowsXCTestCoordinateFallback: Bool {
80-
fallbackPolicy.allowsXCTestCoordinateFallback(accessibilityHealth: accessibilityHealth)
81-
}
82-
8374
var allowsKeyboardProbe: Bool {
8475
keyboardPolicy.allowsProbe(accessibilityHealth: accessibilityHealth)
8576
}
@@ -115,22 +106,59 @@ func shouldProbeCoordinateTapTextInput(xCTestChannelPenalized: Bool) -> Bool {
115106
!xCTestChannelPenalized
116107
}
117108

118-
func sequenceHasSynthesizedCoordinateStep(_ steps: [SequenceStep]) -> Bool {
119-
steps.contains { step in
120-
step.synthesized == true && step.kind == "tap"
121-
}
109+
/// A synthesized `tap` step in a `sequence` is a standalone coordinate tap and follows its policy.
110+
func synthesizedPolicyKind(forSequenceStep step: SequenceStep) -> SynthesizedGesturePolicyKind? {
111+
step.synthesized == true && step.kind == "tap" ? .coordinateTap : nil
112+
}
113+
114+
/// A synthesized gesture's result for its call site, which owns the XCTest coordinate gesture.
115+
enum SynthesizedGestureAttempt {
116+
case performed(timing: (gestureStartUptimeMs: Double, gestureEndUptimeMs: Double))
117+
/// Synthesis failed and the policy allows the call site's XCTest coordinate gesture.
118+
case xctestFallback(message: String, hint: String?)
119+
/// Synthesis failed and the policy refuses an XCTest coordinate gesture.
120+
case refused(
121+
timing: (gestureStartUptimeMs: Double, gestureEndUptimeMs: Double),
122+
message: String,
123+
hint: String?
124+
)
122125
}
123126

124127
extension RunnerTests {
125128
func synthesizedSequenceCoordinateContext(
126129
steps: [SequenceStep],
127130
app: XCUIApplication
128131
) -> SynthesizedCoordinateContext? {
129-
guard sequenceHasSynthesizedCoordinateStep(steps) else { return nil }
130-
return synthesizedCoordinateContext(
131-
app: app,
132-
policy: synthesizedGesturePolicy(.synthesizedDrag)
132+
guard let kind = steps.lazy.compactMap(synthesizedPolicyKind(forSequenceStep:)).first else {
133+
return nil
134+
}
135+
return synthesizedCoordinateContext(app: app, policy: synthesizedGesturePolicy(kind))
136+
}
137+
138+
/// `context` is nil when no window frame resolved; `kind`'s fallback policy then reads the
139+
/// runner's current accessibility health.
140+
func performSynthesizedGesture(
141+
_ app: XCUIApplication,
142+
kind: SynthesizedGesturePolicyKind,
143+
context: SynthesizedCoordinateContext?,
144+
synthesize: () -> RunnerInteractionOutcome
145+
) -> SynthesizedGestureAttempt {
146+
let (timing, outcome) = performGesture(app, idleTimeout: false, synthesize)
147+
guard case .unsupported(let message, let hint) = outcome else {
148+
logSynthesizedGesturePolicyDecision(kind: kind, context: context, fallbackAttempted: false)
149+
return .performed(timing: timing)
150+
}
151+
let fallbackAllowed = synthesizedGesturePolicy(kind).fallbackPolicy.allowsXCTestCoordinateFallback(
152+
accessibilityHealth: context?.accessibilityHealth ?? runnerAccessibilityHealth
153+
)
154+
logSynthesizedGesturePolicyDecision(
155+
kind: kind,
156+
context: context,
157+
fallbackAttempted: fallbackAllowed
133158
)
159+
return fallbackAllowed
160+
? .xctestFallback(message: message, hint: hint)
161+
: .refused(timing: timing, message: message, hint: hint)
134162
}
135163

136164
func logSynthesizedGesturePolicyDecision(
@@ -162,11 +190,12 @@ extension RunnerTests {
162190
return "AGENT_DEVICE_RUNNER_SYNTHESIZED_GESTURE_POLICY kind=\(kind.rawValue)"
163191
+ " context=unavailable fallbackAttempted=\(fallbackAttempted)"
164192
}
193+
let fallbackPolicy = synthesizedGesturePolicy(kind).fallbackPolicy
165194
return "AGENT_DEVICE_RUNNER_SYNTHESIZED_GESTURE_POLICY kind=\(kind.rawValue)"
166195
+ " axHealth=\(context.accessibilityHealth.rawValue) frameSource=window"
167196
+ " keyboardPolicy=\(context.keyboardPolicy.rawValue)"
168-
+ " fallbackPolicy=\(context.fallbackPolicy.rawValue)"
169-
+ " fallbackAllowed=\(context.allowsXCTestCoordinateFallback)"
197+
+ " fallbackPolicy=\(fallbackPolicy.rawValue)"
198+
+ " fallbackAllowed=\(fallbackPolicy.allowsXCTestCoordinateFallback(accessibilityHealth: context.accessibilityHealth))"
170199
+ " fallbackAttempted=\(fallbackAttempted)"
171200
}
172201
}

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -384,7 +384,6 @@ extension RunnerTests {
384384
referenceFrame: referenceFrame,
385385
resolvedWindow: window,
386386
keyboardPolicy: policy.keyboardPolicy,
387-
fallbackPolicy: policy.fallbackPolicy,
388387
accessibilityHealth: health
389388
)
390389
#else

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedTextEntry.swift‎

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -270,11 +270,4 @@ extension RunnerTests {
270270
) -> Bool {
271271
repairMode == .replacement && hasX && hasY && xCTestChannelPenalized
272272
}
273-
274-
static func shouldFallbackFromSynthesizedTextEntryFocus(
275-
_ outcome: RunnerInteractionOutcome
276-
) -> Bool {
277-
if case .unsupported = outcome { return true }
278-
return false
279-
}
280273
}

0 commit comments

Comments
 (0)