Skip to content

Commit ab81d9d

Browse files
thymikeethymikee
committed
fix: address iOS synthesized drag review
1 parent cdfc2fc commit ab81d9d

6 files changed

Lines changed: 184 additions & 68 deletions

File tree

‎apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift‎

Lines changed: 125 additions & 57 deletions
Original file line numberDiff line numberDiff line change
@@ -182,7 +182,7 @@ extension RunnerTests {
182182
XCTAssertNil(xctestRecordedFailureResponse(command: tapCommand, response: runnerFatalResponse))
183183
}
184184

185-
func testSkipAppActivationPreflightIncludesCoordinateOnlySynthesizedGestures() throws {
185+
func testSkipAppActivationPreflightOnlyIncludesCoordinateOnlySynthesizedTaps() throws {
186186
currentApp = app
187187
currentBundleId = nil
188188
defer {
@@ -192,25 +192,8 @@ extension RunnerTests {
192192
let tap = try runnerCommandFixture(
193193
#"{"command":"tap","commandId":"tap-1","x":10,"y":20,"synthesized":true}"#
194194
)
195-
let drag = try runnerCommandFixture(
196-
#"{"command":"drag","commandId":"drag-1","x":10,"y":20,"x2":30,"y2":40,"synthesized":true}"#
197-
)
198-
let scroll = try runnerCommandFixture(
199-
#"{"command":"scroll","commandId":"scroll-1","direction":"down","pixels":400}"#
200-
)
201-
let sequence = try runnerCommandFixture(
202-
"""
203-
{"command":"sequence","commandId":"seq-1","steps":[
204-
{"kind":"tap","x":10,"y":20,"synthesized":true},
205-
{"kind":"drag","x":10,"y":200,"x2":10,"y2":100,"synthesized":true}
206-
]}
207-
"""
208-
)
209195

210196
XCTAssertTrue(shouldSkipAppActivationPreflight(tap))
211-
XCTAssertTrue(shouldSkipAppActivationPreflight(drag))
212-
XCTAssertTrue(shouldSkipAppActivationPreflight(scroll))
213-
XCTAssertTrue(shouldSkipAppActivationPreflight(sequence))
214197
}
215198

216199
func testSkipAppActivationPreflightRejectsSelectorAndMixedSequenceGestures() throws {
@@ -250,6 +233,33 @@ extension RunnerTests {
250233
XCTAssertFalse(shouldSkipAppActivationPreflight(scroll))
251234
}
252235

236+
func testSkipAppActivationPreflightKeepsDragScrollAndSequenceOnForegroundGuard() throws {
237+
currentApp = app
238+
currentBundleId = nil
239+
defer {
240+
currentApp = nil
241+
currentBundleId = nil
242+
}
243+
let drag = try runnerCommandFixture(
244+
#"{"command":"drag","commandId":"drag-1","x":10,"y":20,"x2":30,"y2":40,"synthesized":true}"#
245+
)
246+
let scroll = try runnerCommandFixture(
247+
#"{"command":"scroll","commandId":"scroll-1","direction":"down","pixels":400}"#
248+
)
249+
let sequence = try runnerCommandFixture(
250+
"""
251+
{"command":"sequence","commandId":"seq-1","steps":[
252+
{"kind":"tap","x":10,"y":20,"synthesized":true},
253+
{"kind":"drag","x":10,"y":200,"x2":10,"y2":100,"synthesized":true}
254+
]}
255+
"""
256+
)
257+
258+
XCTAssertFalse(shouldSkipAppActivationPreflight(drag))
259+
XCTAssertFalse(shouldSkipAppActivationPreflight(scroll))
260+
XCTAssertFalse(shouldSkipAppActivationPreflight(sequence))
261+
}
262+
253263
func testExecuteDispatchedReturnsBusyBeforeMainThreadFastPath() throws {
254264
let command = try runnerCommandFixture(#"{"command":"snapshot","commandId":"snapshot-busy"}"#)
255265
abandonedMainThreadWorkCount = 1
@@ -855,7 +865,9 @@ extension RunnerTests {
855865
y2: frame.minY + plan.y2,
856866
durationMs: command.durationMs,
857867
synthesized: shouldUseSynthesizedScrollPath(),
858-
message: "scrolled"
868+
message: "scrolled",
869+
synthesizedReferenceFrame: frame,
870+
allowSynthesizedCoordinateFallback: false
859871
)
860872
case .desktopScroll:
861873
guard let direction = command.direction,
@@ -1218,7 +1230,9 @@ extension RunnerTests {
12181230
y2: Double,
12191231
durationMs: Double?,
12201232
synthesized: Bool,
1221-
message: String
1233+
message: String,
1234+
synthesizedReferenceFrame: CGRect? = nil,
1235+
allowSynthesizedCoordinateFallback: Bool = true
12221236
) -> Response {
12231237
let commandName = dragCommandName(message: message)
12241238
guard x.isFinite, y.isFinite, x2.isFinite, y2.isFinite else {
@@ -1234,7 +1248,9 @@ extension RunnerTests {
12341248
x2: x2,
12351249
y2: y2,
12361250
durationMs: durationMs,
1237-
message: message
1251+
message: message,
1252+
referenceFrame: synthesizedReferenceFrame,
1253+
allowCoordinateFallback: allowSynthesizedCoordinateFallback
12381254
) {
12391255
return synthesizedResponse
12401256
}
@@ -1295,11 +1311,33 @@ extension RunnerTests {
12951311
x2: Double,
12961312
y2: Double,
12971313
durationMs: Double?,
1298-
message: String
1314+
message: String,
1315+
referenceFrame: CGRect?,
1316+
allowCoordinateFallback: Bool
12991317
) -> Response? {
13001318
#if os(iOS)
1301-
guard let plan = axFreeSynthesizedDragPlan(app: activeApp, x: x, y: y, x2: x2, y2: y2)
1319+
guard let plan = axFreeSynthesizedDragPlan(
1320+
app: activeApp,
1321+
x: x,
1322+
y: y,
1323+
x2: x2,
1324+
y2: y2,
1325+
referenceFrame: referenceFrame,
1326+
avoidKeyboardWhenSafe: true
1327+
)
13021328
else {
1329+
if allowCoordinateFallback {
1330+
return executeCoordinateDragFallback(
1331+
activeApp: activeApp,
1332+
x: x,
1333+
y: y,
1334+
x2: x2,
1335+
y2: y2,
1336+
durationMs: durationMs,
1337+
message: message,
1338+
fallback: nil
1339+
)
1340+
}
13031341
return Response(
13041342
ok: false,
13051343
error: ErrorPayload(
@@ -1323,21 +1361,77 @@ extension RunnerTests {
13231361
y: plan.points.y,
13241362
x2: plan.points.x2,
13251363
y2: plan.points.y2,
1326-
durationMs: durationMs
1364+
durationMs: durationMs,
1365+
referenceFrame: plan.referenceFrame
13271366
)
13281367
}
13291368
if case .performed = outcome {
13301369
return gestureResponse(message: message, timing: timing, frame: .drag(dragFrame))
13311370
}
1371+
if allowCoordinateFallback {
1372+
return executeCoordinateDragFallback(
1373+
activeApp: activeApp,
1374+
x: plan.points.x,
1375+
y: plan.points.y,
1376+
x2: plan.points.x2,
1377+
y2: plan.points.y2,
1378+
durationMs: durationMs,
1379+
message: message,
1380+
fallback: gestureFallback(strategy: "xctest-coordinate-drag", from: outcome)
1381+
)
1382+
}
13321383
return unsupportedResponse(for: outcome)
13331384
#else
13341385
return nil
13351386
#endif
13361387
}
13371388

1389+
private func executeCoordinateDragFallback(
1390+
activeApp: XCUIApplication,
1391+
x: Double,
1392+
y: Double,
1393+
x2: Double,
1394+
y2: Double,
1395+
durationMs: Double?,
1396+
message: String,
1397+
fallback: GestureFallback?
1398+
) -> Response {
1399+
let dragPoints = keyboardAvoidingDragPoints(app: activeApp, x: x, y: y, x2: x2, y2: y2)
1400+
let dragFrame = resolvedDragVisualizationFrame(
1401+
app: activeApp,
1402+
x: dragPoints.x,
1403+
y: dragPoints.y,
1404+
x2: dragPoints.x2,
1405+
y2: dragPoints.y2
1406+
)
1407+
let holdDuration = synthesizedSwipeFallbackHoldDuration(durationMs: durationMs ?? 250)
1408+
let (timing, outcome) = performGesture(activeApp) {
1409+
dragAt(
1410+
app: activeApp,
1411+
x: dragPoints.x,
1412+
y: dragPoints.y,
1413+
x2: dragPoints.x2,
1414+
y2: dragPoints.y2,
1415+
holdDuration: holdDuration
1416+
)
1417+
}
1418+
if let response = unsupportedResponse(for: outcome) {
1419+
return response
1420+
}
1421+
return gestureResponse(
1422+
message: message,
1423+
timing: timing,
1424+
frame: .drag(dragFrame),
1425+
fallback: fallback
1426+
)
1427+
}
1428+
13381429
private func scrollReferenceFrame(app: XCUIApplication) -> CGRect {
13391430
#if os(iOS)
1340-
return synthesizedGestureReferenceFrame(app: app) ?? CGRect(x: 0, y: 0, width: 0, height: 0)
1431+
guard let frame = synthesizedGestureReferenceFrame(app: app) else {
1432+
return CGRect(x: 0, y: 0, width: 0, height: 0)
1433+
}
1434+
return synthesizedFrameAvoidingKeyboardWhenSafe(app: app, frame: frame)
13411435
#else
13421436
return resolvedTouchReferenceFrame(app: app, appFrame: app.frame)
13431437
#endif
@@ -1387,42 +1481,16 @@ extension RunnerTests {
13871481

13881482
private func shouldSkipAppActivationPreflight(_ command: Command) -> Bool {
13891483
#if os(iOS)
1390-
// Coordinate-only synthesized gestures can run after an AX-fatal screen because they do not
1484+
// Coordinate-only synthesized taps can run after an AX-fatal screen because they do not
13911485
// need app activation, window lookup, keyboard lookup, or element resolution. Selector/text
13921486
// interactions intentionally stay on the normal AX path because they need an element query.
1487+
// Scroll/drag/sequence keep the normal foreground guard and stabilization path.
13931488
guard command.text == nil, command.selectorKey == nil else { return false }
13941489
guard hasCachedTargetForActivationSkip(command: command) else { return false }
1395-
switch command.command {
1396-
case .tap:
1397-
return command.synthesized == true
1398-
&& command.x != nil
1399-
&& command.y != nil
1400-
case .drag:
1401-
return command.synthesized == true
1402-
&& command.x != nil
1403-
&& command.y != nil
1404-
&& command.x2 != nil
1405-
&& command.y2 != nil
1406-
case .scroll:
1407-
return true
1408-
case .sequence:
1409-
guard let steps = command.steps, !steps.isEmpty else { return false }
1410-
return steps.allSatisfy { step in
1411-
if step.kind == "tap" {
1412-
return step.synthesized == true && step.x != nil && step.y != nil
1413-
}
1414-
if step.kind == "drag" {
1415-
return step.synthesized == true
1416-
&& step.x != nil
1417-
&& step.y != nil
1418-
&& step.x2 != nil
1419-
&& step.y2 != nil
1420-
}
1421-
return false
1422-
}
1423-
default:
1424-
return false
1425-
}
1490+
return command.command == .tap
1491+
&& command.synthesized == true
1492+
&& command.x != nil
1493+
&& command.y != nil
14261494
#else
14271495
return false
14281496
#endif

‎apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift‎

Lines changed: 49 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -792,7 +792,8 @@ extension RunnerTests {
792792
y: Double,
793793
x2: Double,
794794
y2: Double,
795-
durationMs: Double
795+
durationMs: Double,
796+
referenceFrame: CGRect? = nil
796797
) -> RunnerInteractionOutcome {
797798
#if os(iOS)
798799
guard x.isFinite, y.isFinite, x2.isFinite, y2.isFinite else {
@@ -802,7 +803,7 @@ extension RunnerTests {
802803
)
803804
}
804805
let orientation = Int(RunnerSynthesizedGesture.interfaceOrientation(forApplication: app))
805-
guard let frame = synthesizedGestureReferenceFrame(app: app) else {
806+
guard let frame = referenceFrame ?? synthesizedGestureReferenceFrame(app: app) else {
806807
return .unsupported(
807808
message: "synthesized coordinate drag could not resolve a finite screen frame",
808809
hint: "Retry after the app is foregrounded, or use a plain screenshot to choose coordinates."
@@ -837,7 +838,7 @@ extension RunnerTests {
837838
#endif
838839
}
839840

840-
func synthesizedTapAt(app: XCUIApplication, x: Double, y: Double) -> RunnerInteractionOutcome {
841+
func synthesizedTapAt(app: XCUIApplication, x: Double, y: Double, referenceFrame: CGRect? = nil) -> RunnerInteractionOutcome {
841842
#if os(iOS)
842843
guard x.isFinite, y.isFinite else {
843844
return .unsupported(
@@ -846,7 +847,7 @@ extension RunnerTests {
846847
)
847848
}
848849
let orientation = Int(RunnerSynthesizedGesture.interfaceOrientation(forApplication: app))
849-
guard let frame = synthesizedGestureReferenceFrame(app: app) else {
850+
guard let frame = referenceFrame ?? synthesizedGestureReferenceFrame(app: app) else {
850851
return .unsupported(
851852
message: "synthesized coordinate tap could not resolve a finite screen frame",
852853
hint: "Retry after the app is foregrounded, or use a plain screenshot to choose coordinates."
@@ -1027,16 +1028,21 @@ extension RunnerTests {
10271028
x: Double,
10281029
y: Double,
10291030
x2: Double,
1030-
y2: Double
1031+
y2: Double,
1032+
referenceFrame: CGRect? = nil,
1033+
avoidKeyboardWhenSafe: Bool = false
10311034
) -> SynthesizedDragPlan? {
10321035
#if os(iOS)
10331036
guard x.isFinite, y.isFinite, x2.isFinite, y2.isFinite,
1034-
let frame = synthesizedGestureReferenceFrame(app: app)
1037+
let frame = referenceFrame ?? synthesizedGestureReferenceFrame(app: app)
10351038
else {
10361039
return nil
10371040
}
1041+
let points = avoidKeyboardWhenSafe
1042+
? keyboardAvoidingSynthesizedDragPoints(app: app, x: x, y: y, x2: x2, y2: y2)
1043+
: DragPoints(x: x, y: y, x2: x2, y2: y2)
10381044
return SynthesizedDragPlan(
1039-
points: DragPoints(x: x, y: y, x2: x2, y2: y2),
1045+
points: points,
10401046
referenceFrame: frame
10411047
)
10421048
#else
@@ -1061,7 +1067,7 @@ extension RunnerTests {
10611067
)
10621068
}
10631069

1064-
func synthesizedGestureReferenceFrame(app _: XCUIApplication) -> CGRect? {
1070+
func synthesizedGestureReferenceFrame(app: XCUIApplication) -> CGRect? {
10651071
#if os(iOS)
10661072
return finiteSynthesizedReferenceFrame(
10671073
appFrame: .zero,
@@ -1094,6 +1100,40 @@ extension RunnerTests {
10941100
return CGRect(x: 0, y: 0, width: screenshotSize.width, height: screenshotSize.height)
10951101
}
10961102

1103+
func synthesizedFrameAvoidingKeyboardWhenSafe(app: XCUIApplication, frame: CGRect) -> CGRect {
1104+
#if os(iOS)
1105+
guard shouldProbeKeyboardForSynthesizedGesture() else { return frame }
1106+
return frameAvoidingKeyboard(app: app, frame: frame)
1107+
#else
1108+
return frame
1109+
#endif
1110+
}
1111+
1112+
func keyboardAvoidingSynthesizedDragPoints(
1113+
app: XCUIApplication,
1114+
x: Double,
1115+
y: Double,
1116+
x2: Double,
1117+
y2: Double
1118+
) -> DragPoints {
1119+
#if os(iOS)
1120+
guard shouldProbeKeyboardForSynthesizedGesture() else {
1121+
return DragPoints(x: x, y: y, x2: x2, y2: y2)
1122+
}
1123+
return keyboardAvoidingDragPoints(app: app, x: x, y: y, x2: x2, y2: y2)
1124+
#else
1125+
return DragPoints(x: x, y: y, x2: x2, y2: y2)
1126+
#endif
1127+
}
1128+
1129+
private func shouldProbeKeyboardForSynthesizedGesture() -> Bool {
1130+
#if os(iOS)
1131+
return !lastSnapshotHadAccessibilityUnavailable
1132+
#else
1133+
return false
1134+
#endif
1135+
}
1136+
10971137
private func isUsableReferenceFrame(_ frame: CGRect) -> Bool {
10981138
return !frame.isNull
10991139
&& !frame.isEmpty
@@ -1452,7 +1492,7 @@ extension RunnerTests {
14521492
finiteSynthesizedReferenceFrame(
14531493
appFrame: .infinite,
14541494
fallbackBounds: .zero,
1455-
fallbackScreenshotSize: { CGSize(width: .infinity, height: 932) }
1495+
fallbackScreenshotSize: { CGSize(width: CGFloat.infinity, height: 932) }
14561496
)
14571497
)
14581498
}

0 commit comments

Comments
 (0)