Skip to content

Commit e9f7502

Browse files
thymikeeclaude
andcommitted
fix(ios): route synthesized gestures to the sole screen when no window resolves
On a deep app tree (Bluesky's home feed at depth 62) XCTest's windows query fails with kAXErrorIllegalArgument. Synthesized taps and scrolls need no tree, but they could not name a window: the runner paid about 2s of XCTest retries per app.frame read, then failed (scroll after 7.8s) or fell back to an XCTest coordinate tap that crashed the runner after 18s. When no window resolves on an iPhone with exactly one screen, take the screen in the app's interface orientation as the reference frame and route the record to that screen. Other devices keep requiring a window. Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent 6ac6611 commit e9f7502

7 files changed

Lines changed: 138 additions & 11 deletions

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedGesture.m‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -331,10 +331,10 @@ static RunnerTapSynthesisStatus RunnerTrySynthesizeTap(
331331
if (missing != nil) return missing;
332332

333333
NSUInteger displayID = 0;
334-
if (resolvedWindow == nil) {
335-
return @"private XCTest event synthesis unavailable: no resolved application window";
336-
}
337-
NSString *displayError = RunnerResolveWindowDisplayID(resolvedWindow, &displayID);
334+
// A nil window is the caller's sole-screen route: the app tree could not name its window.
335+
NSString *displayError = resolvedWindow == nil
336+
? RunnerResolveSoleScreenDisplayID(&displayID)
337+
: RunnerResolveWindowDisplayID(resolvedWindow, &displayID);
338338
if (displayError != nil) return displayError;
339339
NSInteger interfaceOrientation =
340340
((RunnerMsgSendInteger)objc_msgSend)(application, bridge->interfaceOrientationSelector);

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -259,13 +259,23 @@ extension RunnerTests {
259259
/// records, and the chosen window is read again on its way to a display ID
260260
/// (`RunnerResolveWindowDisplayID`).
261261
func resolveRunnerWindow(app: XCUIApplication) -> (window: XCUIElement?, frame: CGRect) {
262+
guard let resolved = resolveRunnerWindowElement(app: app) else {
263+
return (nil, app.frame)
264+
}
265+
return (resolved.window, resolved.frame)
266+
}
267+
268+
/// The app's first window with a usable frame, or nil. Unlike `resolveRunnerWindow` it never falls
269+
/// back to `app.frame`: when the windows query fails because XCTest cannot serialize the app's
270+
/// tree, that read fails the same way after XCTest's own retries (about 2s on Bluesky's feed).
271+
func resolveRunnerWindowElement(app: XCUIApplication) -> (window: XCUIElement, frame: CGRect)? {
262272
for window in app.windows.allElementsBoundByIndex where window.exists {
263273
let frame = window.frame
264274
if !frame.isEmpty {
265275
return (window, frame)
266276
}
267277
}
268-
return (nil, app.frame)
278+
return nil
269279
}
270280

271281
func onScreenWindowFrame(app: XCUIApplication) -> CGRect {

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedGesturePolicy.swift‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -57,8 +57,10 @@ struct SynthesizedGesturePolicy: Equatable, Hashable {
5757
struct SynthesizedCoordinateContext {
5858
let referenceFrame: CGRect
5959
/// The window `referenceFrame` was measured on. Synthesized records route their display ID
60-
/// through this same window so geometry and routing can never name different windows.
61-
let resolvedWindow: XCUIElement
60+
/// through this same window so geometry and routing can never name different windows. Nil is the
61+
/// sole-screen route: XCTest could not resolve the window, `referenceFrame` is the device's only
62+
/// screen, and the record is routed to that screen.
63+
let resolvedWindow: XCUIElement?
6264
let keyboardPolicy: SynthesizedKeyboardPolicy
6365
let accessibilityHealth: RunnerAccessibilityHealth
6466

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift‎

Lines changed: 56 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -388,11 +388,17 @@ extension RunnerTests {
388388
) -> SynthesizedCoordinateContext? {
389389
#if os(iOS)
390390
let health = mainOwned.accessibilityHealth
391-
let resolved = resolveRunnerWindow(app: app)
392-
guard let window = resolved.window else {
391+
let window: XCUIElement?
392+
let referenceFrame: CGRect
393+
if let resolved = resolveRunnerWindowElement(app: app) {
394+
window = resolved.window
395+
referenceFrame = resolved.frame
396+
} else if let soleScreenFrame = soleScreenReferenceFrame(app: app) {
397+
window = nil
398+
referenceFrame = soleScreenFrame
399+
} else {
393400
return nil
394401
}
395-
let referenceFrame = resolved.frame
396402
guard referenceFrame.width.isFinite, referenceFrame.height.isFinite,
397403
referenceFrame.width > 0, referenceFrame.height > 0
398404
else {
@@ -410,6 +416,52 @@ extension RunnerTests {
410416
}
411417

412418

419+
#if os(iOS)
420+
/// The reference frame of the sole-screen route, read without the app's accessibility tree: on an
421+
/// iPhone with one screen a full-screen app's window is that screen. A deep app tree makes the
422+
/// windows query fail with `kAXErrorIllegalArgument`, and without this route every synthesized
423+
/// tap and scroll on such a screen failed even though synthesis itself needs no tree. Anything
424+
/// that can show an app in part of a screen (iPad, more than one screen) keeps requiring a window.
425+
@MainActor
426+
func soleScreenReferenceFrame(app: XCUIApplication) -> CGRect? {
427+
guard UIDevice.current.userInterfaceIdiom == .phone, XCUIScreen.screens.count == 1 else {
428+
return nil
429+
}
430+
let screen = UIScreen.main
431+
let scale = screen.nativeScale
432+
guard scale > 0 else { return nil }
433+
let frame = Self.soleScreenReferenceFrame(
434+
nativePortraitSize: CGSize(
435+
width: screen.nativeBounds.width / scale,
436+
height: screen.nativeBounds.height / scale
437+
),
438+
interfaceOrientation: Int(RunnerSynthesizedGesture.interfaceOrientation(forApplication: app))
439+
)
440+
NSLog(
441+
"AGENT_DEVICE_RUNNER_SYNTHESIZED_SOLE_SCREEN_ROUTE resolved=%@",
442+
frame.map { NSCoder.string(for: $0) } ?? "none"
443+
)
444+
return frame
445+
}
446+
#endif
447+
448+
/// The screen in the app's interface orientation, anchored at the origin like a window frame. An
449+
/// unknown orientation cannot say which way to lay the screen, so it names no frame.
450+
static func soleScreenReferenceFrame(
451+
nativePortraitSize size: CGSize,
452+
interfaceOrientation: Int
453+
) -> CGRect? {
454+
guard size.width > 0, size.height > 0 else { return nil }
455+
switch interfaceOrientation {
456+
case RunnerInterfaceOrientation.portrait, RunnerInterfaceOrientation.portraitUpsideDown:
457+
return CGRect(origin: .zero, size: size)
458+
case RunnerInterfaceOrientation.landscapeLeft, RunnerInterfaceOrientation.landscapeRight:
459+
return CGRect(x: 0, y: 0, width: size.height, height: size.width)
460+
default:
461+
return nil
462+
}
463+
}
464+
413465
func keyboardAvoidingSynthesizedDragPoints(
414466
app: XCUIApplication,
415467
x: Double,
@@ -594,7 +646,7 @@ extension RunnerTests {
594646
]
595647
}
596648
}
597-
let resolvedWindow = resolveRunnerWindow(app: app).window
649+
let resolvedWindow = resolveRunnerWindowElement(app: app)?.window
598650
if let message = RunnerSynthesizedGesture.synthesizeGesture(
599651
withApplication: app,
600652
resolvedWindow: resolvedWindow,

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.h‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,13 @@ FOUNDATION_EXPORT NSString * _Nullable RunnerResolveWindowDisplayID(
7777
NSUInteger *displayID
7878
);
7979

80+
// Reads the display ID of the device's only screen, for a gesture whose app window XCTest cannot
81+
// resolve because the app's accessibility tree is too deep to serialize. With exactly one screen
82+
// that screen hosts the app, so no window is needed to name it; with any other count this returns
83+
// a message and leaves `displayID` untouched, because only a window can say which panel lights the
84+
// app.
85+
FOUNDATION_EXPORT NSString * _Nullable RunnerResolveSoleScreenDisplayID(NSUInteger *displayID);
86+
8087
FOUNDATION_EXPORT NSString * _Nullable RunnerRequireClass(
8188
Class cls,
8289
NSString *className,

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.m‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
#import "RunnerXCTestEventBridge.h"
22
#import "RunnerObjCExceptionCatcher.h"
33
#import <CoreGraphics/CoreGraphics.h>
4+
#import <XCTest/XCTest.h>
45
#import <math.h>
56

67
NSString * _Nullable RunnerResolveXCTestEventBridge(
@@ -214,6 +215,30 @@ BOOL RunnerResolveApplicationScreen(
214215
// routed by the same window its reference frame was measured on. The window still resolves here:
215216
// its frame is read before its screen, and a window that answers empty or raises fails closed with
216217
// the same reasons the application walk gives.
218+
NSString * _Nullable RunnerResolveSoleScreenDisplayID(NSUInteger *displayID) {
219+
__block NSUInteger screenCount = 0;
220+
__block NSUInteger readDisplayID = 0;
221+
NSString *exception = [RunnerObjCExceptionCatcher catchException:^{
222+
NSArray<XCUIScreen *> *screens = XCUIScreen.screens;
223+
screenCount = screens.count;
224+
if (screenCount != 1) return;
225+
NSNumber *identifier = [screens.firstObject valueForKey:@"displayID"];
226+
if ([identifier isKindOfClass:NSNumber.class]) {
227+
readDisplayID = identifier.unsignedIntegerValue;
228+
}
229+
}];
230+
if (exception != nil || screenCount != 1 || readDisplayID == 0) {
231+
NSLog(
232+
@"AGENT_DEVICE_RUNNER_APP_SCREEN_UNRESOLVED stage=sole-screen screens=%lu raised=%@",
233+
(unsigned long)screenCount,
234+
exception != nil ? @"yes" : @"no"
235+
);
236+
return @"private XCTest event synthesis unavailable: no resolved application window and no sole screen to route to";
237+
}
238+
*displayID = readDisplayID;
239+
return nil;
240+
}
241+
217242
NSString * _Nullable RunnerResolveWindowDisplayID(id window, NSUInteger *displayID) {
218243
__block CGRect frame = CGRectNull;
219244
NSString *frameException = [RunnerObjCExceptionCatcher catchException:^{

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedInteractionTests.swift‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,37 @@ import AgentDeviceSnapshotPresentation
33

44
#if AGENT_DEVICE_RUNNER_UNIT_TESTS
55
extension RunnerTests {
6+
func testSoleScreenReferenceFrameLaysTheScreenInTheAppsOrientation() {
7+
let portrait = CGSize(width: 402, height: 874)
8+
for orientation in [RunnerInterfaceOrientation.portrait, RunnerInterfaceOrientation.portraitUpsideDown] {
9+
XCTAssertEqual(
10+
Self.soleScreenReferenceFrame(nativePortraitSize: portrait, interfaceOrientation: orientation),
11+
CGRect(x: 0, y: 0, width: 402, height: 874)
12+
)
13+
}
14+
for orientation in [RunnerInterfaceOrientation.landscapeLeft, RunnerInterfaceOrientation.landscapeRight] {
15+
XCTAssertEqual(
16+
Self.soleScreenReferenceFrame(nativePortraitSize: portrait, interfaceOrientation: orientation),
17+
CGRect(x: 0, y: 0, width: 874, height: 402)
18+
)
19+
}
20+
}
21+
22+
func testSoleScreenReferenceFrameRefusesAnUnknownOrientationOrAnEmptyScreen() {
23+
XCTAssertNil(
24+
Self.soleScreenReferenceFrame(
25+
nativePortraitSize: CGSize(width: 402, height: 874),
26+
interfaceOrientation: RunnerInterfaceOrientation.unknown
27+
)
28+
)
29+
XCTAssertNil(
30+
Self.soleScreenReferenceFrame(
31+
nativePortraitSize: .zero,
32+
interfaceOrientation: RunnerInterfaceOrientation.portrait
33+
)
34+
)
35+
}
36+
637
func testSynthesizedDispatchInvertsCaptureOnEveryInterfaceOrientation() {
738
let frames = [
839
CGRect(x: 0, y: 0, width: 430, height: 932),

0 commit comments

Comments
 (0)