Skip to content

Commit b9748e2

Browse files
thymikeethymikee
committed
fix(ios-runner): route synthesized display IDs through the resolved window
The gesture reference frame is booked against the window resolveRunnerWindow picks, but record creation re-walked windows.firstMatch for the display ID. With an empty window 0 beside a lit window N the reference lands on N while the gesture routes to window 0's display, or fails outright. The bridge now takes the resolved window: the gesture entry points accept it, record creation reads the display from it, and every remaining firstMatch geometry read (keyboard avoidance, touch reference frame, system edge gestures) goes through the shared resolver. The interaction anchor and its frame come from one resolution pass, keeping the coordinate offset on the window the tap was measured against. Unit tests cover empty-window-0, absent windows, and the pure window-index rule.
1 parent 8001513 commit b9748e2

9 files changed

Lines changed: 130 additions & 50 deletions

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedGesture.h‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,34 +8,42 @@ FOUNDATION_EXPORT double RunnerControlledScrollProgress(double t);
88

99
@interface RunnerSynthesizedGesture : NSObject
1010

11+
// `resolvedWindow` is the app window the caller already resolved for the gesture's reference
12+
// frame. The record's display ID is read from that same window so geometry and routing can never
13+
// name different windows; pass nil to fall back to the application's first window.
1114
+ (NSString * _Nullable)synthesizeSwipeWithApplication:(id)application
15+
resolvedWindow:(id _Nullable)resolvedWindow
1216
x:(double)x
1317
y:(double)y
1418
x2:(double)x2
1519
y2:(double)y2
1620
durationMs:(double)durationMs;
1721

1822
+ (NSString * _Nullable)synthesizeContinuousDragWithApplication:(id)application
23+
resolvedWindow:(id _Nullable)resolvedWindow
1924
x:(double)x
2025
y:(double)y
2126
x2:(double)x2
22-
y2:(double)y2
27+
y2:(double)y2
2328
durationMs:(double)durationMs;
2429

2530
+ (NSString * _Nullable)synthesizeControlledScrollWithApplication:(id)application
31+
resolvedWindow:(id _Nullable)resolvedWindow
2632
x:(double)x
2733
y:(double)y
2834
x2:(double)x2
29-
y2:(double)y2
35+
y2:(double)y2
3036
durationMs:(double)durationMs;
3137

3238
+ (NSString * _Nullable)synthesizeTapWithApplication:(id)application
39+
resolvedWindow:(id _Nullable)resolvedWindow
3340
x:(double)x
3441
y:(double)y;
3542

3643
// Each pointer is an ordered array of { x, y, offsetMs } samples. The first sample
3744
// starts contact; subsequent samples move it; all pointers lift at their final offset.
3845
+ (NSString * _Nullable)synthesizeGestureWithApplication:(id)application
46+
resolvedWindow:(id _Nullable)resolvedWindow
3947
pointerSamples:(NSArray<NSArray<NSDictionary<NSString *, NSNumber *> *> *> *)pointerSamples;
4048

4149
// UIInterfaceOrientation of the app (1 portrait, 2 upsideDown, 3 landscapeRight,

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedGesture.m‎

Lines changed: 27 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@ typedef id (*RunnerDragPointerPathFactory)(
3737
);
3838
static NSString * _Nullable RunnerCreateEventRecord(
3939
id application,
40+
id _Nullable resolvedWindow,
4041
NSString *recordName,
4142
RunnerGestureEventBridge *bridge,
4243
id *record
@@ -74,13 +75,14 @@ static id RunnerSampledDragPointerPath(
7475
static double RunnerSmoothstepProgress(double t);
7576
static NSString * _Nullable RunnerTrySynthesizeDrag(
7677
id application,
78+
id _Nullable resolvedWindow,
7779
CGPoint start,
7880
CGPoint end,
7981
double durationMs,
8082
NSString *recordName,
8183
RunnerDragPointerPathFactory pathFactory
8284
);
83-
static NSString * _Nullable RunnerTrySynthesizeTap(id application, CGPoint point);
85+
static NSString * _Nullable RunnerTrySynthesizeTap(id application, id _Nullable resolvedWindow, CGPoint point);
8486
// XCTest's proven swipe profile reaches the endpoint in 100 ms, then holds for the planned
8587
// fling duration. Fast movement is what lets UIKit distinguish a fling from a timed pan.
8688
static const NSTimeInterval RunnerSwipeMovementDurationSeconds = 0.1;
@@ -94,14 +96,16 @@ static id RunnerTapPointerPath(
9496
@implementation RunnerSynthesizedGesture
9597

9698
+ (NSString * _Nullable)synthesizeSwipeWithApplication:(id)application
97-
x:(double)x
98-
y:(double)y
99-
x2:(double)x2
100-
y2:(double)y2
99+
resolvedWindow:(id _Nullable)resolvedWindow
100+
x:(double)x
101+
y:(double)y
102+
x2:(double)x2
103+
y2:(double)y2
101104
durationMs:(double)durationMs {
102105
@try {
103106
return RunnerTrySynthesizeDrag(
104107
application,
108+
resolvedWindow,
105109
CGPointMake(x, y),
106110
CGPointMake(x2, y2),
107111
durationMs,
@@ -114,6 +118,7 @@ + (NSString * _Nullable)synthesizeSwipeWithApplication:(id)application
114118
}
115119

116120
+ (NSString * _Nullable)synthesizeContinuousDragWithApplication:(id)application
121+
resolvedWindow:(id _Nullable)resolvedWindow
117122
x:(double)x
118123
y:(double)y
119124
x2:(double)x2
@@ -122,6 +127,7 @@ + (NSString * _Nullable)synthesizeContinuousDragWithApplication:(id)application
122127
@try {
123128
return RunnerTrySynthesizeDrag(
124129
application,
130+
resolvedWindow,
125131
CGPointMake(x, y),
126132
CGPointMake(x2, y2),
127133
durationMs,
@@ -134,14 +140,16 @@ + (NSString * _Nullable)synthesizeContinuousDragWithApplication:(id)application
134140
}
135141

136142
+ (NSString * _Nullable)synthesizeControlledScrollWithApplication:(id)application
143+
resolvedWindow:(id _Nullable)resolvedWindow
137144
x:(double)x
138145
y:(double)y
139146
x2:(double)x2
140-
y2:(double)y2
141-
durationMs:(double)durationMs {
147+
y2:(double)y2
148+
durationMs:(double)durationMs {
142149
@try {
143150
return RunnerTrySynthesizeDrag(
144151
application,
152+
resolvedWindow,
145153
CGPointMake(x, y),
146154
CGPointMake(x2, y2),
147155
durationMs,
@@ -154,22 +162,25 @@ + (NSString * _Nullable)synthesizeControlledScrollWithApplication:(id)applicatio
154162
}
155163

156164
+ (NSString * _Nullable)synthesizeTapWithApplication:(id)application
165+
resolvedWindow:(id _Nullable)resolvedWindow
157166
x:(double)x
158167
y:(double)y {
159168
@try {
160-
return RunnerTrySynthesizeTap(application, CGPointMake(x, y));
169+
return RunnerTrySynthesizeTap(application, resolvedWindow, CGPointMake(x, y));
161170
} @catch (NSException *exception) {
162171
return RunnerFormatXCTestException(exception, @"private XCTest event synthesis failed");
163172
}
164173
}
165174

166175
+ (NSString * _Nullable)synthesizeGestureWithApplication:(id)application
176+
resolvedWindow:(id _Nullable)resolvedWindow
167177
pointerSamples:(NSArray<NSArray<NSDictionary<NSString *, NSNumber *> *> *> *)pointerSamples {
168178
@try {
169179
RunnerGestureEventBridge bridge;
170180
id record = nil;
171181
NSString *error = RunnerCreateEventRecord(
172182
application,
183+
resolvedWindow,
173184
@"agent-device-gesture-plan",
174185
&bridge,
175186
&record
@@ -224,6 +235,7 @@ + (NSInteger)interfaceOrientationForApplication:(id)application {
224235

225236
static NSString * _Nullable RunnerTrySynthesizeDrag(
226237
id application,
238+
id _Nullable resolvedWindow,
227239
CGPoint start,
228240
CGPoint end,
229241
double durationMs,
@@ -232,7 +244,7 @@ + (NSInteger)interfaceOrientationForApplication:(id)application {
232244
) {
233245
RunnerGestureEventBridge bridge;
234246
id record = nil;
235-
NSString *error = RunnerCreateEventRecord(application, recordName, &bridge, &record);
247+
NSString *error = RunnerCreateEventRecord(application, resolvedWindow, recordName, &bridge, &record);
236248
if (error != nil) return error;
237249

238250
id path = pathFactory(&bridge, start, end, durationMs);
@@ -244,11 +256,12 @@ + (NSInteger)interfaceOrientationForApplication:(id)application {
244256
return RunnerSynthesizeEventRecord(&bridge, record);
245257
}
246258

247-
static NSString * _Nullable RunnerTrySynthesizeTap(id application, CGPoint point) {
259+
static NSString * _Nullable RunnerTrySynthesizeTap(id application, id _Nullable resolvedWindow, CGPoint point) {
248260
RunnerGestureEventBridge bridge;
249261
id record = nil;
250262
NSString *error = RunnerCreateEventRecord(
251263
application,
264+
resolvedWindow,
252265
@"agent-device-tap",
253266
&bridge,
254267
&record
@@ -311,6 +324,7 @@ + (NSInteger)interfaceOrientationForApplication:(id)application {
311324

312325
static NSString * _Nullable RunnerCreateEventRecord(
313326
id application,
327+
id _Nullable resolvedWindow,
314328
NSString *recordName,
315329
RunnerGestureEventBridge *bridge,
316330
id *record
@@ -319,7 +333,9 @@ + (NSInteger)interfaceOrientationForApplication:(id)application {
319333
if (missing != nil) return missing;
320334

321335
NSUInteger displayID = 0;
322-
NSString *displayError = RunnerResolveApplicationDisplayID(application, &displayID);
336+
NSString *displayError = resolvedWindow != nil
337+
? RunnerResolveWindowDisplayID(resolvedWindow, &displayID)
338+
: RunnerResolveApplicationDisplayID(application, &displayID);
323339
if (displayError != nil) return displayError;
324340
NSInteger interfaceOrientation =
325341
((RunnerMsgSendInteger)objc_msgSend)(application, bridge->interfaceOrientationSelector);

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -5,9 +5,10 @@ import AgentDeviceSnapshotPresentation
55
import ObjectiveC.runtime
66

77
private final class RunnerSynthesizedSwipeFailureStub: NSObject {
8-
@objc(synthesizeSwipeWithApplication:x:y:x2:y2:durationMs:)
8+
@objc(synthesizeSwipeWithApplication:resolvedWindow:x:y:x2:y2:durationMs:)
99
class func synthesizeSwipe(
1010
application: XCUIApplication,
11+
resolvedWindow: Any?,
1112
x: Double,
1213
y: Double,
1314
x2: Double,
@@ -19,8 +20,8 @@ private final class RunnerSynthesizedSwipeFailureStub: NSObject {
1920
}
2021

2122
private final class RunnerSynthesizedTapFailureStub: NSObject {
22-
@objc(synthesizeTapWithApplication:x:y:)
23-
class func synthesizeTap(application: XCUIApplication, x: Double, y: Double) -> String? {
23+
@objc(synthesizeTapWithApplication:resolvedWindow:x:y:)
24+
class func synthesizeTap(application: XCUIApplication, resolvedWindow: Any?, x: Double, y: Double) -> String? {
2425
"forced private synthesis failure"
2526
}
2627
}
@@ -261,7 +262,7 @@ extension RunnerTests {
261262
#if os(iOS)
262263
func testSinglePointerFlingFallsBackToXCTestCoordinateDragWhenPrivateSynthesisFails() throws {
263264
let selector = NSSelectorFromString(
264-
"synthesizeSwipeWithApplication:x:y:x2:y2:durationMs:"
265+
"synthesizeSwipeWithApplication:resolvedWindow:x:y:x2:y2:durationMs:"
265266
)
266267
guard
267268
let synthesizedSwipeMethod = class_getClassMethod(RunnerSynthesizedGesture.self, selector),
@@ -305,7 +306,7 @@ extension RunnerTests {
305306
}
306307

307308
func testSelectorTapFallsBackToXCTestCoordinateWhenPrivateSynthesisFails() throws {
308-
let selector = NSSelectorFromString("synthesizeTapWithApplication:x:y:")
309+
let selector = NSSelectorFromString("synthesizeTapWithApplication:resolvedWindow:x:y:")
309310
guard
310311
let synthesizedTapMethod = class_getClassMethod(RunnerSynthesizedGesture.self, selector),
311312
let failureStubMethod = class_getClassMethod(RunnerSynthesizedTapFailureStub.self, selector)

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift‎

Lines changed: 27 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -76,7 +76,7 @@ extension RunnerTests {
7676

7777
private func performCoordinateBackGesture(app: XCUIApplication) {
7878
#if !os(tvOS)
79-
let target = app.windows.firstMatch.exists ? app.windows.firstMatch : app
79+
let target = resolveRunnerWindow(app: app).window ?? app
8080
let start = target.coordinate(withNormalizedOffset: CGVector(dx: 0.05, dy: 0.5))
8181
let end = target.coordinate(withNormalizedOffset: CGVector(dx: 0.8, dy: 0.5))
8282
start.press(forDuration: 0.05, thenDragTo: end)
@@ -106,7 +106,7 @@ extension RunnerTests {
106106

107107
private func performCoordinateAppSwitcherGesture(app: XCUIApplication) {
108108
#if !os(tvOS)
109-
let target = app.windows.firstMatch.exists ? app.windows.firstMatch : app
109+
let target = resolveRunnerWindow(app: app).window ?? app
110110
let start = target.coordinate(withNormalizedOffset: CGVector(dx: 0.5, dy: 0.99))
111111
let end = target.coordinate(withNormalizedOffset: CGVector(dx: 0.5, dy: 0.7))
112112
start.press(forDuration: 0.6, thenDragTo: end)
@@ -220,20 +220,30 @@ extension RunnerTests {
220220
// out-of-window coordinates still pass containment. Falls back to app.frame
221221
// when no window frame is readable.
222222
/// The app window interactions and captures are booked against: the first window the app
223-
/// reports with a non-empty frame, the application itself when none qualifies. The viewport,
224-
/// the interaction root, and the synthesized reference frame all resolve through here, so a
223+
/// reports with a non-empty frame, and no window at all when none qualifies. The viewport,
224+
/// the interaction anchor, and the synthesized reference frame all resolve through here, so a
225225
/// fold, a sheet, or a rotation moves every consumer to the same window in the same pass.
226-
/// On iOS the resolved element must be the one whose `screen.displayID` a gesture routes by;
227-
/// reading the frame first is what makes that screen read mean the resolved window.
228-
func resolveRunnerWindow(app: XCUIApplication) -> (element: XCUIElement, frame: CGRect) {
229-
for window in app.windows.allElementsBoundByIndex {
230-
guard window.exists else { continue }
231-
let frame = window.frame
232-
if !frame.isEmpty {
233-
return (window, frame)
226+
/// Synthesized gestures route their display ID through the returned window so the record's
227+
/// display can never name a different window than the one the reference frame was measured on.
228+
func resolveRunnerWindow(app: XCUIApplication) -> (window: XCUIElement?, frame: CGRect) {
229+
let windows = app.windows.allElementsBoundByIndex
230+
var frames: [CGRect?] = []
231+
for window in windows {
232+
frames.append(window.exists ? window.frame : nil)
233+
}
234+
if let index = Self.firstUsableWindowIndex(frames: frames) {
235+
return (windows[index], frames[index] ?? .zero)
236+
}
237+
return (nil, app.frame)
238+
}
239+
240+
static func firstUsableWindowIndex(frames: [CGRect?]) -> Int? {
241+
for (index, frame) in frames.enumerated() {
242+
if let frame, !frame.isEmpty {
243+
return index
234244
}
235245
}
236-
return (app, app.frame)
246+
return nil
237247
}
238248

239249
func onScreenWindowFrame(app: XCUIApplication) -> CGRect {
@@ -515,10 +525,6 @@ extension RunnerTests {
515525
return performCoordinateDrag(app: app, x: x, y: y, x2: x2, y2: y2, holdDuration: holdDuration)
516526
}
517527

518-
private func interactionRoot(app: XCUIApplication) -> XCUIElement {
519-
resolveRunnerWindow(app: app).element
520-
}
521-
522528
private func performCoordinateTap(app: XCUIApplication, x: Double, y: Double) -> RunnerInteractionOutcome {
523529
#if os(tvOS)
524530
return .unsupported(
@@ -578,11 +584,11 @@ extension RunnerTests {
578584

579585
#if !os(tvOS)
580586
private func interactionCoordinate(app: XCUIApplication, x: Double, y: Double) -> XCUICoordinate {
581-
let root = interactionRoot(app: app)
587+
let resolved = resolveRunnerWindow(app: app)
588+
let root = resolved.window ?? app
582589
let origin = root.coordinate(withNormalizedOffset: CGVector(dx: 0, dy: 0))
583-
let rootFrame = root.frame
584-
let offsetX = x - Double(rootFrame.origin.x)
585-
let offsetY = y - Double(rootFrame.origin.y)
590+
let offsetX = x - Double(resolved.frame.origin.x)
591+
let offsetY = y - Double(resolved.frame.origin.y)
586592
return origin.withOffset(CGVector(dx: offsetX, dy: offsetY))
587593
}
588594
#endif

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedGesturePolicy.swift‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,13 +59,17 @@ struct SynthesizedGesturePolicy: Equatable, Hashable {
5959

6060
struct SynthesizedCoordinateContext {
6161
let referenceFrame: CGRect
62+
/// The window `referenceFrame` was measured on. Synthesized records route their display ID
63+
/// through this same window so geometry and routing can never name different windows.
64+
let resolvedWindow: XCUIElement?
6265
let keyboardPolicy: SynthesizedKeyboardPolicy
6366
let fallbackPolicy: SynthesizedFallbackPolicy
6467
let accessibilityHealth: RunnerAccessibilityHealth
6568

6669
func withReferenceFrame(_ frame: CGRect) -> SynthesizedCoordinateContext {
6770
SynthesizedCoordinateContext(
6871
referenceFrame: frame,
72+
resolvedWindow: resolvedWindow,
6973
keyboardPolicy: keyboardPolicy,
7074
fallbackPolicy: fallbackPolicy,
7175
accessibilityHealth: accessibilityHealth

0 commit comments

Comments
 (0)