Skip to content

Commit 7c79a37

Browse files
thymikeeclaude
andcommitted
fix(ios): settle back over a full window and keep each platform branch compiling
- Settling now needs three agreeing samples over two verification delays, at least the before-to-after gap, so a screen that changes every 0.5 s cannot pass for settled and read as the swipe's effect. The samples leave out the status bar. - A swipe cut short after touch-down is BACK_SWIPE_UNVERIFIED with a check-with-snapshot hint, never a retryable refusal. - A physical device drags from 2 pt inside the leading edge to 0.8 of the width. Driven on a simulator, that drag popped Settings > General and a React Native header-less screen 9 times in 10 each. - visionOS takes an explicit branch that refuses before dispatch, so the UITest target builds for every platform again. - The back swipe geometry mirrors the right-edge preset, held by contracts/fixtures/back-swipe-geometry.json from both sides. - The protocol doc records the backSystem removal; the flag help and commands.md describe the iOS swipe and its change threshold. Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent 734dc29 commit 7c79a37

11 files changed

Lines changed: 251 additions & 39 deletions

File tree

‎apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerAppFixtures.m‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -601,6 +601,14 @@ - (void)viewDidLoad {
601601
}
602602
}];
603603
}
604+
// `--slow-toggle` repaints the whole screen every 0.5 s, slower than one verification delay, as
605+
// a carousel does: one quiet pair of samples must not pass it for settled.
606+
if ([NSProcessInfo.processInfo.arguments containsObject:@"--slow-toggle"]) {
607+
__weak UIView *view = self.view;
608+
self.animationTimer = [NSTimer scheduledTimerWithTimeInterval:0.5 repeats:YES block:^(NSTimer *timer) {
609+
view.backgroundColor = [view.backgroundColor isEqual:UIColor.blackColor] ? UIColor.whiteColor : UIColor.blackColor;
610+
}];
611+
}
604612
}
605613

606614
if ([NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-alert-replacement-regression"]) {

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -106,18 +106,19 @@ extension RunnerTests {
106106
}
107107
}
108108

109-
/// The left-edge back swipe across `surface`, from 8 pt inside the leading edge. A simulator
110-
/// sends it as one edge-masked touch, because UIKit's interactive pop ignores an XCTest drag that
111-
/// lacks the mask; a device has no such touch and takes the XCTest drag from the same origin.
109+
/// The left-edge back swipe across `surface`. A simulator sends it as one edge-masked touch at
110+
/// `backSwipeEndpoints`. A device has no such touch and takes an XCTest drag that starts 2 pt
111+
/// inside the leading edge: driven on a simulator, that drag popped Settings > General and a React
112+
/// Native header-less screen 9 times in 10 each, where one starting 20 pt in popped 2 in 6.
112113
@MainActor
113114
func performBackGesture(app: XCUIApplication, surface: CGRect) throws -> TouchDispatch {
114115
#if targetEnvironment(simulator)
115116
return try screenEdgeBackSwipe(app: app, surface: surface)
116117
#else
117118
let target = try normalizedOffsetAnchor(app: app)
118119
let start = target.coordinate(withNormalizedOffset: CGVector(dx: 0, dy: 0.5))
119-
.withOffset(CGVector(dx: 8, dy: 0))
120-
let end = target.coordinate(withNormalizedOffset: CGVector(dx: 0.85, dy: 0.5))
120+
.withOffset(CGVector(dx: 2, dy: 0))
121+
let end = target.coordinate(withNormalizedOffset: CGVector(dx: 0.8, dy: 0.5))
121122
start.press(forDuration: 0.05, thenDragTo: end)
122123
return .sent
123124
#endif

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift‎

Lines changed: 81 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,11 @@ import XCTest
33

44
extension RunnerTests {
55
static let backSwipeVerificationDelay: TimeInterval = 0.25
6-
static let backSwipeSettleSamples = 4
6+
/// Three agreeing samples span two verification delays, at least the gap from the last before
7+
/// sample to the after sample, so a screen that changes on its own more slowly than one delay
8+
/// cannot pass for settled and then read as the swipe's effect.
9+
static let backSwipeStableSamples = 3
10+
static let backSwipeSettleSampleBudget = 6
711

812
/// What one `back` concluded. `.refused` dispatched nothing. `.unconfirmed` ran the back gesture,
913
/// or may have, without proof that it navigated back.
@@ -13,6 +17,25 @@ extension RunnerTests {
1317
case unconfirmed(ErrorPayload)
1418
}
1519

20+
/// The back swipe's endpoints and duration across `surface`. Mirrors the `right-edge` fling preset
21+
/// (`buildSwipePresetGesturePlan` and `clampGestureCoordinate` in packages/contracts/src/scroll-gesture.ts):
22+
/// 1% to 85% of the width at half the height, truncated, then held 8 pt inside the edges.
23+
/// contracts/fixtures/back-swipe-geometry.json holds both sides to the same table.
24+
static let backSwipeDurationMs = 100.0
25+
26+
static func backSwipeEndpoints(in surface: CGRect) -> (start: CGPoint, end: CGPoint) {
27+
func axis(_ size: CGFloat, percent: CGFloat) -> CGFloat {
28+
let margin: CGFloat = 8
29+
let upper = max(margin, (size - margin).rounded())
30+
return min(upper, max(margin, (size * percent / 100).rounded(.towardZero)))
31+
}
32+
let y = surface.minY + axis(surface.height, percent: 50)
33+
return (
34+
CGPoint(x: surface.minX + axis(surface.width, percent: 1), y: y),
35+
CGPoint(x: surface.minX + axis(surface.width, percent: 85), y: y)
36+
)
37+
}
38+
1639
/// The three answers a before/after visual comparison can give. `unobserved` carries the typed
1740
/// refusal that made the pair uncomparable: a display that refuses to be sampled proves nothing
1841
/// either way, and reading it as "unchanged" would claim the swipe did nothing.
@@ -48,7 +71,7 @@ extension RunnerTests {
4871
#elseif os(tvOS)
4972
_ = pressTvRemote(.menu)
5073
return .performed
51-
#else
74+
#elseif os(iOS)
5275
guard let context = try synthesizedCoordinateContext(app: app) else {
5376
return .refused(Self.backScreenRefusal(.unresolvedWindow))
5477
}
@@ -77,6 +100,15 @@ extension RunnerTests {
77100
)
78101
NSLog("AGENT_DEVICE_RUNNER_BACK_SWIPE_VERIFICATION changed=%@", observation.logToken)
79102
return Self.backSwipeOutcome(observation: observation)
103+
#else
104+
// visionOS has no edge swipe that pops a navigation stack.
105+
return .refused(
106+
ErrorPayload(
107+
code: RunnerWireErrorCode.backSwipeNotDispatched,
108+
message: "back has no gesture on this platform, so nothing was sent.",
109+
hint: "Press the visible back control by its ref."
110+
)
111+
)
80112
#endif
81113
}
82114

@@ -88,8 +120,13 @@ extension RunnerTests {
88120
return nil
89121
case .notSent(let message, let hint):
90122
return .refused(ErrorPayload(code: RunnerWireErrorCode.backSwipeNotDispatched, message: message, hint: hint))
91-
case .interrupted(let message, let hint):
92-
return .unconfirmed(ErrorPayload(code: "UNSUPPORTED_OPERATION", message: message, hint: hint))
123+
case .interrupted(let message, _):
124+
return .unconfirmed(
125+
backSwipeUnverified(
126+
because: "it was cut short after it touched down (\(message))",
127+
hint: "Check the screen with snapshot before you retry; the swipe may already have navigated."
128+
)
129+
)
93130
}
94131
}
95132
#endif
@@ -103,27 +140,28 @@ extension RunnerTests {
103140
)
104141
}
105142

106-
/// Samples until two frames a verification delay apart agree, so a screen still changing on its
107-
/// own, such as one still rendering after a launch, is not read as the swipe's effect. `settled`
108-
/// is false when the samples ran out first.
143+
/// Samples a verification delay apart until `backSwipeStableSamples` in a row agree with the
144+
/// first of them. `settled` is false when the budget ran out first.
109145
private func settledBackVisualState(
110146
app: XCUIApplication,
111147
context: SynthesizedCoordinateContext
112148
) -> Result<(sample: Data, settled: Bool), RunnerAppScreenCaptureFailure> {
113-
var previous: Data
114-
switch captureBackVisualState(app: app, context: context) {
115-
case .success(let sample): previous = sample
116-
case .failure(let refusal): return .failure(refusal)
117-
}
118-
for _ in 0..<Self.backSwipeSettleSamples {
119-
sleepFor(Self.backSwipeVerificationDelay)
149+
var stableRun: [Data] = []
150+
for index in 0..<Self.backSwipeSettleSampleBudget {
151+
if index > 0 { sleepFor(Self.backSwipeVerificationDelay) }
120152
switch captureBackVisualState(app: app, context: context) {
121-
case .success(let next) where !Self.visualSamplesDiffer(next, previous): return .success((next, true))
122-
case .success(let next): previous = next
123-
case .failure(let refusal): return .failure(refusal)
153+
case .failure(let refusal):
154+
return .failure(refusal)
155+
case .success(let sample):
156+
if let first = stableRun.first, Self.visualSamplesDiffer(first, sample) {
157+
stableRun = [sample]
158+
} else {
159+
stableRun.append(sample)
160+
}
161+
if stableRun.count >= Self.backSwipeStableSamples { return .success((sample, true)) }
124162
}
125163
}
126-
return .success((previous, false))
164+
return .success((stableRun[stableRun.count - 1], false))
127165
}
128166

129167
/// The frame the swipe compares, as a coarse grayscale grid when the display answered and the
@@ -139,7 +177,11 @@ extension RunnerTests {
139177
from: context.resolvedWindow == nil && requestLitPanel == nil
140178
? captureSoleAppScreen(app: app)
141179
: captureResolvedAppScreen(app: app, litPanel: requestLitPanel),
142-
encoding: { runnerCGImage(from: $0.image).flatMap(Self.visualGrid) }
180+
encoding: { screen in
181+
// The status bar changes on its own (the clock, signal, a recording indicator).
182+
let statusBar = screenTopSafeAreaInset() * CGFloat(screen.pixelsPerPoint)
183+
return runnerCGImage(from: screen.image).flatMap { Self.visualGrid(of: $0, excludingTop: statusBar) }
184+
}
143185
)
144186
#else
145187
return .failure(.unresolvedScreen)
@@ -163,9 +205,26 @@ extension RunnerTests {
163205
static let visualCellTolerance = 24
164206
static let visualChangedCellFraction = 0.01
165207

166-
/// The screen drawn into a small grayscale grid. A back navigation redraws most of it, while a
167-
/// spinner, a clock tick, or a material shimmer moves few cells or moves them by little.
168-
static func visualGrid(of image: CGImage) -> Data? {
208+
#if os(iOS)
209+
/// The screen's top inset, read from the runner's own window scenes: the status-bar band is a
210+
/// screen-level fact, so the runner's full-screen host window reports the one the app sees.
211+
private func screenTopSafeAreaInset() -> CGFloat {
212+
UIApplication.shared.connectedScenes
213+
.compactMap { $0 as? UIWindowScene }
214+
.flatMap(\.windows)
215+
.map(\.safeAreaInsets.top)
216+
.max() ?? 0
217+
}
218+
#endif
219+
220+
/// The screen below its top `excludingTop` pixels, drawn into a small grayscale grid. A back
221+
/// navigation redraws most of it, while a spinner, a caret, or a material shimmer moves few
222+
/// cells or moves them by little.
223+
static func visualGrid(of fullImage: CGImage, excludingTop topPixels: CGFloat = 0) -> Data? {
224+
let top = min(max(topPixels, 0), CGFloat(fullImage.height - 1))
225+
guard let image = fullImage.cropping(
226+
to: CGRect(x: 0, y: top, width: CGFloat(fullImage.width), height: CGFloat(fullImage.height) - top)
227+
) else { return nil }
169228
let (width, height) = visualGridSize
170229
var cells = [UInt8](repeating: 0, count: width * height)
171230
let drawn = cells.withUnsafeMutableBytes { buffer -> Bool in

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+ScreenEdgeGesture.swift‎

Lines changed: 6 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -49,24 +49,23 @@ extension RunnerTests {
4949
return .sent
5050
}
5151

52-
/// The left-edge back swipe as the `right-edge` gesture preset plans it: from 8 pt inside the
53-
/// leading edge to 85% of the width at mid-height, in 100 ms.
52+
/// The left-edge back swipe at `backSwipeEndpoints`, as one edge-masked touch.
5453
@MainActor
5554
func screenEdgeBackSwipe(app: XCUIApplication, surface: CGRect) throws -> TouchDispatch {
56-
let y = Double(surface.midY)
55+
let (start, end) = Self.backSwipeEndpoints(in: surface)
5756
let samples = [
58-
RunnerGestureSample(offsetMs: 0, point: RunnerGesturePoint(x: Double(surface.minX + 8), y: y)),
57+
RunnerGestureSample(offsetMs: 0, point: RunnerGesturePoint(x: Double(start.x), y: Double(start.y))),
5958
RunnerGestureSample(
60-
offsetMs: 100,
61-
point: RunnerGesturePoint(x: Double(surface.minX + surface.width * 0.85), y: y)
59+
offsetMs: Self.backSwipeDurationMs,
60+
point: RunnerGesturePoint(x: Double(end.x), y: Double(end.y))
6261
),
6362
]
6463
return try dispatchScreenEdgeTouch(app: app, plan: RunnerGesturePlan(
6564
topology: "single",
6665
intent: "fling",
6766
executionProfile: "endpoint-hold",
6867
contactOrigin: runnerScreenEdgeContactOrigin,
69-
durationMs: 100,
68+
durationMs: Self.backSwipeDurationMs,
7069
viewport: RunnerGestureViewport(
7170
x: Double(surface.minX),
7271
y: Double(surface.minY),

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+ModelsTests.swift‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -123,7 +123,7 @@ extension RunnerTests {
123123
}
124124
}
125125

126-
private func runnerContractFixture(_ name: String) throws -> Any {
126+
func runnerContractFixture(_ name: String) throws -> Any {
127127
let fixtureURL = URL(fileURLWithPath: #filePath)
128128
.deletingLastPathComponent()
129129
.deletingLastPathComponent()

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+NavigationTests.swift‎

Lines changed: 39 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,20 @@ extension RunnerTests {
4444
XCTAssertEqual(payload.code, RunnerWireErrorCode.backSwipeUnverified)
4545
}
4646

47+
/// A screen that changes on its own every 0.5 s gives one quiet pair of samples 0.25 s apart, then
48+
/// changes before the after sample. Settling needs three agreeing samples, so it is never settled
49+
/// and back reports an unknown outcome instead of crediting the swipe.
50+
@MainActor
51+
func testInAppBackOnAScreenChangingSlowerThanOneDelayIsNotPerformed() throws {
52+
_ = try launchBackFixture(["--agent-device-back-regression", "--slow-toggle"])
53+
defer { endBackFixture() }
54+
55+
guard case .unconfirmed(let payload) = try performInAppBack(app: app) else {
56+
return XCTFail("a screen that toggles on its own must not read as performed")
57+
}
58+
XCTAssertEqual(payload.code, RunnerWireErrorCode.backSwipeUnverified)
59+
}
60+
4761
@MainActor
4862
func testInAppBackDoesNotSwipeOffTheLitPanel() throws {
4963
let status = try launchBackFixture(["--agent-device-back-regression"])
@@ -103,7 +117,9 @@ extension RunnerTests {
103117
else {
104118
return XCTFail("a swipe cut short after touch-down must not claim nothing was dispatched")
105119
}
106-
XCTAssertEqual(interrupted.code, "UNSUPPORTED_OPERATION")
120+
// It may already have navigated, so it is the unknown outcome, never a retryable refusal.
121+
XCTAssertEqual(interrupted.code, RunnerWireErrorCode.backSwipeUnverified)
122+
XCTAssertTrue(interrupted.hint?.contains("snapshot") == true)
107123
}
108124

109125
/// Fails when the fixture reports a back within a short window, so a gesture the app handles
@@ -227,6 +243,28 @@ extension RunnerTests {
227243
XCTAssertTrue(Self.visualSamplesDiffer(base, Data(repeating: 100, count: 999)), "a different grid size is uncomparable")
228244
}
229245

246+
/// The runner's back swipe and the host's `right-edge` preset read the same golden table.
247+
func testBackSwipeEndpointsMatchTheHostRightEdgePreset() throws {
248+
let cases = try XCTUnwrap(runnerContractFixture("back-swipe-geometry.json") as? [[String: Any]])
249+
XCTAssertFalse(cases.isEmpty)
250+
func point(_ value: Any?) throws -> CGPoint {
251+
let object = try XCTUnwrap(value as? [String: Double])
252+
return CGPoint(x: try XCTUnwrap(object["x"]), y: try XCTUnwrap(object["y"]))
253+
}
254+
for fixture in cases {
255+
let name = fixture["name"] as? String ?? "?"
256+
let surface = try XCTUnwrap(fixture["surface"] as? [String: Double], name)
257+
let rect = CGRect(
258+
x: try XCTUnwrap(surface["x"]), y: try XCTUnwrap(surface["y"]),
259+
width: try XCTUnwrap(surface["width"]), height: try XCTUnwrap(surface["height"])
260+
)
261+
let endpoints = Self.backSwipeEndpoints(in: rect)
262+
XCTAssertEqual(endpoints.start, try point(fixture["start"]), name)
263+
XCTAssertEqual(endpoints.end, try point(fixture["end"]), name)
264+
XCTAssertEqual(Self.backSwipeDurationMs, fixture["durationMs"] as? Double, name)
265+
}
266+
}
267+
230268
func testBackSwipeOutcomeNamesNoChangeAndAnUnsampledScreenApart() {
231269
switch Self.backSwipeOutcome(observation: .unchanged) {
232270
case .unconfirmed(let payload):

‎apple/runner/RUNNER_PROTOCOL.md‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -130,6 +130,16 @@ than silently applied:
130130
{ "command": "appState", "appBundleId": "com.example.app" }
131131
```
132132

133+
```json
134+
{ "command": "backInApp", "appBundleId": "com.example.app" }
135+
```
136+
137+
`backInApp` is the only back command. On iOS the runner answers it with the left-edge back swipe and
138+
verifies a change of the app's screen; its refusals are `BACK_SWIPE_NO_CHANGE`,
139+
`BACK_SWIPE_UNVERIFIED`, and `BACK_SWIPE_NOT_DISPATCHED`. The `backSystem` command was removed
140+
(#4460). This protocol carries no version: the daemon and the runner are built from the same
141+
agent-device version, and a configured `iosXctestrunFile` must be too, so no peer still sends it.
142+
133143
`appState` answers `data.applicationState` with the named app's `XCUIApplication.State` by name
134144
(`runningForeground`, `runningBackground`, `runningBackgroundSuspended`, `notRunning`, `unknown`).
135145
It is a lifecycle read, so the activation preflight is skipped and the state reported is the one
Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
[
2+
{
3+
"name": "iPhone portrait",
4+
"surface": {
5+
"x": 0,
6+
"y": 0,
7+
"width": 402,
8+
"height": 874
9+
},
10+
"start": {
11+
"x": 8,
12+
"y": 437
13+
},
14+
"end": {
15+
"x": 341,
16+
"y": 437
17+
},
18+
"durationMs": 100
19+
},
20+
{
21+
"name": "iPad portrait",
22+
"surface": {
23+
"x": 0,
24+
"y": 0,
25+
"width": 1024,
26+
"height": 1366
27+
},
28+
"start": {
29+
"x": 10,
30+
"y": 683
31+
},
32+
"end": {
33+
"x": 870,
34+
"y": 683
35+
},
36+
"durationMs": 100
37+
},
38+
{
39+
"name": "offset window",
40+
"surface": {
41+
"x": 100,
42+
"y": 50,
43+
"width": 300,
44+
"height": 600
45+
},
46+
"start": {
47+
"x": 108,
48+
"y": 350
49+
},
50+
"end": {
51+
"x": 355,
52+
"y": 350
53+
},
54+
"durationMs": 100
55+
}
56+
]

0 commit comments

Comments
 (0)