Skip to content

Commit 19cb1fd

Browse files
authored
fix(ios): report capture refusals as typed reasons in back-fallback and recording (#2766)
* fix(ios): report capture refusals as typed reasons in back-fallback and recording #2741 routed screenshots through the resolved app display but left two failure contracts papered over (#2728): - The in-app `back` coordinate fallback read a nil screenshot sample as "no visual change" and sampled SpringBoard when the app resolved no window, so a two-home-screen comparison reported "in-app back control is not available" after a real leading tap had already run. The check now observes only the app's own resolved window and reports the display refusal as an unverified outcome; an observable no-change still reads as unavailable. - `record start` collapsed every no-frame bootstrap into an untyped NSError. It now surfaces the shared APP_SCREEN_* reason, so a runtime with no resolvable window fails closed with a typed code. Refuse a zero-pixel capture at the capture type so a required consumer cannot mistake it for a usable frame. macOS/tvOS keep their host-display capture and generic bootstrap error. Focused runner unit tests pin the observation split, the refusal-selection, and the typed-vs-generic error mapping. * test(ios): pin the capture-refusal contracts behind the code that produces them - testRecordStartThrowsTheCaptureRefusalItReceived drives ScreenRecorder.start with an always-refusing capture and maps the actual thrown error through recordingStartErrorPayload, so the bootstrap forwarding is covered (iOS PR lane). - navigationFallbackSample(resolvingApp:systemSurface:encoding:) now holds the back fallback's app-only decision; the test hands it a system surface that fails if consulted, so a revert to SpringBoard sampling turns it red. - navigationVisualSample + inAppBackOutcome keep the refusal->code mapping and the unknown-outcome decision testable on the macOS host lane.
1 parent 9918e72 commit 19cb1fd

9 files changed

Lines changed: 486 additions & 91 deletions

File tree

‎.github/workflows/ios.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -154,6 +154,7 @@ jobs:
154154
-xctestrun "$XCTESTRUN_PATH" \
155155
-destination "platform=iOS Simulator,id=${{ steps.ios-simulator.outputs.simulator-udid }}" \
156156
-only-testing:AgentDeviceRunnerUITests/RunnerTests/testSinglePointerFlingFallsBackToXCTestCoordinateDragWhenPrivateSynthesisFails \
157+
-only-testing:AgentDeviceRunnerUITests/RunnerTests/testRecordStartThrowsTheCaptureRefusalItReceived \
157158
-only-testing:AgentDeviceRunnerUITests/RunnerTests/testAlertDispatchResolvesItsOwnModalWithoutCoordinateTapRoutingProbe \
158159
-only-testing:AgentDeviceRunnerUITests/RunnerTests/testAlertResolutionCannotBypassRequestedDeadline \
159160
-only-testing:AgentDeviceRunnerUITests/RunnerTests/testTypeWithoutResolvedInputReturnsTypedFailureBeforeDispatchingText \

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAppScreenCapture.swift‎

Lines changed: 21 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,21 @@ enum RunnerAppScreenCaptureFailure: String, Error {
5454
}
5555
}
5656

57+
extension RunnerTests {
58+
/// The target rule, kept apart from the two queries so the rule itself is testable: an unresolved
59+
/// window asks the system surface, and nothing else does.
60+
func selectObservedScreenCapture(
61+
resolving: () -> Result<CapturedAppScreen, RunnerAppScreenCaptureFailure>,
62+
fallingBack: () -> Result<CapturedAppScreen, RunnerAppScreenCaptureFailure>
63+
) -> Result<CapturedAppScreen, RunnerAppScreenCaptureFailure> {
64+
let outcome = resolving()
65+
if case .failure(.unresolvedWindow) = outcome {
66+
return fallingBack()
67+
}
68+
return outcome
69+
}
70+
}
71+
5772
#if canImport(UIKit) && os(iOS)
5873
extension RunnerTests {
5974
/// Captures the display hosting `app` instead of `XCUIScreen.main`.
@@ -103,19 +118,6 @@ extension RunnerTests {
103118
)
104119
}
105120

106-
/// The target rule, kept apart from the two queries so the rule itself is testable: an unresolved
107-
/// window asks the system surface, and nothing else does.
108-
func selectObservedScreenCapture(
109-
resolving: () -> Result<CapturedAppScreen, RunnerAppScreenCaptureFailure>,
110-
fallingBack: () -> Result<CapturedAppScreen, RunnerAppScreenCaptureFailure>
111-
) -> Result<CapturedAppScreen, RunnerAppScreenCaptureFailure> {
112-
let outcome = resolving()
113-
if case .failure(.unresolvedWindow) = outcome {
114-
return fallingBack()
115-
}
116-
return outcome
117-
}
118-
119121
private static func resolveCapturedAppScreen(
120122
app: XCUIApplication
121123
) -> Result<CapturedAppScreen, RunnerAppScreenCaptureFailure> {
@@ -140,6 +142,12 @@ extension RunnerTests {
140142
guard let cgImage = runnerCGImage(from: upright) else {
141143
return .failure(.unrenderableImage)
142144
}
145+
// A zero-pixel image is a capture that did not happen, not a tiny one. Refusing it here — at the
146+
// type that owns the fact — keeps a required consumer (a recording sizing its writer from this
147+
// frame) from mistaking it for a usable frame and falling back to an untyped error (#2728).
148+
guard cgImage.width > 0, cgImage.height > 0 else {
149+
return .failure(.unrenderableImage)
150+
}
143151
return .success(
144152
CapturedAppScreen(
145153
image: upright,

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift‎

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1466,13 +1466,14 @@ extension RunnerTests {
14661466
fps: command.fps.map { Int32($0) }
14671467
)
14681468
try recorder.start { [weak self] in
1469-
return self?.captureRunnerFrame(app: activeApp)
1469+
guard let self else { return .failure(.unresolvedScreen) }
1470+
return self.captureRunnerFrameResult(app: activeApp)
14701471
}
14711472
activeRecording = recorder
14721473
return Response(ok: true, data: DataPayload(message: "recording started"))
14731474
} catch {
14741475
activeRecording = nil
1475-
return Response(ok: false, error: ErrorPayload(message: "failed to start recording: \(error.localizedDescription)"))
1476+
return Response(ok: false, error: Self.recordingStartErrorPayload(for: error))
14761477
}
14771478
case .recordStop:
14781479
guard let recorder = activeRecording else {
@@ -2120,11 +2121,20 @@ extension RunnerTests {
21202121
)
21212122
#endif
21222123
case .back, .backInApp:
2123-
if tapInAppBackControl(app: activeApp) {
2124+
switch tapInAppBackControl(app: activeApp) {
2125+
case .performed:
21242126
let message = command.command == .back ? "back" : "backInApp"
21252127
return Response(ok: true, data: DataPayload(message: message))
2128+
case .unavailable:
2129+
return Response(
2130+
ok: false,
2131+
error: ErrorPayload(message: "in-app back control is not available")
2132+
)
2133+
case .unverified(let error):
2134+
// The fallback gesture ran but the display refused to be sampled. Reporting the typed refusal
2135+
// keeps an unknown outcome from being laundered into a definitive "no back control" (#2728).
2136+
return Response(ok: false, error: error)
21262137
}
2127-
return Response(ok: false, error: ErrorPayload(message: "in-app back control is not available"))
21282138
case .backSystem:
21292139
if performSystemBackAction(app: activeApp) {
21302140
return Response(ok: true, data: DataPayload(message: "backSystem"))

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Lifecycle.swift‎

Lines changed: 39 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -42,31 +42,59 @@ enum RunnerInteractionIdleWaits {
4242
extension RunnerTests {
4343
// MARK: - Recording
4444

45-
/// One frame for the recording pump and the keyboard settle sample.
45+
/// One frame for a caller that tolerates a dropped one — keyboard settling, which skips a sample it
46+
/// cannot take and keeps polling. A frame that must exist goes through `captureRunnerFrameResult`,
47+
/// which says why it refused.
4648
///
4749
/// On iOS the frame comes from the display owning a window, because a foldable's
4850
/// `XCUIScreen.main` can be the dark outer panel while the app runs on the inner one — a stream of
49-
/// identical black frames would then read as a settled screen and as a finished recording (#2728).
50-
/// An observation with no session window falls to the system surface's window, which is what the
51-
/// home screen is. macOS keeps recording the host display the way it always has.
51+
/// identical black frames would then read as a settled screen (#2728). An observation with no
52+
/// session window falls to the system surface's window, which is what the home screen is. macOS
53+
/// keeps the host display it always recorded.
5254
func captureRunnerFrame(app: XCUIApplication) -> RunnerImage? {
53-
#if os(iOS)
54-
guard case .success(let captured) = captureObservedScreen(app: app) else {
55+
switch captureRunnerFrameResult(app: app) {
56+
case .success(let captured):
57+
return captured.image
58+
case .failure:
5559
return nil
5660
}
57-
return captured.image
61+
}
62+
63+
/// The same frame as `captureRunnerFrame`, but carrying the reason it refused, so a required first
64+
/// frame — a recording's bootstrap, which sizes the whole writer from it — fails closed with a
65+
/// typed code rather than a message. The ongoing pump reads the same result and ignores a refusal
66+
/// the way it ignored the `nil` it used to get; only a frame that must exist owes a reason (#2728).
67+
func captureRunnerFrameResult(
68+
app: XCUIApplication
69+
) -> Result<CapturedAppScreen, RunnerAppScreenCaptureFailure> {
70+
#if os(iOS)
71+
return captureObservedScreen(app: app)
5872
#else
59-
var image: RunnerImage?
73+
var outcome: Result<CapturedAppScreen, RunnerAppScreenCaptureFailure> = .failure(
74+
.unrenderableImage
75+
)
6076
let capture = {
61-
let screenshot = XCUIScreen.main.screenshot()
62-
image = screenshot.image
77+
let image = XCUIScreen.main.screenshot().image
78+
if let cgImage = runnerCGImage(from: image) {
79+
// The host display has no resolved-panel facts to report; the recorder reads only the image
80+
// and its pixel size, so these two are inert placeholders, not measurements the host scales by.
81+
outcome = .success(
82+
CapturedAppScreen(
83+
image: image,
84+
displayID: 0,
85+
pixelWidth: cgImage.width,
86+
pixelHeight: cgImage.height,
87+
pixelsPerPoint: 1
88+
)
89+
)
90+
}
6391
}
6492
if Thread.isMainThread {
6593
capture()
6694
} else {
6795
DispatchQueue.main.sync(execute: capture)
6896
}
69-
return image
97+
return outcome
7098
#endif
7199
}
72100

0 commit comments

Comments
 (0)