Skip to content

Commit 00bec14

Browse files
fix(ios): capture the display that owns a window, not the main screen
A foldable lights one panel at a time while XCUIScreen.main names one fixed panel, so an app on the other panel was captured as a valid PNG of black. The runner now asks the target app's window first and the system surface's window second, reads the display off that same window instance, and encodes the capture upright, reporting the display ID, the pixel size it encoded, and the pixels-per-point of that image so the host normalizes density from the capture instead of a panel it guessed. The home screen is SpringBoard's window, so a capture with no session app still has a display that owns a window; a capture fails closed with a typed reason only when neither resolves one. An app with no window is refused on the query's own answer rather than on geometry alone. The screenshot command skips the app-activation preflight, so it resolves the requested bundle id for observation without ever foregrounding an app to see which display it is on. Co-authored-by: Apex <[email protected]>
1 parent b9748e2 commit 00bec14

17 files changed

Lines changed: 1153 additions & 114 deletions
Lines changed: 175 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,175 @@
1+
import XCTest
2+
#if canImport(UIKit)
3+
import UIKit
4+
#endif
5+
6+
/// One capture of the display that actually hosts the target app, with the facts needed to read its
7+
/// pixels without guessing a panel: which display was captured, how many pixels the encoded image
8+
/// holds, and how many of them represent one point.
9+
struct CapturedAppScreen {
10+
let image: RunnerImage
11+
let displayID: UInt
12+
let pixelWidth: Int
13+
let pixelHeight: Int
14+
let pixelsPerPoint: Double
15+
}
16+
17+
/// Why the runner could not name the display hosting the target app. Carried as a reason code
18+
/// rather than as message text so a required capture can fail closed with a code the host keys on,
19+
/// and an optional observation can report "unknown" without parsing a sentence (#2728).
20+
enum RunnerAppScreenCaptureFailure: String, Error {
21+
case unresolvedWindow = "APP_SCREEN_WINDOW_UNRESOLVED"
22+
case unresolvedScreen = "APP_SCREEN_UNRESOLVED"
23+
case unrenderableImage = "APP_SCREEN_CAPTURE_UNRENDERABLE"
24+
25+
init(_ failure: RunnerApplicationScreenFailure) {
26+
switch failure {
27+
case .unresolvedWindow:
28+
self = .unresolvedWindow
29+
default:
30+
self = .unresolvedScreen
31+
}
32+
}
33+
34+
var message: String {
35+
switch self {
36+
case .unresolvedWindow:
37+
return "Neither the target app nor the system surface resolved a window, so no display could be named."
38+
case .unresolvedScreen:
39+
return "The resolved app window did not report the display it is shown on."
40+
case .unrenderableImage:
41+
return "The resolved display handed back an image that could not be drawn upright."
42+
}
43+
}
44+
45+
var hint: String {
46+
switch self {
47+
case .unresolvedWindow:
48+
return "The capture follows the display that owns a window, so this is a runtime that is answering no window for any process; runner.log names the window frame and the query answer it refused."
49+
case .unresolvedScreen:
50+
return "Retry the capture. If it persists, the XCTest runtime is not reporting the window's display; runner.log names the window frame it resolved."
51+
case .unrenderableImage:
52+
return "Retry the capture. The display was resolved, so this is the capture itself failing, not a panel question."
53+
}
54+
}
55+
}
56+
57+
#if canImport(UIKit) && os(iOS)
58+
extension RunnerTests {
59+
/// Captures the display hosting `app` instead of `XCUIScreen.main`.
60+
///
61+
/// On a foldable, `XCUIScreen.main` names one fixed panel while the app can be on the other, and
62+
/// the panel that is dark still answers with a valid PNG of black — a capture that looks like
63+
/// evidence and proves nothing. The window's own screen is the only source that follows the app,
64+
/// and on a panel the system presents turned it hands back an image whose buffer is sideways, so
65+
/// the pixels are redrawn upright at that image's own logical size and scale before anyone
66+
/// encodes or compares them (#2728).
67+
///
68+
/// Never activates `app`: an observation must not change which app is foregrounded just to get a
69+
/// screen, so the caller passes the app context it already intends to observe.
70+
func captureResolvedAppScreen(
71+
app: XCUIApplication
72+
) -> Result<CapturedAppScreen, RunnerAppScreenCaptureFailure> {
73+
var outcome: Result<CapturedAppScreen, RunnerAppScreenCaptureFailure> = .failure(
74+
.unresolvedScreen
75+
)
76+
let capture = {
77+
outcome = Self.resolveCapturedAppScreen(app: app)
78+
}
79+
if Thread.isMainThread {
80+
capture()
81+
} else {
82+
DispatchQueue.main.sync(execute: capture)
83+
}
84+
return outcome
85+
}
86+
87+
/// Captures the display a caller should observe, asking `app` first and the system surface second.
88+
///
89+
/// A capture with no session app, or with one that is not running, still has a window on screen:
90+
/// the home screen is SpringBoard's window, and its panel is the lit one. Asking the system surface
91+
/// keeps that capture honest — it names a display a window actually occupies instead of reaching
92+
/// for `XCUIScreen.main`, which on a foldable is one fixed panel whether or not it is dark (#2728).
93+
///
94+
/// Only an unresolved *window* moves on to the system surface. A window that resolved and then
95+
/// refused to name its display is the failure the host has to see, and no second question about a
96+
/// different process should bury it.
97+
func captureObservedScreen(app: XCUIApplication) -> Result<
98+
CapturedAppScreen, RunnerAppScreenCaptureFailure
99+
> {
100+
selectObservedScreenCapture(
101+
resolving: { captureResolvedAppScreen(app: app) },
102+
fallingBack: { captureResolvedAppScreen(app: springboard) }
103+
)
104+
}
105+
106+
/// The target rule, kept apart from the two queries so the rule itself is testable: an unresolved
107+
/// window asks the system surface, and nothing else does.
108+
func selectObservedScreenCapture(
109+
resolving: () -> Result<CapturedAppScreen, RunnerAppScreenCaptureFailure>,
110+
fallingBack: () -> Result<CapturedAppScreen, RunnerAppScreenCaptureFailure>
111+
) -> Result<CapturedAppScreen, RunnerAppScreenCaptureFailure> {
112+
let outcome = resolving()
113+
if case .failure(.unresolvedWindow) = outcome {
114+
return fallingBack()
115+
}
116+
return outcome
117+
}
118+
119+
private static func resolveCapturedAppScreen(
120+
app: XCUIApplication
121+
) -> Result<CapturedAppScreen, RunnerAppScreenCaptureFailure> {
122+
var screen: AnyObject?
123+
var displayID: UInt = 0
124+
var resolution: RunnerApplicationScreenFailure = .unresolvedScreen
125+
guard RunnerResolveApplicationScreen(app, &screen, &displayID, &resolution),
126+
let captureScreen = screen as? XCUIScreen
127+
else {
128+
return .failure(RunnerAppScreenCaptureFailure(resolution))
129+
}
130+
let image = captureScreen.screenshot().image
131+
guard let upright = runnerUprightCaptureImage(image) else {
132+
return .failure(.unrenderableImage)
133+
}
134+
// A scale that is not a finite positive number cannot be reported to a host that divides by it,
135+
// and cannot be drawn at, so the capture is refused rather than encoded with a value JSON cannot
136+
// carry (#2728).
137+
guard upright.scale.isFinite, upright.scale > 0 else {
138+
return .failure(.unrenderableImage)
139+
}
140+
guard let cgImage = runnerCGImage(from: upright) else {
141+
return .failure(.unrenderableImage)
142+
}
143+
return .success(
144+
CapturedAppScreen(
145+
image: upright,
146+
displayID: displayID,
147+
pixelWidth: cgImage.width,
148+
pixelHeight: cgImage.height,
149+
pixelsPerPoint: Double(upright.scale)
150+
)
151+
)
152+
}
153+
154+
/// Redraws a turned capture so its encoded rows run the way the panel is presented, and leaves an
155+
/// already-upright capture untouched so a single-panel capture keeps the exact bytes it produced
156+
/// before panels existed. The draw box is the image's own oriented logical size and the renderer's
157+
/// scale is the image's own scale: neither comes from a nominal panel size, and the one-pixel
158+
/// difference between what XCTest hands back and what CoreDevice calls the panel is preserved
159+
/// rather than stretched away (#2728).
160+
private static func runnerUprightCaptureImage(_ image: UIImage) -> UIImage? {
161+
if image.imageOrientation == .up {
162+
return image
163+
}
164+
guard image.size.width > 0, image.size.height > 0, image.scale.isFinite, image.scale > 0 else {
165+
return nil
166+
}
167+
let format = UIGraphicsImageRendererFormat()
168+
format.scale = image.scale
169+
let renderer = UIGraphicsImageRenderer(size: image.size, format: format)
170+
return renderer.image { _ in
171+
image.draw(in: CGRect(origin: .zero, size: image.size))
172+
}
173+
}
174+
}
175+
#endif

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift‎

Lines changed: 44 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -1466,7 +1466,7 @@ extension RunnerTests {
14661466
fps: command.fps.map { Int32($0) }
14671467
)
14681468
try recorder.start { [weak self] in
1469-
return self?.captureRunnerFrame()
1469+
return self?.captureRunnerFrame(app: activeApp)
14701470
}
14711471
activeRecording = recorder
14721472
return Response(ok: true, data: DataPayload(message: "recording started"))
@@ -2055,7 +2055,6 @@ extension RunnerTests {
20552055
case .snapshot:
20562056
return try executeSnapshotPrepared(command: command, activeApp: activeApp)
20572057
case .screenshot:
2058-
let screenshot: XCUIScreenshot
20592058
#if os(macOS)
20602059
// macOS keeps the app-targeted capture behavior for window-level screenshots.
20612060
if let bundleId = command.appBundleId, !bundleId.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty {
@@ -2065,37 +2064,60 @@ extension RunnerTests {
20652064
// Brief wait for the app transition animation to complete
20662065
sleepFor(0.5)
20672066
}
2067+
let screenshot: XCUIScreenshot
20682068
if command.fullscreen == true {
20692069
screenshot = XCUIScreen.main.screenshot()
20702070
} else if let bundleId = command.appBundleId, !bundleId.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty {
20712071
screenshot = screenshotRoot(app: activeApp).screenshot()
20722072
} else {
20732073
screenshot = XCUIScreen.main.screenshot()
20742074
}
2075-
#else
2076-
screenshot = XCUIScreen.main.screenshot()
2077-
#endif
2078-
guard let pngData = runnerPngData(for: screenshot.image) else {
2079-
return Response(ok: false, error: ErrorPayload(message: "Failed to encode screenshot as PNG"))
2080-
}
2081-
if command.inlineScreenshot == true {
2075+
return screenshotResponse(
2076+
image: screenshot.image,
2077+
inlineScreenshot: command.inlineScreenshot == true
2078+
)
2079+
#elseif os(iOS)
2080+
// A foldable lights one panel at a time while `XCUIScreen.main` names one fixed panel, so an
2081+
// app on the other panel is captured as a valid PNG of nothing. The capture comes from the
2082+
// display owning a window, and a display that no window can name fails this required capture
2083+
// instead of answering with the main screen (#2728).
2084+
//
2085+
// `.screenshot` is a runner-lifecycle command, which skips the preflight that resolves the
2086+
// session app, so on a fresh runner process `activeApp` is still the runner host app. The
2087+
// requested bundle id is resolved here and never activated: an observation must not change
2088+
// which app is foregrounded in order to see which display it is on (#2728).
2089+
switch captureObservedScreen(app: resolveAppWithoutActivation(command: command)) {
2090+
case .failure(let failure):
20822091
return Response(
2083-
ok: true,
2084-
data: DataPayload(imageBase64: pngData.base64EncodedString())
2092+
ok: false,
2093+
error: ErrorPayload(code: failure.rawValue, message: failure.message, hint: failure.hint)
2094+
)
2095+
case .success(let captured):
2096+
// The facts travel to the host in the response and to the operator in runner.log, because a
2097+
// capture whose density is in question is argued from what the capture itself measured.
2098+
NSLog(
2099+
"AGENT_DEVICE_RUNNER_SCREEN_CAPTURE display=%lu pixels=%ldx%ld pixelsPerPoint=%g",
2100+
captured.displayID,
2101+
captured.pixelWidth,
2102+
captured.pixelHeight,
2103+
captured.pixelsPerPoint
2104+
)
2105+
return screenshotResponse(
2106+
image: captured.image,
2107+
inlineScreenshot: command.inlineScreenshot == true,
2108+
metadata: ScreenshotMetadataPayload(
2109+
displayID: captured.displayID,
2110+
pixelWidth: captured.pixelWidth,
2111+
pixelHeight: captured.pixelHeight,
2112+
pixelsPerPoint: captured.pixelsPerPoint
2113+
)
20852114
)
20862115
}
2087-
let fileName = "screenshot-\(Int(Date().timeIntervalSince1970 * 1000)).png"
2088-
let filePath = (NSTemporaryDirectory() as NSString).appendingPathComponent(fileName)
2089-
do {
2090-
try pngData.write(to: URL(fileURLWithPath: filePath))
2091-
} catch {
2092-
return Response(ok: false, error: ErrorPayload(message: "Failed to write screenshot: \(error.localizedDescription)"))
2093-
}
2094-
#if os(macOS)
2095-
return Response(ok: true, data: DataPayload(message: filePath))
20962116
#else
2097-
// Return path relative to app container root (tmp/ maps to NSTemporaryDirectory)
2098-
return Response(ok: true, data: DataPayload(message: "tmp/\(fileName)"))
2117+
return screenshotResponse(
2118+
image: XCUIScreen.main.screenshot().image,
2119+
inlineScreenshot: command.inlineScreenshot == true
2120+
)
20992121
#endif
21002122
case .back, .backInApp:
21012123
if tapInAppBackControl(app: activeApp) {

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Keyboard.swift‎

Lines changed: 20 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -212,6 +212,7 @@ extension RunnerTests {
212212
if tapKeyboardDismissControl(app: app) {
213213
_ = keyboard.waitForNonExistence(timeout: KeyboardDismissObservationTiming.timeout)
214214
waitForScreenshotStability(
215+
app: app,
215216
timeout: KeyboardDismissObservationTiming.settleTimeout,
216217
sampleInterval: KeyboardDismissObservationTiming.settleSampleInterval,
217218
requiredConsecutiveMatches: KeyboardDismissObservationTiming.settleRequiredConsecutiveMatches
@@ -240,23 +241,38 @@ extension RunnerTests {
240241
// so far, so it is unit-testable without a real screenshot pipeline; this loop
241242
// is the thin, untestable I/O shell around it.
242243
private func waitForScreenshotStability(
244+
app: XCUIApplication,
243245
timeout: TimeInterval,
244246
sampleInterval: TimeInterval,
245247
requiredConsecutiveMatches: Int
246248
) {
247249
let deadline = Date().addingTimeInterval(timeout)
248-
var samples: [Data?] = [screenshotFingerprintForStabilityCheck()]
250+
var samples: [Data?] = [screenshotFingerprintForStabilityCheck(app: app)]
249251
while Date() < deadline {
250252
sleepFor(sampleInterval)
251-
samples.append(screenshotFingerprintForStabilityCheck())
253+
samples.append(screenshotFingerprintForStabilityCheck(app: app))
252254
if runnerScreenshotStabilitySettled(samples, requiredConsecutiveMatches: requiredConsecutiveMatches) {
255+
logScreenshotStability(samples: samples, settled: true)
253256
return
254257
}
255258
}
259+
logScreenshotStability(samples: samples, settled: false)
260+
}
261+
262+
/// Says what the settle actually looked at: samples that are all `nil` means the capture refused to
263+
/// produce a frame, which is a different fact from a screen that never held still, and the two read
264+
/// identically from the outside (#2728).
265+
private func logScreenshotStability(samples: [Data?], settled: Bool) {
266+
NSLog(
267+
"AGENT_DEVICE_RUNNER_KEYBOARD_STABILITY samples=%ld captured=%ld settled=%@",
268+
samples.count,
269+
samples.compactMap { $0 }.count,
270+
settled ? "yes" : "no"
271+
)
256272
}
257273

258-
private func screenshotFingerprintForStabilityCheck() -> Data? {
259-
guard let image = captureRunnerFrame() else { return nil }
274+
private func screenshotFingerprintForStabilityCheck(app: XCUIApplication) -> Data? {
275+
guard let image = captureRunnerFrame(app: app) else { return nil }
260276
return runnerPngData(for: image)
261277
}
262278

0 commit comments

Comments
 (0)