Skip to content

Commit 80079f7

Browse files
fix(ios): capture the display that owns a window, not the main screen
A foldable lights one panel at a time while XCUIScreen.main names one fixed panel, so an app on the other panel was captured as a valid PNG of black. The runner now asks the target app's window first and the system surface's window second, reads the display off that same window instance, and encodes the capture upright, reporting the display ID, the pixel size it encoded, and the pixels-per-point of that image so the host normalizes density from the capture instead of a panel it guessed. The home screen is SpringBoard's window, so a capture with no session app still has a display that owns a window; a capture fails closed with a typed reason only when neither resolves one. An app with no window is refused on the query's own answer rather than on geometry alone. The screenshot command skips the app-activation preflight, so it resolves the requested bundle id for observation without ever foregrounding an app to see which display it is on. Co-authored-by: Apex <[email protected]>
1 parent 073b568 commit 80079f7

17 files changed

Lines changed: 1125 additions & 114 deletions
Lines changed: 175 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,175 @@
1+
import XCTest
2+
#if canImport(UIKit)
3+
import UIKit
4+
#endif
5+
6+
/// One capture of the display that actually hosts the target app, with the facts needed to read its
7+
/// pixels without guessing a panel: which display was captured, how many pixels the encoded image
8+
/// holds, and how many of them represent one point.
9+
struct CapturedAppScreen {
10+
let image: RunnerImage
11+
let displayID: UInt
12+
let pixelWidth: Int
13+
let pixelHeight: Int
14+
let pixelsPerPoint: Double
15+
}
16+
17+
/// Why the runner could not name the display hosting the target app. Carried as a reason code
18+
/// rather than as message text so a required capture can fail closed with a code the host keys on,
19+
/// and an optional observation can report "unknown" without parsing a sentence (#2728).
20+
enum RunnerAppScreenCaptureFailure: String, Error {
21+
case unresolvedWindow = "APP_SCREEN_WINDOW_UNRESOLVED"
22+
case unresolvedScreen = "APP_SCREEN_UNRESOLVED"
23+
case unrenderableImage = "APP_SCREEN_CAPTURE_UNRENDERABLE"
24+
25+
init(_ failure: RunnerApplicationScreenFailure) {
26+
switch failure {
27+
case .unresolvedWindow:
28+
self = .unresolvedWindow
29+
default:
30+
self = .unresolvedScreen
31+
}
32+
}
33+
34+
var message: String {
35+
switch self {
36+
case .unresolvedWindow:
37+
return "Neither the target app nor the system surface resolved a window, so no display could be named."
38+
case .unresolvedScreen:
39+
return "The resolved app window did not report the display it is shown on."
40+
case .unrenderableImage:
41+
return "The resolved display handed back an image that could not be drawn upright."
42+
}
43+
}
44+
45+
var hint: String {
46+
switch self {
47+
case .unresolvedWindow:
48+
return "The capture follows the display that owns a window, so this is a runtime that is answering no window for any process; runner.log names the window frame and the query answer it refused."
49+
case .unresolvedScreen:
50+
return "Retry the capture. If it persists, the XCTest runtime is not reporting the window's display; runner.log names the window frame it resolved."
51+
case .unrenderableImage:
52+
return "Retry the capture. The display was resolved, so this is the capture itself failing, not a panel question."
53+
}
54+
}
55+
}
56+
57+
#if canImport(UIKit) && os(iOS)
58+
extension RunnerTests {
59+
/// Captures the display hosting `app` instead of `XCUIScreen.main`.
60+
///
61+
/// On a foldable, `XCUIScreen.main` names one fixed panel while the app can be on the other, and
62+
/// the panel that is dark still answers with a valid PNG of black — a capture that looks like
63+
/// evidence and proves nothing. The window's own screen is the only source that follows the app,
64+
/// and on a panel the system presents turned it hands back an image whose buffer is sideways, so
65+
/// the pixels are redrawn upright at that image's own logical size and scale before anyone
66+
/// encodes or compares them (#2728).
67+
///
68+
/// Never activates `app`: an observation must not change which app is foregrounded just to get a
69+
/// screen, so the caller passes the app context it already intends to observe.
70+
func captureResolvedAppScreen(
71+
app: XCUIApplication
72+
) -> Result<CapturedAppScreen, RunnerAppScreenCaptureFailure> {
73+
var outcome: Result<CapturedAppScreen, RunnerAppScreenCaptureFailure> = .failure(
74+
.unresolvedScreen
75+
)
76+
let capture = {
77+
outcome = Self.resolveCapturedAppScreen(app: app)
78+
}
79+
if Thread.isMainThread {
80+
capture()
81+
} else {
82+
DispatchQueue.main.sync(execute: capture)
83+
}
84+
return outcome
85+
}
86+
87+
/// Captures the display a caller should observe, asking `app` first and the system surface second.
88+
///
89+
/// A capture with no session app, or with one that is not running, still has a window on screen:
90+
/// the home screen is SpringBoard's window, and its panel is the lit one. Asking the system surface
91+
/// keeps that capture honest — it names a display a window actually occupies instead of reaching
92+
/// for `XCUIScreen.main`, which on a foldable is one fixed panel whether or not it is dark (#2728).
93+
///
94+
/// Only an unresolved *window* moves on to the system surface. A window that resolved and then
95+
/// refused to name its display is the failure the host has to see, and no second question about a
96+
/// different process should bury it.
97+
func captureObservedScreen(app: XCUIApplication) -> Result<
98+
CapturedAppScreen, RunnerAppScreenCaptureFailure
99+
> {
100+
selectObservedScreenCapture(
101+
resolving: { captureResolvedAppScreen(app: app) },
102+
fallingBack: { captureResolvedAppScreen(app: springboard) }
103+
)
104+
}
105+
106+
/// The target rule, kept apart from the two queries so the rule itself is testable: an unresolved
107+
/// window asks the system surface, and nothing else does.
108+
func selectObservedScreenCapture(
109+
resolving: () -> Result<CapturedAppScreen, RunnerAppScreenCaptureFailure>,
110+
fallingBack: () -> Result<CapturedAppScreen, RunnerAppScreenCaptureFailure>
111+
) -> Result<CapturedAppScreen, RunnerAppScreenCaptureFailure> {
112+
let outcome = resolving()
113+
if case .failure(.unresolvedWindow) = outcome {
114+
return fallingBack()
115+
}
116+
return outcome
117+
}
118+
119+
private static func resolveCapturedAppScreen(
120+
app: XCUIApplication
121+
) -> Result<CapturedAppScreen, RunnerAppScreenCaptureFailure> {
122+
var screen: AnyObject?
123+
var displayID: UInt = 0
124+
var resolution: RunnerApplicationScreenFailure = .unresolvedScreen
125+
guard RunnerResolveApplicationScreen(app, &screen, &displayID, &resolution),
126+
let captureScreen = screen as? XCUIScreen
127+
else {
128+
return .failure(RunnerAppScreenCaptureFailure(resolution))
129+
}
130+
let image = captureScreen.screenshot().image
131+
guard let upright = runnerUprightCaptureImage(image) else {
132+
return .failure(.unrenderableImage)
133+
}
134+
// A scale that is not a finite positive number cannot be reported to a host that divides by it,
135+
// and cannot be drawn at, so the capture is refused rather than encoded with a value JSON cannot
136+
// carry (#2728).
137+
guard upright.scale.isFinite, upright.scale > 0 else {
138+
return .failure(.unrenderableImage)
139+
}
140+
guard let cgImage = runnerCGImage(from: upright) else {
141+
return .failure(.unrenderableImage)
142+
}
143+
return .success(
144+
CapturedAppScreen(
145+
image: upright,
146+
displayID: displayID,
147+
pixelWidth: cgImage.width,
148+
pixelHeight: cgImage.height,
149+
pixelsPerPoint: Double(upright.scale)
150+
)
151+
)
152+
}
153+
154+
/// Redraws a turned capture so its encoded rows run the way the panel is presented, and leaves an
155+
/// already-upright capture untouched so a single-panel capture keeps the exact bytes it produced
156+
/// before panels existed. The draw box is the image's own oriented logical size and the renderer's
157+
/// scale is the image's own scale: neither comes from a nominal panel size, and the one-pixel
158+
/// difference between what XCTest hands back and what CoreDevice calls the panel is preserved
159+
/// rather than stretched away (#2728).
160+
private static func runnerUprightCaptureImage(_ image: UIImage) -> UIImage? {
161+
if image.imageOrientation == .up {
162+
return image
163+
}
164+
guard image.size.width > 0, image.size.height > 0, image.scale.isFinite, image.scale > 0 else {
165+
return nil
166+
}
167+
let format = UIGraphicsImageRendererFormat()
168+
format.scale = image.scale
169+
let renderer = UIGraphicsImageRenderer(size: image.size, format: format)
170+
return renderer.image { _ in
171+
image.draw(in: CGRect(origin: .zero, size: image.size))
172+
}
173+
}
174+
}
175+
#endif

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift‎

Lines changed: 44 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -1465,7 +1465,7 @@ extension RunnerTests {
14651465
fps: command.fps.map { Int32($0) }
14661466
)
14671467
try recorder.start { [weak self] in
1468-
return self?.captureRunnerFrame()
1468+
return self?.captureRunnerFrame(app: activeApp)
14691469
}
14701470
activeRecording = recorder
14711471
return Response(ok: true, data: DataPayload(message: "recording started"))
@@ -2054,7 +2054,6 @@ extension RunnerTests {
20542054
case .snapshot:
20552055
return try executeSnapshotPrepared(command: command, activeApp: activeApp)
20562056
case .screenshot:
2057-
let screenshot: XCUIScreenshot
20582057
#if os(macOS)
20592058
// macOS keeps the app-targeted capture behavior for window-level screenshots.
20602059
if let bundleId = command.appBundleId, !bundleId.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty {
@@ -2064,37 +2063,60 @@ extension RunnerTests {
20642063
// Brief wait for the app transition animation to complete
20652064
sleepFor(0.5)
20662065
}
2066+
let screenshot: XCUIScreenshot
20672067
if command.fullscreen == true {
20682068
screenshot = XCUIScreen.main.screenshot()
20692069
} else if let bundleId = command.appBundleId, !bundleId.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty {
20702070
screenshot = screenshotRoot(app: activeApp).screenshot()
20712071
} else {
20722072
screenshot = XCUIScreen.main.screenshot()
20732073
}
2074-
#else
2075-
screenshot = XCUIScreen.main.screenshot()
2076-
#endif
2077-
guard let pngData = runnerPngData(for: screenshot.image) else {
2078-
return Response(ok: false, error: ErrorPayload(message: "Failed to encode screenshot as PNG"))
2079-
}
2080-
if command.inlineScreenshot == true {
2074+
return screenshotResponse(
2075+
image: screenshot.image,
2076+
inlineScreenshot: command.inlineScreenshot == true
2077+
)
2078+
#elseif os(iOS)
2079+
// A foldable lights one panel at a time while `XCUIScreen.main` names one fixed panel, so an
2080+
// app on the other panel is captured as a valid PNG of nothing. The capture comes from the
2081+
// display owning a window, and a display that no window can name fails this required capture
2082+
// instead of answering with the main screen (#2728).
2083+
//
2084+
// `.screenshot` is a runner-lifecycle command, which skips the preflight that resolves the
2085+
// session app, so on a fresh runner process `activeApp` is still the runner host app. The
2086+
// requested bundle id is resolved here and never activated: an observation must not change
2087+
// which app is foregrounded in order to see which display it is on (#2728).
2088+
switch captureObservedScreen(app: resolveAppWithoutActivation(command: command)) {
2089+
case .failure(let failure):
20812090
return Response(
2082-
ok: true,
2083-
data: DataPayload(imageBase64: pngData.base64EncodedString())
2091+
ok: false,
2092+
error: ErrorPayload(code: failure.rawValue, message: failure.message, hint: failure.hint)
2093+
)
2094+
case .success(let captured):
2095+
// The facts travel to the host in the response and to the operator in runner.log, because a
2096+
// capture whose density is in question is argued from what the capture itself measured.
2097+
NSLog(
2098+
"AGENT_DEVICE_RUNNER_SCREEN_CAPTURE display=%lu pixels=%ldx%ld pixelsPerPoint=%g",
2099+
captured.displayID,
2100+
captured.pixelWidth,
2101+
captured.pixelHeight,
2102+
captured.pixelsPerPoint
2103+
)
2104+
return screenshotResponse(
2105+
image: captured.image,
2106+
inlineScreenshot: command.inlineScreenshot == true,
2107+
metadata: ScreenshotMetadataPayload(
2108+
displayID: captured.displayID,
2109+
pixelWidth: captured.pixelWidth,
2110+
pixelHeight: captured.pixelHeight,
2111+
pixelsPerPoint: captured.pixelsPerPoint
2112+
)
20842113
)
20852114
}
2086-
let fileName = "screenshot-\(Int(Date().timeIntervalSince1970 * 1000)).png"
2087-
let filePath = (NSTemporaryDirectory() as NSString).appendingPathComponent(fileName)
2088-
do {
2089-
try pngData.write(to: URL(fileURLWithPath: filePath))
2090-
} catch {
2091-
return Response(ok: false, error: ErrorPayload(message: "Failed to write screenshot: \(error.localizedDescription)"))
2092-
}
2093-
#if os(macOS)
2094-
return Response(ok: true, data: DataPayload(message: filePath))
20952115
#else
2096-
// Return path relative to app container root (tmp/ maps to NSTemporaryDirectory)
2097-
return Response(ok: true, data: DataPayload(message: "tmp/\(fileName)"))
2116+
return screenshotResponse(
2117+
image: XCUIScreen.main.screenshot().image,
2118+
inlineScreenshot: command.inlineScreenshot == true
2119+
)
20982120
#endif
20992121
case .back, .backInApp:
21002122
if tapInAppBackControl(app: activeApp) {

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Keyboard.swift‎

Lines changed: 20 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -212,6 +212,7 @@ extension RunnerTests {
212212
if tapKeyboardDismissControl(app: app) {
213213
_ = keyboard.waitForNonExistence(timeout: KeyboardDismissObservationTiming.timeout)
214214
waitForScreenshotStability(
215+
app: app,
215216
timeout: KeyboardDismissObservationTiming.settleTimeout,
216217
sampleInterval: KeyboardDismissObservationTiming.settleSampleInterval,
217218
requiredConsecutiveMatches: KeyboardDismissObservationTiming.settleRequiredConsecutiveMatches
@@ -240,23 +241,38 @@ extension RunnerTests {
240241
// so far, so it is unit-testable without a real screenshot pipeline; this loop
241242
// is the thin, untestable I/O shell around it.
242243
private func waitForScreenshotStability(
244+
app: XCUIApplication,
243245
timeout: TimeInterval,
244246
sampleInterval: TimeInterval,
245247
requiredConsecutiveMatches: Int
246248
) {
247249
let deadline = Date().addingTimeInterval(timeout)
248-
var samples: [Data?] = [screenshotFingerprintForStabilityCheck()]
250+
var samples: [Data?] = [screenshotFingerprintForStabilityCheck(app: app)]
249251
while Date() < deadline {
250252
sleepFor(sampleInterval)
251-
samples.append(screenshotFingerprintForStabilityCheck())
253+
samples.append(screenshotFingerprintForStabilityCheck(app: app))
252254
if runnerScreenshotStabilitySettled(samples, requiredConsecutiveMatches: requiredConsecutiveMatches) {
255+
logScreenshotStability(samples: samples, settled: true)
253256
return
254257
}
255258
}
259+
logScreenshotStability(samples: samples, settled: false)
260+
}
261+
262+
/// Says what the settle actually looked at: samples that are all `nil` means the capture refused to
263+
/// produce a frame, which is a different fact from a screen that never held still, and the two read
264+
/// identically from the outside (#2728).
265+
private func logScreenshotStability(samples: [Data?], settled: Bool) {
266+
NSLog(
267+
"AGENT_DEVICE_RUNNER_KEYBOARD_STABILITY samples=%ld captured=%ld settled=%@",
268+
samples.count,
269+
samples.compactMap { $0 }.count,
270+
settled ? "yes" : "no"
271+
)
256272
}
257273

258-
private func screenshotFingerprintForStabilityCheck() -> Data? {
259-
guard let image = captureRunnerFrame() else { return nil }
274+
private func screenshotFingerprintForStabilityCheck(app: XCUIApplication) -> Data? {
275+
guard let image = captureRunnerFrame(app: app) else { return nil }
260276
return runnerPngData(for: image)
261277
}
262278

0 commit comments

Comments
 (0)