Skip to content

Commit 6ac9e42

Browse files
thymikeeclaude
andcommitted
feat(macos-helper): drive app sessions through accessibility actions
Add an `app` surface to the macOS helper so an app session can be served without XCTest Automation Mode while the app stays behind other windows: - snapshot targets the session app by bundle id and turns on the Chromium/Electron accessibility tree before traversal - press, fill, type, and scroll act through AXPress, focus, value, selected text, and scroll bar values; pointer actions with no accessibility equivalent are refused with a typed reason instead of posting process events an inactive app drops - screenshot captures the app's front window alone via ScreenCaptureKit Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
1 parent d396f3b commit 6ac9e42

6 files changed

Lines changed: 843 additions & 24 deletions

File tree

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
import AppKit
2+
import CoreGraphics
3+
import Foundation
4+
import ScreenCaptureKit
5+
6+
/// Captures the session app's front window by itself: windows above it, the ghost cursor, and
7+
/// the rest of the desktop are not in the image, so the app may sit behind the user's work.
8+
func captureAppWindowScreenshot(app: NSRunningApplication, outPath: String) throws {
9+
guard #available(macOS 14.0, *) else {
10+
throw HelperError.commandFailed("app window screenshots require macOS 14 or newer")
11+
}
12+
let pid = app.processIdentifier
13+
let frontWindowID = frontWindowNumber(pid: pid)
14+
let content = try awaitShareableContent()
15+
let candidates = content.windows.filter {
16+
$0.owningApplication?.processID == pid && $0.windowLayer == 0 && $0.frame.width > 0
17+
&& $0.frame.height > 0
18+
}
19+
guard
20+
let window = candidates.first(where: { Int($0.windowID) == frontWindowID })
21+
?? candidates.max(by: { $0.frame.width * $0.frame.height < $1.frame.width * $1.frame.height })
22+
else {
23+
throw HelperError.commandFailed(
24+
"screenshot could not find a window for the app",
25+
details: ["reason": "window-not-found", "bundleId": app.bundleIdentifier ?? ""]
26+
)
27+
}
28+
29+
let filter = SCContentFilter(desktopIndependentWindow: window)
30+
let configuration = SCStreamConfiguration()
31+
let scale = CGFloat(filter.pointPixelScale)
32+
configuration.width = Int(filter.contentRect.width * scale)
33+
configuration.height = Int(filter.contentRect.height * scale)
34+
configuration.showsCursor = false
35+
configuration.ignoreShadowsSingleWindow = true
36+
37+
let semaphore = DispatchSemaphore(value: 0)
38+
var capturedImage: CGImage?
39+
var capturedError: Error?
40+
SCScreenshotManager.captureImage(contentFilter: filter, configuration: configuration) { image, error in
41+
capturedImage = image
42+
capturedError = error
43+
semaphore.signal()
44+
}
45+
semaphore.wait()
46+
if let error = capturedError as NSError? {
47+
throw screenshotFailure(error, surface: "app")
48+
}
49+
guard let capturedImage else {
50+
throw HelperError.commandFailed("screenshot failed")
51+
}
52+
try writePNG(capturedImage, to: outPath)
53+
}
54+
55+
@available(macOS 14.0, *)
56+
private func awaitShareableContent() throws -> SCShareableContent {
57+
let semaphore = DispatchSemaphore(value: 0)
58+
var shareable: SCShareableContent?
59+
var failure: Error?
60+
SCShareableContent.getExcludingDesktopWindows(true, onScreenWindowsOnly: true) { content, error in
61+
shareable = content
62+
failure = error
63+
semaphore.signal()
64+
}
65+
semaphore.wait()
66+
if let error = failure as NSError? {
67+
throw screenshotFailure(error, surface: "app")
68+
}
69+
guard let shareable else {
70+
throw HelperError.commandFailed("screenshot could not list shareable windows")
71+
}
72+
return shareable
73+
}

0 commit comments

Comments
 (0)