Skip to content

Commit 307821e

Browse files
committed
fix: dismiss inaccessible macos privacy sheet visually
1 parent 2f1ff03 commit 307821e

3 files changed

Lines changed: 139 additions & 58 deletions

File tree

‎.github/workflows/macos.yml‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -161,7 +161,10 @@ jobs:
161161
162162
attempts_after_capture=5
163163
for attempt in {1..60}; do
164-
if "$helper" alert dismiss --surface frontmost-app; then
164+
if "$helper" alert dismiss \
165+
--surface frontmost-app \
166+
--screenshot "$screenshot_path"; then
167+
sleep 1
165168
break
166169
fi
167170
if [ -f "$screenshot_path" ]; then
Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,116 @@
1+
import AppKit
2+
import ApplicationServices
3+
import CoreGraphics
4+
import Foundation
5+
import ImageIO
6+
import Vision
7+
8+
struct VisualAlertMatch {
9+
let buttonLabel: String
10+
}
11+
12+
func dismissLocalNetworkPermissionAlertVisually(
13+
screenshotPath: String,
14+
app: NSRunningApplication
15+
) throws -> VisualAlertMatch? {
16+
guard FileManager.default.fileExists(atPath: screenshotPath) else {
17+
return nil
18+
}
19+
let screenshotUrl = URL(fileURLWithPath: screenshotPath)
20+
guard let source = CGImageSourceCreateWithURL(screenshotUrl as CFURL, nil),
21+
let image = CGImageSourceCreateImageAtIndex(source, 0, nil)
22+
else {
23+
return nil
24+
}
25+
26+
let request = VNRecognizeTextRequest()
27+
request.recognitionLevel = .accurate
28+
request.recognitionLanguages = ["en-US"]
29+
request.usesLanguageCorrection = false
30+
do {
31+
try VNImageRequestHandler(cgImage: image).perform([request])
32+
} catch {
33+
throw HelperError.commandFailed(
34+
"alert screenshot recognition failed",
35+
details: ["error": error.localizedDescription]
36+
)
37+
}
38+
39+
let recognized = (request.results ?? []).compactMap { observation -> (String, CGRect)? in
40+
guard let candidate = observation.topCandidates(1).first else {
41+
return nil
42+
}
43+
return (candidate.string, observation.boundingBox)
44+
}
45+
let recognizedText = recognized.map(\.0).joined(separator: " ")
46+
guard recognizedText.localizedCaseInsensitiveContains("local network"),
47+
let denial = recognized.first(where: { text, _ in
48+
normalizeVisualAlertText(text).contains("don't allow")
49+
}),
50+
let windowRect = bestMatchingWindowRect(
51+
app: app,
52+
screenshotWidth: image.width,
53+
screenshotHeight: image.height
54+
)
55+
else {
56+
return nil
57+
}
58+
59+
let point = CGPoint(
60+
x: windowRect.x + Double(denial.1.midX) * windowRect.width,
61+
y: windowRect.y + (1 - Double(denial.1.midY)) * windowRect.height
62+
)
63+
guard let move = CGEvent(
64+
mouseEventSource: nil,
65+
mouseType: .mouseMoved,
66+
mouseCursorPosition: point,
67+
mouseButton: .left
68+
),
69+
let down = CGEvent(
70+
mouseEventSource: nil,
71+
mouseType: .leftMouseDown,
72+
mouseCursorPosition: point,
73+
mouseButton: .left
74+
),
75+
let up = CGEvent(
76+
mouseEventSource: nil,
77+
mouseType: .leftMouseUp,
78+
mouseCursorPosition: point,
79+
mouseButton: .left
80+
)
81+
else {
82+
throw HelperError.commandFailed(
83+
"alert action failed",
84+
details: ["reason": "event_creation_failed"]
85+
)
86+
}
87+
move.post(tap: .cghidEventTap)
88+
down.post(tap: .cghidEventTap)
89+
up.post(tap: .cghidEventTap)
90+
return VisualAlertMatch(buttonLabel: denial.0)
91+
}
92+
93+
private func normalizeVisualAlertText(_ value: String) -> String {
94+
value
95+
.replacingOccurrences(of: "’", with: "'")
96+
.trimmingCharacters(in: .whitespacesAndNewlines)
97+
.lowercased()
98+
}
99+
100+
private func bestMatchingWindowRect(
101+
app: NSRunningApplication,
102+
screenshotWidth: Int,
103+
screenshotHeight: Int
104+
) -> RectResponse? {
105+
guard screenshotWidth > 0, screenshotHeight > 0 else {
106+
return nil
107+
}
108+
let screenshotAspectRatio = Double(screenshotWidth) / Double(screenshotHeight)
109+
return windows(of: AXUIElementCreateApplication(app.processIdentifier))
110+
.compactMap(rectAttribute)
111+
.filter { $0.width > 0 && $0.height > 0 }
112+
.min { left, right in
113+
abs(left.width / left.height - screenshotAspectRatio)
114+
< abs(right.width / right.height - screenshotAspectRatio)
115+
}
116+
}

‎apple/macos-helper/Sources/AgentDeviceMacOSHelper/main.swift‎

Lines changed: 19 additions & 57 deletions
Original file line numberDiff line numberDiff line change
@@ -291,8 +291,26 @@ struct AgentDeviceMacOSHelper {
291291
normalizedSurface == "frontmost-app"
292292
? findFocusedAlertElement()
293293
?? findAlertElement(appElement: AXUIElementCreateApplication(app.processIdentifier))
294-
?? findBlockingLocalNetworkPermissionAlert()
295294
: findAlertElement(appElement: AXUIElementCreateApplication(app.processIdentifier))
295+
if alertElement == nil,
296+
action == "dismiss",
297+
normalizedSurface == "frontmost-app",
298+
let screenshotPath = optionValue(arguments: Array(arguments.dropFirst()), name: "--screenshot"),
299+
let visualMatch = try dismissLocalNetworkPermissionAlertVisually(
300+
screenshotPath: screenshotPath,
301+
app: app
302+
)
303+
{
304+
return SuccessEnvelope(
305+
data: AlertResponse(
306+
title: "Local Network",
307+
role: "visual",
308+
buttons: [visualMatch.buttonLabel],
309+
action: action,
310+
bundleId: app.bundleIdentifier
311+
)
312+
)
313+
}
296314
guard let alertElement else {
297315
// `reason` is the typed channel the host retries on; the message is for humans only.
298316
throw HelperError.commandFailed(
@@ -671,62 +689,6 @@ private func findFocusedAlertElement() -> AXUIElement? {
671689
return nil
672690
}
673691

674-
private func findBlockingLocalNetworkPermissionAlert() -> AXUIElement? {
675-
guard let windowInfoList = CGWindowListCopyWindowInfo(
676-
[.optionOnScreenOnly, .excludeDesktopElements],
677-
kCGNullWindowID
678-
) as? [[String: Any]] else {
679-
return nil
680-
}
681-
682-
var inspectedProcessIdentifiers: Set<pid_t> = []
683-
for windowInfo in windowInfoList {
684-
guard let processIdentifier = (windowInfo[kCGWindowOwnerPID as String] as? NSNumber)?.int32Value,
685-
inspectedProcessIdentifiers.insert(processIdentifier).inserted
686-
else {
687-
continue
688-
}
689-
let appElement = AXUIElementCreateApplication(processIdentifier)
690-
for window in windows(of: appElement) {
691-
var remainingNodes = 200
692-
guard elementTreeContainsLocalNetworkText(window, remainingNodes: &remainingNodes) else {
693-
continue
694-
}
695-
let hasDenialAction = collectButtons(root: window).contains { button in
696-
let label = resolveElementLabel(button)
697-
.trimmingCharacters(in: .whitespacesAndNewlines)
698-
.lowercased()
699-
return label == "don't allow" || label == "don’t allow"
700-
}
701-
if hasDenialAction {
702-
return window
703-
}
704-
}
705-
}
706-
return nil
707-
}
708-
709-
private func elementTreeContainsLocalNetworkText(
710-
_ element: AXUIElement,
711-
remainingNodes: inout Int
712-
) -> Bool {
713-
guard remainingNodes > 0 else {
714-
return false
715-
}
716-
remainingNodes -= 1
717-
if let text = readableText(for: element),
718-
text.localizedCaseInsensitiveContains("local network")
719-
{
720-
return true
721-
}
722-
for child in children(of: element) {
723-
if elementTreeContainsLocalNetworkText(child, remainingNodes: &remainingNodes) {
724-
return true
725-
}
726-
}
727-
return false
728-
}
729-
730692
private func findAlertElementRecursively(root: AXUIElement, depth: Int) -> AXUIElement? {
731693
if depth > 4 {
732694
return nil

0 commit comments

Comments
 (0)