Repository navigation
Expand file tree
/
Copy pathmain.swift
More file actions
2056 lines (1829 loc) · 105 KB
/
Copy pathmain.swift
File metadata and controls
2056 lines (1829 loc) · 105 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
import MCP
import Foundation
import CoreGraphics // Still needed for CGPoint, CGEventFlags
import ApplicationServices // For AXUIElement APIs (scroll-into-view, window bounds)
import AppKit // For NSEvent.mouseLocation (cursor position save/restore)
import MacosUseSDK // <-- Import the SDK
// --- Helper to serialize Swift structs to JSON String ---
func serializeToJsonString<T: Encodable>(_ value: T) -> String? {
let encoder = JSONEncoder()
// Use pretty printing for easier debugging of the output if needed
encoder.outputFormatting = [.sortedKeys, .withoutEscapingSlashes]
do {
let jsonData = try encoder.encode(value)
return String(data: jsonData, encoding: .utf8)
} catch {
fputs("error: serializeToJsonString: failed to encode value to JSON: \(error)\n", stderr)
return nil
}
}
// --- Function to get arguments from MCP Value ---
// Helper to extract typed values safely
func getRequiredString(from args: [String: Value]?, key: String) throws -> String {
guard let val = args?[key]?.stringValue else {
throw MCPError.invalidParams("Missing or invalid required string argument: '\(key)'")
}
return val
}
func getRequiredDouble(from args: [String: Value]?, key: String) throws -> Double {
guard let value = args?[key] else {
throw MCPError.invalidParams("Missing required number argument: '\(key)'")
}
switch value {
case .int(let intValue):
fputs("log: getRequiredDouble: converting int \(intValue) to double for key '\(key)'\n", stderr)
return Double(intValue)
case .double(let doubleValue):
return doubleValue
case .string(let stringValue):
if let doubleValue = Double(stringValue) {
fputs("log: getRequiredDouble: converting string \"\(stringValue)\" to double for key '\(key)'\n", stderr)
return doubleValue
}
throw MCPError.invalidParams("Invalid type for required number argument: '\(key)', string \"\(stringValue)\" is not a valid number")
default:
throw MCPError.invalidParams("Invalid type for required number argument: '\(key)', expected Int, Double, or numeric String, got \(value)")
}
}
func getRequiredInt(from args: [String: Value]?, key: String) throws -> Int {
guard let value = args?[key] else {
throw MCPError.invalidParams("Missing required integer argument: '\(key)'")
}
// Allow conversion from Double if it's an exact integer
if let doubleValue = value.doubleValue {
if let intValue = Int(exactly: doubleValue) {
fputs("log: getRequiredInt: converting exact double \(doubleValue) to int for key '\(key)'\n", stderr)
return intValue
} else {
fputs("warning: getRequiredInt: received non-exact double \(doubleValue) for key '\(key)', expecting integer.\n", stderr)
throw MCPError.invalidParams("Invalid type for required integer argument: '\(key)', received non-exact Double \(doubleValue)")
}
}
// Allow conversion from String (MCP clients may send numbers as strings)
if let stringValue = value.stringValue, let intValue = Int(stringValue) {
fputs("log: getRequiredInt: converting string \"\(stringValue)\" to int for key '\(key)'\n", stderr)
return intValue
}
// Otherwise, require it to be an Int directly
guard let intValue = value.intValue else {
throw MCPError.invalidParams("Invalid type for required integer argument: '\(key)', expected Int, exact Double, or numeric String, got \(value)")
}
return intValue
}
// --- Get Optional arguments ---
// Helper for optional values
func getOptionalDouble(from args: [String: Value]?, key: String) throws -> Double? {
guard let value = args?[key] else { return nil } // Key not present is valid for optional
if value.isNull { return nil } // Explicit null is also valid
switch value {
case .int(let intValue):
fputs("log: getOptionalDouble: converting int \(intValue) to double for key '\(key)'\n", stderr)
return Double(intValue)
case .double(let doubleValue):
return doubleValue
case .string(let stringValue):
if let doubleValue = Double(stringValue) {
fputs("log: getOptionalDouble: converting string \"\(stringValue)\" to double for key '\(key)'\n", stderr)
return doubleValue
}
throw MCPError.invalidParams("Invalid type for optional number argument: '\(key)', string \"\(stringValue)\" is not a valid number")
default:
throw MCPError.invalidParams("Invalid type for optional number argument: '\(key)', expected Int, Double, or numeric String, got \(value)")
}
}
func getOptionalInt(from args: [String: Value]?, key: String) throws -> Int? {
guard let value = args?[key] else { return nil } // Key not present is valid for optional
if value.isNull { return nil } // Explicit null is also valid
if let doubleValue = value.doubleValue {
if let intValue = Int(exactly: doubleValue) {
fputs("log: getOptionalInt: converting exact double \(doubleValue) to int for key '\(key)'\n", stderr)
return intValue
} else {
fputs("warning: getOptionalInt: received non-exact double \(doubleValue) for key '\(key)', expecting integer.\n", stderr)
throw MCPError.invalidParams("Invalid type for optional integer argument: '\(key)', received non-exact Double \(doubleValue)")
}
}
// Allow conversion from String (MCP clients may send numbers as strings)
if let stringValue = value.stringValue, let intValue = Int(stringValue) {
fputs("log: getOptionalInt: converting string \"\(stringValue)\" to int for key '\(key)'\n", stderr)
return intValue
}
guard let intValue = value.intValue else {
throw MCPError.invalidParams("Invalid type for optional integer argument: '\(key)', expected Int, exact Double, or numeric String, got \(value)")
}
return intValue
}
func getOptionalBool(from args: [String: Value]?, key: String) throws -> Bool? {
guard let value = args?[key] else { return nil } // Key not present
if value.isNull { return nil } // Explicit null
guard let boolValue = value.boolValue else {
throw MCPError.invalidParams("Invalid type for optional boolean argument: '\(key)', expected Bool, got \(value)")
}
return boolValue
}
// --- NEW Helper to parse modifier flags ---
func parseFlags(from value: Value?) throws -> CGEventFlags {
guard let arrayValue = value?.arrayValue else {
// No flags provided or not an array, return empty flags
return []
}
var flags: CGEventFlags = []
for flagValue in arrayValue {
guard let flagString = flagValue.stringValue else {
throw MCPError.invalidParams("Invalid modifierFlags array: contains non-string element \(flagValue)")
}
switch flagString.lowercased() {
// Standard modifiers
case "capslock", "caps": flags.insert(.maskAlphaShift)
case "shift": flags.insert(.maskShift)
case "control", "ctrl": flags.insert(.maskControl)
case "option", "opt", "alt": flags.insert(.maskAlternate)
case "command", "cmd": flags.insert(.maskCommand)
// Other potentially useful flags
case "help": flags.insert(.maskHelp)
case "function", "fn": flags.insert(.maskSecondaryFn)
case "numericpad", "numpad": flags.insert(.maskNumericPad)
// Non-keyed state (less common for press simulation)
// case "noncoalesced": flags.insert(.maskNonCoalesced)
default:
fputs("warning: parseFlags: unknown modifier flag string '\(flagString)', ignoring.\n", stderr)
// Optionally throw an error:
// throw MCPError.invalidParams("Unknown modifier flag: '\(flagString)'")
}
}
return flags
}
// --- Enriched Data Structures (adds in_viewport metadata) ---
struct EnrichedElementData: Codable {
var role: String
var text: String?
var x: Double?
var y: Double?
var width: Double?
var height: Double?
var in_viewport: Bool?
}
struct EnrichedResponseData: Codable {
let app_name: String
var elements: [EnrichedElementData]
var stats: Statistics
let processing_time_seconds: String
}
/// Diff element: role, text, viewport status, and coordinates for spatial targeting
struct DiffElementData: Codable {
var role: String
var text: String?
var in_viewport: Bool?
var x: Double?
var y: Double?
var width: Double?
var height: Double?
}
struct DiffAttributeChange: Codable {
let attributeName: String
let addedText: String?
let removedText: String?
let oldValue: String?
let newValue: String?
}
struct DiffModifiedElement: Codable {
let before: DiffElementData
let after: DiffElementData
let changes: [DiffAttributeChange]
}
struct EnrichedTraversalDiff: Codable {
let added: [DiffElementData]
let removed: [DiffElementData]
let modified: [DiffModifiedElement]
}
/// Simplified response: returns either a full traversal (open/refresh) or a diff (click/type/press)
struct ToolResponse: Codable {
var openResult: AppOpenerResult?
var traversalPid: pid_t?
var traversal: EnrichedResponseData? // for open/refresh: full current state
var diff: EnrichedTraversalDiff? // for click/type/press: what changed
var primaryActionError: String?
var traversalError: String?
// Cross-app handoff: populated when a different app became frontmost after the action
var appSwitchPid: pid_t?
var appSwitchAppName: String?
var appSwitchTraversal: EnrichedResponseData?
// Sheet/dialog detection
var sheetDetected: Bool?
// Window bounds from traversal (used to match correct CGWindow for screenshots)
var windowBounds: CGRect?
}
// --- Sheet Detection ---
/// Check if the app has an AXSheet child (file dialogs, save sheets, etc.)
/// and return its bounds for viewport scoping.
func findSheetBounds(pid: pid_t) -> CGRect? {
let appElement = AXUIElementCreateApplication(pid)
AXUIElementSetMessagingTimeout(appElement, 5.0)
// Check all windows for AXSheet children
var windowsRef: CFTypeRef?
guard AXUIElementCopyAttributeValue(appElement, "AXWindows" as CFString, &windowsRef) == .success,
let windows = windowsRef as? [AXUIElement] else { return nil }
for window in windows {
AXUIElementSetMessagingTimeout(window, 5.0)
// Look for AXSheet role among children
var childCount: CFIndex = 0
guard AXUIElementGetAttributeValueCount(window, kAXChildrenAttribute as CFString, &childCount) == .success,
childCount > 0 else { continue }
let fetchCount = min(CFIndex(50), childCount)
var childrenRef: CFArray?
guard AXUIElementCopyAttributeValues(window, kAXChildrenAttribute as CFString, 0, fetchCount, &childrenRef) == .success,
let cfArray = childrenRef else { continue }
let children = cfArray as [AnyObject]
for child in children {
let childElement = child as! AXUIElement
AXUIElementSetMessagingTimeout(childElement, 5.0)
var roleRef: CFTypeRef?
guard AXUIElementCopyAttributeValue(childElement, kAXRoleAttribute as CFString, &roleRef) == .success,
let role = roleRef as? String else { continue }
if role == "AXSheet" {
if let frame = getAXElementFrame(childElement) {
fputs("log: findSheetBounds: found AXSheet at \(frame)\n", stderr)
return frame
}
}
}
}
return nil
}
// --- Viewport Detection Helpers ---
/// Extract window bounds from traversal data by finding the AXWindow element
func getWindowBoundsFromTraversal(_ responseData: ResponseData?) -> CGRect? {
guard let response = responseData else { return nil }
for element in response.elements {
if element.role == "AXWindow",
let x = element.x, let y = element.y,
let w = element.width, let h = element.height {
return CGRect(x: x, y: y, width: w, height: h)
}
}
return nil
}
/// Extract bounds for ALL windows from traversal data.
/// Used for multi-window viewport detection (e.g. Sparkle update dialogs).
func getAllWindowBoundsFromTraversal(_ responseData: ResponseData?) -> [CGRect] {
guard let response = responseData else { return [] }
return response.elements.compactMap { element in
guard element.role == "AXWindow",
let x = element.x, let y = element.y,
let w = element.width, let h = element.height else { return nil }
return CGRect(x: x, y: y, width: w, height: h)
}
}
/// Get bounds for ALL windows of a PID from the accessibility API
func getAllWindowBoundsFromAPI(pid: pid_t) -> [CGRect] {
let appElement = AXUIElementCreateApplication(pid)
AXUIElementSetMessagingTimeout(appElement, 5.0)
var windowsRef: CFTypeRef?
guard AXUIElementCopyAttributeValue(appElement, "AXWindows" as CFString, &windowsRef) == .success,
let windows = windowsRef as? [AXUIElement] else { return [] }
return windows.compactMap { getAXElementFrame($0) }
}
/// Check if a point falls within any of the given window bounds
func isPointInAnyWindow(_ point: CGPoint, windows: [CGRect]) -> Bool {
return windows.contains { $0.contains(point) }
}
/// Find the window (element + frame) whose frame contains the given point.
/// Searches all AXWindows of the app; falls back to AXMainWindow if none matches.
func getWindowContainingPoint(appElement: AXUIElement, point: CGPoint) -> (element: AXUIElement, bounds: CGRect)? {
var windowsRef: CFTypeRef?
if AXUIElementCopyAttributeValue(appElement, "AXWindows" as CFString, &windowsRef) == .success,
let windows = windowsRef as? [AXUIElement] {
for window in windows {
AXUIElementSetMessagingTimeout(window, 5.0)
guard let frame = getAXElementFrame(window) else { continue }
if frame.contains(point) {
fputs("log: getWindowContainingPoint: matched window \(frame) for point \(point)\n", stderr)
return (window, frame)
}
}
}
// Fallback to main window
var winRef: CFTypeRef?
guard AXUIElementCopyAttributeValue(appElement, "AXMainWindow" as CFString, &winRef) == .success else { return nil }
let win = winRef as! AXUIElement
AXUIElementSetMessagingTimeout(win, 5.0)
guard let frame = getAXElementFrame(win) else { return nil }
fputs("log: getWindowContainingPoint: no window contains \(point), falling back to main window \(frame)\n", stderr)
return (win, frame)
}
/// Get window bounds directly from the accessibility API
func getWindowBoundsFromAPI(pid: pid_t) -> CGRect? {
let appElement = AXUIElementCreateApplication(pid)
AXUIElementSetMessagingTimeout(appElement, 5.0)
var windowValue: CFTypeRef?
guard AXUIElementCopyAttributeValue(appElement, "AXMainWindow" as CFString, &windowValue) == .success else {
fputs("warning: getWindowBoundsFromAPI: could not get main window for pid \(pid)\n", stderr)
return nil
}
let windowElement = windowValue as! AXUIElement
var positionValue: CFTypeRef?
guard AXUIElementCopyAttributeValue(windowElement, "AXPosition" as CFString, &positionValue) == .success else {
fputs("warning: getWindowBoundsFromAPI: could not get window position\n", stderr)
return nil
}
var position = CGPoint.zero
AXValueGetValue(positionValue as! AXValue, .cgPoint, &position)
var sizeValue: CFTypeRef?
guard AXUIElementCopyAttributeValue(windowElement, "AXSize" as CFString, &sizeValue) == .success else {
fputs("warning: getWindowBoundsFromAPI: could not get window size\n", stderr)
return nil
}
var size = CGSize.zero
AXValueGetValue(sizeValue as! AXValue, .cgSize, &size)
return CGRect(origin: position, size: size)
}
/// Capture a screenshot of the window(s) belonging to a given PID and save as PNG.
/// If `clickPoint` is provided (screen coordinates), draws a red crosshair at that location.
/// Returns the file path on success, nil on failure.
///
/// IMPORTANT: The actual CGWindowListCreateImage call runs in a **subprocess**
/// (screenshot-helper) so that the ReplayKit framework — loaded as a side-effect
/// by macOS — dies with the subprocess instead of spinning at ~19% CPU forever
/// in the parent MCP server process.
func captureWindowScreenshot(pid: pid_t, outputPath: String, clickPoint: CGPoint? = nil, traversalWindowBounds: CGRect? = nil) -> String? {
// Get the list of windows for this PID
guard let windowList = CGWindowListCopyWindowInfo([.optionOnScreenOnly, .excludeDesktopElements], kCGNullWindowID) as? [[String: Any]] else {
fputs("warning: captureWindowScreenshot: could not get window list\n", stderr)
return nil
}
// Find the correct window for this PID.
// Strategy: if we have traversal window bounds, match the CGWindow whose bounds
// best overlap with the traversal window. Otherwise fall back to largest window.
var targetWindowID: CGWindowID? = nil
var windowBoundsDict: CFDictionary? = nil
var bestScore: CGFloat = 0
for window in windowList {
guard let ownerPID = window[kCGWindowOwnerPID as String] as? pid_t,
ownerPID == pid,
let layer = window[kCGWindowLayer as String] as? Int,
layer == 0,
let windowID = window[kCGWindowNumber as String] as? CGWindowID else {
continue
}
let bounds = window[kCGWindowBounds as String] as! CFDictionary
var rect = CGRect.zero
CGRectMakeWithDictionaryRepresentation(bounds, &rect)
let score: CGFloat
if let twb = traversalWindowBounds {
let intersection = rect.intersection(twb)
score = intersection.isNull ? 0 : intersection.width * intersection.height
} else {
score = rect.width * rect.height
}
if score > bestScore {
bestScore = score
targetWindowID = windowID
windowBoundsDict = bounds
}
}
if let windowID = targetWindowID {
fputs("log: captureWindowScreenshot: selected window \(windowID) (score=\(Int(bestScore)))\n", stderr)
}
guard let windowID = targetWindowID else {
fputs("warning: captureWindowScreenshot: no on-screen window found for PID \(pid)\n", stderr)
return nil
}
// Find the screenshot-helper binary next to our own executable
let myPath = CommandLine.arguments[0]
let myDir = (myPath as NSString).deletingLastPathComponent
let helperPath = (myDir as NSString).appendingPathComponent("screenshot-helper")
guard FileManager.default.fileExists(atPath: helperPath) else {
fputs("warning: captureWindowScreenshot: screenshot-helper not found at \(helperPath)\n", stderr)
return nil
}
// Build arguments for the subprocess
var helperArgs = [String(windowID), outputPath]
if let clickPoint = clickPoint, let boundsDict = windowBoundsDict {
var windowRect = CGRect.zero
CGRectMakeWithDictionaryRepresentation(boundsDict, &windowRect)
helperArgs += ["--click", "\(clickPoint.x),\(clickPoint.y)"]
helperArgs += ["--bounds", "\(windowRect.origin.x),\(windowRect.origin.y),\(windowRect.width),\(windowRect.height)"]
fputs("log: captureWindowScreenshot: invoking subprocess with click=\(clickPoint.x),\(clickPoint.y) bounds=\(windowRect)\n", stderr)
}
fputs("log: captureWindowScreenshot: launching screenshot-helper for window \(windowID)...\n", stderr)
// Run screenshot-helper in a subprocess — ReplayKit dies when it exits
let process = Process()
process.executableURL = URL(fileURLWithPath: helperPath)
process.arguments = helperArgs
let stdoutPipe = Pipe()
let stderrPipe = Pipe()
process.standardOutput = stdoutPipe
process.standardError = stderrPipe
do {
try process.run()
} catch {
fputs("warning: captureWindowScreenshot: failed to launch screenshot-helper: \(error)\n", stderr)
return nil
}
// Wait with timeout
let timeoutSeconds = 5.0
let deadline = DispatchTime.now() + timeoutSeconds
let group = DispatchGroup()
group.enter()
DispatchQueue.global().async {
process.waitUntilExit()
group.leave()
}
if group.wait(timeout: deadline) == .timedOut {
process.terminate()
fputs("warning: captureWindowScreenshot: screenshot-helper timed out (\(timeoutSeconds)s)\n", stderr)
return nil
}
// Forward stderr from the helper
let stderrData = stderrPipe.fileHandleForReading.readDataToEndOfFile()
if !stderrData.isEmpty, let stderrStr = String(data: stderrData, encoding: .utf8) {
fputs(stderrStr, stderr)
}
guard process.terminationStatus == 0 else {
fputs("warning: captureWindowScreenshot: screenshot-helper exited with status \(process.terminationStatus)\n", stderr)
return nil
}
let stdoutData = stdoutPipe.fileHandleForReading.readDataToEndOfFile()
let result = String(data: stdoutData, encoding: .utf8)?.trimmingCharacters(in: .whitespacesAndNewlines)
if let path = result, !path.isEmpty {
fputs("log: captureWindowScreenshot: screenshot saved via subprocess to \(path)\n", stderr)
return path
}
return nil
}
/// Enrich a ResponseData with in_viewport metadata for each element
func enrichResponseData(_ response: ResponseData, windowBounds: CGRect?, allWindowBounds: [CGRect] = []) -> EnrichedResponseData {
// Use all window bounds if available; fall back to single windowBounds
let boundsToCheck: [CGRect]
if !allWindowBounds.isEmpty {
boundsToCheck = allWindowBounds
} else if let bounds = windowBounds {
boundsToCheck = [bounds]
} else {
boundsToCheck = []
}
let enrichedElements = response.elements.map { element -> EnrichedElementData in
let inViewport: Bool?
if let x = element.x, let y = element.y, !boundsToCheck.isEmpty {
inViewport = isPointInAnyWindow(CGPoint(x: x, y: y), windows: boundsToCheck)
} else {
inViewport = nil
}
return EnrichedElementData(
role: element.role, text: element.text,
x: element.x, y: element.y,
width: element.width, height: element.height,
in_viewport: inViewport
)
}
return EnrichedResponseData(
app_name: response.app_name,
elements: enrichedElements,
stats: response.stats,
processing_time_seconds: response.processing_time_seconds
)
}
/// Look up text for a container element (AXRow, AXCell) by:
/// 1. Coordinate containment — find a text-bearing child within the element's bounds
/// 2. List proximity — the traversal is depth-first, so children follow the parent;
/// find the element in the flat list and check the next few entries for text.
/// This handles off-screen elements whose children have no coordinates.
func findTextForElement(_ element: ElementData, in traversal: ResponseData?) -> String? {
if let text = element.text, !text.isEmpty { return text }
guard let elements = traversal?.elements else { return nil }
// Strategy 1: coordinate containment (works for visible elements)
if let x = element.x, let y = element.y,
let w = element.width, let h = element.height {
let bounds = CGRect(x: x, y: y, width: w, height: h)
for el in elements {
if let elText = el.text, !elText.isEmpty,
let elX = el.x, let elY = el.y,
bounds.contains(CGPoint(x: elX, y: elY)) {
return elText
}
}
}
// Strategy 2: list proximity (works for off-screen elements)
// Use approximate coordinate matching (±2px) to handle floating-point differences
if let x = element.x, let y = element.y {
for (i, el) in elements.enumerated() {
if el.role == element.role,
let elX = el.x, let elY = el.y,
abs(elX - x) < 2, abs(elY - y) < 2 {
// Found the element — look at next few entries for text
for j in (i + 1)..<min(i + 6, elements.count) {
if let text = elements[j].text, !text.isEmpty {
return text
}
// Stop if we hit another row (left the subtree)
if elements[j].role.contains("AXRow") && j > i + 1 { break }
}
break
}
}
}
return nil
}
/// Returns true if the role represents a scroll-bar component (noise in diffs).
func isScrollBarNoise(_ role: String) -> Bool {
let lower = role.lowercased()
return lower.contains("scrollbar") || lower.contains("scroll bar") ||
lower.contains("value indicator") ||
lower.contains("page button") || lower.contains("arrow button")
}
/// Returns true if the role is a structural container that's noise without text.
func isStructuralNoise(_ role: String, text: String?) -> Bool {
if let text = text, !text.isEmpty { return false }
let lower = role.lowercased()
return lower.contains("axrow") || lower.contains("outline row") ||
lower.contains("axcell") || lower.contains("cell") ||
lower.contains("axcolumn") || lower.contains("column") ||
lower.contains("axmenu") || lower.contains("menu")
}
/// Build a ToolResponse from an ActionResult.
/// For actions with diff: returns only the diff (enriched with in_viewport on added elements).
/// For open/refresh: returns the full traversal (enriched with in_viewport).
func buildToolResponse(_ result: ActionResult, hasDiff: Bool) -> ToolResponse {
var windowBounds = getWindowBoundsFromTraversal(result.traversalAfter)
?? getWindowBoundsFromTraversal(result.traversalBefore)
if windowBounds == nil {
if let pid = result.traversalPid ?? result.openResult?.pid {
windowBounds = getWindowBoundsFromAPI(pid: pid)
}
}
// Collect ALL window bounds for multi-window viewport detection.
// An element is "visible" if it falls within ANY window of the app.
var allWindowBounds = getAllWindowBoundsFromTraversal(result.traversalAfter)
if allWindowBounds.isEmpty {
allWindowBounds = getAllWindowBoundsFromTraversal(result.traversalBefore)
}
if allWindowBounds.isEmpty, let pid = result.traversalPid ?? result.openResult?.pid {
allWindowBounds = getAllWindowBoundsFromAPI(pid: pid)
}
// Check for AXSheet (file dialogs, save sheets) — use sheet bounds for viewport
var sheetDetected = false
if let pid = result.traversalPid ?? result.openResult?.pid {
if let sheetBounds = findSheetBounds(pid: pid) {
windowBounds = sheetBounds
sheetDetected = true
}
}
var response = ToolResponse()
response.openResult = result.openResult
response.traversalPid = result.traversalPid
response.primaryActionError = result.primaryActionError
response.traversalError = result.traversalAfterError ?? result.traversalBeforeError
response.windowBounds = windowBounds
response.sheetDetected = sheetDetected ? true : nil
if hasDiff, let rawDiff = result.traversalDiff {
let coordinateAttrs: Set<String> = ["x", "y", "width", "height"]
// Filter noise, strip coordinates, add in_viewport, resolve text
let filteredAdded = rawDiff.added
.filter { !isScrollBarNoise($0.role) }
.map { element -> DiffElementData in
let inViewport: Bool?
if let x = element.x, let y = element.y, !allWindowBounds.isEmpty {
inViewport = isPointInAnyWindow(CGPoint(x: x, y: y), windows: allWindowBounds)
} else {
inViewport = nil
}
let text = findTextForElement(element, in: result.traversalAfter)
return DiffElementData(role: element.role, text: text, in_viewport: inViewport,
x: element.x, y: element.y, width: element.width, height: element.height)
}
.filter { !isStructuralNoise($0.role, text: $0.text) }
let filteredRemoved = rawDiff.removed
.filter { !isScrollBarNoise($0.role) }
.map { element -> DiffElementData in
let text = findTextForElement(element, in: result.traversalBefore)
return DiffElementData(role: element.role, text: text, in_viewport: nil,
x: element.x, y: element.y, width: element.width, height: element.height)
}
.filter { !isStructuralNoise($0.role, text: $0.text) }
// Filter modified: skip scroll-bar noise, drop coordinate-only changes
var filteredModified: [DiffModifiedElement] = []
for mod in rawDiff.modified {
if isScrollBarNoise(mod.before.role) || isScrollBarNoise(mod.after.role) { continue }
let meaningfulChanges = mod.changes.filter { !coordinateAttrs.contains($0.attributeName) }
if meaningfulChanges.isEmpty { continue }
let diffChanges = meaningfulChanges.map {
DiffAttributeChange(
attributeName: $0.attributeName,
addedText: $0.addedText,
removedText: $0.removedText,
oldValue: $0.oldValue,
newValue: $0.newValue
)
}
let beforeVP: Bool?
if let x = mod.before.x, let y = mod.before.y, !allWindowBounds.isEmpty {
beforeVP = isPointInAnyWindow(CGPoint(x: x, y: y), windows: allWindowBounds)
} else { beforeVP = nil }
let afterVP: Bool?
if let x = mod.after.x, let y = mod.after.y, !allWindowBounds.isEmpty {
afterVP = isPointInAnyWindow(CGPoint(x: x, y: y), windows: allWindowBounds)
} else { afterVP = nil }
let beforeText = findTextForElement(mod.before, in: result.traversalBefore)
let afterText = findTextForElement(mod.after, in: result.traversalAfter)
filteredModified.append(DiffModifiedElement(
before: DiffElementData(role: mod.before.role, text: beforeText, in_viewport: beforeVP),
after: DiffElementData(role: mod.after.role, text: afterText, in_viewport: afterVP),
changes: diffChanges
))
}
response.diff = EnrichedTraversalDiff(
added: filteredAdded,
removed: filteredRemoved,
modified: filteredModified
)
} else if let after = result.traversalAfter {
// Full traversal for open/refresh
response.traversal = enrichResponseData(after, windowBounds: windowBounds, allWindowBounds: allWindowBounds)
}
return response
}
// --- Compact Summary Builder (file-based MCP responses) ---
/// Build a concise text summary for the MCP response instead of returning the full JSON.
/// The full JSON is written to a file; this summary contains just the key info + file path.
func buildCompactSummary(toolName: String, params: CallTool.Parameters, toolResponse: ToolResponse, filepath: String, fileSize: Int, screenshotPath: String? = nil) -> String {
var lines: [String] = []
// Status line
let status = (toolResponse.primaryActionError != nil || toolResponse.traversalError != nil) ? "error" : "success"
lines.append("status: \(status)")
// PID and app
if let pid = toolResponse.traversalPid {
lines.append("pid: \(pid)")
}
if let appName = toolResponse.traversal?.app_name ?? toolResponse.openResult?.appName {
lines.append("app: \(appName)")
}
if toolResponse.sheetDetected == true {
lines.append("dialog: AXSheet detected (viewport scoped to sheet bounds)")
}
// File path + metadata
lines.append("file: \(filepath)")
let elementCount: Int
if let traversal = toolResponse.traversal {
elementCount = traversal.elements.count
} else {
let added = toolResponse.diff?.added.count ?? 0
let removed = toolResponse.diff?.removed.count ?? 0
let modified = toolResponse.diff?.modified.count ?? 0
elementCount = added + removed + modified
}
lines.append("file_size: \(fileSize) bytes (\(elementCount) elements)")
lines.append("hint: grep -n 'AXButton' \(filepath) # search by role or text")
if let screenshotPath = screenshotPath {
lines.append("screenshot: \(screenshotPath)")
}
// Errors if any
if let err = toolResponse.primaryActionError {
lines.append("error: \(err)")
}
if let err = toolResponse.traversalError {
lines.append("traversal_error: \(err)")
}
// Tool-specific summary line
let summaryLine: String
switch toolName {
case "macos-use_open_application_and_traverse":
let identifier = params.arguments?["identifier"]?.stringValue ?? "unknown"
if let traversal = toolResponse.traversal {
let total = traversal.elements.count
let visible = traversal.elements.filter { $0.in_viewport == true }.count
summaryLine = "Opened \(identifier) (PID:\(toolResponse.traversalPid ?? 0)). \(total) elements, \(visible) visible."
} else {
summaryLine = "Opened \(identifier) (PID:\(toolResponse.traversalPid ?? 0))."
}
case "macos-use_click_and_traverse":
let isDoubleClick = params.arguments?["doubleClick"]?.boolValue ?? false
let isRightClick = params.arguments?["rightClick"]?.boolValue ?? false
let clickType = isDoubleClick ? "Double-clicked" : isRightClick ? "Right-clicked" : "Clicked"
let diffSummary = buildDiffSummary(toolResponse.diff)
if let elemSearch = params.arguments?["element"]?.stringValue {
let roleFilter = params.arguments?["role"]?.stringValue
let roleDesc = roleFilter != nil ? " [\(roleFilter!)]" : ""
summaryLine = "\(clickType) element '\(elemSearch)'\(roleDesc). \(diffSummary)"
} else {
let x = params.arguments?["x"]?.doubleValue ?? params.arguments?["x"]?.intValue.map(Double.init) ?? 0
let y = params.arguments?["y"]?.doubleValue ?? params.arguments?["y"]?.intValue.map(Double.init) ?? 0
summaryLine = "\(clickType) at (\(Int(x)),\(Int(y))). \(diffSummary)"
}
case "macos-use_type_and_traverse":
let text = params.arguments?["text"]?.stringValue ?? ""
let truncatedText = text.count > 40 ? String(text.prefix(40)) + "..." : text
let diffSummary = buildDiffSummary(toolResponse.diff)
summaryLine = "Typed '\(truncatedText)'. \(diffSummary)"
case "macos-use_press_key_and_traverse":
let keyName = params.arguments?["keyName"]?.stringValue ?? "unknown"
let mods = params.arguments?["modifierFlags"]?.arrayValue?.compactMap { $0.stringValue }.joined(separator: "+")
let keyDesc = (mods != nil && !mods!.isEmpty) ? "\(mods!)+\(keyName)" : keyName
let diffSummary = buildDiffSummary(toolResponse.diff)
summaryLine = "Pressed \(keyDesc). \(diffSummary)"
case "macos-use_scroll_and_traverse":
let x = params.arguments?["x"]?.doubleValue ?? params.arguments?["x"]?.intValue.map(Double.init) ?? 0
let y = params.arguments?["y"]?.doubleValue ?? params.arguments?["y"]?.intValue.map(Double.init) ?? 0
let deltaY = params.arguments?["deltaY"]?.intValue ?? 0
let diffSummary = buildDiffSummary(toolResponse.diff)
summaryLine = "Scrolled deltaY=\(deltaY) at (\(Int(x)),\(Int(y))). \(diffSummary)"
case "macos-use_refresh_traversal":
if let traversal = toolResponse.traversal {
let total = traversal.elements.count
let visible = traversal.elements.filter { $0.in_viewport == true }.count
summaryLine = "Refreshed PID \(toolResponse.traversalPid ?? 0) (\(traversal.app_name)). \(total) elements, \(visible) visible."
} else {
summaryLine = "Refreshed PID \(toolResponse.traversalPid ?? 0)."
}
case "macos-use_set_value_and_traverse":
let x = params.arguments?["x"]?.doubleValue ?? params.arguments?["x"]?.intValue.map(Double.init) ?? 0
let y = params.arguments?["y"]?.doubleValue ?? params.arguments?["y"]?.intValue.map(Double.init) ?? 0
let value = params.arguments?["value"]?.stringValue ?? ""
let truncatedValue = value.count > 40 ? String(value.prefix(40)) + "..." : value
let diffSummary = buildDiffSummary(toolResponse.diff)
summaryLine = "AX set value '\(truncatedValue)' at (\(Int(x)),\(Int(y))). \(diffSummary)"
case "macos-use_press_ax_and_traverse":
let x = params.arguments?["x"]?.doubleValue ?? params.arguments?["x"]?.intValue.map(Double.init) ?? 0
let y = params.arguments?["y"]?.doubleValue ?? params.arguments?["y"]?.intValue.map(Double.init) ?? 0
let diffSummary = buildDiffSummary(toolResponse.diff)
summaryLine = "AX pressed element at (\(Int(x)),\(Int(y))). \(diffSummary)"
case "macos-use_set_selected_and_traverse":
let x = params.arguments?["x"]?.doubleValue ?? params.arguments?["x"]?.intValue.map(Double.init) ?? 0
let y = params.arguments?["y"]?.doubleValue ?? params.arguments?["y"]?.intValue.map(Double.init) ?? 0
let sel = params.arguments?["selected"]?.boolValue ?? true
let diffSummary = buildDiffSummary(toolResponse.diff)
summaryLine = "AX set selected=\(sel) at (\(Int(x)),\(Int(y))). \(diffSummary)"
default:
summaryLine = "Tool \(toolName) completed."
}
lines.append("summary: \(summaryLine)")
// Append notable text changes from diff (up to 3, truncated)
if let diff = toolResponse.diff {
var textChanges: [String] = []
for mod in diff.modified.prefix(5) {
for change in mod.changes {
if change.attributeName == "text" || change.attributeName == "AXValue" {
let oldVal = truncate(change.oldValue ?? change.removedText ?? "", maxLen: 60)
let newVal = truncate(change.newValue ?? change.addedText ?? "", maxLen: 60)
if !oldVal.isEmpty || !newVal.isEmpty {
textChanges.append(" '\(oldVal)' -> '\(newVal)'")
}
}
}
if textChanges.count >= 3 { break }
}
if !textChanges.isEmpty {
lines.append("text_changes:")
lines.append(contentsOf: textChanges.prefix(3))
}
}
// Inline visible interactive elements
if let traversal = toolResponse.traversal {
// Full traversal (open/refresh): show all visible interactive elements
let visLines = buildVisibleElementsSection(elements: traversal.elements, label: "visible_elements")
lines.append(contentsOf: visLines)
} else if let diff = toolResponse.diff, !diff.added.isEmpty {
// Diff (click/type/press/scroll): show newly added visible elements
let visLines = buildVisibleElementsSection(elements: diff.added, label: "visible_elements", interactiveCap: 20, textCap: 10)
lines.append(contentsOf: visLines)
}
// Cross-app handoff: a different app became frontmost after the action
if let switchPid = toolResponse.appSwitchPid {
let switchName = toolResponse.appSwitchAppName ?? "Unknown"
lines.append("app_switch: \(switchName) (PID: \(switchPid)) is now frontmost")
if let switchTraversal = toolResponse.appSwitchTraversal {
let total = switchTraversal.elements.count
let visible = switchTraversal.elements.filter { $0.in_viewport == true }.count
lines.append("app_switch_elements: \(total) total, \(visible) visible")
let visLines = buildVisibleElementsSection(elements: switchTraversal.elements, label: "app_switch_visible_elements")
lines.append(contentsOf: visLines)
}
}
return lines.joined(separator: "\n")
}
/// Build a short diff summary string like "3 added, 2 removed, 1 modified."
func buildDiffSummary(_ diff: EnrichedTraversalDiff?) -> String {
guard let diff = diff else { return "No diff." }
var parts: [String] = []
if !diff.added.isEmpty { parts.append("\(diff.added.count) added") }
if !diff.removed.isEmpty { parts.append("\(diff.removed.count) removed") }
if !diff.modified.isEmpty { parts.append("\(diff.modified.count) modified") }
return parts.isEmpty ? "No changes." : parts.joined(separator: ", ") + "."
}
/// Truncate a string to maxLen characters
func truncate(_ s: String, maxLen: Int) -> String {
s.count > maxLen ? String(s.prefix(maxLen)) + "..." : s
}
/// Protocol for element types that can be displayed in visible elements section
protocol VisibleElement {
var role: String { get }
var text: String? { get }
var in_viewport: Bool? { get }
var x: Double? { get }
var y: Double? { get }
var width: Double? { get }
var height: Double? { get }
}
extension EnrichedElementData: VisibleElement {}
extension DiffElementData: VisibleElement {}
/// Interactive role prefixes worth showing inline in the compact summary
private let interactiveRolePrefixes: [String] = [
"AXButton", "AXLink", "AXTextField", "AXTextArea", "AXCheckBox",
"AXRadioButton", "AXPopUpButton", "AXComboBox", "AXSlider",
"AXMenuItem", "AXMenuButton", "AXTab"
]
/// Check if a role string matches any interactive prefix
private func isInteractiveRole(_ role: String) -> Bool {
interactiveRolePrefixes.contains { role.hasPrefix($0) }
}
/// Check if a role string is static text
private func isStaticTextRole(_ role: String) -> Bool {
role.hasPrefix("AXStaticText")
}
/// Build a visible_elements section from a list of elements
func buildVisibleElementsSection<T: VisibleElement>(elements: [T], label: String, interactiveCap: Int = 30, textCap: Int = 10) -> [String] {
var interactive: [String] = []
var staticText: [String] = []
for el in elements {
guard el.in_viewport == true else { continue }
guard let text = el.text, !text.isEmpty else { continue }
let truncatedText = truncate(text, maxLen: 50)
let pos: String
if let x = el.x, let y = el.y, let w = el.width, let h = el.height {
pos = " (\(Int(x)),\(Int(y)) \(Int(w))×\(Int(h)))"
} else {
pos = ""
}
let line = " [\(el.role)] \"\(truncatedText)\"\(pos)"
if isInteractiveRole(el.role) {
if interactive.count < interactiveCap {
interactive.append(line)
}
} else if isStaticTextRole(el.role) {
if staticText.count < textCap {
staticText.append(line)
}
}
}
if interactive.isEmpty && staticText.isEmpty { return [] }
var result = ["\(label):"]
result.append(contentsOf: interactive)
result.append(contentsOf: staticText)
return result
}
// --- Flat Text Response File Builder ---
/// Format a single element as a grep-friendly text line
func formatElementLine(_ el: VisibleElement, prefix: String = " ") -> String {
var parts: [String] = []
parts.append("[\(el.role)]")
if let text = el.text, !text.isEmpty {
let truncated = text.count > 80 ? String(text.prefix(80)) + "..." : text
parts.append("\"\(truncated)\"")
}
if let x = el.x, let y = el.y {