Skip to content

Commit 4e617c8

Browse files
thymikeeclaude
andcommitted
feat(macos): draw a ghost cursor for native app backend actions
Each helper action on a native-backend app session draws its own pointer, glides it to the target, and pulses on delivery, so a person can follow the agent while keeping the real pointer. Only a delivered action's point seeds the next glide, and a point no display shows any more restarts from the user's cursor. AGENT_DEVICE_MACOS_GHOST_CURSOR=0 turns it off; the host budgets the animation on top of each helper deadline. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
1 parent 87340b8 commit 4e617c8

9 files changed

Lines changed: 308 additions & 25 deletions

File tree

‎apple/macos-helper/Sources/AgentDeviceMacOSHelper/BackgroundInteraction.swift‎

Lines changed: 17 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -82,7 +82,8 @@ func requireSessionApplication(bundleId: String?) throws -> NSRunningApplication
8282

8383
func pressInBackground(
8484
_ request: MouseClickRequest,
85-
app: NSRunningApplication
85+
app: NSRunningApplication,
86+
cursor: GhostCursor?
8687
) throws -> BackgroundPressResponse {
8788
guard !request.doubleClick, request.holdMs <= 0 else {
8889
throw refusal(
@@ -92,6 +93,7 @@ func pressInBackground(
9293
)
9394
}
9495
let point = CGPoint(x: request.x, y: request.y)
96+
cursor?.move(to: point)
9597
guard let target = resolvePressTarget(app: app, point: point),
9698
let mechanism = perform(target)
9799
else {
@@ -103,6 +105,7 @@ func pressInBackground(
103105
guard perform(target) != nil else { break }
104106
clicks += 1
105107
}
108+
cursor?.pulse()
106109
return BackgroundPressResponse(
107110
x: request.x,
108111
y: request.y,
@@ -118,10 +121,14 @@ func pressInBackground(
118121
func typeInBackground(
119122
text: String,
120123
delayMs: Int,
121-
app: NSRunningApplication
124+
app: NSRunningApplication,
125+
cursor: GhostCursor?
122126
) throws -> BackgroundTextResponse {
123127
let appElement = AXUIElementCreateApplication(app.processIdentifier)
124128
let focused = elementAttribute(appElement, attribute: kAXFocusedUIElementAttribute as String)
129+
if let focused, let rect = rectAttribute(focused) {
130+
cursor?.move(to: CGPoint(x: rect.x + rect.width / 2, y: rect.y + rect.height / 2))
131+
}
125132
let focusedRole = focused.map(role(of:))
126133
if text == "\n", let focused, actionNames(of: focused).contains(kAXConfirmAction as String),
127134
AXUIElementPerformAction(focused, kAXConfirmAction as CFString) == .success
@@ -144,8 +151,10 @@ func typeInBackground(
144151
func fillInBackground(
145152
point: CGPoint,
146153
text: String,
147-
app: NSRunningApplication
154+
app: NSRunningApplication,
155+
cursor: GhostCursor?
148156
) throws -> BackgroundTextResponse {
157+
cursor?.move(to: point)
149158
let chain = elementAtPoint(in: app, point: point).map(pressSearchChain) ?? []
150159
guard
151160
let input = chain.first(where: isTextInput)
@@ -159,6 +168,7 @@ func fillInBackground(
159168
else {
160169
throw refusal(.noTextInput, "the text input refused the new value", app: app)
161170
}
171+
cursor?.pulse()
162172
return BackgroundTextResponse(bundleId: app.bundleIdentifier, mechanism: .axValue, role: role(of: input))
163173
}
164174

@@ -167,7 +177,8 @@ func scrollInBackground(
167177
direction: String,
168178
amount: Double?,
169179
pixels: Double?,
170-
app: NSRunningApplication
180+
app: NSRunningApplication,
181+
cursor: GhostCursor?
171182
) throws -> BackgroundScrollResponse {
172183
let isVertical = direction == "up" || direction == "down"
173184
guard isVertical || direction == "left" || direction == "right" else {
@@ -185,6 +196,7 @@ func scrollInBackground(
185196
pixels: pixels
186197
)
187198
let center = CGPoint(x: frame.midX, y: frame.midY)
199+
cursor?.move(to: center)
188200
let revealsLaterContent = direction == "down" || direction == "right"
189201
guard
190202
performScrollBarScroll(
@@ -196,6 +208,7 @@ func scrollInBackground(
196208
else {
197209
throw refusal(.noScrollBar, "no scroll area with a settable scroll bar under the window center", app: app)
198210
}
211+
cursor?.pulse()
199212
// Reported as the equivalent drag, start to end, the way the runner reports a desktop scroll.
200213
let half = travel / 2
201214
let sign: Double = revealsLaterContent ? 1 : -1
Lines changed: 181 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,181 @@
1+
import AppKit
2+
import QuartzCore
3+
4+
/// A drawn pointer that shows where a background action lands while the user keeps the real
5+
/// one. The helper exits after each command, so the last point is persisted and the next
6+
/// action glides from it; a stale point restarts from the user's own cursor.
7+
final class GhostCursor {
8+
private static let size = CGSize(width: 64, height: 64)
9+
/// Where the arrow's tip sits inside the panel, from its top-left corner; it leaves room for
10+
/// the pulse ring's full radius.
11+
private static let tipOffset = CGPoint(x: 22, y: 22)
12+
private static let staleAfter: TimeInterval = 30
13+
private static let positionURL = URL(fileURLWithPath: NSTemporaryDirectory())
14+
.appendingPathComponent("agent-device-ghost-cursor.json")
15+
16+
private let panel: NSPanel
17+
private let view: GhostCursorView
18+
private let primaryScreenHeight: CGFloat
19+
private var position: CGPoint
20+
21+
/// Nil when no display is attached; the action still runs, just unseen.
22+
static func show() -> GhostCursor? {
23+
_ = NSApplication.shared
24+
NSApp.setActivationPolicy(.accessory)
25+
guard let primary = NSScreen.screens.first else { return nil }
26+
return GhostCursor(primaryScreenHeight: primary.frame.height)
27+
}
28+
29+
private init(primaryScreenHeight: CGFloat) {
30+
self.primaryScreenHeight = primaryScreenHeight
31+
view = GhostCursorView(frame: CGRect(origin: .zero, size: Self.size), tip: Self.tipOffset)
32+
panel = NSPanel(
33+
contentRect: CGRect(origin: .zero, size: Self.size),
34+
styleMask: [.borderless, .nonactivatingPanel],
35+
backing: .buffered,
36+
defer: false
37+
)
38+
panel.isOpaque = false
39+
panel.backgroundColor = .clear
40+
panel.hasShadow = false
41+
panel.ignoresMouseEvents = true
42+
panel.level = .screenSaver
43+
panel.collectionBehavior = [.canJoinAllSpaces, .stationary, .ignoresCycle, .fullScreenAuxiliary]
44+
panel.contentView = view
45+
position =
46+
Self.restoredPosition(primaryScreenHeight: primaryScreenHeight)
47+
?? Self.userCursorPosition(primaryScreenHeight: primaryScreenHeight)
48+
place(at: position)
49+
panel.orderFrontRegardless()
50+
flush(for: 0.016)
51+
}
52+
53+
/// Glides to a point in global top-left coordinates, the space AX and CGEvent share.
54+
func move(to target: CGPoint, duration: TimeInterval = 0.22) {
55+
let start = position
56+
let steps = max(1, Int(duration / 0.016))
57+
for step in 1...steps {
58+
let t = Double(step) / Double(steps)
59+
let eased = t < 0.5 ? 2 * t * t : 1 - pow(-2 * t + 2, 2) / 2
60+
place(
61+
at: CGPoint(
62+
x: start.x + (target.x - start.x) * eased,
63+
y: start.y + (target.y - start.y) * eased
64+
)
65+
)
66+
flush(for: 0.016)
67+
}
68+
position = target
69+
}
70+
71+
/// A ring at the tip, marking the moment the action was delivered.
72+
func pulse(duration: TimeInterval = 0.18) {
73+
let steps = max(1, Int(duration / 0.016))
74+
for step in 1...steps {
75+
view.ringProgress = CGFloat(step) / CGFloat(steps)
76+
view.display()
77+
flush(for: 0.016)
78+
}
79+
view.ringProgress = nil
80+
view.display()
81+
}
82+
83+
/// Hides the cursor; only a delivered action's point becomes the next action's start.
84+
func finish(delivered: Bool, linger: TimeInterval = 0.25) {
85+
flush(for: linger)
86+
panel.orderOut(nil)
87+
if delivered { persist(position) }
88+
}
89+
90+
private func place(at tip: CGPoint) {
91+
panel.setFrameOrigin(
92+
CGPoint(
93+
x: tip.x - Self.tipOffset.x,
94+
y: primaryScreenHeight - tip.y - Self.size.height + Self.tipOffset.y
95+
)
96+
)
97+
}
98+
99+
private func flush(for interval: TimeInterval) {
100+
CATransaction.flush()
101+
RunLoop.current.run(until: Date(timeIntervalSinceNow: interval))
102+
}
103+
104+
private static func userCursorPosition(primaryScreenHeight: CGFloat) -> CGPoint {
105+
let location = NSEvent.mouseLocation
106+
return CGPoint(x: location.x, y: primaryScreenHeight - location.y)
107+
}
108+
109+
/// The last delivered point, unless it is stale or no display shows it any more.
110+
private static func restoredPosition(primaryScreenHeight: CGFloat) -> CGPoint? {
111+
guard let attributes = try? FileManager.default.attributesOfItem(atPath: positionURL.path),
112+
let modified = attributes[.modificationDate] as? Date,
113+
Date().timeIntervalSince(modified) < staleAfter,
114+
let data = try? Data(contentsOf: positionURL),
115+
let stored = try? JSONDecoder().decode([Double].self, from: data),
116+
stored.count == 2
117+
else {
118+
return nil
119+
}
120+
let point = CGPoint(x: stored[0], y: stored[1])
121+
let appKitPoint = CGPoint(x: point.x, y: primaryScreenHeight - point.y)
122+
return NSScreen.screens.contains { $0.frame.contains(appKitPoint) } ? point : nil
123+
}
124+
125+
private func persist(_ point: CGPoint) {
126+
guard let data = try? JSONEncoder().encode([Double(point.x), Double(point.y)]) else { return }
127+
try? data.write(to: Self.positionURL, options: .atomic)
128+
}
129+
}
130+
131+
private final class GhostCursorView: NSView {
132+
private let tip: CGPoint
133+
var ringProgress: CGFloat?
134+
135+
init(frame: CGRect, tip: CGPoint) {
136+
self.tip = tip
137+
super.init(frame: frame)
138+
}
139+
140+
required init?(coder: NSCoder) {
141+
nil
142+
}
143+
144+
override var isFlipped: Bool { true }
145+
146+
override func draw(_ dirtyRect: NSRect) {
147+
NSColor.clear.setFill()
148+
dirtyRect.fill()
149+
if let progress = ringProgress {
150+
let radius = 6 + 14 * progress
151+
let ring = NSBezierPath(
152+
ovalIn: CGRect(x: tip.x - radius, y: tip.y - radius, width: radius * 2, height: radius * 2)
153+
)
154+
ring.lineWidth = 3
155+
NSColor.systemPurple.withAlphaComponent(0.85 * (1 - progress)).setStroke()
156+
ring.stroke()
157+
}
158+
let arrow = NSBezierPath()
159+
arrow.move(to: tip)
160+
arrow.line(to: CGPoint(x: tip.x, y: tip.y + 22))
161+
arrow.line(to: CGPoint(x: tip.x + 6, y: tip.y + 17))
162+
arrow.line(to: CGPoint(x: tip.x + 10, y: tip.y + 26))
163+
arrow.line(to: CGPoint(x: tip.x + 14, y: tip.y + 24))
164+
arrow.line(to: CGPoint(x: tip.x + 10, y: tip.y + 15))
165+
arrow.line(to: CGPoint(x: tip.x + 17, y: tip.y + 15))
166+
arrow.close()
167+
arrow.lineJoinStyle = .round
168+
let shadow = NSShadow()
169+
shadow.shadowBlurRadius = 3
170+
shadow.shadowOffset = CGSize(width: 0, height: -1)
171+
shadow.shadowColor = NSColor.black.withAlphaComponent(0.35)
172+
NSGraphicsContext.saveGraphicsState()
173+
shadow.set()
174+
NSColor.systemPurple.setFill()
175+
arrow.fill()
176+
NSGraphicsContext.restoreGraphicsState()
177+
arrow.lineWidth = 1.5
178+
NSColor.white.setStroke()
179+
arrow.stroke()
180+
}
181+
}

‎apple/macos-helper/Sources/AgentDeviceMacOSHelper/main.swift‎

Lines changed: 32 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -419,7 +419,9 @@ struct AgentDeviceMacOSHelper {
419419
if surface == "app" {
420420
let app = try requireSessionApplication(bundleId: bundleId)
421421
return SuccessEnvelope(
422-
data: try pressInBackground(request, app: app)
422+
data: try withGhostCursor(arguments) { cursor in
423+
try pressInBackground(request, app: app, cursor: cursor)
424+
}
423425
)
424426
}
425427
try pressAtPosition(request)
@@ -450,7 +452,9 @@ struct AgentDeviceMacOSHelper {
450452
bundleId: try optionValue(arguments: arguments, name: "--bundle-id").map(validatedBundleId)
451453
)
452454
return SuccessEnvelope(
453-
data: try typeInBackground(text: text, delayMs: delayMs, app: app)
455+
data: try withGhostCursor(arguments) { cursor in
456+
try typeInBackground(text: text, delayMs: delayMs, app: app, cursor: cursor)
457+
}
454458
)
455459
}
456460

@@ -467,7 +471,10 @@ struct AgentDeviceMacOSHelper {
467471
bundleId: try optionValue(arguments: arguments, name: "--bundle-id").map(validatedBundleId)
468472
)
469473
return SuccessEnvelope(
470-
data: try fillInBackground(point: CGPoint(x: x, y: y), text: text, app: app)
474+
data: try withGhostCursor(arguments) { cursor in
475+
try fillInBackground(
476+
point: CGPoint(x: x, y: y), text: text, app: app, cursor: cursor)
477+
}
471478
)
472479
}
473480

@@ -484,12 +491,15 @@ struct AgentDeviceMacOSHelper {
484491
bundleId: try optionValue(arguments: arguments, name: "--bundle-id").map(validatedBundleId)
485492
)
486493
return SuccessEnvelope(
487-
data: try scrollInBackground(
488-
direction: direction,
489-
amount: amount,
490-
pixels: pixels,
491-
app: app
492-
)
494+
data: try withGhostCursor(arguments) { cursor in
495+
try scrollInBackground(
496+
direction: direction,
497+
amount: amount,
498+
pixels: pixels,
499+
app: app,
500+
cursor: cursor
501+
)
502+
}
493503
)
494504
}
495505

@@ -542,6 +552,19 @@ private func optionValue(arguments: [String], name: String) -> String? {
542552
return arguments[index + 1]
543553
}
544554

555+
/// Runs a background action under the ghost cursor when the host asked for one.
556+
private func withGhostCursor<T>(_ arguments: [String], _ body: (GhostCursor?) throws -> T) throws -> T {
557+
let cursor = arguments.contains("--ghost-cursor") ? GhostCursor.show() : nil
558+
do {
559+
let result = try body(cursor)
560+
cursor?.finish(delivered: true)
561+
return result
562+
} catch {
563+
cursor?.finish(delivered: false)
564+
throw error
565+
}
566+
}
567+
545568
private func positiveDoubleOption(arguments: [String], name: String) throws -> Double? {
546569
guard let raw = optionValue(arguments: arguments, name: name) else { return nil }
547570
guard let value = Double(raw), value.isFinite, value > 0 else {

‎docs/adr/0031-macos-native-app-backend.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,11 @@ test with wrapper groups that all claim `AXPress`, so a group is pressed only wh
5353
contains the point. This resolves the same point the daemon computed from the snapshot node; the
5454
daemon dispatch paths and their ADR 0011 guarantees are unchanged.
5555

56+
**Ghost cursor.** Each helper action draws its own pointer, glides to the target, and pulses on
57+
delivery so a person can follow the agent without losing the real pointer.
58+
`AGENT_DEVICE_MACOS_GHOST_CURSOR=0` disables it. The helper is one-shot, so the cursor lives for
59+
the action and persists only its last position.
60+
5661
## Rejected alternatives
5762

5863
- **Suppressing Automation Mode.** `automationmodetool` removes the authentication prompt, not the

‎docs/adr/README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@
3232
| [0028 Capability-Family Cell Vocabulary — One Runtime Source (Proposed)](0028-capability-family-cell-vocabulary.md) | adding a capability operation family, `UnavailablePlatformRuntimeFacts` / `UNAVAILABLE_CELLS`, `INTERACTOR_OPERATIONS`, and why an eight-package fan-out recurs |
3333
| [0029 Daemon Policy](0029-daemon-policy.md) | `AGENT_DEVICE_DAEMON_POLICY`, confining a daemon's commands, devices, or device shutdown, and where operator rules are enforced for batch/replay steps |
3434
| [0030 Process Lock Exclusion](0030-process-lock-exclusion.md) | process-lock publication/reclaim/release, retained mutation guards, and the single-protocol upgrade boundary |
35-
| [0031 macOS Native App Backend](0031-macos-native-app-backend.md) | `AGENT_DEVICE_MACOS_APP_BACKEND`, driving macOS app sessions without XCTest Automation Mode, and why pointer actions are accessibility actions only |
35+
| [0031 macOS Native App Backend](0031-macos-native-app-backend.md) | `AGENT_DEVICE_MACOS_APP_BACKEND`, driving macOS app sessions without XCTest Automation Mode, why pointer actions are accessibility actions only, and the ghost cursor |
3636

3737
ADRs record *why*; the registries and gates they describe are the living source of truth — when
3838
prose and a registry disagree, the registry wins and the ADR needs a follow-up.

0 commit comments

Comments
 (0)