Skip to content

Commit ce33ec9

Browse files
feat(cua): show Cua Driver calls as computer use in every provider
Cua tool calls read like Codex's own computer use in every harness: the target app's name in a plain title ("Clicked in Safari"), its icon, and a computer source. The shared MCP presentation recognizes the `cua-driver` server, so Claude, Codex, Cursor, the ACP family, OpenCode 1.x, OpenCode 2, and Pi all reach it. Completed calls teach the thread which app owns each pid, so later calls that address a pid or window name the app. App icons resolve on every host platform: Spotlight on macOS, launcher entries and the hicolor theme on Linux, and the executable's associated icon (found through running processes or the Start Menu) on Windows. Ported from #10899 (Julius), with the title table updated for Cua 0.34. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
1 parent 7418a60 commit ce33ec9

18 files changed

Lines changed: 1699 additions & 206 deletions

File tree

‎apps/server/src/assets/NativeAppIconResolver.ts‎

Lines changed: 17 additions & 180 deletions
Original file line numberDiff line numberDiff line change
@@ -1,32 +1,32 @@
11
import type { ToolActivityNativeAppReference } from "@t3tools/contracts";
22
import * as Cache from "effect/Cache";
3-
import * as Cause from "effect/Cause";
4-
import * as Crypto from "effect/Crypto";
5-
import * as Effect from "effect/Effect";
6-
import * as Clock from "effect/Clock";
3+
import type * as Cause from "effect/Cause";
74
import * as Context from "effect/Context";
85
import * as Duration from "effect/Duration";
6+
import * as Effect from "effect/Effect";
97
import * as Exit from "effect/Exit";
108
import * as FileSystem from "effect/FileSystem";
119
import * as Layer from "effect/Layer";
12-
import * as Option from "effect/Option";
13-
import * as Path from "effect/Path";
14-
import * as PlatformError from "effect/PlatformError";
10+
import type * as PlatformError from "effect/PlatformError";
1511
import * as Semaphore from "effect/Semaphore";
16-
import * as Hex from "effect/encoding/Hex";
17-
import { ChildProcess, ChildProcessSpawner } from "effect/process";
1812
import * as HostProcess from "@t3tools/shared/HostProcess";
1913

20-
import * as ServerConfig from "../config.ts";
14+
import { darwinSource } from "./nativeAppIcons/darwin.ts";
15+
import { linuxSource } from "./nativeAppIcons/linux.ts";
16+
import { win32Source } from "./nativeAppIcons/win32.ts";
17+
import type { NativeAppIconSource } from "./nativeAppIcons/source.ts";
2118

22-
const ICON_SIZE = 64;
23-
const COMMAND_TIMEOUT = "5 seconds";
2419
const RESOLUTION_CACHE_TTL = Duration.hours(1);
2520
const RESOLUTION_CACHE_MAX_ENTRIES = 256;
2621

22+
/** One source per platform; a platform without one resolves nothing. */
23+
const SOURCES: ReadonlyArray<NativeAppIconSource> = [darwinSource, linuxSource, win32Source];
24+
25+
/** Resolves and caches host application icons without exposing host paths to clients. */
2726
export class NativeAppIconResolver extends Context.Service<
2827
NativeAppIconResolver,
2928
{
29+
/** Returns a cached image path for the application, or `null` when no icon is available. */
3030
readonly resolve: (app: ToolActivityNativeAppReference) => Effect.Effect<string | null>;
3131
}
3232
>()("t3/assets/NativeAppIconResolver") {}
@@ -39,188 +39,28 @@ function appFromCacheKey(key: string): ToolActivityNativeAppReference {
3939
return JSON.parse(key) as ToolActivityNativeAppReference;
4040
}
4141

42-
const existingFile = Effect.fn("NativeAppIconResolver.existingFile")(function* (filePath: string) {
43-
const fileSystem = yield* FileSystem.FileSystem;
44-
const info = yield* fileSystem.stat(filePath).pipe(
45-
Effect.asSome,
46-
Effect.catchTags({
47-
PlatformError: (error) =>
48-
error.reason._tag === "NotFound" ? Effect.succeedNone : Effect.fail(error),
49-
}),
50-
);
51-
return Option.isSome(info) && info.value.type === "File" ? filePath : null;
52-
});
53-
54-
const commandOutput = Effect.fn("NativeAppIconResolver.commandOutput")(function* (
55-
command: string,
56-
args: ReadonlyArray<string>,
57-
) {
58-
const spawner = yield* ChildProcessSpawner.ChildProcessSpawner;
59-
return yield* spawner
60-
.string(ChildProcess.make(command, args, { stdin: "ignore", stderr: "ignore" }))
61-
.pipe(Effect.timeout(COMMAND_TIMEOUT));
62-
});
63-
64-
const plistValue = Effect.fn("NativeAppIconResolver.plistValue")(function* (
65-
infoPlistPath: string,
66-
key: string,
67-
) {
68-
return yield* commandOutput("/usr/bin/plutil", [
69-
"-extract",
70-
key,
71-
"raw",
72-
"-o",
73-
"-",
74-
infoPlistPath,
75-
]).pipe(
76-
Effect.map((value) => value.trim()),
77-
Effect.orElseSucceed(() => ""),
78-
);
79-
});
80-
81-
function escapeSpotlightString(value: string): string {
82-
return value.replace(/([\\'*?])/gu, "\\$1");
83-
}
84-
8542
function containsControlCharacter(value: string): boolean {
8643
return [...value].some((character) => {
8744
const codePoint = character.codePointAt(0) ?? 0;
8845
return codePoint <= 31 || codePoint === 127;
8946
});
9047
}
9148

92-
const resolveApplicationPath = Effect.fn("NativeAppIconResolver.resolveApplicationPath")(function* (
93-
app: ToolActivityNativeAppReference,
94-
) {
95-
const path = yield* Path.Path;
96-
const query =
97-
app._tag === "app-id"
98-
? `kMDItemCFBundleIdentifier == '${app.appId}'`
99-
: `kMDItemContentType == 'com.apple.application-bundle' && kMDItemDisplayName == '${escapeSpotlightString(app.displayName)}'`;
100-
const spotlightOutput = yield* commandOutput("/usr/bin/mdfind", [query]);
101-
const candidates = spotlightOutput
102-
.split(/\r?\n/u)
103-
.map((value) => value.trim())
104-
.filter((value) => value.endsWith(".app"));
105-
const matchingCandidates =
106-
app._tag === "app-id"
107-
? candidates
108-
: candidates.filter(
109-
(value) =>
110-
path.basename(value, ".app").toLocaleLowerCase() ===
111-
app.displayName.toLocaleLowerCase(),
112-
);
113-
const rankedCandidates = matchingCandidates.length > 0 ? matchingCandidates : candidates;
114-
let mostRecentlyUsed: { readonly path: string; readonly lastUsed: string } | null = null;
115-
for (const candidate of rankedCandidates) {
116-
const lastUsed = yield* commandOutput("/usr/bin/mdls", [
117-
"-raw",
118-
"-name",
119-
"kMDItemLastUsedDate",
120-
candidate,
121-
]).pipe(
122-
Effect.map((value) => value.trim()),
123-
Effect.orElseSucceed(() => ""),
124-
);
125-
if (!mostRecentlyUsed || lastUsed > mostRecentlyUsed.lastUsed) {
126-
mostRecentlyUsed = { path: candidate, lastUsed };
127-
}
128-
}
129-
return mostRecentlyUsed?.path ?? null;
130-
});
131-
132-
const resolveNativeAppIconUncached = Effect.fn("NativeAppIconResolver.resolveUncached")(function* (
133-
app: ToolActivityNativeAppReference,
134-
) {
135-
const fileSystem = yield* FileSystem.FileSystem;
136-
const path = yield* Path.Path;
137-
const config = yield* ServerConfig.ServerConfig;
138-
const appPath = yield* resolveApplicationPath(app);
139-
if (!appPath) return null;
140-
141-
const canonicalAppPath = yield* fileSystem.realPath(appPath);
142-
const infoPlistPath = path.join(canonicalAppPath, "Contents", "Info.plist");
143-
const resourcesDirectory = path.join(canonicalAppPath, "Contents", "Resources");
144-
const iconName =
145-
(yield* plistValue(infoPlistPath, "CFBundleIconFile")) ||
146-
(yield* plistValue(infoPlistPath, "CFBundleIconName"));
147-
if (iconName && path.basename(iconName) !== iconName) return null;
148-
const iconFileName = iconName ? (path.extname(iconName) ? iconName : `${iconName}.icns`) : null;
149-
const resourceEntries = yield* fileSystem
150-
.readDirectory(resourcesDirectory)
151-
.pipe(Effect.orElseSucceed(() => []));
152-
const sourceIconCandidate =
153-
(iconFileName ? yield* existingFile(path.join(resourcesDirectory, iconFileName)) : null) ??
154-
(yield* existingFile(path.join(resourcesDirectory, "AppIcon.icns"))) ??
155-
(resourceEntries.find((entry) => entry.toLowerCase().endsWith(".icns"))
156-
? yield* existingFile(
157-
path.join(
158-
resourcesDirectory,
159-
resourceEntries.find((entry) => entry.toLowerCase().endsWith(".icns"))!,
160-
),
161-
)
162-
: null);
163-
if (!sourceIconCandidate) return null;
164-
const sourceIconPath = yield* fileSystem.realPath(sourceIconCandidate);
165-
const relativeSource = path.relative(resourcesDirectory, sourceIconPath);
166-
if (
167-
relativeSource === ".." ||
168-
relativeSource.startsWith(`..${path.sep}`) ||
169-
path.isAbsolute(relativeSource)
170-
) {
171-
return null;
172-
}
173-
174-
const appVersion =
175-
(yield* plistValue(infoPlistPath, "CFBundleVersion")) ||
176-
(yield* plistValue(infoPlistPath, "CFBundleShortVersionString"));
177-
const crypto = yield* Crypto.Crypto;
178-
const cacheKey = yield* crypto
179-
.digest(
180-
"SHA-256",
181-
new TextEncoder().encode(`${canonicalAppPath}\0${appVersion}\0${sourceIconPath}`),
182-
)
183-
.pipe(Effect.map(Hex.encode), Effect.orDie);
184-
const cacheDirectory = path.join(config.providerStatusCacheDir, "native-app-icons");
185-
const cachePath = path.join(cacheDirectory, `${cacheKey}.png`);
186-
if (yield* existingFile(cachePath)) return cachePath;
187-
188-
yield* fileSystem.makeDirectory(cacheDirectory, { recursive: true });
189-
const temporaryPath = path.join(
190-
cacheDirectory,
191-
`.${cacheKey}-${process.pid}-${(yield* Clock.currentTimeMillis).toString(36)}-${yield* crypto.randomUUIDv4().pipe(Effect.orDie)}.png`,
192-
);
193-
yield* commandOutput("/usr/bin/sips", [
194-
"-z",
195-
String(ICON_SIZE),
196-
String(ICON_SIZE),
197-
"-s",
198-
"format",
199-
"png",
200-
sourceIconPath,
201-
"--out",
202-
temporaryPath,
203-
]).pipe(
204-
Effect.tap(() => fileSystem.rename(temporaryPath, cachePath)),
205-
Effect.ensuring(
206-
fileSystem.remove(temporaryPath).pipe(Effect.catchTags({ PlatformError: () => Effect.void })),
207-
),
208-
);
209-
return yield* existingFile(cachePath);
210-
});
211-
21249
/** @public Service construction is part of the canonical Effect module API. */
21350
export const make = Effect.gen(function* () {
21451
const fileSystem = yield* FileSystem.FileSystem;
21552
const hostPlatform = yield* HostProcess.Platform;
53+
const source = SOURCES.find((candidate) => candidate.platform === hostPlatform);
21654
const resolutionSemaphore = yield* Semaphore.make(2);
21755
const resolutionCache: Cache.Cache<
21856
string,
21957
string | null,
22058
PlatformError.PlatformError | Cause.TimeoutError
22159
> = yield* Cache.makeWith(
22260
(key: string) =>
223-
resolutionSemaphore.withPermits(1)(resolveNativeAppIconUncached(appFromCacheKey(key))),
61+
source
62+
? resolutionSemaphore.withPermits(1)(source.resolve(appFromCacheKey(key)))
63+
: Effect.succeed(null),
22464
{
22565
capacity: RESOLUTION_CACHE_MAX_ENTRIES,
22666
timeToLive: Exit.match({
@@ -242,10 +82,7 @@ export const make = Effect.gen(function* () {
24282
const resolveAttempt = Effect.fn("NativeAppIconResolver.resolve")(function* (
24383
app: ToolActivityNativeAppReference,
24484
) {
245-
if (
246-
hostPlatform !== "darwin" ||
247-
(app._tag === "display-name" && containsControlCharacter(app.displayName))
248-
) {
85+
if (!source || (app._tag === "display-name" && containsControlCharacter(app.displayName))) {
24986
return null;
25087
}
25188

0 commit comments

Comments
 (0)