Skip to content

Commit d12b801

Browse files
committed
feat(desktop): add opt-in Cua Driver computer use
1 parent 6c58362 commit d12b801

38 files changed

Lines changed: 2630 additions & 11 deletions

‎apps/desktop/package.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
"@t3tools/shared": "workspace:*",
2323
"@t3tools/ssh": "workspace:*",
2424
"@t3tools/tailscale": "workspace:*",
25+
"@trycua/cua-driver": "0.24.0",
2526
"dbus-next": "0.10.2",
2627
"effect": "catalog:",
2728
"electron": "44.1.0",

‎apps/desktop/src/app/DesktopApp.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ import * as DesktopServerExposure from "../backend/DesktopServerExposure.ts";
2727
import * as DesktopAppSettings from "../settings/DesktopAppSettings.ts";
2828
import * as DesktopShellEnvironment from "../shell/DesktopShellEnvironment.ts";
2929
import * as DesktopState from "./DesktopState.ts";
30+
import { DesktopCuaDriver } from "../cua/DesktopCuaDriver.ts";
3031
import * as DesktopRemoteUpdates from "../updates/DesktopRemoteUpdates.ts";
3132
import * as DesktopUpdates from "../updates/DesktopUpdates.ts";
3233
import * as DesktopSnapShot from "../snapShot/DesktopSnapShot.ts";
@@ -243,6 +244,7 @@ const startup = Effect.gen(function* () {
243244
const preReadyElectronOptions = yield* DesktopPreReadyPlatform.DesktopPreReadyElectronOptions;
244245
const safeStorage = yield* ElectronSafeStorage.ElectronSafeStorage;
245246
const updates = yield* DesktopUpdates.DesktopUpdates;
247+
const cuaDriver = yield* DesktopCuaDriver;
246248
const environment = yield* DesktopEnvironment.DesktopEnvironment;
247249

248250
yield* shellEnvironment.installIntoProcess;
@@ -300,6 +302,7 @@ const startup = Effect.gen(function* () {
300302
yield* applicationMenu.configure;
301303
yield* updates.configure;
302304
yield* DesktopRemoteUpdates.listen;
305+
yield* cuaDriver.listen;
303306
yield* linuxUrlHandler.register;
304307
yield* bootstrap.pipe(Effect.catchCause((cause) => fatalStartupCause("bootstrap", cause)));
305308
}).pipe(Effect.withSpan("desktop.startup"));

‎apps/desktop/src/backend/DesktopBackendManager.test.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -163,6 +163,9 @@ function makeTestInstance(input: MakeInstanceInput) {
163163
latest: Effect.succeed(Option.none()),
164164
changes: Stream.empty,
165165
encoded: input.desktopTelemetryStream ?? Stream.empty,
166+
encodedForSource: () => input.desktopTelemetryStream ?? Stream.empty,
167+
cuaRequests: Stream.empty,
168+
publishCuaReport: () => Effect.void,
166169
handleControl: () => Effect.void,
167170
handleControlForSource: (_sourceId, message) =>
168171
(input.desktopTelemetryPublisher?.handleControl ?? (() => Effect.void))(message),

‎apps/desktop/src/backend/DesktopBackendManager.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -906,7 +906,7 @@ export const makeBackendInstance = Effect.fn("makeBackendInstance")(function* (
906906

907907
const program = runBackendProcess({
908908
...config.value,
909-
desktopTelemetryStream: desktopTelemetryPublisher.encoded,
909+
desktopTelemetryStream: desktopTelemetryPublisher.encodedForSource(spec.id),
910910
onDesktopTelemetryControl: (message) =>
911911
desktopTelemetryPublisher.handleControlForSource(spec.id, message),
912912
onStarted: Effect.fn("desktop.backendInstance.onStarted")(function* (pid) {

‎apps/desktop/src/backend/DesktopBackendPool.test.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,9 @@ function makePoolLayer(
7070
latest: Effect.succeed(Option.none()),
7171
changes: Stream.empty,
7272
encoded: Stream.empty,
73+
encodedForSource: () => Stream.empty,
74+
cuaRequests: Stream.empty,
75+
publishCuaReport: () => Effect.void,
7376
handleControl: () => Effect.void,
7477
handleControlForSource: () => Effect.void,
7578
removeControlSource: () => Effect.void,
Lines changed: 288 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,288 @@
1+
import * as NodePath from "@effect/platform-node/NodePath";
2+
import { assert, describe, it } from "@effect/vitest";
3+
import {
4+
DesktopHostTelemetryMessage,
5+
type DesktopCuaDriverReport,
6+
type DesktopCuaDriverRequest,
7+
} from "@t3tools/contracts";
8+
import * as Fiber from "effect/Fiber";
9+
import * as Exit from "effect/Exit";
10+
import * as Scope from "effect/Scope";
11+
import * as TestClock from "effect/testing/TestClock";
12+
import * as Effect from "effect/Effect";
13+
import * as Layer from "effect/Layer";
14+
import * as Queue from "effect/Queue";
15+
import * as Schema from "effect/Schema";
16+
import * as Stream from "effect/Stream";
17+
18+
import * as DesktopConfig from "../app/DesktopConfig.ts";
19+
import * as DesktopEnvironment from "../app/DesktopEnvironment.ts";
20+
import { DesktopTelemetryPublisher } from "../telemetry/DesktopTelemetryPublisher.ts";
21+
import { make, type DesktopCuaDriverDependencies } from "./DesktopCuaDriver.ts";
22+
23+
const encodeMessage = Schema.encodeEffect(Schema.fromJsonString(DesktopHostTelemetryMessage));
24+
const decodeMessage = Schema.decodeEffect(Schema.fromJsonString(DesktopHostTelemetryMessage));
25+
26+
const descriptor = {
27+
command: "/native/driver",
28+
args: ["mcp", "--session", "opaque"],
29+
environment: [{ name: "CUA_SESSION", value: "test-value" }],
30+
};
31+
const environmentLayer = (platform: NodeJS.Platform = "linux") =>
32+
DesktopEnvironment.layer({
33+
dirname: "/app",
34+
homeDirectory: "/home/test",
35+
platform,
36+
processArch: "x64",
37+
appVersion: "1.0.0",
38+
appPath: "/app",
39+
isPackaged: true,
40+
resourcesPath: "/resources",
41+
runningUnderArm64Translation: false,
42+
}).pipe(Layer.provide(Layer.mergeAll(NodePath.layerPosix, DesktopConfig.layerTest({}))));
43+
44+
const fixture = Effect.fn(function* (
45+
failStart = false,
46+
denyPermissions = false,
47+
control: { holdStart?: boolean; holdStop?: boolean } = {},
48+
) {
49+
const requests = yield* Queue.unbounded<DesktopCuaDriverRequest>();
50+
const reports = yield* Queue.unbounded<DesktopCuaDriverReport>();
51+
const exits: Array<() => void> = [];
52+
const starting = yield* Queue.unbounded<void>();
53+
const stopping = yield* Queue.unbounded<void>();
54+
const destroyed = yield* Queue.unbounded<void>();
55+
const startGate = Promise.withResolvers<void>();
56+
const stopGate = Promise.withResolvers<void>();
57+
let starts = 0;
58+
let stops = 0;
59+
let destroys = 0;
60+
const dependencies: DesktopCuaDriverDependencies = {
61+
loadElectron: async () => ({
62+
requestMacOSPermissions: () => ({ accessibility: false, screenRecording: false }),
63+
hasRequiredMacOSPermissions: () => false,
64+
}),
65+
loadEmbedded: async () => ({
66+
EmbeddedCuaDriverHost: class {
67+
async start() {
68+
starts++;
69+
Queue.offerUnsafe(starting, undefined);
70+
if (control.holdStart) await startGate.promise;
71+
if (failStart) throw new Error("private host details");
72+
return {
73+
socketPath: "/socket",
74+
pid: 100,
75+
generation: String(starts),
76+
driverVersion: "0.24",
77+
contractVersion: "1",
78+
mcpProtocolVersion: "1",
79+
mcp: descriptor,
80+
};
81+
}
82+
async stop() {
83+
stops++;
84+
Queue.offerUnsafe(stopping, undefined);
85+
if (control.holdStop) await stopGate.promise;
86+
}
87+
uniffiDestroy() {
88+
destroys++;
89+
Queue.offerUnsafe(destroyed, undefined);
90+
}
91+
waitForExit(generation: string, options?: { signal: AbortSignal }) {
92+
return new Promise<{ generation: string; success: boolean }>((resolve) => {
93+
exits.push(() => resolve({ generation, success: false }));
94+
options?.signal.addEventListener(
95+
"abort",
96+
() => resolve({ generation, success: false }),
97+
{ once: true },
98+
);
99+
});
100+
}
101+
},
102+
}),
103+
};
104+
const publisher = DesktopTelemetryPublisher.of({
105+
latest: Effect.succeedNone,
106+
changes: Stream.empty,
107+
encoded: Stream.empty,
108+
encodedForSource: () => Stream.empty,
109+
handleControl: () => Effect.void,
110+
handleControlForSource: () => Effect.void,
111+
removeControlSource: () => Effect.void,
112+
updateRequests: Stream.empty,
113+
updateCommits: Stream.empty,
114+
updateCancellations: Stream.empty,
115+
publishUpdateReport: () => Effect.void,
116+
cuaRequests: Stream.fromQueue(requests),
117+
publishCuaReport: (report) => Queue.offer(reports, report).pipe(Effect.asVoid),
118+
});
119+
const driver = yield* make(dependencies).pipe(
120+
Effect.provideService(DesktopTelemetryPublisher, publisher),
121+
Effect.provide(environmentLayer(denyPermissions ? "darwin" : "linux")),
122+
);
123+
yield* driver.listen;
124+
return {
125+
request: (requestId: string, enabled: boolean) =>
126+
Queue.offer(requests, { version: 1, type: "cuaDriverRequest", requestId, enabled }),
127+
next: Queue.take(reports),
128+
starting: Queue.take(starting),
129+
stopping: Queue.take(stopping),
130+
destroyed: Queue.take(destroyed),
131+
releaseStart: Effect.sync(() => startGate.resolve()),
132+
releaseStop: Effect.sync(() => stopGate.resolve()),
133+
exits,
134+
counts: () => ({ starts, stops, destroys }),
135+
};
136+
});
137+
138+
describe("DesktopCuaDriver", () => {
139+
it.effect("returns the exact descriptor, reuses the host, and stops it once", () =>
140+
Effect.scoped(
141+
Effect.gen(function* () {
142+
const test = yield* fixture();
143+
yield* test.request("start", true);
144+
const ready = yield* test.next;
145+
const json = yield* encodeMessage(ready);
146+
assert.deepEqual(yield* decodeMessage(json), {
147+
version: 1,
148+
type: "cuaDriverReport",
149+
requestId: "start",
150+
status: "ready",
151+
mcp: descriptor,
152+
});
153+
yield* test.request("same", true);
154+
assert.equal((yield* test.next).requestId, "same");
155+
yield* test.request("stop", false);
156+
assert.equal((yield* test.next).status, "stopped");
157+
yield* test.request("stop-again", false);
158+
yield* test.next;
159+
assert.deepEqual(test.counts(), { starts: 1, stops: 1, destroys: 1 });
160+
}),
161+
),
162+
);
163+
164+
it.effect("reports a crash against the active request and allows a new host", () =>
165+
Effect.scoped(
166+
Effect.gen(function* () {
167+
const test = yield* fixture();
168+
yield* test.request("first", true);
169+
yield* test.next;
170+
yield* test.request("current", true);
171+
yield* test.next;
172+
yield* Effect.sync(() => test.exits[0]!());
173+
const report = yield* test.next;
174+
assert.equal(report.status, "unavailable");
175+
assert.equal(report.requestId, "current");
176+
yield* test.request("restart", true);
177+
assert.equal((yield* test.next).status, "ready");
178+
assert.deepEqual(test.counts(), { starts: 2, stops: 1, destroys: 1 });
179+
}),
180+
),
181+
);
182+
183+
it.effect("cleans up failed starts without exposing native error details", () =>
184+
Effect.scoped(
185+
Effect.gen(function* () {
186+
const test = yield* fixture(true);
187+
yield* test.request("failed", true);
188+
const report = yield* test.next;
189+
assert.equal(report.status, "unavailable");
190+
if (report.status !== "ready")
191+
assert.equal(report.message?.includes("private host details"), false);
192+
assert.deepEqual(test.counts(), { starts: 1, stops: 1, destroys: 1 });
193+
}),
194+
),
195+
);
196+
197+
it.effect("reports missing permissions without constructing a native host", () =>
198+
Effect.scoped(
199+
Effect.gen(function* () {
200+
const test = yield* fixture(false, true);
201+
yield* test.request("permissions", true);
202+
const report = yield* test.next;
203+
assert.equal(report.status, "unavailable");
204+
if (report.status !== "ready") assert.include(report.message ?? "", "Accessibility");
205+
assert.deepEqual(test.counts(), { starts: 0, stops: 0, destroys: 0 });
206+
}),
207+
),
208+
);
209+
210+
it.effect(
211+
"times out a hanging start and cleans up its late result without publishing ready",
212+
() =>
213+
Effect.scoped(
214+
Effect.gen(function* () {
215+
const test = yield* fixture(false, false, { holdStart: true });
216+
yield* test.request("hanging", true);
217+
yield* test.starting;
218+
yield* TestClock.adjust("23 seconds");
219+
assert.equal((yield* test.next).status, "unavailable");
220+
yield* test.request("disable", false);
221+
yield* TestClock.adjust("3 seconds");
222+
assert.equal((yield* test.next).status, "unavailable");
223+
assert.deepEqual(test.counts(), { starts: 1, stops: 0, destroys: 0 });
224+
yield* test.releaseStart;
225+
yield* test.destroyed;
226+
yield* test.request("disabled", false);
227+
const report = yield* test.next;
228+
assert.equal(report.requestId, "disabled");
229+
assert.equal(report.status, "stopped");
230+
assert.deepEqual(test.counts(), { starts: 1, stops: 1, destroys: 1 });
231+
}),
232+
),
233+
);
234+
235+
it.effect("bounds a hanging stop and prevents another native host until cleanup settles", () =>
236+
Effect.scoped(
237+
Effect.gen(function* () {
238+
const test = yield* fixture(false, false, { holdStop: true });
239+
yield* test.request("start", true);
240+
yield* test.next;
241+
yield* test.request("stop", false);
242+
yield* test.stopping;
243+
yield* TestClock.adjust("3 seconds");
244+
assert.equal((yield* test.next).status, "unavailable");
245+
yield* test.request("too-soon", true);
246+
assert.equal((yield* test.next).status, "unavailable");
247+
assert.deepEqual(test.counts(), { starts: 1, stops: 1, destroys: 0 });
248+
yield* test.releaseStop;
249+
yield* test.destroyed;
250+
yield* test.request("restart", true);
251+
assert.equal((yield* test.next).status, "ready");
252+
assert.deepEqual(test.counts(), { starts: 2, stops: 1, destroys: 1 });
253+
}),
254+
),
255+
);
256+
257+
it.effect("closes its scope while start hangs and cleans up after late completion", () =>
258+
Effect.gen(function* () {
259+
const scope = yield* Scope.make();
260+
const test = yield* fixture(false, false, { holdStart: true }).pipe(
261+
Effect.provideService(Scope.Scope, scope),
262+
);
263+
yield* test.request("start", true);
264+
yield* test.starting;
265+
const closing = yield* Scope.close(scope, Exit.void).pipe(Effect.forkChild);
266+
yield* TestClock.adjust("3 seconds");
267+
yield* Fiber.join(closing);
268+
assert.deepEqual(test.counts(), { starts: 1, stops: 0, destroys: 0 });
269+
yield* test.releaseStart;
270+
yield* test.destroyed;
271+
assert.deepEqual(test.counts(), { starts: 1, stops: 1, destroys: 1 });
272+
}),
273+
);
274+
275+
it.effect("cleans up the active host when its scope closes", () =>
276+
Effect.gen(function* () {
277+
const test = yield* Effect.scoped(
278+
Effect.gen(function* () {
279+
const test = yield* fixture();
280+
yield* test.request("start", true);
281+
yield* test.next;
282+
return test;
283+
}),
284+
);
285+
assert.deepEqual(test.counts(), { starts: 1, stops: 1, destroys: 1 });
286+
}),
287+
);
288+
});

0 commit comments

Comments
 (0)