Skip to content

Commit 3f61c22

Browse files
feat(cua): mention installed applications in the composer
1 parent bbb4369 commit 3f61c22

27 files changed

Lines changed: 535 additions & 16 deletions

‎apps/mobile/src/features/threads/ThreadFeed.tsx‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import { Button } from "../../components/ui/Button";
22
import * as Dialog from "../../components/ui/Dialog";
3+
import { parseNativeAppMention } from "@t3tools/shared/nativeAppMentions";
34
import { useOpenPullRequest } from "../../native/useOpenPullRequest";
45
import { useWorkspaceSettledContentWidth } from "../layout/workspace-content-width";
56
import { ThreadMarkdownVideo, ThreadMediaVisibleContext } from "./ThreadMarkdownVideo";
@@ -2288,6 +2289,7 @@ function UserMessageContent(props: UserMessageContentProps) {
22882289
return `[${ref.label}${available ? "" : " (unavailable)"}](t3-context://v1/${ref.kind}/${ref.contextId})`;
22892290
});
22902291
const onLinkPress = (href: string) => {
2292+
if (parseNativeAppMention(href)) return;
22912293
const reference = parseComposerContextHref(href);
22922294
if (!reference) return props.linkHandlers.onLinkPress?.(href);
22932295
const record = props.context?.records.find(

‎apps/server/src/auth/RpcAuthorization.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,7 @@ export const RPC_REQUIRED_SCOPES = {
6262
[WS_METHODS.serverProbe]: AuthOrchestrationReadScope,
6363
[WS_METHODS.serverGetConfig]: AuthOrchestrationReadScope,
6464
[WS_METHODS.serverCheckComputerUse]: AuthOrchestrationReadScope,
65+
[WS_METHODS.serverListApplications]: AuthOrchestrationReadScope,
6566
// Operate: the check reads the host's browser profiles; the action installs a tool.
6667
[WS_METHODS.serverCheckBrowserTabs]: AuthOrchestrationOperateScope,
6768
[WS_METHODS.serverRunBrowserTabsAction]: AuthOrchestrationOperateScope,
Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
import { it } from "@effect/vitest";
2+
import { expect } from "vite-plus/test";
3+
import * as NodeServices from "@effect/platform-node/NodeServices";
4+
import * as Effect from "effect/Effect";
5+
import * as CuaApplications from "./CuaApplications.ts";
6+
import * as CuaDriver from "./CuaDriver.ts";
7+
8+
it.effect(
9+
"lists and deduplicates native identities across platforms, then releases its client",
10+
() =>
11+
Effect.gen(function* () {
12+
let released = 0;
13+
const windows = {
14+
name: "Calculator",
15+
bundleId: "Microsoft.WindowsCalculator_8wekyb3d8bbwe!App",
16+
pid: 42,
17+
running: true,
18+
};
19+
const sdk = {
20+
CuaDriver: {
21+
connect: () => ({
22+
listApps: async () => ({
23+
apps: [
24+
windows,
25+
windows,
26+
{
27+
name: "Helium",
28+
bundleId: "C:\\Apps\\helium.exe",
29+
launchPath: "C:\\Apps\\helium.exe",
30+
pid: 1,
31+
running: true,
32+
},
33+
{
34+
name: "Terminal",
35+
launchPath: "/usr/share/applications/terminal.desktop",
36+
pid: 0,
37+
running: false,
38+
},
39+
{ name: "", pid: 0, running: false },
40+
],
41+
}),
42+
shutdown: async () => {},
43+
uniffiDestroy: () => {
44+
released++;
45+
},
46+
}),
47+
},
48+
ListAppsInput: { new: () => ({}) },
49+
} as unknown as CuaApplications.Sdk;
50+
const service = yield* CuaApplications.make({ loadSdk: Effect.succeed(sdk) });
51+
expect(yield* service.list).toEqual([
52+
{ name: "Calculator", bundleId: windows.bundleId, pid: 42 },
53+
{ name: "Helium", launchPath: "C:\\Apps\\helium.exe", pid: 1 },
54+
{ name: "Terminal", launchPath: "/usr/share/applications/terminal.desktop" },
55+
]);
56+
expect(released).toBe(1);
57+
}).pipe(
58+
Effect.provideService(
59+
CuaDriver.CuaDriver,
60+
CuaDriver.CuaDriver.of({
61+
enabled: Effect.succeed(true),
62+
acquire: Effect.succeedSome({
63+
command: "driver",
64+
args: [],
65+
environment: [],
66+
socketPath: "test",
67+
}),
68+
lastFailure: Effect.succeedNone,
69+
status: Effect.succeed({ platform: "win32", running: true }),
70+
}),
71+
),
72+
Effect.provide(NodeServices.layer),
73+
),
74+
);
75+
76+
it.effect("does not start the driver when Computer Use is disabled", () =>
77+
Effect.gen(function* () {
78+
const service = yield* CuaApplications.make({ loadSdk: Effect.die("must not load") });
79+
expect(yield* service.list).toEqual([]);
80+
}).pipe(
81+
Effect.provideService(
82+
CuaDriver.CuaDriver,
83+
CuaDriver.CuaDriver.of({
84+
enabled: Effect.succeed(false),
85+
acquire: Effect.die("must not start"),
86+
lastFailure: Effect.succeedNone,
87+
status: Effect.succeed({ platform: "win32", running: false }),
88+
}),
89+
),
90+
Effect.provide(NodeServices.layer),
91+
),
92+
);
Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,94 @@
1+
import * as NodeModule from "node:module";
2+
import { CuaApplication, CuaApplicationsError } from "@t3tools/contracts";
3+
import type * as CuaSdk from "@trycua/cua-driver";
4+
import * as Context from "effect/Context";
5+
import * as Effect from "effect/Effect";
6+
import * as FileSystem from "effect/FileSystem";
7+
import * as Layer from "effect/Layer";
8+
import * as Option from "effect/Option";
9+
import * as Path from "effect/Path";
10+
import * as Schema from "effect/Schema";
11+
import * as CuaDriver from "./CuaDriver.ts";
12+
13+
export type Sdk = Pick<typeof CuaSdk, "CuaDriver" | "ListAppsInput">;
14+
type Client = Pick<CuaSdk.CuaDriver, "listApps" | "shutdown" | "uniffiDestroy">;
15+
const isApplication = Schema.is(CuaApplication);
16+
const requireSdk = NodeModule.createRequire(import.meta.url);
17+
const loadSdk = Effect.gen(function* () {
18+
const fs = yield* FileSystem.FileSystem;
19+
const path = yield* Path.Path;
20+
for (const root of requireSdk.resolve.paths("@trycua/cua-driver") ?? []) {
21+
const dir = path.join(root, "@trycua", "cua-driver");
22+
if (yield* fs.exists(path.join(dir, "package.json"))) {
23+
return yield* Effect.try(() => requireSdk(path.join(dir, "dist", "index.js")) as Sdk);
24+
}
25+
}
26+
return yield* new CuaApplicationsError({ cause: "Cua Driver SDK is not installed." });
27+
}).pipe(Effect.mapError((cause) => new CuaApplicationsError({ cause })));
28+
29+
export class CuaApplications extends Context.Service<
30+
CuaApplications,
31+
{
32+
readonly list: Effect.Effect<ReadonlyArray<CuaApplication>, CuaApplicationsError>;
33+
}
34+
>()("t3/cua/CuaApplications") {}
35+
36+
export const make = Effect.fn("CuaApplications.make")(function* (
37+
options: { loadSdk?: Effect.Effect<Sdk, CuaApplicationsError> } = {},
38+
) {
39+
const driver = yield* CuaDriver.CuaDriver;
40+
const fs = yield* FileSystem.FileSystem;
41+
const path = yield* Path.Path;
42+
const sdk =
43+
options.loadSdk ??
44+
loadSdk.pipe(
45+
Effect.provideService(FileSystem.FileSystem, fs),
46+
Effect.provideService(Path.Path, path),
47+
);
48+
const list = Effect.gen(function* () {
49+
if (!(yield* driver.enabled)) return [];
50+
const config = yield* driver.acquire;
51+
if (Option.isNone(config) || !config.value.socketPath) return [];
52+
const module = yield* sdk;
53+
return yield* Effect.scoped(
54+
Effect.gen(function* () {
55+
const client = yield* Effect.acquireRelease(
56+
Effect.try(() => module.CuaDriver.connect(config.value.socketPath!) as unknown as Client),
57+
(client) =>
58+
Effect.promise(() => client.shutdown()).pipe(
59+
Effect.ignore,
60+
Effect.andThen(Effect.sync(() => client.uniffiDestroy())),
61+
),
62+
);
63+
const result = yield* Effect.tryPromise((signal) =>
64+
client.listApps(module.ListAppsInput.new({}), { signal }),
65+
);
66+
const apps: CuaApplication[] = [];
67+
const seen = new Set<string>();
68+
for (const app of result.apps) {
69+
const packagedId = /^shell:appsFolder\\(.+)$/iu.exec(app.launchPath ?? "")?.[1];
70+
const bundleId =
71+
packagedId ??
72+
(app.bundleId && /^[A-Za-z0-9._-]+(?:![A-Za-z0-9._-]+)?$/u.test(app.bundleId)
73+
? app.bundleId
74+
: undefined);
75+
const candidate = {
76+
name: app.name,
77+
...(bundleId ? { bundleId } : {}),
78+
...(app.launchPath ? { launchPath: app.launchPath } : {}),
79+
...(app.running && app.pid > 0 ? { pid: app.pid } : {}),
80+
};
81+
if (!isApplication(candidate)) continue;
82+
const key = bundleId ?? app.launchPath ?? app.name;
83+
if (seen.has(key)) continue;
84+
seen.add(key);
85+
apps.push(candidate);
86+
}
87+
return apps.sort((a, b) => a.name.localeCompare(b.name));
88+
}),
89+
);
90+
}).pipe(Effect.mapError((cause) => new CuaApplicationsError({ cause })));
91+
return CuaApplications.of({ list });
92+
});
93+
94+
export const layer = Layer.effect(CuaApplications, make());

‎apps/server/src/observability/RpcInstrumentation.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ const RPC_AGGREGATES = {
3333
[WS_METHODS.projectsMutate]: "orchestration",
3434
[WS_METHODS.serverProbe]: "server",
3535
[WS_METHODS.serverCheckComputerUse]: "server",
36+
[WS_METHODS.serverListApplications]: "server",
3637
[WS_METHODS.serverCheckBrowserTabs]: "server",
3738
[WS_METHODS.serverRunBrowserTabsAction]: "server",
3839
[WS_METHODS.serverGetConfig]: "server",

‎apps/server/src/server.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -174,6 +174,7 @@ import * as TraceDiagnostics from "./diagnostics/TraceDiagnostics.ts";
174174
import * as TriageService from "./triage/TriageService.ts";
175175
import * as DesktopTelemetryReceiver from "./resourceTelemetry/DesktopTelemetryReceiver.ts";
176176
import * as CuaDriver from "./cua/CuaDriver.ts";
177+
import * as CuaApplications from "./cua/CuaApplications.ts";
177178
import * as BrowserTabs from "./mcp/BrowserTabs.ts";
178179
import * as CuaWindowPreview from "./cua/CuaWindowPreview.ts";
179180
import * as NativeTelemetryClient from "./resourceTelemetry/NativeTelemetryClient.ts";
@@ -593,6 +594,7 @@ const layerProviderInstallationRefresh = Layer.effectDiscard(
593594
const layerRuntimeCoreDependenciesBase = Layer.mergeAll(
594595
AgentAwarenessRelay.layer,
595596
CuaWindowPreview.layer,
597+
CuaApplications.layer,
596598
HeldHooksWaker.layer,
597599
layerThreadSettlementWorker,
598600
layerIdleCompactionWorker,

‎apps/server/src/ws.ts‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -188,6 +188,7 @@ import { terminalOutputStream, withTerminalOutputWindow } from "./terminal/Outpu
188188
import * as ServerBrowser from "./preview/ServerBrowser.ts";
189189
import * as DeviceService from "./device/DeviceService.ts";
190190
import * as CuaDriver from "./cua/CuaDriver.ts";
191+
import * as CuaApplications from "./cua/CuaApplications.ts";
191192
import * as BrowserTabs from "./mcp/BrowserTabs.ts";
192193
import * as CuaWindowPreview from "./cua/CuaWindowPreview.ts";
193194
import { remoteSshDeviceHosts } from "./device/localSshDeviceHost.ts";
@@ -2187,6 +2188,10 @@ export const layerWsRpc = (
21872188
: Effect.succeed({ platform: "other", running: false } as const),
21882189
),
21892190
),
2191+
[WS_METHODS.serverListApplications]: () =>
2192+
Effect.flatMap(Effect.serviceOption(CuaApplications.CuaApplications), (service) =>
2193+
Option.isSome(service) ? service.value.list : Effect.succeed([]),
2194+
),
21902195
[WS_METHODS.serverCheckBrowserTabs]: (_input) =>
21912196
Effect.serviceOption(BrowserTabs.BrowserTabs).pipe(
21922197
Effect.flatMap((tabs) =>

‎apps/web/src/components/ChatMarkdown.tsx‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,8 @@ import type { Pluggable, PluggableList } from "unified";
9292
import { defaultUrlTransform } from "react-markdown";
9393
import { parseAssistantCitationHref } from "@t3tools/shared/assistantCitations";
9494
import { parseComposerContextHref } from "@t3tools/shared/composerContextReferences";
95+
import { parseNativeAppMention } from "@t3tools/shared/nativeAppMentions";
96+
import { NativeAppChip } from "./NativeAppChip";
9597
import { parseThreadLinkHref } from "@t3tools/shared/threadLinks";
9698
import { AssistantCitationChip } from "./chat/AssistantCitationChip";
9799
import { MarkdownThreadLink } from "./chat/MarkdownThreadLink";
@@ -400,7 +402,7 @@ const CHAT_MARKDOWN_LITERAL_HTML_REHYPE_PLUGINS = [
400402

401403
function markdownUrlTransform(href: string) {
402404
if (parseAssistantCitationHref(href)) return href;
403-
if (parseComposerContextHref(href)) return href;
405+
if (parseComposerContextHref(href) || parseNativeAppMention(href)) return href;
404406
if (parseThreadLinkHref(href)) return href;
405407
if (isWindowsDrivePathHref(href)) return href;
406408
return rewriteMarkdownFileUriHref(href) ?? defaultUrlTransform(href);
@@ -450,7 +452,7 @@ function fileLinkSourcesKey(
450452
imageBaseDir: string | undefined,
451453
): string {
452454
const hrefs = extractMarkdownLinkHrefs(renderCodexInlineDirectivesAsMarkdown(text))
453-
.filter((href) => !parseComposerContextHref(href))
455+
.filter((href) => !parseComposerContextHref(href) && !parseNativeAppMention(href))
454456
.map(normalizeMarkdownLinkHrefKey)
455457
.filter((href) => resolveMarkdownFileLinkMeta(href, cwd, imageBaseDir ?? cwd));
456458
const spans = extractInlineCodeSpans(text).filter((span) =>
@@ -1078,6 +1080,8 @@ const CHAT_MARKDOWN_COMPONENTS = {
10781080
} = use(ChatMarkdownRendererContext);
10791081
const text = use(ChatMarkdownTextContext);
10801082
const citation = href ? parseAssistantCitationHref(href) : null;
1083+
const application = href ? parseNativeAppMention(href) : null;
1084+
if (application) return <NativeAppChip {...application} />;
10811085
if (citation) return <AssistantCitationChip citation={citation} />;
10821086
const threadLink = href ? parseThreadLinkHref(href) : null;
10831087
if (threadLink) {

‎apps/web/src/components/ComposerPromptEditorTiptap.tsx‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
import { documentChipKindForFileName } from "~/pierre-icons";
22
import { isProviderSkillUserInvocable } from "@t3tools/client-runtime/providerSkills";
33
import { formatProviderSkillDisplayName } from "@t3tools/shared/inlineSkills";
4+
import { parseNativeAppMention } from "@t3tools/shared/nativeAppMentions";
5+
import { NativeAppChip } from "./NativeAppChip";
46
import {
57
Extension,
68
findChildren,
@@ -510,6 +512,35 @@ const ComposerContextReferenceExtension = Node.create({
510512
},
511513
});
512514

515+
const ComposerApplicationExtension = Node.create({
516+
name: "composer-application",
517+
group: "inline",
518+
inline: true,
519+
atom: true,
520+
selectable: true,
521+
addAttributes() {
522+
return { href: { default: "" }, source: { default: "" } };
523+
},
524+
parseHTML() {
525+
return [{ tag: "span[data-composer-application]" }];
526+
},
527+
renderHTML({ HTMLAttributes }) {
528+
return ["span", { "data-composer-application": "", ...HTMLAttributes }];
529+
},
530+
addNodeView() {
531+
return ReactNodeViewRenderer(ComposerApplicationNodeView);
532+
},
533+
});
534+
535+
function ComposerApplicationNodeView({ node }: NodeViewProps) {
536+
const mention = parseNativeAppMention(String(node.attrs.href));
537+
return (
538+
<NodeViewWrapper as="span" className={CHIP_NODE_SELECTION_CLASS_NAME}>
539+
{mention ? <NativeAppChip {...mention} /> : String(node.attrs.source)}
540+
</NodeViewWrapper>
541+
);
542+
}
543+
513544
function ComposerContextReferenceNodeView({ node }: NodeViewProps) {
514545
return (
515546
<NodeViewWrapper as="span" className={CHIP_NODE_SELECTION_CLASS_NAME}>
@@ -836,6 +867,7 @@ function createComposerExtensions(richText: boolean) {
836867
ComposerUndoGroupingExtension,
837868
ComposerTextDirectionExtension,
838869
ComposerMentionExtension,
870+
ComposerApplicationExtension,
839871
ComposerSkillExtension,
840872
ComposerCitationExtension,
841873
ComposerContextReferenceExtension,
Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
import { EnvironmentId, type CuaApplication } from "@t3tools/contracts";
2+
import { AppWindowIcon } from "lucide-react";
3+
import { useState } from "react";
4+
import { useAssetUrlState } from "~/assets/assetUrls";
5+
import { ContextChip, ContextChipLabel } from "./ContextChip";
6+
7+
export function NativeAppIcon({
8+
app,
9+
environmentId,
10+
}: {
11+
app: CuaApplication;
12+
environmentId: string;
13+
}) {
14+
const reference =
15+
app.bundleId && /^[A-Za-z0-9._-]+(?:![A-Za-z0-9._-]+)?$/u.test(app.bundleId)
16+
? { _tag: "app-id" as const, appId: app.bundleId }
17+
: { _tag: "display-name" as const, displayName: app.name };
18+
const icon = useAssetUrlState(EnvironmentId.make(environmentId), {
19+
_tag: "native-app-icon",
20+
app: reference,
21+
});
22+
const [loadedUrl, setLoadedUrl] = useState<string | null>(null);
23+
const [failedUrl, setFailedUrl] = useState<string | null>(null);
24+
const url = icon._tag === "Success" ? icon.url : null;
25+
const loaded = url !== null && loadedUrl === url && failedUrl !== url;
26+
return (
27+
<span
28+
aria-hidden="true"
29+
className="relative inline-flex size-4 shrink-0 items-center justify-center"
30+
>
31+
{!loaded ? <AppWindowIcon className="size-4" /> : null}
32+
{url !== null && failedUrl !== url ? (
33+
<img
34+
src={url}
35+
alt=""
36+
hidden={!loaded}
37+
className="absolute inset-0 size-full object-contain"
38+
onLoad={() => setLoadedUrl(url)}
39+
onError={() => setFailedUrl(url)}
40+
/>
41+
) : null}
42+
</span>
43+
);
44+
}
45+
46+
export function NativeAppChip(props: { app: CuaApplication; environmentId: string }) {
47+
return (
48+
<ContextChip kind="mention" title={`Application: ${props.app.name}`}>
49+
<NativeAppIcon {...props} />
50+
<ContextChipLabel>{props.app.name}</ContextChipLabel>
51+
</ContextChip>
52+
);
53+
}

0 commit comments

Comments
 (0)