Skip to content

Commit 824b743

Browse files
feat(server): count which T3 MCP tools agents use
Records an anonymous mcp.tool.invoked event (tool name, calling driver kind) for every T3 MCP tool call, and an mcp.agent.delegated event for delegate_task, create_threads and t3_thread_launch that relates the calling driver/model to the target driver/model. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
1 parent 37de6cb commit 824b743

7 files changed

Lines changed: 295 additions & 20 deletions

File tree

‎apps/server/src/mcp/McpHttpServer.ts‎

Lines changed: 61 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -11,13 +11,15 @@ import * as Schema from "effect/Schema";
1111
import * as Sink from "effect/Sink";
1212
import * as Stream from "effect/Stream";
1313
import type * as Types from "effect/Types";
14-
import { McpProtocol, McpSchema, McpServer, Tool } from "effect/unstable/ai";
14+
import { McpProtocol, McpSchema, McpServer, Tool, type Toolkit } from "effect/unstable/ai";
1515
import { HttpRouter, HttpServerRequest, HttpServerResponse } from "effect/unstable/http";
1616
import { PreviewAutomationError } from "@t3tools/contracts";
1717

1818
import packageJson from "../../package.json" with { type: "json" };
1919
import * as ServerConfig from "../config.ts";
2020
import * as DeviceService from "../device/DeviceService.ts";
21+
import * as ProviderRegistry from "../provider/Services/ProviderRegistry.ts";
22+
import * as AnalyticsService from "../telemetry/AnalyticsService.ts";
2123
import * as McpInvocationContext from "./McpInvocationContext.ts";
2224
import * as OrchestratorMcpService from "./OrchestratorMcpService.ts";
2325
import { PreviewControlsToolkit } from "./toolkits/previewControls/tools.ts";
@@ -647,61 +649,101 @@ const registerDeviceScreenshot = Effect.fn("McpHttpServer.registerDeviceScreensh
647649
);
648650
});
649651

650-
const PreviewStandardToolkitRegistrationLive = McpServer.toolkit(PreviewStandardToolkit).pipe(
652+
/**
653+
* Runs a tool registration against a server that records one anonymous
654+
* `mcp.tool.invoked` event per call: the tool name and the calling agent's
655+
* driver kind. Arguments, results, and ids are never recorded.
656+
*/
657+
const withToolAnalytics = <E, R>(registration: Effect.Effect<void, E, R>) =>
658+
Effect.gen(function* () {
659+
const server = yield* McpServer.McpServer;
660+
const analytics = yield* Effect.serviceOption(AnalyticsService.AnalyticsService);
661+
if (Option.isNone(analytics)) return yield* registration;
662+
const registry = yield* Effect.serviceOption(ProviderRegistry.ProviderRegistry);
663+
const record = (tool: string) =>
664+
Effect.gen(function* () {
665+
const invocation = yield* Effect.serviceOption(McpInvocationContext.McpInvocationContext);
666+
const providers = Option.isSome(registry) ? yield* registry.value.getProviders : [];
667+
const caller = Option.isSome(invocation)
668+
? providers.find(
669+
(provider) => provider.instanceId === invocation.value.providerInstanceId,
670+
)
671+
: undefined;
672+
yield* analytics.value.record("mcp.tool.invoked", {
673+
tool,
674+
provider: caller?.driver ?? "unknown",
675+
});
676+
}).pipe(Effect.ignoreCause);
677+
const recordingServer = McpServer.McpServer.of({
678+
...server,
679+
addTool: (options) =>
680+
server.addTool({
681+
...options,
682+
handle: (payload) =>
683+
record(options.tool.name).pipe(Effect.andThen(options.handle(payload))),
684+
}),
685+
});
686+
return yield* registration.pipe(Effect.provideService(McpServer.McpServer, recordingServer));
687+
});
688+
689+
const toolkit = <Tools extends Record<string, Tool.Any>>(tools: Toolkit.Toolkit<Tools>) =>
690+
Layer.effectDiscard(withToolAnalytics(McpServer.registerToolkit(tools))).pipe(
691+
Layer.provide(McpServer.McpServer.layer),
692+
);
693+
694+
const PreviewStandardToolkitRegistrationLive = toolkit(PreviewStandardToolkit).pipe(
651695
Layer.provide(PreviewStandardToolkitHandlersLive),
652696
);
653697

654-
const PreviewSnapshotRegistrationLive = Layer.effectDiscard(registerPreviewSnapshot()).pipe(
655-
Layer.provide(PreviewSnapshotToolkitHandlersLive),
656-
);
698+
const PreviewSnapshotRegistrationLive = Layer.effectDiscard(
699+
withToolAnalytics(registerPreviewSnapshot()),
700+
).pipe(Layer.provide(PreviewSnapshotToolkitHandlersLive));
657701

658702
export const PreviewToolkitRegistrationLive = Layer.mergeAll(
659703
PreviewStandardToolkitRegistrationLive,
660704
PreviewSnapshotRegistrationLive,
661705
);
662706

663-
export const OrchestratorToolkitRegistrationLive = McpServer.toolkit(OrchestratorToolkit).pipe(
707+
export const OrchestratorToolkitRegistrationLive = toolkit(OrchestratorToolkit).pipe(
664708
Layer.provide(OrchestratorToolkitHandlersLive),
665709
Layer.provide(OrchestratorMcpService.layer),
666710
Layer.provide(ThreadMetadataMcpService.layer),
667711
);
668712

669-
export const ThreadToolkitRegistrationLive = McpServer.toolkit(ThreadToolkit).pipe(
713+
export const ThreadToolkitRegistrationLive = toolkit(ThreadToolkit).pipe(
670714
Layer.provide(ThreadToolkitHandlersLive),
671715
);
672716

673-
const WorktreeToolkitRegistrationLive = McpServer.toolkit(WorktreeToolkit).pipe(
717+
const WorktreeToolkitRegistrationLive = toolkit(WorktreeToolkit).pipe(
674718
Layer.provide(WorktreeToolkitHandlersLive),
675719
Layer.provide(WorktreeMcpService.layer),
676720
);
677721

678-
const PreviewControlsRegistrationLive = McpServer.toolkit(PreviewControlsToolkit).pipe(
722+
const PreviewControlsRegistrationLive = toolkit(PreviewControlsToolkit).pipe(
679723
Layer.provide(PreviewControlsHandlersLive),
680724
);
681725

682-
const EnvironmentRegistrationLive = McpServer.toolkit(EnvironmentToolkit).pipe(
726+
const EnvironmentRegistrationLive = toolkit(EnvironmentToolkit).pipe(
683727
Layer.provide(EnvironmentHandlersLive),
684728
);
685729

686-
const ProjectRegistrationLive = McpServer.toolkit(ProjectToolkit).pipe(
687-
Layer.provide(ProjectHandlersLive),
688-
);
730+
const ProjectRegistrationLive = toolkit(ProjectToolkit).pipe(Layer.provide(ProjectHandlersLive));
689731

690-
const AttachmentRegistrationLive = McpServer.toolkit(AttachmentToolkit).pipe(
732+
const AttachmentRegistrationLive = toolkit(AttachmentToolkit).pipe(
691733
Layer.provide(AttachmentHandlersLive),
692734
);
693735

694-
export const PullRequestsToolkitRegistrationLive = McpServer.toolkit(PullRequestsToolkit).pipe(
736+
export const PullRequestsToolkitRegistrationLive = toolkit(PullRequestsToolkit).pipe(
695737
Layer.provide(PullRequestsToolkitHandlersLive),
696738
);
697739

698-
const DeviceStandardToolkitRegistrationLive = McpServer.toolkit(DeviceStandardToolkit).pipe(
740+
const DeviceStandardToolkitRegistrationLive = toolkit(DeviceStandardToolkit).pipe(
699741
Layer.provide(DeviceStandardToolkitHandlersLive),
700742
);
701743

702-
const DeviceScreenshotRegistrationLive = Layer.effectDiscard(registerDeviceScreenshot()).pipe(
703-
Layer.provide(DeviceScreenshotToolkitHandlersLive),
704-
);
744+
const DeviceScreenshotRegistrationLive = Layer.effectDiscard(
745+
withToolAnalytics(registerDeviceScreenshot()),
746+
).pipe(Layer.provide(DeviceScreenshotToolkitHandlersLive));
705747

706748
export const DeviceToolkitRegistrationLive = Layer.mergeAll(
707749
DeviceStandardToolkitRegistrationLive,

‎apps/server/src/mcp/OrchestratorMcpService.ts‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,8 @@ import {
6969
} from "../orchestration-v2/SubagentProjection.ts";
7070
import * as ThreadManagementService from "../orchestration-v2/ThreadManagementService.ts";
7171
import * as ProviderRegistry from "../provider/Services/ProviderRegistry.ts";
72+
import * as AnalyticsService from "../telemetry/AnalyticsService.ts";
73+
import { delegationProperties } from "../telemetry/ProviderDimensions.ts";
7274
import * as ScheduledTaskService from "../scheduledTasks/ScheduledTaskService.ts";
7375
import type { McpInvocationScope } from "./McpInvocationContext.ts";
7476

@@ -760,6 +762,14 @@ const make = Effect.gen(function* () {
760762
const providerRegistry = yield* ProviderRegistry.ProviderRegistry;
761763
const providerAdapters = yield* ProviderAdapterRegistry.ProviderAdapterRegistryV2;
762764
const scheduledTasks = yield* ScheduledTaskService.ScheduledTaskService;
765+
const analytics = yield* Effect.serviceOption(AnalyticsService.AnalyticsService);
766+
767+
const recordDelegation = (input: Parameters<typeof delegationProperties>[0]) =>
768+
Option.isNone(analytics)
769+
? Effect.void
770+
: analytics.value
771+
.record("mcp.agent.delegated", delegationProperties(input))
772+
.pipe(Effect.ignoreCause);
763773

764774
const requireCapability = (scope: McpInvocationScope) =>
765775
scope.capabilities.has("orchestration")
@@ -1438,6 +1448,12 @@ const make = Effect.gen(function* () {
14381448
);
14391449
}
14401450
const taskId = taskEvent.event.payload.id;
1451+
yield* recordDelegation({
1452+
tool: "delegate_task",
1453+
providers,
1454+
caller: parent.thread.modelSelection,
1455+
target: target.modelSelection,
1456+
});
14411457

14421458
if (input.mode !== "wait") {
14431459
return yield* readTask(scope, taskId, false, true);
@@ -1705,6 +1721,12 @@ const make = Effect.gen(function* () {
17051721
),
17061722
),
17071723
);
1724+
yield* recordDelegation({
1725+
tool: "create_threads",
1726+
providers,
1727+
caller: parent.thread.modelSelection,
1728+
target: target.modelSelection,
1729+
});
17081730
return {
17091731
threadId,
17101732
runId: run?.id ?? null,

‎apps/server/src/mcp/toolkits/core.test.ts‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,8 @@ import { OrchestratorProjectionError } from "../../orchestration-v2/Orchestrator
1717
import * as ThreadManagement from "../../orchestration-v2/ThreadManagementService.ts";
1818
import * as McpHttpServer from "../McpHttpServer.ts";
1919
import * as McpInvocationContext from "../McpInvocationContext.ts";
20+
import * as ProviderRegistry from "../../provider/Services/ProviderRegistry.ts";
21+
import * as AnalyticsService from "../../telemetry/AnalyticsService.ts";
2022
import { OrchestratorToolkit } from "./orchestrator/tools.ts";
2123
import { PreviewToolkit } from "./preview/tools.ts";
2224
import { PreviewControlsToolkit } from "./previewControls/tools.ts";
@@ -157,6 +159,54 @@ it.effect("returns a bounded public failure without serializing storage causes",
157159
),
158160
);
159161

162+
it.effect("records one anonymous analytics event per tool call", () => {
163+
const recorded: Array<{ event: string; properties: unknown }> = [];
164+
return Effect.gen(function* () {
165+
const server = yield* McpServer.McpServer;
166+
yield* server
167+
.callTool({ name: "t3_thread_organize", arguments: { action: "pin" } })
168+
.pipe(
169+
Effect.provideService(McpInvocationContext.McpInvocationContext, scope),
170+
Effect.provideService(McpSchema.McpServerClient, client),
171+
);
172+
expect(recorded).toEqual([
173+
{
174+
event: "mcp.tool.invoked",
175+
properties: { tool: "t3_thread_organize", provider: "codex" },
176+
},
177+
]);
178+
}).pipe(
179+
Effect.provide(
180+
McpHttpServer.ThreadToolkitRegistrationLive.pipe(
181+
Layer.provideMerge(McpServer.McpServer.layer),
182+
Layer.provide(NodeCrypto.layer),
183+
Layer.provide(
184+
Layer.mock(ThreadManagement.ThreadManagementService)({
185+
getThreadShell: () => Effect.succeed(null),
186+
}),
187+
),
188+
Layer.provide(
189+
Layer.mock(ProviderRegistry.ProviderRegistry)({
190+
getProviders: Effect.succeed([
191+
{ instanceId: ProviderInstanceId.make("codex"), driver: "codex" } as never,
192+
]),
193+
}),
194+
),
195+
Layer.provide(
196+
Layer.succeed(
197+
AnalyticsService.AnalyticsService,
198+
AnalyticsService.AnalyticsService.of({
199+
record: (event, properties) =>
200+
Effect.sync(() => void recorded.push({ event, properties })),
201+
flush: Effect.void,
202+
}),
203+
),
204+
),
205+
),
206+
),
207+
);
208+
});
209+
160210
it("keeps MCP preference output allowlisted and Unicode-bounded", () => {
161211
const settings = {
162212
...DEFAULT_SERVER_SETTINGS,

‎apps/server/src/mcp/toolkits/project/handlers.ts‎

Lines changed: 24 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,20 @@
1-
import { MessageId, ThreadId, OrchestratorMcpFailure, ProjectId } from "@t3tools/contracts";
1+
import {
2+
MessageId,
3+
type ModelSelection,
4+
ThreadId,
5+
OrchestratorMcpFailure,
6+
ProjectId,
7+
} from "@t3tools/contracts";
28
import * as Effect from "effect/Effect";
39
import * as Option from "effect/Option";
410
import * as ThreadMessageIntake from "../../../orchestration-v2/ThreadMessageIntake.ts";
511
import * as Claims from "../../../orchestration-v2/AttachmentClaims.ts";
612
import * as Project from "../../../project/ProjectService.ts";
713
import * as ManagedProjectFolders from "../../../project/ManagedProjectFolders.ts";
814
import * as Repositories from "../../../sourceControl/SourceControlRepositoryService.ts";
15+
import * as ProviderRegistry from "../../../provider/Services/ProviderRegistry.ts";
16+
import * as AnalyticsService from "../../../telemetry/AnalyticsService.ts";
17+
import { delegationProperties } from "../../../telemetry/ProviderDimensions.ts";
918
import { newCommandId, readCaller, readMutationCaller, unavailable } from "../../threadAccess.ts";
1019
import { ProjectToolkit } from "./tools.ts";
1120

@@ -37,6 +46,19 @@ const mutation = Effect.gen(function* () {
3746
});
3847
return yield* Project.ProjectService;
3948
});
49+
/** Records which agent launched a thread on which provider, without ids or prompts. */
50+
const recordLaunch = (caller: ModelSelection, target: ModelSelection) =>
51+
Effect.gen(function* () {
52+
const analytics = yield* Effect.serviceOption(AnalyticsService.AnalyticsService);
53+
const registry = yield* Effect.serviceOption(ProviderRegistry.ProviderRegistry);
54+
if (Option.isNone(analytics) || Option.isNone(registry)) return;
55+
const providers = yield* registry.value.getProviders;
56+
yield* analytics.value.record(
57+
"mcp.agent.delegated",
58+
delegationProperties({ tool: "t3_thread_launch", providers, caller, target }),
59+
);
60+
}).pipe(Effect.ignoreCause);
61+
4062
export const ProjectHandlersLive = ProjectToolkit.toLayer({
4163
t3_thread_launch: (input) =>
4264
Effect.gen(function* () {
@@ -107,6 +129,7 @@ export const ProjectHandlersLive = ProjectToolkit.toLayer({
107129
);
108130
const thread = result.projection.thread;
109131
const run = result.projection.runs.find((run) => run.userMessageId === messageId);
132+
yield* recordLaunch(caller.modelSelection, thread.modelSelection);
110133
return {
111134
threadId: thread.id,
112135
projectId: thread.projectId,
Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
import { ProviderInstanceId, type ServerProvider } from "@t3tools/contracts";
2+
import { expect, it } from "@effect/vitest";
3+
4+
import { delegationProperties } from "./ProviderDimensions.ts";
5+
6+
const provider = (
7+
instanceId: string,
8+
driver: string,
9+
models: ReadonlyArray<{ slug: string; isCustom: boolean }>,
10+
) =>
11+
({
12+
instanceId: ProviderInstanceId.make(instanceId),
13+
driver,
14+
models,
15+
}) as unknown as ServerProvider;
16+
17+
const providers = [
18+
provider("work-codex", "codex", [{ slug: "gpt-5.5", isCustom: false }]),
19+
provider("claudeAgent", "claudeAgent", [
20+
{ slug: "claude-opus-5-5", isCustom: false },
21+
{ slug: "my-private-finetune", isCustom: true },
22+
]),
23+
provider("opencode", "opencode", [{ slug: "ollama/acme-internal", isCustom: false }]),
24+
];
25+
26+
const selection = (instanceId: string, model: string) => ({
27+
instanceId: ProviderInstanceId.make(instanceId),
28+
model,
29+
});
30+
31+
it("relates the delegating and receiving agents by driver and catalog model", () => {
32+
expect(
33+
delegationProperties({
34+
tool: "delegate_task",
35+
providers,
36+
caller: selection("work-codex", "gpt-5.5"),
37+
target: selection("claudeAgent", "claude-opus-5-5"),
38+
}),
39+
).toEqual({
40+
tool: "delegate_task",
41+
callerProvider: "codex",
42+
callerModel: "gpt-5.5",
43+
targetProvider: "claudeAgent",
44+
targetModel: "claude-opus-5-5",
45+
crossProvider: true,
46+
});
47+
});
48+
49+
it("omits custom, user-configured, and unknown models and instance names", () => {
50+
const properties = delegationProperties({
51+
tool: "create_threads",
52+
providers,
53+
caller: selection("claudeAgent", "my-private-finetune"),
54+
target: selection("opencode", "ollama/acme-internal"),
55+
});
56+
expect(properties).toEqual({
57+
tool: "create_threads",
58+
callerProvider: "claudeAgent",
59+
targetProvider: "opencode",
60+
crossProvider: true,
61+
});
62+
expect(
63+
delegationProperties({
64+
tool: "t3_thread_launch",
65+
providers,
66+
caller: selection("removed-instance", "gpt-5.5"),
67+
target: selection("work-codex", "gpt-5.5"),
68+
}),
69+
).toEqual({
70+
tool: "t3_thread_launch",
71+
callerProvider: "unknown",
72+
targetProvider: "codex",
73+
targetModel: "gpt-5.5",
74+
crossProvider: true,
75+
});
76+
});

0 commit comments

Comments
 (0)