Skip to content

Commit a30d041

Browse files
feat(server): count which T3 MCP tools agents use
Records anonymous mcp.tool.invoked events from one wrapper around MCP tool registration and mcp.delegated_task.finished events from a subagent worker. Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent 9077e81 commit a30d041

8 files changed

Lines changed: 957 additions & 3 deletions

File tree

‎apps/server/src/mcp/McpHttpServer.ts‎

Lines changed: 202 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -13,14 +13,28 @@ import * as Stream from "effect/Stream";
1313
import type * as Types from "effect/Types";
1414
import { AiError, McpProtocol, McpSchema, McpServer, Tool, Toolkit } from "effect/ai";
1515
import { HttpRouter, HttpServerRequest, HttpServerResponse } from "effect/http";
16-
import { OrchestratorMcpFailure, PreviewAutomationError } from "@t3tools/contracts";
16+
import {
17+
OrchestratorMcpFailure,
18+
PreviewAutomationError,
19+
type RunId,
20+
ThreadId,
21+
} from "@t3tools/contracts";
1722

1823
import packageJson from "../../package.json" with { type: "json" };
1924
import * as ServerConfig from "../config.ts";
2025
import * as DeviceService from "../device/DeviceService.ts";
2126
import * as HtmlRender from "../htmlRender/HtmlRender.ts";
2227
import * as ThreadCommandExecutor from "../orchestration-v2/ThreadCommandExecutor.ts";
2328
import * as ThreadManagementService from "../orchestration-v2/ThreadManagementService.ts";
29+
import * as ProviderRegistry from "../provider/ProviderRegistry.ts";
30+
import * as AnalyticsService from "../telemetry/AnalyticsService.ts";
31+
import {
32+
type AgentThread,
33+
agentToolProperties,
34+
handoffSettings,
35+
MAX_REPORTED_DEPTH,
36+
toolOutcome,
37+
} from "../telemetry/ProviderDimensions.ts";
2438
import * as McpInvocationContext from "./McpInvocationContext.ts";
2539
import * as McpToolAccess from "./McpToolAccess.ts";
2640
import * as OrchestratorMcpService from "./OrchestratorMcpService.ts";
@@ -746,6 +760,186 @@ const registerHtmlPreview = Effect.fn("McpHttpServer.registerHtmlPreview")(funct
746760
);
747761
});
748762

763+
/**
764+
* Tools that hand work to another thread's agent: they create it, message it,
765+
* steer it, or move context into it. Their events name the receiving agent too.
766+
*/
767+
const HANDOFF_TOOLS: ReadonlySet<string> = new Set([
768+
"delegate_task",
769+
"create_threads",
770+
"t3_thread_launch",
771+
"t3_thread_send",
772+
"t3_thread_send_attachments",
773+
"t3_thread_fork",
774+
"t3_thread_merge_back",
775+
"t3_queue_edit",
776+
"t3_queue_promote_to_steer",
777+
"t3_pending_request_respond",
778+
"schedule_task",
779+
"run_scheduled_task_now",
780+
]);
781+
782+
/** Handoff tools whose result is only `{ sequence }`; their input names the receiving thread. */
783+
const INPUT_TARGET_TOOLS: ReadonlySet<string> = new Set([
784+
"t3_queue_edit",
785+
"t3_queue_promote_to_steer",
786+
"t3_pending_request_respond",
787+
]);
788+
789+
const resultThreadKeys = ["childThreadId", "targetThreadId", "boundThreadId", "threadId"] as const;
790+
791+
/** Thread ids a handoff tool result names: the thread that received the work. */
792+
const resultThreadIds = (content: unknown): ReadonlyArray<string> => {
793+
if (typeof content !== "object" || content === null) return [];
794+
const record = content as Readonly<Record<string, unknown>>;
795+
if (Array.isArray(record.threads)) return record.threads.flatMap(resultThreadIds);
796+
const key = resultThreadKeys.find((candidate) => typeof record[candidate] === "string");
797+
return key === undefined ? [] : [record[key] as string];
798+
};
799+
800+
/**
801+
* Thread ids that received a handoff. The input's `threadId` is used only after
802+
* the call succeeded, so it is one the handler already resolved and authorized.
803+
*/
804+
const handoffTargetIds = (
805+
tool: string,
806+
args: unknown,
807+
result: McpSchema.CallToolResult | undefined,
808+
): ReadonlyArray<string> => {
809+
if (!INPUT_TARGET_TOOLS.has(tool)) return resultThreadIds(result?.structuredContent);
810+
if (result === undefined || toolOutcome(result).outcome !== "ok") return [];
811+
const threadId = (args as { readonly threadId?: unknown } | null | undefined)?.threadId;
812+
return typeof threadId === "string" ? [threadId] : [];
813+
};
814+
815+
/**
816+
* Runs a tool registration against a server that records one anonymous
817+
* `mcp.tool.invoked` event per call: the tool, the calling agent's provider,
818+
* the outcome with its failure code, and the duration. A handoff tool also
819+
* reports the caller's model, origin, and delegation depth, the settings the
820+
* agent chose, and the provider, model, and modes of each thread that received
821+
* the work. Ids, prompts, and free text are never recorded.
822+
*/
823+
const withToolAnalytics = <A, E, R>(registration: Effect.Effect<A, E, R>) =>
824+
Effect.gen(function* () {
825+
const server = yield* McpServer.McpServer;
826+
const analytics = yield* Effect.serviceOption(AnalyticsService.AnalyticsService);
827+
if (Option.isNone(analytics)) return yield* registration;
828+
const registry = yield* Effect.serviceOption(ProviderRegistry.ProviderRegistry);
829+
const threads = yield* Effect.serviceOption(ThreadManagementService.ThreadManagementService);
830+
const shellOf = (threadId: string) =>
831+
Option.isNone(threads)
832+
? Effect.succeed(null)
833+
: threads.value
834+
.getThreadShell(ThreadId.make(threadId))
835+
.pipe(Effect.orElseSucceed(() => null));
836+
// Delegation depth: parents walked from the caller, capped.
837+
const depthOf = (thread: AgentThread) =>
838+
Effect.gen(function* () {
839+
let depth = 0;
840+
let parentId =
841+
thread.lineage.relationshipToParent === "subagent" ? thread.lineage.parentThreadId : null;
842+
while (parentId !== null && depth < MAX_REPORTED_DEPTH) {
843+
depth += 1;
844+
const parent = yield* shellOf(parentId);
845+
parentId =
846+
parent?.lineage.relationshipToParent === "subagent"
847+
? parent.lineage.parentThreadId
848+
: null;
849+
}
850+
return depth;
851+
});
852+
// Whether the caller's active run was started by a scheduled task.
853+
const scheduledRunOf = (threadId: string, runId: RunId | null) =>
854+
Option.isNone(threads) || runId === null
855+
? Effect.succeed(false)
856+
: threads.value
857+
.getThreadRecords(ThreadId.make(threadId), ["runs", "messages"], {
858+
runIds: [runId],
859+
messageRoles: ["user"],
860+
})
861+
.pipe(
862+
Effect.map((records) => {
863+
const run = records.runs.find((candidate) => candidate.id === runId);
864+
return records.messages.some(
865+
(message) =>
866+
message.id === run?.userMessageId && message.scheduledTaskId !== undefined,
867+
);
868+
}),
869+
Effect.orElseSucceed(() => false),
870+
);
871+
const record = (
872+
tool: string,
873+
args: unknown,
874+
result: McpSchema.CallToolResult | undefined,
875+
durationMs: number,
876+
) =>
877+
Effect.gen(function* () {
878+
// Client credentials (no calling thread) report an unknown caller.
879+
const invocation = Option.getOrUndefined(
880+
yield* Effect.serviceOption(McpInvocationContext.McpInvocationContext),
881+
)?.thread;
882+
const providers = Option.isSome(registry) ? yield* registry.value.getProviders : [];
883+
// Only handoff tools pay for thread lookups; the rest report the
884+
// caller's provider from the credential.
885+
const handoff = HANDOFF_TOOLS.has(tool);
886+
const caller =
887+
handoff && invocation !== undefined
888+
? ((yield* shellOf(invocation.threadId)) ?? undefined)
889+
: undefined;
890+
const targetIds = handoff
891+
? [...new Set(handoffTargetIds(tool, args, result))].filter(
892+
(threadId) => threadId !== invocation?.threadId,
893+
)
894+
: [];
895+
const targets = yield* Effect.forEach(targetIds, shellOf);
896+
for (const properties of agentToolProperties({
897+
tool,
898+
providers,
899+
caller,
900+
...(invocation === undefined
901+
? {}
902+
: { callerProviderInstanceId: invocation.providerInstanceId }),
903+
...(caller === undefined || invocation === undefined
904+
? {}
905+
: {
906+
callerDepth: yield* depthOf(caller),
907+
callerScheduledRun: yield* scheduledRunOf(invocation.threadId, caller.activeRunId),
908+
}),
909+
targets: targets.filter((thread) => thread !== null),
910+
outcome: toolOutcome(result),
911+
...(handoff ? { settings: handoffSettings(tool, args, result?.structuredContent) } : {}),
912+
durationMs,
913+
})) {
914+
yield* analytics.value.record("mcp.tool.invoked", properties);
915+
}
916+
}).pipe(Effect.ignoreCause);
917+
const recordingServer = McpServer.McpServer.of({
918+
...server,
919+
addTool: (options) =>
920+
server.addTool({
921+
...options,
922+
handle: (payload) =>
923+
Effect.gen(function* () {
924+
const startedAt = yield* Clock.currentTimeMillis;
925+
return yield* options.handle(payload).pipe(
926+
Effect.onExit((exit) => {
927+
const result = exit._tag === "Success" ? exit.value : undefined;
928+
// A call that asks for input is retried with it, and that call is the one counted.
929+
if (result instanceof McpSchema.InputRequired) return Effect.void;
930+
return Clock.currentTimeMillis.pipe(
931+
Effect.flatMap((endedAt) =>
932+
record(options.tool.name, payload, result, endedAt - startedAt),
933+
),
934+
);
935+
}),
936+
);
937+
}),
938+
}),
939+
});
940+
return yield* registration.pipe(Effect.provideService(McpServer.McpServer, recordingServer));
941+
});
942+
749943
/**
750944
* `McpServer.toolkit` asks for every service its tools declare when it
751945
* registers them, but the auth middleware provides `McpInvocationContext` to
@@ -756,7 +950,9 @@ export const toolkitRegistration = <Tools extends Record<string, Tool.Any>, EX,
756950
toolkit: Toolkit.Toolkit<Tools>,
757951
handlers: McpToolAccess.HandlersLayer<Tools, EX, RX>,
758952
) => {
759-
const registration = McpServer.toolkit(decodeAsAdvertised(toolkit));
953+
const registration = Layer.effectDiscard(
954+
withToolAnalytics(McpServer.registerToolkit(decodeAsAdvertised(toolkit))),
955+
).pipe(Layer.provide(McpServer.McpServer.layer));
760956
// @effect-diagnostics-next-line unsafeEffectTypeAssertion:off - the auth middleware provides it per request.
761957
const registered = registration as Layer.Layer<
762958
never,
@@ -769,7 +965,10 @@ export const toolkitRegistration = <Tools extends Record<string, Tool.Any>, EX,
769965
const imageToolRegistration = <Tools extends Record<string, Tool.Any>, A, E, R, EX, RX>(
770966
register: Effect.Effect<A, E, R>,
771967
handlers: McpToolAccess.HandlersLayer<Tools, EX, RX>,
772-
) => Layer.effectDiscard(register).pipe(Layer.provide(McpToolAccess.HandlersLayer.layer(handlers)));
968+
) =>
969+
Layer.effectDiscard(withToolAnalytics(register)).pipe(
970+
Layer.provide(McpToolAccess.HandlersLayer.layer(handlers)),
971+
);
773972

774973
export const layerHtmlToolkit = Layer.mergeAll(
775974
toolkitRegistration(HtmlRenderToolkit, HtmlHandlers.layerRender),

‎apps/server/src/mcp/toolkits/core.test.ts‎

Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@ import * as McpHttpServer from "../McpHttpServer.ts";
4949
import * as McpInvocationContext from "../McpInvocationContext.ts";
5050
import * as McpToolAccessTestkit from "../McpToolAccess.testkit.ts";
5151
import { dispatchFailure } from "../threadAccess.ts";
52+
import * as AnalyticsService from "../../telemetry/AnalyticsService.ts";
5253
import { OrchestratorToolkit } from "./orchestrator/tools.ts";
5354
import { PreviewToolkit } from "./preview/tools.ts";
5455
import { PreviewControlsToolkit } from "./previewControls/tools.ts";
@@ -362,6 +363,123 @@ it.effect("keeps unexpected handler defects private through the production regis
362363
),
363364
);
364365

366+
it.effect("records the calling agent and the agent a tool acted on", () => {
367+
const recorded: Array<{ event: string; properties: unknown }> = [];
368+
const shell = (id: ThreadId, instanceId: string, model: string) =>
369+
({
370+
id,
371+
projectId: "mcp-core-project",
372+
providerInstanceId: ProviderInstanceId.make(instanceId),
373+
modelSelection: { instanceId: ProviderInstanceId.make(instanceId), model },
374+
runtimeMode: "full-access",
375+
interactionMode: "default",
376+
archivedAt: null,
377+
deletedAt: null,
378+
activeRunId: "mcp-core-run",
379+
createdBy: "agent",
380+
creationSource: "mcp",
381+
lineage: {
382+
parentThreadId: id === threadId ? ThreadId.make("mcp-core-parent") : null,
383+
relationshipToParent: id === threadId ? "subagent" : null,
384+
rootThreadId: ThreadId.make("mcp-core-parent"),
385+
},
386+
}) as never;
387+
return Effect.gen(function* () {
388+
const server = yield* McpServer.McpServer;
389+
yield* server
390+
.callTool({ name: "t3_thread_fork", arguments: { sourcePoint: { type: "latest_stable" } } })
391+
.pipe(
392+
Effect.provideService(McpInvocationContext.McpInvocationContext, scope),
393+
Effect.provideService(McpSchema.McpServerClient, client),
394+
);
395+
expect(recorded).toHaveLength(1);
396+
const [{ event, properties }] = recorded as [{ event: string; properties: object }];
397+
expect(event).toBe("mcp.tool.invoked");
398+
expect(properties).toMatchObject({
399+
tool: "t3_thread_fork",
400+
outcome: "ok",
401+
callerProvider: "codex",
402+
callerModel: "gpt-5.5",
403+
callerOrigin: "agent",
404+
callerDepth: 1,
405+
targetProvider: "claudeAgent",
406+
targetModel: "claude-opus-5-5",
407+
targetRuntimeMode: "full-access",
408+
targetInteractionMode: "default",
409+
crossProvider: true,
410+
});
411+
expect(properties).toHaveProperty("durationMs");
412+
expect(Object.values(properties).some((value) => String(value).includes("mcp-core"))).toBe(
413+
false,
414+
);
415+
416+
// Queue tools return only a sequence, so the receiving thread comes from the input.
417+
yield* server
418+
.callTool({
419+
name: "t3_queue_edit",
420+
arguments: { threadId: "mcp-core-other", queuedRunId: "mcp-core-queued", text: "secret" },
421+
})
422+
.pipe(
423+
Effect.provideService(McpInvocationContext.McpInvocationContext, scope),
424+
Effect.provideService(McpSchema.McpServerClient, client),
425+
);
426+
expect(recorded[1]?.properties).toMatchObject({
427+
tool: "t3_queue_edit",
428+
outcome: "ok",
429+
callerProvider: "codex",
430+
targetProvider: "claudeAgent",
431+
crossProvider: true,
432+
});
433+
}).pipe(
434+
Effect.provide(
435+
layerThreadToolkit.pipe(
436+
Layer.provideMerge(McpServer.McpServer.layer),
437+
Layer.provide(NodeCrypto.layer),
438+
Layer.provide(
439+
Layer.mock(ThreadManagement.ThreadManagementService)({
440+
getThreadShell: (id) =>
441+
Effect.succeed(
442+
id === threadId
443+
? shell(threadId, "codex", "gpt-5.5")
444+
: shell(id, "claudeAgent", "claude-opus-5-5"),
445+
),
446+
getThreadRecords: () => Effect.succeed({ runs: [], messages: [] } as never),
447+
getProjectThreadRecords: () =>
448+
Effect.succeed({ thread: shell(threadId, "codex", "gpt-5.5") } as never),
449+
dispatch: () => Effect.succeed({ sequence: 1 } as never),
450+
}),
451+
),
452+
Layer.provide(
453+
Layer.mock(ProviderRegistry.ProviderRegistry)({
454+
getProviders: Effect.succeed([
455+
{
456+
instanceId: ProviderInstanceId.make("codex"),
457+
driver: "codex",
458+
models: [{ slug: "gpt-5.5", isCustom: false }],
459+
} as never,
460+
{
461+
instanceId: ProviderInstanceId.make("claudeAgent"),
462+
driver: "claudeAgent",
463+
models: [{ slug: "claude-opus-5-5", isCustom: false }],
464+
} as never,
465+
]),
466+
}),
467+
),
468+
Layer.provide(
469+
Layer.succeed(
470+
AnalyticsService.AnalyticsService,
471+
AnalyticsService.AnalyticsService.of({
472+
record: (event, properties) =>
473+
Effect.sync(() => void recorded.push({ event, properties })),
474+
flush: Effect.void,
475+
}),
476+
),
477+
),
478+
),
479+
),
480+
);
481+
});
482+
365483
it("keeps MCP preference output allowlisted and Unicode-bounded", () => {
366484
const settings = {
367485
...DEFAULT_SERVER_SETTINGS,

‎apps/server/src/server.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@ import * as PullRequestReviewGuides from "./persistence/PullRequestReviewGuides.
4545
import * as PullRequestReviewGuideService from "./pullRequest/PullRequestReviewGuideService.ts";
4646
import * as ServerLifecycleEvents from "./serverLifecycleEvents.ts";
4747
import * as AnalyticsService from "./telemetry/AnalyticsService.ts";
48+
import * as DelegatedTaskAnalytics from "./telemetry/DelegatedTaskAnalytics.ts";
4849
import * as ProviderEventIngestor from "./orchestration-v2/ProviderEventIngestor.ts";
4950
import * as ModelManifest from "./provider/ModelManifest.ts";
5051
import * as ResetCreditCoordinator from "./provider/resetCreditCoordinator.ts";
@@ -554,6 +555,7 @@ const layerRuntimeCoreDependenciesBase = Layer.mergeAll(
554555
),
555556
),
556557
layerThreadPullRequestWorker,
558+
DelegatedTaskAnalytics.layer,
557559
Layer.effectDiscard(
558560
Effect.gen(function* () {
559561
const service = yield* PullRequestSyncReactor.PullRequestSyncReactor;

0 commit comments

Comments
 (0)