Skip to content

Commit e8cbd3d

Browse files
feat(server): report outcome, handoff settings, and lineage for agent tools
mcp.tool.invoked now carries outcome and failure code, duration, and for handoff tools the caller's model, origin (user/agent/system/scheduler) and delegation depth, the settings the agent chose (delegate mode and wait timeout, delivery, workspace, batch size, explicit vs inherited target, schedule binding), and each receiving thread's runtime and interaction mode. A new mcp.delegated_task.finished event records each app-owned delegated task's final status and runtime with both providers. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
1 parent 4f91804 commit e8cbd3d

8 files changed

Lines changed: 579 additions & 88 deletions

File tree

‎apps/server/src/mcp/McpHttpServer.ts‎

Lines changed: 95 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,13 @@ import * as ServerConfig from "../config.ts";
2020
import * as DeviceService from "../device/DeviceService.ts";
2121
import * as ProviderRegistry from "../provider/Services/ProviderRegistry.ts";
2222
import * as AnalyticsService from "../telemetry/AnalyticsService.ts";
23-
import { agentToolProperties } from "../telemetry/ProviderDimensions.ts";
23+
import {
24+
type AgentThread,
25+
agentToolProperties,
26+
handoffSettings,
27+
MAX_REPORTED_DEPTH,
28+
toolOutcome,
29+
} from "../telemetry/ProviderDimensions.ts";
2430
import * as ThreadManagement from "../orchestration-v2/ThreadManagementService.ts";
2531
import * as McpInvocationContext from "./McpInvocationContext.ts";
2632
import * as OrchestratorMcpService from "./OrchestratorMcpService.ts";
@@ -683,10 +689,11 @@ const resultThreadIds = (content: unknown): ReadonlyArray<string> => {
683689

684690
/**
685691
* Runs a tool registration against a server that records one anonymous
686-
* `mcp.tool.invoked` event per call. The event names the tool and the calling
687-
* agent's provider and model. A handoff tool also names the provider and model
688-
* of each thread that received the work, so delegation between providers is
689-
* visible. Arguments, results, ids, and text are never recorded.
692+
* `mcp.tool.invoked` event per call: the tool, the calling agent's provider,
693+
* the outcome with its failure code, and the duration. A handoff tool also
694+
* reports the caller's model, origin, and delegation depth, the settings the
695+
* agent chose, and the provider, model, and modes of each thread that received
696+
* the work. Ids, prompts, and free text are never recorded.
690697
*/
691698
const withToolAnalytics = <E, R>(registration: Effect.Effect<void, E, R>) =>
692699
Effect.gen(function* () {
@@ -695,39 +702,88 @@ const withToolAnalytics = <E, R>(registration: Effect.Effect<void, E, R>) =>
695702
if (Option.isNone(analytics)) return yield* registration;
696703
const registry = yield* Effect.serviceOption(ProviderRegistry.ProviderRegistry);
697704
const threads = yield* Effect.serviceOption(ThreadManagement.ThreadManagementService);
698-
const modelSelectionOf = (threadId: string) =>
705+
const shellOf = (threadId: string) =>
699706
Option.isNone(threads)
700-
? Effect.succeed(undefined)
701-
: threads.value.getThreadShell(ThreadId.make(threadId)).pipe(
702-
Effect.map((shell) => shell?.modelSelection),
703-
Effect.orElseSucceed(() => undefined),
704-
);
705-
const record = (tool: string, result: McpSchema.CallToolResult | undefined) =>
707+
? Effect.succeed(null)
708+
: threads.value
709+
.getThreadShell(ThreadId.make(threadId))
710+
.pipe(Effect.orElseSucceed(() => null));
711+
// Delegation depth: parents walked from the caller, capped.
712+
const depthOf = (thread: AgentThread) =>
713+
Effect.gen(function* () {
714+
let depth = 0;
715+
let parentId =
716+
thread.lineage.relationshipToParent === "subagent" ? thread.lineage.parentThreadId : null;
717+
while (parentId !== null && depth < MAX_REPORTED_DEPTH) {
718+
depth += 1;
719+
const parent = yield* shellOf(parentId);
720+
parentId =
721+
parent?.lineage.relationshipToParent === "subagent"
722+
? parent.lineage.parentThreadId
723+
: null;
724+
}
725+
return depth;
726+
});
727+
// Whether the caller's active run was started by a scheduled task.
728+
const scheduledRunOf = (threadId: string, runId: string | null) =>
729+
Option.isNone(threads) || runId === null
730+
? Effect.succeed(false)
731+
: threads.value
732+
.getThreadRecords(ThreadId.make(threadId), ["runs", "messages"], {
733+
runIds: [runId as never],
734+
messageRoles: ["user"],
735+
})
736+
.pipe(
737+
Effect.map((records) => {
738+
const run = records.runs.find((candidate) => candidate.id === runId);
739+
return records.messages.some(
740+
(message) =>
741+
message.id === run?.userMessageId && message.scheduledTaskId !== undefined,
742+
);
743+
}),
744+
Effect.orElseSucceed(() => false),
745+
);
746+
const record = (
747+
tool: string,
748+
args: unknown,
749+
result: McpSchema.CallToolResult | undefined,
750+
durationMs: number,
751+
) =>
706752
Effect.gen(function* () {
707753
const invocation = Option.getOrUndefined(
708754
yield* Effect.serviceOption(McpInvocationContext.McpInvocationContext),
709755
);
710756
const providers = Option.isSome(registry) ? yield* registry.value.getProviders : [];
711757
// Only handoff tools pay for thread lookups; the rest report the
712-
// caller's provider from the credential and skip its model.
758+
// caller's provider from the credential.
713759
const handoff = HANDOFF_TOOLS.has(tool);
714760
const caller =
715-
invocation === undefined
716-
? undefined
717-
: handoff
718-
? yield* modelSelectionOf(invocation.threadId)
719-
: { instanceId: invocation.providerInstanceId, model: "" };
761+
handoff && invocation !== undefined
762+
? ((yield* shellOf(invocation.threadId)) ?? undefined)
763+
: undefined;
720764
const targetIds = handoff
721765
? [...new Set(resultThreadIds(result?.structuredContent))].filter(
722766
(threadId) => threadId !== invocation?.threadId,
723767
)
724768
: [];
725-
const targets = yield* Effect.forEach(targetIds, modelSelectionOf);
769+
const targets = yield* Effect.forEach(targetIds, shellOf);
726770
for (const properties of agentToolProperties({
727771
tool,
728772
providers,
729773
caller,
730-
targets: targets.filter((selection) => selection !== undefined),
774+
...(invocation === undefined
775+
? {}
776+
: { callerProviderInstanceId: invocation.providerInstanceId }),
777+
...(caller === undefined || invocation === undefined
778+
? {}
779+
: {
780+
callerDepth: yield* depthOf(caller),
781+
callerScheduledRun: yield* scheduledRunOf(invocation.threadId, caller.activeRunId),
782+
}),
783+
targets: targets.filter((thread) => thread !== null),
784+
outcome: toolOutcome(result),
785+
...(handoff ? { settings: handoffSettings(tool, args, result?.structuredContent) } : {}),
786+
durationMs,
731787
})) {
732788
yield* analytics.value.record("mcp.tool.invoked", properties);
733789
}
@@ -738,13 +794,25 @@ const withToolAnalytics = <E, R>(registration: Effect.Effect<void, E, R>) =>
738794
server.addTool({
739795
...options,
740796
handle: (payload) =>
741-
options
742-
.handle(payload)
743-
.pipe(
744-
Effect.onExit((exit) =>
745-
record(options.tool.name, exit._tag === "Success" ? exit.value : undefined),
746-
),
747-
),
797+
Effect.gen(function* () {
798+
const startedAt = yield* Clock.currentTimeMillis;
799+
return yield* options
800+
.handle(payload)
801+
.pipe(
802+
Effect.onExit((exit) =>
803+
Clock.currentTimeMillis.pipe(
804+
Effect.flatMap((endedAt) =>
805+
record(
806+
options.tool.name,
807+
payload,
808+
exit._tag === "Success" ? exit.value : undefined,
809+
endedAt - startedAt,
810+
),
811+
),
812+
),
813+
),
814+
);
815+
}),
748816
}),
749817
});
750818
return yield* registration.pipe(Effect.provideService(McpServer.McpServer, recordingServer));

‎apps/server/src/mcp/toolkits/core.test.ts‎

Lines changed: 28 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -172,6 +172,13 @@ it.effect("records the calling agent and the agent a tool acted on", () => {
172172
archivedAt: null,
173173
deletedAt: null,
174174
activeRunId: "mcp-core-run",
175+
createdBy: "agent",
176+
creationSource: "mcp",
177+
lineage: {
178+
parentThreadId: id === threadId ? ThreadId.make("mcp-core-parent") : null,
179+
relationshipToParent: id === threadId ? "subagent" : null,
180+
rootThreadId: ThreadId.make("mcp-core-parent"),
181+
},
175182
}) as never;
176183
return Effect.gen(function* () {
177184
const server = yield* McpServer.McpServer;
@@ -181,19 +188,26 @@ it.effect("records the calling agent and the agent a tool acted on", () => {
181188
Effect.provideService(McpInvocationContext.McpInvocationContext, scope),
182189
Effect.provideService(McpSchema.McpServerClient, client),
183190
);
184-
expect(recorded).toEqual([
185-
{
186-
event: "mcp.tool.invoked",
187-
properties: {
188-
tool: "t3_thread_fork",
189-
callerProvider: "codex",
190-
callerModel: "gpt-5.5",
191-
targetProvider: "claudeAgent",
192-
targetModel: "claude-opus-5-5",
193-
crossProvider: true,
194-
},
195-
},
196-
]);
191+
expect(recorded).toHaveLength(1);
192+
const [{ event, properties }] = recorded as [{ event: string; properties: object }];
193+
expect(event).toBe("mcp.tool.invoked");
194+
expect(properties).toMatchObject({
195+
tool: "t3_thread_fork",
196+
outcome: "ok",
197+
callerProvider: "codex",
198+
callerModel: "gpt-5.5",
199+
callerOrigin: "agent",
200+
callerDepth: 1,
201+
targetProvider: "claudeAgent",
202+
targetModel: "claude-opus-5-5",
203+
targetRuntimeMode: "full-access",
204+
targetInteractionMode: "default",
205+
crossProvider: true,
206+
});
207+
expect(properties).toHaveProperty("durationMs");
208+
expect(Object.values(properties).some((value) => String(value).includes("mcp-core"))).toBe(
209+
false,
210+
);
197211
}).pipe(
198212
Effect.provide(
199213
McpHttpServer.ThreadToolkitRegistrationLive.pipe(
@@ -207,6 +221,7 @@ it.effect("records the calling agent and the agent a tool acted on", () => {
207221
? shell(threadId, "codex", "gpt-5.5")
208222
: shell(id, "claudeAgent", "claude-opus-5-5"),
209223
),
224+
getThreadRecords: () => Effect.succeed({ runs: [], messages: [] } as never),
210225
getProjectThreadRecords: () =>
211226
Effect.succeed({ thread: shell(threadId, "codex", "gpt-5.5") } as never),
212227
dispatch: () => Effect.succeed({ sequence: 1 } as never),

‎apps/server/src/server.ts‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@ import * as SqlitePersistence from "./persistence/Layers/Sqlite.ts";
4646
import * as PullRequestFilesViewed from "./persistence/PullRequestFilesViewed.ts";
4747
import * as ServerLifecycleEvents from "./serverLifecycleEvents.ts";
4848
import * as AnalyticsService from "./telemetry/AnalyticsService.ts";
49+
import * as DelegatedTaskAnalytics from "./telemetry/DelegatedTaskAnalytics.ts";
4950
import * as ProviderEventIngestor from "./orchestration-v2/ProviderEventIngestor.ts";
5051
import * as ModelManifest from "./provider/ModelManifest.ts";
5152
import * as ResetCreditCoordinator from "./provider/Layers/resetCreditCoordinator.ts";
@@ -470,6 +471,10 @@ const ThreadSettlementWorkerLive = Layer.effectDiscard(
470471
ThreadSettlementService.make.pipe(Effect.flatMap((service) => service.start())),
471472
).pipe(Layer.provide(PullRequestServiceLive), Layer.provide(ProjectionStoreV2.layer));
472473

474+
const DelegatedTaskAnalyticsLive = Layer.effectDiscard(
475+
DelegatedTaskAnalytics.make.pipe(Effect.flatMap((service) => service.start())),
476+
);
477+
473478
const ThreadPullRequestWorkerLive = Layer.effectDiscard(
474479
ThreadPullRequestService.make.pipe(Effect.flatMap((service) => service.start())),
475480
).pipe(Layer.provide(PullRequestServiceLive));
@@ -513,6 +518,7 @@ const RuntimeCoreDependenciesBaseLive = Layer.mergeAll(
513518
Layer.provide(ProjectionStoreV2.layer),
514519
),
515520
ThreadPullRequestWorkerLive,
521+
DelegatedTaskAnalyticsLive,
516522
Layer.effectDiscard(
517523
Effect.gen(function* () {
518524
const service = yield* PullRequestSyncReactor.PullRequestSyncReactor;
Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
import { ProviderInstanceId, ThreadId } from "@t3tools/contracts";
2+
import { expect, it } from "@effect/vitest";
3+
import * as DateTime from "effect/DateTime";
4+
import * as Effect from "effect/Effect";
5+
import * as Layer from "effect/Layer";
6+
7+
import * as Orchestrator from "../orchestration-v2/Orchestrator.ts";
8+
import * as ProviderRegistry from "../provider/Services/ProviderRegistry.ts";
9+
import * as AnalyticsService from "./AnalyticsService.ts";
10+
import * as DelegatedTaskAnalytics from "./DelegatedTaskAnalytics.ts";
11+
12+
const parentThreadId = ThreadId.make("parent-private");
13+
14+
const subagentEvent = (status: string, origin = "app_owned") =>
15+
({
16+
type: "subagent.updated",
17+
payload: {
18+
id: "task-private",
19+
threadId: parentThreadId,
20+
origin,
21+
status,
22+
providerInstanceId: ProviderInstanceId.make("claudeAgent"),
23+
model: "claude-opus-5-5",
24+
prompt: "secret task",
25+
completionWake: "always",
26+
startedAt: DateTime.makeUnsafe(0),
27+
completedAt: DateTime.makeUnsafe(90_000),
28+
},
29+
}) as never;
30+
31+
it.effect("records each delegated task's final outcome once, without ids or prompts", () => {
32+
const recorded: Array<{ event: string; properties: unknown }> = [];
33+
return Effect.gen(function* () {
34+
const analytics = yield* DelegatedTaskAnalytics.make;
35+
yield* analytics.onSubagent(subagentEvent("running"));
36+
yield* analytics.onSubagent(subagentEvent("completed", "provider_native"));
37+
yield* analytics.onSubagent(subagentEvent("completed"));
38+
yield* analytics.onSubagent(subagentEvent("completed"));
39+
expect(recorded).toEqual([
40+
{
41+
event: "mcp.delegated_task.finished",
42+
properties: {
43+
status: "completed",
44+
callerProvider: "codex",
45+
callerModel: "gpt-5.5",
46+
targetProvider: "claudeAgent",
47+
targetModel: "claude-opus-5-5",
48+
crossProvider: true,
49+
completionWake: "always",
50+
durationSeconds: 90,
51+
},
52+
},
53+
]);
54+
}).pipe(
55+
Effect.provide(
56+
Layer.mergeAll(
57+
Layer.mock(Orchestrator.OrchestratorV2)({
58+
getThreadShell: () =>
59+
Effect.succeed({
60+
modelSelection: { instanceId: ProviderInstanceId.make("codex"), model: "gpt-5.5" },
61+
} as never),
62+
}),
63+
Layer.mock(ProviderRegistry.ProviderRegistry)({
64+
getProviders: Effect.succeed([
65+
{
66+
instanceId: ProviderInstanceId.make("codex"),
67+
driver: "codex",
68+
models: [{ slug: "gpt-5.5", isCustom: false }],
69+
} as never,
70+
{
71+
instanceId: ProviderInstanceId.make("claudeAgent"),
72+
driver: "claudeAgent",
73+
models: [{ slug: "claude-opus-5-5", isCustom: false }],
74+
} as never,
75+
]),
76+
}),
77+
Layer.succeed(
78+
AnalyticsService.AnalyticsService,
79+
AnalyticsService.AnalyticsService.of({
80+
record: (event, properties) =>
81+
Effect.sync(() => void recorded.push({ event, properties })),
82+
flush: Effect.void,
83+
}),
84+
),
85+
),
86+
),
87+
);
88+
});

0 commit comments

Comments
 (0)