Skip to content

Commit 9c30a1a

Browse files
authored
perf(server): linked calls reuse sessions while a tool waits (#26567)
1 parent 00b2d0e commit 9c30a1a

2 files changed

Lines changed: 181 additions & 2 deletions

File tree

Lines changed: 177 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,177 @@
1+
import { NodeHttpServer } from "@effect/platform-node";
2+
import { expect, it } from "@effect/vitest";
3+
import * as Context from "effect/Context";
4+
import * as DateTime from "effect/DateTime";
5+
import * as Deferred from "effect/Deferred";
6+
import * as Effect from "effect/Effect";
7+
import * as Fiber from "effect/Fiber";
8+
import * as Layer from "effect/Layer";
9+
import * as Ref from "effect/Ref";
10+
import * as Schema from "effect/Schema";
11+
import { Tool, Toolkit } from "effect/ai";
12+
import { HttpClient, HttpClientResponse } from "effect/http";
13+
14+
import * as McpHttpServer from "../mcp/McpHttpServer.ts";
15+
import * as McpInvocationContext from "../mcp/McpInvocationContext.ts";
16+
import * as McpToolAccess from "../mcp/McpToolAccess.ts";
17+
import * as PeerLinks from "./PeerLinks.ts";
18+
import * as PeerMcpClient from "./PeerMcpClient.ts";
19+
import { descriptorOf, layerLinkingEnvironment, linkTo, servePeer } from "./PeerLinks.testkit.ts";
20+
21+
const Result = Schema.Struct({ modes: Schema.String });
22+
const toolkit = Toolkit.make(
23+
Tool.make("probe", {
24+
description: "Returns this request's mode limits.",
25+
success: Result,
26+
dependencies: [McpInvocationContext.McpInvocationContext],
27+
}),
28+
Tool.make("hold", {
29+
description: "Holds a tool request open.",
30+
success: Result,
31+
dependencies: [McpInvocationContext.McpInvocationContext],
32+
}),
33+
);
34+
35+
it.effect.each([false, true])(
36+
"reuses an initialized session before its first call finishes (cancelled: %s)",
37+
(cancelFirst) =>
38+
Effect.scoped(
39+
Effect.gen(function* () {
40+
const entered = yield* Deferred.make<void>();
41+
const release = yield* Deferred.make<void>();
42+
const modes = McpInvocationContext.McpInvocationContext.pipe(
43+
Effect.map((scope) => {
44+
const ceiling = McpInvocationContext.clientModeCeiling(scope.client);
45+
return { modes: `${ceiling.runtimeMode}/${ceiling.interactionMode}` };
46+
}),
47+
);
48+
const handlers = McpToolAccess.toLayer(toolkit, {
49+
probe: McpToolAccess.reads(() => modes),
50+
hold: McpToolAccess.reads(() =>
51+
Deferred.succeed(entered, undefined).pipe(
52+
Effect.andThen(Deferred.await(release)),
53+
Effect.andThen(modes),
54+
),
55+
),
56+
});
57+
const box = descriptorOf("environment-box", "Box");
58+
const b = yield* servePeer(box, McpHttpServer.toolkitRegistration(toolkit, handlers));
59+
const context = yield* Layer.build(
60+
layerLinkingEnvironment(descriptorOf("environment-a", "A")),
61+
);
62+
const links = Context.get(context, PeerLinks.PeerLinks);
63+
const client = Context.get(context, PeerMcpClient.PeerMcpClient);
64+
yield* linkTo(links, b);
65+
const held = yield* client
66+
.call({
67+
environmentId: box.environmentId,
68+
tool: "hold",
69+
arguments: {},
70+
limits: { runtimeMode: "full-access", interactionMode: "default" },
71+
success: Result,
72+
})
73+
.pipe(Effect.forkScoped);
74+
yield* Deferred.await(entered);
75+
if (cancelFirst) yield* Fiber.interrupt(held);
76+
const before = (yield* Ref.get(b.bearers)).length;
77+
expect(
78+
yield* client.call({
79+
environmentId: box.environmentId,
80+
tool: "probe",
81+
arguments: {},
82+
limits: { runtimeMode: "approval-required", interactionMode: "plan" },
83+
success: Result,
84+
}),
85+
).toEqual({ modes: "approval-required/plan" });
86+
// One tool POST; no second initialize or initialized notification.
87+
expect((yield* Ref.get(b.bearers)).length - before).toBe(1);
88+
yield* Deferred.succeed(release, undefined);
89+
if (!cancelFirst) expect(yield* Fiber.join(held)).toEqual({ modes: "auto/default" });
90+
}),
91+
).pipe(Effect.provide(NodeHttpServer.layerTest)),
92+
);
93+
94+
it.effect("renews a missing peer session once and reuses its replacement", () =>
95+
Effect.gen(function* () {
96+
const sessionHeaders: Array<string | undefined> = [];
97+
const statuses = [200, 202, 404, 200, 202, 200, 200];
98+
const http = HttpClient.make((request) =>
99+
Effect.sync(() => {
100+
const index = sessionHeaders.length;
101+
sessionHeaders.push(request.headers["mcp-session-id"]);
102+
const status = statuses[index];
103+
if (status === undefined) throw new Error("Unexpected retry");
104+
return HttpClientResponse.fromWeb(
105+
request,
106+
new Response(
107+
index >= 5
108+
? JSON.stringify({
109+
result: {
110+
content: [{ type: "text", text: '{"modes":"approval-required/plan"}' }],
111+
},
112+
})
113+
: "",
114+
{
115+
status,
116+
headers:
117+
index === 0
118+
? { "mcp-session-id": "old" }
119+
: index === 3
120+
? { "mcp-session-id": "renewed" }
121+
: {},
122+
},
123+
),
124+
);
125+
}),
126+
);
127+
const box = descriptorOf("environment-box", "Box");
128+
const context = yield* Layer.build(
129+
PeerMcpClient.layer.pipe(
130+
Layer.provide(
131+
Layer.mergeAll(
132+
Layer.succeed(HttpClient.HttpClient, http),
133+
Layer.mock(PeerLinks.PeerLinks)({
134+
resolve: () =>
135+
Effect.succeed({
136+
link: {
137+
environmentId: box.environmentId,
138+
label: box.label,
139+
urls: ["https://peer.example"],
140+
access: "auto",
141+
linkedAt: DateTime.makeUnsafe(0),
142+
expiresAt: DateTime.makeUnsafe(1),
143+
lastReachedAt: null,
144+
lastError: null,
145+
},
146+
url: "https://peer.example",
147+
token: "fixture",
148+
}),
149+
recordOutcome: () => Effect.void,
150+
}),
151+
),
152+
),
153+
),
154+
);
155+
const client = Context.get(context, PeerMcpClient.PeerMcpClient);
156+
for (let call = 0; call < 2; call++) {
157+
expect(
158+
yield* client.call({
159+
environmentId: box.environmentId,
160+
tool: "probe",
161+
arguments: {},
162+
limits: { runtimeMode: "approval-required", interactionMode: "plan" },
163+
success: Result,
164+
}),
165+
).toEqual({ modes: "approval-required/plan" });
166+
}
167+
expect(sessionHeaders).toEqual([
168+
undefined,
169+
"old",
170+
"old",
171+
undefined,
172+
"renewed",
173+
"renewed",
174+
"renewed",
175+
]);
176+
}).pipe(Effect.scoped),
177+
);

‎apps/server/src/peer/PeerMcpClient.ts‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -175,13 +175,15 @@ const make = Effect.gen(function* () {
175175
);
176176
const existing = (yield* Ref.get(sessions)).get(sessionKey);
177177
const sessionId = existing ?? (yield* openSession(resolved.url, resolved.token, limitHeader));
178+
// The initialized session outlives a tool request, including a long wait or cancellation.
179+
if (existing === undefined) {
180+
yield* Ref.update(sessions, (current) => new Map(current).set(sessionKey, sessionId));
181+
}
178182
let response = yield* callOnce(sessionId);
179183
if (response.status === 404) {
180184
const renewed = yield* openSession(resolved.url, resolved.token, limitHeader);
181185
yield* Ref.update(sessions, (current) => new Map(current).set(sessionKey, renewed));
182186
response = yield* callOnce(renewed);
183-
} else if (existing === undefined) {
184-
yield* Ref.update(sessions, (current) => new Map(current).set(sessionKey, sessionId));
185187
}
186188
if (response.status === 401) {
187189
return yield* new OrchestratorMcpFailure({

0 commit comments

Comments
 (0)