Skip to content

Commit bd731da

Browse files
only21milmaria-rcksjuliusmarminge
authored
fix(server): linked calls fail fast when a peer stalls the MCP handshake (pingdotgg#28876)
Co-authored-by: maria-rcks <[email protected]> Co-authored-by: Julius Marminge <[email protected]>
1 parent 0508906 commit bd731da

3 files changed

Lines changed: 136 additions & 48 deletions

File tree

‎apps/server/src/peer/PeerLinks.ts‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -47,10 +47,10 @@ const PeerHttpApi = HttpApi.make(EnvironmentHttpApi.identifier)
4747
/** The redirect registered for a link. Nothing ever listens there: the code comes back in the response. */
4848
const LINK_REDIRECT_URI = "http://127.0.0.1/t3code-peer-link";
4949
/**
50-
* How long a peer's descriptor may take before a route counts as unreachable.
51-
* A tailnet or relayed route to a machine that is waking up takes longer.
50+
* How long a peer's descriptor or MCP handshake may take before a route counts
51+
* as unreachable. A tailnet or relayed route to a machine that is waking up takes longer.
5252
*/
53-
const probeTimeout = (origin: string) =>
53+
export const probeTimeout = (origin: string) =>
5454
isLocalLoopbackHost(new URL(origin).hostname) ? Duration.seconds(3) : Duration.seconds(10);
5555
/** How long a listing waits on a link's addresses; a real call still gives a waking peer `probeTimeout`. */
5656
const LIST_PROBE_TIMEOUT = Duration.seconds(3);

‎apps/server/src/peer/PeerMcpClient.test.ts‎

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,12 +9,15 @@ import {
99
import * as Context from "effect/Context";
1010
import * as DateTime from "effect/DateTime";
1111
import * as Deferred from "effect/Deferred";
12+
import * as Duration from "effect/Duration";
1213
import * as Effect from "effect/Effect";
1314
import * as Fiber from "effect/Fiber";
1415
import * as Layer from "effect/Layer";
16+
import * as Queue from "effect/Queue";
1517
import * as Ref from "effect/Ref";
1618
import * as Schema from "effect/Schema";
1719
import { Tool, Toolkit } from "effect/ai";
20+
import * as TestClock from "effect/testing/TestClock";
1821
import { HttpClient, HttpClientResponse } from "effect/http";
1922

2023
import * as McpHttpServer from "../mcp/McpHttpServer.ts";
@@ -44,6 +47,49 @@ const toolkit = Toolkit.make(
4447
}),
4548
);
4649

50+
/** One `probe` call to a linked peer at a remote address, answered by `http`. */
51+
const probeThrough = Effect.fn(function* (http: HttpClient.HttpClient) {
52+
const box = descriptorOf("environment-box", "Box");
53+
const context = yield* Layer.build(
54+
PeerMcpClient.layer.pipe(
55+
Layer.provide(
56+
Layer.mergeAll(
57+
Layer.succeed(HttpClient.HttpClient, http),
58+
Layer.mock(PeerLinks.PeerLinks)({
59+
resolve: () =>
60+
Effect.succeed({
61+
link: {
62+
environmentId: box.environmentId,
63+
label: box.label,
64+
urls: ["https://peer.example"],
65+
access: "auto",
66+
linkedAt: DateTime.makeUnsafe(0),
67+
expiresAt: DateTime.makeUnsafe(1),
68+
lastReachedAt: null,
69+
lastError: null,
70+
},
71+
url: "https://peer.example",
72+
token: "fixture",
73+
}),
74+
recordOutcome: () => Effect.void,
75+
}),
76+
),
77+
),
78+
),
79+
);
80+
return Context.get(context, PeerMcpClient.PeerMcpClient).call({
81+
environmentId: box.environmentId,
82+
tool: "probe",
83+
arguments: {},
84+
limits: { runtimeMode: "approval-required", interactionMode: "plan" },
85+
success: Result,
86+
});
87+
});
88+
89+
const probeResult = JSON.stringify({
90+
result: { content: [{ type: "text", text: '{"modes":"approval-required/plan"}' }] },
91+
});
92+
4793
const refusal = new OrchestratorMcpFailure({
4894
code: "thread_not_found",
4995
message: "Thread thread-gone was not found.",
@@ -295,3 +341,45 @@ it.effect("renews a missing peer session once and reuses its replacement", () =>
295341
]);
296342
}).pipe(Effect.scoped),
297343
);
344+
345+
it.effect("bounds the session handshake, not a tool call that waits longer", () =>
346+
Effect.gen(function* () {
347+
const arrived = yield* Queue.unbounded<number>();
348+
const answer = yield* Deferred.make<void>();
349+
let requests = 0;
350+
const http = HttpClient.make((request) =>
351+
Effect.gen(function* () {
352+
const index = requests++;
353+
yield* Queue.offer(arrived, index);
354+
// The first initialize, then the next initialized notification, never answer;
355+
// the third session's tool call answers late.
356+
if (index === 0 || index === 2) return yield* Effect.never;
357+
if (index === 5) yield* Deferred.await(answer);
358+
return HttpClientResponse.fromWeb(
359+
request,
360+
new Response(index === 5 ? probeResult : "", {
361+
status: index === 4 ? 202 : 200,
362+
headers: { "mcp-session-id": "session" },
363+
}),
364+
);
365+
}),
366+
);
367+
const probe = yield* probeThrough(http);
368+
369+
for (const requestsBeforeStall of [1, 2]) {
370+
const stalled = yield* probe.pipe(Effect.flip, Effect.forkScoped);
371+
yield* Queue.takeN(arrived, requestsBeforeStall);
372+
yield* TestClock.adjust(Duration.seconds(10));
373+
expect(yield* Fiber.join(stalled)).toMatchObject({
374+
code: "orchestration_error",
375+
message: "The linked environment stopped answering.",
376+
});
377+
}
378+
379+
const waiting = yield* Effect.forkScoped(probe);
380+
yield* Queue.takeN(arrived, 3);
381+
yield* TestClock.adjust(Duration.minutes(1));
382+
yield* Deferred.succeed(answer, undefined);
383+
expect(yield* Fiber.join(waiting)).toEqual({ modes: "approval-required/plan" });
384+
}).pipe(Effect.scoped),
385+
);

‎apps/server/src/peer/PeerMcpClient.ts‎

Lines changed: 45 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -90,6 +90,11 @@ const linkedEnvironmentsOffThere = new OrchestratorMcpFailure({
9090
"The linked environment has Linked environments turned off. Turn it on in its Settings → Labs.",
9191
});
9292

93+
const stoppedAnswering = new OrchestratorMcpFailure({
94+
code: "orchestration_error",
95+
message: "The linked environment stopped answering.",
96+
});
97+
9398
/** The body of an MCP response: JSON, or a single server-sent event carrying JSON. */
9499
const responseJson = (contentType: string | undefined, text: string) =>
95100
contentType?.includes("text/event-stream") === true
@@ -119,45 +124,45 @@ const make = Effect.gen(function* () {
119124
),
120125
);
121126

122-
const openSession = Effect.fn("PeerMcpClient.openSession")(function* (
123-
url: string,
124-
token: string,
125-
limitHeader: Record<string, string>,
126-
) {
127-
const initialized = yield* post(
128-
url,
129-
token,
130-
{
131-
jsonrpc: "2.0",
132-
id: 0,
133-
method: "initialize",
134-
params: {
135-
protocolVersion: PROTOCOL_VERSION,
136-
capabilities: {},
137-
clientInfo: { name: "t3code-peer", version: "1" },
127+
const openSession = Effect.fn("PeerMcpClient.openSession")(
128+
function* (url: string, token: string, limitHeader: Record<string, string>) {
129+
const initialized = yield* post(
130+
url,
131+
token,
132+
{
133+
jsonrpc: "2.0",
134+
id: 0,
135+
method: "initialize",
136+
params: {
137+
protocolVersion: PROTOCOL_VERSION,
138+
capabilities: {},
139+
clientInfo: { name: "t3code-peer", version: "1" },
140+
},
138141
},
139-
},
140-
limitHeader,
141-
);
142-
const sessionId = initialized.headers["mcp-session-id"];
143-
if (initialized.status === 403) return yield* linkedEnvironmentsOffThere;
144-
if (initialized.status !== 200 || sessionId === undefined) {
145-
return yield* new OrchestratorMcpFailure({
146-
code: initialized.status === 401 ? "capability_denied" : "orchestration_error",
147-
message:
148-
initialized.status === 401
149-
? "The linked environment no longer accepts this link. It may have been revoked there; link it again."
150-
: `The linked environment refused to start an MCP session (HTTP ${initialized.status}).`,
151-
});
152-
}
153-
yield* post(
154-
url,
155-
token,
156-
{ jsonrpc: "2.0", method: "notifications/initialized" },
157-
{ ...limitHeader, "mcp-session-id": sessionId, "mcp-protocol-version": PROTOCOL_VERSION },
158-
);
159-
return sessionId;
160-
});
142+
limitHeader,
143+
);
144+
const sessionId = initialized.headers["mcp-session-id"];
145+
if (initialized.status === 403) return yield* linkedEnvironmentsOffThere;
146+
if (initialized.status !== 200 || sessionId === undefined) {
147+
return yield* new OrchestratorMcpFailure({
148+
code: initialized.status === 401 ? "capability_denied" : "orchestration_error",
149+
message:
150+
initialized.status === 401
151+
? "The linked environment no longer accepts this link. It may have been revoked there; link it again."
152+
: `The linked environment refused to start an MCP session (HTTP ${initialized.status}).`,
153+
});
154+
}
155+
yield* post(
156+
url,
157+
token,
158+
{ jsonrpc: "2.0", method: "notifications/initialized" },
159+
{ ...limitHeader, "mcp-session-id": sessionId, "mcp-protocol-version": PROTOCOL_VERSION },
160+
);
161+
return sessionId;
162+
},
163+
// Only the handshake is bounded: a forwarded tool call may wait for minutes.
164+
(effect, url) => Effect.timeout(effect, PeerLinks.probeTimeout(url)),
165+
);
161166

162167
const call: PeerMcpClient["Service"]["call"] = Effect.fn("PeerMcpClient.call")(
163168
function* (input) {
@@ -285,13 +290,8 @@ const make = Effect.gen(function* () {
285290
Effect.tap(() => links.recordOutcome(input.environmentId, { error: null })),
286291
// Recorded above with the transport's own message; the agent hears it plainly.
287292
Effect.catchTags({
288-
HttpClientError: () =>
289-
Effect.fail(
290-
new OrchestratorMcpFailure({
291-
code: "orchestration_error",
292-
message: "The linked environment stopped answering.",
293-
}),
294-
),
293+
HttpClientError: () => Effect.fail(stoppedAnswering),
294+
TimeoutError: () => Effect.fail(stoppedAnswering),
295295
}),
296296
Effect.flatMap(Effect.fromResult),
297297
),

0 commit comments

Comments
 (0)