Skip to content

Commit a2a13b5

Browse files
robertnisipeanuclaudejuliusmarminge
authored
fix(server): t3-code tools keep working after a turn waits over a day (pingdotgg#18552)
Co-authored-by: Claude Opus 5.5 <[email protected]> Co-authored-by: Julius Marminge <[email protected]>
1 parent 60f2246 commit a2a13b5

2 files changed

Lines changed: 141 additions & 63 deletions

File tree

‎apps/server/src/orchestration-v2/RunExecutionService.test.ts‎

Lines changed: 126 additions & 62 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,11 @@
1+
import * as NodeServices from "@effect/platform-node/NodeServices";
12
import { assert, it, vi } from "@effect/vitest";
23
import {
34
CheckpointScopeId,
45
ChatAttachmentId,
56
ChatFileAttachment,
67
CommandId,
8+
EnvironmentId,
79
EventId,
810
MessageId,
911
NodeId,
@@ -32,10 +34,16 @@ import * as Exit from "effect/Exit";
3234
import * as DateTime from "effect/DateTime";
3335
import * as Deferred from "effect/Deferred";
3436
import * as Effect from "effect/Effect";
37+
import * as Fiber from "effect/Fiber";
3538
import * as Layer from "effect/Layer";
3639
import * as Option from "effect/Option";
3740
import * as Ref from "effect/Ref";
3841
import * as Stream from "effect/Stream";
42+
import { HttpServer } from "effect/http";
43+
import * as NetAddress from "effect/net/NetAddress";
44+
import * as TestClock from "effect/testing/TestClock";
45+
46+
import * as ServerEnvironment from "../environment/ServerEnvironment.ts";
3947

4048
import * as McpSessionRegistry from "../mcp/McpSessionRegistry.ts";
4149
import * as ServerSettings from "../serverSettings.ts";
@@ -1096,75 +1104,128 @@ it.effect(
10961104
}).pipe(Effect.provide(layerRunExecutionTest)),
10971105
);
10981106

1107+
const layerMcpSessionRegistryTest = McpSessionRegistry.layer.pipe(
1108+
Layer.provide(
1109+
Layer.mergeAll(
1110+
NodeServices.layer,
1111+
Layer.mock(HttpServer.HttpServer)({
1112+
address: NetAddress.inetAddressFromIpStringUnsafe("127.0.0.1", 43123),
1113+
}),
1114+
Layer.mock(ServerEnvironment.ServerEnvironment)({
1115+
getEnvironmentId: Effect.succeed(EnvironmentId.make("environment:mcp-liveness")),
1116+
}),
1117+
),
1118+
),
1119+
);
1120+
10991121
it.effect("refreshes MCP credential liveness before calling the provider", () =>
11001122
Effect.gen(function* () {
1101-
const runExecution = yield* RunExecutionService.RunExecutionServiceV2;
11021123
const order = yield* Ref.make<ReadonlyArray<string>>([]);
1103-
const threadId = ThreadId.make("thread:run-execution-mcp-liveness");
1124+
const key = "run-execution-mcp-liveness";
11041125
const touchActiveMcpThread = vi
11051126
.spyOn(McpSessionRegistry, "touchActiveMcpThread")
1106-
.mockImplementation((touchedThreadId) =>
1107-
Ref.update(order, (entries) => [...entries, `touch:${touchedThreadId}`]),
1127+
.mockImplementation((threadId) =>
1128+
Ref.update(order, (entries) => [...entries, `touch:${threadId}`]),
11081129
);
11091130

1110-
yield* runExecution
1111-
.startRootRun({
1112-
commandId: CommandId.make("command:run-execution-mcp-liveness"),
1113-
appThread: { id: threadId } as OrchestrationV2AppThread,
1114-
providerSessionId: ProviderSessionId.make("session:run-execution-mcp-liveness"),
1115-
session: {
1116-
events: Stream.never,
1117-
startTurn: () => Ref.update(order, (entries) => [...entries, "start-turn"]),
1118-
} as unknown as ProviderAdapter.ProviderAdapterV2SessionRuntime,
1119-
run: {
1120-
id: RunId.make("run:run-execution-mcp-liveness"),
1121-
threadId,
1122-
ordinal: 1,
1123-
providerInstanceId: ProviderInstanceId.make("codex"),
1124-
} as OrchestrationV2Run,
1125-
rootNode: {
1126-
id: NodeId.make("node:run-execution-mcp-liveness"),
1127-
} as OrchestrationV2ExecutionNode,
1128-
checkpointScope: {
1129-
id: CheckpointScopeId.make("checkpoint-scope:run-execution-mcp-liveness"),
1130-
} as OrchestrationV2CheckpointScope,
1131-
providerThread: {
1132-
id: ProviderThreadId.make("provider-thread:run-execution-mcp-liveness"),
1133-
driver,
1134-
} as OrchestrationV2ProviderThread,
1135-
attempt: {
1136-
id: RunAttemptId.make("attempt:run-execution-mcp-liveness"),
1137-
providerTurnId: null,
1138-
} as OrchestrationV2RunAttempt,
1139-
attemptId: RunAttemptId.make("attempt:run-execution-mcp-liveness"),
1140-
providerTurnOrdinal: 1,
1141-
message: {
1142-
messageId: MessageId.make("message:run-execution-mcp-liveness"),
1143-
text: "Keep the MCP credential alive.",
1144-
attachments: [],
1145-
createdBy: "user",
1146-
creationSource: "web",
1147-
},
1148-
modelSelection: {
1149-
instanceId: ProviderInstanceId.make("codex"),
1150-
model: "gpt-5.4",
1151-
},
1152-
runtimePolicy: {
1153-
runtimeMode: "full-access",
1154-
interactionMode: "default",
1155-
cwd: process.cwd(),
1156-
approvalPolicy: "never",
1157-
sandboxPolicy: {
1158-
type: "readOnly",
1159-
access: { type: "fullAccess" },
1160-
networkAccess: false,
1161-
},
1162-
},
1163-
})
1164-
.pipe(Effect.ensuring(Effect.sync(() => touchActiveMcpThread.mockRestore())));
1131+
yield* captureRootRunTermination({
1132+
key,
1133+
shouldFinalizeRun: () => Effect.succeed(true),
1134+
startTurn: () => Ref.update(order, (entries) => [...entries, "start-turn"]),
1135+
}).pipe(Effect.ensuring(Effect.sync(() => touchActiveMcpThread.mockRestore())));
11651136

1166-
assert.deepEqual(yield* Ref.get(order), [`touch:${threadId}`, "start-turn"]);
1167-
}).pipe(Effect.provide(layerRunExecutionTest)),
1137+
assert.deepEqual(yield* Ref.get(order), [
1138+
`touch:${backgroundScenarioIds(key).threadId}`,
1139+
"start-turn",
1140+
]);
1141+
}),
1142+
);
1143+
1144+
it.effect.each([
1145+
["keeps the MCP credential alive for a 44-hour turn without MCP traffic", false],
1146+
["expires the MCP credential at 44 hours when ingestion ended at 2 hours", true],
1147+
] as const)("%s", ([, endEarly]) =>
1148+
Effect.gen(function* () {
1149+
const key = `mcp-liveness:${endEarly}`;
1150+
const { threadId } = backgroundScenarioIds(key);
1151+
const registry = yield* McpSessionRegistry.McpSessionRegistry;
1152+
const issued = yield* registry.issue({
1153+
threadId,
1154+
providerInstanceId: ProviderInstanceId.make("codex"),
1155+
});
1156+
const token = issued.config.authorizationHeader.replace(/^Bearer\s+/, "");
1157+
const started = yield* Deferred.make<void>();
1158+
const end = yield* Deferred.make<void>();
1159+
const run = yield* captureRootRunTermination({
1160+
key,
1161+
shouldFinalizeRun: () => Effect.succeed(true),
1162+
startTurn: () => Deferred.succeed(started, undefined).pipe(Effect.asVoid),
1163+
events: (ids) =>
1164+
Stream.fromEffect(Deferred.await(end)).pipe(
1165+
Stream.map(() => rootTerminalEvent(ids, "completed")),
1166+
),
1167+
}).pipe(Effect.forkChild);
1168+
1169+
yield* Effect.gen(function* () {
1170+
yield* Deferred.await(started);
1171+
yield* TestClock.adjust("2 hours");
1172+
if (endEarly) {
1173+
yield* Deferred.succeed(end, undefined);
1174+
yield* Fiber.join(run);
1175+
}
1176+
yield* TestClock.adjust("42 hours");
1177+
assert.equal(
1178+
(yield* registry.resolve(token))?.thread.threadId,
1179+
endEarly ? undefined : threadId,
1180+
);
1181+
}).pipe(
1182+
Effect.ensuring(
1183+
Deferred.succeed(end, undefined).pipe(Effect.andThen(Fiber.join(run)), Effect.orDie),
1184+
),
1185+
);
1186+
}).pipe(Effect.provide(layerMcpSessionRegistryTest)),
1187+
);
1188+
1189+
it.effect(
1190+
"does not renew an MCP credential after startup is interrupted during the app context read",
1191+
() =>
1192+
Effect.gen(function* () {
1193+
const key = "mcp-liveness:abandoned-startup";
1194+
const { threadId } = backgroundScenarioIds(key);
1195+
const registry = yield* McpSessionRegistry.McpSessionRegistry;
1196+
const issued = yield* registry.issue({
1197+
threadId,
1198+
providerInstanceId: ProviderInstanceId.make("codex"),
1199+
});
1200+
const token = issued.config.authorizationHeader.replace(/^Bearer\s+/, "");
1201+
const contextReadStarted = yield* Deferred.make<void>();
1202+
const end = yield* Deferred.make<void>();
1203+
const closed = yield* Deferred.make<void>();
1204+
const run = yield* captureRootRunTermination({
1205+
key,
1206+
shouldFinalizeRun: () => Effect.succeed(true),
1207+
appContext: Deferred.succeed(contextReadStarted, undefined).pipe(
1208+
Effect.andThen(Effect.never),
1209+
),
1210+
events: (ids) =>
1211+
Stream.fromEffect(Deferred.await(end)).pipe(
1212+
Stream.map(() => rootTerminalEvent(ids, "completed")),
1213+
),
1214+
onSubscriptionClose: Deferred.succeed(closed, undefined).pipe(Effect.asVoid),
1215+
}).pipe(Effect.interruptible, Effect.forkChild);
1216+
1217+
yield* Effect.gen(function* () {
1218+
yield* Deferred.await(contextReadStarted);
1219+
yield* Fiber.interrupt(run);
1220+
assert.isTrue(Exit.hasInterrupts(yield* Fiber.await(run)));
1221+
yield* TestClock.adjust("44 hours");
1222+
assert.isUndefined(yield* registry.resolve(token));
1223+
}).pipe(
1224+
Effect.ensuring(
1225+
Deferred.succeed(end, undefined).pipe(Effect.andThen(Deferred.await(closed))),
1226+
),
1227+
);
1228+
}).pipe(Effect.provide(layerMcpSessionRegistryTest)),
11681229
);
11691230

11701231
it.effect("starts the provider when checkpoint baseline capture fails", () =>
@@ -4057,6 +4118,7 @@ function captureRootRunTermination(input: {
40574118
>;
40584119
readonly startTurn?: ProviderAdapter.ProviderAdapterV2SessionRuntime["startTurn"];
40594120
readonly refreshAfterTurn?: Effect.Effect<void>;
4121+
readonly onSubscriptionClose?: Effect.Effect<void>;
40604122
readonly shouldStartProviderTurn?: RunExecutionService.RunExecutionServiceV2StartRootRunInput["shouldStartProviderTurn"];
40614123
readonly appContext?: ReturnType<McpAppModelContext.McpAppModelContext["Service"]["forThread"]>;
40624124
}) {
@@ -4178,7 +4240,9 @@ function captureRootRunTermination(input: {
41784240
: []),
41794241
rootTerminalEvent(ids, "interrupted"),
41804242
] satisfies ReadonlyArray<ProviderAdapter.ProviderAdapterV2Event>),
4181-
close: Deferred.succeed(ingestionDone, undefined),
4243+
close: Deferred.succeed(ingestionDone, undefined).pipe(
4244+
Effect.andThen(input.onSubscriptionClose ?? Effect.void),
4245+
),
41824246
}),
41834247
startTurn: input.startTurn ?? (() => Effect.void),
41844248
} as unknown as ProviderAdapter.ProviderAdapterV2SessionRuntime,

‎apps/server/src/orchestration-v2/RunExecutionService.ts‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1485,7 +1485,13 @@ export const layer: Layer.Layer<
14851485
// A provider turn is a sign that the session is still alive. Keep
14861486
// its already-issued MCP credential valid even when the agent goes
14871487
// a long time between browser-tool calls.
1488-
yield* McpSessionRegistry.touchActiveMcpThread(input.run.threadId);
1488+
const touchMcpCredential = McpSessionRegistry.touchActiveMcpThread(input.run.threadId).pipe(
1489+
Effect.ignoreCause({
1490+
log: true,
1491+
message: "Failed to refresh MCP credential liveness.",
1492+
}),
1493+
);
1494+
yield* touchMcpCredential;
14891495
const appContext = (yield* mcpAppModelContext
14901496
.forThread(input.run.threadId)
14911497
.pipe(
@@ -1550,6 +1556,14 @@ export const layer: Layer.Layer<
15501556
))
15511557
: input.session.startTurn(turnInput);
15521558
yield* Effect.andThen(shouldStart, startTurn).pipe(
1559+
// Wait for acceptance so abandoned startup cannot leave a refresher
1560+
// on the already-detached ingestion fiber.
1561+
Effect.tap(() =>
1562+
Fiber.await(providerEventFiber).pipe(
1563+
Effect.raceFirst(touchMcpCredential.pipe(Effect.delay("1 hour"), Effect.forever)),
1564+
Effect.forkDetach,
1565+
),
1566+
),
15531567
Effect.catchCause((cause) =>
15541568
Effect.logError("orchestration V2 provider turn start failed", {
15551569
runId: input.run.id,

0 commit comments

Comments
 (0)