Skip to content

Commit be865b2

Browse files
fix(server): keep a Pi turn whose prompt was sent, clean up sessions before a stuck close
Pi: an interrupt that lands after startTurn sent the prompt no longer clears the turn. Pi is already running that prompt, so the turn stays installed and keeps its events; Stop (interruptTurn) or settlement ends it. A turn interrupted before its prompt went out is still cleared, and a typed failure still clears it as before. The send and the record that it happened are uninterruptible together. ProviderSessionManager: an interrupted or failed open now drops its reservation and clears the session it set up before closing the session scope, so an adapter finalizer that never finishes cannot hold that cleanup up. The scope close still runs afterwards. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
1 parent aade036 commit be865b2

4 files changed

Lines changed: 177 additions & 31 deletions

File tree

‎apps/server/src/orchestration-v2/Adapters/PiAdapterV2.test.ts‎

Lines changed: 73 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -89,7 +89,10 @@ interface FakePi {
8989
readonly resolveDeferredState: (data: unknown) => Effect.Effect<void>;
9090
/** Reject the next `get_state` request. */
9191
readonly failNextState: () => void;
92-
readonly deferNextLifecycle: (type: "switch_session" | "new_session") => void;
92+
/** Leave the next request of this type unanswered. */
93+
readonly deferNextLifecycle: (
94+
type: "switch_session" | "new_session" | "set_session_name",
95+
) => void;
9396
readonly queueModels: (models: ReadonlyArray<unknown>) => void;
9497
readonly vetoNextNewSession: () => void;
9598
/** Every request received by the fake process. */
@@ -1184,7 +1187,43 @@ describe("PiAdapterV2", () => {
11841187
}).pipe(Effect.scoped, Effect.provide(testLayer)),
11851188
);
11861189

1187-
it.effect("clears a turn whose start is interrupted while it is installed", () =>
1190+
it.effect("leaves no turn behind when a start is interrupted before its prompt", () =>
1191+
Effect.gen(function* () {
1192+
const fake = yield* makeFakePi;
1193+
const { runtime, takeEvent } = yield* openRuntime(fake);
1194+
const providerThread = yield* runtime.ensureThread({
1195+
threadId: THREAD_ID,
1196+
modelSelection: modelSelection("default"),
1197+
runtimePolicy,
1198+
});
1199+
// The start waits on naming the session before it sends the prompt.
1200+
fake.deferNextLifecycle("set_session_name");
1201+
const starting = yield* startTurn(runtime, providerThread).pipe(Effect.forkChild);
1202+
yield* fake.takeRequest("set_session_name");
1203+
const interrupted = yield* Fiber.interrupt(starting).pipe(
1204+
Effect.andThen(Fiber.await(starting)),
1205+
);
1206+
assert.isTrue(Exit.isFailure(interrupted) && Cause.hasInterruptsOnly(interrupted.cause));
1207+
1208+
// Nothing is left installed, so the next turn starts and completes.
1209+
yield* startTurn(runtime, providerThread, "default", [], "Second turn", undefined, 2);
1210+
yield* fake.takeRequest("prompt");
1211+
yield* fake.emit({ type: "agent_start" });
1212+
yield* fake.emit({ type: "agent_settled" });
1213+
const terminal = yield* takeEvent((event) => event.type === "turn.terminal");
1214+
assert.isTrue(terminal.type === "turn.terminal" && terminal.status === "completed");
1215+
// Only the second turn's prompt ever reached Pi.
1216+
assert.deepStrictEqual(
1217+
fake
1218+
.allRequests()
1219+
.filter((request) => request["type"] === "prompt")
1220+
.map((request) => request["message"]),
1221+
["Second turn"],
1222+
);
1223+
}).pipe(Effect.scoped, Effect.provide(testLayer)),
1224+
);
1225+
1226+
it.effect("keeps a turn whose start is interrupted after its prompt is sent", () =>
11881227
Effect.gen(function* () {
11891228
const fake = yield* makeFakePi;
11901229
const { runtime, takeEvent } = yield* openRuntime(fake);
@@ -1195,15 +1234,20 @@ describe("PiAdapterV2", () => {
11951234
});
11961235
const appThread = yield* makeAppThread("default");
11971236
const runId = RunId.make(`run:${THREAD_ID}:1`);
1198-
// The install block reads `runOrdinal` once the turn is installed and its
1199-
// prompt sent. Interrupting the starting fiber there is a Stop that lands
1200-
// between installing the turn and returning, with no wait to race.
1237+
// The install block first reads `runOrdinal` after it sent the prompt.
1238+
// Interrupting the starting fiber there is a Stop that lands once Pi is
1239+
// already running the prompt, with no wait to race. Later reads (the
1240+
// event pump settling the turn) are left alone.
1241+
let interruptOnRead = true;
12011242
const turnInput = {
12021243
appThread,
12031244
threadId: THREAD_ID,
12041245
runId,
12051246
get runOrdinal() {
1206-
Fiber.getCurrent()?.interruptUnsafe();
1247+
if (interruptOnRead) {
1248+
interruptOnRead = false;
1249+
Fiber.getCurrent()?.interruptUnsafe();
1250+
}
12071251
return 1;
12081252
},
12091253
providerTurnOrdinal: 1,
@@ -1224,14 +1268,33 @@ describe("PiAdapterV2", () => {
12241268
const interrupted = yield* Fiber.await(starting);
12251269
assert.isTrue(Exit.isFailure(interrupted) && Cause.hasInterruptsOnly(interrupted.cause));
12261270
yield* fake.takeRequest("prompt");
1271+
const running = yield* takeEvent(
1272+
(event) =>
1273+
event.type === "provider_turn.updated" && event.providerTurn.status === "running",
1274+
);
1275+
const providerTurnId =
1276+
running.type === "provider_turn.updated" ? running.providerTurn.id : undefined;
1277+
assert.isDefined(providerTurnId);
12271278

1228-
// A turn left installed would reject this one as already active.
1229-
yield* startTurn(runtime, providerThread, "default", [], "Second turn", undefined, 2);
1230-
yield* fake.takeRequest("prompt");
1279+
// Pi is running that prompt, so the turn still owns the thread.
1280+
const rejected = yield* startTurn(
1281+
runtime,
1282+
providerThread,
1283+
"default",
1284+
[],
1285+
"Second turn",
1286+
undefined,
1287+
2,
1288+
).pipe(Effect.flip);
1289+
assert.include(String(rejected.cause), "already has an active turn");
1290+
1291+
// Its events stay its own, and Stop settles it as interrupted.
12311292
yield* fake.emit({ type: "agent_start" });
1293+
yield* runtime.interruptTurn({ providerThread, providerTurnId: providerTurnId! });
1294+
yield* fake.takeRequest("abort");
12321295
yield* fake.emit({ type: "agent_settled" });
12331296
const terminal = yield* takeEvent((event) => event.type === "turn.terminal");
1234-
assert.isTrue(terminal.type === "turn.terminal" && terminal.status === "completed");
1297+
assert.isTrue(terminal.type === "turn.terminal" && terminal.status === "interrupted");
12351298
}).pipe(Effect.scoped, Effect.provide(testLayer)),
12361299
);
12371300

‎apps/server/src/orchestration-v2/Adapters/PiAdapterV2.ts‎

Lines changed: 22 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -2342,24 +2342,34 @@ export function makePiAdapterV2(
23422342
activeProviderRetry: null,
23432343
failure: null,
23442344
};
2345+
// Set together with the send, with no gap for an interrupt to land in.
2346+
let sent = false;
23452347
// Only the install/send/start-event boundary excludes the event
23462348
// pump. Earlier correlated requests must leave the pump free so
23472349
// project trust, login, and session-switch dialogs can be shown
23482350
// and answered instead of deadlocking the caller.
23492351
yield* Effect.gen(function* () {
23502352
state.activeTurn = activeTurn;
23512353
if (compactCommand !== null) {
2352-
yield* connection.send(compactRpcRecord(compactCommand));
2354+
yield* connection.send(compactRpcRecord(compactCommand)).pipe(
2355+
Effect.tap(() => Effect.sync(() => (sent = true))),
2356+
Effect.uninterruptible,
2357+
);
23532358
pendingCompactResponses.push({
23542359
providerTurnId: providerTurn.id,
23552360
kind: "turn_start",
23562361
});
23572362
} else if (payload !== null) {
2358-
yield* connection.send({
2359-
type: "prompt",
2360-
message: payload.message,
2361-
...(payload.images.length === 0 ? {} : { images: payload.images }),
2362-
});
2363+
yield* connection
2364+
.send({
2365+
type: "prompt",
2366+
message: payload.message,
2367+
...(payload.images.length === 0 ? {} : { images: payload.images }),
2368+
})
2369+
.pipe(
2370+
Effect.tap(() => Effect.sync(() => (sent = true))),
2371+
Effect.uninterruptible,
2372+
);
23632373
pendingPromptResponses.push({
23642374
providerTurnId: providerTurn.id,
23652375
kind: "turn_start",
@@ -2382,10 +2392,13 @@ export function makePiAdapterV2(
23822392
}
23832393
}).pipe(
23842394
sessionEventPermit.withPermits(1),
2385-
// On interrupt too: a Stop that lands here must not leave the
2386-
// turn installed, or every later turn is rejected as active.
2387-
Effect.onError(() =>
2395+
// A turn interrupted before its prompt went out is cleared, or
2396+
// every later turn is rejected as active. Once the prompt is out,
2397+
// Pi is running it: the turn stays installed so its events stay
2398+
// its own, and Stop (interruptTurn) or settlement ends it.
2399+
Effect.onError((cause) =>
23882400
Effect.sync(() => {
2401+
if (sent && Cause.hasInterruptsOnly(cause)) return;
23892402
if (state.activeTurn === activeTurn) state.activeTurn = null;
23902403
}),
23912404
),

‎apps/server/src/orchestration-v2/ProviderSessionManager.test.ts‎

Lines changed: 74 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -296,6 +296,8 @@ function makeProviderAdapter(
296296
readonly beforeUnload?: Effect.Effect<void>;
297297
/** Registers the process's closeCount finalizer before `beforeOpen` runs. */
298298
readonly spawnBeforeOpen?: boolean;
299+
/** Completed when a hanging scope close reaches its wedged finalizer. */
300+
readonly scopeCloseReached?: Deferred.Deferred<void>;
299301
} = {},
300302
): ProviderAdapterV2Shape {
301303
const countClose = Effect.addFinalizer(() =>
@@ -304,6 +306,15 @@ function makeProviderAdapter(
304306
closeCount: current.closeCount + 1,
305307
})),
306308
);
309+
// Registered after countClose so it runs first on scope close, wedging the
310+
// close before the closeCount finalizer, like a provider process that never
311+
// yields its message stream.
312+
const hangClose = Effect.addFinalizer(() =>
313+
(options.scopeCloseReached === undefined
314+
? Effect.void
315+
: Deferred.succeed(options.scopeCloseReached, undefined)
316+
).pipe(Effect.andThen(Effect.never)),
317+
);
307318
return {
308319
instanceId: ProviderInstanceId.make("codex"),
309320
driver: CODEX_DRIVER,
@@ -313,6 +324,7 @@ function makeProviderAdapter(
313324
Effect.gen(function* () {
314325
if (options.spawnBeforeOpen === true) {
315326
yield* countClose;
327+
if (options.hangSessionScopeClose === true) yield* hangClose;
316328
}
317329
if (options.mcpConfigs !== undefined && options.spawnBeforeOpen === true) {
318330
yield* Ref.update(options.mcpConfigs, (configs) => [
@@ -347,12 +359,7 @@ function makeProviderAdapter(
347359
});
348360
if (options.spawnBeforeOpen !== true) {
349361
yield* countClose;
350-
}
351-
if (options.hangSessionScopeClose === true) {
352-
// Registered last so it runs first on scope close, wedging the
353-
// close before the closeCount finalizer, like a provider process
354-
// that never yields its message stream.
355-
yield* Effect.addFinalizer(() => Effect.never);
362+
if (options.hangSessionScopeClose === true) yield* hangClose;
356363
}
357364

358365
return {
@@ -425,6 +432,7 @@ function makeTestLayer(input: {
425432
readonly hangSessionScopeClose?: boolean;
426433
readonly beforeUnload?: Effect.Effect<void>;
427434
readonly spawnBeforeOpen?: boolean;
435+
readonly scopeCloseReached?: Deferred.Deferred<void>;
428436
readonly serverSettingsLayer?: ReturnType<typeof ServerSettings.layerTest>;
429437
readonly projectServiceLayer?: Layer.Layer<ProjectService.ProjectService>;
430438
}) {
@@ -448,6 +456,9 @@ function makeTestLayer(input: {
448456
: { hangSessionScopeClose: input.hangSessionScopeClose }),
449457
...(input.beforeUnload === undefined ? {} : { beforeUnload: input.beforeUnload }),
450458
...(input.spawnBeforeOpen === undefined ? {} : { spawnBeforeOpen: input.spawnBeforeOpen }),
459+
...(input.scopeCloseReached === undefined
460+
? {}
461+
: { scopeCloseReached: input.scopeCloseReached }),
451462
}),
452463
);
453464
const providerEventIngestorTestLayer = ProviderEventIngestor.layer.pipe(
@@ -989,6 +1000,63 @@ it.effect("ProviderSessionManagerV2 cleans up an open interrupted mid-handshake"
9891000
}),
9901001
);
9911002

1003+
it.effect("ProviderSessionManagerV2 cleans up an interrupted open whose scope close hangs", () =>
1004+
Effect.gen(function* () {
1005+
const state = yield* Ref.make(emptyState);
1006+
const mcpConfigs = yield* Ref.make<
1007+
ReadonlyArray<McpProviderSession.McpProviderSessionConfig | undefined>
1008+
>([]);
1009+
const handshakeStarted = yield* Deferred.make<void>();
1010+
const scopeCloseReached = yield* Deferred.make<void>();
1011+
yield* Effect.gen(function* () {
1012+
const eventSink = yield* EventSink.EventSinkV2;
1013+
const idAllocator = yield* IdAllocator.IdAllocatorV2;
1014+
const manager = yield* ProviderSessionManager.ProviderSessionManagerV2;
1015+
const registry = yield* McpSessionRegistry.McpSessionRegistry;
1016+
const now = yield* DateTime.now;
1017+
const threadId = ThreadId.make("thread-provider-session-manager-interrupted-hung-open");
1018+
const providerSessionId = yield* idAllocator.allocate.providerSession({
1019+
providerInstanceId: modelSelection.instanceId,
1020+
threadId,
1021+
});
1022+
yield* eventSink.write({
1023+
events: [yield* makeThreadCreatedEvent({ idAllocator, threadId, now })],
1024+
});
1025+
1026+
// Detached so the layer's close does not wait on the wedged scope.
1027+
const opening = yield* manager
1028+
.open({ threadId, providerSessionId, modelSelection, runtimePolicy })
1029+
.pipe(Effect.forkDetach);
1030+
yield* Deferred.await(handshakeStarted);
1031+
const issued = (yield* Ref.get(mcpConfigs)).at(-1);
1032+
const token = issued?.authorizationHeader.replace(/^Bearer\s+/, "");
1033+
assert.isDefined(token);
1034+
1035+
// The interrupt starts cleanup; the scope close then never finishes.
1036+
opening.interruptUnsafe();
1037+
yield* Deferred.await(scopeCloseReached);
1038+
1039+
// The session cleanup already ran, ahead of the stuck close.
1040+
assert.isUndefined(yield* registry.resolve(token!));
1041+
assert.isUndefined(McpProviderSession.readMcpProviderSession(threadId));
1042+
assert.equal((yield* Ref.get(state)).closeCount, 0);
1043+
}).pipe(
1044+
Effect.provide(
1045+
makeTestLayer({
1046+
state,
1047+
idleTimeoutMs: 60_000,
1048+
mcpConfigs,
1049+
beforeOpen: () =>
1050+
Deferred.succeed(handshakeStarted, undefined).pipe(Effect.andThen(Effect.never)),
1051+
spawnBeforeOpen: true,
1052+
hangSessionScopeClose: true,
1053+
scopeCloseReached,
1054+
}),
1055+
),
1056+
);
1057+
}),
1058+
);
1059+
9921060
it.effect("ProviderSessionManagerV2 stops a session still opening when its layer shuts down", () =>
9931061
Effect.gen(function* () {
9941062
const state = yield* Ref.make(emptyState);

‎apps/server/src/orchestration-v2/ProviderSessionManager.ts‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1787,18 +1787,20 @@ export const layerWithOptions = (
17871787
Effect.provideService(Scope.Scope, sessionScope),
17881788
// Any failure, including a Stop that interrupts a slow
17891789
// handshake, stops the provider process this open started.
1790+
// The session cleanup runs first, so an adapter finalizer
1791+
// that never finishes cannot hold it up; the process it
1792+
// was set up for is being stopped either way.
17901793
Effect.onError(() =>
1791-
Scope.close(sessionScope, Exit.void).pipe(
1792-
Effect.ignore,
1793-
Effect.andThen(dropReservation),
1794-
// Revoke only a credential this open freshly minted: a
1795-
// reused credential is held by another live provider
1796-
// process and must survive this open's failure.
1794+
dropReservation.pipe(
1795+
// Clear only a session this open freshly set up: a reused
1796+
// one is held by another live provider process and must
1797+
// survive this open's failure.
17971798
Effect.andThen(
17981799
prepared.issued
17991800
? clearMcpSession(input.threadId, mcpCredentialId)
18001801
: Effect.void,
18011802
),
1803+
Effect.ensuring(Scope.close(sessionScope, Exit.void).pipe(Effect.ignore)),
18021804
),
18031805
),
18041806
Effect.mapError(

0 commit comments

Comments
 (0)