Skip to content

Commit 5beedc5

Browse files
fix(server): a stuck session close no longer holds up an interrupted open
The cleanup for an interrupted open waited on the session scope's close with no limit, under the session's open lock. An adapter finalizer that never finished held the interrupter, every later open of that session, a worker slot, and layer shutdown. That close and the layer's close of in-flight session scopes now use the same 30s time box as release. A turn start that is stopped before the provider accepts it now undoes its busy mark, so the session can still go idle and be released. The device hub's cleanup now covers its spawn too. The Pi startTurn send bookkeeping is dropped: everything after the permit is synchronous, so no Stop can land between the send and its pending entry, and its tests passed without it. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
1 parent e7741ec commit 5beedc5

5 files changed

Lines changed: 234 additions & 341 deletions

File tree

‎apps/server/src/device/LocalDeviceHost.ts‎

Lines changed: 54 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -358,60 +358,60 @@ export const make = Effect.fn("LocalDeviceHost.make")(function* () {
358358
);
359359
const origin = `http://127.0.0.1:${port}`;
360360
const scope = yield* Scope.make("sequential");
361-
const child = yield* spawner
362-
.spawn(
363-
ChildProcess.make(
364-
nodePath,
365-
[
366-
hubTool.entryPath,
367-
"--port",
368-
String(port),
369-
"--host",
370-
"127.0.0.1",
371-
"--hide-sidebar",
372-
"--hide-boot-device",
373-
],
374-
{
375-
detached: false,
376-
shell: false,
377-
stdout: "pipe",
378-
stderr: "pipe",
379-
env: hubEnvironment(),
380-
},
381-
),
382-
)
383-
.pipe(
384-
Effect.provideService(Scope.Scope, scope),
385-
Effect.mapError(
386-
(cause) =>
387-
new DeviceHost.DeviceHostError({
388-
hostId,
389-
step: "starting the device hub",
390-
cause,
391-
}),
392-
),
393-
);
394-
const startedAtMillis = yield* Clock.currentTimeMillis;
395-
const hub: HubProcess = { child, scope, origin, startedAtMillis, nodePath };
396-
yield* Effect.forkIn(observeHubOutput(hub), scope);
397-
yield* waitForHttpReady({
398-
baseUrl: origin,
399-
path: "/readyz",
400-
timeoutMs: HUB_READY_TIMEOUT_MS,
401-
makeError: (info) =>
402-
new DeviceHost.DeviceHostError({
403-
hostId,
404-
step: "waiting for the device hub to answer",
405-
cause: info.cause,
406-
}),
407-
}).pipe(
408-
Effect.provideService(HttpClient.HttpClient, httpClient),
409-
// On interrupt too: the hub is not recorded yet, so nothing else stops it.
410-
Effect.onError(() => stopHub(hub)),
411-
);
412-
yield* recordHub(hub, hubTool);
413-
yield* Effect.logInfo("Device hub started", { pid: Number(child.pid), port });
414-
return hub;
361+
// On failure or interrupt, from the spawn on: the hub is not recorded yet,
362+
// so nothing else stops it.
363+
return yield* Effect.gen(function* () {
364+
const child = yield* spawner
365+
.spawn(
366+
ChildProcess.make(
367+
nodePath,
368+
[
369+
hubTool.entryPath,
370+
"--port",
371+
String(port),
372+
"--host",
373+
"127.0.0.1",
374+
"--hide-sidebar",
375+
"--hide-boot-device",
376+
],
377+
{
378+
detached: false,
379+
shell: false,
380+
stdout: "pipe",
381+
stderr: "pipe",
382+
env: hubEnvironment(),
383+
},
384+
),
385+
)
386+
.pipe(
387+
Effect.provideService(Scope.Scope, scope),
388+
Effect.mapError(
389+
(cause) =>
390+
new DeviceHost.DeviceHostError({
391+
hostId,
392+
step: "starting the device hub",
393+
cause,
394+
}),
395+
),
396+
);
397+
const startedAtMillis = yield* Clock.currentTimeMillis;
398+
const hub: HubProcess = { child, scope, origin, startedAtMillis, nodePath };
399+
yield* Effect.forkIn(observeHubOutput(hub), scope);
400+
yield* waitForHttpReady({
401+
baseUrl: origin,
402+
path: "/readyz",
403+
timeoutMs: HUB_READY_TIMEOUT_MS,
404+
makeError: (info) =>
405+
new DeviceHost.DeviceHostError({
406+
hostId,
407+
step: "waiting for the device hub to answer",
408+
cause: info.cause,
409+
}),
410+
}).pipe(Effect.provideService(HttpClient.HttpClient, httpClient));
411+
yield* recordHub(hub, hubTool);
412+
yield* Effect.logInfo("Device hub started", { pid: Number(child.pid), port });
413+
return hub;
414+
}).pipe(Effect.onError(() => Scope.close(scope, Exit.void).pipe(Effect.ignore)));
415415
});
416416

417417
const observeHubOutput = (hub: HubProcess) =>

‎apps/server/src/orchestration-v2/Adapters/PiAdapterV2.test.ts‎

Lines changed: 1 addition & 185 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,6 @@ import * as Cause from "effect/Cause";
2020
import * as DateTime from "effect/DateTime";
2121
import * as Duration from "effect/Duration";
2222
import * as Effect from "effect/Effect";
23-
import * as Exit from "effect/Exit";
2423
import * as Fiber from "effect/Fiber";
2524
import * as FileSystem from "effect/FileSystem";
2625
import * as Layer from "effect/Layer";
@@ -89,10 +88,7 @@ interface FakePi {
8988
readonly resolveDeferredState: (data: unknown) => Effect.Effect<void>;
9089
/** Reject the next `get_state` request. */
9190
readonly failNextState: () => void;
92-
/** Leave the next request of this type unanswered. */
93-
readonly deferNextLifecycle: (
94-
type: "switch_session" | "new_session" | "set_session_name",
95-
) => void;
91+
readonly deferNextLifecycle: (type: "switch_session" | "new_session") => void;
9692
readonly queueModels: (models: ReadonlyArray<unknown>) => void;
9793
readonly vetoNextNewSession: () => void;
9894
/** Every request received by the fake process. */
@@ -1187,186 +1183,6 @@ describe("PiAdapterV2", () => {
11871183
}).pipe(Effect.scoped, Effect.provide(testLayer)),
11881184
);
11891185

1190-
it.effect("leaves no turn behind when a start is interrupted before its prompt", () =>
1191-
Effect.gen(function* () {
1192-
const fake = yield* makeFakePi;
1193-
const { runtime, takeEvent } = yield* openRuntime(fake);
1194-
const providerThread = yield* runtime.ensureThread({
1195-
threadId: THREAD_ID,
1196-
modelSelection: modelSelection("default"),
1197-
runtimePolicy,
1198-
});
1199-
// The start waits on naming the session before it sends the prompt.
1200-
fake.deferNextLifecycle("set_session_name");
1201-
const starting = yield* startTurn(runtime, providerThread).pipe(Effect.forkChild);
1202-
yield* fake.takeRequest("set_session_name");
1203-
const interrupted = yield* Fiber.interrupt(starting).pipe(
1204-
Effect.andThen(Fiber.await(starting)),
1205-
);
1206-
assert.isTrue(Exit.isFailure(interrupted) && Cause.hasInterruptsOnly(interrupted.cause));
1207-
1208-
// Nothing is left installed, so the next turn starts and completes.
1209-
yield* startTurn(runtime, providerThread, "default", [], "Second turn", undefined, 2);
1210-
yield* fake.takeRequest("prompt");
1211-
yield* fake.emit({ type: "agent_start" });
1212-
yield* fake.emit({ type: "agent_settled" });
1213-
const terminal = yield* takeEvent((event) => event.type === "turn.terminal");
1214-
assert.isTrue(terminal.type === "turn.terminal" && terminal.status === "completed");
1215-
// Only the second turn's prompt ever reached Pi.
1216-
assert.deepStrictEqual(
1217-
fake
1218-
.allRequests()
1219-
.filter((request) => request["type"] === "prompt")
1220-
.map((request) => request["message"]),
1221-
["Second turn"],
1222-
);
1223-
}).pipe(Effect.scoped, Effect.provide(testLayer)),
1224-
);
1225-
1226-
it.effect("keeps a turn whose start is interrupted after its prompt is sent", () =>
1227-
Effect.gen(function* () {
1228-
const fake = yield* makeFakePi;
1229-
const { runtime, takeEvent } = yield* openRuntime(fake);
1230-
const providerThread = yield* runtime.ensureThread({
1231-
threadId: THREAD_ID,
1232-
modelSelection: modelSelection("default"),
1233-
runtimePolicy,
1234-
});
1235-
const appThread = yield* makeAppThread("default");
1236-
const runId = RunId.make(`run:${THREAD_ID}:1`);
1237-
// The install block first reads `runOrdinal` after it sent the prompt.
1238-
// Interrupting the starting fiber there is a Stop that lands once Pi is
1239-
// already running the prompt, with no wait to race. Later reads (the
1240-
// event pump settling the turn) are left alone.
1241-
let interruptOnRead = true;
1242-
const turnInput = {
1243-
appThread,
1244-
threadId: THREAD_ID,
1245-
runId,
1246-
get runOrdinal() {
1247-
if (interruptOnRead) {
1248-
interruptOnRead = false;
1249-
Fiber.getCurrent()?.interruptUnsafe();
1250-
}
1251-
return 1;
1252-
},
1253-
providerTurnOrdinal: 1,
1254-
attemptId: RunAttemptId.make(`run-attempt:${runId}:1`),
1255-
rootNodeId: NodeId.make(`node:${runId}:root`),
1256-
providerThread,
1257-
message: {
1258-
messageId: `message:${THREAD_ID}:1` as never,
1259-
text: "Hello pi",
1260-
attachments: [],
1261-
createdBy: "user" as const,
1262-
creationSource: "web" as const,
1263-
},
1264-
modelSelection: modelSelection("default"),
1265-
runtimePolicy,
1266-
};
1267-
const starting = yield* runtime.startTurn(turnInput).pipe(Effect.forkChild);
1268-
const interrupted = yield* Fiber.await(starting);
1269-
assert.isTrue(Exit.isFailure(interrupted) && Cause.hasInterruptsOnly(interrupted.cause));
1270-
yield* fake.takeRequest("prompt");
1271-
const running = yield* takeEvent(
1272-
(event) =>
1273-
event.type === "provider_turn.updated" && event.providerTurn.status === "running",
1274-
);
1275-
const providerTurnId =
1276-
running.type === "provider_turn.updated" ? running.providerTurn.id : undefined;
1277-
assert.isDefined(providerTurnId);
1278-
1279-
// Pi is running that prompt, so the turn still owns the thread.
1280-
const rejected = yield* startTurn(
1281-
runtime,
1282-
providerThread,
1283-
"default",
1284-
[],
1285-
"Second turn",
1286-
undefined,
1287-
2,
1288-
).pipe(Effect.flip);
1289-
assert.include(String(rejected.cause), "already has an active turn");
1290-
1291-
// Its events stay its own, and Stop settles it as interrupted.
1292-
yield* fake.emit({ type: "agent_start" });
1293-
yield* runtime.interruptTurn({ providerThread, providerTurnId: providerTurnId! });
1294-
yield* fake.takeRequest("abort");
1295-
yield* fake.emit({ type: "agent_settled" });
1296-
const terminal = yield* takeEvent((event) => event.type === "turn.terminal");
1297-
assert.isTrue(terminal.type === "turn.terminal" && terminal.status === "interrupted");
1298-
}).pipe(Effect.scoped, Effect.provide(testLayer)),
1299-
);
1300-
1301-
it.effect("fails a kept turn when Pi rejects the prompt it was interrupted after sending", () =>
1302-
Effect.gen(function* () {
1303-
const fake = yield* makeFakePi;
1304-
const { runtime, takeEvent } = yield* openRuntime(fake);
1305-
const providerThread = yield* runtime.ensureThread({
1306-
threadId: THREAD_ID,
1307-
modelSelection: modelSelection("default"),
1308-
runtimePolicy,
1309-
});
1310-
const appThread = yield* makeAppThread("default");
1311-
const runId = RunId.make(`run:${THREAD_ID}:1`);
1312-
// The install block first reads `runOrdinal` after it sent the prompt.
1313-
// Interrupting the starting fiber there is a Stop that lands once Pi is
1314-
// already running the prompt, with no wait to race. Later reads (the
1315-
// event pump settling the turn) are left alone.
1316-
let interruptOnRead = true;
1317-
const turnInput = {
1318-
appThread,
1319-
threadId: THREAD_ID,
1320-
runId,
1321-
get runOrdinal() {
1322-
if (interruptOnRead) {
1323-
interruptOnRead = false;
1324-
Fiber.getCurrent()?.interruptUnsafe();
1325-
}
1326-
return 1;
1327-
},
1328-
providerTurnOrdinal: 1,
1329-
attemptId: RunAttemptId.make(`run-attempt:${runId}:1`),
1330-
rootNodeId: NodeId.make(`node:${runId}:root`),
1331-
providerThread,
1332-
message: {
1333-
messageId: `message:${THREAD_ID}:1` as never,
1334-
text: "Hello pi",
1335-
attachments: [],
1336-
createdBy: "user" as const,
1337-
creationSource: "web" as const,
1338-
},
1339-
modelSelection: modelSelection("default"),
1340-
runtimePolicy,
1341-
};
1342-
const starting = yield* runtime.startTurn(turnInput).pipe(Effect.forkChild);
1343-
const interrupted = yield* Fiber.await(starting);
1344-
assert.isTrue(Exit.isFailure(interrupted) && Cause.hasInterruptsOnly(interrupted.cause));
1345-
yield* fake.takeRequest("prompt");
1346-
const running = yield* takeEvent(
1347-
(event) =>
1348-
event.type === "provider_turn.updated" && event.providerTurn.status === "running",
1349-
);
1350-
const providerTurnId =
1351-
running.type === "provider_turn.updated" ? running.providerTurn.id : undefined;
1352-
assert.isDefined(providerTurnId);
1353-
1354-
// The kept turn still has the pending response that settles or fails it.
1355-
yield* fake.emit({
1356-
type: "response",
1357-
command: "prompt",
1358-
success: false,
1359-
error: "Prompt rejected",
1360-
});
1361-
const terminal = yield* takeEvent((event) => event.type === "turn.terminal");
1362-
assert.isTrue(
1363-
terminal.type === "turn.terminal" &&
1364-
terminal.status === "failed" &&
1365-
terminal.providerTurnId === providerTurnId,
1366-
);
1367-
}).pipe(Effect.scoped, Effect.provide(testLayer)),
1368-
);
1369-
13701186
it.effect("leaves /compacted as an ordinary prompt", () =>
13711187
Effect.gen(function* () {
13721188
const fake = yield* makeFakePi;

‎apps/server/src/orchestration-v2/Adapters/PiAdapterV2.ts‎

Lines changed: 15 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -2342,48 +2342,28 @@ export function makePiAdapterV2(
23422342
activeProviderRetry: null,
23432343
failure: null,
23442344
};
2345-
// Set together with the send and its pending-response entry, with
2346-
// no gap for an interrupt to land in: a sent turn always has the
2347-
// entry that settles or fails it.
2348-
let sent = false;
23492345
// Only the install/send/start-event boundary excludes the event
23502346
// pump. Earlier correlated requests must leave the pump free so
23512347
// project trust, login, and session-switch dialogs can be shown
23522348
// and answered instead of deadlocking the caller.
23532349
yield* Effect.gen(function* () {
23542350
state.activeTurn = activeTurn;
23552351
if (compactCommand !== null) {
2356-
yield* connection.send(compactRpcRecord(compactCommand)).pipe(
2357-
Effect.tap(() =>
2358-
Effect.sync(() => {
2359-
pendingCompactResponses.push({
2360-
providerTurnId: providerTurn.id,
2361-
kind: "turn_start",
2362-
});
2363-
sent = true;
2364-
}),
2365-
),
2366-
Effect.uninterruptible,
2367-
);
2352+
yield* connection.send(compactRpcRecord(compactCommand));
2353+
pendingCompactResponses.push({
2354+
providerTurnId: providerTurn.id,
2355+
kind: "turn_start",
2356+
});
23682357
} else if (payload !== null) {
2369-
yield* connection
2370-
.send({
2371-
type: "prompt",
2372-
message: payload.message,
2373-
...(payload.images.length === 0 ? {} : { images: payload.images }),
2374-
})
2375-
.pipe(
2376-
Effect.tap(() =>
2377-
Effect.sync(() => {
2378-
pendingPromptResponses.push({
2379-
providerTurnId: providerTurn.id,
2380-
kind: "turn_start",
2381-
});
2382-
sent = true;
2383-
}),
2384-
),
2385-
Effect.uninterruptible,
2386-
);
2358+
yield* connection.send({
2359+
type: "prompt",
2360+
message: payload.message,
2361+
...(payload.images.length === 0 ? {} : { images: payload.images }),
2362+
});
2363+
pendingPromptResponses.push({
2364+
providerTurnId: providerTurn.id,
2365+
kind: "turn_start",
2366+
});
23872367
}
23882368
yield* emit({
23892369
type: "provider_turn.updated",
@@ -2402,13 +2382,8 @@ export function makePiAdapterV2(
24022382
}
24032383
}).pipe(
24042384
sessionEventPermit.withPermits(1),
2405-
// A turn interrupted before its prompt went out is cleared, or
2406-
// every later turn is rejected as active. Once the prompt is out,
2407-
// Pi is running it: the turn stays installed so its events stay
2408-
// its own, and Stop (interruptTurn) or settlement ends it.
2409-
Effect.onError((cause) =>
2385+
Effect.tapError(() =>
24102386
Effect.sync(() => {
2411-
if (sent && Cause.hasInterruptsOnly(cause)) return;
24122387
if (state.activeTurn === activeTurn) state.activeTurn = null;
24132388
}),
24142389
),

0 commit comments

Comments
 (0)