Skip to content

Commit a3d73c9

Browse files
fix(server): settle Claude background agents when their process exits (pingdotgg#19464)
Co-authored-by: Julius Marminge <[email protected]>
1 parent 3279826 commit a3d73c9

4 files changed

Lines changed: 178 additions & 73 deletions

File tree

‎apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts‎

Lines changed: 71 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -9292,59 +9292,84 @@ describe("ClaudeAdapterV2 background wake turns", () => {
92929292
),
92939293
);
92949294

9295-
it.effect("ends a subagent's open call when the CLI exits while the root turn is idle", () =>
9296-
Effect.scoped(
9297-
Effect.gen(function* () {
9298-
const TOOL_USE_ID = "toolu-idle-exit";
9299-
const BASH = "toolu-idle-exit-bash";
9300-
const harness = yield* makeWakeHarness;
9301-
const now = yield* DateTime.now;
9302-
const bashStatuses = () =>
9303-
harness.events.flatMap((event) =>
9304-
event.type === "turn_item.updated" && event.turnItem.nativeItemRef?.nativeId === BASH
9305-
? [`${event.turnItem.type}:${event.turnItem.status}`]
9306-
: [],
9295+
it.effect.each(["exit", "Stop"] as const)(
9296+
"settles a plain background subagent and its open call on %s after the root turn completes",
9297+
(reason) =>
9298+
Effect.scoped(
9299+
Effect.gen(function* () {
9300+
const TOOL_USE_ID = "toolu-idle-exit";
9301+
const BASH = "toolu-idle-exit-bash";
9302+
const harness = yield* makeWakeHarnessWithOptions({ close: Queue.shutdown });
9303+
const now = yield* DateTime.now;
9304+
const bashStatuses = () =>
9305+
harness.events.flatMap((event) =>
9306+
event.type === "turn_item.updated" && event.turnItem.nativeItemRef?.nativeId === BASH
9307+
? [`${event.turnItem.type}:${event.turnItem.status}`]
9308+
: [],
9309+
);
9310+
yield* harness.runtime.startTurn(
9311+
makeClaudeTestTurnInput({
9312+
threadId: harness.threadId,
9313+
providerThread: harness.providerThread,
9314+
now,
9315+
attemptId: RunAttemptId.make("attempt-claude-idle-exit"),
9316+
text: "Audit in the background.",
9317+
attachments: [],
9318+
}),
93079319
);
9308-
yield* harness.runtime.startTurn(
9309-
makeClaudeTestTurnInput({
9310-
threadId: harness.threadId,
9311-
providerThread: harness.providerThread,
9312-
now,
9313-
attemptId: RunAttemptId.make("attempt-claude-idle-exit"),
9314-
text: "Audit in the background.",
9315-
attachments: [],
9316-
}),
9317-
);
9318-
for (const frame of [
9319-
makeSubagentTaskStartedFrame({
9320-
taskId: "task-idle-exit",
9321-
toolUseId: TOOL_USE_ID,
9322-
uuid: "00000000-0000-4000-8000-000000000981",
9323-
}),
9324-
...makeSubagentAssistantFrames({
9320+
yield* harness.offerAndWait(
9321+
makeSubagentTaskStartedFrame({
9322+
taskId: "task-idle-exit",
9323+
toolUseId: TOOL_USE_ID,
9324+
uuid: "00000000-0000-4000-8000-000000000981",
9325+
}),
9326+
);
9327+
const started = yield* Queue.take(harness.subagentReceipts);
9328+
assert.equal(started.subagent.status, "running");
9329+
for (const frame of makeSubagentAssistantFrames({
93259330
parentToolUseId: TOOL_USE_ID,
93269331
uuid: "00000000-0000-4000-8000-000000000982",
93279332
bashToolUseId: BASH,
9328-
}),
9329-
makeResultFrame({
9330-
uuid: "00000000-0000-4000-8000-000000000983",
9331-
result: "Auditing in the background.",
9332-
}),
9333-
]) {
9334-
yield* Queue.offer(harness.sdkMessages, frame);
9335-
}
9336-
yield* awaitUntil(() => harness.terminalEvents().length === 1, "launch turn terminal");
9337-
assert.deepEqual(bashStatuses(), ["command_execution:running"]);
9333+
})) {
9334+
yield* harness.offerAndWait(frame);
9335+
}
9336+
yield* harness.offerAndWait(
9337+
makeResultFrame({
9338+
uuid: "00000000-0000-4000-8000-000000000983",
9339+
result: "Auditing in the background.",
9340+
}),
9341+
);
9342+
const terminal = yield* Queue.take(harness.terminalReceipts);
9343+
assert.equal(terminal.status, "completed");
9344+
assert.isTrue(yield* harness.hasPendingBackgroundWork);
9345+
assert.deepEqual(bashStatuses(), ["command_execution:running"]);
93389346

9339-
yield* Queue.shutdown(harness.sdkMessages);
9340-
yield* awaitUntil(() => bashStatuses().length === 2, "call ended");
9341-
assert.deepEqual(bashStatuses(), ["command_execution:running", "command_execution:failed"]);
9342-
}).pipe(
9343-
Effect.provide(
9344-
Layer.mergeAll(IdAllocator.layer, McpProviderSessions.layer, NodeServices.layer),
9347+
if (reason === "Stop") {
9348+
yield* harness.runtime.interruptTurn({
9349+
providerThread: harness.providerThread,
9350+
providerTurnId: terminal.providerTurnId,
9351+
requestRuntimeRestart: true,
9352+
});
9353+
} else {
9354+
yield* Queue.shutdown(harness.sdkMessages);
9355+
}
9356+
let ended = yield* Queue.take(harness.subagentReceipts);
9357+
while (ended.subagent.status === "running") {
9358+
ended = yield* Queue.take(harness.subagentReceipts);
9359+
}
9360+
assert.equal(ended.subagent.nativeTaskRef?.nativeId, "task-idle-exit");
9361+
assert.equal(ended.subagent.status, reason === "Stop" ? "cancelled" : "failed");
9362+
assert.isFalse(yield* harness.hasPendingBackgroundWork);
9363+
assert.deepEqual(bashStatuses(), [
9364+
"command_execution:running",
9365+
reason === "Stop" ? "command_execution:interrupted" : "command_execution:failed",
9366+
]);
9367+
}).pipe(
9368+
Effect.provide(
9369+
Layer.mergeAll(IdAllocator.layer, McpProviderSessions.layer, NodeServices.layer),
9370+
),
93459371
),
93469372
),
9347-
),
93489373
);
93499374

93509375
it.effect("ends a subagent's open call when Stop's close of the CLI times out", () =>

‎apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6396,11 +6396,12 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* (
63966396
cause?: Cause.Cause<ClaudeAgentSdkQueryRunnerError>,
63976397
) {
63986398
const current = yield* Ref.get(activeTurn);
6399-
const context =
6400-
current?.nativeThreadId === closedQuery.nativeThreadId ? current : closedQuery.lastTurn;
6399+
// A replacement query can reuse the native thread id; only this query's turn is ours.
6400+
const context = closedQuery.lastTurn;
64016401
if (context === null) return;
64026402
const interrupted =
64036403
closedQuery.stopping || (yield* Ref.get(interruptedTurns)).has(context.providerTurnId);
6404+
// Finalization can yield to a replacement process; capture ownership and outcomes first.
64046405
const subagents = yield* Ref.get(sessionSubagentsByTaskId);
64056406
const buffered = (yield* Ref.get(wakeBuffers)).get(closedQuery.nativeThreadId)?.messages;
64066407
if (current === context) {
@@ -6420,9 +6421,9 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* (
64206421
} else {
64216422
yield* resetBackgroundTaskStateForNativeThreadProcess(closedQuery.nativeThreadId);
64226423
}
6424+
// Preserve buffered outcomes before Stop clears the wake buffer.
64236425
for (const [taskId, subagent] of subagents) {
64246426
if (
6425-
subagent.task.workflow === undefined ||
64266427
subagent.task.status !== "running" ||
64276428
closedQuery.subagentsFromEarlierProcesses.has(subagent)
64286429
)

‎apps/server/src/orchestration-v2/testkit/OrchestratorReplayRecovery.integration.test.ts‎

Lines changed: 47 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import * as FileSystem from "effect/FileSystem";
55
import * as Layer from "effect/Layer";
66
import * as Path from "effect/Path";
77
import * as Schema from "effect/Schema";
8+
import * as SqlClient from "effect/sql/SqlClient";
89
import * as CodexReplay from "effect-codex-app-server/replay";
910
import {
1011
type OrchestrationV2ThreadProjection,
@@ -25,6 +26,7 @@ import {
2526
} from "../Adapters/CursorAdapterV2.testkit.ts";
2627
import * as CursorAdapterV2Testkit from "../Adapters/CursorAdapterV2.testkit.ts";
2728
import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator";
29+
import * as EffectWorker from "../EffectWorker.ts";
2830
import * as SqlitePersistence from "../../persistence/Sqlite.ts";
2931
import { provideDeterministicTestRuntime } from "./DeterministicRuntime.ts";
3032
import {
@@ -46,8 +48,14 @@ import {
4648
assertUserMessagesInclude,
4749
projectionFor,
4850
} from "./fixtures/shared.ts";
49-
import type { OrchestratorV2ScenarioResult } from "./OrchestratorScenario.ts";
50-
import { runOrchestratorV2ProviderReplayScenario } from "./ProviderReplayHarness.ts";
51+
import {
52+
runOrchestratorV2Scenario,
53+
type OrchestratorV2ScenarioResult,
54+
} from "./OrchestratorScenario.ts";
55+
import {
56+
layerProviderReplay,
57+
runOrchestratorV2ProviderReplayScenario,
58+
} from "./ProviderReplayHarness.ts";
5159
import { checkpointWorkspace } from "@t3tools/provider-testing/replayWorkspace";
5260
import { materializeReplayTranscriptRuntimeInstructions } from "./ReplayRuntimeInstructions.ts";
5361
import {
@@ -111,11 +119,15 @@ const readClaudeCancelledSubagentResumeTranscript = Effect.fn(
111119
const firstOutcome = indices.find(
112120
(index) => frameField(index, "subtype") === "task_notification",
113121
);
114-
if (firstSubagentFrame === undefined || firstOutcome === undefined) {
122+
const firstExit = recorded.entries.findIndex((entry) => entry.type === "runtime_exit");
123+
if (firstSubagentFrame === undefined || firstOutcome === undefined || firstExit < 0) {
115124
throw new Error("Expected the recorded subagent's first run.");
116125
}
117126
const entries = recorded.entries
118-
.filter((_, index) => index < firstSubagentFrame || index > firstOutcome)
127+
// A CLI exit settles the task; hold at query.open:2 until the crash snapshot instead.
128+
.filter(
129+
(_, index) => (index < firstSubagentFrame || index > firstOutcome) && index !== firstExit,
130+
)
119131
.map((entry) => {
120132
if (entry.type !== "expect_outbound" || entry.label !== "prompt.offer:2") return entry;
121133
const frame = entry.frame as {
@@ -524,18 +536,31 @@ describe("orchestrator replay recovery", () => {
524536
const childRootStatus = (child: OrchestrationV2ThreadProjection | undefined) =>
525537
child?.nodes.find((node) => node.kind === "root_turn" && node.runId === null)?.status;
526538

539+
const snapshotPath = path.join(tempDir, "restart-state.sqlite");
540+
const beforeScenario = {
541+
name: `${transcript.scenario}:cancelled-first-runtime`,
542+
transcript,
543+
commands: phase1Commands,
544+
steps: phase1Steps,
545+
projectionThreadIds: materialized.projectionThreadIds,
546+
runtimePolicyOverride: { cwd: workspace },
547+
};
527548
const before = yield* Effect.scoped(
528-
runOrchestratorV2ProviderReplayScenario(
529-
{
530-
name: `${transcript.scenario}:cancelled-first-runtime`,
531-
transcript,
532-
commands: phase1Commands,
533-
steps: phase1Steps,
534-
projectionThreadIds: materialized.projectionThreadIds,
535-
runtimePolicyOverride: { cwd: workspace },
536-
},
537-
harness,
538-
options,
549+
Effect.gen(function* () {
550+
const result = yield* runOrchestratorV2Scenario(beforeScenario);
551+
const worker = yield* EffectWorker.OrchestrationEffectWorkerV2;
552+
yield* worker.drain();
553+
// Preserve the orphan before scoped teardown settles its provider work.
554+
const sql = yield* SqlClient.SqlClient;
555+
yield* sql`VACUUM INTO ${snapshotPath}`;
556+
return result;
557+
}).pipe(
558+
Effect.provide(
559+
Layer.merge(
560+
layerProviderReplay(beforeScenario, harness, options),
561+
options.databaseLayer,
562+
),
563+
),
539564
),
540565
);
541566
const settled = projectionFor(before, transcript.scenario);
@@ -555,7 +580,13 @@ describe("orchestrator replay recovery", () => {
555580
runtimePolicyOverride: { cwd: workspace },
556581
},
557582
harness,
558-
{ ...options, recoverOnStartup: true, continueThreadsAfterServerUpdate: false },
583+
{
584+
databaseLayer: SqlitePersistence.layerFromPath(snapshotPath).pipe(
585+
Layer.provide(NodeServices.layer),
586+
),
587+
recoverOnStartup: true,
588+
continueThreadsAfterServerUpdate: false,
589+
},
559590
),
560591
);
561592
yield* assertComplete;

‎apps/server/src/orchestration-v2/testkit/OrchestratorReplayRestartBackgroundNote.integration.test.ts‎

Lines changed: 56 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,21 @@
11
import * as NodeServices from "@effect/platform-node/NodeServices";
22
import { assert, describe, it } from "@effect/vitest";
3-
import { ProviderDriverKind, type ProviderReplayEntry } from "@t3tools/contracts";
3+
import { EventId, ProviderDriverKind, type ProviderReplayEntry } from "@t3tools/contracts";
4+
import * as DateTime from "effect/DateTime";
45
import * as Effect from "effect/Effect";
56
import * as FileSystem from "effect/FileSystem";
67
import * as Layer from "effect/Layer";
78
import * as Path from "effect/Path";
9+
import * as SqlClient from "effect/sql/SqlClient";
810

911
import {
1012
ClaudeOrchestratorReplayHarness,
1113
makeClaudeRestartReplayHarness,
1214
} from "../Adapters/ClaudeAdapterV2.testkit.ts";
1315
import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator";
1416
import * as EffectWorker from "../EffectWorker.ts";
17+
import * as EventSink from "../EventSink.ts";
18+
import * as Orchestrator from "../Orchestrator.ts";
1519
import * as SqlitePersistence from "../../persistence/Sqlite.ts";
1620
import { provideDeterministicTestRuntime } from "./DeterministicRuntime.ts";
1721
import { CLAUDE_BACKGROUND_SUBAGENT_AFTER_ROOT_PROMPT } from "./fixtures/claude_background_subagent_after_root/input.ts";
@@ -113,8 +117,7 @@ const readRestartTranscript = Effect.fn("readRestartTranscript")(function* (
113117
...recorded,
114118
entries: [
115119
...recorded.entries.slice(0, rootResult + 1),
116-
117-
{ type: "runtime_exit", status: "success" },
120+
// Keep the query alive until the crash snapshot is captured.
118121
{
119122
type: "expect_outbound",
120123
label: "query.open:resume",
@@ -157,6 +160,7 @@ const runRestart = Effect.fn("runRestart")(function* (input: {
157160
const layerDatabase = SqlitePersistence.layerFromPath(path.join(tempDir, "state.sqlite")).pipe(
158161
Layer.provide(NodeServices.layer),
159162
);
163+
const snapshotPath = path.join(tempDir, "restart-state.sqlite");
160164
const scenario = (name: string, steps: typeof materialized.steps) => ({
161165
name: `${SCENARIO}:${name}`,
162166
transcript,
@@ -166,10 +170,54 @@ const runRestart = Effect.fn("runRestart")(function* (input: {
166170
runtimePolicyOverride: { cwd: workspace },
167171
});
168172

173+
const beforeScenario = scenario("before-restart", phase1Steps);
169174
const before = yield* Effect.scoped(
170-
runOrchestratorV2ProviderReplayScenario(scenario("before-restart", phase1Steps), harness, {
171-
databaseLayer: layerDatabase,
172-
}),
175+
Effect.gen(function* () {
176+
const result = yield* runOrchestratorV2Scenario(beforeScenario);
177+
const worker = yield* EffectWorker.OrchestrationEffectWorkerV2;
178+
yield* worker.drain();
179+
const orchestrator = yield* Orchestrator.OrchestratorV2;
180+
const eventSink = yield* EventSink.EventSinkV2;
181+
const threadId = materialized.projectionThreadIds[0];
182+
if (threadId === undefined) {
183+
return yield* Effect.die("restart scenario has no projection thread");
184+
}
185+
const projection = yield* orchestrator.getThreadProjection(threadId);
186+
const session = projection.providerSessions[0];
187+
if (session === undefined) {
188+
return yield* Effect.die("restart scenario has no provider session");
189+
}
190+
// Snapshot the legacy running task before exit cleanup settles it, then
191+
// seed the stopped session that older runtimes could leave after a crash.
192+
const now = yield* DateTime.now;
193+
yield* eventSink.write({
194+
events: [
195+
{
196+
id: EventId.make("event:restart-background-note:stopped-provider-session"),
197+
type: "provider-session.updated",
198+
threadId,
199+
providerInstanceId: session.providerInstanceId,
200+
occurredAt: now,
201+
payload: { ...session, status: "stopped", updatedAt: now, lastError: null },
202+
},
203+
],
204+
});
205+
const sql = yield* SqlClient.SqlClient;
206+
yield* sql`VACUUM INTO ${snapshotPath}`;
207+
return result;
208+
}).pipe(
209+
Effect.provide(
210+
Layer.merge(
211+
ProviderReplayHarness.layerProviderReplay(beforeScenario, harness, {
212+
databaseLayer: layerDatabase,
213+
}),
214+
layerDatabase,
215+
),
216+
),
217+
),
218+
);
219+
const snapshotDatabase = SqlitePersistence.layerFromPath(snapshotPath).pipe(
220+
Layer.provide(NodeServices.layer),
173221
);
174222
const settled = projectionFor(before, SCENARIO);
175223
assert.equal(settled.runs[0]?.status, "completed");
@@ -184,7 +232,7 @@ const runRestart = Effect.fn("runRestart")(function* (input: {
184232
}).pipe(
185233
Effect.provide(
186234
ProviderReplayHarness.layerProviderReplay(restartScenario, harness, {
187-
databaseLayer: layerDatabase,
235+
databaseLayer: snapshotDatabase,
188236
recoverOnStartup: true,
189237
continueThreadsAfterServerUpdate: input.continueThreadsAfterServerUpdate,
190238
}),
@@ -201,7 +249,7 @@ const runRestart = Effect.fn("runRestart")(function* (input: {
201249

202250
const after = yield* Effect.scoped(
203251
runOrchestratorV2ProviderReplayScenario(scenario("after-restart", phase2Steps), harness, {
204-
databaseLayer: layerDatabase,
252+
databaseLayer: snapshotDatabase,
205253
continueThreadsAfterServerUpdate: input.continueThreadsAfterServerUpdate,
206254
}),
207255
);

0 commit comments

Comments
 (0)