Skip to content

Commit b4d3d51

Browse files
fix(server): Claude model changes no longer kill running background agents (#14726)
Co-authored-by: Claude Opus 5.5 (1M context) <[email protected]>
1 parent 8bc40b4 commit b4d3d51

3 files changed

Lines changed: 258 additions & 15 deletions

File tree

‎apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts‎

Lines changed: 184 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,7 @@ import {
6262
type ProviderAdapterV2TurnInput,
6363
} from "../ProviderAdapter.ts";
6464
import type { ProviderContinuationRequest } from "../ProviderContinuationRequests.ts";
65+
import { makeProviderFailure } from "../ProviderFailure.ts";
6566
import * as ClaudeAdapterV2 from "./ClaudeAdapterV2.ts";
6667
import * as IdAllocator from "../IdAllocator.ts";
6768

@@ -6608,12 +6609,25 @@ describe("ClaudeAdapterV2 background wake turns", () => {
66086609
assert.equal(processQueues.length, 1);
66096610
const firstProcess = processQueues[0]!;
66106611
yield* Queue.offer(firstProcess, wakeTaskStarted);
6612+
// The shell leaves the roster before its notification arrives, so
6613+
// nothing runs in this process any more and a model change may
6614+
// replace it. Wake eligibility outlives the empty level.
6615+
yield* Queue.offer(
6616+
firstProcess,
6617+
claudeSdkFrame({
6618+
type: "system",
6619+
subtype: "background_tasks_changed",
6620+
tasks: [],
6621+
uuid: "00000000-0000-4000-8000-000000000603",
6622+
session_id: WAKE_NATIVE_SESSION,
6623+
}),
6624+
);
66116625
yield* Queue.offer(firstProcess, turnOneResult);
66126626
yield* awaitUntil(
66136627
() => events.some((event) => event.type === "turn.terminal"),
66146628
"first turn terminal",
66156629
);
6616-
assert.isTrue(yield* hasPendingBackgroundWork);
6630+
assert.isFalse(yield* hasPendingBackgroundWork);
66176631

66186632
const alternateModel = {
66196633
...CLAUDE_TEST_MODEL_SELECTION,
@@ -7117,8 +7131,172 @@ describe("ClaudeAdapterV2 background wake turns", () => {
71177131
),
71187132
);
71197133

7134+
it.effect("refuses a model change that would kill a running background subagent", () =>
7135+
Effect.scoped(
7136+
Effect.gen(function* () {
7137+
const SUBAGENT_TASK_ID = "task-model-change-running-subagent";
7138+
const SUBAGENT_TOOL_USE_ID = "toolu-model-change-running-subagent";
7139+
const fileSystem = yield* FileSystem.FileSystem;
7140+
const idAllocator = yield* IdAllocator.IdAllocatorV2;
7141+
const attachmentsDir = yield* fileSystem.makeTempDirectoryScoped({
7142+
prefix: "t3-claude-v2-model-change-running-subagent-",
7143+
});
7144+
const processQueues: Array<Queue.Queue<SDKMessage>> = [];
7145+
const events: Array<ProviderAdapterV2Event> = [];
7146+
const adapter = ClaudeAdapterV2.makeClaudeAdapterV2({
7147+
instanceId: ClaudeAdapterV2.CLAUDE_DEFAULT_INSTANCE_ID,
7148+
settings: DEFAULT_CLAUDE_SETTINGS,
7149+
environment: {},
7150+
attachmentsDir,
7151+
fileSystem,
7152+
path: yield* Path.Path,
7153+
idAllocator,
7154+
continuationRequests: { offer: () => Effect.void },
7155+
queryRunner: {
7156+
allocateSessionId: Effect.succeed(WAKE_NATIVE_SESSION),
7157+
open: () =>
7158+
Effect.gen(function* () {
7159+
const sdkMessages = yield* Queue.unbounded<SDKMessage>();
7160+
processQueues.push(sdkMessages);
7161+
return {
7162+
messages: Stream.fromQueue(sdkMessages),
7163+
offer: () => Effect.void,
7164+
setModel: () => Effect.void,
7165+
interrupt: Effect.void,
7166+
close: Queue.shutdown(sdkMessages),
7167+
};
7168+
}),
7169+
forkSession: () => Effect.die("unused forkSession"),
7170+
subagentLaunchToolUseId: () => Effect.succeed(null),
7171+
assertComplete: Effect.void,
7172+
},
7173+
});
7174+
const threadId = ThreadId.make("thread-claude-model-change-running-subagent");
7175+
const runtime = yield* adapter.openSession({
7176+
threadId,
7177+
providerSessionId: ProviderSessionId.make(
7178+
"provider-session-claude-model-change-running-subagent",
7179+
),
7180+
modelSelection: CLAUDE_TEST_MODEL_SELECTION,
7181+
runtimePolicy: CLAUDE_TEST_RUNTIME_POLICY,
7182+
});
7183+
const providerThread = yield* runtime.ensureThread({
7184+
threadId,
7185+
modelSelection: CLAUDE_TEST_MODEL_SELECTION,
7186+
runtimePolicy: CLAUDE_TEST_RUNTIME_POLICY,
7187+
});
7188+
yield* runtime.events.pipe(
7189+
Stream.runForEach((event) =>
7190+
Effect.sync(() => {
7191+
events.push(event);
7192+
}),
7193+
),
7194+
Effect.forkScoped,
7195+
);
7196+
const terminals = () => events.filter((event) => event.type === "turn.terminal");
7197+
const now = yield* DateTime.now;
7198+
7199+
yield* runtime.startTurn(
7200+
makeClaudeTestTurnInput({
7201+
threadId,
7202+
providerThread,
7203+
now,
7204+
attemptId: RunAttemptId.make("attempt-claude-model-change-running-subagent-a"),
7205+
text: "Spawn a background subagent and stop.",
7206+
attachments: [],
7207+
}),
7208+
);
7209+
const firstProcess = processQueues[0]!;
7210+
yield* Queue.offer(
7211+
firstProcess,
7212+
claudeSdkFrame({
7213+
type: "system",
7214+
subtype: "task_started",
7215+
task_id: SUBAGENT_TASK_ID,
7216+
tool_use_id: SUBAGENT_TOOL_USE_ID,
7217+
description: "Background research",
7218+
subagent_type: "general-purpose",
7219+
task_type: "local_agent",
7220+
prompt: "Research, then report.",
7221+
uuid: "00000000-0000-4000-8000-000000000901",
7222+
session_id: WAKE_NATIVE_SESSION,
7223+
}),
7224+
);
7225+
yield* Queue.offer(
7226+
firstProcess,
7227+
makeResultFrame({
7228+
uuid: "00000000-0000-4000-8000-000000000902",
7229+
result: "Spawned the subagent in the background.",
7230+
}),
7231+
);
7232+
yield* awaitUntil(() => terminals().length === 1, "first turn terminal");
7233+
const settledTurn = terminals()[0]!;
7234+
7235+
// The subagent runs inside the first CLI process. Another model needs
7236+
// another process, so the turn must not start and close this one.
7237+
const alternateModel = {
7238+
...CLAUDE_TEST_MODEL_SELECTION,
7239+
model: "claude-haiku-4-5-20251001",
7240+
} satisfies ModelSelection;
7241+
const switchTurn = (attempt: string) =>
7242+
runtime.startTurn(
7243+
makeClaudeTestTurnInput({
7244+
threadId,
7245+
providerThread: { ...providerThread, status: "active" },
7246+
now,
7247+
attemptId: RunAttemptId.make(attempt),
7248+
text: "Switch model while the subagent runs.",
7249+
attachments: [],
7250+
providerTurnOrdinal: 2,
7251+
modelSelection: alternateModel,
7252+
}),
7253+
);
7254+
const refused = yield* switchTurn("attempt-claude-model-change-running-subagent-b").pipe(
7255+
Effect.flip,
7256+
);
7257+
assert.equal(
7258+
makeProviderFailure({ cause: refused, class: "provider_error" }).message,
7259+
new ClaudeAdapterV2.ClaudeBackgroundWorkBlocksQueryReplacementError().message,
7260+
);
7261+
assert.lengthOf(processQueues, 1);
7262+
7263+
// Stop ends the background work, so the switch may replace the process.
7264+
yield* runtime.interruptTurn({
7265+
providerThread,
7266+
providerTurnId: settledTurn.providerTurnId,
7267+
requestRuntimeRestart: true,
7268+
});
7269+
yield* switchTurn("attempt-claude-model-change-running-subagent-c");
7270+
assert.lengthOf(processQueues, 2);
7271+
7272+
// The stopped subagent never reports its end, so it must not block
7273+
// later changes on the replacement process either.
7274+
yield* Queue.offer(
7275+
processQueues[1]!,
7276+
makeResultFrame({
7277+
uuid: "00000000-0000-4000-8000-000000000903",
7278+
result: "Switched model.",
7279+
}),
7280+
);
7281+
yield* awaitUntil(() => terminals().length === 2, "switched turn terminal");
7282+
yield* runtime.startTurn(
7283+
makeClaudeTestTurnInput({
7284+
threadId,
7285+
providerThread: { ...providerThread, status: "active" },
7286+
now,
7287+
attemptId: RunAttemptId.make("attempt-claude-model-change-running-subagent-d"),
7288+
text: "Switch back.",
7289+
attachments: [],
7290+
providerTurnOrdinal: 3,
7291+
}),
7292+
);
7293+
assert.lengthOf(processQueues, 3);
7294+
}).pipe(Effect.provide(Layer.merge(IdAllocator.layer, NodeServices.layer))),
7295+
),
7296+
);
7297+
71207298
it.effect(
7121-
"clears process-scoped roster when same-native-thread replacement open fails after close",
7299+
"keeps the process and its roster when a model change meets a running background shell",
71227300
() =>
71237301
Effect.scoped(
71247302
Effect.gen(function* () {
@@ -7236,17 +7414,10 @@ describe("ClaudeAdapterV2 background wake turns", () => {
72367414
)
72377415
.pipe(Effect.exit);
72387416
assert.isTrue(Exit.isFailure(failedStart));
7239-
// Old process was closed before the failed open: roster must not stick.
7240-
yield* awaitUntil(
7241-
() =>
7242-
providerThreadRosterEvents(events).some(
7243-
(event) =>
7244-
event.providerThread.status === "idle" &&
7245-
(event.providerThread.pendingBackgroundTasks?.length ?? 0) === 0,
7246-
),
7247-
"roster cleared after failed same-thread replacement open",
7248-
);
7249-
assert.isFalse(yield* hasPendingBackgroundWork);
7417+
// The shell runs in the first process, so it is never closed and
7418+
// no replacement is opened.
7419+
assert.equal(openCount, 1);
7420+
assert.isTrue(yield* hasPendingBackgroundWork);
72507421
}).pipe(Effect.provide(Layer.merge(IdAllocator.layer, NodeServices.layer))),
72517422
),
72527423
);

‎apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts‎

Lines changed: 72 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -340,6 +340,15 @@ export class ClaudeAgentSdkQueryRunnerError extends Schema.TaggedError<ClaudeAge
340340
}
341341
}
342342

343+
export class ClaudeBackgroundWorkBlocksQueryReplacementError extends Schema.TaggedError<ClaudeBackgroundWorkBlocksQueryReplacementError>()(
344+
"ClaudeBackgroundWorkBlocksQueryReplacementError",
345+
{},
346+
) {
347+
override get message(): string {
348+
return "Claude is still running background agents or commands, and this model or setting change would end them. Wait for them to finish, or press Stop, then send the message again.";
349+
}
350+
}
351+
343352
export interface ClaudeAgentSdkQueryRunnerShape {
344353
readonly allocateSessionId: Effect.Effect<string, ClaudeAgentSdkQueryRunnerError>;
345354
readonly open: (
@@ -2667,6 +2676,12 @@ interface ClaudeLiveQueryContext {
26672676
// uuid before any echo, so it echoes, but a resume's own turns can still
26682677
// run ahead of that prompt.
26692678
promptEchoMode: "unknown" | "acknowledged" | "early" | "result_only";
2679+
// Stop, rollback or fork is closing this process; its work is ending.
2680+
stopping: boolean;
2681+
// Registry entries still running when this process opened. Their process
2682+
// is gone and never reports their end; any later task_started replaces the
2683+
// entry, so an entry still in this set runs nowhere.
2684+
readonly subagentsFromEarlierProcesses: ReadonlySet<ActiveClaudeSubagent>;
26702685
}
26712686

26722687
interface ActiveClaudeToolCall {
@@ -6753,6 +6768,37 @@ export function makeClaudeAdapterV2(
67536768
}),
67546769
);
67556770

6771+
// Work the live process still runs. A subagent whose completion is
6772+
// already buffered is done: the buffer outlives the process.
6773+
const liveProcessRunsBackgroundWork = Effect.fnUntraced(function* (
6774+
live: ClaudeLiveQueryContext,
6775+
) {
6776+
if (
6777+
rosterForNativeThread(
6778+
yield* Ref.get(pendingBackgroundTasksByNativeThread),
6779+
live.nativeThreadId,
6780+
).size > 0
6781+
) {
6782+
return true;
6783+
}
6784+
const buffered = (yield* Ref.get(wakeBuffers)).get(live.nativeThreadId)?.messages ?? [];
6785+
for (const [taskId, subagent] of yield* Ref.get(sessionSubagentsByTaskId)) {
6786+
if (
6787+
subagent.task.status === "running" &&
6788+
!live.subagentsFromEarlierProcesses.has(subagent) &&
6789+
!buffered.some(
6790+
(message) =>
6791+
message.type === "system" &&
6792+
message.subtype === "task_notification" &&
6793+
message.task_id === taskId,
6794+
)
6795+
) {
6796+
return true;
6797+
}
6798+
}
6799+
return false;
6800+
});
6801+
67566802
const openQuery = Effect.fnUntraced(function* (
67576803
turnInput: ProviderAdapter.ProviderAdapterV2TurnInput,
67586804
nativeThreadId: string,
@@ -6770,15 +6816,32 @@ export function makeClaudeAdapterV2(
67706816
const compiledSelection = compileClaudeModelSelection(turnInput.modelSelection);
67716817
const resumeSessionAt = yield* getNativeConversationHeadId(turnInput.providerThread);
67726818
const existing = yield* Ref.get(queryContext);
6819+
// A continuation prompts nothing: it drains output the live process
6820+
// already produced, so it keeps that process whatever its selection.
67736821
if (
67746822
existing !== null &&
67756823
existing.nativeThreadId === nativeThreadId &&
6776-
existing.queryPolicyKey === queryPolicyKey &&
6777-
existing.selectionKey === compiledSelection.queryIdentity
6824+
(isClaudeProviderContinuationTurn(turnInput) ||
6825+
(existing.queryPolicyKey === queryPolicyKey &&
6826+
existing.selectionKey === compiledSelection.queryIdentity))
67786827
) {
67796828
return existing;
67806829
}
67816830

6831+
// Background agents and shells run inside the CLI process, so a
6832+
// new selection would kill them and lose their results. Refuse until
6833+
// they finish or the user presses Stop, which closes the process.
6834+
// Another native thread on this session is one the app thread has
6835+
// left (Claude sessions serve one app thread), so it is replaced.
6836+
if (
6837+
existing !== null &&
6838+
existing.nativeThreadId === nativeThreadId &&
6839+
!existing.stopping &&
6840+
(yield* liveProcessRunsBackgroundWork(existing))
6841+
) {
6842+
return yield* new ClaudeBackgroundWorkBlocksQueryReplacementError();
6843+
}
6844+
67826845
// openQuery owns one live process. Closing it for another native
67836846
// thread kills that sibling's CLI; it can never emit a roster clear,
67846847
// so drop its process-scoped Waiting/wake state immediately. Closing
@@ -6875,6 +6938,12 @@ export function makeClaudeAdapterV2(
68756938
selectionKey: compiledSelection.queryIdentity,
68766939
closed,
68776940
promptEchoMode: "unknown",
6941+
stopping: false,
6942+
subagentsFromEarlierProcesses: new Set(
6943+
[...(yield* Ref.get(sessionSubagentsByTaskId)).values()].filter(
6944+
(subagent) => subagent.task.status === "running",
6945+
),
6946+
),
68786947
};
68796948
yield* Ref.set(queryContext, context);
68806949
yield* querySession.messages.pipe(
@@ -7239,6 +7308,7 @@ export function makeClaudeAdapterV2(
72397308
return;
72407309
}
72417310

7311+
existing.stopping = true;
72427312
yield* existing.query.close.pipe(Effect.ignore);
72437313
const closed = yield* Deferred.await(existing.closed).pipe(
72447314
Effect.timeoutOption("10 seconds"),

‎apps/server/src/orchestration-v2/ProviderFailure.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,8 @@ function causeMessage(cause: unknown): string | undefined {
3636
switch ((cause as Record<string, unknown>)._tag) {
3737
case "ContextHandoffBudgetError":
3838
return new ContextHandoffBudgetError().message;
39+
case "ClaudeBackgroundWorkBlocksQueryReplacementError":
40+
return stringField(cause, "message");
3941
case "ContextHandoffDeliveryUncertainError":
4042
return "T3 could not confirm whether conversation history reached the provider. Retry the turn to recover the session.";
4143
case "ProviderAdapterTurnStartError":

0 commit comments

Comments
 (0)