Skip to content

Commit c5d3f59

Browse files
committed
fix(server): recover stalled Claude sessions
1 parent f0953d1 commit c5d3f59

8 files changed

Lines changed: 715 additions & 82 deletions

File tree

‎apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts‎

Lines changed: 170 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3579,6 +3579,176 @@ describe("ClaudeAdapterV2 background wake turns", () => {
35793579
),
35803580
);
35813581

3582+
it.effect.each([{ status: "compacting" }, { status: "requesting" }] as const)(
3583+
"terminalizes a stalled Claude $status phase and marks its session invalid",
3584+
(input) =>
3585+
Effect.gen(function* () {
3586+
const closeRequested = yield* Deferred.make<void>();
3587+
const harness = yield* makeWakeHarnessWithOptions({
3588+
close: (sdkMessages) =>
3589+
Deferred.succeed(closeRequested, undefined).pipe(
3590+
Effect.andThen(Queue.shutdown(sdkMessages)),
3591+
),
3592+
});
3593+
yield* harness.runtime.startTurn(
3594+
makeClaudeTestTurnInput({
3595+
threadId: harness.threadId,
3596+
providerThread: harness.providerThread,
3597+
now: yield* DateTime.now,
3598+
attemptId: RunAttemptId.make(`attempt-claude-stalled-${input.status}`),
3599+
text: input.status === "compacting" ? "/compact" : "Continue.",
3600+
attachments: [],
3601+
}),
3602+
);
3603+
yield* harness.offerAndWait(
3604+
claudeSdkFrame({
3605+
type: "system",
3606+
subtype: "status",
3607+
status: input.status,
3608+
uuid: `status-${input.status}`,
3609+
session_id: WAKE_NATIVE_SESSION,
3610+
}),
3611+
);
3612+
3613+
yield* TestClock.adjust("2 minutes");
3614+
yield* Deferred.await(closeRequested);
3615+
const terminal = yield* Queue.take(harness.terminalReceipts);
3616+
assert.equal(terminal.status, "failed");
3617+
if (terminal.status === "failed") {
3618+
assert.equal(terminal.failure.code, "native_session_stalled");
3619+
assert.equal(terminal.failure.retryable, true);
3620+
}
3621+
assert.lengthOf(harness.terminalEvents(), 1);
3622+
}).pipe(
3623+
Effect.scoped,
3624+
Effect.provide(
3625+
Layer.mergeAll(
3626+
IdAllocator.layer,
3627+
McpProviderSessions.layer,
3628+
NodeServices.layer,
3629+
TestClock.layer(),
3630+
),
3631+
),
3632+
),
3633+
);
3634+
3635+
it.effect("does not let rate-limit telemetry extend a stalled request deadline", () =>
3636+
Effect.gen(function* () {
3637+
const closeRequested = yield* Deferred.make<void>();
3638+
const harness = yield* makeWakeHarnessWithOptions({
3639+
close: (sdkMessages) =>
3640+
Deferred.succeed(closeRequested, undefined).pipe(
3641+
Effect.andThen(Queue.shutdown(sdkMessages)),
3642+
),
3643+
});
3644+
yield* harness.runtime.startTurn(
3645+
makeClaudeTestTurnInput({
3646+
threadId: harness.threadId,
3647+
providerThread: harness.providerThread,
3648+
now: yield* DateTime.now,
3649+
attemptId: RunAttemptId.make("attempt-claude-stalled-through-rate-limit-event"),
3650+
text: "Continue.",
3651+
attachments: [],
3652+
}),
3653+
);
3654+
yield* harness.offerAndWait(
3655+
claudeSdkFrame({
3656+
type: "system",
3657+
subtype: "status",
3658+
status: "requesting",
3659+
uuid: "status-requesting-before-rate-limit-event",
3660+
session_id: WAKE_NATIVE_SESSION,
3661+
}),
3662+
);
3663+
yield* TestClock.adjust("1 minute");
3664+
yield* harness.offerAndWait(
3665+
claudeSdkFrame({
3666+
type: "rate_limit_event",
3667+
rate_limit_info: { status: "allowed" },
3668+
uuid: "00000000-0000-4000-8000-0000000002f3",
3669+
session_id: WAKE_NATIVE_SESSION,
3670+
}),
3671+
);
3672+
3673+
yield* TestClock.adjust("1 minute");
3674+
assert.isTrue(yield* Deferred.isDone(closeRequested));
3675+
yield* Deferred.await(closeRequested);
3676+
const terminal = yield* Queue.take(harness.terminalReceipts);
3677+
assert.equal(terminal.status, "failed");
3678+
if (terminal.status === "failed") {
3679+
assert.equal(terminal.failure.code, "native_session_stalled");
3680+
}
3681+
}).pipe(
3682+
Effect.scoped,
3683+
Effect.provide(
3684+
Layer.mergeAll(
3685+
IdAllocator.layer,
3686+
McpProviderSessions.layer,
3687+
NodeServices.layer,
3688+
TestClock.layer(),
3689+
),
3690+
),
3691+
),
3692+
);
3693+
3694+
it.effect("keeps a completed Claude compaction from tripping the stall timeout", () =>
3695+
Effect.gen(function* () {
3696+
const closeRequested = yield* Deferred.make<void>();
3697+
const harness = yield* makeWakeHarnessWithOptions({
3698+
close: () => Deferred.succeed(closeRequested, undefined),
3699+
});
3700+
yield* harness.runtime.startTurn(
3701+
makeClaudeTestTurnInput({
3702+
threadId: harness.threadId,
3703+
providerThread: harness.providerThread,
3704+
now: yield* DateTime.now,
3705+
attemptId: RunAttemptId.make("attempt-claude-completed-compaction-timeout"),
3706+
text: "/compact",
3707+
attachments: [],
3708+
}),
3709+
);
3710+
yield* harness.offerAndWait(
3711+
claudeSdkFrame({
3712+
type: "system",
3713+
subtype: "status",
3714+
status: "compacting",
3715+
uuid: "status-completing-compaction",
3716+
session_id: WAKE_NATIVE_SESSION,
3717+
}),
3718+
);
3719+
yield* TestClock.adjust("1 minute");
3720+
yield* harness.offerAndWait(
3721+
claudeSdkFrame({
3722+
type: "system",
3723+
subtype: "compact_boundary",
3724+
compact_metadata: { trigger: "manual", pre_tokens: 1500, post_tokens: 400 },
3725+
uuid: "00000000-0000-4000-8000-0000000002f1",
3726+
session_id: WAKE_NATIVE_SESSION,
3727+
}),
3728+
);
3729+
yield* TestClock.adjust("2 minutes");
3730+
assert.isFalse(yield* Deferred.isDone(closeRequested));
3731+
yield* harness.offerAndWait(
3732+
makeResultFrame({
3733+
uuid: "00000000-0000-4000-8000-0000000002f2",
3734+
result: "Compacted conversation.",
3735+
}),
3736+
);
3737+
const terminal = yield* Queue.take(harness.terminalReceipts);
3738+
assert.equal(terminal.status, "completed");
3739+
}).pipe(
3740+
Effect.scoped,
3741+
Effect.provide(
3742+
Layer.mergeAll(
3743+
IdAllocator.layer,
3744+
McpProviderSessions.layer,
3745+
NodeServices.layer,
3746+
TestClock.layer(),
3747+
),
3748+
),
3749+
),
3750+
);
3751+
35823752
it.effect.each(["completed", "interrupted"] as const)(
35833753
"projects Claude thinking blocks when %s",
35843754
(status) =>

‎apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts‎

Lines changed: 114 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -176,6 +176,7 @@ import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator";
176176
import {
177177
makeProviderFailure,
178178
makeProviderRetryTurnItem,
179+
NATIVE_SESSION_STALLED_FAILURE_CODE,
179180
} from "@t3tools/provider-core/server/failure";
180181
import { turnScopedSelectionTransition } from "@t3tools/provider-core/server/selectionTransition";
181182
import { providerMessageTextWithAttachmentPaths } from "@t3tools/provider-core/server/attachmentPrompt";
@@ -312,6 +313,7 @@ const WORKFLOW_MEMBER_TRANSCRIPT_FLUSH_MS = 500;
312313
// A steer waits at most this long, from its request, for its foreground
313314
// subagents to move to the background before it is sent.
314315
const BACKGROUND_MOVE_TIMEOUT_MS = 5_000;
316+
const CLAUDE_STALLED_PHASE_TIMEOUT = "2 minutes";
315317

316318
const CLAUDE_GROUP_WATCH_SCRIPT = [
317319
"{ trap '' TERM; read -r _ <&3; kill -TERM 0; sleep 5; kill -KILL 0; } </dev/null >/dev/null 2>&1 &",
@@ -3477,6 +3479,13 @@ interface ActiveClaudeTurnContext {
34773479
nativeWakeStarted: boolean;
34783480
gatedFramesBeforeEcho: number;
34793481
readonly heldRootFrames: Array<SDKMessage>;
3482+
stalledPhase: ClaudeStalledPhase | null;
3483+
}
3484+
3485+
type ClaudeStalledPhaseKind = "compaction" | "request";
3486+
3487+
interface ClaudeStalledPhase {
3488+
readonly kind: ClaudeStalledPhaseKind;
34803489
}
34813490

34823491
interface ActiveClaudeProviderRetry {
@@ -3519,6 +3528,10 @@ interface ClaudeLiveQueryContext {
35193528
receivedInit: boolean;
35203529
permissionMode: PermissionMode;
35213530
stopping: boolean;
3531+
terminalFailure: OrchestrationV2ProviderFailure | undefined;
3532+
// Registry entries still running when this process opened. Their process
3533+
// is gone and never reports their end; any later task_started replaces the
3534+
// entry, so an entry still in this set runs nowhere.
35223535
readonly subagentsFromEarlierProcesses: ReadonlySet<ActiveClaudeSubagent>;
35233536
}
35243537

@@ -7031,10 +7044,12 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* (
70317044
...(interrupted
70327045
? {}
70337046
: {
7034-
failure: makeProviderFailure({
7035-
cause: cause === undefined ? undefined : Cause.squash(cause),
7036-
class: "transport_error",
7037-
}),
7047+
failure:
7048+
closedQuery.terminalFailure ??
7049+
makeProviderFailure({
7050+
cause: cause === undefined ? undefined : Cause.squash(cause),
7051+
class: "transport_error",
7052+
}),
70387053
}),
70397054
});
70407055
} else {
@@ -8724,6 +8739,98 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* (
87248739
),
87258740
);
87268741

8742+
const stalledPhaseFailure = (kind: ClaudeStalledPhaseKind) =>
8743+
makeProviderFailure({
8744+
class: "transport_error",
8745+
code: NATIVE_SESSION_STALLED_FAILURE_CODE,
8746+
message:
8747+
kind === "compaction"
8748+
? "Claude context compaction made no progress for two minutes. T3 closed the stalled session; the next turn will continue in a fresh Claude session."
8749+
: "Claude returned no model or tool activity for two minutes. T3 closed the stalled session; the next turn will continue in a fresh Claude session.",
8750+
retryable: true,
8751+
});
8752+
8753+
const closeStalledQuery = Effect.fn("ClaudeAdapterV2.closeStalledQuery")(function* (
8754+
context: ActiveClaudeTurnContext,
8755+
liveQuery: ClaudeLiveQueryContext,
8756+
phase: ClaudeStalledPhase,
8757+
) {
8758+
if (
8759+
(yield* Ref.get(activeTurn)) !== context ||
8760+
(yield* Ref.get(queryContext)) !== liveQuery ||
8761+
context.stalledPhase !== phase
8762+
) {
8763+
return;
8764+
}
8765+
context.stalledPhase = null;
8766+
liveQuery.terminalFailure = stalledPhaseFailure(phase.kind);
8767+
yield* Effect.logWarning("orchestration-v2.claude-stalled-phase-timeout", {
8768+
providerSessionId: input.providerSessionId,
8769+
providerThreadId: context.input.providerThread.id,
8770+
providerTurnId: context.providerTurnId,
8771+
nativeThreadId: context.nativeThreadId,
8772+
phase: phase.kind,
8773+
});
8774+
const closed = yield* liveQuery.query.close.pipe(
8775+
Effect.ignore,
8776+
Effect.andThen(Deferred.await(liveQuery.closed)),
8777+
Effect.timeoutOption("10 seconds"),
8778+
);
8779+
if (Option.isSome(closed)) return;
8780+
8781+
yield* Effect.logWarning("orchestration-v2.claude-stalled-query-close-timeout", {
8782+
providerSessionId: input.providerSessionId,
8783+
providerThreadId: context.input.providerThread.id,
8784+
providerTurnId: context.providerTurnId,
8785+
nativeThreadId: context.nativeThreadId,
8786+
phase: phase.kind,
8787+
});
8788+
const ownsLiveQuery = yield* Ref.modify(queryContext, (current) =>
8789+
current?.query === liveQuery.query ? [true, null] : [false, current],
8790+
);
8791+
if (ownsLiveQuery) {
8792+
yield* finalizeActiveTurnAfterQueryExit(liveQuery);
8793+
yield* endSubagentCallsOfClosedQuery(liveQuery.nativeThreadId, "failed");
8794+
}
8795+
yield* Deferred.succeed(liveQuery.closed, undefined);
8796+
});
8797+
8798+
// This runs for every SDK frame, including streaming events; avoid a span per frame.
8799+
const observeClaudeStalledPhase = Effect.fnUntraced(function* (observation: {
8800+
readonly query: ClaudeAgentSdkQuerySession;
8801+
readonly message: SDKMessage;
8802+
}) {
8803+
const context = yield* Ref.get(activeTurn);
8804+
const liveQuery = yield* Ref.get(queryContext);
8805+
if (context === null || liveQuery?.query !== observation.query) return;
8806+
const message = observation.message;
8807+
const kind: ClaudeStalledPhaseKind | null =
8808+
message.type === "system" &&
8809+
message.subtype === "status" &&
8810+
message.status === "compacting"
8811+
? "compaction"
8812+
: (message.type === "system" &&
8813+
message.subtype === "status" &&
8814+
message.status === "requesting") ||
8815+
(message.type === "system" && message.subtype === "api_retry")
8816+
? "request"
8817+
: null;
8818+
if (kind === null) {
8819+
if (message.type === "rate_limit_event") return;
8820+
context.stalledPhase = null;
8821+
return;
8822+
}
8823+
if (context.stalledPhase?.kind === kind) return;
8824+
8825+
const phase: ClaudeStalledPhase = { kind };
8826+
context.stalledPhase = phase;
8827+
yield* Effect.sleep(CLAUDE_STALLED_PHASE_TIMEOUT).pipe(
8828+
Effect.andThen(closeStalledQuery(context, liveQuery, phase)),
8829+
Effect.forkIn(sessionScope),
8830+
);
8831+
});
8832+
8833+
// Gives held root output to the pending prompt turn.
87278834
const releaseHeldRootFrames = Effect.fnUntraced(function* (
87288835
context: ActiveClaudeTurnContext,
87298836
) {
@@ -8743,6 +8850,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* (
87438850
readonly message: SDKMessage;
87448851
}) {
87458852
const message = input.message;
8853+
yield* observeClaudeStalledPhase(input);
87468854
const context = yield* Ref.get(activeTurn);
87478855
const liveQuery = yield* Ref.get(queryContext);
87488856
if (
@@ -9413,6 +9521,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* (
94139521
receivedInit: false,
94149522
permissionMode: queryOptions.permissionMode,
94159523
stopping: false,
9524+
terminalFailure: undefined,
94169525
subagentsFromEarlierProcesses: new Set(
94179526
[...(yield* Ref.get(sessionSubagentsByTaskId)).values()].filter(
94189527
(subagent) => subagent.task.status === "running",
@@ -9580,6 +9689,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* (
95809689
nativeWakeStarted: false,
95819690
gatedFramesBeforeEcho: 0,
95829691
heldRootFrames: [],
9692+
stalledPhase: null,
95839693
};
95849694
const attachTurn = Effect.gen(function* () {
95859695
querySession.lastTurn = context;

0 commit comments

Comments
 (0)