Skip to content

Commit c5f81f3

Browse files
committed
fix(server): recover stalled Claude sessions
1 parent 2d19435 commit c5f81f3

5 files changed

Lines changed: 338 additions & 7 deletions

File tree

‎apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts‎

Lines changed: 111 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3579,6 +3579,117 @@ describe("ClaudeAdapterV2 background wake turns", () => {
35793579
),
35803580
);
35813581

3582+
it.effect.each([{ status: "compacting" }, { status: "requesting" }] as const)(
3583+
"terminalizes a stalled Claude $status phase and marks its session invalid",
3584+
(input) =>
3585+
Effect.gen(function* () {
3586+
const closeRequested = yield* Deferred.make<void>();
3587+
const harness = yield* makeWakeHarnessWithOptions({
3588+
close: (sdkMessages) =>
3589+
Deferred.succeed(closeRequested, undefined).pipe(
3590+
Effect.andThen(Queue.shutdown(sdkMessages)),
3591+
),
3592+
});
3593+
yield* harness.runtime.startTurn(
3594+
makeClaudeTestTurnInput({
3595+
threadId: harness.threadId,
3596+
providerThread: harness.providerThread,
3597+
now: yield* DateTime.now,
3598+
attemptId: RunAttemptId.make(`attempt-claude-stalled-${input.status}`),
3599+
text: input.status === "compacting" ? "/compact" : "Continue.",
3600+
attachments: [],
3601+
}),
3602+
);
3603+
yield* harness.offerAndWait(
3604+
claudeSdkFrame({
3605+
type: "system",
3606+
subtype: "status",
3607+
status: input.status,
3608+
uuid: `status-${input.status}`,
3609+
session_id: WAKE_NATIVE_SESSION,
3610+
}),
3611+
);
3612+
3613+
yield* TestClock.adjust("2 minutes");
3614+
yield* Deferred.await(closeRequested);
3615+
const terminal = yield* Queue.take(harness.terminalReceipts);
3616+
assert.equal(terminal.status, "failed");
3617+
if (terminal.status === "failed") {
3618+
assert.equal(terminal.failure.code, "native_session_stalled");
3619+
assert.equal(terminal.failure.retryable, true);
3620+
}
3621+
assert.lengthOf(harness.terminalEvents(), 1);
3622+
}).pipe(
3623+
Effect.scoped,
3624+
Effect.provide(
3625+
Layer.mergeAll(
3626+
IdAllocator.layer,
3627+
McpProviderSessions.layer,
3628+
NodeServices.layer,
3629+
TestClock.layer(),
3630+
),
3631+
),
3632+
),
3633+
);
3634+
3635+
it.effect("keeps a completed Claude compaction from tripping the stall timeout", () =>
3636+
Effect.gen(function* () {
3637+
const closeRequested = yield* Deferred.make<void>();
3638+
const harness = yield* makeWakeHarnessWithOptions({
3639+
close: () => Deferred.succeed(closeRequested, undefined),
3640+
});
3641+
yield* harness.runtime.startTurn(
3642+
makeClaudeTestTurnInput({
3643+
threadId: harness.threadId,
3644+
providerThread: harness.providerThread,
3645+
now: yield* DateTime.now,
3646+
attemptId: RunAttemptId.make("attempt-claude-completed-compaction-timeout"),
3647+
text: "/compact",
3648+
attachments: [],
3649+
}),
3650+
);
3651+
yield* harness.offerAndWait(
3652+
claudeSdkFrame({
3653+
type: "system",
3654+
subtype: "status",
3655+
status: "compacting",
3656+
uuid: "status-completing-compaction",
3657+
session_id: WAKE_NATIVE_SESSION,
3658+
}),
3659+
);
3660+
yield* TestClock.adjust("1 minute");
3661+
yield* harness.offerAndWait(
3662+
claudeSdkFrame({
3663+
type: "system",
3664+
subtype: "compact_boundary",
3665+
compact_metadata: { trigger: "manual", pre_tokens: 1500, post_tokens: 400 },
3666+
uuid: "00000000-0000-4000-8000-0000000002f1",
3667+
session_id: WAKE_NATIVE_SESSION,
3668+
}),
3669+
);
3670+
yield* TestClock.adjust("2 minutes");
3671+
assert.isFalse(yield* Deferred.isDone(closeRequested));
3672+
yield* harness.offerAndWait(
3673+
makeResultFrame({
3674+
uuid: "00000000-0000-4000-8000-0000000002f2",
3675+
result: "Compacted conversation.",
3676+
}),
3677+
);
3678+
const terminal = yield* Queue.take(harness.terminalReceipts);
3679+
assert.equal(terminal.status, "completed");
3680+
}).pipe(
3681+
Effect.scoped,
3682+
Effect.provide(
3683+
Layer.mergeAll(
3684+
IdAllocator.layer,
3685+
McpProviderSessions.layer,
3686+
NodeServices.layer,
3687+
TestClock.layer(),
3688+
),
3689+
),
3690+
),
3691+
);
3692+
35823693
it.effect.each(["completed", "interrupted"] as const)(
35833694
"projects Claude thinking blocks when %s",
35843695
(status) =>

‎apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts‎

Lines changed: 112 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -176,6 +176,7 @@ import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator";
176176
import {
177177
makeProviderFailure,
178178
makeProviderRetryTurnItem,
179+
NATIVE_SESSION_STALLED_FAILURE_CODE,
179180
} from "@t3tools/provider-core/server/failure";
180181
import { turnScopedSelectionTransition } from "@t3tools/provider-core/server/selectionTransition";
181182
import { providerMessageTextWithAttachmentPaths } from "@t3tools/provider-core/server/attachmentPrompt";
@@ -312,6 +313,7 @@ const WORKFLOW_MEMBER_TRANSCRIPT_FLUSH_MS = 500;
312313
// A steer waits at most this long, from its request, for its foreground
313314
// subagents to move to the background before it is sent.
314315
const BACKGROUND_MOVE_TIMEOUT_MS = 5_000;
316+
const CLAUDE_STALLED_PHASE_TIMEOUT = "2 minutes";
315317

316318
const CLAUDE_GROUP_WATCH_SCRIPT = [
317319
"{ trap '' TERM; read -r _ <&3; kill -TERM 0; sleep 5; kill -KILL 0; } </dev/null >/dev/null 2>&1 &",
@@ -3477,6 +3479,13 @@ interface ActiveClaudeTurnContext {
34773479
nativeWakeStarted: boolean;
34783480
gatedFramesBeforeEcho: number;
34793481
readonly heldRootFrames: Array<SDKMessage>;
3482+
stalledPhase: ClaudeStalledPhase | null;
3483+
}
3484+
3485+
type ClaudeStalledPhaseKind = "compaction" | "request";
3486+
3487+
interface ClaudeStalledPhase {
3488+
readonly kind: ClaudeStalledPhaseKind;
34803489
}
34813490

34823491
interface ActiveClaudeProviderRetry {
@@ -3519,6 +3528,10 @@ interface ClaudeLiveQueryContext {
35193528
receivedInit: boolean;
35203529
permissionMode: PermissionMode;
35213530
stopping: boolean;
3531+
terminalFailure: OrchestrationV2ProviderFailure | undefined;
3532+
// Registry entries still running when this process opened. Their process
3533+
// is gone and never reports their end; any later task_started replaces the
3534+
// entry, so an entry still in this set runs nowhere.
35223535
readonly subagentsFromEarlierProcesses: ReadonlySet<ActiveClaudeSubagent>;
35233536
}
35243537

@@ -7031,10 +7044,12 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* (
70317044
...(interrupted
70327045
? {}
70337046
: {
7034-
failure: makeProviderFailure({
7035-
cause: cause === undefined ? undefined : Cause.squash(cause),
7036-
class: "transport_error",
7037-
}),
7047+
failure:
7048+
closedQuery.terminalFailure ??
7049+
makeProviderFailure({
7050+
cause: cause === undefined ? undefined : Cause.squash(cause),
7051+
class: "transport_error",
7052+
}),
70387053
}),
70397054
});
70407055
} else {
@@ -8724,6 +8739,96 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* (
87248739
),
87258740
);
87268741

8742+
const stalledPhaseFailure = (kind: ClaudeStalledPhaseKind) =>
8743+
makeProviderFailure({
8744+
class: "transport_error",
8745+
code: NATIVE_SESSION_STALLED_FAILURE_CODE,
8746+
message:
8747+
kind === "compaction"
8748+
? "Claude context compaction made no progress for two minutes. T3 closed the stalled session; the next turn will continue in a fresh Claude session."
8749+
: "Claude returned no model or tool activity for two minutes. T3 closed the stalled session; the next turn will continue in a fresh Claude session.",
8750+
retryable: true,
8751+
});
8752+
8753+
const closeStalledQuery = Effect.fnUntraced(function* (
8754+
context: ActiveClaudeTurnContext,
8755+
liveQuery: ClaudeLiveQueryContext,
8756+
phase: ClaudeStalledPhase,
8757+
) {
8758+
if (
8759+
(yield* Ref.get(activeTurn)) !== context ||
8760+
(yield* Ref.get(queryContext)) !== liveQuery ||
8761+
context.stalledPhase !== phase
8762+
) {
8763+
return;
8764+
}
8765+
context.stalledPhase = null;
8766+
liveQuery.terminalFailure = stalledPhaseFailure(phase.kind);
8767+
yield* Effect.logWarning("orchestration-v2.claude-stalled-phase-timeout", {
8768+
providerSessionId: input.providerSessionId,
8769+
providerThreadId: context.input.providerThread.id,
8770+
providerTurnId: context.providerTurnId,
8771+
nativeThreadId: context.nativeThreadId,
8772+
phase: phase.kind,
8773+
});
8774+
const closed = yield* liveQuery.query.close.pipe(
8775+
Effect.ignore,
8776+
Effect.andThen(Deferred.await(liveQuery.closed)),
8777+
Effect.timeoutOption("10 seconds"),
8778+
);
8779+
if (Option.isSome(closed)) return;
8780+
8781+
yield* Effect.logWarning("orchestration-v2.claude-stalled-query-close-timeout", {
8782+
providerSessionId: input.providerSessionId,
8783+
providerThreadId: context.input.providerThread.id,
8784+
providerTurnId: context.providerTurnId,
8785+
nativeThreadId: context.nativeThreadId,
8786+
phase: phase.kind,
8787+
});
8788+
const ownsLiveQuery = yield* Ref.modify(queryContext, (current) =>
8789+
current?.query === liveQuery.query ? [true, null] : [false, current],
8790+
);
8791+
if (ownsLiveQuery) {
8792+
yield* finalizeActiveTurnAfterQueryExit(liveQuery);
8793+
yield* endSubagentCallsOfClosedQuery(liveQuery.nativeThreadId, "failed");
8794+
}
8795+
yield* Deferred.succeed(liveQuery.closed, undefined);
8796+
});
8797+
8798+
const observeClaudeStalledPhase = Effect.fnUntraced(function* (observation: {
8799+
readonly query: ClaudeAgentSdkQuerySession;
8800+
readonly message: SDKMessage;
8801+
}) {
8802+
const context = yield* Ref.get(activeTurn);
8803+
const liveQuery = yield* Ref.get(queryContext);
8804+
if (context === null || liveQuery?.query !== observation.query) return;
8805+
const message = observation.message;
8806+
const kind: ClaudeStalledPhaseKind | null =
8807+
message.type === "system" &&
8808+
message.subtype === "status" &&
8809+
message.status === "compacting"
8810+
? "compaction"
8811+
: (message.type === "system" &&
8812+
message.subtype === "status" &&
8813+
message.status === "requesting") ||
8814+
(message.type === "system" && message.subtype === "api_retry")
8815+
? "request"
8816+
: null;
8817+
if (kind === null) {
8818+
context.stalledPhase = null;
8819+
return;
8820+
}
8821+
if (context.stalledPhase?.kind === kind) return;
8822+
8823+
const phase: ClaudeStalledPhase = { kind };
8824+
context.stalledPhase = phase;
8825+
yield* Effect.sleep(CLAUDE_STALLED_PHASE_TIMEOUT).pipe(
8826+
Effect.andThen(closeStalledQuery(context, liveQuery, phase)),
8827+
Effect.forkIn(sessionScope),
8828+
);
8829+
});
8830+
8831+
// Gives held root output to the pending prompt turn.
87278832
const releaseHeldRootFrames = Effect.fnUntraced(function* (
87288833
context: ActiveClaudeTurnContext,
87298834
) {
@@ -8743,6 +8848,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* (
87438848
readonly message: SDKMessage;
87448849
}) {
87458850
const message = input.message;
8851+
yield* observeClaudeStalledPhase(input);
87468852
const context = yield* Ref.get(activeTurn);
87478853
const liveQuery = yield* Ref.get(queryContext);
87488854
if (
@@ -9413,6 +9519,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* (
94139519
receivedInit: false,
94149520
permissionMode: queryOptions.permissionMode,
94159521
stopping: false,
9522+
terminalFailure: undefined,
94169523
subagentsFromEarlierProcesses: new Set(
94179524
[...(yield* Ref.get(sessionSubagentsByTaskId)).values()].filter(
94189525
(subagent) => subagent.task.status === "running",
@@ -9580,6 +9687,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* (
95809687
nativeWakeStarted: false,
95819688
gatedFramesBeforeEcho: 0,
95829689
heldRootFrames: [],
9690+
stalledPhase: null,
95839691
};
95849692
const attachTurn = Effect.gen(function* () {
95859693
querySession.lastTurn = context;

‎apps/server/src/orchestration-v2/ProviderTurnStartService.ts‎

Lines changed: 19 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@ import * as ProviderSessionManager from "./ProviderSessionManager.ts";
5050
import {
5151
knownProviderFailureMessage,
5252
makeProviderFailure,
53+
nativeSessionInvalidatedByProviderFailure,
5354
} from "@t3tools/provider-core/server/failure";
5455
import * as RunExecutionService from "./RunExecutionService.ts";
5556
import * as RuntimePolicy from "./RuntimePolicy.ts";
@@ -751,15 +752,26 @@ export const layer: Layer.Layer<
751752
handoff.delivery?.nativeThreadId === providerThread.nativeThreadRef?.nativeId &&
752753
handoff.delivery?.status === "pending",
753754
);
755+
const providerNativeThreadId = providerThread.nativeThreadRef?.nativeId;
756+
const invalidatedNativeSession =
757+
providerNativeThreadId != null &&
758+
nativeSessionInvalidatedByProviderFailure({
759+
providerThreadId: providerThread.id,
760+
nativeThreadId: providerNativeThreadId,
761+
attempts: projection.attempts,
762+
turnItems: projection.turnItems,
763+
});
754764
const resumed = yield* Effect.result(
755-
uncertainDelivery
765+
uncertainDelivery || invalidatedNativeSession
756766
? Effect.fail(
757767
new ProviderAdapter.ProviderAdapterTurnStartError({
758768
driver: session.driver,
759769
threadId: projection.thread.id,
760770
providerThreadId: providerThread.id,
761771
runId,
762-
cause: "Uncertain native history injection",
772+
cause: uncertainDelivery
773+
? "Uncertain native history injection"
774+
: "Native provider session invalidated after a stalled turn",
763775
}),
764776
)
765777
: session.resumeThread({
@@ -777,7 +789,11 @@ export const layer: Layer.Layer<
777789
driver: session.driver,
778790
providerThreadId: providerThread.id,
779791
runId,
780-
reason: uncertainDelivery ? "uncertain_history_delivery" : "resume_failed",
792+
reason: uncertainDelivery
793+
? "uncertain_history_delivery"
794+
: invalidatedNativeSession
795+
? "native_session_invalidated"
796+
: "resume_failed",
781797
errorTag: resumed.failure._tag,
782798
});
783799
const replacement = yield* loadFromProvider(

0 commit comments

Comments
 (0)