Skip to content

Commit 00766e2

Browse files
authored
fix(server): fail any provider turn stuck in a retry (pingdotgg#33152)
1 parent 836b86a commit 00766e2

6 files changed

Lines changed: 426 additions & 10 deletions

File tree

‎apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts‎

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10256,11 +10256,15 @@ describe("ClaudeAdapterV2 background wake turns", () => {
1025610256
retryDelayMs: 1_500,
1025710257
});
1025810258

10259+
// The stream starting again settles the retry before any assistant message completes.
1025910260
yield* Queue.offer(
1026010261
harness.sdkMessages,
10261-
makeAssistantTextFrame({
10262-
uuid: "00000000-0000-4000-8000-000000000202",
10263-
text: "Opening GitHub.",
10262+
claudeSdkFrame({
10263+
type: "stream_event",
10264+
event: { type: "message_start", message: { id: "api-retry-recovered-message" } },
10265+
parent_tool_use_id: null,
10266+
session_id: WAKE_NATIVE_SESSION,
10267+
uuid: "00000000-0000-4000-8000-000000000204",
1026410268
}),
1026510269
);
1026610270
yield* harness.until(() => retryItems().length === 2);
@@ -10269,6 +10273,13 @@ describe("ClaudeAdapterV2 background wake turns", () => {
1026910273
assert.equal(recoveredRetry?.id, runningRetry?.id);
1027010274
assert.equal(recoveredRetry?.status, "completed");
1027110275
assert.equal(recoveredRetry?.title, "Provider recovered");
10276+
yield* Queue.offer(
10277+
harness.sdkMessages,
10278+
makeAssistantTextFrame({
10279+
uuid: "00000000-0000-4000-8000-000000000202",
10280+
text: "Opening GitHub.",
10281+
}),
10282+
);
1027210283

1027310284
yield* Queue.offer(
1027410285
harness.sdkMessages,

‎apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10430,6 +10430,16 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* (
1043010430
const message = input.message;
1043110431
const context = yield* Ref.get(activeTurn);
1043210432
const liveQuery = yield* Ref.get(queryContext);
10433+
// The API answering again is recovery; a long text or tool input block emits nothing
10434+
// else. Judged here, in arrival order: held and buffered frames are handled later.
10435+
if (
10436+
context !== null &&
10437+
liveQuery?.query === input.query &&
10438+
message.type === "stream_event" &&
10439+
message.event.type === "message_start"
10440+
) {
10441+
yield* completeProviderRetry(context, yield* DateTime.now);
10442+
}
1043310443
if (
1043410444
context !== null &&
1043510445
context.pendingSteers.size > 0 &&

‎apps/server/src/orchestration-v2/ProviderSessionManager.test.ts‎

Lines changed: 193 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,10 @@ import * as ThreadCommandExecutor from "./ThreadCommandExecutor.ts";
5555
import * as ProviderSessionManager from "./ProviderSessionManager.ts";
5656
import * as CommandOutputHub from "./CommandOutputHub.ts";
5757
import * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter";
58+
import {
59+
makeProviderFailure,
60+
makeProviderRetryTurnItem,
61+
} from "@t3tools/provider-core/server/failure";
5862

5963
const layerTestDatabase = SqlitePersistence.layerMemory;
6064
const layerTestStores = Layer.merge(EventStore.layer, ProjectionStore.layer).pipe(
@@ -2023,6 +2027,195 @@ it.effect("ProviderSessionManagerV2 drains subscribers when the provider stops",
20232027
}),
20242028
);
20252029

2030+
it.effect("ProviderSessionManagerV2 fails a turn whose provider retry never resumes", () =>
2031+
Effect.gen(function* () {
2032+
const state = yield* Ref.make(emptyState);
2033+
const interrupted = yield* Deferred.make<void>();
2034+
const effect = Effect.gen(function* () {
2035+
const eventSink = yield* EventSink.EventSinkV2;
2036+
const idAllocator = yield* IdAllocator.IdAllocatorV2;
2037+
const manager = yield* ProviderSessionManager.ProviderSessionManagerV2;
2038+
const now = yield* DateTime.now;
2039+
const threadId = ThreadId.make("thread-provider-session-manager-retry-stall");
2040+
const providerSessionId = yield* idAllocator.allocate.providerSession({
2041+
providerInstanceId: modelSelection.instanceId,
2042+
threadId,
2043+
});
2044+
const providerThread = makeProviderThread({ idAllocator, threadId, providerSessionId, now });
2045+
yield* eventSink.write({
2046+
events: [
2047+
yield* makeThreadCreatedEvent({ idAllocator, threadId, now }),
2048+
{
2049+
id: yield* idAllocator.allocate.event({ threadId }),
2050+
type: "provider-thread.updated",
2051+
threadId,
2052+
occurredAt: now,
2053+
payload: providerThread,
2054+
},
2055+
],
2056+
});
2057+
const runtime = yield* manager.open({
2058+
threadId,
2059+
providerSessionId,
2060+
modelSelection,
2061+
runtimePolicy,
2062+
});
2063+
const subscription = yield* runtime.subscribeEvents!;
2064+
const published = yield* Queue.unbounded<void>();
2065+
// The pump updates the stall watch before it publishes a turn item.
2066+
const collected = yield* subscription.events.pipe(
2067+
Stream.tap((event) =>
2068+
event.type === "turn_item.updated" ? Queue.offer(published, undefined) : Effect.void,
2069+
),
2070+
Stream.runCollect,
2071+
Effect.forkScoped,
2072+
);
2073+
const adapterQueue = (yield* Ref.get(state)).eventQueues.get(String(providerSessionId));
2074+
assert.isDefined(adapterQueue);
2075+
const providerTurnId = idAllocator.derive.providerTurn({
2076+
driver: CODEX_DRIVER,
2077+
nativeTurnId: "retry-stall-turn",
2078+
});
2079+
const failure = makeProviderFailure({
2080+
message: "Rate limit exceeded",
2081+
code: "rate_limited",
2082+
class: "usage_limit",
2083+
retryable: true,
2084+
});
2085+
const retry = { attempt: 2, maxAttempts: 5, retryDelayMs: 60_000 };
2086+
const retryItem = (status: "running" | "interrupted") =>
2087+
makeProviderRetryTurnItem({
2088+
driver: CODEX_DRIVER,
2089+
threadId,
2090+
runId: null,
2091+
nodeId: null,
2092+
providerThreadId: providerThread.id,
2093+
providerTurnId,
2094+
itemOrdinal: 3,
2095+
failure,
2096+
retry,
2097+
status,
2098+
startedAt: now,
2099+
updatedAt: now,
2100+
});
2101+
yield* Queue.offer(adapterQueue!, {
2102+
type: "turn_item.updated",
2103+
driver: CODEX_DRIVER,
2104+
turnItem: retryItem("running"),
2105+
});
2106+
yield* Queue.take(published);
2107+
const offerOutput = (outputThreadId: ThreadId) =>
2108+
Queue.offer(adapterQueue!, {
2109+
type: "turn_item.updated",
2110+
driver: CODEX_DRIVER,
2111+
turnItem: {
2112+
id: idAllocator.derive.turnItemFromProviderItem({
2113+
driver: CODEX_DRIVER,
2114+
nativeItemId: `retry-stall-output-${outputThreadId}`,
2115+
}),
2116+
threadId: outputThreadId,
2117+
runId: null,
2118+
nodeId: null,
2119+
providerThreadId: providerThread.id,
2120+
providerTurnId,
2121+
nativeItemRef: null,
2122+
parentItemId: null,
2123+
ordinal: 4,
2124+
type: "compaction",
2125+
driver: CODEX_DRIVER,
2126+
status: "running",
2127+
title: "Compacting context",
2128+
startedAt: now,
2129+
completedAt: null,
2130+
updatedAt: now,
2131+
},
2132+
}).pipe(Effect.andThen(Queue.take(published)));
2133+
// The stall window is 15 minutes plus the provider's announced retry delay.
2134+
yield* TestClock.adjust("15 minutes");
2135+
assert.equal((yield* Ref.get(state)).interruptCount, 0);
2136+
// Output on the retry's thread proves the provider is alive, so a new window starts.
2137+
yield* offerOutput(threadId);
2138+
yield* TestClock.adjust("1 minute");
2139+
// Another thread's output lets the timer start its next wait and does not count.
2140+
yield* offerOutput(ThreadId.make("thread-provider-session-manager-retry-stall-other"));
2141+
yield* TestClock.adjust("15 minutes");
2142+
assert.equal((yield* Ref.get(state)).interruptCount, 0);
2143+
yield* TestClock.adjust("1 minute");
2144+
yield* Deferred.await(interrupted);
2145+
yield* Queue.offer(adapterQueue!, {
2146+
type: "turn_item.updated",
2147+
driver: CODEX_DRIVER,
2148+
turnItem: retryItem("interrupted"),
2149+
});
2150+
yield* Queue.offer(adapterQueue!, {
2151+
type: "provider_turn.updated",
2152+
driver: CODEX_DRIVER,
2153+
threadId,
2154+
providerTurn: {
2155+
id: providerTurnId,
2156+
providerThreadId: providerThread.id,
2157+
nodeId: idAllocator.derive.rootNode({
2158+
runId: idAllocator.derive.run({ threadId, ordinal: 1 }),
2159+
}),
2160+
runAttemptId: null,
2161+
nativeTurnRef: null,
2162+
ordinal: 1,
2163+
status: "interrupted",
2164+
startedAt: now,
2165+
completedAt: now,
2166+
},
2167+
});
2168+
yield* Queue.offer(adapterQueue!, {
2169+
type: "turn.terminal",
2170+
driver: CODEX_DRIVER,
2171+
providerThreadId: providerThread.id,
2172+
providerTurnId,
2173+
runOrdinal: 1,
2174+
status: "interrupted",
2175+
failure: null,
2176+
threadDisposition: "reusable",
2177+
});
2178+
yield* Queue.offer(adapterQueue!, {
2179+
type: "provider_session.updated",
2180+
driver: CODEX_DRIVER,
2181+
providerSession: { ...runtime.providerSession, status: "stopped", updatedAt: now },
2182+
});
2183+
yield* Queue.end(adapterQueue!);
2184+
2185+
const events = Array.from(yield* Fiber.join(collected));
2186+
const terminal = events.find((event) => event.type === "turn.terminal");
2187+
const providerTurn = events.find((event) => event.type === "provider_turn.updated");
2188+
assert.equal(
2189+
providerTurn?.type === "provider_turn.updated" ? providerTurn.providerTurn.status : null,
2190+
"failed",
2191+
);
2192+
assert.deepEqual(terminal, {
2193+
type: "turn.terminal",
2194+
driver: CODEX_DRIVER,
2195+
providerThreadId: providerThread.id,
2196+
providerTurnId,
2197+
runOrdinal: 1,
2198+
failureItemOrdinal: 3,
2199+
status: "failed",
2200+
failure: { ...failure, retryable: false },
2201+
retry,
2202+
retryStartedAt: now,
2203+
threadDisposition: "reusable",
2204+
});
2205+
});
2206+
2207+
yield* effect.pipe(
2208+
Effect.provide(
2209+
layerTest({
2210+
state,
2211+
idleTimeoutMs: 60 * 60_000,
2212+
interruptTurn: Deferred.succeed(interrupted, undefined).pipe(Effect.asVoid),
2213+
}),
2214+
),
2215+
);
2216+
}),
2217+
);
2218+
20262219
it.effect(
20272220
"ProviderSessionManagerV2 issues MCP credentials before opening and revokes them on close",
20282221
() =>

0 commit comments

Comments
 (0)