Skip to content

Commit 6c6ea8e

Browse files
committed
fix(server): fail any provider turn stuck in a retry
1 parent 521ac7c commit 6c6ea8e

2 files changed

Lines changed: 253 additions & 3 deletions

File tree

‎apps/server/src/orchestration-v2/ProviderSessionManager.test.ts‎

Lines changed: 129 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,10 @@ import * as ThreadCommandExecutor from "./ThreadCommandExecutor.ts";
5454
import * as ProviderSessionManager from "./ProviderSessionManager.ts";
5555
import * as CommandOutputHub from "./CommandOutputHub.ts";
5656
import * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter";
57+
import {
58+
makeProviderFailure,
59+
makeProviderRetryTurnItem,
60+
} from "@t3tools/provider-core/server/failure";
5761

5862
const layerTestDatabase = SqlitePersistence.layerMemory;
5963
const layerTestStores = Layer.merge(EventStore.layer, ProjectionStore.layer).pipe(
@@ -2022,6 +2026,131 @@ it.effect("ProviderSessionManagerV2 drains subscribers when the provider stops",
20222026
}),
20232027
);
20242028

2029+
it.effect("ProviderSessionManagerV2 fails a turn whose provider retry never resumes", () =>
2030+
Effect.gen(function* () {
2031+
const state = yield* Ref.make(emptyState);
2032+
const interrupted = yield* Deferred.make<void>();
2033+
const effect = Effect.gen(function* () {
2034+
const eventSink = yield* EventSink.EventSinkV2;
2035+
const idAllocator = yield* IdAllocator.IdAllocatorV2;
2036+
const manager = yield* ProviderSessionManager.ProviderSessionManagerV2;
2037+
const now = yield* DateTime.now;
2038+
const threadId = ThreadId.make("thread-provider-session-manager-retry-stall");
2039+
const providerSessionId = yield* idAllocator.allocate.providerSession({
2040+
providerInstanceId: modelSelection.instanceId,
2041+
threadId,
2042+
});
2043+
const providerThread = makeProviderThread({ idAllocator, threadId, providerSessionId, now });
2044+
yield* eventSink.write({
2045+
events: [
2046+
yield* makeThreadCreatedEvent({ idAllocator, threadId, now }),
2047+
{
2048+
id: yield* idAllocator.allocate.event({ threadId }),
2049+
type: "provider-thread.updated",
2050+
threadId,
2051+
occurredAt: now,
2052+
payload: providerThread,
2053+
},
2054+
],
2055+
});
2056+
const runtime = yield* manager.open({
2057+
threadId,
2058+
providerSessionId,
2059+
modelSelection,
2060+
runtimePolicy,
2061+
});
2062+
const subscription = yield* runtime.subscribeEvents!;
2063+
const collected = yield* subscription.events.pipe(Stream.runCollect, Effect.forkScoped);
2064+
const adapterQueue = (yield* Ref.get(state)).eventQueues.get(String(providerSessionId));
2065+
assert.isDefined(adapterQueue);
2066+
const providerTurnId = idAllocator.derive.providerTurn({
2067+
driver: CODEX_DRIVER,
2068+
nativeTurnId: "retry-stall-turn",
2069+
});
2070+
const failure = makeProviderFailure({
2071+
message: "Rate limit exceeded",
2072+
code: "rate_limited",
2073+
class: "usage_limit",
2074+
retryable: true,
2075+
});
2076+
const retry = { attempt: 2, maxAttempts: 5, retryDelayMs: 60_000 };
2077+
const retryItem = (status: "running" | "interrupted") =>
2078+
makeProviderRetryTurnItem({
2079+
driver: CODEX_DRIVER,
2080+
threadId,
2081+
runId: null,
2082+
nodeId: null,
2083+
providerThreadId: providerThread.id,
2084+
providerTurnId,
2085+
itemOrdinal: 3,
2086+
failure,
2087+
retry,
2088+
status,
2089+
startedAt: now,
2090+
updatedAt: now,
2091+
});
2092+
yield* Queue.offer(adapterQueue!, {
2093+
type: "turn_item.updated",
2094+
driver: CODEX_DRIVER,
2095+
turnItem: retryItem("running"),
2096+
});
2097+
// The stall window starts after the provider's announced retry delay.
2098+
yield* TestClock.adjust("15 minutes");
2099+
assert.equal((yield* Ref.get(state)).interruptCount, 0);
2100+
yield* TestClock.adjust("1 minute");
2101+
yield* Deferred.await(interrupted);
2102+
yield* Queue.offer(adapterQueue!, {
2103+
type: "turn_item.updated",
2104+
driver: CODEX_DRIVER,
2105+
turnItem: retryItem("interrupted"),
2106+
});
2107+
yield* Queue.offer(adapterQueue!, {
2108+
type: "turn.terminal",
2109+
driver: CODEX_DRIVER,
2110+
providerThreadId: providerThread.id,
2111+
providerTurnId,
2112+
runOrdinal: 1,
2113+
status: "interrupted",
2114+
failure: null,
2115+
threadDisposition: "reusable",
2116+
});
2117+
yield* Queue.offer(adapterQueue!, {
2118+
type: "provider_session.updated",
2119+
driver: CODEX_DRIVER,
2120+
providerSession: { ...runtime.providerSession, status: "stopped", updatedAt: now },
2121+
});
2122+
yield* Queue.end(adapterQueue!);
2123+
2124+
const terminal = Array.from(yield* Fiber.join(collected)).find(
2125+
(event) => event.type === "turn.terminal",
2126+
);
2127+
assert.deepEqual(terminal, {
2128+
type: "turn.terminal",
2129+
driver: CODEX_DRIVER,
2130+
providerThreadId: providerThread.id,
2131+
providerTurnId,
2132+
runOrdinal: 1,
2133+
failureItemOrdinal: 3,
2134+
status: "failed",
2135+
failure: { ...failure, retryable: false },
2136+
retry,
2137+
retryStartedAt: now,
2138+
threadDisposition: "reusable",
2139+
});
2140+
});
2141+
2142+
yield* effect.pipe(
2143+
Effect.provide(
2144+
layerTest({
2145+
state,
2146+
idleTimeoutMs: 60 * 60_000,
2147+
interruptTurn: Deferred.succeed(interrupted, undefined).pipe(Effect.asVoid),
2148+
}),
2149+
),
2150+
);
2151+
}),
2152+
);
2153+
20252154
it.effect(
20262155
"ProviderSessionManagerV2 issues MCP credentials before opening and revokes them on close",
20272156
() =>

‎apps/server/src/orchestration-v2/ProviderSessionManager.ts‎

Lines changed: 124 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,9 @@ import {
1111
ProviderInstanceId,
1212
ProviderSessionId,
1313
ThreadId,
14+
type OrchestrationV2TurnItem,
1415
type ProviderThreadId,
16+
type ProviderTurnId,
1517
} from "@t3tools/contracts";
1618
import * as Cause from "effect/Cause";
1719
import * as Clock from "effect/Clock";
@@ -64,6 +66,12 @@ const busyTurnPrefix = (providerThreadId: ProviderThreadId) => `${providerThread
6466
const busyTurnKey = (providerThreadId: ProviderThreadId, runOrdinal: number) =>
6567
`${busyTurnPrefix(providerThreadId)}${runOrdinal}`;
6668
const UNLOAD_THREAD_TIMEOUT_MS = 10 * 1000;
69+
// Providers can announce a retry and never resume; Codex fails its own stalls sooner.
70+
const PROVIDER_RETRY_STALL_TIMEOUT = Duration.minutes(15);
71+
type ProviderRetryItem = Extract<OrchestrationV2TurnItem, { readonly type: "error" }> & {
72+
readonly providerTurnId: ProviderTurnId;
73+
readonly providerThreadId: ProviderThreadId;
74+
};
6775
const WORKING_RUN_STATUSES: ReadonlySet<string> = new Set([
6876
"preparing",
6977
"starting",
@@ -2165,16 +2173,128 @@ export const layerWithOptions = (
21652173
),
21662174
);
21672175

2176+
const makeRetryStallWatch = (entry: LiveSessionEntry) => {
2177+
const runningRetries = new Map<ProviderTurnId, ProviderRetryItem>();
2178+
const stalledRetries = new Map<ProviderTurnId, ProviderRetryItem>();
2179+
// Stops the stalled turn; its interrupted terminal is then reported as the retry's failure.
2180+
const stopStalledRetry = Effect.fn("ProviderSessionManager.stopStalledRetry")(
2181+
function* (item: ProviderRetryItem) {
2182+
if (runningRetries.get(item.providerTurnId) !== item) return;
2183+
const current = (yield* Ref.get(sessions)).get(
2184+
sessionKey(entry.runtime.providerSessionId),
2185+
);
2186+
if (current?.runtime !== entry.runtime) return;
2187+
const { providerThread } = yield* projectionStore.getProviderControlContext(
2188+
item.threadId,
2189+
{ providerThreadId: item.providerThreadId, providerTurnId: item.providerTurnId },
2190+
);
2191+
if (providerThread === undefined) return;
2192+
stalledRetries.set(item.providerTurnId, item);
2193+
yield* entry.runtime.interruptTurn({
2194+
providerThread,
2195+
providerTurnId: item.providerTurnId,
2196+
});
2197+
},
2198+
Effect.catchCause((cause) =>
2199+
Effect.logWarning("orchestration-v2.provider-retry-stall-stop-failed", {
2200+
providerSessionId: entry.runtime.providerSessionId,
2201+
cause,
2202+
}),
2203+
),
2204+
);
2205+
const trackProviderRetry = (
2206+
event: ProviderAdapter.ProviderAdapterV2Event,
2207+
): {
2208+
readonly event: ProviderAdapter.ProviderAdapterV2Event;
2209+
readonly armed?: ProviderRetryItem;
2210+
} => {
2211+
if (event.type === "turn_item.updated") {
2212+
const item = event.turnItem;
2213+
if (
2214+
item.type !== "error" ||
2215+
item.retry === undefined ||
2216+
item.providerTurnId === null ||
2217+
item.providerThreadId === null
2218+
)
2219+
return { event };
2220+
const retryItem = {
2221+
...item,
2222+
providerTurnId: item.providerTurnId,
2223+
providerThreadId: item.providerThreadId,
2224+
};
2225+
if (item.status === "running") {
2226+
runningRetries.set(retryItem.providerTurnId, retryItem);
2227+
return { event, armed: retryItem };
2228+
}
2229+
runningRetries.delete(retryItem.providerTurnId);
2230+
// Adapters mark the retry stopped before the interrupted terminal; only recovery clears it.
2231+
if (item.status === "completed") stalledRetries.delete(retryItem.providerTurnId);
2232+
return { event };
2233+
}
2234+
if (event.type === "provider_turn.updated") {
2235+
if (event.providerTurn.status === "running") return { event };
2236+
runningRetries.delete(event.providerTurn.id);
2237+
const stalled = stalledRetries.has(event.providerTurn.id);
2238+
return stalled &&
2239+
(event.providerTurn.status === "interrupted" ||
2240+
event.providerTurn.status === "cancelled")
2241+
? { event: { ...event, providerTurn: { ...event.providerTurn, status: "failed" } } }
2242+
: { event };
2243+
}
2244+
if (event.type !== "turn.terminal") return { event };
2245+
runningRetries.delete(event.providerTurnId);
2246+
const stalled = stalledRetries.get(event.providerTurnId);
2247+
stalledRetries.delete(event.providerTurnId);
2248+
if (
2249+
stalled === undefined ||
2250+
(event.status !== "interrupted" && event.status !== "cancelled")
2251+
)
2252+
return { event };
2253+
return {
2254+
event: {
2255+
type: "turn.terminal",
2256+
driver: event.driver,
2257+
providerThreadId: event.providerThreadId,
2258+
providerTurnId: event.providerTurnId,
2259+
runOrdinal: event.runOrdinal,
2260+
failureItemOrdinal: stalled.ordinal,
2261+
status: "failed",
2262+
failure: { ...stalled.failure, retryable: false },
2263+
...(stalled.retry === undefined ? {} : { retry: stalled.retry }),
2264+
...(stalled.startedAt === null ? {} : { retryStartedAt: stalled.startedAt }),
2265+
threadDisposition: event.threadDisposition,
2266+
},
2267+
};
2268+
};
2269+
return {
2270+
track: trackProviderRetry,
2271+
arm: (item: ProviderRetryItem) =>
2272+
stopStalledRetry(item).pipe(
2273+
Effect.delay(
2274+
Duration.sum(
2275+
PROVIDER_RETRY_STALL_TIMEOUT,
2276+
Duration.millis(item.retry?.retryDelayMs ?? 0),
2277+
),
2278+
),
2279+
Effect.forkIn(layerScope, { startImmediately: true }),
2280+
Effect.asVoid,
2281+
),
2282+
};
2283+
};
2284+
21682285
const startEventPump = (entry: LiveSessionEntry) => {
21692286
let stoppedByProvider = false;
21702287
let lastSessionStatus = entry.runtime.providerSession.status;
2288+
const retryStalls = makeRetryStallWatch(entry);
21712289
return entry.runtime.events.pipe(
2172-
Stream.runForEach((event) => {
2290+
Stream.runForEach((nativeEvent) => {
21732291
if (shutdownSignal.received) return Effect.void;
21742292
// Live output bypasses persistence and subscribers so floods cannot delay domain events.
2175-
if (event.type === "command_output.delta") {
2176-
return commandOutput.append(event);
2293+
if (nativeEvent.type === "command_output.delta") {
2294+
return commandOutput.append(nativeEvent);
21772295
}
2296+
const { event, armed } = retryStalls.track(nativeEvent);
2297+
const armRetryStall = armed === undefined ? Effect.void : retryStalls.arm(armed);
21782298
const releaseLiveOutput =
21792299
event.type === "turn_item.updated" &&
21802300
event.turnItem.type === "command_execution" &&
@@ -2204,6 +2324,7 @@ export const layerWithOptions = (
22042324
? Effect.void
22052325
: touchActivity(entry.runtime.providerSessionId, { providerActivity: true }),
22062326
).pipe(
2327+
Effect.andThen(armRetryStall),
22072328
Effect.andThen(
22082329
event.type === "provider_session.updated"
22092330
? persistProviderSessionUpdate(entry, event)

0 commit comments

Comments
 (0)