@@ -11,7 +11,9 @@ import {
1111 ProviderInstanceId ,
1212 ProviderSessionId ,
1313 ThreadId ,
14+ type OrchestrationV2TurnItem ,
1415 type ProviderThreadId ,
16+ type ProviderTurnId ,
1517} from "@t3tools/contracts" ;
1618import * as Cause from "effect/Cause" ;
1719import * as Clock from "effect/Clock" ;
@@ -64,6 +66,12 @@ const busyTurnPrefix = (providerThreadId: ProviderThreadId) => `${providerThread
6466const busyTurnKey = ( providerThreadId : ProviderThreadId , runOrdinal : number ) =>
6567 `${ busyTurnPrefix ( providerThreadId ) } ${ runOrdinal } ` ;
6668const UNLOAD_THREAD_TIMEOUT_MS = 10 * 1000 ;
69+ // Providers can announce a retry and never resume; Codex fails its own stalls sooner.
70+ const PROVIDER_RETRY_STALL_TIMEOUT = Duration . minutes ( 15 ) ;
71+ type ProviderRetryItem = Extract < OrchestrationV2TurnItem , { readonly type : "error" } > & {
72+ readonly providerTurnId : ProviderTurnId ;
73+ readonly providerThreadId : ProviderThreadId ;
74+ } ;
6775const WORKING_RUN_STATUSES : ReadonlySet < string > = new Set ( [
6876 "preparing" ,
6977 "starting" ,
@@ -2165,16 +2173,128 @@ export const layerWithOptions = (
21652173 ) ,
21662174 ) ;
21672175
2176+ const makeRetryStallWatch = ( entry : LiveSessionEntry ) => {
2177+ const runningRetries = new Map < ProviderTurnId , ProviderRetryItem > ( ) ;
2178+ const stalledRetries = new Map < ProviderTurnId , ProviderRetryItem > ( ) ;
2179+ // Stops the stalled turn; its interrupted terminal is then reported as the retry's failure.
2180+ const stopStalledRetry = Effect . fn ( "ProviderSessionManager.stopStalledRetry" ) (
2181+ function * ( item : ProviderRetryItem ) {
2182+ if ( runningRetries . get ( item . providerTurnId ) !== item ) return ;
2183+ const current = ( yield * Ref . get ( sessions ) ) . get (
2184+ sessionKey ( entry . runtime . providerSessionId ) ,
2185+ ) ;
2186+ if ( current ?. runtime !== entry . runtime ) return ;
2187+ const { providerThread } = yield * projectionStore . getProviderControlContext (
2188+ item . threadId ,
2189+ { providerThreadId : item . providerThreadId , providerTurnId : item . providerTurnId } ,
2190+ ) ;
2191+ if ( providerThread === undefined ) return ;
2192+ stalledRetries . set ( item . providerTurnId , item ) ;
2193+ yield * entry . runtime . interruptTurn ( {
2194+ providerThread,
2195+ providerTurnId : item . providerTurnId ,
2196+ } ) ;
2197+ } ,
2198+ Effect . catchCause ( ( cause ) =>
2199+ Effect . logWarning ( "orchestration-v2.provider-retry-stall-stop-failed" , {
2200+ providerSessionId : entry . runtime . providerSessionId ,
2201+ cause,
2202+ } ) ,
2203+ ) ,
2204+ ) ;
2205+ const trackProviderRetry = (
2206+ event : ProviderAdapter . ProviderAdapterV2Event ,
2207+ ) : {
2208+ readonly event : ProviderAdapter . ProviderAdapterV2Event ;
2209+ readonly armed ?: ProviderRetryItem ;
2210+ } => {
2211+ if ( event . type === "turn_item.updated" ) {
2212+ const item = event . turnItem ;
2213+ if (
2214+ item . type !== "error" ||
2215+ item . retry === undefined ||
2216+ item . providerTurnId === null ||
2217+ item . providerThreadId === null
2218+ )
2219+ return { event } ;
2220+ const retryItem = {
2221+ ...item ,
2222+ providerTurnId : item . providerTurnId ,
2223+ providerThreadId : item . providerThreadId ,
2224+ } ;
2225+ if ( item . status === "running" ) {
2226+ runningRetries . set ( retryItem . providerTurnId , retryItem ) ;
2227+ return { event, armed : retryItem } ;
2228+ }
2229+ runningRetries . delete ( retryItem . providerTurnId ) ;
2230+ // Adapters mark the retry stopped before the interrupted terminal; only recovery clears it.
2231+ if ( item . status === "completed" ) stalledRetries . delete ( retryItem . providerTurnId ) ;
2232+ return { event } ;
2233+ }
2234+ if ( event . type === "provider_turn.updated" ) {
2235+ if ( event . providerTurn . status === "running" ) return { event } ;
2236+ runningRetries . delete ( event . providerTurn . id ) ;
2237+ const stalled = stalledRetries . has ( event . providerTurn . id ) ;
2238+ return stalled &&
2239+ ( event . providerTurn . status === "interrupted" ||
2240+ event . providerTurn . status === "cancelled" )
2241+ ? { event : { ...event , providerTurn : { ...event . providerTurn , status : "failed" } } }
2242+ : { event } ;
2243+ }
2244+ if ( event . type !== "turn.terminal" ) return { event } ;
2245+ runningRetries . delete ( event . providerTurnId ) ;
2246+ const stalled = stalledRetries . get ( event . providerTurnId ) ;
2247+ stalledRetries . delete ( event . providerTurnId ) ;
2248+ if (
2249+ stalled === undefined ||
2250+ ( event . status !== "interrupted" && event . status !== "cancelled" )
2251+ )
2252+ return { event } ;
2253+ return {
2254+ event : {
2255+ type : "turn.terminal" ,
2256+ driver : event . driver ,
2257+ providerThreadId : event . providerThreadId ,
2258+ providerTurnId : event . providerTurnId ,
2259+ runOrdinal : event . runOrdinal ,
2260+ failureItemOrdinal : stalled . ordinal ,
2261+ status : "failed" ,
2262+ failure : { ...stalled . failure , retryable : false } ,
2263+ ...( stalled . retry === undefined ? { } : { retry : stalled . retry } ) ,
2264+ ...( stalled . startedAt === null ? { } : { retryStartedAt : stalled . startedAt } ) ,
2265+ threadDisposition : event . threadDisposition ,
2266+ } ,
2267+ } ;
2268+ } ;
2269+ return {
2270+ track : trackProviderRetry ,
2271+ arm : ( item : ProviderRetryItem ) =>
2272+ stopStalledRetry ( item ) . pipe (
2273+ Effect . delay (
2274+ Duration . sum (
2275+ PROVIDER_RETRY_STALL_TIMEOUT ,
2276+ Duration . millis ( item . retry ?. retryDelayMs ?? 0 ) ,
2277+ ) ,
2278+ ) ,
2279+ Effect . forkIn ( layerScope , { startImmediately : true } ) ,
2280+ Effect . asVoid ,
2281+ ) ,
2282+ } ;
2283+ } ;
2284+
21682285 const startEventPump = ( entry : LiveSessionEntry ) => {
21692286 let stoppedByProvider = false ;
21702287 let lastSessionStatus = entry . runtime . providerSession . status ;
2288+ const retryStalls = makeRetryStallWatch ( entry ) ;
21712289 return entry . runtime . events . pipe (
2172- Stream . runForEach ( ( event ) => {
2290+ Stream . runForEach ( ( nativeEvent ) => {
21732291 if ( shutdownSignal . received ) return Effect . void ;
21742292 // Live output bypasses persistence and subscribers so floods cannot delay domain events.
2175- if ( event . type === "command_output.delta" ) {
2176- return commandOutput . append ( event ) ;
2293+ if ( nativeEvent . type === "command_output.delta" ) {
2294+ return commandOutput . append ( nativeEvent ) ;
21772295 }
2296+ const { event, armed } = retryStalls . track ( nativeEvent ) ;
2297+ const armRetryStall = armed === undefined ? Effect . void : retryStalls . arm ( armed ) ;
21782298 const releaseLiveOutput =
21792299 event . type === "turn_item.updated" &&
21802300 event . turnItem . type === "command_execution" &&
@@ -2204,6 +2324,7 @@ export const layerWithOptions = (
22042324 ? Effect . void
22052325 : touchActivity ( entry . runtime . providerSessionId , { providerActivity : true } ) ,
22062326 ) . pipe (
2327+ Effect . andThen ( armRetryStall ) ,
22072328 Effect . andThen (
22082329 event . type === "provider_session.updated"
22092330 ? persistProviderSessionUpdate ( entry , event )
0 commit comments