@@ -55,6 +55,10 @@ import * as ThreadCommandExecutor from "./ThreadCommandExecutor.ts";
5555import * as ProviderSessionManager from "./ProviderSessionManager.ts" ;
5656import * as CommandOutputHub from "./CommandOutputHub.ts" ;
5757import * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter" ;
58+ import {
59+ makeProviderFailure ,
60+ makeProviderRetryTurnItem ,
61+ } from "@t3tools/provider-core/server/failure" ;
5862
5963const layerTestDatabase = SqlitePersistence . layerMemory ;
6064const layerTestStores = Layer . merge ( EventStore . layer , ProjectionStore . layer ) . pipe (
@@ -2023,6 +2027,195 @@ it.effect("ProviderSessionManagerV2 drains subscribers when the provider stops",
20232027 } ) ,
20242028) ;
20252029
2030+ it . effect ( "ProviderSessionManagerV2 fails a turn whose provider retry never resumes" , ( ) =>
2031+ Effect . gen ( function * ( ) {
2032+ const state = yield * Ref . make ( emptyState ) ;
2033+ const interrupted = yield * Deferred . make < void > ( ) ;
2034+ const effect = Effect . gen ( function * ( ) {
2035+ const eventSink = yield * EventSink . EventSinkV2 ;
2036+ const idAllocator = yield * IdAllocator . IdAllocatorV2 ;
2037+ const manager = yield * ProviderSessionManager . ProviderSessionManagerV2 ;
2038+ const now = yield * DateTime . now ;
2039+ const threadId = ThreadId . make ( "thread-provider-session-manager-retry-stall" ) ;
2040+ const providerSessionId = yield * idAllocator . allocate . providerSession ( {
2041+ providerInstanceId : modelSelection . instanceId ,
2042+ threadId,
2043+ } ) ;
2044+ const providerThread = makeProviderThread ( { idAllocator, threadId, providerSessionId, now } ) ;
2045+ yield * eventSink . write ( {
2046+ events : [
2047+ yield * makeThreadCreatedEvent ( { idAllocator, threadId, now } ) ,
2048+ {
2049+ id : yield * idAllocator . allocate . event ( { threadId } ) ,
2050+ type : "provider-thread.updated" ,
2051+ threadId,
2052+ occurredAt : now ,
2053+ payload : providerThread ,
2054+ } ,
2055+ ] ,
2056+ } ) ;
2057+ const runtime = yield * manager . open ( {
2058+ threadId,
2059+ providerSessionId,
2060+ modelSelection,
2061+ runtimePolicy,
2062+ } ) ;
2063+ const subscription = yield * runtime . subscribeEvents ! ;
2064+ const published = yield * Queue . unbounded < void > ( ) ;
2065+ // The pump updates the stall watch before it publishes a turn item.
2066+ const collected = yield * subscription . events . pipe (
2067+ Stream . tap ( ( event ) =>
2068+ event . type === "turn_item.updated" ? Queue . offer ( published , undefined ) : Effect . void ,
2069+ ) ,
2070+ Stream . runCollect ,
2071+ Effect . forkScoped ,
2072+ ) ;
2073+ const adapterQueue = ( yield * Ref . get ( state ) ) . eventQueues . get ( String ( providerSessionId ) ) ;
2074+ assert . isDefined ( adapterQueue ) ;
2075+ const providerTurnId = idAllocator . derive . providerTurn ( {
2076+ driver : CODEX_DRIVER ,
2077+ nativeTurnId : "retry-stall-turn" ,
2078+ } ) ;
2079+ const failure = makeProviderFailure ( {
2080+ message : "Rate limit exceeded" ,
2081+ code : "rate_limited" ,
2082+ class : "usage_limit" ,
2083+ retryable : true ,
2084+ } ) ;
2085+ const retry = { attempt : 2 , maxAttempts : 5 , retryDelayMs : 60_000 } ;
2086+ const retryItem = ( status : "running" | "interrupted" ) =>
2087+ makeProviderRetryTurnItem ( {
2088+ driver : CODEX_DRIVER ,
2089+ threadId,
2090+ runId : null ,
2091+ nodeId : null ,
2092+ providerThreadId : providerThread . id ,
2093+ providerTurnId,
2094+ itemOrdinal : 3 ,
2095+ failure,
2096+ retry,
2097+ status,
2098+ startedAt : now ,
2099+ updatedAt : now ,
2100+ } ) ;
2101+ yield * Queue . offer ( adapterQueue ! , {
2102+ type : "turn_item.updated" ,
2103+ driver : CODEX_DRIVER ,
2104+ turnItem : retryItem ( "running" ) ,
2105+ } ) ;
2106+ yield * Queue . take ( published ) ;
2107+ const offerOutput = ( outputThreadId : ThreadId ) =>
2108+ Queue . offer ( adapterQueue ! , {
2109+ type : "turn_item.updated" ,
2110+ driver : CODEX_DRIVER ,
2111+ turnItem : {
2112+ id : idAllocator . derive . turnItemFromProviderItem ( {
2113+ driver : CODEX_DRIVER ,
2114+ nativeItemId : `retry-stall-output-${ outputThreadId } ` ,
2115+ } ) ,
2116+ threadId : outputThreadId ,
2117+ runId : null ,
2118+ nodeId : null ,
2119+ providerThreadId : providerThread . id ,
2120+ providerTurnId,
2121+ nativeItemRef : null ,
2122+ parentItemId : null ,
2123+ ordinal : 4 ,
2124+ type : "compaction" ,
2125+ driver : CODEX_DRIVER ,
2126+ status : "running" ,
2127+ title : "Compacting context" ,
2128+ startedAt : now ,
2129+ completedAt : null ,
2130+ updatedAt : now ,
2131+ } ,
2132+ } ) . pipe ( Effect . andThen ( Queue . take ( published ) ) ) ;
2133+ // The stall window is 15 minutes plus the provider's announced retry delay.
2134+ yield * TestClock . adjust ( "15 minutes" ) ;
2135+ assert . equal ( ( yield * Ref . get ( state ) ) . interruptCount , 0 ) ;
2136+ // Output on the retry's thread proves the provider is alive, so a new window starts.
2137+ yield * offerOutput ( threadId ) ;
2138+ yield * TestClock . adjust ( "1 minute" ) ;
2139+ // Another thread's output lets the timer start its next wait and does not count.
2140+ yield * offerOutput ( ThreadId . make ( "thread-provider-session-manager-retry-stall-other" ) ) ;
2141+ yield * TestClock . adjust ( "15 minutes" ) ;
2142+ assert . equal ( ( yield * Ref . get ( state ) ) . interruptCount , 0 ) ;
2143+ yield * TestClock . adjust ( "1 minute" ) ;
2144+ yield * Deferred . await ( interrupted ) ;
2145+ yield * Queue . offer ( adapterQueue ! , {
2146+ type : "turn_item.updated" ,
2147+ driver : CODEX_DRIVER ,
2148+ turnItem : retryItem ( "interrupted" ) ,
2149+ } ) ;
2150+ yield * Queue . offer ( adapterQueue ! , {
2151+ type : "provider_turn.updated" ,
2152+ driver : CODEX_DRIVER ,
2153+ threadId,
2154+ providerTurn : {
2155+ id : providerTurnId ,
2156+ providerThreadId : providerThread . id ,
2157+ nodeId : idAllocator . derive . rootNode ( {
2158+ runId : idAllocator . derive . run ( { threadId, ordinal : 1 } ) ,
2159+ } ) ,
2160+ runAttemptId : null ,
2161+ nativeTurnRef : null ,
2162+ ordinal : 1 ,
2163+ status : "interrupted" ,
2164+ startedAt : now ,
2165+ completedAt : now ,
2166+ } ,
2167+ } ) ;
2168+ yield * Queue . offer ( adapterQueue ! , {
2169+ type : "turn.terminal" ,
2170+ driver : CODEX_DRIVER ,
2171+ providerThreadId : providerThread . id ,
2172+ providerTurnId,
2173+ runOrdinal : 1 ,
2174+ status : "interrupted" ,
2175+ failure : null ,
2176+ threadDisposition : "reusable" ,
2177+ } ) ;
2178+ yield * Queue . offer ( adapterQueue ! , {
2179+ type : "provider_session.updated" ,
2180+ driver : CODEX_DRIVER ,
2181+ providerSession : { ...runtime . providerSession , status : "stopped" , updatedAt : now } ,
2182+ } ) ;
2183+ yield * Queue . end ( adapterQueue ! ) ;
2184+
2185+ const events = Array . from ( yield * Fiber . join ( collected ) ) ;
2186+ const terminal = events . find ( ( event ) => event . type === "turn.terminal" ) ;
2187+ const providerTurn = events . find ( ( event ) => event . type === "provider_turn.updated" ) ;
2188+ assert . equal (
2189+ providerTurn ?. type === "provider_turn.updated" ? providerTurn . providerTurn . status : null ,
2190+ "failed" ,
2191+ ) ;
2192+ assert . deepEqual ( terminal , {
2193+ type : "turn.terminal" ,
2194+ driver : CODEX_DRIVER ,
2195+ providerThreadId : providerThread . id ,
2196+ providerTurnId,
2197+ runOrdinal : 1 ,
2198+ failureItemOrdinal : 3 ,
2199+ status : "failed" ,
2200+ failure : { ...failure , retryable : false } ,
2201+ retry,
2202+ retryStartedAt : now ,
2203+ threadDisposition : "reusable" ,
2204+ } ) ;
2205+ } ) ;
2206+
2207+ yield * effect . pipe (
2208+ Effect . provide (
2209+ layerTest ( {
2210+ state,
2211+ idleTimeoutMs : 60 * 60_000 ,
2212+ interruptTurn : Deferred . succeed ( interrupted , undefined ) . pipe ( Effect . asVoid ) ,
2213+ } ) ,
2214+ ) ,
2215+ ) ;
2216+ } ) ,
2217+ ) ;
2218+
20262219it . effect (
20272220 "ProviderSessionManagerV2 issues MCP credentials before opening and revokes them on close" ,
20282221 ( ) =>
0 commit comments