@@ -2214,6 +2214,340 @@ describe("CodexAdapterV2 post-settle continuation", () => {
22142214 ) ,
22152215 ) ;
22162216
2217+ it . effect . each ( [
2218+ "recover" ,
2219+ "completion-only" ,
2220+ "exhaust" ,
2221+ "stop" ,
2222+ "stop-continued" ,
2223+ "steer" ,
2224+ "start-fails" ,
2225+ ] as const ) ( "handles terminal Codex capacity failures when retries %s" , ( outcome ) =>
2226+ Effect . scoped (
2227+ Effect . gen ( function * ( ) {
2228+ const scenario = `capacity-${ outcome } ` ;
2229+ const nativeThreadId = `native-${ scenario } ` ;
2230+ const nativeTurnId = ( attempt : number ) => `${ nativeThreadId } -${ attempt } ` ;
2231+ const prompt = "Continue the task." ;
2232+ const capacityError = {
2233+ message : "Selected model is at capacity. Please try a different model." ,
2234+ codexErrorInfo : "serverOverloaded" ,
2235+ } ;
2236+ const failureEntries = ( attempt : number ) : Array < CodexReplay . CodexAppServerReplayEntry > => [
2237+ {
2238+ type : "emit_inbound" ,
2239+ frame : {
2240+ method : "error" ,
2241+ params : {
2242+ threadId : nativeThreadId ,
2243+ turnId : nativeTurnId ( attempt ) ,
2244+ willRetry : false ,
2245+ error : capacityError ,
2246+ } ,
2247+ } ,
2248+ } ,
2249+ {
2250+ type : "emit_inbound" ,
2251+ frame : {
2252+ method : "turn/completed" ,
2253+ params : {
2254+ threadId : nativeThreadId ,
2255+ turn : {
2256+ ...makeCodexReplayTurn ( { id : nativeTurnId ( attempt ) , status : "failed" } ) ,
2257+ error : capacityError ,
2258+ } ,
2259+ } ,
2260+ } ,
2261+ } ,
2262+ ] ;
2263+ const entries = codexReplayPreamble ( {
2264+ nativeThreadId,
2265+ nativeTurnId : nativeTurnId ( 0 ) ,
2266+ prompt,
2267+ } ) ;
2268+ entries . push (
2269+ ...( outcome === "completion-only" ? failureEntries ( 0 ) . slice ( 1 ) : failureEntries ( 0 ) ) ,
2270+ ) ;
2271+ if ( outcome === "completion-only" ) entries . push ( ...failureEntries ( 0 ) . slice ( 1 ) ) ;
2272+ const retries = outcome === "exhaust" ? 5 : outcome === "stop" ? 0 : 1 ;
2273+ const steeredText = "Use the smaller change." ;
2274+ for ( let attempt = 1 ; attempt <= retries ; attempt ++ ) {
2275+ const start = codexReplayPreamble ( {
2276+ nativeThreadId,
2277+ nativeTurnId : nativeTurnId ( attempt ) ,
2278+ prompt,
2279+ } ) . slice ( - 3 ) ;
2280+ for ( const entry of start ) {
2281+ if ( entry . type === "expect_outbound" ) {
2282+ entries . push ( {
2283+ ...entry ,
2284+ frame : {
2285+ id : 3 + attempt ,
2286+ method : "turn/start" ,
2287+ params : {
2288+ threadId : nativeThreadId ,
2289+ input : [ ] ,
2290+ cwd : "/workspace" ,
2291+ model : "gpt-5.4" ,
2292+ approvalPolicy : "never" ,
2293+ approvalsReviewer : "user" ,
2294+ sandboxPolicy : { type : "dangerFullAccess" } ,
2295+ summary : "detailed" ,
2296+ collaborationMode : { mode : "default" , settings : { model : "gpt-5.4" } } ,
2297+ } ,
2298+ } ,
2299+ } ) ;
2300+ } else if ( entry . type === "emit_inbound" && entry . label === "turn/start" ) {
2301+ entries . push (
2302+ outcome === "start-fails"
2303+ ? {
2304+ type : "emit_inbound" ,
2305+ frame : {
2306+ id : 3 + attempt ,
2307+ error : { code : - 32603 , message : "Start failed" } ,
2308+ } ,
2309+ }
2310+ : withReplayRequestId ( entry , 3 + attempt ) ,
2311+ ) ;
2312+ } else if ( outcome !== "start-fails" ) {
2313+ entries . push ( entry ) ;
2314+ }
2315+ }
2316+ if ( outcome === "exhaust" ) entries . push ( ...failureEntries ( attempt ) ) ;
2317+ if ( outcome === "recover" || outcome === "completion-only" ) {
2318+ entries . push ( {
2319+ type : "emit_inbound" ,
2320+ frame : {
2321+ method : "turn/completed" ,
2322+ params : {
2323+ threadId : nativeThreadId ,
2324+ turn : makeCodexReplayTurn ( { id : nativeTurnId ( attempt ) , status : "completed" } ) ,
2325+ } ,
2326+ } ,
2327+ } ) ;
2328+ }
2329+ if ( outcome === "stop-continued" || outcome === "steer" ) {
2330+ const steering = outcome === "steer" ;
2331+ entries . push (
2332+ {
2333+ type : "expect_outbound" ,
2334+ frame : {
2335+ id : 5 ,
2336+ method : steering ? "turn/steer" : "turn/interrupt" ,
2337+ params : steering
2338+ ? {
2339+ threadId : nativeThreadId ,
2340+ expectedTurnId : nativeTurnId ( attempt ) ,
2341+ input : [ { type : "text" , text : steeredText } ] ,
2342+ }
2343+ : { threadId : nativeThreadId , turnId : nativeTurnId ( attempt ) } ,
2344+ } ,
2345+ } ,
2346+ {
2347+ type : "emit_inbound" ,
2348+ frame : { id : 5 , result : steering ? { turnId : nativeTurnId ( attempt ) } : { } } ,
2349+ } ,
2350+ {
2351+ type : "emit_inbound" ,
2352+ frame : {
2353+ method : "turn/completed" ,
2354+ params : {
2355+ threadId : nativeThreadId ,
2356+ turn : makeCodexReplayTurn ( {
2357+ id : nativeTurnId ( attempt ) ,
2358+ status : steering ? "completed" : "interrupted" ,
2359+ } ) ,
2360+ } ,
2361+ } ,
2362+ } ,
2363+ ) ;
2364+ }
2365+ }
2366+ const milestones = yield * Queue . unbounded < ProviderAdapter . ProviderAdapterV2Event > ( ) ;
2367+ const starts : Array < unknown > = [ ] ;
2368+ const harness = yield * makeCodexReplayHarness (
2369+ makeCodexReplayTranscript ( { scenario, entries } ) ,
2370+ ( event ) =>
2371+ event . type === "turn.terminal" ||
2372+ ( event . type === "turn_item.updated" &&
2373+ event . turnItem . type === "error" &&
2374+ event . turnItem . status === "running" )
2375+ ? Queue . offer ( milestones , event )
2376+ : Effect . void ,
2377+ ( method , params ) =>
2378+ Effect . sync ( ( ) => {
2379+ if ( method === "turn/start" ) starts . push ( params ) ;
2380+ } ) ,
2381+ ) ;
2382+ const turnInput = makeCodexTestTurnInput ( {
2383+ threadId : harness . threadId ,
2384+ providerThread : harness . providerThread ,
2385+ now : yield * DateTime . now ,
2386+ attemptId : RunAttemptId . make ( scenario ) ,
2387+ text : prompt ,
2388+ } ) ;
2389+ yield * harness . runtime . startTurn ( turnInput ) ;
2390+ for ( let attempt = 1 ; attempt <= Math . max ( retries , 1 ) ; attempt ++ ) {
2391+ const milestone = yield * Queue . take ( milestones ) ;
2392+ assert . equal ( milestone . type , "turn_item.updated" ) ;
2393+ if ( milestone . type !== "turn_item.updated" || milestone . turnItem . type !== "error" ) return ;
2394+ const providerTurnId = milestone . turnItem . providerTurnId ;
2395+ assert . isNotNull ( providerTurnId ) ;
2396+ if ( providerTurnId === null ) return ;
2397+ assert . deepEqual ( milestone . turnItem . retry , {
2398+ attempt,
2399+ maxAttempts : 5 ,
2400+ retryDelayMs : Math . min ( 2_000 * 2 ** ( attempt - 1 ) , 30_000 ) ,
2401+ } ) ;
2402+ assert . isEmpty ( harness . terminalEvents ( ) ) ;
2403+ if ( outcome === "stop" ) {
2404+ yield * harness . runtime . interruptTurn ( {
2405+ providerThread : harness . providerThread ,
2406+ providerTurnId,
2407+ } ) ;
2408+ }
2409+ const steering =
2410+ outcome === "steer"
2411+ ? yield * harness . runtime
2412+ . steerTurn ( {
2413+ ...turnInput ,
2414+ providerTurnId,
2415+ message : { ...turnInput . message , text : steeredText } ,
2416+ } )
2417+ . pipe ( Effect . forkScoped )
2418+ : undefined ;
2419+ yield * TestClock . adjust ( Math . min ( 2_000 * 2 ** ( attempt - 1 ) , 30_000 ) ) ;
2420+ if ( steering !== undefined ) yield * Fiber . join ( steering ) ;
2421+ if ( outcome === "stop-continued" ) {
2422+ yield * harness . until ( ( ) =>
2423+ harness . events . some (
2424+ ( event ) =>
2425+ event . type === "provider_turn.updated" &&
2426+ event . providerTurn . nativeTurnRef ?. nativeId === nativeTurnId ( 1 ) ,
2427+ ) ,
2428+ ) ;
2429+ yield * harness . runtime . interruptTurn ( {
2430+ providerThread : harness . providerThread ,
2431+ providerTurnId,
2432+ } ) ;
2433+ }
2434+ }
2435+ yield * harness . awaitTerminal ( 1 ) ;
2436+ const terminal = harness . terminalEvents ( ) [ 0 ] ;
2437+ assert . equal (
2438+ terminal ?. status ,
2439+ outcome === "recover" || outcome === "completion-only" || outcome === "steer"
2440+ ? "completed"
2441+ : outcome === "stop" || outcome === "stop-continued"
2442+ ? "interrupted"
2443+ : "failed" ,
2444+ ) ;
2445+ assert . lengthOf ( starts , 1 + retries ) ;
2446+ assert . lengthOf ( harness . terminalEvents ( ) , 1 ) ;
2447+ const retryItems = harness . events . flatMap ( ( event ) =>
2448+ event . type === "turn_item.updated" && event . turnItem . type === "error"
2449+ ? [ event . turnItem ]
2450+ : [ ] ,
2451+ ) ;
2452+ assert . isTrue (
2453+ [ "completed" , "failed" , "interrupted" ] . includes ( retryItems . at ( - 1 ) ?. status ?? "" ) ,
2454+ ) ;
2455+ assert . lengthOf ( new Set ( retryItems . map ( ( item ) => item . id ) ) , 1 ) ;
2456+
2457+ if ( outcome === "recover" ) {
2458+ const fs = yield * FileSystem . FileSystem ;
2459+ const cwd = yield * fs . makeTempDirectoryScoped ( { prefix : "t3-capacity-retry-" } ) ;
2460+ const localTranscript = yield * decodeReplayTranscriptJson (
2461+ ( yield * encodeReplayTranscriptJson (
2462+ makeCodexReplayTranscript ( { scenario, entries } ) ,
2463+ ) ) . replaceAll ( yield * encodeStringJson ( "/workspace" ) , yield * encodeStringJson ( cwd ) ) ,
2464+ ) ;
2465+ const driver = yield * CodexReplay . makeReplayDriver ( localTranscript ) ;
2466+ yield * Effect . gen ( function * ( ) {
2467+ const orchestrator = yield * Orchestrator . OrchestratorV2 ;
2468+ const worker = yield * EffectWorker . OrchestrationEffectWorkerV2 ;
2469+ const threadId = ThreadId . make ( "thread:capacity-retry" ) ;
2470+ yield * orchestrator . dispatch ( {
2471+ type : "thread.create" ,
2472+ commandId : CommandId . make ( "create-capacity-retry" ) ,
2473+ threadId,
2474+ projectId : ProjectId . make ( "project:capacity-retry" ) ,
2475+ title : "Capacity retry" ,
2476+ modelSelection : CODEX_TEST_MODEL_SELECTION ,
2477+ runtimeMode : "full-access" ,
2478+ interactionMode : "default" ,
2479+ branch : null ,
2480+ worktreePath : cwd ,
2481+ createdBy : "user" ,
2482+ creationSource : "web" ,
2483+ } ) ;
2484+ const retrying = yield * orchestrator . streamDomainEvents . pipe (
2485+ Stream . filter (
2486+ ( event ) =>
2487+ event . type === "turn-item.updated" &&
2488+ event . payload . type === "error" &&
2489+ event . payload . retry != null &&
2490+ event . payload . status === "running" ,
2491+ ) ,
2492+ Stream . runHead ,
2493+ Effect . forkChild ( { startImmediately : true } ) ,
2494+ ) ;
2495+ yield * orchestrator . dispatch ( {
2496+ type : "message.dispatch" ,
2497+ commandId : CommandId . make ( "start-capacity-retry" ) ,
2498+ threadId,
2499+ messageId : MessageId . make ( "message:capacity-retry" ) ,
2500+ text : prompt ,
2501+ attachments : [ ] ,
2502+ createdBy : "user" ,
2503+ creationSource : "web" ,
2504+ dispatchMode : { type : "start_immediately" } ,
2505+ } ) ;
2506+ yield * worker . drain ( ) ;
2507+ yield * Fiber . join ( retrying ) ;
2508+ assert . equal (
2509+ ( yield * orchestrator . getThreadProjection ( threadId ) ) . runs . at ( - 1 ) ?. status ,
2510+ "running" ,
2511+ ) ;
2512+ const completed = yield * orchestrator . streamDomainEvents . pipe (
2513+ Stream . filter (
2514+ ( event ) => event . type === "run.updated" && event . payload . status === "waiting" ,
2515+ ) ,
2516+ Stream . runHead ,
2517+ Effect . forkChild ( { startImmediately : true } ) ,
2518+ ) ;
2519+ yield * TestClock . adjust ( "2 seconds" ) ;
2520+ yield * Fiber . join ( completed ) ;
2521+ yield * worker . drain ( ) ;
2522+ const projection = yield * orchestrator . getThreadProjection ( threadId ) ;
2523+ assert . equal ( projection . runs . at ( - 1 ) ?. status , "completed" ) ;
2524+ assert . deepEqual (
2525+ projection . providerTurns . map ( ( turn ) => turn . status ) ,
2526+ [ "failed" , "completed" ] ,
2527+ ) ;
2528+ assert . lengthOf (
2529+ projection . messages . filter ( ( message ) => message . role === "user" ) ,
2530+ 1 ,
2531+ ) ;
2532+ assert . isNull ( ( yield * Ref . get ( driver . state ) ) . failure ) ;
2533+ } ) . pipe (
2534+ Effect . provide (
2535+ ProviderReplayHarness . layerWithRegistry (
2536+ { name : "codex-capacity-retry" , runtimePolicyOverride : { cwd } } ,
2537+ CodexAdapterV2Testkit . layer ( { transcript : localTranscript , driver } ) ,
2538+ { runEffectWorker : false } ,
2539+ ) ,
2540+ ) ,
2541+ ) ;
2542+ }
2543+ } ) . pipe (
2544+ Effect . provide (
2545+ Layer . mergeAll ( IdAllocator . layer , McpProviderSessions . layer , NodeServices . layer ) ,
2546+ ) ,
2547+ ) ,
2548+ ) ,
2549+ ) ;
2550+
22172551 it . effect ( "resolves retryable app-server errors on resumed provider activity" , ( ) =>
22182552 Effect . scoped (
22192553 Effect . gen ( function * ( ) {
0 commit comments