@@ -563,6 +563,159 @@ it.effect.each(
563563 } ,
564564) ;
565565
566+ it . effect . each ( [ false , true ] ) ( "groups shutdown commits with rollback=%s" , ( fail ) => {
567+ let commits = 0 ;
568+ const tracer = Tracer . make ( {
569+ span ( options ) {
570+ const span = new Tracer . NativeSpan ( options ) ;
571+ const end = span . end . bind ( span ) ;
572+ span . end = ( time , exit ) => {
573+ end ( time , exit ) ;
574+ if ( span . name === "COMMIT" ) commits ++ ;
575+ } ;
576+ return span ;
577+ } ,
578+ } ) ;
579+ return Effect . gen ( function * ( ) {
580+ const sink = yield * EventSink . EventSinkV2 ;
581+ const events = yield * EventStore . EventStoreV2 ;
582+ const projections = yield * ProjectionStore . ProjectionStoreV2 ;
583+ const sql = yield * SqlClient . SqlClient ;
584+ const outbox = yield * EffectOutbox . EffectOutboxV2 ;
585+ const now = yield * DateTime . now ;
586+ const threads = Array . from ( { length : 65 } , ( _ , i ) =>
587+ ThreadId . make ( `shutdown-group:${ String ( i ) . padStart ( 2 , "0" ) } ` ) ,
588+ ) ;
589+ const sharedSession = {
590+ id : ProviderSessionId . make ( "shutdown-shared" ) ,
591+ driver : providerDriver ,
592+ providerInstanceId,
593+ status : "ready" as const ,
594+ cwd : process . cwd ( ) ,
595+ model : modelSelection . model ,
596+ capabilities : CodexProviderCapabilitiesV2 ,
597+ createdAt : now ,
598+ updatedAt : now ,
599+ lastError : null ,
600+ } ;
601+ yield * sink . write ( {
602+ events : threads . flatMap ( ( threadId , i ) : OrchestrationV2DomainEvent [ ] => [
603+ threadCreatedEvent ( { id : `shutdown-create:${ i } ` , thread : makeThread ( threadId , now ) , now } ) ,
604+ ...( i < 2
605+ ? [
606+ {
607+ id : EventId . make ( `shutdown-session:${ i } ` ) ,
608+ type : "provider-session.attached" as const ,
609+ threadId,
610+ occurredAt : now ,
611+ payload : sharedSession ,
612+ } ,
613+ ]
614+ : [ ] ) ,
615+ {
616+ id : EventId . make ( `shutdown-provider:${ i } ` ) ,
617+ type : "provider-thread.updated" ,
618+ threadId,
619+ occurredAt : now ,
620+ payload : {
621+ id : ProviderThreadId . make ( `shutdown-provider:${ i } ` ) ,
622+ driver : providerDriver ,
623+ providerInstanceId,
624+ providerSessionId : i < 2 ? sharedSession . id : null ,
625+ appThreadId : threadId ,
626+ ownerNodeId : null ,
627+ nativeThreadRef : null ,
628+ nativeConversationHeadRef : null ,
629+ status : i === 1 || i === 32 ? "idle" : "active" ,
630+ firstRunOrdinal : null ,
631+ lastRunOrdinal : null ,
632+ handoffIds : [ ] ,
633+ forkedFrom : null ,
634+ createdAt : now ,
635+ updatedAt : now ,
636+ } ,
637+ } ,
638+ ] ) ,
639+ } ) ;
640+ yield * outbox . enqueue (
641+ threads . map ( ( threadId , i ) => ( {
642+ id : `shutdown-effect:${ i } ` ,
643+ commandId : CommandId . make ( `shutdown-command:${ i } ` ) ,
644+ threadId,
645+ request : { type : "provider-turn.start" as const , runId : RunId . make ( `shutdown-run:${ i } ` ) } ,
646+ } ) ) ,
647+ ) ;
648+ let eventlessNotifications = 0 ;
649+ let armed = fail ;
650+ const recovery = yield * ProviderRuntimeRecovery . make . pipe (
651+ Effect . provide ( ServerSettings . layerTest ( ) ) ,
652+ Effect . provideService ( EffectOutbox . EffectOutboxV2 , {
653+ ...outbox ,
654+ signalCancellations : ( ids ) =>
655+ Effect . gen ( function * ( ) {
656+ assert . isTrue ( Option . isNone ( yield * Effect . serviceOption ( sql . transactionService ) ) ) ;
657+ eventlessNotifications += ids . length ;
658+ yield * outbox . signalCancellations ( ids ) ;
659+ } ) ,
660+ } ) ,
661+ Effect . provideService (
662+ EffectWorker . OrchestrationEffectWorkerV2 ,
663+ EffectWorker . OrchestrationEffectWorkerV2 . of ( {
664+ awaitWork : Effect . void ,
665+ runRecoveryOnce : Effect . succeed ( false ) ,
666+ runOnce : Effect . succeed ( false ) ,
667+ nextClaimableAt : Effect . succeedNone ,
668+ drain : ( ) => Effect . succeed ( 0 ) ,
669+ } ) ,
670+ ) ,
671+ Effect . provideService ( EventSink . EventSinkV2 , {
672+ ...sink ,
673+ commitCommand : ( input ) =>
674+ armed && input . threadId === threads [ 33 ]
675+ ? Effect . die ( "injected shutdown failure" )
676+ : sink . commitCommand ( input ) ,
677+ } ) ,
678+ ) ;
679+ commits = 0 ;
680+ const result = yield * recovery . reconcile ( "shutdown" ) . pipe ( Effect . exit ) ;
681+ assert . equal ( Exit . isFailure ( result ) , fail ) ;
682+ assert . equal ( commits , fail ? 1 : 3 ) ;
683+ const receipts =
684+ yield * sql `SELECT aggregate_id FROM orchestration_command_receipts WHERE command_type = 'provider-runtime.reconcile' ORDER BY aggregate_id` ;
685+ assert . deepEqual (
686+ receipts . map ( ( r ) => r . aggregate_id ) ,
687+ threads . slice ( 0 , fail ? 32 : 65 ) . filter ( ( _ , i ) => i !== 1 && i !== 32 ) ,
688+ ) ;
689+ assert . deepEqual (
690+ yield * projections . getRecoveryThreadIds ( "runtime" ) ,
691+ fail ? threads . slice ( 32 ) : [ ] ,
692+ ) ;
693+ assert . equal ( eventlessNotifications , fail ? 1 : 2 ) ;
694+ for ( const [ i ] of threads . entries ( ) ) {
695+ const effect = Option . getOrThrow ( yield * outbox . get ( `shutdown-effect:${ i } ` ) ) ;
696+ assert . equal ( effect . status , ! fail || i < 32 ? "cancelled" : "pending" ) ;
697+ }
698+ assert . equal (
699+ ( yield * events . read ( { afterSequence : 0 } ) . pipe ( Stream . runCollect ) ) . filter (
700+ ( e ) =>
701+ e . event . type === "provider-session.updated" && e . event . payload . id === sharedSession . id ,
702+ ) . length ,
703+ 1 ,
704+ ) ;
705+ armed = false ;
706+ yield * recovery . reconcile ( "shutdown" ) ;
707+ assert . deepEqual ( yield * projections . getRecoveryThreadIds ( "runtime" ) , [ ] ) ;
708+ const highWater = yield * sink . latestSequence ( ) ;
709+ yield * recovery . reconcile ( "shutdown" ) ;
710+ assert . equal ( yield * sink . latestSequence ( ) , highWater ) ;
711+ } ) . pipe (
712+ Effect . provide ( Layer . fresh ( layerTest ) ) ,
713+ Effect . withTracer ( tracer ) ,
714+ Effect . withTracerEnabled ( true ) ,
715+ Effect . provideService ( Statement . SpanPropagationEnabled , true ) ,
716+ ) ;
717+ } ) ;
718+
566719it . layer ( layerTest ) ( "orchestration V2 foundation persistence" , ( it ) => {
567720 it . effect ( "projects oversized tool bodies once before replay and live RPC retention" , ( ) =>
568721 Effect . scoped (
0 commit comments