@@ -948,6 +948,10 @@ export const layer: Layer.Layer<
948948 ) ;
949949 const rootTerminalSeen = yield * Ref . make ( false ) ;
950950 const rootRunFinalized = yield * Ref . make ( false ) ;
951+ // Set when an event write fails. The stream then stops writing but
952+ // keeps routing until it has seen the root provider turn, so
953+ // recovery always knows which turn to stop.
954+ const lostIngestion = yield * Ref . make < Cause . Cause < unknown > | null > ( null ) ;
951955 const providerThreadOwnerLost = yield * Ref . make ( false ) ;
952956 const activeChildProviderTurns = yield * Ref . make < ReadonlySet < ProviderTurnId > > ( new Set ( ) ) ;
953957 const activeChildSubagents = yield * Ref . make < ReadonlySet < NodeId > > ( new Set ( ) ) ;
@@ -1271,6 +1275,7 @@ export const layer: Layer.Layer<
12711275 ) ,
12721276 Stream . tap ( ( event ) =>
12731277 Effect . gen ( function * ( ) {
1278+ if ( ( yield * Ref . get ( lostIngestion ) ) !== null ) return ;
12741279 let storedEventCount = 0 ;
12751280 const deliveredEvent = filterAssistantEvent (
12761281 event ,
@@ -1286,37 +1291,45 @@ export const layer: Layer.Layer<
12861291 const isRootProviderThreadUpdate =
12871292 event . type === "provider_thread.updated" &&
12881293 event . providerThread . id === input . providerThread . id ;
1289- const storedEvents = yield * providerEventIngestor . ingestNormalized ( {
1290- analyticsContext : {
1291- modelSelection : input . modelSelection ,
1292- runtimeMode : input . runtimePolicy . runtimeMode ,
1293- interactionMode : input . runtimePolicy . interactionMode ,
1294- } ,
1295- providerSessionId : input . providerSessionId ,
1296- providerInstanceId : input . run . providerInstanceId ,
1297- threadId : input . run . threadId ,
1298- runId : input . run . id ,
1299- nodeId : input . rootNode . id ,
1300- event : deliveredEvent ,
1301- ...( isRootProviderThreadUpdate
1302- ? rootTerminalAlreadySeen
1303- ? {
1304- writeIfProviderThreadOwner : {
1305- providerThreadId : input . providerThread . id ,
1306- runId : input . run . id ,
1307- activeAttemptId : input . attempt . id ,
1308- expectedLastRunOrdinal : input . run . ordinal ,
1309- } ,
1310- }
1311- : {
1312- writeIfRunCurrent : {
1313- runId : input . run . id ,
1314- activeAttemptId : input . attempt . id ,
1315- expectedStatus : "running" as const ,
1316- } ,
1317- }
1318- : { } ) ,
1319- } ) ;
1294+ const storedEvents = yield * providerEventIngestor
1295+ . ingestNormalized ( {
1296+ analyticsContext : {
1297+ modelSelection : input . modelSelection ,
1298+ runtimeMode : input . runtimePolicy . runtimeMode ,
1299+ interactionMode : input . runtimePolicy . interactionMode ,
1300+ } ,
1301+ providerSessionId : input . providerSessionId ,
1302+ providerInstanceId : input . run . providerInstanceId ,
1303+ threadId : input . run . threadId ,
1304+ runId : input . run . id ,
1305+ nodeId : input . rootNode . id ,
1306+ event : deliveredEvent ,
1307+ ...( isRootProviderThreadUpdate
1308+ ? rootTerminalAlreadySeen
1309+ ? {
1310+ writeIfProviderThreadOwner : {
1311+ providerThreadId : input . providerThread . id ,
1312+ runId : input . run . id ,
1313+ activeAttemptId : input . attempt . id ,
1314+ expectedLastRunOrdinal : input . run . ordinal ,
1315+ } ,
1316+ }
1317+ : {
1318+ writeIfRunCurrent : {
1319+ runId : input . run . id ,
1320+ activeAttemptId : input . attempt . id ,
1321+ expectedStatus : "running" as const ,
1322+ } ,
1323+ }
1324+ : { } ) ,
1325+ } )
1326+ . pipe (
1327+ Effect . catchCauseIf (
1328+ ( cause ) => ! Cause . hasInterruptsOnly ( cause ) ,
1329+ ( cause ) => Ref . set ( lostIngestion , cause ) . pipe ( Effect . as ( null ) ) ,
1330+ ) ,
1331+ ) ;
1332+ if ( storedEvents === null ) return ;
13201333 storedEventCount = storedEvents . length ;
13211334 if (
13221335 isRootProviderThreadUpdate &&
@@ -1350,11 +1363,22 @@ export const layer: Layer.Layer<
13501363 yield * trackChildLifecycle ( event , deliveredEvent !== null ) ;
13511364 } ) ,
13521365 ) ,
1353- Stream . takeUntilEffect ( ( ) => shouldStopProviderEventIngestion ) ,
1366+ Stream . takeUntilEffect ( ( ) =>
1367+ Effect . gen ( function * ( ) {
1368+ if ( ( yield * Ref . get ( lostIngestion ) ) !== null ) {
1369+ return ( yield * Ref . get ( eventRouting ) ) . rootProviderTurnId !== null ;
1370+ }
1371+ return yield * shouldStopProviderEventIngestion ;
1372+ } ) ,
1373+ ) ,
13541374 Stream . runDrain ,
13551375 Effect . mapError ( ( cause ) => new RunExecutionIngestError ( { runId : input . run . id , cause } ) ) ,
13561376 Effect . flatMap ( ( ) =>
13571377 Effect . gen ( function * ( ) {
1378+ const lost = yield * Ref . get ( lostIngestion ) ;
1379+ if ( lost !== null ) {
1380+ return yield * new RunExecutionIngestError ( { runId : input . run . id , cause : lost } ) ;
1381+ }
13581382 const terminal = yield * Ref . get ( terminalEvent ) ;
13591383 if ( terminal === null ) {
13601384 return ;
0 commit comments