55 isProviderNativeSubagentThread ,
66 type OrchestrationV2DomainEvent ,
77 type ProviderThreadId ,
8+ type RunAttemptId ,
89 type OrchestrationV2RestartCancelledBackgroundWork ,
910 type OrchestrationV2Subagent ,
1011 type OrchestrationV2ThreadProjection ,
@@ -23,7 +24,11 @@ import * as EventSink from "./EventSink.ts";
2324import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator" ;
2425import * as ProjectionStore from "./ProjectionStore.ts" ;
2526import * as ServerSettings from "../serverSettings.ts" ;
26- import { restartContinuationRun } from "./RestartContinuation.ts" ;
27+ import {
28+ restartContinuationBlocked ,
29+ restartContinuationRun ,
30+ restartStopRequested ,
31+ } from "./RestartContinuation.ts" ;
2732import { isBufferedProviderNotification } from "./NotificationMailbox.ts" ;
2833import { queuedRunsInDeliveryOrder } from "./QueuedRunOrder.ts" ;
2934import {
@@ -53,6 +58,8 @@ export class ProviderRuntimeRecoveryError extends Schema.TaggedError<ProviderRun
5358
5459const isProviderRuntimeRecoveryError = Schema . is ( ProviderRuntimeRecoveryError ) ;
5560
61+ const MAX_RESENDS = 3 ;
62+
5663export interface ProviderRuntimeRecoverySummary {
5764 readonly terminalizedRuns : number ;
5865 readonly stoppedSessions : number ;
@@ -205,14 +212,71 @@ export const make = Effect.gen(function* () {
205212 return [ ...sessions . values ( ) ] ;
206213 } ;
207214
215+ /**
216+ * A run is marked running before its provider accepts the prompt, so only a provider turn of
217+ * one of its attempts proves the prompt arrived. Recovery treats the others as never started.
218+ */
219+ const rewindUnacceptedRuns = Effect . fn ( "ProviderRuntimeRecoveryService.rewindUnacceptedRuns" ) (
220+ function * ( projection : ProjectionStore . ProjectionRuntimeRecoveryState ) {
221+ const runIdByAttempt = new Map (
222+ projection . runs . flatMap ( ( run ) =>
223+ ( run . status === "running" || ( run . status === "starting" && run . startedAt !== null ) ) &&
224+ run . activeAttemptId !== null
225+ ? [ [ run . activeAttemptId , run . id ] as const ]
226+ : [ ] ,
227+ ) ,
228+ ) ;
229+ if ( runIdByAttempt . size === 0 ) return projection ;
230+ const unaccepted = new Set ( runIdByAttempt . values ( ) ) ;
231+ const accept = ( turns : ReadonlyArray < { readonly runAttemptId : RunAttemptId | null } > ) => {
232+ for ( const turn of turns ) {
233+ const runId =
234+ turn . runAttemptId === null ? undefined : runIdByAttempt . get ( turn . runAttemptId ) ;
235+ if ( runId !== undefined ) unaccepted . delete ( runId ) ;
236+ }
237+ } ;
238+ accept ( projection . providerTurns ) ;
239+ if ( unaccepted . size === 0 ) return projection ;
240+ for ( const attempt of projection . attempts ) {
241+ if ( unaccepted . has ( attempt . runId ) ) runIdByAttempt . set ( attempt . id , attempt . runId ) ;
242+ }
243+ // The recovery projection omits ended provider turns, such as one a steer replaced.
244+ accept (
245+ ( yield * projections
246+ . getThreadRecords ( projection . thread . id , [ "providerTurns" ] , {
247+ providerTurnAttemptIds : [ ...runIdByAttempt ] . flatMap ( ( [ attemptId , runId ] ) =>
248+ unaccepted . has ( runId ) ? [ attemptId ] : [ ] ,
249+ ) ,
250+ } )
251+ . pipe (
252+ Effect . mapError (
253+ ( cause ) =>
254+ new ProviderRuntimeRecoveryError ( {
255+ operation : "read-projections" ,
256+ threadId : projection . thread . id ,
257+ cause,
258+ } ) ,
259+ ) ,
260+ ) ) . providerTurns ,
261+ ) ;
262+ return {
263+ ...projection ,
264+ runs : projection . runs . map ( ( run ) =>
265+ unaccepted . has ( run . id ) ? { ...run , status : "starting" as const , startedAt : null } : run ,
266+ ) ,
267+ } ;
268+ } ,
269+ ) ;
270+
208271 const reconcileProjection = Effect . fn ( "ProviderRuntimeRecoveryService.reconcileProjection" ) (
209272 function * (
210- projection : ProjectionStore . ProjectionRuntimeRecoveryState ,
273+ stored : ProjectionStore . ProjectionRuntimeRecoveryState ,
211274 trigger : "startup" | "shutdown" ,
212275 continueAfterRestart : boolean ,
213276 stoppedThisPass : ReadonlyMap < string , RecoverySession > = new Map ( ) ,
214277 ) {
215278 const now = yield * DateTime . now ;
279+ const projection = yield * rewindUnacceptedRuns ( stored ) ;
216280 let runs = [ ] as Array < OrchestrationV2ThreadProjection [ "runs" ] [ number ] > ;
217281 for ( const run of nonterminalRuns ( projection ) ) {
218282 if ( run . status === "waiting" ) {
@@ -379,7 +443,105 @@ export const make = Effect.gen(function* () {
379443 const retryableRuns = unstartedRuns . filter ( ( run ) =>
380444 retryableMessageIds . has ( run . userMessageId ) ,
381445 ) ;
382- const retryableRunIds = new Set ( retryableRuns . map ( ( run ) => run . id ) ) ;
446+ // Nobody reviews a delegated, agent-sent or scheduled prompt, so it is sent again instead.
447+ const unattendedRuns = continueAfterRestart
448+ ? unstartedRuns . filter ( ( run ) =>
449+ queuedMessages . some (
450+ ( message ) =>
451+ message . id === run . userMessageId &&
452+ ! retryableMessageIds . has ( message . id ) &&
453+ message . notification === undefined &&
454+ message . delegatedCompletion === undefined &&
455+ ( message . scheduledTaskId !== undefined || message . creationSource === "mcp" ) ,
456+ ) ,
457+ )
458+ : [ ] ;
459+ const resentRuns : Array < OrchestrationV2ThreadProjection [ "runs" ] [ number ] > = [ ] ;
460+ const resends : Array < EffectOutbox . PendingOrchestrationEffectV2 > = [ ] ;
461+ for ( const run of unattendedRuns ) {
462+ if (
463+ restartContinuationBlocked ( projection , run , now , false ) ||
464+ ( yield * restartStopRequested ( projections . getThreadRecords , projection . thread , run ) . pipe (
465+ Effect . mapError (
466+ ( cause ) =>
467+ new ProviderRuntimeRecoveryError ( {
468+ operation : "read-projections" ,
469+ threadId : projection . thread . id ,
470+ cause,
471+ } ) ,
472+ ) ,
473+ ) )
474+ )
475+ continue ;
476+ const resendCommandId = CommandId . make ( `command:restart-resend:${ run . id } ` ) ;
477+ const earlierResends = yield * outbox . listByCommandId ( resendCommandId ) . pipe (
478+ Effect . mapError (
479+ ( cause ) =>
480+ new ProviderRuntimeRecoveryError ( {
481+ operation : "reconcile" ,
482+ threadId : projection . thread . id ,
483+ cause,
484+ } ) ,
485+ ) ,
486+ ) ;
487+ // A prompt whose start keeps crashing the server would otherwise be sent forever.
488+ if ( earlierResends . filter ( ( effect ) => effect . attemptCount > 0 ) . length >= MAX_RESENDS ) {
489+ yield * Effect . logWarning ( "Not resending a prompt that restarts cut too many times" , {
490+ threadId : projection . thread . id ,
491+ runId : run . id ,
492+ } ) ;
493+ continue ;
494+ }
495+ resentRuns . push ( run ) ;
496+ resends . push ( {
497+ id : `effect:restart-resend:${ run . id } :${ commandId } ` ,
498+ commandId : resendCommandId ,
499+ threadId : projection . thread . id ,
500+ request : { type : "provider-turn.start" , runId : run . id } ,
501+ } ) ;
502+ }
503+ for ( const run of resentRuns ) {
504+ events . push ( {
505+ id : yield * allocateEventId ( ) ,
506+ type : "run.updated" ,
507+ threadId : projection . thread . id ,
508+ runId : run . id ,
509+ providerInstanceId : run . providerInstanceId ,
510+ occurredAt : now ,
511+ payload : run ,
512+ } ) ;
513+ }
514+ const keptRuns = [ ...retryableRuns , ...resentRuns ] ;
515+ for ( const run of keptRuns ) {
516+ for ( const attempt of projection . attempts . filter (
517+ ( candidate ) => candidate . id === run . activeAttemptId && candidate . status === "running" ,
518+ ) ) {
519+ events . push ( {
520+ id : yield * allocateEventId ( ) ,
521+ type : "run-attempt.updated" ,
522+ threadId : projection . thread . id ,
523+ runId : run . id ,
524+ nodeId : attempt . rootNodeId ,
525+ providerInstanceId : run . providerInstanceId ,
526+ occurredAt : now ,
527+ payload : { ...attempt , status : "pending" , startedAt : null } ,
528+ } ) ;
529+ }
530+ for ( const node of projection . nodes . filter (
531+ ( candidate ) => candidate . id === run . rootNodeId && candidate . status === "running" ,
532+ ) ) {
533+ events . push ( {
534+ id : yield * allocateEventId ( ) ,
535+ type : "node.updated" ,
536+ threadId : projection . thread . id ,
537+ runId : run . id ,
538+ nodeId : node . id ,
539+ providerInstanceId : run . providerInstanceId ,
540+ occurredAt : now ,
541+ payload : { ...node , status : "pending" , startedAt : null } ,
542+ } ) ;
543+ }
544+ }
383545 const retryableItems =
384546 retryableRuns . length === 0
385547 ? [ ]
@@ -416,7 +578,7 @@ export const make = Effect.gen(function* () {
416578 } ,
417579 } ) ;
418580 }
419- runs = runs . filter ( ( run ) => ! retryableRunIds . has ( run . id ) ) ;
581+ runs = runs . filter ( ( run ) => ! keptRuns . some ( ( kept ) => kept . id === run . id ) ) ;
420582 const heldRuns = [
421583 ...retryableRuns ,
422584 ...queuedRunsInDeliveryOrder ( { runs : queuedRuns , messages : queuedMessages } ) ,
@@ -875,6 +1037,7 @@ export const make = Effect.gen(function* () {
8751037 closedRequests : requests . length ,
8761038 retiredEffects,
8771039 cancelledEffectIds,
1040+ resends,
8781041 } ;
8791042 } ,
8801043 ) ;
@@ -894,6 +1057,7 @@ export const make = Effect.gen(function* () {
8941057 let stoppedSessions = 0 ;
8951058 let closedRequests = 0 ;
8961059 let retiredEffects = 0 ;
1060+ const resends : Array < EffectOutbox . PendingOrchestrationEffectV2 > = [ ] ;
8971061 const stoppedThisPass = new Map < string , RecoverySession > ( ) ;
8981062 const grouped = trigger === "shutdown" && threadIds . length > 1 ;
8991063 const groupSize = grouped ? 32 : 1 ;
@@ -923,6 +1087,7 @@ export const make = Effect.gen(function* () {
9231087 }
9241088 }
9251089 cancelledEffectIds . push ( ...result . cancelledEffectIds ) ;
1090+ resends . push ( ...result . resends ) ;
9261091 terminalizedRuns += result . terminalizedRuns ;
9271092 stoppedSessions += result . stoppedSessions ;
9281093 closedRequests += result . closedRequests ;
@@ -944,6 +1109,17 @@ export const make = Effect.gen(function* () {
9441109 ( cause ) => new ProviderRuntimeRecoveryError ( { operation : "drain-outbox" , cause } ) ,
9451110 ) ,
9461111 ) ;
1112+ // After the drain, which cancels every unsettled provider start. A crash before this write
1113+ // leaves the run unstarted, so the next recovery sends it again.
1114+ if ( resends . length > 0 ) {
1115+ yield * eventSink
1116+ . writeWithEffects ( { events : [ ] , effects : resends } )
1117+ . pipe (
1118+ Effect . mapError (
1119+ ( cause ) => new ProviderRuntimeRecoveryError ( { operation : "reconcile" , cause } ) ,
1120+ ) ,
1121+ ) ;
1122+ }
9471123 return {
9481124 terminalizedRuns,
9491125 stoppedSessions,
@@ -979,7 +1155,10 @@ export const make = Effect.gen(function* () {
9791155 . continueThreadsAfterServerUpdate
9801156 )
9811157 return ;
982- const run = restartContinuationRun ( projection , yield * DateTime . now ) ;
1158+ const run = restartContinuationRun (
1159+ yield * rewindUnacceptedRuns ( projection ) ,
1160+ yield * DateTime . now ,
1161+ ) ;
9831162 if ( ! run ) return ;
9841163 const commandId = CommandId . make ( `command:restart-prepare:${ run . id } ` ) ;
9851164 yield * eventSink . writeWithEffects ( {
0 commit comments