@@ -25,6 +25,7 @@ import * as ProjectionStore from "./ProjectionStore.ts";
2525import * as ServerSettings from "../serverSettings.ts" ;
2626import { restartContinuationRun } from "./RestartContinuation.ts" ;
2727import { isBufferedProviderNotification } from "./NotificationMailbox.ts" ;
28+ import { queuedRunsInDeliveryOrder } from "./QueuedRunOrder.ts" ;
2829import {
2930 endOrphanedNativeSubagent ,
3031 endRunlessRootTurns ,
@@ -210,7 +211,7 @@ export const make = Effect.gen(function* () {
210211 stoppedThisPass : ReadonlyMap < string , RecoverySession > = new Map ( ) ,
211212 ) {
212213 const now = yield * DateTime . now ;
213- const runs = [ ] as Array < OrchestrationV2ThreadProjection [ "runs" ] [ number ] > ;
214+ let runs = [ ] as Array < OrchestrationV2ThreadProjection [ "runs" ] [ number ] > ;
214215 for ( const run of nonterminalRuns ( projection ) ) {
215216 if ( run . status === "waiting" ) {
216217 const checkpointEffects = yield * outbox
@@ -325,13 +326,25 @@ export const make = Effect.gen(function* () {
325326 cancelledBackgroundNativeIds . add ( item . nativeItemRef . nativeId ) ;
326327 }
327328 } ;
329+ const unstartedRuns = runs . filter (
330+ ( run ) =>
331+ run . status === "starting" &&
332+ neverStarted ( run ) &&
333+ ! projection . providerTurns . some ( ( turn ) => turn . runAttemptId === run . activeAttemptId ) &&
334+ ! projection . runtimeRequests . some (
335+ ( request ) =>
336+ request . status === "pending" &&
337+ projection . nodes . some ( ( node ) => node . id === request . nodeId && node . runId === run . id ) ,
338+ ) ,
339+ ) ;
328340 const queuedRuns = projection . runs . filter ( ( run ) => run . status === "queued" ) ;
341+ const pendingRuns = [ ...unstartedRuns , ...queuedRuns ] ;
329342 const queuedMessages =
330- queuedRuns . length === 0
343+ pendingRuns . length === 0
331344 ? [ ]
332345 : ( yield * projections
333346 . getThreadRecords ( projection . thread . id , [ "messages" ] , {
334- messageIds : queuedRuns . map ( ( run ) => run . userMessageId ) ,
347+ messageIds : pendingRuns . map ( ( run ) => run . userMessageId ) ,
335348 } )
336349 . pipe (
337350 Effect . mapError (
@@ -346,20 +359,82 @@ export const make = Effect.gen(function* () {
346359 const bufferedMessageIds = new Set (
347360 queuedMessages . filter ( isBufferedProviderNotification ) . map ( ( message ) => message . id ) ,
348361 ) ;
349- for ( const run of queuedRuns ) {
362+ const retryableMessageIds = new Set (
363+ queuedMessages
364+ . filter (
365+ ( message ) =>
366+ message . createdBy === "user" &&
367+ message . scheduledTaskId === undefined &&
368+ message . notification === undefined &&
369+ message . delegatedCompletion === undefined ,
370+ )
371+ . map ( ( message ) => message . id ) ,
372+ ) ;
373+ // A saved user message the provider never accepted stays available through the held queue.
374+ const retryableRuns = unstartedRuns . filter ( ( run ) =>
375+ retryableMessageIds . has ( run . userMessageId ) ,
376+ ) ;
377+ const retryableRunIds = new Set ( retryableRuns . map ( ( run ) => run . id ) ) ;
378+ const retryableItems =
379+ retryableRuns . length === 0
380+ ? [ ]
381+ : ( yield * projections
382+ . getThreadRecords ( projection . thread . id , [ "turnItems" ] , {
383+ turnItemTypes : [ "user_message" ] ,
384+ turnItemRunIds : retryableRuns . map ( ( run ) => run . id ) ,
385+ } )
386+ . pipe (
387+ Effect . mapError (
388+ ( cause ) =>
389+ new ProviderRuntimeRecoveryError ( {
390+ operation : "read-projections" ,
391+ threadId : projection . thread . id ,
392+ cause,
393+ } ) ,
394+ ) ,
395+ ) ) . turnItems ;
396+ for ( const item of retryableItems ) {
397+ if ( item . type !== "user_message" || ! retryableMessageIds . has ( item . messageId ) ) continue ;
398+ events . push ( {
399+ id : yield * allocateEventId ( ) ,
400+ type : "turn-item.updated" ,
401+ threadId : projection . thread . id ,
402+ ...( item . runId === null ? { } : { runId : item . runId } ) ,
403+ occurredAt : now ,
404+ payload : {
405+ ...item ,
406+ inputIntent : "queued_turn" ,
407+ status : "pending" ,
408+ startedAt : null ,
409+ completedAt : null ,
410+ updatedAt : now ,
411+ } ,
412+ } ) ;
413+ }
414+ runs = runs . filter ( ( run ) => ! retryableRunIds . has ( run . id ) ) ;
415+ const heldRuns = [
416+ ...retryableRuns ,
417+ ...queuedRunsInDeliveryOrder ( { runs : queuedRuns , messages : queuedMessages } ) ,
418+ ] ;
419+ for ( const [ index , run ] of heldRuns . entries ( ) ) {
350420 if ( bufferedMessageIds . has ( run . userMessageId ) ) {
351421 runs . push ( run ) ;
352422 continue ;
353423 }
354- if ( run . queueHeld === true ) continue ;
424+ if ( run . queueHeld === true && retryableRuns . length === 0 ) continue ;
355425 events . push ( {
356426 id : yield * allocateEventId ( ) ,
357427 type : "run.updated" ,
358428 threadId : projection . thread . id ,
359429 runId : run . id ,
360430 providerInstanceId : run . providerInstanceId ,
361431 occurredAt : now ,
362- payload : { ...run , queueHeld : true } ,
432+ payload : {
433+ ...run ,
434+ status : "queued" ,
435+ queueHeld : true ,
436+ ...( retryableRuns . length === 0 ? { } : { queuePosition : index + 1 } ) ,
437+ } ,
363438 } ) ;
364439 }
365440 for ( const question of questions ) {
0 commit comments