@@ -20,6 +20,7 @@ import * as Effect from "effect/Effect";
2020import * as Layer from "effect/Layer" ;
2121import * as ServerSettings from "../serverSettings.ts" ;
2222import { restartContinuationRun , continueRestartedRun } from "./RestartContinuation.ts" ;
23+ import { continuationStartRetryDelayMs } from "./RestartBackgroundNote.ts" ;
2324import * as ThreadManagementService from "./ThreadManagementService.ts" ;
2425import * as ProviderRuntimeRecovery from "./ProviderRuntimeRecoveryService.ts" ;
2526import * as ProjectionStore from "./ProjectionStore.ts" ;
@@ -46,6 +47,8 @@ function makeProjection() {
4647 providerInstanceId : instanceId ,
4748 archivedAt : null ,
4849 deletedAt : null ,
50+ lineage : { parentThreadId : null , relationshipToParent : null } ,
51+ forkedFrom : null ,
4952 } ,
5053 runs : [
5154 {
@@ -839,10 +842,13 @@ it.effect("does not cancel or resume a run that completes while shutdown intent
839842 } ) ,
840843) ;
841844
842- const continuationDispatches = ( projection : OrchestrationV2ThreadProjection ) =>
845+ const continuationDispatches = (
846+ projection : OrchestrationV2ThreadProjection ,
847+ sourceRunId : RunId = runId ,
848+ ) =>
843849 Effect . gen ( function * ( ) {
844850 const dispatches : Array < Extract < OrchestrationV2Command , { type : "message.dispatch" } > > = [ ] ;
845- yield * continueRestartedRun ( { threadId, sourceRunId : runId } ) . pipe (
851+ yield * continueRestartedRun ( { threadId, sourceRunId } ) . pipe (
846852 Effect . provide (
847853 Layer . merge (
848854 Layer . mock ( ThreadManagementService . ThreadManagementService ) ( {
@@ -943,6 +949,118 @@ it.effect("continues a resumed queued run that ran after an earlier continuation
943949 } ) ,
944950) ;
945951
952+ it ( "backs off twice from a continuation the provider never took, then lets it fail" , ( ) => {
953+ const base = makeProjection ( ) ;
954+ const failedStart = ( id : string , of : RunId ) => ( {
955+ ...base . runs [ 0 ] ! ,
956+ id : RunId . make ( id ) ,
957+ activeAttemptId : RunAttemptId . make ( `attempt:${ id } ` ) ,
958+ status : "failed" as const ,
959+ restartContinuationOfRunId : of ,
960+ } ) ;
961+ const first = failedStart ( "run:retry-1" , runId ) ;
962+ const second = failedStart ( "run:retry-2" , first . id ) ;
963+ const third = failedStart ( "run:retry-3" , second . id ) ;
964+ const projection = ( failureClass = "provider_error" , providerTurns = base . providerTurns ) =>
965+ ( {
966+ runs : [ { ...base . runs [ 0 ] ! , status : "cancelled" } , first , second , third ] ,
967+ providerTurns,
968+ turnItems : [ first , second , third ] . map ( ( run ) => ( {
969+ id : `error:${ run . id } ` ,
970+ type : "error" ,
971+ status : "failed" ,
972+ runId : run . id ,
973+ nodeId : run . rootNodeId ,
974+ ordinal : 1 ,
975+ updatedAt : now ,
976+ failure : { class : failureClass , message : "Failed." , code : null , retryable : null } ,
977+ } ) ) ,
978+ } ) as unknown as Parameters < typeof continuationStartRetryDelayMs > [ 1 ] ;
979+ assert . deepEqual (
980+ [ first , second , third ] . map ( ( run ) => continuationStartRetryDelayMs ( run , projection ( ) ) ) ,
981+ [ 60_000 , 180_000 , undefined ] ,
982+ ) ;
983+ for ( const failureClass of [ "usage_limit" , "permission_error" , "validation_error" ] )
984+ assert . isUndefined ( continuationStartRetryDelayMs ( first , projection ( failureClass ) ) ) ;
985+ // Once the provider took the prompt, starting again would repeat the turn.
986+ assert . isUndefined (
987+ continuationStartRetryDelayMs (
988+ first ,
989+ projection ( "provider_error" , [
990+ { ...base . providerTurns [ 0 ] ! , runAttemptId : first . activeAttemptId , status : "failed" } ,
991+ ] as never ) ,
992+ ) ,
993+ ) ;
994+ } ) ;
995+
996+ it . effect ( "continues again after a continuation fails before its prompt" , ( ) =>
997+ Effect . gen ( function * ( ) {
998+ const cut = cutMidTurn ( ) ;
999+ const failedStart = {
1000+ ...cut . runs [ 0 ] ! ,
1001+ id : RunId . make ( "run:failed-start" ) ,
1002+ ordinal : 2 ,
1003+ activeAttemptId : RunAttemptId . make ( "attempt:failed-start" ) ,
1004+ status : "failed" ,
1005+ restartContinuationOfRunId : runId ,
1006+ } ;
1007+ const dispatches = yield * continuationDispatches (
1008+ { ...cut , runs : [ ...cut . runs , failedStart ] } as unknown as OrchestrationV2ThreadProjection ,
1009+ failedStart . id ,
1010+ ) ;
1011+ assert . deepEqual (
1012+ dispatches . map ( ( command ) => command . restartContinuationOfRunId ) ,
1013+ [ failedStart . id ] ,
1014+ ) ;
1015+ } ) ,
1016+ ) ;
1017+
1018+ it . effect . each ( [
1019+ [ "running" , null , 0 ] ,
1020+ [ "completed" , "Done." , 1 ] ,
1021+ ] as const ) (
1022+ "a parent's Stop on the task's run blocks the child only while the task is %s" ,
1023+ ( [ status , result , expected ] ) =>
1024+ Effect . gen ( function * ( ) {
1025+ const parentThreadId = ThreadId . make ( "thread:parent" ) ;
1026+ const parentRunId = RunId . make ( "run:parent" ) ;
1027+ const taskId = NodeId . make ( "node:task" ) ;
1028+ const cut = cutMidTurn ( ) ;
1029+ const child = {
1030+ ...cut ,
1031+ thread : {
1032+ ...cut . thread ,
1033+ lineage : { parentThreadId, relationshipToParent : "subagent" } ,
1034+ forkedFrom : { type : "node" , nodeId : taskId } ,
1035+ } ,
1036+ } as unknown as OrchestrationV2ThreadProjection ;
1037+ const parent = {
1038+ thread : { id : parentThreadId } ,
1039+ subagents : [ { id : taskId , runId : parentRunId , origin : "app_owned" , status, result } ] ,
1040+ turnItems : [ { id : "turn-item:stop" , runId : parentRunId , type : "run_interrupt_request" } ] ,
1041+ } ;
1042+ let dispatched = 0 ;
1043+ yield * continueRestartedRun ( { threadId, sourceRunId : runId } ) . pipe (
1044+ Effect . provide (
1045+ Layer . merge (
1046+ Layer . mock ( ThreadManagementService . ThreadManagementService ) ( {
1047+ getThreadRecords : ( id ) =>
1048+ Effect . succeed ( ( id === parentThreadId ? parent : child ) as never ) ,
1049+ recoverDelegatedTask : ( ) => Effect . void ,
1050+ dispatch : ( ) =>
1051+ Effect . sync ( ( ) => {
1052+ dispatched += 1 ;
1053+ return { } as never ;
1054+ } ) ,
1055+ } ) ,
1056+ ServerSettings . layerTest ( { continueThreadsAfterServerUpdate : true } ) ,
1057+ ) ,
1058+ ) ,
1059+ ) ;
1060+ assert . equal ( dispatched , expected ) ;
1061+ } ) ,
1062+ ) ;
1063+
9461064it . effect ( "does not continue a run the user asked to stop before the restart" , ( ) =>
9471065 Effect . gen ( function * ( ) {
9481066 const texts = yield * continuationTexts ( {
0 commit comments