@@ -7388,15 +7388,17 @@ it.layer(layerTest)("usage-limit recovery", (it) => {
73887388 } ) ,
73897389 ) ;
73907390
7391- it . effect . each ( [ true , false ] ) (
7392- "preserves early recovery with global auto-resume off when retry reports reset : %s" ,
7393- ( retryReportsReset ) =>
7391+ it . effect . each ( [ "reset" , "no-reset" , "snoozed" ] as const ) (
7392+ "preserves early recovery with global auto-resume off: %s" ,
7393+ ( variant ) =>
73947394 Effect . gen ( function * ( ) {
7395+ const retryReportsReset = variant === "reset" ;
7396+ const snoozed = variant === "snoozed" ;
73957397 const orchestrator = yield * Orchestrator . OrchestratorV2 ;
73967398 const events = yield * EventSink . EventSinkV2 ;
73977399 const projections = yield * ProjectionStore . ProjectionStoreV2 ;
7398- const threadId = ThreadId . make ( `recovery:early-reset:${ retryReportsReset } ` ) ;
7399- const projectId = ProjectId . make ( `recovery:project:early-reset:${ retryReportsReset } ` ) ;
7400+ const threadId = ThreadId . make ( `recovery:early-reset:${ variant } ` ) ;
7401+ const projectId = ProjectId . make ( `recovery:project:early-reset:${ variant } ` ) ;
74007402 const now = yield * DateTime . now ;
74017403 yield * seedProject ( {
74027404 projectId,
@@ -7407,7 +7409,7 @@ it.layer(layerTest)("usage-limit recovery", (it) => {
74077409 } ) ;
74087410 yield * orchestrator . dispatch ( {
74097411 type : "thread.create" ,
7410- commandId : CommandId . make ( `recovery:create:early-reset:${ retryReportsReset } ` ) ,
7412+ commandId : CommandId . make ( `recovery:create:early-reset:${ variant } ` ) ,
74117413 threadId,
74127414 projectId,
74137415 title : "Limited thread" ,
@@ -7421,9 +7423,9 @@ it.layer(layerTest)("usage-limit recovery", (it) => {
74217423 } ) ;
74227424 yield * orchestrator . dispatch ( {
74237425 type : "message.dispatch" ,
7424- commandId : CommandId . make ( `recovery:message:early-reset:${ retryReportsReset } ` ) ,
7426+ commandId : CommandId . make ( `recovery:message:early-reset:${ variant } ` ) ,
74257427 threadId,
7426- messageId : MessageId . make ( `recovery:message:early-reset:${ retryReportsReset } ` ) ,
7428+ messageId : MessageId . make ( `recovery:message:early-reset:${ variant } ` ) ,
74277429 text : "Work on this." ,
74287430 attachments : [ ] ,
74297431 dispatchMode : { type : "defer_start" } ,
@@ -7435,19 +7437,19 @@ it.layer(layerTest)("usage-limit recovery", (it) => {
74357437 yield * events . write ( {
74367438 events : [
74377439 {
7438- id : EventId . make ( `recovery:run:early-reset:${ retryReportsReset } ` ) ,
7440+ id : EventId . make ( `recovery:run:early-reset:${ variant } ` ) ,
74397441 type : "run.updated" ,
74407442 threadId,
74417443 occurredAt : now ,
74427444 payload : { ...run , status : "failed" , completedAt : now } ,
74437445 } ,
74447446 {
7445- id : EventId . make ( `recovery:error:early-reset:${ retryReportsReset } ` ) ,
7447+ id : EventId . make ( `recovery:error:early-reset:${ variant } ` ) ,
74467448 type : "turn-item.updated" ,
74477449 threadId,
74487450 occurredAt : now ,
74497451 payload : {
7450- id : TurnItemId . make ( `recovery:error:early-reset:${ retryReportsReset } ` ) ,
7452+ id : TurnItemId . make ( `recovery:error:early-reset:${ variant } ` ) ,
74517453 type : "error" ,
74527454 threadId,
74537455 runId : run . id ,
@@ -7476,7 +7478,7 @@ it.layer(layerTest)("usage-limit recovery", (it) => {
74767478 const shell = ( yield * orchestrator . getShellSnapshot ( ) ) . threads . find (
74777479 ( thread ) => thread . id === threadId ,
74787480 ) ! ;
7479- const arm = limitRecoveryCommand ( shell , true , DateTime . toEpochMillis ( now ) ) ;
7481+ const arm = limitRecoveryCommand ( shell , true , DateTime . toEpochMillis ( now ) , snoozed ) ;
74807482 assert ( arm ?. type === "thread.metadata.update" && arm . limitRecovery != null ) ;
74817483 yield * orchestrator . autoArmLimitRecovery ( {
74827484 commandId : arm . commandId ,
@@ -7543,6 +7545,23 @@ it.layer(layerTest)("usage-limit recovery", (it) => {
75437545 yield * sweep ( reading ( checkedNow , 100 ) ) ;
75447546 yield * sweep ( reading ( checkedNow , 0 ) , alternateInstanceId ) ;
75457547 assert . lengthOf ( ( yield * orchestrator . getThreadProjection ( threadId ) ) . runs , 1 ) ;
7548+ if ( snoozed ) {
7549+ // A sidebar snooze holds the thread even when it ends at the reset.
7550+ yield * orchestrator . dispatch ( {
7551+ type : "thread.snooze" ,
7552+ commandId : CommandId . make ( `recovery:user-snooze:early-reset:${ variant } ` ) ,
7553+ threadId,
7554+ snoozedUntil : resetAt ,
7555+ } ) ;
7556+ yield * sweep ( reading ( checkedNow , 0 ) ) ;
7557+ assert . lengthOf ( ( yield * orchestrator . getThreadProjection ( threadId ) ) . runs , 1 ) ;
7558+ yield * orchestrator . dispatch ( {
7559+ type : "thread.metadata.update" ,
7560+ commandId : CommandId . make ( `recovery:resnooze:early-reset:${ variant } ` ) ,
7561+ threadId,
7562+ limitRecovery : { runId : run . id , resetAt, snooze : true } ,
7563+ } ) ;
7564+ }
75467565 const staleResume = limitRecoveryCommand (
75477566 armed ! ,
75487567 true ,
@@ -7553,7 +7572,7 @@ it.layer(layerTest)("usage-limit recovery", (it) => {
75537572 assert ( staleResume ?. type === "message.dispatch" ) ;
75547573 yield * orchestrator . dispatch ( {
75557574 type : "thread.metadata.update" ,
7556- commandId : CommandId . make ( `recovery:cancel:early-reset:${ retryReportsReset } ` ) ,
7575+ commandId : CommandId . make ( `recovery:cancel:early-reset:${ variant } ` ) ,
75577576 threadId,
75587577 limitRecovery : { runId : run . id , resetAt, autoResume : false } ,
75597578 } ) ;
@@ -7562,7 +7581,7 @@ it.layer(layerTest)("usage-limit recovery", (it) => {
75627581 assert . lengthOf ( ( yield * orchestrator . getThreadProjection ( threadId ) ) . runs , 1 ) ;
75637582 yield * orchestrator . dispatch ( {
75647583 type : "thread.metadata.update" ,
7565- commandId : CommandId . make ( `recovery:rearm:early-reset:${ retryReportsReset } ` ) ,
7584+ commandId : CommandId . make ( `recovery:rearm:early-reset:${ variant } ` ) ,
75667585 threadId,
75677586 limitRecovery : { runId : run . id , resetAt, autoResume : true } ,
75687587 } ) ;
@@ -7576,7 +7595,7 @@ it.layer(layerTest)("usage-limit recovery", (it) => {
75767595 const nodeId = run . rootNodeId ;
75777596 assert ( nodeId !== null ) ;
75787597 const request = {
7579- id : RuntimeRequestId . make ( `recovery:pending-race:${ retryReportsReset } ` ) ,
7598+ id : RuntimeRequestId . make ( `recovery:pending-race:${ variant } ` ) ,
75807599 nodeId,
75817600 providerTurnId : null ,
75827601 nativeRequestRef : null ,
@@ -7589,7 +7608,7 @@ it.layer(layerTest)("usage-limit recovery", (it) => {
75897608 yield * events . write ( {
75907609 events : [
75917610 {
7592- id : EventId . make ( `recovery:pending-race:${ retryReportsReset } ` ) ,
7611+ id : EventId . make ( `recovery:pending-race:${ variant } ` ) ,
75937612 type : "runtime-request.updated" ,
75947613 threadId,
75957614 nodeId,
@@ -7603,7 +7622,7 @@ it.layer(layerTest)("usage-limit recovery", (it) => {
76037622 yield * events . write ( {
76047623 events : [
76057624 {
7606- id : EventId . make ( `recovery:pending-race:resolved:${ retryReportsReset } ` ) ,
7625+ id : EventId . make ( `recovery:pending-race:resolved:${ variant } ` ) ,
76077626 type : "runtime-request.updated" ,
76087627 threadId,
76097628 nodeId,
@@ -7616,6 +7635,7 @@ it.layer(layerTest)("usage-limit recovery", (it) => {
76167635 const resumed = yield * orchestrator . getThreadProjection ( threadId ) ;
76177636 assert . lengthOf ( resumed . runs , 2 ) ;
76187637 assert . equal ( resumed . runs [ 1 ] ?. status , "starting" ) ;
7638+ assert . isNull ( resumed . thread . snoozedUntil ) ;
76197639
76207640 // The reading can omit the limit that stopped the run, such as a
76217641 // per-model weekly limit. The early run then fails on the same limit.
@@ -7626,19 +7646,19 @@ it.layer(layerTest)("usage-limit recovery", (it) => {
76267646 yield * events . write ( {
76277647 events : [
76287648 {
7629- id : EventId . make ( `recovery:run:early-reset:again:${ retryReportsReset } ` ) ,
7649+ id : EventId . make ( `recovery:run:early-reset:again:${ variant } ` ) ,
76307650 type : "run.updated" ,
76317651 threadId,
76327652 occurredAt : checkedNow ,
76337653 payload : { ...earlyRun , status : "failed" , completedAt : checkedNow } ,
76347654 } ,
76357655 {
7636- id : EventId . make ( `recovery:error:early-reset:again:${ retryReportsReset } ` ) ,
7656+ id : EventId . make ( `recovery:error:early-reset:again:${ variant } ` ) ,
76377657 type : "turn-item.updated" ,
76387658 threadId,
76397659 occurredAt : checkedNow ,
76407660 payload : {
7641- id : TurnItemId . make ( `recovery:error:early-reset:again:${ retryReportsReset } ` ) ,
7661+ id : TurnItemId . make ( `recovery:error:early-reset:again:${ variant } ` ) ,
76427662 type : "error" ,
76437663 threadId,
76447664 runId : earlyRun . id ,
0 commit comments