11import { SourceControlProviderKind } from "@t3tools/contracts" ;
22import { limitRecoveryCommand } from "./UsageLimitRecoveryWorker.ts" ;
3+ import * as UsageLimitRecoveryWorker from "./UsageLimitRecoveryWorker.ts" ;
4+ import * as ProviderRegistry from "../provider/ProviderRegistry.ts" ;
35import * as McpProviderSessions from "@t3tools/provider-core/server/McpProviderSessions" ;
46import * as SourceControlProviderRegistry from "../sourceControl/SourceControlProviderRegistry.ts" ;
57import * as NodeServices from "@effect/platform-node/NodeServices" ;
@@ -18,6 +20,7 @@ import {
1820 TurnItemId ,
1921 type ModelSelection ,
2022 type OrchestrationV2Run ,
23+ type ServerProviderUsageLimits ,
2124 ProjectId ,
2225 type PullRequestDetail ,
2326 type PullRequestComment ,
@@ -6281,6 +6284,45 @@ it.layer(layerTest)("usage-limit recovery", (it) => {
62816284 checkedAt : DateTime . formatIso ( checkedAt ) ,
62826285 windows : [ { id : "five_hour" , kind : "session" as const , label : "5h" , usedPercent } ] ,
62836286 } ) ;
6287+ const sweep = (
6288+ usageLimits : ServerProviderUsageLimits ,
6289+ instanceId = shell . providerInstanceId ,
6290+ ) =>
6291+ Effect . scoped (
6292+ Layer . build (
6293+ UsageLimitRecoveryWorker . layer . pipe (
6294+ Layer . provide ( [
6295+ Layer . succeed ( ProjectionStore . ProjectionStoreV2 , projections ) ,
6296+ Layer . succeed ( Orchestrator . OrchestratorV2 , orchestrator ) ,
6297+ Layer . mock ( ThreadManagementService . ThreadManagementService ) ( {
6298+ dispatch : orchestrator . dispatch ,
6299+ } ) ,
6300+ ServerSettings . layerTest ( ) ,
6301+ Layer . mock ( ProviderRegistry . ProviderRegistry ) ( {
6302+ getProviders : Effect . succeed ( [
6303+ {
6304+ instanceId,
6305+ driver,
6306+ enabled : true ,
6307+ installed : true ,
6308+ version : null ,
6309+ status : "ready" ,
6310+ auth : { status : "authenticated" } ,
6311+ checkedAt : usageLimits . checkedAt ,
6312+ models : [ ] ,
6313+ slashCommands : [ ] ,
6314+ skills : [ ] ,
6315+ usageLimits,
6316+ } ,
6317+ ] ) ,
6318+ } ) ,
6319+ Layer . mock ( Scheduler . Scheduler ) ( {
6320+ register : ( _name , work ) => Effect . orDie ( work ) ,
6321+ } ) ,
6322+ ] ) ,
6323+ ) ,
6324+ ) ,
6325+ ) ;
62846326 const armed = ( yield * projections . getLimitRecoveryCandidates ( {
62856327 now : checkedNow ,
62866328 autoResume : true ,
@@ -6291,9 +6333,34 @@ it.layer(layerTest)("usage-limit recovery", (it) => {
62916333 // Readings from before the failure, or ones that still show the limit, keep waiting.
62926334 assert . isNull ( limitRecoveryCommand ( armed ! , true , nowMs , false , reading ( now , 0 ) ) ) ;
62936335 assert . isNull ( limitRecoveryCommand ( armed ! , true , nowMs , false , reading ( checkedNow , 100 ) ) ) ;
6294- const resume = limitRecoveryCommand ( armed ! , true , nowMs , false , reading ( checkedNow , 0 ) ) ;
6295- assert ( resume ?. type === "message.dispatch" ) ;
6296- yield * orchestrator . dispatch ( resume ) ;
6336+ yield * sweep ( reading ( now , 0 ) ) ;
6337+ yield * sweep ( reading ( checkedNow , 100 ) ) ;
6338+ yield * sweep ( reading ( checkedNow , 0 ) , alternateInstanceId ) ;
6339+ assert . lengthOf ( ( yield * orchestrator . getThreadProjection ( threadId ) ) . runs , 1 ) ;
6340+ const staleResume = limitRecoveryCommand (
6341+ armed ! ,
6342+ true ,
6343+ nowMs ,
6344+ false ,
6345+ reading ( checkedNow , 0 ) ,
6346+ ) ;
6347+ assert ( staleResume ?. type === "message.dispatch" ) ;
6348+ yield * orchestrator . dispatch ( {
6349+ type : "thread.metadata.update" ,
6350+ commandId : CommandId . make ( "recovery:cancel:early-reset" ) ,
6351+ threadId,
6352+ limitRecovery : { runId : run . id , resetAt, autoResume : false } ,
6353+ } ) ;
6354+ yield * orchestrator . dispatch ( staleResume ) ;
6355+ yield * sweep ( reading ( checkedNow , 0 ) ) ;
6356+ assert . lengthOf ( ( yield * orchestrator . getThreadProjection ( threadId ) ) . runs , 1 ) ;
6357+ yield * orchestrator . dispatch ( {
6358+ type : "thread.metadata.update" ,
6359+ commandId : CommandId . make ( "recovery:rearm:early-reset" ) ,
6360+ threadId,
6361+ limitRecovery : { runId : run . id , resetAt, autoResume : true } ,
6362+ } ) ;
6363+ yield * sweep ( reading ( checkedNow , 0 ) ) ;
62976364 const resumed = yield * orchestrator . getThreadProjection ( threadId ) ;
62986365 assert . lengthOf ( resumed . runs , 2 ) ;
62996366 assert . equal ( resumed . runs [ 1 ] ?. status , "starting" ) ;
@@ -6360,6 +6427,8 @@ it.layer(layerTest)("usage-limit recovery", (it) => {
63606427 snooze : false ,
63616428 } ) ) . find ( ( row ) => row . id === threadId ) ! ;
63626429 assert . equal ( rearmed . limitRecovery ?. waitForReset , true ) ;
6430+ yield * sweep ( reading ( laterNow , 0 ) ) ;
6431+ assert . lengthOf ( ( yield * orchestrator . getThreadProjection ( threadId ) ) . runs , 2 ) ;
63636432 assert . isNull (
63646433 limitRecoveryCommand (
63656434 rearmed ,
@@ -6372,6 +6441,9 @@ it.layer(layerTest)("usage-limit recovery", (it) => {
63726441 const onTime = limitRecoveryCommand ( rearmed , true , Date . parse ( resetAt ) ) ;
63736442 assert ( onTime ?. type === "message.dispatch" ) ;
63746443 assert . isUndefined ( onTime . usageLimitClearedAt ) ;
6444+ yield * TestClock . setTime ( Date . parse ( resetAt ) ) ;
6445+ yield * orchestrator . dispatch ( onTime ) ;
6446+ assert . lengthOf ( ( yield * orchestrator . getThreadProjection ( threadId ) ) . runs , 3 ) ;
63756447 } ) ,
63766448 ) ;
63776449
0 commit comments