@@ -27,6 +27,7 @@ import * as Fiber from "effect/Fiber";
2727import * as Layer from "effect/Layer" ;
2828import * as Stream from "effect/Stream" ;
2929import * as Tracer from "effect/Tracer" ;
30+ import * as TestClock from "effect/testing/TestClock" ;
3031
3132import * as CheckpointStore from "../checkpointing/CheckpointStore.ts" ;
3233import * as ServerConfig from "../config.ts" ;
@@ -46,6 +47,7 @@ import * as Orchestrator from "./Orchestrator.ts";
4647import { continueRestartedRun } from "./RestartContinuation.ts" ;
4748import * as RuntimeLayer from "./runtimeLayer.ts" ;
4849import * as ThreadCommandExecutor from "./ThreadCommandExecutor.ts" ;
50+ import { limitRecoveryCommand } from "./UsageLimitRecoveryWorker.ts" ;
4951import * as ProviderTurnStartServiceTestkit from "./ProviderTurnStartService.testkit.ts" ;
5052import type * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter" ;
5153import * as ProviderContinuationRequests from "@t3tools/provider-core/server/ProviderContinuationRequests" ;
@@ -2120,6 +2122,111 @@ it.layer(Layer.merge(layerTest, ProviderContinuationRequests.layer))(
21202122 } ) ,
21212123 ) ;
21222124
2125+ it . effect ( "tracks a child's usage-limit auto-resume as a follow-up" , ( ) =>
2126+ Effect . gen ( function * ( ) {
2127+ const orchestrator = yield * Orchestrator . OrchestratorV2 ;
2128+ const eventSink = yield * EventSink . EventSinkV2 ;
2129+ const now = yield * DateTime . now ;
2130+ const parentThreadId = ThreadId . make ( "thread:limit-resume-parent" ) ;
2131+ const projectId = ProjectId . make ( "project:limit-resume" ) ;
2132+ const parentRunId = RunId . make ( "run:limit-resume-parent" ) ;
2133+ const rootNodeId = NodeId . make ( "node:limit-resume-parent" ) ;
2134+ yield * seedParentWithTerminalTask ( {
2135+ threadId : parentThreadId ,
2136+ projectId,
2137+ runId : parentRunId ,
2138+ rootNodeId,
2139+ taskId : NodeId . make ( "node:limit-resume-seed" ) ,
2140+ deliveryState : "acknowledged" ,
2141+ now,
2142+ } ) ;
2143+ const child = yield * seedRestartCancelledChild ( {
2144+ parentThreadId,
2145+ projectId,
2146+ parentRunId,
2147+ rootNodeId,
2148+ name : "limit-resume-child" ,
2149+ completionWake : "always" ,
2150+ continuationPending : false ,
2151+ runStatus : "completed" ,
2152+ now,
2153+ } ) ;
2154+ const childRun = ( yield * orchestrator . getThreadProjection ( child . childThreadId ) ) . runs [ 0 ] ! ;
2155+ yield * eventSink . write ( {
2156+ commandId : reconcileCommandId ( "limit-resume-child-limited" ) ,
2157+ events : [
2158+ {
2159+ id : EventId . make ( "event:limit-resume-child-failed" ) ,
2160+ type : "run.updated" ,
2161+ threadId : child . childThreadId ,
2162+ runId : childRun . id ,
2163+ occurredAt : now ,
2164+ payload : { ...childRun , status : "failed" , completedAt : now } ,
2165+ } ,
2166+ {
2167+ id : EventId . make ( "event:limit-resume-child-error" ) ,
2168+ type : "turn-item.updated" ,
2169+ threadId : child . childThreadId ,
2170+ occurredAt : now ,
2171+ payload : {
2172+ id : TurnItemId . make ( "turn-item:limit-resume-child-error" ) ,
2173+ type : "error" ,
2174+ threadId : child . childThreadId ,
2175+ runId : childRun . id ,
2176+ nodeId : childRun . rootNodeId ,
2177+ providerThreadId : null ,
2178+ providerTurnId : null ,
2179+ nativeItemRef : null ,
2180+ parentItemId : null ,
2181+ ordinal : 1 ,
2182+ status : "failed" ,
2183+ title : "Usage limit reached" ,
2184+ startedAt : now ,
2185+ completedAt : now ,
2186+ updatedAt : now ,
2187+ failure : {
2188+ class : "usage_limit" ,
2189+ message : "Plan limit reached." ,
2190+ code : "usageLimitExceeded" ,
2191+ retryable : null ,
2192+ resetAt : DateTime . formatIso ( DateTime . add ( now , { minutes : 1 } ) ) ,
2193+ } ,
2194+ } ,
2195+ } ,
2196+ ] ,
2197+ } ) ;
2198+ yield * orchestrator . recoverDelegatedTasks ;
2199+ const childShell = Effect . map ( orchestrator . getShellSnapshot ( ) , ( snapshot ) =>
2200+ snapshot . threads . find ( ( thread ) => thread . id === child . childThreadId ) ,
2201+ ) ;
2202+ const arm = limitRecoveryCommand ( ( yield * childShell ) ! , true , DateTime . toEpochMillis ( now ) ) ;
2203+ assert ( arm ?. type === "thread.metadata.update" && arm . limitRecovery != null ) ;
2204+ yield * orchestrator . autoArmLimitRecovery ( {
2205+ commandId : arm . commandId ,
2206+ threadId : arm . threadId ,
2207+ limitRecovery : arm . limitRecovery ,
2208+ } ) ;
2209+ yield * TestClock . adjust ( "1 minute" ) ;
2210+ const resume = limitRecoveryCommand (
2211+ ( yield * childShell ) ! ,
2212+ true ,
2213+ DateTime . toEpochMillis ( yield * DateTime . now ) ,
2214+ ) ;
2215+ assert ( resume ?. type === "message.dispatch" ) ;
2216+ yield * orchestrator . dispatch ( resume ) ;
2217+
2218+ const resumed = ( yield * orchestrator . getThreadProjection ( child . childThreadId ) ) . runs . find (
2219+ ( run ) => run . userMessageId === resume . messageId ,
2220+ ) ;
2221+ assert . isDefined ( resumed ?. delegatedTaskId ) ;
2222+ const followup = ( yield * orchestrator . getThreadProjection ( parentThreadId ) ) . subagents . find (
2223+ ( task ) => task . id === resumed ?. delegatedTaskId ,
2224+ ) ;
2225+ assert . equal ( followup ?. childThreadId , child . childThreadId ) ;
2226+ assert . isNull ( followup ?. result ) ;
2227+ } ) ,
2228+ ) ;
2229+
21232230 it . effect ( "serializes a recovered child answer with its parent and tracks the follow-up" , ( ) =>
21242231 Effect . gen ( function * ( ) {
21252232 const orchestrator = yield * Orchestrator . OrchestratorV2 ;
0 commit comments