@@ -49,6 +49,8 @@ import * as ThreadCommandExecutor from "./ThreadCommandExecutor.ts";
4949import * as ProviderTurnStartServiceTestkit from "./ProviderTurnStartService.testkit.ts" ;
5050import type * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter" ;
5151import * as ProviderContinuationRequests from "@t3tools/provider-core/server/ProviderContinuationRequests" ;
52+ import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator" ;
53+ import * as ProviderContinuationService from "./ProviderContinuationService.ts" ;
5254
5355const layerPlatformTest = Layer . merge (
5456 Layer . mergeAll ( NodeServices . layer , NodeCrypto . layer ) ,
@@ -152,6 +154,12 @@ const layerTest = Layer.mergeAll(
152154 Layer . provide ( layerPlatformTest ) ,
153155) ;
154156
157+ // ProviderContinuationService consumes the orchestrator's continuation queue, as in production.
158+ const layerContinuationWorkerTest = ProviderContinuationService . layer . pipe (
159+ Layer . provide ( Layer . merge ( ProviderContinuationRequests . layer , IdAllocator . layer ) ) ,
160+ Layer . provideMerge ( layerTest ) ,
161+ ) ;
162+
155163const seedParentWithTerminalTask = ( input : {
156164 readonly threadId : ThreadId ;
157165 readonly projectId : ProjectId ;
@@ -1370,6 +1378,251 @@ const seedRestartCancelledChild = (input: {
13701378 return { taskId, childThreadId, childRunId } ;
13711379 } ) ;
13721380
1381+ it . layer ( layerContinuationWorkerTest ) ( "interrupted completion wakes" , ( it ) => {
1382+ it . effect . each ( [
1383+ { name : "an interrupt keeps" , holdQueue : false , siblingFinishesFirst : false } ,
1384+ { name : "Stop disposes" , holdQueue : true , siblingFinishesFirst : false } ,
1385+ { name : "an interrupt reserves" , holdQueue : false , siblingFinishesFirst : true } ,
1386+ ] ) ( "$name a completion wake's other siblings" , ( { holdQueue, siblingFinishesFirst } ) =>
1387+ Effect . gen ( function * ( ) {
1388+ const orchestrator = yield * Orchestrator . OrchestratorV2 ;
1389+ const sink = yield * EventSink . EventSinkV2 ;
1390+ const now = yield * DateTime . now ;
1391+ const key = holdQueue
1392+ ? "sibling-stop"
1393+ : siblingFinishesFirst
1394+ ? "sibling-finished-interrupt"
1395+ : "sibling-interrupt" ;
1396+ const threadId = ThreadId . make ( `${ key } -parent` ) ;
1397+ const originRunId = RunId . make ( `${ key } -origin` ) ;
1398+ const incomingTaskId = NodeId . make ( `${ key } -incoming` ) ;
1399+ yield * seedParentWithTerminalTask ( {
1400+ threadId,
1401+ projectId : ProjectId . make ( `${ key } -project` ) ,
1402+ runId : originRunId ,
1403+ rootNodeId : NodeId . make ( `${ key } -origin-root` ) ,
1404+ taskId : incomingTaskId ,
1405+ deliveryState : "claimed" ,
1406+ deliveryTaskIds : [ incomingTaskId ] ,
1407+ completionWake : "always" ,
1408+ now,
1409+ } ) ;
1410+ const originRootId = NodeId . make ( `${ key } -origin-root` ) ;
1411+ yield * sink . write ( {
1412+ events : [
1413+ {
1414+ id : EventId . make ( `event:${ key } -origin-root` ) ,
1415+ type : "node.updated" ,
1416+ threadId,
1417+ runId : originRunId ,
1418+ nodeId : originRootId ,
1419+ occurredAt : now ,
1420+ payload : {
1421+ id : originRootId ,
1422+ threadId,
1423+ runId : originRunId ,
1424+ parentNodeId : null ,
1425+ rootNodeId : originRootId ,
1426+ kind : "root_turn" ,
1427+ status : "running" ,
1428+ countsForRun : true ,
1429+ providerThreadId : null ,
1430+ providerTurnId : null ,
1431+ nativeItemRef : null ,
1432+ runtimeRequestId : null ,
1433+ checkpointScopeId : null ,
1434+ startedAt : now ,
1435+ completedAt : null ,
1436+ } ,
1437+ } ,
1438+ ] ,
1439+ } ) ;
1440+ yield * orchestrator . dispatch ( {
1441+ type : "delegated_task.request" ,
1442+ createdBy : "agent" ,
1443+ creationSource : "mcp" ,
1444+ commandId : CommandId . make ( `${ key } -sibling` ) ,
1445+ parentThreadId : threadId ,
1446+ parentRunId : originRunId ,
1447+ parentNodeId : originRootId ,
1448+ task : "Keep working" ,
1449+ modelSelection,
1450+ runtimeMode : "full-access" ,
1451+ interactionMode : "default" ,
1452+ completionWake : "always" ,
1453+ } ) ;
1454+ const seeded = yield * orchestrator . getThreadProjection ( threadId ) ;
1455+ const origin = seeded . runs . find ( ( run ) => run . id === originRunId ) ! ;
1456+ const sibling = seeded . subagents . find ( ( task ) => task . id !== incomingTaskId ) ! ;
1457+ yield * sink . write ( {
1458+ events : [
1459+ {
1460+ ...runEvent ( { threadId, runId : originRunId , ordinal : 1 , status : "completed" , now } ) ,
1461+ payload : { ...origin , status : "completed" , completedAt : now } ,
1462+ } ,
1463+ ] ,
1464+ } ) ;
1465+ const delivery = origin . delegatedCompletion ! . delivery ! ;
1466+ yield * orchestrator . dispatch ( {
1467+ type : "message.dispatch" ,
1468+ createdBy : "agent" ,
1469+ creationSource : "server" ,
1470+ commandId : CommandId . make ( `${ key } -wake` ) ,
1471+ threadId,
1472+ messageId : delivery . messageId ,
1473+ text : "Delegated task completed" ,
1474+ attachments : [ ] ,
1475+ modelSelection,
1476+ dispatchMode : { type : "queue_after_active" } ,
1477+ delegatedCompletion : {
1478+ parentRunId : originRunId ,
1479+ generation : delivery . generation ,
1480+ taskIds : delivery . taskIds ,
1481+ } ,
1482+ } ) ;
1483+ yield * orchestrator . resumeQueuedRuns ;
1484+ const wake = ( yield * orchestrator . getThreadProjection ( threadId ) ) . runs . find (
1485+ ( run ) => run . userMessageId === delivery . messageId ,
1486+ ) ! ;
1487+ const childRun = ( yield * orchestrator . getThreadProjection ( sibling . childThreadId ! ) ) . runs [ 0 ] ! ;
1488+ const completeSibling = Effect . gen ( function * ( ) {
1489+ const afterSequence = yield * sink . latestSequence ( ) ;
1490+ yield * sink . write ( {
1491+ events : [
1492+ {
1493+ ...runEvent ( {
1494+ threadId : sibling . childThreadId ! ,
1495+ runId : childRun . id ,
1496+ ordinal : childRun . ordinal ,
1497+ status : "completed" ,
1498+ now,
1499+ } ) ,
1500+ payload : { ...childRun , status : "completed" , completedAt : now } ,
1501+ } ,
1502+ ] ,
1503+ } ) ;
1504+ return yield * sink . stream ( { afterSequence, eventType : "subagent.updated" } ) . pipe (
1505+ Stream . filter (
1506+ ( stored ) =>
1507+ stored . event . type === "subagent.updated" &&
1508+ stored . event . payload . id === sibling . id &&
1509+ stored . event . payload . status === "completed" ,
1510+ ) ,
1511+ Stream . take ( 1 ) ,
1512+ Stream . runDrain ,
1513+ Effect . as ( afterSequence ) ,
1514+ ) ;
1515+ } ) ;
1516+ if ( siblingFinishesFirst ) {
1517+ yield * completeSibling ;
1518+ assert . equal (
1519+ ( yield * orchestrator . getThreadProjection ( threadId ) ) . subagents . find (
1520+ ( task ) => task . id === sibling . id ,
1521+ ) ?. completionDelivery ?. state ,
1522+ "pending" ,
1523+ ) ;
1524+ }
1525+ const beforeInterrupt = yield * sink . latestSequence ( ) ;
1526+ const interrupted = yield * orchestrator . dispatch ( {
1527+ type : "run.interrupt" ,
1528+ commandId : CommandId . make ( `${ key } -interrupt` ) ,
1529+ threadId,
1530+ runId : wake . id ,
1531+ ...( holdQueue ? { holdQueue } : { } ) ,
1532+ } ) ;
1533+ // The wake's terminal may claim the sibling right after commit, so read what the interrupt
1534+ // itself wrote rather than a later projection.
1535+ const interruptDelivery = new Map (
1536+ interrupted . storedEvents . flatMap ( ( { event } ) =>
1537+ event . type === "subagent.updated"
1538+ ? [ [ event . payload . id , event . payload . completionDelivery ?. state ] as const ]
1539+ : [ ] ,
1540+ ) ,
1541+ ) ;
1542+ const interruptCohort = interrupted . storedEvents . flatMap ( ( { event } ) =>
1543+ event . type === "run.updated" && event . payload . id === originRunId
1544+ ? [ event . payload . delegatedCompletion ]
1545+ : [ ] ,
1546+ ) ;
1547+ assert . equal ( interruptDelivery . get ( incomingTaskId ) , "disposed" ) ;
1548+ assert . equal ( interruptCohort . at ( - 1 ) ?. disposition , holdQueue ? "stopped" : "open" ) ;
1549+ if ( holdQueue ) {
1550+ assert . equal ( interruptDelivery . get ( sibling . id ) , "disposed" ) ;
1551+ return ;
1552+ }
1553+ assert . isFalse ( interruptDelivery . has ( sibling . id ) ) ;
1554+
1555+ const afterSequence = siblingFinishesFirst ? beforeInterrupt : yield * completeSibling ;
1556+ // The wake's terminal, or the sibling's own completion, reserves the successor.
1557+ yield * sink . stream ( { afterSequence, eventType : "subagent.updated" } ) . pipe (
1558+ Stream . filter (
1559+ ( stored ) =>
1560+ stored . event . type === "subagent.updated" &&
1561+ stored . event . payload . id === sibling . id &&
1562+ stored . event . payload . completionDelivery ?. state === "claimed" ,
1563+ ) ,
1564+ Stream . take ( 1 ) ,
1565+ Stream . runDrain ,
1566+ ) ;
1567+ const woken = yield * orchestrator . getThreadProjection ( threadId ) ;
1568+ const successor = woken . runs . find ( ( run ) => run . id === originRunId ) ?. delegatedCompletion
1569+ ?. delivery ;
1570+ assert . equal ( successor ?. generation , delivery . generation + 1 ) ;
1571+ assert . deepEqual ( successor ?. taskIds , [ sibling . id ] ) ;
1572+ assert . notEqual ( successor ?. messageId , delivery . messageId ) ;
1573+
1574+ // ProviderContinuationService consumes the claim's offer and dispatches the successor.
1575+ yield * sink . stream ( { afterSequence, eventType : "message.updated" } ) . pipe (
1576+ Stream . filter (
1577+ ( stored ) =>
1578+ stored . event . type === "message.updated" &&
1579+ stored . event . payload . id === successor ! . messageId ,
1580+ ) ,
1581+ Stream . take ( 1 ) ,
1582+ Stream . runDrain ,
1583+ ) ;
1584+ // Repeated recovery re-offers every open delivery; none may wake the parent twice.
1585+ yield * orchestrator . resumeQueuedRuns ;
1586+ yield * orchestrator . recoverDelegatedTasks ;
1587+ yield * orchestrator . recoverDelegatedTasks ;
1588+ yield * orchestrator . resumeQueuedRuns ;
1589+ const delivered = yield * orchestrator . getThreadProjection ( threadId ) ;
1590+ const cohort = delivered . runs . find ( ( run ) => run . id === originRunId ) ?. delegatedCompletion ;
1591+ assert . equal ( cohort ?. disposition , "open" ) ;
1592+ assert . equal ( cohort ?. nextGeneration , delivery . generation + 2 ) ;
1593+ assert . deepEqual ( cohort ?. delivery , successor ) ;
1594+ assert . deepEqual (
1595+ delivered . messages
1596+ . filter ( ( message ) => message . delegatedCompletion ?. parentRunId === originRunId )
1597+ . map ( ( message ) => [ message . id , message . delegatedCompletion ] ) ,
1598+ [
1599+ [
1600+ delivery . messageId ,
1601+ {
1602+ parentRunId : originRunId ,
1603+ generation : delivery . generation ,
1604+ taskIds : [ incomingTaskId ] ,
1605+ } ,
1606+ ] ,
1607+ [
1608+ successor ! . messageId ,
1609+ { parentRunId : originRunId , generation : successor ! . generation , taskIds : [ sibling . id ] } ,
1610+ ] ,
1611+ ] ,
1612+ ) ;
1613+ const successorRuns = delivered . runs . filter (
1614+ ( run ) => run . userMessageId === successor ! . messageId ,
1615+ ) ;
1616+ assert . equal ( successorRuns . length , 1 ) ;
1617+ assert . equal ( successorRuns [ 0 ] ?. status , "starting" ) ;
1618+ assert . equal (
1619+ delivered . subagents . find ( ( task ) => task . id === sibling . id ) ?. completionDelivery ?. state ,
1620+ "claimed" ,
1621+ ) ;
1622+ } ) ,
1623+ ) ;
1624+ } ) ;
1625+
13731626it . layer ( Layer . merge ( layerTest , ProviderContinuationRequests . layer ) ) (
13741627 "subagent follow-ups" ,
13751628 ( it ) => {
0 commit comments