@@ -638,6 +638,45 @@ function nextQueuedRun(
638638 return queuedRunsInDeliveryOrder ( projection ) [ 0 ] ;
639639}
640640
641+ function followsCurrentConversation (
642+ message : OrchestrationV2ConversationMessage | undefined ,
643+ ) : boolean {
644+ return (
645+ message ?. notification ?. source . kind === "monitor" && ! isBufferedProviderNotification ( message )
646+ ) ;
647+ }
648+
649+ function queuedRunModelSelection (
650+ projection : Pick < OrchestrationV2ThreadProjection , "thread" | "messages" > ,
651+ run : OrchestrationV2Run ,
652+ ) : ModelSelection {
653+ const message = projection . messages . find ( ( candidate ) => candidate . id === run . userMessageId ) ;
654+ // App-owned monitor updates follow the agent currently handling the thread.
655+ return followsCurrentConversation ( message )
656+ ? projection . thread . modelSelection
657+ : run . modelSelection ;
658+ }
659+
660+ function usageLimitHoldsQueue (
661+ projection : Pick < OrchestrationV2ThreadProjection , "runs" | "turnItems" | "providerSessions" > ,
662+ providerInstanceId : ModelSelection [ "instanceId" ] ,
663+ ) : boolean {
664+ let latestSession : OrchestrationV2ThreadProjection [ "providerSessions" ] [ number ] | undefined ;
665+ for ( const session of projection . providerSessions ) {
666+ if (
667+ session . providerInstanceId === providerInstanceId &&
668+ ( latestSession === undefined ||
669+ DateTime . isGreaterThan ( session . updatedAt , latestSession . updatedAt ) )
670+ ) {
671+ latestSession = session ;
672+ }
673+ }
674+ return (
675+ usageLimitBlockedRun ( projection . runs , projection . turnItems , latestSession ?. lastError ?? null )
676+ ?. providerInstanceId === providerInstanceId
677+ ) ;
678+ }
679+
641680function latestStableRun (
642681 projection : Pick < OrchestrationV2ThreadProjection , "runs" > ,
643682) : OrchestrationV2Run | null {
@@ -1328,20 +1367,21 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
13281367 return ;
13291368 }
13301369
1331- // The limit already stopped this thread. Starting the queue would send
1332- // every waiting message and drop it from the queue as each one fails.
1333- const sessionError =
1334- projection . providerSessions
1335- . filter ( ( session ) => session . providerInstanceId === projection . thread . providerInstanceId )
1336- . toSorted (
1337- ( left , right ) =>
1338- DateTime . toEpochMillis ( right . updatedAt ) - DateTime . toEpochMillis ( left . updatedAt ) ,
1339- ) [ 0 ] ?. lastError ?? null ;
1340- if ( usageLimitBlockedRun ( projection . runs , projection . turnItems , sessionError ) !== null ) {
1370+ const storedQueuedRun = nextQueuedRun ( projection ) ;
1371+ if ( storedQueuedRun === undefined ) {
13411372 return ;
13421373 }
1343- const queuedRun = nextQueuedRun ( projection ) ;
1344- if ( queuedRun === undefined ) {
1374+ const queuedMessage = projection . messages . find (
1375+ ( candidate ) => candidate . id === storedQueuedRun . userMessageId ,
1376+ ) ;
1377+ const modelSelection = queuedRunModelSelection ( projection , storedQueuedRun ) ;
1378+ const queuedRun = {
1379+ ...storedQueuedRun ,
1380+ modelSelection,
1381+ providerInstanceId : modelSelection . instanceId ,
1382+ } ;
1383+ // A limited provider would fail and consume every waiting message.
1384+ if ( usageLimitHoldsQueue ( projection , queuedRun . providerInstanceId ) ) {
13451385 return ;
13461386 }
13471387 // A message queued behind a worktree that was cancelled or never created
@@ -1420,9 +1460,6 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
14201460
14211461 const rootNode = projection . nodes . find ( ( candidate ) => candidate . id === rootNodeId ) ;
14221462 const attempt = projection . attempts . find ( ( candidate ) => candidate . id === attemptId ) ;
1423- const queuedMessage = projection . messages . find (
1424- ( candidate ) => candidate . id === queuedRun . userMessageId ,
1425- ) ;
14261463 const legacyQueuedTurnItem = projection . turnItems . find (
14271464 (
14281465 candidate ,
@@ -1431,7 +1468,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
14311468 candidate . runId === queuedRun . id &&
14321469 candidate . messageId === queuedRun . userMessageId ,
14331470 ) ;
1434- const queuedProviderThread = projection . providerThreads . find (
1471+ const storedProviderThread = projection . providerThreads . find (
14351472 ( candidate ) => candidate . id === providerThreadId ,
14361473 ) ;
14371474 const storedCheckpointScope = projection . checkpointScopes . find (
@@ -1441,7 +1478,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
14411478 rootNode === undefined ||
14421479 attempt === undefined ||
14431480 queuedMessage === undefined ||
1444- queuedProviderThread === undefined ||
1481+ storedProviderThread === undefined ||
14451482 ( rootNode . checkpointScopeId !== null && storedCheckpointScope === undefined )
14461483 ) {
14471484 return yield * new OrchestratorDispatchError ( {
@@ -1453,27 +1490,83 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
14531490
14541491 const commandId = CommandId . make ( `command:system:start-queued:${ queuedRun . id } ` ) ;
14551492 const now = yield * DateTime . now ;
1456- const selectionChanged = ! modelSelectionsEqual (
1457- projection . thread . modelSelection ,
1458- queuedRun . modelSelection ,
1493+ const targetAdapter = yield * providerAdapters . get ( queuedRun . providerInstanceId ) . pipe (
1494+ Effect . mapError (
1495+ ( cause ) =>
1496+ new OrchestratorDispatchError ( {
1497+ commandId,
1498+ commandType : "message.dispatch" ,
1499+ cause,
1500+ } ) ,
1501+ ) ,
14591502 ) ;
1460- const switchPlan = selectionChanged
1461- ? yield * providerSwitchService
1462- . plan ( { projection, targetModelSelection : queuedRun . modelSelection } )
1463- . pipe (
1464- Effect . mapError (
1465- ( cause ) =>
1466- new OrchestratorDispatchError ( {
1467- commandId,
1468- commandType : "message.dispatch" ,
1469- cause,
1470- } ) ,
1471- ) ,
1472- )
1473- : null ;
14741503 const activeProviderThread = projection . providerThreads . find (
14751504 ( candidate ) => candidate . id === projection . thread . activeProviderThreadId ,
14761505 ) ;
1506+ const followsCurrent = followsCurrentConversation ( queuedMessage ) ;
1507+ const targetProviderThread = followsCurrent
1508+ ? activeProviderThread ?. providerInstanceId === queuedRun . providerInstanceId
1509+ ? activeProviderThread
1510+ : rootProviderThreadsForProvider ( projection , queuedRun . providerInstanceId ) [ 0 ]
1511+ : storedProviderThread ;
1512+ const queuedProviderThread : OrchestrationV2ProviderThread = targetProviderThread ?? {
1513+ id : idAllocator . derive . providerThread ( {
1514+ driver : targetAdapter . driver ,
1515+ nativeThreadId : `pending:${ queuedRun . id } ` ,
1516+ } ) ,
1517+ driver : targetAdapter . driver ,
1518+ providerInstanceId : queuedRun . providerInstanceId ,
1519+ providerSessionId : null ,
1520+ appThreadId : threadId ,
1521+ ownerNodeId : null ,
1522+ nativeThreadRef : null ,
1523+ nativeConversationHeadRef : null ,
1524+ status : "not_loaded" ,
1525+ firstRunOrdinal : null ,
1526+ lastRunOrdinal : null ,
1527+ handoffIds : [ ] ,
1528+ forkedFrom : null ,
1529+ createdAt : now ,
1530+ updatedAt : now ,
1531+ } ;
1532+ const selectionChanged = ! modelSelectionsEqual (
1533+ projection . thread . modelSelection ,
1534+ queuedRun . modelSelection ,
1535+ ) ;
1536+ // The picker may already have moved ahead of the native conversation.
1537+ const switchFromActiveProvider =
1538+ followsCurrent &&
1539+ activeProviderThread !== undefined &&
1540+ activeProviderThread . providerInstanceId !== queuedRun . providerInstanceId ;
1541+ const switchPlan =
1542+ selectionChanged || switchFromActiveProvider
1543+ ? yield * providerSwitchService
1544+ . plan ( {
1545+ projection : switchFromActiveProvider
1546+ ? {
1547+ ...projection ,
1548+ thread : {
1549+ ...projection . thread ,
1550+ modelSelection : {
1551+ ...projection . thread . modelSelection ,
1552+ instanceId : activeProviderThread . providerInstanceId ,
1553+ } ,
1554+ } ,
1555+ }
1556+ : projection ,
1557+ targetModelSelection : queuedRun . modelSelection ,
1558+ } )
1559+ . pipe (
1560+ Effect . mapError (
1561+ ( cause ) =>
1562+ new OrchestratorDispatchError ( {
1563+ commandId,
1564+ commandType : "message.dispatch" ,
1565+ cause,
1566+ } ) ,
1567+ ) ,
1568+ )
1569+ : null ;
14771570 const canResumeAcrossInstances =
14781571 switchPlan ?. instanceChanged === true &&
14791572 switchPlan . transition . type === "restart_and_resume" &&
@@ -1488,16 +1581,6 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
14881581 nativeMetadata : activeProviderThread . nativeMetadata ,
14891582 }
14901583 : queuedProviderThread ;
1491- const targetAdapter = yield * providerAdapters . get ( queuedRun . providerInstanceId ) . pipe (
1492- Effect . mapError (
1493- ( cause ) =>
1494- new OrchestratorDispatchError ( {
1495- commandId,
1496- commandType : "message.dispatch" ,
1497- cause,
1498- } ) ,
1499- ) ,
1500- ) ;
15011584 const targetCapabilities = yield * targetAdapter . getCapabilities ( ) . pipe (
15021585 Effect . mapError (
15031586 ( cause ) =>
@@ -1690,6 +1773,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
16901773 } ;
16911774 const startingRun : OrchestrationV2Run = {
16921775 ...queuedRun ,
1776+ providerThreadId : queuedProviderThread . id ,
16931777 ordinal,
16941778 status : "starting" ,
16951779 queuePosition : null ,
@@ -1728,6 +1812,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
17281812 ? { }
17291813 : { senderThreadId : queuedMessage . senderThreadId } ) ,
17301814 } ) ,
1815+ providerThreadId : queuedProviderThread . id ,
17311816 inputIntent : "queued_turn" ,
17321817 startedAt : now ,
17331818 completedAt : now ,
@@ -1774,26 +1859,45 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
17741859 strategy : activeHandoff . strategy ,
17751860 summary : activeHandoff . summaryText ,
17761861 } ;
1777- const checkpointEvents : ReadonlyArray < Omit < OrchestrationV2DomainEvent , "id" > > =
1778- storedCheckpointScope === undefined
1862+ const executionIdentityEvents : ReadonlyArray < Omit < OrchestrationV2DomainEvent , "id" > > =
1863+ storedCheckpointScope === undefined ||
1864+ storedProviderThread . id !== queuedProviderThread . id ||
1865+ attempt . providerInstanceId !== queuedRun . providerInstanceId
17791866 ? [
17801867 {
17811868 type : "checkpoint-scope.created" ,
17821869 threadId,
17831870 runId : queuedRun . id ,
1784- nodeId : rootNode . id ,
1871+ nodeId : rootNodeId ,
17851872 providerInstanceId : queuedRun . providerInstanceId ,
17861873 occurredAt : now ,
1787- payload : checkpointScope ,
1874+ payload : { ... checkpointScope , providerThreadId : queuedProviderThread . id } ,
17881875 } ,
17891876 {
17901877 type : "node.updated" ,
17911878 threadId,
17921879 runId : queuedRun . id ,
1793- nodeId : rootNode . id ,
1880+ nodeId : rootNodeId ,
1881+ providerInstanceId : queuedRun . providerInstanceId ,
1882+ occurredAt : now ,
1883+ payload : {
1884+ ...rootNode ,
1885+ providerThreadId : queuedProviderThread . id ,
1886+ checkpointScopeId : checkpointScope . id ,
1887+ } ,
1888+ } ,
1889+ {
1890+ type : "run-attempt.updated" ,
1891+ threadId,
1892+ runId : queuedRun . id ,
1893+ nodeId : rootNodeId ,
17941894 providerInstanceId : queuedRun . providerInstanceId ,
17951895 occurredAt : now ,
1796- payload : { ...rootNode , checkpointScopeId : checkpointScope . id } ,
1896+ payload : {
1897+ ...attempt ,
1898+ providerInstanceId : queuedRun . providerInstanceId ,
1899+ providerThreadId : queuedProviderThread . id ,
1900+ } ,
17971901 } ,
17981902 ]
17991903 : [ ] ;
@@ -1906,7 +2010,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
19062010 reason : "Provider or model selection changed." ,
19072011 } ,
19082012 } ) ) ,
1909- ...checkpointEvents ,
2013+ ...executionIdentityEvents ,
19102014 {
19112015 type : "provider-thread.updated" ,
19122016 threadId,
@@ -11250,8 +11354,8 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
1125011354 case "queue.resume" : {
1125111355 const projection = yield * loadProjectionForCommand (
1125211356 command ,
11253- [ "runs" , "turnItems" , "providerSessions" ] ,
11254- { turnItemTypes : [ "error" ] } ,
11357+ [ "runs" , "turnItems" , "providerSessions" , "messages" ] ,
11358+ { turnItemTypes : [ "error" ] , messageRoles : [ "user" ] } ,
1125511359 ) ;
1125611360 if ( projection . thread . archivedAt !== null || projection . thread . deletedAt !== null ) {
1125711361 return yield * new OrchestratorDispatchError ( {
@@ -11260,16 +11364,15 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
1126011364 cause : `Thread ${ command . threadId } is not active.` ,
1126111365 } ) ;
1126211366 }
11263- const sessionError =
11264- projection . providerSessions
11265- . filter (
11266- ( session ) => session . providerInstanceId === projection . thread . providerInstanceId ,
11267- )
11268- . toSorted (
11269- ( left , right ) =>
11270- DateTime . toEpochMillis ( right . updatedAt ) - DateTime . toEpochMillis ( left . updatedAt ) ,
11271- ) [ 0 ] ?. lastError ?? null ;
11272- if ( usageLimitBlockedRun ( projection . runs , projection . turnItems , sessionError ) !== null ) {
11367+ const nextRun = nextQueuedRun ( projection ) ;
11368+ if (
11369+ usageLimitHoldsQueue (
11370+ projection ,
11371+ nextRun === undefined
11372+ ? projection . thread . providerInstanceId
11373+ : queuedRunModelSelection ( projection , nextRun ) . instanceId ,
11374+ )
11375+ ) {
1127311376 return yield * new OrchestratorDispatchError ( {
1127411377 commandId : command . commandId ,
1127511378 commandType : command . type ,
0 commit comments