Skip to content

Commit f8bcc3c

Browse files
voltcrashclaudejuliusmarminge
authored
fix(server): queued PR updates follow the takeover provider (pingdotgg#21314)
Co-authored-by: Claude Opus 5.5 <[email protected]> Co-authored-by: Julius Marminge <[email protected]>
1 parent 3c18818 commit f8bcc3c

3 files changed

Lines changed: 671 additions & 68 deletions

File tree

‎apps/server/src/orchestration-v2/Orchestrator.ts‎

Lines changed: 166 additions & 63 deletions
Original file line numberDiff line numberDiff line change
@@ -638,6 +638,45 @@ function nextQueuedRun(
638638
return queuedRunsInDeliveryOrder(projection)[0];
639639
}
640640

641+
function followsCurrentConversation(
642+
message: OrchestrationV2ConversationMessage | undefined,
643+
): boolean {
644+
return (
645+
message?.notification?.source.kind === "monitor" && !isBufferedProviderNotification(message)
646+
);
647+
}
648+
649+
function queuedRunModelSelection(
650+
projection: Pick<OrchestrationV2ThreadProjection, "thread" | "messages">,
651+
run: OrchestrationV2Run,
652+
): ModelSelection {
653+
const message = projection.messages.find((candidate) => candidate.id === run.userMessageId);
654+
// App-owned monitor updates follow the agent currently handling the thread.
655+
return followsCurrentConversation(message)
656+
? projection.thread.modelSelection
657+
: run.modelSelection;
658+
}
659+
660+
function usageLimitHoldsQueue(
661+
projection: Pick<OrchestrationV2ThreadProjection, "runs" | "turnItems" | "providerSessions">,
662+
providerInstanceId: ModelSelection["instanceId"],
663+
): boolean {
664+
let latestSession: OrchestrationV2ThreadProjection["providerSessions"][number] | undefined;
665+
for (const session of projection.providerSessions) {
666+
if (
667+
session.providerInstanceId === providerInstanceId &&
668+
(latestSession === undefined ||
669+
DateTime.isGreaterThan(session.updatedAt, latestSession.updatedAt))
670+
) {
671+
latestSession = session;
672+
}
673+
}
674+
return (
675+
usageLimitBlockedRun(projection.runs, projection.turnItems, latestSession?.lastError ?? null)
676+
?.providerInstanceId === providerInstanceId
677+
);
678+
}
679+
641680
function latestStableRun(
642681
projection: Pick<OrchestrationV2ThreadProjection, "runs">,
643682
): OrchestrationV2Run | null {
@@ -1328,20 +1367,21 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
13281367
return;
13291368
}
13301369

1331-
// The limit already stopped this thread. Starting the queue would send
1332-
// every waiting message and drop it from the queue as each one fails.
1333-
const sessionError =
1334-
projection.providerSessions
1335-
.filter((session) => session.providerInstanceId === projection.thread.providerInstanceId)
1336-
.toSorted(
1337-
(left, right) =>
1338-
DateTime.toEpochMillis(right.updatedAt) - DateTime.toEpochMillis(left.updatedAt),
1339-
)[0]?.lastError ?? null;
1340-
if (usageLimitBlockedRun(projection.runs, projection.turnItems, sessionError) !== null) {
1370+
const storedQueuedRun = nextQueuedRun(projection);
1371+
if (storedQueuedRun === undefined) {
13411372
return;
13421373
}
1343-
const queuedRun = nextQueuedRun(projection);
1344-
if (queuedRun === undefined) {
1374+
const queuedMessage = projection.messages.find(
1375+
(candidate) => candidate.id === storedQueuedRun.userMessageId,
1376+
);
1377+
const modelSelection = queuedRunModelSelection(projection, storedQueuedRun);
1378+
const queuedRun = {
1379+
...storedQueuedRun,
1380+
modelSelection,
1381+
providerInstanceId: modelSelection.instanceId,
1382+
};
1383+
// A limited provider would fail and consume every waiting message.
1384+
if (usageLimitHoldsQueue(projection, queuedRun.providerInstanceId)) {
13451385
return;
13461386
}
13471387
// A message queued behind a worktree that was cancelled or never created
@@ -1420,9 +1460,6 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
14201460

14211461
const rootNode = projection.nodes.find((candidate) => candidate.id === rootNodeId);
14221462
const attempt = projection.attempts.find((candidate) => candidate.id === attemptId);
1423-
const queuedMessage = projection.messages.find(
1424-
(candidate) => candidate.id === queuedRun.userMessageId,
1425-
);
14261463
const legacyQueuedTurnItem = projection.turnItems.find(
14271464
(
14281465
candidate,
@@ -1431,7 +1468,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
14311468
candidate.runId === queuedRun.id &&
14321469
candidate.messageId === queuedRun.userMessageId,
14331470
);
1434-
const queuedProviderThread = projection.providerThreads.find(
1471+
const storedProviderThread = projection.providerThreads.find(
14351472
(candidate) => candidate.id === providerThreadId,
14361473
);
14371474
const storedCheckpointScope = projection.checkpointScopes.find(
@@ -1441,7 +1478,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
14411478
rootNode === undefined ||
14421479
attempt === undefined ||
14431480
queuedMessage === undefined ||
1444-
queuedProviderThread === undefined ||
1481+
storedProviderThread === undefined ||
14451482
(rootNode.checkpointScopeId !== null && storedCheckpointScope === undefined)
14461483
) {
14471484
return yield* new OrchestratorDispatchError({
@@ -1453,27 +1490,83 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
14531490

14541491
const commandId = CommandId.make(`command:system:start-queued:${queuedRun.id}`);
14551492
const now = yield* DateTime.now;
1456-
const selectionChanged = !modelSelectionsEqual(
1457-
projection.thread.modelSelection,
1458-
queuedRun.modelSelection,
1493+
const targetAdapter = yield* providerAdapters.get(queuedRun.providerInstanceId).pipe(
1494+
Effect.mapError(
1495+
(cause) =>
1496+
new OrchestratorDispatchError({
1497+
commandId,
1498+
commandType: "message.dispatch",
1499+
cause,
1500+
}),
1501+
),
14591502
);
1460-
const switchPlan = selectionChanged
1461-
? yield* providerSwitchService
1462-
.plan({ projection, targetModelSelection: queuedRun.modelSelection })
1463-
.pipe(
1464-
Effect.mapError(
1465-
(cause) =>
1466-
new OrchestratorDispatchError({
1467-
commandId,
1468-
commandType: "message.dispatch",
1469-
cause,
1470-
}),
1471-
),
1472-
)
1473-
: null;
14741503
const activeProviderThread = projection.providerThreads.find(
14751504
(candidate) => candidate.id === projection.thread.activeProviderThreadId,
14761505
);
1506+
const followsCurrent = followsCurrentConversation(queuedMessage);
1507+
const targetProviderThread = followsCurrent
1508+
? activeProviderThread?.providerInstanceId === queuedRun.providerInstanceId
1509+
? activeProviderThread
1510+
: rootProviderThreadsForProvider(projection, queuedRun.providerInstanceId)[0]
1511+
: storedProviderThread;
1512+
const queuedProviderThread: OrchestrationV2ProviderThread = targetProviderThread ?? {
1513+
id: idAllocator.derive.providerThread({
1514+
driver: targetAdapter.driver,
1515+
nativeThreadId: `pending:${queuedRun.id}`,
1516+
}),
1517+
driver: targetAdapter.driver,
1518+
providerInstanceId: queuedRun.providerInstanceId,
1519+
providerSessionId: null,
1520+
appThreadId: threadId,
1521+
ownerNodeId: null,
1522+
nativeThreadRef: null,
1523+
nativeConversationHeadRef: null,
1524+
status: "not_loaded",
1525+
firstRunOrdinal: null,
1526+
lastRunOrdinal: null,
1527+
handoffIds: [],
1528+
forkedFrom: null,
1529+
createdAt: now,
1530+
updatedAt: now,
1531+
};
1532+
const selectionChanged = !modelSelectionsEqual(
1533+
projection.thread.modelSelection,
1534+
queuedRun.modelSelection,
1535+
);
1536+
// The picker may already have moved ahead of the native conversation.
1537+
const switchFromActiveProvider =
1538+
followsCurrent &&
1539+
activeProviderThread !== undefined &&
1540+
activeProviderThread.providerInstanceId !== queuedRun.providerInstanceId;
1541+
const switchPlan =
1542+
selectionChanged || switchFromActiveProvider
1543+
? yield* providerSwitchService
1544+
.plan({
1545+
projection: switchFromActiveProvider
1546+
? {
1547+
...projection,
1548+
thread: {
1549+
...projection.thread,
1550+
modelSelection: {
1551+
...projection.thread.modelSelection,
1552+
instanceId: activeProviderThread.providerInstanceId,
1553+
},
1554+
},
1555+
}
1556+
: projection,
1557+
targetModelSelection: queuedRun.modelSelection,
1558+
})
1559+
.pipe(
1560+
Effect.mapError(
1561+
(cause) =>
1562+
new OrchestratorDispatchError({
1563+
commandId,
1564+
commandType: "message.dispatch",
1565+
cause,
1566+
}),
1567+
),
1568+
)
1569+
: null;
14771570
const canResumeAcrossInstances =
14781571
switchPlan?.instanceChanged === true &&
14791572
switchPlan.transition.type === "restart_and_resume" &&
@@ -1488,16 +1581,6 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
14881581
nativeMetadata: activeProviderThread.nativeMetadata,
14891582
}
14901583
: queuedProviderThread;
1491-
const targetAdapter = yield* providerAdapters.get(queuedRun.providerInstanceId).pipe(
1492-
Effect.mapError(
1493-
(cause) =>
1494-
new OrchestratorDispatchError({
1495-
commandId,
1496-
commandType: "message.dispatch",
1497-
cause,
1498-
}),
1499-
),
1500-
);
15011584
const targetCapabilities = yield* targetAdapter.getCapabilities().pipe(
15021585
Effect.mapError(
15031586
(cause) =>
@@ -1690,6 +1773,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
16901773
};
16911774
const startingRun: OrchestrationV2Run = {
16921775
...queuedRun,
1776+
providerThreadId: queuedProviderThread.id,
16931777
ordinal,
16941778
status: "starting",
16951779
queuePosition: null,
@@ -1728,6 +1812,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
17281812
? {}
17291813
: { senderThreadId: queuedMessage.senderThreadId }),
17301814
}),
1815+
providerThreadId: queuedProviderThread.id,
17311816
inputIntent: "queued_turn",
17321817
startedAt: now,
17331818
completedAt: now,
@@ -1774,26 +1859,45 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
17741859
strategy: activeHandoff.strategy,
17751860
summary: activeHandoff.summaryText,
17761861
};
1777-
const checkpointEvents: ReadonlyArray<Omit<OrchestrationV2DomainEvent, "id">> =
1778-
storedCheckpointScope === undefined
1862+
const executionIdentityEvents: ReadonlyArray<Omit<OrchestrationV2DomainEvent, "id">> =
1863+
storedCheckpointScope === undefined ||
1864+
storedProviderThread.id !== queuedProviderThread.id ||
1865+
attempt.providerInstanceId !== queuedRun.providerInstanceId
17791866
? [
17801867
{
17811868
type: "checkpoint-scope.created",
17821869
threadId,
17831870
runId: queuedRun.id,
1784-
nodeId: rootNode.id,
1871+
nodeId: rootNodeId,
17851872
providerInstanceId: queuedRun.providerInstanceId,
17861873
occurredAt: now,
1787-
payload: checkpointScope,
1874+
payload: { ...checkpointScope, providerThreadId: queuedProviderThread.id },
17881875
},
17891876
{
17901877
type: "node.updated",
17911878
threadId,
17921879
runId: queuedRun.id,
1793-
nodeId: rootNode.id,
1880+
nodeId: rootNodeId,
1881+
providerInstanceId: queuedRun.providerInstanceId,
1882+
occurredAt: now,
1883+
payload: {
1884+
...rootNode,
1885+
providerThreadId: queuedProviderThread.id,
1886+
checkpointScopeId: checkpointScope.id,
1887+
},
1888+
},
1889+
{
1890+
type: "run-attempt.updated",
1891+
threadId,
1892+
runId: queuedRun.id,
1893+
nodeId: rootNodeId,
17941894
providerInstanceId: queuedRun.providerInstanceId,
17951895
occurredAt: now,
1796-
payload: { ...rootNode, checkpointScopeId: checkpointScope.id },
1896+
payload: {
1897+
...attempt,
1898+
providerInstanceId: queuedRun.providerInstanceId,
1899+
providerThreadId: queuedProviderThread.id,
1900+
},
17971901
},
17981902
]
17991903
: [];
@@ -1906,7 +2010,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
19062010
reason: "Provider or model selection changed.",
19072011
},
19082012
})),
1909-
...checkpointEvents,
2013+
...executionIdentityEvents,
19102014
{
19112015
type: "provider-thread.updated",
19122016
threadId,
@@ -11250,8 +11354,8 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
1125011354
case "queue.resume": {
1125111355
const projection = yield* loadProjectionForCommand(
1125211356
command,
11253-
["runs", "turnItems", "providerSessions"],
11254-
{ turnItemTypes: ["error"] },
11357+
["runs", "turnItems", "providerSessions", "messages"],
11358+
{ turnItemTypes: ["error"], messageRoles: ["user"] },
1125511359
);
1125611360
if (projection.thread.archivedAt !== null || projection.thread.deletedAt !== null) {
1125711361
return yield* new OrchestratorDispatchError({
@@ -11260,16 +11364,15 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
1126011364
cause: `Thread ${command.threadId} is not active.`,
1126111365
});
1126211366
}
11263-
const sessionError =
11264-
projection.providerSessions
11265-
.filter(
11266-
(session) => session.providerInstanceId === projection.thread.providerInstanceId,
11267-
)
11268-
.toSorted(
11269-
(left, right) =>
11270-
DateTime.toEpochMillis(right.updatedAt) - DateTime.toEpochMillis(left.updatedAt),
11271-
)[0]?.lastError ?? null;
11272-
if (usageLimitBlockedRun(projection.runs, projection.turnItems, sessionError) !== null) {
11367+
const nextRun = nextQueuedRun(projection);
11368+
if (
11369+
usageLimitHoldsQueue(
11370+
projection,
11371+
nextRun === undefined
11372+
? projection.thread.providerInstanceId
11373+
: queuedRunModelSelection(projection, nextRun).instanceId,
11374+
)
11375+
) {
1127311376
return yield* new OrchestratorDispatchError({
1127411377
commandId: command.commandId,
1127511378
commandType: command.type,

0 commit comments

Comments
 (0)