Skip to content

Commit 8d54c8b

Browse files
fix(server): a steer that restarts a completion wake no longer strands later subagent results (pingdotgg#33469)
Co-authored-by: Claude Opus 5.5 <[email protected]>
1 parent 9e12eca commit 8d54c8b

2 files changed

Lines changed: 429 additions & 83 deletions

File tree

‎apps/server/src/orchestration-v2/Orchestrator.ts‎

Lines changed: 131 additions & 83 deletions
Original file line numberDiff line numberDiff line change
@@ -1600,6 +1600,104 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
16001600
}
16011601
});
16021602

1603+
/** Marks a wake the provider received delivered and reserves the next one for pending siblings. */
1604+
const acceptDelegatedCompletionDelivery = Effect.fn(
1605+
"Orchestrator.acceptDelegatedCompletionDelivery",
1606+
)(function* (input: {
1607+
readonly command: OrchestrationV2ServerCommand;
1608+
readonly events: Ref.Ref<Array<OrchestrationV2DomainEvent>>;
1609+
readonly projection: Pick<
1610+
OrchestrationV2ThreadProjection,
1611+
"thread" | "runs" | "messages" | "subagents"
1612+
>;
1613+
readonly messageId: OrchestrationV2ConversationMessage["id"];
1614+
readonly now: DateTime.Utc;
1615+
}) {
1616+
const { projection, now } = input;
1617+
const message = projection.messages.find((row) => row.id === input.messageId);
1618+
const ownership = message?.delegatedCompletion;
1619+
const parentRun = projection.runs.find((run) => run.id === ownership?.parentRunId);
1620+
const cohort = parentRun?.delegatedCompletion;
1621+
const delivery = cohort?.delivery;
1622+
if (
1623+
parentRun === undefined ||
1624+
cohort === undefined ||
1625+
cohort.disposition !== "open" ||
1626+
delivery == null ||
1627+
delivery.messageId !== input.messageId ||
1628+
delivery.generation !== ownership?.generation
1629+
) {
1630+
return undefined;
1631+
}
1632+
const threadId = projection.thread.id;
1633+
const parentIsLive = hasLiveRun({ runs: [parentRun] });
1634+
const pendingTaskIds =
1635+
projection.thread.archivedAt === null && projection.thread.deletedAt === null
1636+
? projection.subagents
1637+
.filter(
1638+
(task) =>
1639+
task.origin === "app_owned" &&
1640+
task.runId === parentRun.id &&
1641+
task.completionDelivery?.state === "pending" &&
1642+
isTerminalDelegatedTaskStatus(task.status) &&
1643+
(!parentIsLive || task.completionWake === "always"),
1644+
)
1645+
.map((task) => task.id)
1646+
: [];
1647+
const nextDelivery =
1648+
pendingTaskIds.length === 0
1649+
? null
1650+
: {
1651+
generation: cohort.nextGeneration,
1652+
messageId: yield* mapDelegatedCompletionError(
1653+
idAllocator.allocate.message({
1654+
threadId,
1655+
ordinal:
1656+
(yield* mapDelegatedCompletionError(projectionStore.getMessageCount(threadId))) +
1657+
1,
1658+
}),
1659+
),
1660+
taskIds: pendingTaskIds,
1661+
};
1662+
const emitEvent = emit(input.events, input.command);
1663+
const acceptedIds = new Set(delivery.taskIds);
1664+
const pendingIds = new Set(pendingTaskIds);
1665+
for (const task of projection.subagents) {
1666+
const state = pendingIds.has(task.id)
1667+
? "claimed"
1668+
: acceptedIds.has(task.id) && task.completionDelivery?.state === "claimed"
1669+
? "delivered"
1670+
: undefined;
1671+
if (state === undefined) continue;
1672+
yield* emitEvent({
1673+
type: "subagent.updated",
1674+
threadId,
1675+
runId: parentRun.id,
1676+
nodeId: task.id,
1677+
driver: task.driver,
1678+
providerInstanceId: task.providerInstanceId,
1679+
occurredAt: now,
1680+
payload: { ...task, completionDelivery: { state, observedByRunId: null }, updatedAt: now },
1681+
});
1682+
}
1683+
yield* emitEvent({
1684+
type: "run.updated",
1685+
threadId,
1686+
runId: parentRun.id,
1687+
providerInstanceId: parentRun.providerInstanceId,
1688+
occurredAt: now,
1689+
payload: {
1690+
...parentRun,
1691+
delegatedCompletion: {
1692+
...cohort,
1693+
nextGeneration: cohort.nextGeneration + (nextDelivery === null ? 0 : 1),
1694+
delivery: nextDelivery,
1695+
},
1696+
},
1697+
});
1698+
return parentRun.id;
1699+
});
1700+
16031701
const emitQueuedRunCancellation = Effect.fn("Orchestrator.emitQueuedRunCancellation")(
16041702
function* (input: {
16051703
readonly command: OrchestrationV2ServerCommand;
@@ -5698,6 +5796,15 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
56985796
occurredAt: now,
56995797
payload: checkpointScope,
57005798
});
5799+
// The restart replaces the run's input, so the wake it was running can no longer be
5800+
// settled at its terminal. Its provider turn was running, so the wake already landed.
5801+
yield* acceptDelegatedCompletionDelivery({
5802+
command: input.command,
5803+
events: input.events,
5804+
projection: input.projection,
5805+
messageId: targetRun.userMessageId,
5806+
now,
5807+
});
57015808
yield* appendSteeringMessage({
57025809
runId: targetRun.id,
57035810
nodeId: nextRootNodeId,
@@ -13347,95 +13454,26 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
1334713454
events: Ref.Ref<Array<OrchestrationV2DomainEvent>>,
1334813455
) {
1334913456
const projection = yield* getProjectionWithPendingEvents(command.threadId, events);
13350-
const message = projection.messages.find((row) => row.id === command.messageId);
13351-
const ownership = message?.delegatedCompletion;
13352-
const parentRun = projection.runs.find((run) => run.id === ownership?.parentRunId);
13353-
const cohort = parentRun?.delegatedCompletion;
13354-
const delivery = cohort?.delivery;
1335513457
const now = yield* DateTime.now;
13356-
const emitEvent = emit(events, command);
13357-
if (
13358-
parentRun === undefined ||
13359-
cohort === undefined ||
13360-
cohort.disposition !== "open" ||
13361-
delivery == null ||
13362-
delivery.messageId !== command.messageId ||
13363-
delivery.generation !== ownership?.generation
13364-
) {
13365-
yield* emitEvent({
13458+
const parentRunId = yield* acceptDelegatedCompletionDelivery({
13459+
command,
13460+
events,
13461+
projection,
13462+
messageId: command.messageId,
13463+
now,
13464+
});
13465+
if (parentRunId === undefined) {
13466+
yield* emit(
13467+
events,
13468+
command,
13469+
)({
1336613470
type: "thread.metadata-updated",
1336713471
threadId: command.threadId,
1336813472
occurredAt: now,
1336913473
payload: projection.thread,
1337013474
});
13371-
return;
1337213475
}
13373-
const parentIsLive = hasLiveRun({ runs: [parentRun] });
13374-
const pendingTaskIds =
13375-
projection.thread.archivedAt === null && projection.thread.deletedAt === null
13376-
? projection.subagents
13377-
.filter(
13378-
(task) =>
13379-
task.origin === "app_owned" &&
13380-
task.runId === parentRun.id &&
13381-
task.completionDelivery?.state === "pending" &&
13382-
isTerminalDelegatedTaskStatus(task.status) &&
13383-
(!parentIsLive || task.completionWake === "always"),
13384-
)
13385-
.map((task) => task.id)
13386-
: [];
13387-
const nextDelivery =
13388-
pendingTaskIds.length === 0
13389-
? null
13390-
: {
13391-
generation: cohort.nextGeneration,
13392-
messageId: yield* mapDelegatedCompletionError(
13393-
idAllocator.allocate.message({
13394-
threadId: command.threadId,
13395-
ordinal:
13396-
(yield* mapDelegatedCompletionError(
13397-
projectionStore.getMessageCount(projection.thread.id),
13398-
)) + 1,
13399-
}),
13400-
),
13401-
taskIds: pendingTaskIds,
13402-
};
13403-
const acceptedIds = new Set(delivery.taskIds);
13404-
const pendingIds = new Set(pendingTaskIds);
13405-
for (const task of projection.subagents) {
13406-
const state = pendingIds.has(task.id)
13407-
? "claimed"
13408-
: acceptedIds.has(task.id) && task.completionDelivery?.state === "claimed"
13409-
? "delivered"
13410-
: undefined;
13411-
if (state === undefined) continue;
13412-
yield* emitEvent({
13413-
type: "subagent.updated",
13414-
threadId: command.threadId,
13415-
runId: parentRun.id,
13416-
nodeId: task.id,
13417-
driver: task.driver,
13418-
providerInstanceId: task.providerInstanceId,
13419-
occurredAt: now,
13420-
payload: { ...task, completionDelivery: { state, observedByRunId: null }, updatedAt: now },
13421-
});
13422-
}
13423-
yield* emitEvent({
13424-
type: "run.updated",
13425-
threadId: command.threadId,
13426-
runId: parentRun.id,
13427-
providerInstanceId: parentRun.providerInstanceId,
13428-
occurredAt: now,
13429-
payload: {
13430-
...parentRun,
13431-
delegatedCompletion: {
13432-
...cohort,
13433-
nextGeneration: cohort.nextGeneration + (nextDelivery === null ? 0 : 1),
13434-
delivery: nextDelivery,
13435-
},
13436-
},
13437-
});
13438-
return parentRun.id;
13476+
return parentRunId;
1343913477
});
1344013478

1344113479
const dispatchUnsupported = (command: OrchestrationV2ServerCommand) =>
@@ -14125,7 +14163,17 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
1412514163
),
1412614164
);
1412714165
}
14128-
if (command.type === "notification.delivery.accept") {
14166+
if (
14167+
command.type === "notification.delivery.accept" ||
14168+
// A steer restart that settled a wake can reserve the next one for its siblings.
14169+
((command.type === "message.dispatch" ||
14170+
command.type === "queued-message.promote-to-steer") &&
14171+
committed.storedEvents.some(
14172+
({ event }) =>
14173+
event.type === "subagent.updated" &&
14174+
event.payload.completionDelivery?.state === "claimed",
14175+
))
14176+
) {
1412914177
yield* mapDispatchError(command)(offerDelegatedCompletionDeliveries(command.threadId));
1413014178
}
1413114179
if (

0 commit comments

Comments
 (0)