Skip to content

Commit 4dff7de

Browse files
shivamhwpPR Batch Testerclaude
authored
fix(server): preserve sibling wakeups when interrupting a completion wake (pingdotgg#28204)
Co-authored-by: PR Batch Tester <[email protected]> Co-authored-by: Claude Opus 5.5 <[email protected]>
1 parent 9bfecb8 commit 4dff7de

2 files changed

Lines changed: 287 additions & 7 deletions

File tree

‎apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts‎

Lines changed: 253 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,8 @@ import * as ThreadCommandExecutor from "./ThreadCommandExecutor.ts";
4949
import * as ProviderTurnStartServiceTestkit from "./ProviderTurnStartService.testkit.ts";
5050
import type * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter";
5151
import * as ProviderContinuationRequests from "@t3tools/provider-core/server/ProviderContinuationRequests";
52+
import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator";
53+
import * as ProviderContinuationService from "./ProviderContinuationService.ts";
5254

5355
const layerPlatformTest = Layer.merge(
5456
Layer.mergeAll(NodeServices.layer, NodeCrypto.layer),
@@ -152,6 +154,12 @@ const layerTest = Layer.mergeAll(
152154
Layer.provide(layerPlatformTest),
153155
);
154156

157+
// ProviderContinuationService consumes the orchestrator's continuation queue, as in production.
158+
const layerContinuationWorkerTest = ProviderContinuationService.layer.pipe(
159+
Layer.provide(Layer.merge(ProviderContinuationRequests.layer, IdAllocator.layer)),
160+
Layer.provideMerge(layerTest),
161+
);
162+
155163
const seedParentWithTerminalTask = (input: {
156164
readonly threadId: ThreadId;
157165
readonly projectId: ProjectId;
@@ -1370,6 +1378,251 @@ const seedRestartCancelledChild = (input: {
13701378
return { taskId, childThreadId, childRunId };
13711379
});
13721380

1381+
it.layer(layerContinuationWorkerTest)("interrupted completion wakes", (it) => {
1382+
it.effect.each([
1383+
{ name: "an interrupt keeps", holdQueue: false, siblingFinishesFirst: false },
1384+
{ name: "Stop disposes", holdQueue: true, siblingFinishesFirst: false },
1385+
{ name: "an interrupt reserves", holdQueue: false, siblingFinishesFirst: true },
1386+
])("$name a completion wake's other siblings", ({ holdQueue, siblingFinishesFirst }) =>
1387+
Effect.gen(function* () {
1388+
const orchestrator = yield* Orchestrator.OrchestratorV2;
1389+
const sink = yield* EventSink.EventSinkV2;
1390+
const now = yield* DateTime.now;
1391+
const key = holdQueue
1392+
? "sibling-stop"
1393+
: siblingFinishesFirst
1394+
? "sibling-finished-interrupt"
1395+
: "sibling-interrupt";
1396+
const threadId = ThreadId.make(`${key}-parent`);
1397+
const originRunId = RunId.make(`${key}-origin`);
1398+
const incomingTaskId = NodeId.make(`${key}-incoming`);
1399+
yield* seedParentWithTerminalTask({
1400+
threadId,
1401+
projectId: ProjectId.make(`${key}-project`),
1402+
runId: originRunId,
1403+
rootNodeId: NodeId.make(`${key}-origin-root`),
1404+
taskId: incomingTaskId,
1405+
deliveryState: "claimed",
1406+
deliveryTaskIds: [incomingTaskId],
1407+
completionWake: "always",
1408+
now,
1409+
});
1410+
const originRootId = NodeId.make(`${key}-origin-root`);
1411+
yield* sink.write({
1412+
events: [
1413+
{
1414+
id: EventId.make(`event:${key}-origin-root`),
1415+
type: "node.updated",
1416+
threadId,
1417+
runId: originRunId,
1418+
nodeId: originRootId,
1419+
occurredAt: now,
1420+
payload: {
1421+
id: originRootId,
1422+
threadId,
1423+
runId: originRunId,
1424+
parentNodeId: null,
1425+
rootNodeId: originRootId,
1426+
kind: "root_turn",
1427+
status: "running",
1428+
countsForRun: true,
1429+
providerThreadId: null,
1430+
providerTurnId: null,
1431+
nativeItemRef: null,
1432+
runtimeRequestId: null,
1433+
checkpointScopeId: null,
1434+
startedAt: now,
1435+
completedAt: null,
1436+
},
1437+
},
1438+
],
1439+
});
1440+
yield* orchestrator.dispatch({
1441+
type: "delegated_task.request",
1442+
createdBy: "agent",
1443+
creationSource: "mcp",
1444+
commandId: CommandId.make(`${key}-sibling`),
1445+
parentThreadId: threadId,
1446+
parentRunId: originRunId,
1447+
parentNodeId: originRootId,
1448+
task: "Keep working",
1449+
modelSelection,
1450+
runtimeMode: "full-access",
1451+
interactionMode: "default",
1452+
completionWake: "always",
1453+
});
1454+
const seeded = yield* orchestrator.getThreadProjection(threadId);
1455+
const origin = seeded.runs.find((run) => run.id === originRunId)!;
1456+
const sibling = seeded.subagents.find((task) => task.id !== incomingTaskId)!;
1457+
yield* sink.write({
1458+
events: [
1459+
{
1460+
...runEvent({ threadId, runId: originRunId, ordinal: 1, status: "completed", now }),
1461+
payload: { ...origin, status: "completed", completedAt: now },
1462+
},
1463+
],
1464+
});
1465+
const delivery = origin.delegatedCompletion!.delivery!;
1466+
yield* orchestrator.dispatch({
1467+
type: "message.dispatch",
1468+
createdBy: "agent",
1469+
creationSource: "server",
1470+
commandId: CommandId.make(`${key}-wake`),
1471+
threadId,
1472+
messageId: delivery.messageId,
1473+
text: "Delegated task completed",
1474+
attachments: [],
1475+
modelSelection,
1476+
dispatchMode: { type: "queue_after_active" },
1477+
delegatedCompletion: {
1478+
parentRunId: originRunId,
1479+
generation: delivery.generation,
1480+
taskIds: delivery.taskIds,
1481+
},
1482+
});
1483+
yield* orchestrator.resumeQueuedRuns;
1484+
const wake = (yield* orchestrator.getThreadProjection(threadId)).runs.find(
1485+
(run) => run.userMessageId === delivery.messageId,
1486+
)!;
1487+
const childRun = (yield* orchestrator.getThreadProjection(sibling.childThreadId!)).runs[0]!;
1488+
const completeSibling = Effect.gen(function* () {
1489+
const afterSequence = yield* sink.latestSequence();
1490+
yield* sink.write({
1491+
events: [
1492+
{
1493+
...runEvent({
1494+
threadId: sibling.childThreadId!,
1495+
runId: childRun.id,
1496+
ordinal: childRun.ordinal,
1497+
status: "completed",
1498+
now,
1499+
}),
1500+
payload: { ...childRun, status: "completed", completedAt: now },
1501+
},
1502+
],
1503+
});
1504+
return yield* sink.stream({ afterSequence, eventType: "subagent.updated" }).pipe(
1505+
Stream.filter(
1506+
(stored) =>
1507+
stored.event.type === "subagent.updated" &&
1508+
stored.event.payload.id === sibling.id &&
1509+
stored.event.payload.status === "completed",
1510+
),
1511+
Stream.take(1),
1512+
Stream.runDrain,
1513+
Effect.as(afterSequence),
1514+
);
1515+
});
1516+
if (siblingFinishesFirst) {
1517+
yield* completeSibling;
1518+
assert.equal(
1519+
(yield* orchestrator.getThreadProjection(threadId)).subagents.find(
1520+
(task) => task.id === sibling.id,
1521+
)?.completionDelivery?.state,
1522+
"pending",
1523+
);
1524+
}
1525+
const beforeInterrupt = yield* sink.latestSequence();
1526+
const interrupted = yield* orchestrator.dispatch({
1527+
type: "run.interrupt",
1528+
commandId: CommandId.make(`${key}-interrupt`),
1529+
threadId,
1530+
runId: wake.id,
1531+
...(holdQueue ? { holdQueue } : {}),
1532+
});
1533+
// The wake's terminal may claim the sibling right after commit, so read what the interrupt
1534+
// itself wrote rather than a later projection.
1535+
const interruptDelivery = new Map(
1536+
interrupted.storedEvents.flatMap(({ event }) =>
1537+
event.type === "subagent.updated"
1538+
? [[event.payload.id, event.payload.completionDelivery?.state] as const]
1539+
: [],
1540+
),
1541+
);
1542+
const interruptCohort = interrupted.storedEvents.flatMap(({ event }) =>
1543+
event.type === "run.updated" && event.payload.id === originRunId
1544+
? [event.payload.delegatedCompletion]
1545+
: [],
1546+
);
1547+
assert.equal(interruptDelivery.get(incomingTaskId), "disposed");
1548+
assert.equal(interruptCohort.at(-1)?.disposition, holdQueue ? "stopped" : "open");
1549+
if (holdQueue) {
1550+
assert.equal(interruptDelivery.get(sibling.id), "disposed");
1551+
return;
1552+
}
1553+
assert.isFalse(interruptDelivery.has(sibling.id));
1554+
1555+
const afterSequence = siblingFinishesFirst ? beforeInterrupt : yield* completeSibling;
1556+
// The wake's terminal, or the sibling's own completion, reserves the successor.
1557+
yield* sink.stream({ afterSequence, eventType: "subagent.updated" }).pipe(
1558+
Stream.filter(
1559+
(stored) =>
1560+
stored.event.type === "subagent.updated" &&
1561+
stored.event.payload.id === sibling.id &&
1562+
stored.event.payload.completionDelivery?.state === "claimed",
1563+
),
1564+
Stream.take(1),
1565+
Stream.runDrain,
1566+
);
1567+
const woken = yield* orchestrator.getThreadProjection(threadId);
1568+
const successor = woken.runs.find((run) => run.id === originRunId)?.delegatedCompletion
1569+
?.delivery;
1570+
assert.equal(successor?.generation, delivery.generation + 1);
1571+
assert.deepEqual(successor?.taskIds, [sibling.id]);
1572+
assert.notEqual(successor?.messageId, delivery.messageId);
1573+
1574+
// ProviderContinuationService consumes the claim's offer and dispatches the successor.
1575+
yield* sink.stream({ afterSequence, eventType: "message.updated" }).pipe(
1576+
Stream.filter(
1577+
(stored) =>
1578+
stored.event.type === "message.updated" &&
1579+
stored.event.payload.id === successor!.messageId,
1580+
),
1581+
Stream.take(1),
1582+
Stream.runDrain,
1583+
);
1584+
// Repeated recovery re-offers every open delivery; none may wake the parent twice.
1585+
yield* orchestrator.resumeQueuedRuns;
1586+
yield* orchestrator.recoverDelegatedTasks;
1587+
yield* orchestrator.recoverDelegatedTasks;
1588+
yield* orchestrator.resumeQueuedRuns;
1589+
const delivered = yield* orchestrator.getThreadProjection(threadId);
1590+
const cohort = delivered.runs.find((run) => run.id === originRunId)?.delegatedCompletion;
1591+
assert.equal(cohort?.disposition, "open");
1592+
assert.equal(cohort?.nextGeneration, delivery.generation + 2);
1593+
assert.deepEqual(cohort?.delivery, successor);
1594+
assert.deepEqual(
1595+
delivered.messages
1596+
.filter((message) => message.delegatedCompletion?.parentRunId === originRunId)
1597+
.map((message) => [message.id, message.delegatedCompletion]),
1598+
[
1599+
[
1600+
delivery.messageId,
1601+
{
1602+
parentRunId: originRunId,
1603+
generation: delivery.generation,
1604+
taskIds: [incomingTaskId],
1605+
},
1606+
],
1607+
[
1608+
successor!.messageId,
1609+
{ parentRunId: originRunId, generation: successor!.generation, taskIds: [sibling.id] },
1610+
],
1611+
],
1612+
);
1613+
const successorRuns = delivered.runs.filter(
1614+
(run) => run.userMessageId === successor!.messageId,
1615+
);
1616+
assert.equal(successorRuns.length, 1);
1617+
assert.equal(successorRuns[0]?.status, "starting");
1618+
assert.equal(
1619+
delivered.subagents.find((task) => task.id === sibling.id)?.completionDelivery?.state,
1620+
"claimed",
1621+
);
1622+
}),
1623+
);
1624+
});
1625+
13731626
it.layer(Layer.merge(layerTest, ProviderContinuationRequests.layer))(
13741627
"subagent follow-ups",
13751628
(it) => {

‎apps/server/src/orchestration-v2/Orchestrator.ts‎

Lines changed: 34 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -2700,6 +2700,9 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
27002700
readonly disposition: "stopped" | "disposed";
27012701
readonly now: DateTime.Utc;
27022702
readonly cancelQueuedDelivery?: boolean;
2703+
// An interrupted wake disposes only the results it carried. The cohort stays open, and its
2704+
// delivery stays the wake's, so the wake's terminal reserves any sibling's result.
2705+
readonly interruptedWakeTaskIds?: ReadonlyArray<NodeId>;
27032706
}) {
27042707
const parentRun = input.projection.runs.find((candidate) => candidate.id === input.parentRunId);
27052708
if (parentRun === undefined) {
@@ -2713,12 +2716,28 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
27132716
return;
27142717
}
27152718
const emitEvent = emit(input.events, input.command);
2716-
const nextDisposition = cohort?.disposition === "disposed" ? "disposed" : input.disposition;
2717-
const nextCohort = {
2718-
disposition: nextDisposition,
2719-
nextGeneration: cohort?.nextGeneration ?? 1,
2720-
delivery: null,
2721-
} as const;
2719+
const wakeTaskIds = input.interruptedWakeTaskIds;
2720+
const keptTaskIds = new Set(
2721+
wakeTaskIds !== undefined && cohort?.disposition === "open"
2722+
? tasks
2723+
.filter(
2724+
(task) =>
2725+
!wakeTaskIds.includes(task.id) &&
2726+
task.completionDelivery?.state !== "acknowledged" &&
2727+
task.completionDelivery?.state !== "delivered" &&
2728+
task.completionDelivery?.state !== "disposed",
2729+
)
2730+
.map((task) => task.id)
2731+
: [],
2732+
);
2733+
const nextCohort =
2734+
keptTaskIds.size > 0
2735+
? cohort!
2736+
: ({
2737+
disposition: cohort?.disposition === "disposed" ? "disposed" : input.disposition,
2738+
nextGeneration: cohort?.nextGeneration ?? 1,
2739+
delivery: null,
2740+
} as const);
27222741
yield* emitEvent({
27232742
type: "run.updated",
27242743
threadId: parentRun.threadId,
@@ -2733,6 +2752,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
27332752
});
27342753
for (const task of tasks) {
27352754
if (
2755+
keptTaskIds.has(task.id) ||
27362756
task.completionDelivery?.state === "acknowledged" ||
27372757
task.completionDelivery?.state === "delivered" ||
27382758
task.completionDelivery?.state === "disposed"
@@ -2758,7 +2778,11 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
27582778
});
27592779
}
27602780
const deliveryRun = completionDeliveryRun(input.projection, cohort?.delivery ?? null);
2761-
if (input.cancelQueuedDelivery !== false && deliveryRun?.status === "queued") {
2781+
if (
2782+
keptTaskIds.size === 0 &&
2783+
input.cancelQueuedDelivery !== false &&
2784+
deliveryRun?.status === "queued"
2785+
) {
27622786
yield* emitQueuedRunCancellation({
27632787
command: input.command,
27642788
events: input.events,
@@ -11384,6 +11408,9 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
1138411408
parentRunId,
1138511409
disposition: "stopped",
1138611410
now,
11411+
...(parentRunId !== run.id && completionMessage?.delegatedCompletion !== undefined
11412+
? { interruptedWakeTaskIds: completionMessage.delegatedCompletion.taskIds }
11413+
: {}),
1138711414
});
1138811415
}
1138911416
});

0 commit comments

Comments
 (0)