Skip to content

Commit 2a78d51

Browse files
committed
fix(server): finalize each delegated child result by run
1 parent 0f412d5 commit 2a78d51

5 files changed

Lines changed: 228 additions & 101 deletions

File tree

‎apps/server/src/mcp/OrchestratorMcpToolkit.integration.test.ts‎

Lines changed: 103 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@ import * as DateTime from "effect/DateTime";
4141
import * as Deferred from "effect/Deferred";
4242
import * as Effect from "effect/Effect";
4343
import * as FileSystem from "effect/FileSystem";
44+
import * as Fiber from "effect/Fiber";
4445
import * as Layer from "effect/Layer";
4546
import * as Option from "effect/Option";
4647
import * as PubSub from "effect/PubSub";
@@ -253,6 +254,7 @@ function makeDeterministicAdapter(input: {
253254
},
254255
]);
255256
const eventTime = yield* DateTime.now;
257+
// This fixture assigns one native turn per app run.
256258
const providerTurnId = ProviderTurnId.make(
257259
`provider-turn:${input.instanceId}:${turnInput.threadId}:${turnInput.runOrdinal}`,
258260
);
@@ -272,7 +274,7 @@ function makeDeterministicAdapter(input: {
272274
nativeId: `native-turn:${turnInput.threadId}:${turnInput.runOrdinal}`,
273275
strength: "strong",
274276
},
275-
ordinal: turnInput.providerTurnOrdinal,
277+
ordinal: turnInput.runOrdinal,
276278
status: "running",
277279
startedAt: eventTime,
278280
completedAt: null,
@@ -300,7 +302,7 @@ function makeDeterministicAdapter(input: {
300302
nativeId: `native-turn:${turnInput.threadId}:${turnInput.runOrdinal}`,
301303
strength: "strong",
302304
},
303-
ordinal: turnInput.providerTurnOrdinal,
305+
ordinal: turnInput.runOrdinal,
304306
status: "completed",
305307
startedAt: eventTime,
306308
completedAt: eventTime,
@@ -366,7 +368,7 @@ function makeDeterministicAdapter(input: {
366368
nativeId: `native-turn:${turnInput.threadId}:${turnInput.runOrdinal}`,
367369
strength: "strong",
368370
},
369-
ordinal: turnInput.providerTurnOrdinal,
371+
ordinal: turnInput.runOrdinal,
370372
status: "interrupted",
371373
startedAt: completedAt,
372374
completedAt,
@@ -3898,9 +3900,13 @@ describe("orchestrator MCP toolkit", () => {
38983900
}),
38993901
),
39003902
);
3901-
it.live.each(["always", "settled_only"] as const)(
3902-
"wakes the parent for completed child follow-ups after an original %s delegation",
3903-
(completionWake) =>
3903+
it.live.each([
3904+
{ completionWake: "always", delayedDelivery: false },
3905+
{ completionWake: "settled_only", delayedDelivery: false },
3906+
{ completionWake: "always", delayedDelivery: true },
3907+
] as const)(
3908+
"wakes the parent after an original $completionWake delegation with delayed delivery $delayedDelivery",
3909+
({ completionWake, delayedDelivery }) =>
39043910
Effect.scoped(
39053911
Effect.gen(function* () {
39063912
const cwd = yield* checkpointWorkspace("parent-wake-followup-comparison");
@@ -3909,35 +3915,53 @@ describe("orchestrator MCP toolkit", () => {
39093915
const initialGate = yield* Deferred.make<void>();
39103916
const controlGate = yield* Deferred.make<void>();
39113917
const foregroundGate = yield* Deferred.make<void>();
3918+
const selectionEntered = yield* Deferred.make<void>();
3919+
const selectionGate = yield* Deferred.make<void>();
39123920
const foregroundPrompt = "Hold the parent while its child follow-up completes.";
39133921
const initialPrompt = "Original delegated task for follow-up comparison.";
39143922
const controlPrompt = "Control delegated task without follow-ups.";
39153923
const followupPrompt = "Follow-up after the original delegated task completed.";
39163924
const registryLayer = ProviderAdapterRegistry.makeLayer([
3917-
makeDeterministicAdapter({
3918-
instanceId: codexInstanceId,
3919-
driver: ProviderDriverKind.make("codex"),
3920-
capabilities: {
3921-
...CodexProviderCapabilitiesV2,
3922-
turns: { ...CodexProviderCapabilitiesV2.turns, supportsActiveSteering: false },
3923-
},
3924-
capturedTurns,
3925-
shouldComplete: () => true,
3926-
terminalGate: (turn) =>
3927-
turn.message.text === parentPrompt
3928-
? parentGate
3929-
: turn.message.text === initialPrompt
3930-
? initialGate
3931-
: turn.message.text === controlPrompt
3932-
? controlGate
3933-
: turn.message.text === foregroundPrompt
3934-
? foregroundGate
3935-
: undefined,
3936-
response: (turn) => `Result: ${turn.message.text}`,
3937-
}),
3925+
{
3926+
...makeDeterministicAdapter({
3927+
instanceId: codexInstanceId,
3928+
driver: ProviderDriverKind.make("codex"),
3929+
capabilities: {
3930+
...CodexProviderCapabilitiesV2,
3931+
turns: {
3932+
...CodexProviderCapabilitiesV2.turns,
3933+
supportsActiveSteering: false,
3934+
},
3935+
},
3936+
capturedTurns,
3937+
shouldComplete: () => true,
3938+
terminalGate: (turn) =>
3939+
turn.message.text === parentPrompt
3940+
? parentGate
3941+
: turn.message.text === initialPrompt
3942+
? initialGate
3943+
: turn.message.text === controlPrompt
3944+
? controlGate
3945+
: turn.message.text === foregroundPrompt
3946+
? foregroundGate
3947+
: undefined,
3948+
response: (turn) => `Result: ${turn.message.text}`,
3949+
}),
3950+
planSelectionTransition: ({ target }) =>
3951+
Effect.gen(function* () {
3952+
if (target.model === "gpt-5.4-next") {
3953+
yield* Deferred.succeed(selectionEntered, undefined);
3954+
yield* Deferred.await(selectionGate);
3955+
}
3956+
return { type: "apply_on_next_turn" as const };
3957+
}),
3958+
},
39383959
]);
39393960
const layer = makeOrchestratorV2ReplayLayerWithRegistry(
3940-
{ name: "parent-wake-followup-comparison", runtimePolicyOverride: { cwd } },
3961+
{
3962+
name: "parent-wake-followup-comparison",
3963+
runtimePolicyOverride: { cwd },
3964+
},
39413965
registryLayer,
39423966
{ runContinuationWorker: true },
39433967
);
@@ -4069,27 +4093,52 @@ describe("orchestrator MCP toolkit", () => {
40694093
(event) =>
40704094
event.type === "provider-turn.updated" && event.payload.status === "running",
40714095
);
4072-
const childSequence = yield* orchestrator.getThreadEventSequence(worker.childThreadId!);
4073-
yield* orchestrator.dispatch({
4074-
type: "message.dispatch",
4075-
createdBy: "agent",
4076-
creationSource: "mcp",
4077-
commandId: CommandId.make("wake-comparison:followup"),
4078-
threadId: worker.childThreadId!,
4079-
messageId: MessageId.make("wake-comparison:followup"),
4080-
text: followupPrompt,
4081-
attachments: [],
4082-
modelSelection: codexSelection,
4083-
dispatchMode: { type: "start_immediately" },
4084-
});
4085-
const followupTerminal = yield* awaitEvent(
4086-
worker.childThreadId!,
4087-
childSequence,
4088-
(event) => event.type === "run.updated" && event.payload.status === "completed",
4089-
);
4090-
if (followupTerminal.event.type !== "run.updated")
4091-
throw new Error("Missing follow-up run.");
4092-
const followupRunId = followupTerminal.event.payload.id;
4096+
// A slow provider selection holds the actual parent dispatch lock.
4097+
// Both child runs finish before their result handlers acquire it.
4098+
const selectionFiber = delayedDelivery
4099+
? yield* orchestrator
4100+
.dispatch({
4101+
type: "thread.model-selection.set",
4102+
commandId: CommandId.make("wake-comparison:slow-selection"),
4103+
threadId: parentThreadId,
4104+
modelSelection: {
4105+
...codexSelection,
4106+
model: "gpt-5.4-next",
4107+
},
4108+
})
4109+
.pipe(Effect.forkChild)
4110+
: undefined;
4111+
if (delayedDelivery) yield* Deferred.await(selectionEntered);
4112+
const followupRunIds: RunId[] = [];
4113+
for (let index = 0; index < (delayedDelivery ? 2 : 1); index++) {
4114+
const childSequence = yield* orchestrator.getThreadEventSequence(
4115+
worker.childThreadId!,
4116+
);
4117+
yield* orchestrator.dispatch({
4118+
type: "message.dispatch",
4119+
createdBy: "agent",
4120+
creationSource: "mcp",
4121+
commandId: CommandId.make(`wake-comparison:followup:${index}`),
4122+
threadId: worker.childThreadId!,
4123+
messageId: MessageId.make(`wake-comparison:followup:${index}`),
4124+
text: `${followupPrompt} ${index}`,
4125+
attachments: [],
4126+
modelSelection: codexSelection,
4127+
dispatchMode: { type: "start_immediately" },
4128+
});
4129+
const followupTerminal = yield* awaitEvent(
4130+
worker.childThreadId!,
4131+
childSequence,
4132+
(event) => event.type === "run.updated" && event.payload.status === "completed",
4133+
);
4134+
if (followupTerminal.event.type !== "run.updated")
4135+
throw new Error("Missing follow-up run.");
4136+
followupRunIds.push(followupTerminal.event.payload.id);
4137+
}
4138+
if (selectionFiber !== undefined) {
4139+
yield* Deferred.succeed(selectionGate, undefined);
4140+
yield* Fiber.join(selectionFiber);
4141+
}
40934142
// This later independent completion is a barrier on the sequential
40944143
// terminal reactor and continuation worker, so absence needs no sleep.
40954144
const controlSequence = yield* orchestrator.getThreadEventSequence(parentThreadId);
@@ -4141,7 +4190,7 @@ describe("orchestrator MCP toolkit", () => {
41414190
(transfer) =>
41424191
transfer.type === "subagent_result" &&
41434192
transfer.sourceThreadId === worker.childThreadId &&
4144-
transfer.sourcePoint.runId === followupRunId,
4193+
followupRunIds.includes(transfer.sourcePoint.runId!),
41454194
);
41464195
const controlTransfers = finalParent.contextTransfers.filter(
41474196
(transfer) =>
@@ -4154,7 +4203,10 @@ describe("orchestrator MCP toolkit", () => {
41544203
expect(
41554204
followupTransfers,
41564205
"A completed follow-up must deliver its result to the parent",
4157-
).toHaveLength(1);
4206+
).toHaveLength(followupRunIds.length);
4207+
expect(
4208+
new Set(followupTransfers.map((transfer) => transfer.sourcePoint.runId)).size,
4209+
).toBe(followupRunIds.length);
41584210
expect(workerWakes).toHaveLength(2);
41594211
const originalTask = initialParent.subagents.find((task) => task.id === worker.id)!;
41604212
const finalTask = finalParent.subagents.find((task) => task.id === worker.id)!;

‎apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts‎

Lines changed: 22 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -323,43 +323,45 @@ it.layer(TestLayer)("delegated completion delivery repairs", (it) => {
323323
occurredAt: now,
324324
payload: { ...task, childThreadId: childId },
325325
},
326-
{
327-
id: EventId.make("event:follow-up-ack-transfer"),
328-
type: "context-transfer.created",
326+
...[RunId.make("run:original-child"), resultRunId].map((childRunId) => ({
327+
id: EventId.make(`event:follow-up-ack-transfer:${childRunId}`),
328+
type: "context-transfer.created" as const,
329329
threadId,
330330
occurredAt: now,
331331
payload: {
332-
id: ContextTransferId.make("transfer:follow-up-ack"),
333-
type: "subagent_result",
332+
id: ContextTransferId.make(`transfer:follow-up-ack:${childRunId}`),
333+
type: "subagent_result" as const,
334334
sourceThreadId: childId,
335335
targetThreadId: threadId,
336-
sourcePoint: { threadId: childId, runId: resultRunId },
336+
sourcePoint: { threadId: childId, runId: childRunId },
337337
basePoint: null,
338338
sourceProviderInstanceId: modelSelection.instanceId,
339339
targetProviderInstanceId: modelSelection.instanceId,
340340
targetRunId: runId,
341-
status: "consumed",
341+
status: "consumed" as const,
342342
resolution: null,
343-
createdBy: "system",
343+
createdBy: "system" as const,
344344
error: null,
345345
createdAt: now,
346346
updatedAt: now,
347347
consumedAt: now,
348348
},
349-
},
349+
})),
350350
],
351351
});
352-
yield* orchestrator.dispatch({
353-
type: "delegated_task.completion-delivery.acknowledge",
354-
commandId: CommandId.make("command:follow-up-ack-stale"),
355-
parentThreadId: threadId,
356-
taskId,
357-
resultRunId: RunId.make("run:original-child"),
358-
observedByRunId: runId,
359-
});
360-
const unchanged = yield* orchestrator.getThreadProjection(threadId);
361-
assert.equal(unchanged.subagents[0]?.completionDelivery?.state, "claimed");
362-
assert.deepEqual(unchanged.runs[0]?.delegatedCompletion?.delivery?.taskIds, [taskId]);
352+
for (const staleResultRunId of [RunId.make("run:original-child"), undefined]) {
353+
yield* orchestrator.dispatch({
354+
type: "delegated_task.completion-delivery.acknowledge",
355+
commandId: CommandId.make(`command:follow-up-ack-stale:${staleResultRunId ?? "legacy"}`),
356+
parentThreadId: threadId,
357+
taskId,
358+
...(staleResultRunId === undefined ? {} : { resultRunId: staleResultRunId }),
359+
observedByRunId: runId,
360+
});
361+
const unchanged = yield* orchestrator.getThreadProjection(threadId);
362+
assert.equal(unchanged.subagents[0]?.completionDelivery?.state, "claimed");
363+
assert.deepEqual(unchanged.runs[0]?.delegatedCompletion?.delivery?.taskIds, [taskId]);
364+
}
363365
yield* orchestrator.dispatch({
364366
type: "delegated_task.completion-delivery.acknowledge",
365367
commandId: CommandId.make("command:follow-up-ack-current"),

‎apps/server/src/orchestration-v2/Orchestrator.ts‎

Lines changed: 49 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1792,17 +1792,21 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
17921792
// valid observations can race after their read preflight. Re-emit the
17931793
// existing task row so the second dispatch is a successful idempotent
17941794
// no-op rather than "already acknowledged/disposed" or empty-events.
1795-
const latestResultTransfer = projection.contextTransfers.findLast(
1795+
const resultTransfers = projection.contextTransfers.filter(
17961796
(transfer) =>
17971797
transfer.type === "subagent_result" &&
17981798
transfer.sourceThreadId === task.childThreadId &&
17991799
transfer.targetThreadId === command.parentThreadId,
18001800
);
1801+
const latestResultTransfer = resultTransfers.at(-1);
18011802
if (
18021803
(command.type === "delegated_task.completion-delivery.acknowledge" &&
1803-
command.resultRunId !== undefined &&
1804-
latestResultTransfer?.sourcePoint.runId !== undefined &&
1805-
latestResultTransfer.sourcePoint.runId !== command.resultRunId) ||
1804+
(command.resultRunId === undefined
1805+
? // Older callers can acknowledge the original result, but cannot
1806+
// identify which follow-up they read.
1807+
resultTransfers.length > 1
1808+
: latestResultTransfer?.sourcePoint.runId !== undefined &&
1809+
latestResultTransfer.sourcePoint.runId !== command.resultRunId)) ||
18061810
task.completionDelivery?.state === state ||
18071811
(command.type === "delegated_task.completion-delivery.acknowledge" &&
18081812
task.completionDelivery?.state === "disposed")
@@ -8487,7 +8491,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
84878491
* other (stale policy on the terminal row, or a terminal row regressed to
84888492
* running).
84898493
*/
8490-
const finalizeAppOwnedSubagent = (childThreadId: ThreadId) =>
8494+
const finalizeAppOwnedSubagent = (childThreadId: ThreadId, terminalRunId: RunId) =>
84918495
Effect.gen(function* () {
84928496
const childControls = yield* projectionStore.getThreadRecords(
84938497
childThreadId,
@@ -8502,7 +8506,11 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
85028506
) {
85038507
return;
85048508
}
8505-
const progress = delegatedTaskProgress(childControls);
8509+
const progress = delegatedTaskProgress({
8510+
...childControls,
8511+
// The parent lock can defer this event until a later child run finishes.
8512+
runs: childControls.runs.filter((run) => run.id === terminalRunId),
8513+
});
85068514
if (progress.state !== "result_available") return;
85078515
const childRun = progress.resultRun;
85088516
if (childRun === undefined) return;
@@ -9503,8 +9511,11 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
95039511
// child lock here would invert that order, and the keyed executor's
95049512
// semaphores are neither reentrant nor deadlock-aware.
95059513
const parentThreadId = yield* appOwnedSubagentParentThreadId(threadId);
9506-
if (parentThreadId !== undefined) {
9507-
yield* threadDispatch.withLock(parentThreadId, finalizeAppOwnedSubagent(threadId));
9514+
if (parentThreadId !== undefined && stored.event.type === "run.updated") {
9515+
yield* threadDispatch.withLock(
9516+
parentThreadId,
9517+
finalizeAppOwnedSubagent(threadId, stored.event.payload.id),
9518+
);
95089519
}
95099520
if (stored.event.type === "run.updated") {
95109521
yield* threadDispatch.withLock(
@@ -9565,7 +9576,36 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
95659576
const thread = yield* projectionStore.getThreadShell(threadId);
95669577
const parentThreadId = thread?.lineage.parentThreadId;
95679578
if (parentThreadId === undefined || parentThreadId === null) return;
9568-
yield* threadDispatch.withLock(parentThreadId, finalizeAppOwnedSubagent(threadId));
9579+
yield* threadDispatch.withLock(
9580+
parentThreadId,
9581+
Effect.gen(function* () {
9582+
const child = yield* projectionStore.getThreadRecords(threadId, ["runs"]);
9583+
const parent = yield* projectionStore.getThreadRecords(parentThreadId, [
9584+
"contextTransfers",
9585+
]);
9586+
const deliveredRunIds = new Set(
9587+
parent.contextTransfers
9588+
.filter(
9589+
(transfer) =>
9590+
transfer.type === "subagent_result" &&
9591+
transfer.sourceThreadId === threadId &&
9592+
transfer.targetThreadId === parentThreadId,
9593+
)
9594+
.map((transfer) => transfer.sourcePoint.runId),
9595+
);
9596+
for (const run of child.runs.toSorted(
9597+
(left, right) => left.ordinal - right.ordinal,
9598+
)) {
9599+
if (
9600+
["completed", "interrupted", "failed", "cancelled"].includes(run.status) &&
9601+
!deliveredRunIds.has(run.id) &&
9602+
!(run.ordinal === 1 && deliveredRunIds.has(undefined))
9603+
) {
9604+
yield* finalizeAppOwnedSubagent(threadId, run.id);
9605+
}
9606+
}
9607+
}),
9608+
);
95699609
}).pipe(
95709610
Effect.catchCause((cause) =>
95719611
Effect.logWarning("Failed to recover terminal app-owned subagent", {

0 commit comments

Comments
 (0)