Skip to content

Commit 24e7747

Browse files
fix(server): a follow-up delegated task waits for a run its child queued on its own thread (pingdotgg#32504)
1 parent 1bdf470 commit 24e7747

4 files changed

Lines changed: 310 additions & 8 deletions

File tree

‎apps/server/src/mcp/OrchestratorMcpService.ts‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -100,7 +100,7 @@ import type {
100100
ProjectionRecordField,
101101
ProjectionRecordFilter,
102102
} from "../orchestration-v2/ProjectionStore.ts";
103-
import { runOutcome } from "../orchestration-v2/RestartBackgroundNote.ts";
103+
import { delegatedTaskResultRun, runOutcome } from "../orchestration-v2/RestartBackgroundNote.ts";
104104
import * as ThreadManagementService from "../orchestration-v2/ThreadManagementService.ts";
105105
import * as ThreadLaunchService from "../orchestration-v2/ThreadLaunchService.ts";
106106
import * as GitVcsDriver from "../vcs/GitVcsDriver.ts";
@@ -1489,9 +1489,7 @@ const make = Effect.gen(function* () {
14891489
const progress = delegatedTaskProgress(childControls);
14901490
const taskResultRun =
14911491
childRun?.delegatedTaskId === task.id
1492-
? childControls.runs
1493-
.filter((run) => run.delegatedTaskId === task.id)
1494-
.toSorted((a, b) => b.ordinal - a.ordinal)[0]
1492+
? delegatedTaskResultRun(childControls.runs, task.id)
14951493
: progress.resultRun;
14961494
const taskWorkState =
14971495
childRun?.delegatedTaskId === task.id

‎apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts‎

Lines changed: 281 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2498,6 +2498,287 @@ it.layer(Layer.merge(layerTest, ProviderContinuationRequests.layer))(
24982498
}),
24992499
);
25002500

2501+
// The child's follow-up run sends `t3_thread_send` to its own thread before it ends.
2502+
const startSelfQueuedFollowup = (name: string) =>
2503+
Effect.gen(function* () {
2504+
const orchestrator = yield* Orchestrator.OrchestratorV2;
2505+
const eventSink = yield* EventSink.EventSinkV2;
2506+
const requests = yield* ProviderContinuationRequests.ProviderContinuationRequests;
2507+
const now = yield* DateTime.now;
2508+
const parentThreadId = ThreadId.make(`thread:${name}-parent`);
2509+
const projectId = ProjectId.make(`project:${name}`);
2510+
const parentRunId = RunId.make(`run:${name}-parent`);
2511+
const rootNodeId = NodeId.make(`node:${name}-parent`);
2512+
yield* seedParentWithTerminalTask({
2513+
threadId: parentThreadId,
2514+
projectId,
2515+
runId: parentRunId,
2516+
rootNodeId,
2517+
taskId: NodeId.make(`node:${name}-seed`),
2518+
deliveryState: "acknowledged",
2519+
now,
2520+
});
2521+
const child = yield* seedRestartCancelledChild({
2522+
parentThreadId,
2523+
projectId,
2524+
parentRunId,
2525+
rootNodeId,
2526+
name: `${name}-child`,
2527+
completionWake: "always",
2528+
continuationPending: false,
2529+
runStatus: "completed",
2530+
now,
2531+
});
2532+
yield* orchestrator.recoverDelegatedTasks;
2533+
let firstWake = yield* requests.take;
2534+
while (firstWake.threadId !== parentThreadId) firstWake = yield* requests.take;
2535+
yield* orchestrator.dispatch({
2536+
type: "delegated_task.completion-delivery.acknowledge",
2537+
commandId: CommandId.make(`command:${name}-ack`),
2538+
parentThreadId,
2539+
taskId: child.taskId,
2540+
observedByRunId: parentRunId,
2541+
});
2542+
const parentRun = (yield* orchestrator.getThreadProjection(parentThreadId)).runs.find(
2543+
(run) => run.id === parentRunId,
2544+
)!;
2545+
yield* eventSink.write({
2546+
commandId: reconcileCommandId(`${name}-parent-finished`),
2547+
events: [
2548+
{
2549+
id: EventId.make(`event:${name}-parent-finished`),
2550+
type: "run.updated",
2551+
threadId: parentThreadId,
2552+
runId: parentRunId,
2553+
occurredAt: now,
2554+
payload: { ...parentRun, status: "completed", completedAt: now },
2555+
},
2556+
],
2557+
});
2558+
const followupMessageId = MessageId.make(`message:${name}`);
2559+
yield* orchestrator.dispatch({
2560+
type: "message.dispatch",
2561+
createdBy: "agent",
2562+
creationSource: "mcp",
2563+
commandId: CommandId.make(`command:${name}`),
2564+
threadId: child.childThreadId,
2565+
senderThreadId: parentThreadId,
2566+
messageId: followupMessageId,
2567+
text: "Check the same work again.",
2568+
attachments: [],
2569+
dispatchMode: { type: "start_immediately" },
2570+
});
2571+
const queuedRun = (yield* orchestrator.getThreadProjection(child.childThreadId)).runs.find(
2572+
(row) => row.userMessageId === followupMessageId,
2573+
)!;
2574+
const followupTaskId = queuedRun.delegatedTaskId;
2575+
assert.isDefined(followupTaskId);
2576+
const followupRun = { ...queuedRun, status: "running" as const, startedAt: now };
2577+
yield* eventSink.write({
2578+
commandId: reconcileCommandId(`${name}-running`),
2579+
events: [
2580+
{
2581+
id: EventId.make(`event:${name}-running`),
2582+
type: "run.updated",
2583+
threadId: child.childThreadId,
2584+
runId: followupRun.id,
2585+
occurredAt: now,
2586+
payload: followupRun,
2587+
},
2588+
],
2589+
});
2590+
const selfQueuedMessageId = MessageId.make(`message:${name}-self-send`);
2591+
yield* orchestrator.dispatch({
2592+
type: "message.dispatch",
2593+
createdBy: "agent",
2594+
creationSource: "mcp",
2595+
commandId: CommandId.make(`command:${name}-self-send`),
2596+
threadId: child.childThreadId,
2597+
senderThreadId: child.childThreadId,
2598+
messageId: selfQueuedMessageId,
2599+
text: "Finish the work after this turn.",
2600+
attachments: [],
2601+
dispatchMode: { type: "queue_after_active" },
2602+
});
2603+
const selfQueuedRun = (yield* orchestrator.getThreadProjection(
2604+
child.childThreadId,
2605+
)).runs.find((row) => row.userMessageId === selfQueuedMessageId)!;
2606+
assert.equal(selfQueuedRun.status, "queued");
2607+
const finishRun = (step: string, run: OrchestrationV2Run, text: string) =>
2608+
eventSink.write({
2609+
commandId: reconcileCommandId(`${name}-${step}`),
2610+
events: [
2611+
{
2612+
id: EventId.make(`event:${name}-${step}:result`),
2613+
type: "message.updated",
2614+
threadId: child.childThreadId,
2615+
runId: run.id,
2616+
occurredAt: now,
2617+
payload: {
2618+
id: MessageId.make(`message:${name}-${step}:result`),
2619+
threadId: child.childThreadId,
2620+
runId: run.id,
2621+
nodeId: null,
2622+
role: "assistant",
2623+
text,
2624+
attachments: [],
2625+
streaming: false,
2626+
createdBy: "agent",
2627+
creationSource: "server",
2628+
createdAt: now,
2629+
updatedAt: now,
2630+
},
2631+
},
2632+
{
2633+
id: EventId.make(`event:${name}-${step}:finished`),
2634+
type: "run.updated",
2635+
threadId: child.childThreadId,
2636+
runId: run.id,
2637+
occurredAt: now,
2638+
payload: { ...run, status: "completed", startedAt: now, completedAt: now },
2639+
},
2640+
],
2641+
});
2642+
const removeSelfQueuedRun = orchestrator.dispatch({
2643+
type: "queued-run.cancel",
2644+
commandId: CommandId.make(`command:${name}-remove-self-send`),
2645+
threadId: child.childThreadId,
2646+
runId: selfQueuedRun.id,
2647+
});
2648+
const readFollowup = Effect.gen(function* () {
2649+
yield* orchestrator.recoverDelegatedTasks;
2650+
return (yield* orchestrator.getThreadProjection(parentThreadId)).subagents.find(
2651+
(task) => task.id === followupTaskId,
2652+
)!;
2653+
});
2654+
return {
2655+
parentThreadId,
2656+
parentRunId,
2657+
childThreadId: child.childThreadId,
2658+
firstWake,
2659+
followupRun,
2660+
selfQueuedRun,
2661+
finishRun,
2662+
removeSelfQueuedRun,
2663+
readFollowup,
2664+
};
2665+
});
2666+
2667+
it.effect("waits for a run the follow-up's child queued on its own thread", () =>
2668+
Effect.gen(function* () {
2669+
const requests = yield* ProviderContinuationRequests.ProviderContinuationRequests;
2670+
const followup = yield* startSelfQueuedFollowup("self-queued-followup");
2671+
yield* followup.finishRun("handoff", followup.followupRun, "HANDOFF-C");
2672+
const waiting = yield* followup.readFollowup;
2673+
assert.isNull(
2674+
waiting.result,
2675+
"the follow-up task settled on its handoff turn while the run its child queued was pending",
2676+
);
2677+
assert.notEqual(waiting.status, "completed");
2678+
2679+
yield* followup.finishRun("final", followup.selfQueuedRun, "FINAL-D");
2680+
const finished = yield* followup.readFollowup;
2681+
assert.equal(finished.status, "completed");
2682+
assert.equal(finished.result, "FINAL-D");
2683+
assert.equal(finished.completionDelivery?.state, "claimed");
2684+
let wake = yield* requests.take;
2685+
while (
2686+
wake.threadId !== followup.parentThreadId ||
2687+
wake.delegatedCompletion?.messageId === followup.firstWake.delegatedCompletion?.messageId
2688+
) {
2689+
wake = yield* requests.take;
2690+
}
2691+
assert.equal(wake.delegatedCompletion?.parentRunId, followup.parentRunId);
2692+
}),
2693+
);
2694+
2695+
it.effect("keeps a follow-up running when the run its child queued is removed early", () =>
2696+
Effect.gen(function* () {
2697+
const followup = yield* startSelfQueuedFollowup("self-queued-removed-early-followup");
2698+
yield* followup.removeSelfQueuedRun;
2699+
const running = yield* followup.readFollowup;
2700+
assert.isNull(running.result);
2701+
assert.equal(running.status, "running");
2702+
2703+
yield* followup.finishRun("handoff", followup.followupRun, "HANDOFF-C");
2704+
const finished = yield* followup.readFollowup;
2705+
assert.equal(finished.status, "completed");
2706+
assert.equal(finished.result, "HANDOFF-C");
2707+
}),
2708+
);
2709+
2710+
it.effect.each(["removed", "held"] as const)(
2711+
"settles a follow-up on its last turn when the run it queued is %s",
2712+
(ending) =>
2713+
Effect.gen(function* () {
2714+
const eventSink = yield* EventSink.EventSinkV2;
2715+
const now = yield* DateTime.now;
2716+
const followup = yield* startSelfQueuedFollowup(`self-queued-${ending}-followup`);
2717+
if (ending === "held") {
2718+
yield* eventSink.write({
2719+
commandId: reconcileCommandId("self-queued-held-followup-hold"),
2720+
events: [
2721+
{
2722+
id: EventId.make("event:self-queued-held-followup-hold"),
2723+
type: "run.updated",
2724+
threadId: followup.childThreadId,
2725+
runId: followup.selfQueuedRun.id,
2726+
occurredAt: now,
2727+
payload: { ...followup.selfQueuedRun, queueHeld: true },
2728+
},
2729+
],
2730+
});
2731+
}
2732+
yield* followup.finishRun("handoff", followup.followupRun, "HANDOFF-C");
2733+
if (ending === "removed") yield* followup.removeSelfQueuedRun;
2734+
const finished = yield* followup.readFollowup;
2735+
assert.equal(finished.status, "completed");
2736+
assert.equal(finished.result, "HANDOFF-C");
2737+
}),
2738+
);
2739+
2740+
it.effect("waits for every run its child queued when a later one is moved ahead", () =>
2741+
Effect.gen(function* () {
2742+
const orchestrator = yield* Orchestrator.OrchestratorV2;
2743+
const followup = yield* startSelfQueuedFollowup("self-queued-reordered-followup");
2744+
const laterMessageId = MessageId.make("message:self-queued-reordered-followup-later");
2745+
yield* orchestrator.dispatch({
2746+
type: "message.dispatch",
2747+
createdBy: "agent",
2748+
creationSource: "mcp",
2749+
commandId: CommandId.make("command:self-queued-reordered-followup-later"),
2750+
threadId: followup.childThreadId,
2751+
senderThreadId: followup.childThreadId,
2752+
messageId: laterMessageId,
2753+
text: "Do this part first.",
2754+
attachments: [],
2755+
dispatchMode: { type: "queue_after_active" },
2756+
});
2757+
const later = (yield* orchestrator.getThreadProjection(followup.childThreadId)).runs.find(
2758+
(run) => run.userMessageId === laterMessageId,
2759+
)!;
2760+
yield* orchestrator.dispatch({
2761+
type: "queued-run.reorder",
2762+
commandId: CommandId.make("command:self-queued-reordered-followup-reorder"),
2763+
threadId: followup.childThreadId,
2764+
runId: later.id,
2765+
beforeRunId: followup.selfQueuedRun.id,
2766+
});
2767+
assert.equal((yield* followup.readFollowup).status, "running");
2768+
yield* followup.finishRun("handoff", followup.followupRun, "HANDOFF-C");
2769+
yield* followup.finishRun("later", later, "PART-ONE");
2770+
const waiting = yield* followup.readFollowup;
2771+
assert.isNull(waiting.result, "the follow-up settled while an earlier queued run waited");
2772+
2773+
// Promotion moves a queued run after the runs that started ahead of it.
2774+
const promoted = { ...followup.selfQueuedRun, ordinal: later.ordinal + 1 };
2775+
yield* followup.finishRun("final", promoted, "FINAL-D");
2776+
const finished = yield* followup.readFollowup;
2777+
assert.equal(finished.status, "completed");
2778+
assert.equal(finished.result, "FINAL-D");
2779+
}),
2780+
);
2781+
25012782
const outcomes = [
25022783
"completed",
25032784
"failed",

‎apps/server/src/orchestration-v2/Orchestrator.ts‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -124,6 +124,7 @@ import {
124124
cancelledBeforeStart,
125125
continuationFailedBeforePrompt,
126126
continuationStartRetryDelayMs,
127+
delegatedTaskResultRun,
127128
isRestartContinuationSource,
128129
runOutcome,
129130
} from "./RestartBackgroundNote.ts";
@@ -6035,8 +6036,11 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
60356036
projection = applyToProjection(projection, event);
60366037
}
60376038

6038-
const continuedTaskId = projection.runs.find(
6039-
(run) => run.id === command.restartContinuationOfRunId,
6039+
// A run the thread queues for itself continues the task of the run that sent it.
6040+
const continuedTaskId = projection.runs.find((run) =>
6041+
command.restartContinuationOfRunId === undefined
6042+
? command.senderThreadId === command.threadId && isActiveRun(run)
6043+
: run.id === command.restartContinuationOfRunId,
60406044
)?.delegatedTaskId;
60416045

60426046
if (projection.thread.settledOverride !== null) {
@@ -13029,7 +13033,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
1302913033
? options.task.id === forkedFrom.nodeId
1303013034
? options.resultRun
1303113035
: undefined
13032-
: taskRuns.toSorted((a, b) => b.ordinal - a.ordinal)[0];
13036+
: delegatedTaskResultRun(taskRuns, options.task.id);
1303313037
if (childRun === undefined) return;
1303413038
const outcome = runOutcome(childRun);
1303513039
const terminalStatus = delegatedTaskTerminalStatus(outcome);
@@ -14543,7 +14547,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
1454314547
);
1454414548
if (taskId !== undefined) {
1454514549
const runs = child.runs.filter((run) => run.delegatedTaskId === taskId);
14546-
const run = runs.toSorted((a, b) => b.ordinal - a.ordinal)[0];
14550+
const run = delegatedTaskResultRun(runs, taskId);
1454714551
if (run === undefined || delegatedTaskTerminalStatus(run.status) === null) return true;
1454814552
return yield* childAwaitsRestartContinuation(runs, run);
1454914553
}

‎apps/server/src/orchestration-v2/RestartBackgroundNote.ts‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import type {
2+
NodeId,
23
OrchestrationV2PendingBackgroundTask,
34
OrchestrationV2ProviderTurn,
45
OrchestrationV2RestartCancelledBackgroundWork,
@@ -185,6 +186,24 @@ export function cancelledBeforeStart(
185186
return run.status === "cancelled" && neverStarted(run);
186187
}
187188

189+
/** A follow-up task's latest run, passing over queued work it no longer waits on, as the original task does. */
190+
export function delegatedTaskResultRun(
191+
runs: ReadonlyArray<OrchestrationV2Run>,
192+
taskId: NodeId,
193+
): OrchestrationV2Run | undefined {
194+
const taskRuns = runs
195+
.filter((run) => run.delegatedTaskId === taskId)
196+
.toSorted((a, b) => b.ordinal - a.ordinal);
197+
const awaited = taskRuns.filter(
198+
(run) => !cancelledBeforeStart(run) && !(run.status === "queued" && run.queueHeld === true),
199+
);
200+
// Executing work reports the task's state; a reordered queue can finish a newer run before an older one.
201+
const executing = awaited.find((run) =>
202+
["preparing", "starting", "running", "waiting"].includes(run.status),
203+
);
204+
return executing ?? awaited.find((run) => run.status === "queued") ?? awaited[0] ?? taskRuns[0];
205+
}
206+
188207
/** The provider never accepted this continuation's prompt, so starting it again repeats nothing. */
189208
export function continuationFailedBeforePrompt(
190209
run: Pick<OrchestrationV2Run, "status" | "activeAttemptId" | "restartContinuationOfRunId">,

0 commit comments

Comments
 (0)