Skip to content

Commit 9b1d351

Browse files
perf(server): busy threads stop decoding settled subagents and idle pollers skip work (#27711)
Co-authored-by: Claude Opus 5.5 <[email protected]>
1 parent bf270e7 commit 9b1d351

19 files changed

Lines changed: 645 additions & 92 deletions

‎apps/server/src/mcp/OrchestratorMcpService.ts‎

Lines changed: 48 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -513,6 +513,21 @@ function pendingTaskApproval(
513513
return approval;
514514
}
515515

516+
/** Every task `directAppOwnedChildTasks` can return, so callers load only those. */
517+
function directChildTaskIds(
518+
target: Pick<OrchestrationV2ThreadProjection, "thread" | "runs">,
519+
returnedRunIds: ReadonlySet<RunId | null>,
520+
): ReadonlyArray<NodeId> {
521+
const ids = new Set<NodeId>();
522+
if (target.thread.forkedFrom?.type === "node") ids.add(target.thread.forkedFrom.nodeId);
523+
for (const run of target.runs) {
524+
if (returnedRunIds.has(run.id) && run.delegatedTaskId !== undefined) {
525+
ids.add(run.delegatedTaskId);
526+
}
527+
}
528+
return [...ids];
529+
}
530+
516531
function directAppOwnedChildTasks(
517532
parent: Pick<OrchestrationV2ThreadProjection, "thread" | "subagents">,
518533
target: Pick<OrchestrationV2ThreadProjection, "thread" | "runs">,
@@ -1016,7 +1031,6 @@ const make = Effect.gen(function* () {
10161031
[
10171032
"runs",
10181033
"messages",
1019-
"subagents",
10201034
"providerThreads",
10211035
"providerTurns",
10221036
"runtimeRequests",
@@ -1034,6 +1048,15 @@ const make = Effect.gen(function* () {
10341048
),
10351049
);
10361050

1051+
// A dispatcher thread holds thousands of tasks, each with its full prompt; read only the ones named.
1052+
const loadTasks = (threadId: ThreadId, taskIds: ReadonlyArray<NodeId>) =>
1053+
threadManagement.getThreadRecords(threadId, ["subagents"], { subagentIds: taskIds }).pipe(
1054+
Effect.map(({ subagents }) => subagents),
1055+
Effect.mapError((error) =>
1056+
failure("orchestration_error", `Unable to read thread ${threadId}: ${errorMessage(error)}`),
1057+
),
1058+
);
1059+
10371060
// Task waits read on every wake of a parent that may hold thousands of
10381061
// tasks, so they read only the task they watch and what its status needs.
10391062
const loadTaskParent = (threadId: ThreadId, taskId: NodeId) =>
@@ -2385,7 +2408,7 @@ const make = Effect.gen(function* () {
23852408
const childThreadId = current.childThreadId;
23862409
if (childThreadId === null) {
23872410
// The task runs in a linked environment: its thread there is stopped through the link.
2388-
const { subagents } = yield* loadProjection(scope.thread.threadId);
2411+
const subagents = yield* loadTasks(scope.thread.threadId, [input.taskId]);
23892412
const remoteTask = subagents.find(
23902413
(task) =>
23912414
task.id === input.taskId &&
@@ -2428,7 +2451,7 @@ const make = Effect.gen(function* () {
24282451
}
24292452
});
24302453
yield* assertStoppable(childThreadId);
2431-
const parentTask = parentProjection.subagents.find(
2454+
const parentTask = (yield* loadTasks(scope.thread.threadId, [input.taskId])).find(
24322455
(task) => task.id === input.taskId && task.origin === "app_owned",
24332456
);
24342457
const disposeCompletionDelivery =
@@ -2548,9 +2571,11 @@ const make = Effect.gen(function* () {
25482571
runId: parentTask.runId,
25492572
subagentId: parentTask.id,
25502573
});
2551-
const { subagents } = yield* threadManagement.getThreadRecords(scope.thread.threadId, [
2552-
"subagents",
2553-
]);
2574+
const { subagents } = yield* threadManagement.getThreadRecords(
2575+
scope.thread.threadId,
2576+
["subagents"],
2577+
{ subagentIds: [input.taskId] },
2578+
);
25542579
return subagents.some((task) => task.id === input.taskId && task.status === "cancelled");
25552580
}).pipe(Effect.mapError(threadManagementFailure));
25562581
if (isTerminalTaskStatus(current.status)) {
@@ -2847,7 +2872,19 @@ const make = Effect.gen(function* () {
28472872
page.filter((row) => row.sourceThreadId === target.thread.id).map((row) => row.item.runId),
28482873
);
28492874
const tasks =
2850-
parent === undefined ? [] : directAppOwnedChildTasks(parent, target, returnedRunIds);
2875+
parent === undefined
2876+
? []
2877+
: directAppOwnedChildTasks(
2878+
{
2879+
thread: parent.thread,
2880+
subagents: yield* loadTasks(
2881+
parent.thread.id,
2882+
directChildTaskIds(target, returnedRunIds),
2883+
),
2884+
},
2885+
target,
2886+
returnedRunIds,
2887+
);
28512888
if (
28522889
parent !== undefined &&
28532890
scope.thread !== undefined &&
@@ -2978,14 +3015,16 @@ const make = Effect.gen(function* () {
29783015
const ownedTask =
29793016
result.timedOut &&
29803017
status !== "waiting" &&
2981-
parent?.subagents.some(
3018+
parent !== undefined &&
3019+
taskId !== undefined &&
3020+
(yield* loadTasks(parent.thread.id, [taskId])).some(
29823021
(task) =>
29833022
task.id === taskId &&
29843023
task.origin === "app_owned" &&
29853024
task.threadId === parent.thread.id &&
29863025
task.childThreadId === input.threadId &&
29873026
task.completionDelivery?.state !== "disposed",
2988-
) === true;
3027+
);
29893028
// A child waiting on a question or approval needs an answer, not a wake.
29903029
const childRequests = ownedTask
29913030
? (yield* threadManagement

‎apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4944,7 +4944,8 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* (
49444944
});
49454945

49464946
// Claude has no recovered frame; the next assistant message proves recovery.
4947-
const completeProviderRetry = Effect.fn("ClaudeAdapterV2.completeProviderRetry")(function* (
4947+
// Runs on every model output frame and is almost always a no-op; only a pending retry gets a span.
4948+
const completeProviderRetry = Effect.fnUntraced(function* (
49484949
context: ActiveClaudeTurnContext,
49494950
updatedAt: DateTime.Utc,
49504951
) {
@@ -4977,7 +4978,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* (
49774978
startedAt: providerRetry.startedAt,
49784979
updatedAt,
49794980
}),
4980-
});
4981+
}).pipe(Effect.withSpan("ClaudeAdapterV2.completeProviderRetry"));
49814982
});
49824983

49834984
const rememberProviderThread =

‎apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1945,7 +1945,8 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* (
19451945
});
19461946

19471947
// Only new model output proves recovery; local item/completed can arrive during upstream retries.
1948-
const completeProviderRetry = Effect.fn("CodexAdapterV2.completeProviderRetry")(function* (
1948+
// Runs on every model output frame and is almost always a no-op; only a pending retry gets a span.
1949+
const completeProviderRetry = Effect.fnUntraced(function* (
19491950
context: ActiveCodexTurnContext,
19501951
updatedAt: DateTime.Utc,
19511952
) {
@@ -1965,7 +1966,7 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* (
19651966
type: "turn_item.updated",
19661967
driver: CODEX_PROVIDER,
19671968
turnItem: codexRetryTurnItem(context, providerRetry, "completed", updatedAt),
1968-
});
1969+
}).pipe(Effect.withSpan("CodexAdapterV2.completeProviderRetry"));
19691970
});
19701971

19711972
const registerRootTurn = Effect.fn("CodexAdapterV2.registerRootTurn")(function* (input: {

‎apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts‎

Lines changed: 72 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -159,6 +159,11 @@ const seedParentWithTerminalTask = (input: {
159159
readonly deliveryState: "delivered" | "claimed" | "acknowledged" | "disposed";
160160
readonly completionWake?: "always" | "settled_only";
161161
readonly deliveryTaskIds?: ReadonlyArray<NodeId>;
162+
readonly parentRunStatus?: "running" | "completed";
163+
readonly siblingTasks?: ReadonlyArray<{
164+
readonly id: NodeId;
165+
readonly deliveryState: "delivered" | "claimed";
166+
}>;
162167
readonly now: DateTime.Utc;
163168
}) =>
164169
Effect.gen(function* () {
@@ -241,10 +246,10 @@ const seedParentWithTerminalTask = (input: {
241246
userMessageId: MessageId.make(`message:seed-user:${input.threadId}`),
242247
rootNodeId: input.rootNodeId,
243248
activeAttemptId: null,
244-
status: "running",
249+
status: input.parentRunStatus ?? "running",
245250
requestedAt: input.now,
246251
startedAt: input.now,
247-
completedAt: null,
252+
completedAt: input.parentRunStatus === "completed" ? input.now : null,
248253
checkpointId: null,
249254
contextHandoffId: null,
250255
delegatedCompletion: {
@@ -261,22 +266,25 @@ const seedParentWithTerminalTask = (input: {
261266
},
262267
},
263268
},
264-
{
265-
id: EventId.make(`event:seed-task:${input.threadId}`),
266-
type: "subagent.updated",
269+
...[
270+
{ id: input.taskId, deliveryState: input.deliveryState },
271+
...(input.siblingTasks ?? []),
272+
].map((task) => ({
273+
id: EventId.make(`event:seed-task:${input.threadId}:${task.id}`),
274+
type: "subagent.updated" as const,
267275
threadId: input.threadId,
268276
runId: input.runId,
269-
nodeId: input.taskId,
277+
nodeId: task.id,
270278
driver,
271279
providerInstanceId: modelSelection.instanceId,
272280
occurredAt: input.now,
273281
payload: {
274-
id: input.taskId,
282+
id: task.id,
275283
threadId: input.threadId,
276284
runId: input.runId,
277285
parentNodeId: input.rootNodeId,
278-
origin: "app_owned",
279-
createdBy: "agent",
286+
origin: "app_owned" as const,
287+
createdBy: "agent" as const,
280288
driver,
281289
providerInstanceId: modelSelection.instanceId,
282290
providerThreadId: null,
@@ -287,16 +295,16 @@ const seedParentWithTerminalTask = (input: {
287295
model: null,
288296
completionWake: input.completionWake ?? "settled_only",
289297
completionDelivery: {
290-
state: input.deliveryState,
291-
observedByRunId: input.deliveryState === "acknowledged" ? input.runId : null,
298+
state: task.deliveryState,
299+
observedByRunId: task.deliveryState === "acknowledged" ? input.runId : null,
292300
},
293-
status: "completed",
301+
status: "completed" as const,
294302
result: "child finished",
295303
startedAt: input.now,
296304
completedAt: input.now,
297305
updatedAt: input.now,
298306
},
299-
},
307+
})),
300308
],
301309
});
302310
});
@@ -683,6 +691,57 @@ it.layer(layerTest)("delegated completion delivery repairs", (it) => {
683691
}),
684692
);
685693

694+
it.effect("counts already delivered siblings in the completion notice", () =>
695+
Effect.gen(function* () {
696+
const orchestrator = yield* Orchestrator.OrchestratorV2;
697+
const now = yield* DateTime.now;
698+
const threadId = ThreadId.make("thread:delegated-delivery-notice-count");
699+
const runId = RunId.make("run:delegated-delivery-notice-count");
700+
const firstTaskId = NodeId.make("node:delegated-delivery-notice-count-first");
701+
const secondTaskId = NodeId.make("node:delegated-delivery-notice-count-second");
702+
703+
yield* seedParentWithTerminalTask({
704+
threadId,
705+
projectId: ProjectId.make("project:delegated-delivery-notice-count"),
706+
runId,
707+
rootNodeId: NodeId.make("node:delegated-delivery-notice-count-root"),
708+
taskId: firstTaskId,
709+
deliveryState: "claimed",
710+
completionWake: "always",
711+
deliveryTaskIds: [firstTaskId, secondTaskId],
712+
parentRunStatus: "completed",
713+
siblingTasks: [
714+
{ id: secondTaskId, deliveryState: "claimed" },
715+
{
716+
id: NodeId.make("node:delegated-delivery-notice-count-earlier"),
717+
deliveryState: "delivered",
718+
},
719+
],
720+
now,
721+
});
722+
723+
yield* orchestrator.dispatch({
724+
type: "message.dispatch",
725+
commandId: CommandId.make("command:delegated-delivery-notice-count"),
726+
threadId,
727+
messageId: MessageId.make(`message:delegated-delivery:${threadId}`),
728+
text: "Delegated tasks reached a terminal state.",
729+
attachments: [],
730+
dispatchMode: { type: "queue_after_active" },
731+
createdBy: "agent",
732+
creationSource: "server",
733+
delegatedCompletion: { parentRunId: runId, generation: 1, taskIds: [firstTaskId] },
734+
});
735+
736+
const projection = yield* orchestrator.getThreadProjection(threadId);
737+
const notice = projection.turnItems.find((item) => item.type === "notification");
738+
assert.match(
739+
notice?.type === "notification" ? notice.summary : "",
740+
/^2 of 3 delegated tasks finished:/,
741+
);
742+
}),
743+
);
744+
686745
it.effect.each([
687746
{
688747
name: "oversized UTF-8 result",

0 commit comments

Comments
 (0)