Skip to content

Commit ab3a3cd

Browse files
t3dotggclaude
andauthored
perf(server): commands on busy threads stop decoding thousands of finished tool nodes (#19675)
Co-authored-by: Claude Opus 5.5 <[email protected]>
1 parent 742c229 commit ab3a3cd

3 files changed

Lines changed: 147 additions & 4 deletions

File tree

‎apps/server/src/orchestration-v2/Orchestrator.ts‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -940,6 +940,8 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
940940

941941
// Command decisions need control records, not historical assistant/tool output.
942942
// Handoff preparation explicitly reads its history after choosing a strategy.
943+
// This runs inside the write transaction for every command, so finished leaf
944+
// nodes of finished runs stay unread: busy threads hold thousands of them.
943945
const readCommandProjection = (threadId: ThreadId) =>
944946
projectionStore
945947
.getThreadRecords(
@@ -958,7 +960,11 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
958960
"checkpointScopes",
959961
"contextTransfers",
960962
],
961-
{ turnItemTypes: ["user_message", "error"], messageRoles: ["user"] },
963+
{
964+
turnItemTypes: ["user_message", "error"],
965+
messageRoles: ["user"],
966+
omitSettledLeafNodes: true,
967+
},
962968
)
963969
.pipe(
964970
Effect.map((records): OrchestrationV2ThreadProjection => ({

‎apps/server/src/orchestration-v2/ProjectionStore.test.ts‎

Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -382,6 +382,89 @@ const threadWorktreePathsReadActiveThreads = Effect.gen(function* () {
382382
]);
383383
});
384384

385+
const commandReadsSkipSettledLeafNodes = Effect.gen(function* () {
386+
const store = yield* ProjectionStore.ProjectionStoreV2;
387+
const now = yield* DateTime.now;
388+
const threadId = yield* addRolledBackRecoveryCandidate("settled-leaf-nodes");
389+
const settledRun = (yield* store.getThreadProjection(threadId)).runs[0]!;
390+
const activeRun = {
391+
...settledRun,
392+
id: RunId.make("run:settled-leaf-nodes:active"),
393+
ordinal: 2,
394+
userMessageId: MessageId.make("message:settled-leaf-nodes:active"),
395+
rootNodeId: NodeId.make("node:settled-leaf-nodes:active-root"),
396+
status: "running" as const,
397+
completedAt: null,
398+
};
399+
yield* store.apply({
400+
id: EventId.make("event:settled-leaf-nodes:active-run"),
401+
type: "run.created",
402+
threadId,
403+
runId: activeRun.id,
404+
nodeId: activeRun.rootNodeId,
405+
driver,
406+
providerInstanceId,
407+
occurredAt: now,
408+
payload: activeRun,
409+
});
410+
const nodes = [
411+
["settled-root", settledRun, "root_turn", "completed"],
412+
["settled-tool", settledRun, "tool_call", "completed"],
413+
["settled-message", settledRun, "assistant_message", "failed"],
414+
["settled-reasoning", settledRun, "reasoning", "cancelled"],
415+
["settled-subagent", settledRun, "subagent", "completed"],
416+
["settled-run-live-tool", settledRun, "tool_call", "running"],
417+
["active-run-done-tool", activeRun, "tool_call", "completed"],
418+
] as const;
419+
for (const [name, run, kind, status] of nodes) {
420+
const id = NodeId.make(`node:settled-leaf-nodes:${name}`);
421+
yield* store.apply({
422+
id: EventId.make(`event:settled-leaf-nodes:${name}`),
423+
type: "node.updated",
424+
threadId,
425+
runId: run.id,
426+
nodeId: id,
427+
driver,
428+
occurredAt: now,
429+
payload: {
430+
id,
431+
threadId,
432+
runId: run.id,
433+
rootNodeId: kind === "root_turn" ? id : run.rootNodeId!,
434+
parentNodeId: kind === "root_turn" ? null : run.rootNodeId,
435+
kind,
436+
status,
437+
countsForRun: kind === "root_turn",
438+
providerThreadId: null,
439+
providerTurnId: null,
440+
nativeItemRef: null,
441+
runtimeRequestId: null,
442+
checkpointScopeId: null,
443+
startedAt: now,
444+
completedAt: status === "running" ? null : now,
445+
},
446+
});
447+
}
448+
const nodeNames = (records: { readonly nodes: ReadonlyArray<{ readonly id: string }> }) =>
449+
records.nodes
450+
.map((node) => node.id.replace("node:settled-leaf-nodes:", ""))
451+
.filter((name) => nodes.some(([candidate]) => candidate === name))
452+
.toSorted();
453+
454+
assert.deepEqual(
455+
nodeNames(yield* store.getThreadRecords(threadId, ["nodes"], { omitSettledLeafNodes: true })),
456+
["active-run-done-tool", "settled-root", "settled-run-live-tool", "settled-subagent"],
457+
);
458+
assert.deepEqual(
459+
nodeNames(yield* store.getThreadRecords(threadId, ["nodes"])),
460+
nodes.map(([name]) => name).toSorted(),
461+
);
462+
});
463+
464+
it.effect("memory command reads skip finished leaf nodes of finished runs", () =>
465+
commandReadsSkipSettledLeafNodes.pipe(Effect.provide(ProjectionStore.layerMemory)),
466+
);
467+
385468
it.effect("memory worktree path lookup reads only active threads", () =>
386469
threadWorktreePathsReadActiveThreads.pipe(Effect.provide(ProjectionStore.layerMemory)),
387470
);
@@ -657,6 +740,10 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => {
657740
assert.deepEqual(recovery?.restartCancelledBackgroundWork, work);
658741
}),
659742
);
743+
it.effect(
744+
"command reads skip finished leaf nodes of finished runs",
745+
() => commandReadsSkipSettledLeafNodes,
746+
);
660747
it.effect("limits turn-start history to the requested runs, including an empty selection", () =>
661748
Effect.gen(function* () {
662749
const store = yield* ProjectionStore.ProjectionStoreV2;

‎apps/server/src/orchestration-v2/ProjectionStore.ts‎

Lines changed: 53 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -304,7 +304,39 @@ export interface ProjectionRecordFilter {
304304
readonly runIds?: ReadonlyArray<RunId>;
305305
readonly turnItemTypes?: ReadonlyArray<OrchestrationV2TurnItem["type"]>;
306306
readonly turnItemStatuses?: ReadonlyArray<OrchestrationV2TurnItem["status"]>;
307+
// Skips finished tool call, assistant message and reasoning nodes of finished
308+
// runs. Command decisions only read root, subagent and active-run nodes.
309+
readonly omitSettledLeafNodes?: boolean;
307310
}
311+
312+
type ProjectionNode = OrchestrationV2ThreadProjection["nodes"][number];
313+
type ProjectionRun = OrchestrationV2ThreadProjection["runs"][number];
314+
const settledLeafNodeKinds: ReadonlySet<ProjectionNode["kind"]> = new Set([
315+
"tool_call",
316+
"assistant_message",
317+
"reasoning",
318+
]);
319+
const settledNodeStatuses: ReadonlySet<ProjectionNode["status"]> = new Set([
320+
"completed",
321+
"interrupted",
322+
"failed",
323+
"cancelled",
324+
"rolled_back",
325+
]);
326+
const activeRunStatuses: ReadonlySet<ProjectionRun["status"]> = new Set([
327+
"preparing",
328+
"queued",
329+
"starting",
330+
"running",
331+
"waiting",
332+
]);
333+
334+
// In-memory twin of the omitSettledLeafNodes SQL clause in readCanonicalProjection.
335+
const isSettledLeafNode = (node: ProjectionNode, runs: ReadonlyArray<ProjectionRun>) =>
336+
settledLeafNodeKinds.has(node.kind) &&
337+
settledNodeStatuses.has(node.status) &&
338+
node.runId !== null &&
339+
!runs.some((run) => run.id === node.runId && activeRunStatuses.has(run.status));
308340
export type ProjectionRecordField = Exclude<
309341
keyof OrchestrationV2ThreadProjection,
310342
"thread" | "updatedAt" | "visibleTurnItems"
@@ -3069,9 +3101,23 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
30693101
: window === undefined
30703102
? sql<PayloadRow>`
30713103
SELECT payload_json
3072-
FROM orchestration_v2_projection_nodes
3073-
WHERE thread_id = ${threadId}
3074-
ORDER BY COALESCE(started_at, ''), node_id ASC
3104+
FROM orchestration_v2_projection_nodes AS node
3105+
WHERE node.thread_id = ${threadId}
3106+
${
3107+
filter?.omitSettledLeafNodes === true
3108+
? sql`AND NOT (
3109+
node.kind IN ('tool_call', 'assistant_message', 'reasoning')
3110+
AND node.status IN ('completed', 'interrupted', 'failed', 'cancelled', 'rolled_back')
3111+
AND node.run_id IS NOT NULL
3112+
AND NOT EXISTS (
3113+
SELECT 1 FROM orchestration_v2_projection_runs AS run
3114+
WHERE run.run_id = node.run_id
3115+
AND run.status IN ('preparing', 'queued', 'starting', 'running', 'waiting')
3116+
)
3117+
)`
3118+
: sql``
3119+
}
3120+
ORDER BY COALESCE(node.started_at, ''), node.node_id ASC
30753121
`
30763122
: sql<PayloadRow & { readonly position: number }>`
30773123
-- Keep retained IDs outermost so SQLite looks up each node by its
@@ -6423,6 +6469,10 @@ export const layerMemory: Layer.Layer<ProjectionStoreV2> = Layer.effect(
64236469
filter.turnItemStatuses.includes(row.status)) &&
64246470
(filter?.turnItemRunId === undefined || filter.turnItemRunId === row.runId),
64256471
),
6472+
nodes:
6473+
filter?.omitSettledLeafNodes === true
6474+
? projection.nodes.filter((node) => !isSettledLeafNode(node, projection.runs))
6475+
: projection.nodes,
64266476
};
64276477
return Object.fromEntries([
64286478
["thread", projection.thread],

0 commit comments

Comments
 (0)