Skip to content

Commit b7a224c

Browse files
perf(server): sending, editing and steering queued messages no longer read a long thread's whole history (#29521)
1 parent 0f5ecef commit b7a224c

7 files changed

Lines changed: 332 additions & 26 deletions

File tree

‎apps/server/src/orchestration-v2/Orchestrator.control-reads.test.ts‎

Lines changed: 291 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ import * as Result from "effect/Result";
2828
import * as Stream from "effect/Stream";
2929
import * as TestClock from "effect/testing/TestClock";
3030
import * as SqlClient from "effect/sql/SqlClient";
31+
import * as Statement from "effect/sql/Statement";
3132
import {
3233
orchestrationCommandDuration,
3334
orchestrationCommandsTotal,
@@ -2199,3 +2200,293 @@ it.effect("a user message dispatched first stays run 1 when its lineage read is
21992200
);
22002201
}).pipe(Effect.provide(layerGatedLineageRead));
22012202
});
2203+
2204+
it.effect("queued-message commands read a long thread's history without walking all of it", () =>
2205+
Effect.gen(function* () {
2206+
const orchestrator = yield* Orchestrator.OrchestratorV2;
2207+
const projections = yield* ProjectionStore.ProjectionStoreV2;
2208+
const sql = yield* SqlClient.SqlClient;
2209+
const threadId = ThreadId.make("thread:long-history-reads");
2210+
const driver = ProviderDriverKind.make("codex");
2211+
const now = yield* DateTime.now;
2212+
yield* orchestrator.dispatch({
2213+
type: "thread.create",
2214+
commandId: CommandId.make("create-long-history-reads"),
2215+
threadId,
2216+
projectId: ProjectId.make("project:long-history-reads"),
2217+
title: "Long history",
2218+
modelSelection,
2219+
runtimeMode: "full-access",
2220+
interactionMode: "default",
2221+
branch: null,
2222+
worktreePath: null,
2223+
createdBy: "user",
2224+
creationSource: "web",
2225+
});
2226+
const providerThreadId = ProviderThreadId.make("provider-thread:long-history-reads");
2227+
yield* projections.apply({
2228+
id: EventId.make("event:long-history-reads:provider-thread"),
2229+
type: "provider-thread.updated",
2230+
threadId,
2231+
driver,
2232+
providerInstanceId: instanceId,
2233+
occurredAt: now,
2234+
payload: {
2235+
id: providerThreadId,
2236+
appThreadId: threadId,
2237+
ownerNodeId: null,
2238+
driver,
2239+
providerInstanceId: instanceId,
2240+
providerSessionId: null,
2241+
nativeThreadRef: null,
2242+
nativeConversationHeadRef: null,
2243+
status: "idle",
2244+
firstRunOrdinal: 1,
2245+
lastRunOrdinal: 4,
2246+
handoffIds: [],
2247+
forkedFrom: null,
2248+
createdAt: now,
2249+
updatedAt: now,
2250+
pendingBackgroundTasks: [],
2251+
},
2252+
});
2253+
const dispatch = (name: string) =>
2254+
orchestrator.dispatch({
2255+
type: "message.dispatch",
2256+
commandId: CommandId.make(`dispatch:long-history-reads:${name}`),
2257+
threadId,
2258+
messageId: MessageId.make(`message:long-history-reads:${name}`),
2259+
// A compacting target lets promote skip it without a live provider session.
2260+
text: name === "active" ? "/compact" : `Request ${name}`,
2261+
attachments: [],
2262+
dispatchMode: { type: "defer_start" },
2263+
createdBy: "user",
2264+
creationSource: "web",
2265+
});
2266+
const nextOrdinal = Effect.gen(function* () {
2267+
const [row] = yield* sql<{ readonly ordinal: number }>`
2268+
SELECT COALESCE(MAX(ordinal), 0) + 1 AS ordinal
2269+
FROM orchestration_v2_projection_turn_items WHERE thread_id = ${threadId}`;
2270+
return row!.ordinal;
2271+
});
2272+
// Settled history: each run leaves completed leaf nodes, tool output and a reply.
2273+
for (const name of ["1", "2", "3", "active"]) {
2274+
yield* dispatch(name);
2275+
const run = (yield* projections.getThreadProjection(threadId)).runs.at(-1)!;
2276+
const active = name === "active";
2277+
yield* projections.apply({
2278+
id: EventId.make(`event:long-history-reads:${name}:run`),
2279+
type: "run.updated",
2280+
threadId,
2281+
runId: run.id,
2282+
occurredAt: now,
2283+
payload: {
2284+
...run,
2285+
providerThreadId,
2286+
status: active ? "running" : "completed",
2287+
queuePosition: null,
2288+
startedAt: now,
2289+
completedAt: active ? null : now,
2290+
},
2291+
});
2292+
if (active) continue;
2293+
for (const kind of ["tool_call", "assistant_message"] as const) {
2294+
const nodeId = NodeId.make(`node:long-history-reads:${name}:${kind}`);
2295+
yield* projections.apply({
2296+
id: EventId.make(`event:${nodeId}`),
2297+
type: "node.updated",
2298+
threadId,
2299+
runId: run.id,
2300+
nodeId,
2301+
driver,
2302+
occurredAt: now,
2303+
payload: {
2304+
id: nodeId,
2305+
threadId,
2306+
runId: run.id,
2307+
rootNodeId: run.rootNodeId!,
2308+
parentNodeId: run.rootNodeId,
2309+
kind,
2310+
status: "completed",
2311+
countsForRun: false,
2312+
providerThreadId,
2313+
providerTurnId: null,
2314+
nativeItemRef: null,
2315+
runtimeRequestId: null,
2316+
checkpointScopeId: null,
2317+
startedAt: now,
2318+
completedAt: now,
2319+
},
2320+
});
2321+
}
2322+
const replyId = MessageId.make(`message:long-history-reads:${name}:reply`);
2323+
yield* projections.apply({
2324+
id: EventId.make(`event:${replyId}`),
2325+
type: "message.updated",
2326+
threadId,
2327+
occurredAt: now,
2328+
payload: {
2329+
id: replyId,
2330+
threadId,
2331+
runId: run.id,
2332+
nodeId: NodeId.make(`node:long-history-reads:${name}:assistant_message`),
2333+
createdBy: "agent",
2334+
creationSource: "server",
2335+
role: "assistant",
2336+
text: `Reply ${name}`,
2337+
attachments: [],
2338+
streaming: false,
2339+
createdAt: now,
2340+
updatedAt: now,
2341+
},
2342+
});
2343+
const itemBase = {
2344+
threadId,
2345+
runId: run.id,
2346+
providerThreadId,
2347+
providerTurnId: null,
2348+
nativeItemRef: null,
2349+
parentItemId: null,
2350+
status: "completed" as const,
2351+
startedAt: now,
2352+
completedAt: now,
2353+
updatedAt: now,
2354+
};
2355+
for (const item of [
2356+
{
2357+
...itemBase,
2358+
id: TurnItemId.make(`item:long-history-reads:${name}:command`),
2359+
nodeId: NodeId.make(`node:long-history-reads:${name}:tool_call`),
2360+
ordinal: yield* nextOrdinal,
2361+
title: "command",
2362+
type: "command_execution" as const,
2363+
input: "cat build.log",
2364+
},
2365+
{
2366+
...itemBase,
2367+
id: TurnItemId.make(`item:long-history-reads:${name}:reply`),
2368+
nodeId: NodeId.make(`node:long-history-reads:${name}:assistant_message`),
2369+
ordinal: yield* nextOrdinal,
2370+
title: null,
2371+
type: "assistant_message" as const,
2372+
messageId: replyId,
2373+
text: `Reply ${name}`,
2374+
streaming: false,
2375+
},
2376+
]) {
2377+
yield* projections.apply({
2378+
id: EventId.make(`event:${item.id}`),
2379+
type: "turn-item.updated",
2380+
threadId,
2381+
runId: run.id,
2382+
nodeId: item.nodeId,
2383+
driver,
2384+
occurredAt: now,
2385+
payload: item,
2386+
});
2387+
}
2388+
}
2389+
const activeRun = (yield* projections.getThreadProjection(threadId)).runs.at(-1)!;
2390+
// Stop needs a live provider turn to interrupt.
2391+
yield* projections.apply({
2392+
id: EventId.make("event:long-history-reads:active-turn"),
2393+
type: "provider-turn.updated",
2394+
threadId,
2395+
runId: activeRun.id,
2396+
occurredAt: now,
2397+
payload: {
2398+
id: ProviderTurnId.make("provider-turn:long-history-reads"),
2399+
providerThreadId,
2400+
nodeId: activeRun.rootNodeId!,
2401+
runAttemptId: activeRun.activeAttemptId!,
2402+
nativeTurnRef: { driver, nativeId: "native:long-history-reads", strength: "strong" },
2403+
ordinal: 4,
2404+
status: "running",
2405+
startedAt: now,
2406+
completedAt: null,
2407+
},
2408+
});
2409+
2410+
const statements: Array<ReturnType<Statement.Statement<unknown>["compile"]>> = [];
2411+
const observe: Statement.Transformer = (statement) => {
2412+
const compiled = statement.compile();
2413+
// Row reads and the message-ordering pass; a MAX(ordinal) probe reads one index entry.
2414+
if (/_(nodes|turn_items)\b/.test(compiled[0]) && /payload_json|GROUP BY/.test(compiled[0])) {
2415+
statements.push(compiled);
2416+
}
2417+
return Effect.succeed(statement);
2418+
};
2419+
const historyWalks = Effect.fnUntraced(function* (commandType: string) {
2420+
const walks: Array<string> = [];
2421+
for (const [query, params] of statements.splice(0)) {
2422+
const plan = yield* sql.unsafe<{ detail: string }>(`EXPLAIN QUERY PLAN ${query}`, params);
2423+
for (const { detail } of plan) {
2424+
// A history-sized read: an ordering pass over every message item for a narrowed
2425+
// message read, or a node/turn-item row search bounded by nothing but the thread.
2426+
// Walking only a covering index's ids reads no payloads. Stop still loads every
2427+
// message, so its ordering pass is no larger than that read.
2428+
if (
2429+
(detail.includes("TEMP B-TREE FOR GROUP BY") && query.includes("json_each")) ||
2430+
(/^SEARCH .*(node|turn_item).* \(thread_id=\?\)$/.test(detail) &&
2431+
!detail.includes("USING COVERING INDEX")) ||
2432+
// Every active run in the environment, not only this thread's.
2433+
/^SEARCH .*runs.* \(status=\?\)$/.test(detail) ||
2434+
// A status-filtered read that fetches every typed row to check its status.
2435+
(/^SEARCH .*turn_items USING INDEX .* \(thread_id=\? AND type=\?\)$/.test(detail) &&
2436+
query.includes("status IN"))
2437+
) {
2438+
walks.push(`${commandType}: ${detail}\n in: ${query.replace(/\s+/g, " ").trim()}`);
2439+
}
2440+
}
2441+
}
2442+
return walks;
2443+
});
2444+
const observed = <A, E, R>(effect: Effect.Effect<A, E, R>) =>
2445+
effect.pipe(Effect.provideService(Statement.CurrentTransformer, observe));
2446+
2447+
yield* observed(dispatch("queued"));
2448+
const walks = yield* historyWalks("message.dispatch");
2449+
const queuedRun = (yield* projections.getThreadProjection(threadId)).runs.at(-1)!;
2450+
assert.equal(queuedRun.status, "queued");
2451+
yield* observed(
2452+
orchestrator.dispatch({
2453+
type: "queued-run.edit",
2454+
commandId: CommandId.make("edit:long-history-reads"),
2455+
threadId,
2456+
runId: queuedRun.id,
2457+
text: "Request queued, edited",
2458+
}),
2459+
);
2460+
walks.push(...(yield* historyWalks("queued-run.edit")));
2461+
yield* observed(
2462+
orchestrator.dispatch({
2463+
type: "queued-message.promote-to-steer",
2464+
commandId: CommandId.make("promote:long-history-reads"),
2465+
threadId,
2466+
queuedRunId: queuedRun.id,
2467+
targetRunId: activeRun.id,
2468+
}),
2469+
);
2470+
walks.push(...(yield* historyWalks("queued-message.promote-to-steer")));
2471+
yield* observed(
2472+
orchestrator.dispatch({
2473+
type: "run.interrupt",
2474+
commandId: CommandId.make("interrupt:long-history-reads"),
2475+
threadId,
2476+
runId: activeRun.id,
2477+
}),
2478+
);
2479+
walks.push(...(yield* historyWalks("run.interrupt")));
2480+
2481+
const after = yield* projections.getThreadProjection(threadId);
2482+
assert.deepEqual(
2483+
after.messages.filter((message) => message.role === "user").map((message) => message.text),
2484+
["Request 1", "Request 2", "Request 3", "/compact", "Request queued, edited"],
2485+
);
2486+
assert.deepEqual(
2487+
after.providerThreads.map((providerThread) => providerThread.id),
2488+
[providerThreadId],
2489+
);
2490+
assert.deepEqual(walks, [], walks.join("\n"));
2491+
}).pipe(Effect.provide(layerTest)),
2492+
);

‎apps/server/src/orchestration-v2/Orchestrator.ts‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10839,8 +10839,8 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
1083910839
"runtimeRequests",
1084010840
"subagents",
1084110841
],
10842-
// User messages measure a restart's model switch.
10843-
{ turnItemTypes: ["user_message"], messageRoles: ["user"] },
10842+
// User messages measure a restart's model switch. Steering reads root and live nodes only.
10843+
{ turnItemTypes: ["user_message"], messageRoles: ["user"], omitSettledLeafNodes: true },
1084410844
);
1084510845
if (projection.thread.archivedAt !== null || projection.thread.deletedAt !== null) {
1084610846
return yield* new OrchestratorDispatchError({
@@ -12644,6 +12644,8 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
1264412644
"reasoning",
1264512645
],
1264612646
turnItemStatuses: ["pending", "running", "waiting"],
12647+
// Stop reads root nodes and ends open ones; a settled leaf has nothing left to end.
12648+
omitSettledLeafNodes: true,
1264712649
},
1264812650
);
1264912651
const run = projection.runs.find((candidate) => candidate.id === command.runId);

‎apps/server/src/orchestration-v2/ProjectionStore.test.ts‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -755,7 +755,7 @@ it.effect("idle compaction filters keep the latest run, live work and the newest
755755
})
756756
.pipe(Effect.withTracer(command.tracer));
757757
for (const [table, seeks] of [
758-
["turn_items", ["USING INDEX orchestration_v2_projection_turn_items_conversation_idx"]],
758+
["turn_items", ["turn_items_thread_type_idx (thread_id=? AND type=?)"]],
759759
["nodes", ["nodes_settled_leaf_idx (thread_id=? AND <expr>=?)"]],
760760
["subagents", ["USING COVERING INDEX orchestration_v2_projection_subagents_settled_idx"]],
761761
[
@@ -1292,9 +1292,9 @@ it.effect("record reads filter notification sources without dropping matching me
12921292
`EXPLAIN QUERY PLAN ${ordinalLookup}`,
12931293
[threadId, "[]"],
12941294
)).map((row) => row.detail);
1295-
// User-only reads must stay on the partial index instead of scanning every turn item.
1295+
// User-only reads must stay on a type-bounded index instead of scanning every turn item.
12961296
assert.isTrue(
1297-
plan.some((detail) => detail.includes("turn_items_user_message_idx")),
1297+
plan.some((detail) => /turn_items_(user_message|thread_type)_idx/.test(detail)),
12981298
plan.join("\n"),
12991299
);
13001300
// With every user item loaded, the items give the order and the lookup does not run.

0 commit comments

Comments
 (0)