Skip to content

Commit 776f1ea

Browse files
committed
fix(server): turn starts in long threads no longer stall the server
Each turn start reads the thread's runtime recovery state four times. Several of those reads look up unfinished turn items without a type filter, so no partial index applied and SQLite read every item in the thread. On threads with 14,000-17,500 items that cost 75-93 ms per read, so each turn start blocked the event loop for roughly 300-370 ms, freezing every client request, browser stream input included. Add a partial index over unfinished turn items, ordered like the recovery read, and drop a redundant run_id IS NOT NULL filter that steered one subquery onto the full-history run index. The same reads now take 4-7 ms on those threads.
1 parent 1451e7c commit 776f1ea

7 files changed

Lines changed: 64 additions & 4 deletions

File tree

‎apps/server/src/orchestration-v2/ProjectionRecovery.test.ts‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ import * as DateTime from "effect/DateTime";
2121
import * as Effect from "effect/Effect";
2222
import * as Layer from "effect/Layer";
2323
import * as SqlClient from "effect/sql/SqlClient";
24+
import * as Tracer from "effect/Tracer";
2425

2526
import * as SqlitePersistence from "../persistence/Sqlite.ts";
2627
import * as ServerSettings from "../serverSettings.ts";
@@ -369,6 +370,44 @@ it.effect("selects unfinished recovery work without reading settled thread histo
369370
}).pipe(Effect.provide(layerTest)),
370371
);
371372

373+
it.effect("looks up a thread's unfinished turn items without reading its finished history", () =>
374+
Effect.gen(function* () {
375+
const projections = yield* ProjectionStore.ProjectionStoreV2;
376+
const sql = yield* SqlClient.SqlClient;
377+
const threadId = yield* createThread("unfinished-item-lookups");
378+
yield* createRun(threadId, "running");
379+
const statements: Array<string> = [];
380+
const tracer = Tracer.make({
381+
span(options) {
382+
const span = new Tracer.NativeSpan(options);
383+
const end = span.end.bind(span);
384+
span.end = (endTime, exit) => {
385+
end(endTime, exit);
386+
const query = span.attributes.get("db.query.text");
387+
if (typeof query === "string") statements.push(query);
388+
};
389+
return span;
390+
},
391+
});
392+
yield* projections.getRuntimeRecoveryProjection(threadId).pipe(Effect.withTracer(tracer));
393+
394+
const itemLookups: Array<string> = [];
395+
for (const statement of statements) {
396+
const plan = yield* sql.unsafe<{ readonly detail: string }>(
397+
`EXPLAIN QUERY PLAN ${statement}`,
398+
);
399+
itemLookups.push(
400+
...plan.map((row) => row.detail).filter((detail) => /^(?:SEARCH|SCAN) item\b/.test(detail)),
401+
);
402+
}
403+
// Partial indexes hold only unfinished items, so a long thread's history stays unread.
404+
assert.isNotEmpty(itemLookups);
405+
for (const lookup of itemLookups) {
406+
assert.match(lookup, /turn_items_(?:active|recovery)_idx\b/);
407+
}
408+
}).pipe(Effect.provide(layerTest)),
409+
);
410+
372411
it.effect("recovers terminal subagent results until their cross-thread transfer exists", () =>
373412
Effect.gen(function* () {
374413
const projections = yield* ProjectionStore.ProjectionStoreV2;

‎apps/server/src/orchestration-v2/ProjectionStore.ts‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4143,7 +4143,6 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
41434143
WHERE item.thread_id = ${threadId}
41444144
AND item.type IN ('command_execution', 'dynamic_tool', 'subagent')
41454145
AND item.status IN ('pending', 'running', 'waiting')
4146-
AND item.run_id IS NOT NULL
41474146
)
41484147
)
41494148
ORDER BY run.ordinal ASC

‎apps/server/src/persistence/Migrations.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,7 @@ import Migration0061 from "./Migrations/061_PullRequestReviewGuides.ts";
6767
import Migration0062 from "./Migrations/062_ThreadFailureLookupIndex.ts";
6868
import Migration0063 from "./Migrations/063_DropUnusedV1Projections.ts";
6969
import Migration0064 from "./Migrations/064_CanonicalizeThreadPullRequestLinks.ts";
70+
import Migration0065 from "./Migrations/065_TurnItemsActiveStatusIndex.ts";
7071

7172
export const migrationEntries = [
7273
[1, "OrchestrationEvents", Migration0001],
@@ -134,6 +135,7 @@ export const migrationEntries = [
134135
[62, "ThreadFailureLookupIndex", Migration0062],
135136
[63, "DropUnusedV1Projections", Migration0063],
136137
[64, "CanonicalizeThreadPullRequestLinks", Migration0064],
138+
[65, "TurnItemsActiveStatusIndex", Migration0065],
137139
] as const;
138140

139141
export const migrationManifest = migrationEntries.map(([id, name]) => [id, name] as const);

‎apps/server/src/persistence/Migrations/055_OrchestrationV2.test.ts‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ layer("055_OrchestrationV2", (it) => {
1313
Effect.sync(() => {
1414
assert.deepStrictEqual(
1515
migrationEntries.map(([id]) => id),
16-
Array.from({ length: 64 }, (_, index) => index + 1),
16+
Array.from({ length: 65 }, (_, index) => index + 1),
1717
);
1818
}),
1919
);
@@ -36,6 +36,7 @@ layer("055_OrchestrationV2", (it) => {
3636
[62, "ThreadFailureLookupIndex"],
3737
[63, "DropUnusedV1Projections"],
3838
[64, "CanonicalizeThreadPullRequestLinks"],
39+
[65, "TurnItemsActiveStatusIndex"],
3940
]);
4041
assert.deepStrictEqual(yield* runMigrations(), []);
4142

@@ -66,6 +67,7 @@ layer("055_OrchestrationV2", (it) => {
6667
{ migration_id: 62, name: "ThreadFailureLookupIndex" },
6768
{ migration_id: 63, name: "DropUnusedV1Projections" },
6869
{ migration_id: 64, name: "CanonicalizeThreadPullRequestLinks" },
70+
{ migration_id: 65, name: "TurnItemsActiveStatusIndex" },
6971
]);
7072

7173
const tables = yield* sql<{ readonly name: string }>`

‎apps/server/src/persistence/Migrations/064_CanonicalizeThreadPullRequestLinks.test.ts‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -123,8 +123,10 @@ layer("064_CanonicalizeThreadPullRequestLinks", (it) => {
123123
'user', ${v1Payload}, '{}', 1)
124124
`;
125125

126-
assert.deepEqual(yield* runMigrations(), [[64, "CanonicalizeThreadPullRequestLinks"]]);
127-
assert.deepEqual(yield* runMigrations(), []);
126+
assert.deepEqual(yield* runMigrations({ toMigrationInclusive: 64 }), [
127+
[64, "CanonicalizeThreadPullRequestLinks"],
128+
]);
129+
assert.deepEqual(yield* runMigrations({ toMigrationInclusive: 64 }), []);
128130
const rows = yield* sql<{ readonly thread_id: string; readonly payload_json: string }>`
129131
SELECT thread_id, payload_json FROM orchestration_v2_projection_threads
130132
`;
Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
import * as Effect from "effect/Effect";
2+
import * as SqlClient from "effect/sql/SqlClient";
3+
4+
export default Effect.gen(function* () {
5+
const sql = yield* SqlClient.SqlClient;
6+
7+
// ProjectionStore's runtime recovery reads look up a thread's unfinished
8+
// items of any type; without it they read the thread's whole history.
9+
yield* sql`
10+
CREATE INDEX IF NOT EXISTS orchestration_v2_projection_turn_items_active_idx
11+
ON orchestration_v2_projection_turn_items(thread_id, ordinal, turn_item_id)
12+
WHERE status IN ('pending', 'running', 'waiting')
13+
`;
14+
});

‎apps/server/src/persistence/reconcileV2PreviewMigration.test.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@ describe("V2 preview upgrade", () => {
4444
[62, "ThreadFailureLookupIndex"],
4545
[63, "DropUnusedV1Projections"],
4646
[64, "CanonicalizeThreadPullRequestLinks"],
47+
[65, "TurnItemsActiveStatusIndex"],
4748
]);
4849
assert.deepStrictEqual(yield* runMigrations(), []);
4950
assert.deepStrictEqual(yield* sql`SELECT * FROM orchestration_v2_legacy_imports`, imports);
@@ -131,6 +132,7 @@ describe("V2 preview upgrade", () => {
131132
[62, "ThreadFailureLookupIndex"],
132133
[63, "DropUnusedV1Projections"],
133134
[64, "CanonicalizeThreadPullRequestLinks"],
135+
[65, "TurnItemsActiveStatusIndex"],
134136
]);
135137
}).pipe(Effect.provide(NodeSqliteClient.layer({ filename: ":memory:" }))),
136138
);

0 commit comments

Comments
 (0)