Skip to content

Commit 759566f

Browse files
committed
fix(server): turn starts in long threads no longer stall the server
Each turn start reads the thread's runtime recovery state four times. Several of those reads look up unfinished turn items without a type filter, so no partial index applied and SQLite read every item in the thread. On threads with 14,000-17,500 items that cost 75-93 ms per read, so each turn start blocked the event loop for roughly 300-370 ms, freezing every client request, browser stream input included. Add a partial index over unfinished turn items, ordered like the recovery read, and drop a redundant run_id IS NOT NULL filter that steered one subquery onto the full-history run index. The same reads now take 4-7 ms on those threads.
1 parent 67c0fc0 commit 759566f

6 files changed

Lines changed: 60 additions & 2 deletions

File tree

‎apps/server/src/orchestration-v2/ProjectionRecovery.test.ts‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ import * as DateTime from "effect/DateTime";
2121
import * as Effect from "effect/Effect";
2222
import * as Layer from "effect/Layer";
2323
import * as SqlClient from "effect/sql/SqlClient";
24+
import * as Tracer from "effect/Tracer";
2425

2526
import * as SqlitePersistence from "../persistence/Sqlite.ts";
2627
import * as ServerSettings from "../serverSettings.ts";
@@ -369,6 +370,44 @@ it.effect("selects unfinished recovery work without reading settled thread histo
369370
}).pipe(Effect.provide(layerTest)),
370371
);
371372

373+
it.effect("looks up a thread's unfinished turn items without reading its finished history", () =>
374+
Effect.gen(function* () {
375+
const projections = yield* ProjectionStore.ProjectionStoreV2;
376+
const sql = yield* SqlClient.SqlClient;
377+
const threadId = yield* createThread("unfinished-item-lookups");
378+
yield* createRun(threadId, "running");
379+
const statements: Array<string> = [];
380+
const tracer = Tracer.make({
381+
span(options) {
382+
const span = new Tracer.NativeSpan(options);
383+
const end = span.end.bind(span);
384+
span.end = (endTime, exit) => {
385+
end(endTime, exit);
386+
const query = span.attributes.get("db.query.text");
387+
if (typeof query === "string") statements.push(query);
388+
};
389+
return span;
390+
},
391+
});
392+
yield* projections.getRuntimeRecoveryProjection(threadId).pipe(Effect.withTracer(tracer));
393+
394+
const itemLookups: Array<string> = [];
395+
for (const statement of statements) {
396+
const plan = yield* sql.unsafe<{ readonly detail: string }>(
397+
`EXPLAIN QUERY PLAN ${statement}`,
398+
);
399+
itemLookups.push(
400+
...plan.map((row) => row.detail).filter((detail) => /^(?:SEARCH|SCAN) item\b/.test(detail)),
401+
);
402+
}
403+
// Partial indexes hold only unfinished items, so a long thread's history stays unread.
404+
assert.isNotEmpty(itemLookups);
405+
for (const lookup of itemLookups) {
406+
assert.match(lookup, /turn_items_(?:active|recovery)_idx\b/);
407+
}
408+
}).pipe(Effect.provide(layerTest)),
409+
);
410+
372411
it.effect("recovers terminal subagent results until their cross-thread transfer exists", () =>
373412
Effect.gen(function* () {
374413
const projections = yield* ProjectionStore.ProjectionStoreV2;

‎apps/server/src/orchestration-v2/ProjectionStore.ts‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4353,7 +4353,6 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
43534353
WHERE item.thread_id = ${threadId}
43544354
AND item.type IN ('command_execution', 'dynamic_tool', 'subagent')
43554355
AND item.status IN ('pending', 'running', 'waiting')
4356-
AND item.run_id IS NOT NULL
43574356
)
43584357
)
43594358
ORDER BY run.ordinal ASC

‎apps/server/src/persistence/Migrations.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,7 @@ import Migration0063 from "./Migrations/063_DropUnusedV1Projections.ts";
6969
import Migration0064 from "./Migrations/064_CanonicalizeThreadPullRequestLinks.ts";
7070
import Migration0065 from "./Migrations/065_CodexUsageNativeTurnIndex.ts";
7171
import Migration0066 from "./Migrations/066_RepairClientSettlementTimestamps.ts";
72+
import Migration0067 from "./Migrations/067_TurnItemsActiveStatusIndex.ts";
7273

7374
export const migrationEntries = [
7475
[1, "OrchestrationEvents", Migration0001],
@@ -138,6 +139,7 @@ export const migrationEntries = [
138139
[64, "CanonicalizeThreadPullRequestLinks", Migration0064],
139140
[65, "CodexUsageNativeTurnIndex", Migration0065],
140141
[66, "RepairClientSettlementTimestamps", Migration0066],
142+
[67, "TurnItemsActiveStatusIndex", Migration0067],
141143
] as const;
142144

143145
export const migrationManifest = migrationEntries.map(([id, name]) => [id, name] as const);

‎apps/server/src/persistence/Migrations/055_OrchestrationV2.test.ts‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ layer("055_OrchestrationV2", (it) => {
1515
Effect.sync(() => {
1616
assert.deepStrictEqual(
1717
migrationEntries.map(([id]) => id),
18-
Array.from({ length: 66 }, (_, index) => index + 1),
18+
Array.from({ length: 67 }, (_, index) => index + 1),
1919
);
2020
}),
2121
);
@@ -40,6 +40,7 @@ layer("055_OrchestrationV2", (it) => {
4040
[64, "CanonicalizeThreadPullRequestLinks"],
4141
[65, "CodexUsageNativeTurnIndex"],
4242
[66, "RepairClientSettlementTimestamps"],
43+
[67, "TurnItemsActiveStatusIndex"],
4344
]);
4445
assert.deepStrictEqual(yield* runMigrations(), []);
4546

@@ -72,6 +73,7 @@ layer("055_OrchestrationV2", (it) => {
7273
{ migration_id: 64, name: "CanonicalizeThreadPullRequestLinks" },
7374
{ migration_id: 65, name: "CodexUsageNativeTurnIndex" },
7475
{ migration_id: 66, name: "RepairClientSettlementTimestamps" },
76+
{ migration_id: 67, name: "TurnItemsActiveStatusIndex" },
7577
]);
7678

7779
const tables = yield* sql<{ readonly name: string }>`
Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
import * as Effect from "effect/Effect";
2+
import * as SqlClient from "effect/sql/SqlClient";
3+
4+
export default Effect.gen(function* () {
5+
const sql = yield* SqlClient.SqlClient;
6+
7+
// ProjectionStore's runtime recovery reads look up a thread's unfinished
8+
// items of any type; without it they read the thread's whole history.
9+
yield* sql`
10+
CREATE INDEX IF NOT EXISTS orchestration_v2_projection_turn_items_active_idx
11+
ON orchestration_v2_projection_turn_items(thread_id, ordinal, turn_item_id)
12+
WHERE status IN ('pending', 'running', 'waiting')
13+
`;
14+
});

‎apps/server/src/persistence/reconcileV2PreviewMigration.test.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@ describe("V2 preview upgrade", () => {
4646
[64, "CanonicalizeThreadPullRequestLinks"],
4747
[65, "CodexUsageNativeTurnIndex"],
4848
[66, "RepairClientSettlementTimestamps"],
49+
[67, "TurnItemsActiveStatusIndex"],
4950
]);
5051
assert.deepStrictEqual(yield* runMigrations(), []);
5152
assert.deepStrictEqual(yield* sql`SELECT * FROM orchestration_v2_legacy_imports`, imports);
@@ -135,6 +136,7 @@ describe("V2 preview upgrade", () => {
135136
[64, "CanonicalizeThreadPullRequestLinks"],
136137
[65, "CodexUsageNativeTurnIndex"],
137138
[66, "RepairClientSettlementTimestamps"],
139+
[67, "TurnItemsActiveStatusIndex"],
138140
]);
139141
}).pipe(Effect.provide(NodeSqliteClient.layer({ filename: ":memory:" }))),
140142
);

0 commit comments

Comments
 (0)