Skip to content

Commit f8b290d

Browse files
perf(server): index unfinished work during startup recovery
1 parent cfafb4b commit f8b290d

4 files changed

Lines changed: 63 additions & 8 deletions

File tree

‎apps/server/src/orchestration-v2/ProjectionStore.test.ts‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3158,6 +3158,49 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => {
31583158
}),
31593159
);
31603160

3161+
it.effect("selects runtime recovery candidates without scanning historical runs", () =>
3162+
Effect.gen(function* () {
3163+
const store = yield* ProjectionStore.ProjectionStoreV2;
3164+
const sql = yield* SqlClient.SqlClient;
3165+
yield* addRolledBackRecoveryCandidate("indexed-candidates");
3166+
const { statements, tracer } = traceSqlStatements();
3167+
yield* store.getRecoveryThreadIds("runtime").pipe(Effect.withTracer(tracer));
3168+
const statement = statements.find((query) => query.includes("WITH pending_provider_threads"));
3169+
assert.isDefined(statement);
3170+
const plan = yield* sql.unsafe<{ readonly detail: string }>(
3171+
`EXPLAIN QUERY PLAN ${statement}`,
3172+
);
3173+
const runScans = plan.filter((row) =>
3174+
/(?:SCAN|SEARCH) orchestration_v2_projection_runs /.test(row.detail),
3175+
);
3176+
assert.isNotEmpty(runScans);
3177+
for (const { detail } of runScans) assert.include(detail, "runs_recovery_idx");
3178+
}),
3179+
);
3180+
3181+
it.effect("recovers provider threads without walking finished turn items", () =>
3182+
Effect.gen(function* () {
3183+
const store = yield* ProjectionStore.ProjectionStoreV2;
3184+
const sql = yield* SqlClient.SqlClient;
3185+
const threadId = yield* addRolledBackRecoveryCandidate("indexed-runtime");
3186+
const { statements, tracer } = traceSqlStatements();
3187+
yield* store.getRuntimeRecoveryProjection(threadId).pipe(Effect.withTracer(tracer));
3188+
3189+
const statement = statements.find((query) =>
3190+
query.includes("SELECT provider_thread.payload_json"),
3191+
);
3192+
assert.isDefined(statement);
3193+
const plan = yield* sql.unsafe<{ readonly detail: string }>(
3194+
`EXPLAIN QUERY PLAN ${statement}`,
3195+
);
3196+
const itemLookups = plan.filter((row) => row.detail.startsWith("SEARCH item "));
3197+
assert.isNotEmpty(itemLookups);
3198+
for (const { detail } of itemLookups) {
3199+
assert.include(detail, "turn_items_live_idx (thread_id=?)");
3200+
}
3201+
}),
3202+
);
3203+
31613204
it.effect("projects only the latest failed root turn's limit into SQL and memory shells", () =>
31623205
Effect.gen(function* () {
31633206
const store = yield* ProjectionStore.ProjectionStoreV2;

‎apps/server/src/orchestration-v2/ProjectionStore.ts‎

Lines changed: 4 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -3679,15 +3679,11 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
36793679
ELSE 0 END
36803680
)
36813681
SELECT thread_id FROM orchestration_v2_projection_runs
3682-
WHERE status IN ('preparing', 'starting', 'running', 'waiting')
3683-
UNION
3684-
-- A held queue already went through recovery; rereading it on
3685-
-- every boot costs a projection read per held thread.
3686-
SELECT thread_id FROM orchestration_v2_projection_runs
3687-
WHERE status = 'queued'
3688-
AND CASE WHEN json_valid(payload_json)
3682+
-- Match the partial index predicate; held queues still stay excluded.
3683+
WHERE status IN ('queued', 'preparing', 'starting', 'running', 'waiting')
3684+
AND (status <> 'queued' OR CASE WHEN json_valid(payload_json)
36893685
THEN json_extract(payload_json, '$.queueHeld') IS NOT 1
3690-
ELSE 1 END
3686+
ELSE 1 END)
36913687
UNION
36923688
SELECT thread_id FROM orchestration_v2_projection_runtime_requests
36933689
WHERE status = 'pending'

‎apps/server/src/persistence/Migrations.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,7 @@ import Migration0059 from "./Migrations/059_McpAppModelContext.ts";
6565
import Migration0060 from "./Migrations/060_ThreadSnapshotWindowIndexes.ts";
6666
import Migration0061 from "./Migrations/061_PullRequestReviewGuides.ts";
6767
import Migration0062 from "./Migrations/062_ThreadFailureLookupIndex.ts";
68+
import Migration0063 from "./Migrations/063_RuntimeRecoveryItemIndex.ts";
6869

6970
export const migrationEntries = [
7071
[1, "OrchestrationEvents", Migration0001],
@@ -130,6 +131,7 @@ export const migrationEntries = [
130131
[60, "ThreadSnapshotWindowIndexes", Migration0060],
131132
[61, "PullRequestReviewGuides", Migration0061],
132133
[62, "ThreadFailureLookupIndex", Migration0062],
134+
[63, "RuntimeRecoveryItemIndex", Migration0063],
133135
] as const;
134136

135137
export const migrationManifest = migrationEntries.map(([id, name]) => [id, name] as const);
Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
import * as Effect from "effect/Effect";
2+
import * as SqlClient from "effect/sql/SqlClient";
3+
4+
export default Effect.gen(function* () {
5+
const sql = yield* SqlClient.SqlClient;
6+
7+
// Recovery looks up unfinished work across item types. The existing recovery
8+
// index covers only three types, so these reads walk the finished transcript.
9+
yield* sql`
10+
CREATE INDEX orchestration_v2_projection_turn_items_live_idx
11+
ON orchestration_v2_projection_turn_items(thread_id)
12+
WHERE status IN ('pending', 'running', 'waiting')
13+
`;
14+
});

0 commit comments

Comments
 (0)