Skip to content

Commit d833184

Browse files
authored
perf(server): bound runtime recovery reads by thread ownership (pingdotgg#29429)
1 parent 5e966bc commit d833184

2 files changed

Lines changed: 160 additions & 12 deletions

File tree

‎apps/server/src/orchestration-v2/ProjectionRecovery.test.ts‎

Lines changed: 146 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ import * as DateTime from "effect/DateTime";
2222
import * as Effect from "effect/Effect";
2323
import * as Layer from "effect/Layer";
2424
import * as SqlClient from "effect/sql/SqlClient";
25+
import * as Statement from "effect/sql/Statement";
2526

2627
import * as SqlitePersistence from "../persistence/Sqlite.ts";
2728
import * as ServerSettings from "../serverSettings.ts";
@@ -657,6 +658,151 @@ it.effect("includes shared sessions and provider-owned background rosters in rec
657658
}).pipe(Effect.provide(layerTest)),
658659
);
659660

661+
it.effect("bounds runtime provider-thread reads by ownership and preserves recovery liveness", () =>
662+
Effect.gen(function* () {
663+
const projections = yield* ProjectionStore.ProjectionStoreV2;
664+
const sql = yield* SqlClient.SqlClient;
665+
const now = yield* DateTime.now;
666+
const threadId = yield* createThread("ownership");
667+
const other = yield* createThread("other-owner");
668+
const ownerNodeId = NodeId.make("node:recovery:owner");
669+
yield* projections.apply({
670+
id: EventId.make("event:recovery:owner"),
671+
type: "node.updated",
672+
threadId,
673+
nodeId: ownerNodeId,
674+
driver,
675+
occurredAt: now,
676+
payload: {
677+
id: ownerNodeId,
678+
threadId,
679+
runId: null,
680+
parentNodeId: null,
681+
rootNodeId: ownerNodeId,
682+
kind: "root_turn",
683+
status: "completed",
684+
countsForRun: false,
685+
providerThreadId: null,
686+
providerTurnId: null,
687+
nativeItemRef: null,
688+
runtimeRequestId: null,
689+
checkpointScopeId: null,
690+
startedAt: now,
691+
completedAt: now,
692+
},
693+
});
694+
const cases = [
695+
{ name: "direct", ownership: "direct", status: "active", ordinal: 2, included: true },
696+
{ name: "owner", ownership: "owner", status: "active", ordinal: null, included: true },
697+
{ name: "owner-only", ownership: "owner", status: "active", ordinal: 1, included: true },
698+
{ name: "subagent", ownership: "subagent", status: "active", ordinal: 1, included: true },
699+
{ name: "duplicate", ownership: "all", status: "active", ordinal: 1, included: true },
700+
{ name: "roster", ownership: "direct", status: "idle", ordinal: 3, included: true },
701+
{ name: "run", ownership: "owner", status: "idle", ordinal: 4, included: true },
702+
{ name: "item", ownership: "subagent", status: "idle", ordinal: 5, included: true },
703+
{ name: "waiting-run", ownership: "direct", status: "idle", ordinal: 6, included: true },
704+
{ name: "settled", ownership: "all", status: "idle", ordinal: 7, included: false },
705+
{ name: "unrelated", ownership: "none", status: "active", ordinal: 8, included: false },
706+
{ name: "unrelated-run", ownership: "none", status: "idle", ordinal: 9, included: false },
707+
] as const;
708+
for (const entry of cases) {
709+
const id = ProviderThreadId.make(`provider-thread:ownership:${entry.name}`);
710+
yield* projections.apply({
711+
id: EventId.make(`event:${id}`),
712+
type: "provider-thread.updated",
713+
threadId,
714+
driver,
715+
providerInstanceId,
716+
occurredAt: now,
717+
payload: {
718+
id,
719+
appThreadId:
720+
entry.name === "owner-only"
721+
? null
722+
: entry.ownership === "direct" || entry.ownership === "all"
723+
? threadId
724+
: other,
725+
ownerNodeId:
726+
entry.ownership === "owner" || entry.ownership === "all" ? ownerNodeId : null,
727+
driver,
728+
providerInstanceId,
729+
providerSessionId: null,
730+
nativeThreadRef: null,
731+
nativeConversationHeadRef: null,
732+
status: entry.status,
733+
firstRunOrdinal: entry.ordinal,
734+
lastRunOrdinal: entry.ordinal,
735+
handoffIds: [],
736+
forkedFrom: null,
737+
createdAt: now,
738+
updatedAt: now,
739+
pendingBackgroundTasks:
740+
entry.name === "roster"
741+
? [{ taskId: "shell", description: "background", kind: "command" }]
742+
: [],
743+
},
744+
});
745+
if (entry.ownership === "subagent" || entry.ownership === "all") {
746+
for (const suffix of ["a", "b"]) {
747+
yield* sql`
748+
INSERT INTO orchestration_v2_projection_subagents
749+
(subagent_id, thread_id, parent_node_id, provider, provider_thread_id,
750+
origin, status, updated_at, payload_json)
751+
VALUES (${`${id}:${suffix}`}, ${threadId}, ${ownerNodeId}, 'codex', ${id},
752+
'provider_native', 'completed', '', '{}')
753+
`;
754+
}
755+
}
756+
if (entry.name === "run" || entry.name === "unrelated-run") {
757+
yield* createRun(threadId, "waiting", { ordinal: entry.ordinal, providerThreadId: id });
758+
}
759+
}
760+
const waitingProviderThreadId = ProviderThreadId.make("provider-thread:ownership:waiting-run");
761+
const waitingRunId = yield* createRun(threadId, "completed", {
762+
ordinal: 6,
763+
providerThreadId: waitingProviderThreadId,
764+
});
765+
yield* sql`
766+
INSERT INTO orchestration_v2_projection_turn_items
767+
(turn_item_id, thread_id, run_id, node_id, provider_thread_id, ordinal, type,
768+
status, updated_at, payload_json)
769+
VALUES ('item:ownership', ${threadId}, ${waitingRunId}, ${ownerNodeId},
770+
${ProviderThreadId.make("provider-thread:ownership:item")}, 1, 'assistant_message', 'waiting', '', '{}')
771+
`;
772+
const statements: Array<ReturnType<Statement.Statement<unknown>["compile"]>> = [];
773+
const observe: Statement.Transformer = (statement) => {
774+
const compiled = statement.compile();
775+
if (compiled[0].includes("SELECT provider_thread.payload_json")) statements.push(compiled);
776+
return Effect.succeed(statement);
777+
};
778+
const projection = yield* projections
779+
.getRuntimeRecoveryProjection(threadId)
780+
.pipe(Effect.provideService(Statement.CurrentTransformer, observe));
781+
assert.deepEqual(
782+
projection.providerThreads.map((row) => row.id),
783+
cases
784+
.filter((entry) => entry.included)
785+
.toSorted((a, b) => (a.ordinal ?? 0) - (b.ordinal ?? 0) || a.name.localeCompare(b.name))
786+
.map((entry) => ProviderThreadId.make(`provider-thread:ownership:${entry.name}`)),
787+
);
788+
assert.equal(statements.length, 1);
789+
const [query, params] = statements[0]!;
790+
const plan = yield* sql.unsafe<{ detail: string }>(`EXPLAIN QUERY PLAN ${query}`, params);
791+
assert.isFalse(
792+
plan.some((row) => /^SCAN provider_thread(?:$| )/.test(row.detail)),
793+
JSON.stringify(plan),
794+
);
795+
assert.isTrue(
796+
plan.some((row) => row.detail.includes("provider_threads_owner_idx")),
797+
JSON.stringify(plan),
798+
);
799+
assert.isFalse(
800+
plan.some((row) => /SEARCH (owner|n) USING INDEX .*nodes_thread/.test(row.detail)),
801+
JSON.stringify(plan),
802+
);
803+
}).pipe(Effect.provide(layerTest)),
804+
);
805+
660806
it.effect("reads the run that owns a background roster, not a queued or resumed one", () =>
661807
Effect.gen(function* () {
662808
const projections = yield* ProjectionStore.ProjectionStoreV2;

‎apps/server/src/orchestration-v2/ProjectionStore.ts‎

Lines changed: 14 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -4586,18 +4586,20 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
45864586
sql<PayloadRow>`
45874587
SELECT provider_thread.payload_json
45884588
FROM orchestration_v2_projection_provider_threads AS provider_thread
4589-
WHERE (
4590-
provider_thread.thread_id = ${threadId}
4591-
OR EXISTS (
4592-
SELECT 1 FROM orchestration_v2_projection_nodes AS owner
4593-
WHERE owner.node_id = provider_thread.owner_node_id
4594-
AND owner.thread_id = ${threadId}
4595-
)
4596-
OR EXISTS (
4597-
SELECT 1 FROM orchestration_v2_projection_subagents AS subagent
4598-
WHERE subagent.provider_thread_id = provider_thread.provider_thread_id
4599-
AND subagent.thread_id = ${threadId}
4600-
)
4589+
WHERE provider_thread.provider_thread_id IN (
4590+
SELECT provider_thread_id FROM orchestration_v2_projection_provider_threads
4591+
WHERE thread_id = ${threadId}
4592+
UNION
4593+
-- Start with non-null owners; a thread can have a large settled node history.
4594+
SELECT owned.provider_thread_id
4595+
FROM orchestration_v2_projection_provider_threads AS owned
4596+
INDEXED BY orchestration_v2_projection_provider_threads_owner_idx
4597+
CROSS JOIN orchestration_v2_projection_nodes AS owner
4598+
ON owner.node_id = owned.owner_node_id
4599+
WHERE owned.owner_node_id IS NOT NULL AND owner.thread_id = ${threadId}
4600+
UNION
4601+
SELECT provider_thread_id FROM orchestration_v2_projection_subagents
4602+
WHERE thread_id = ${threadId} AND provider_thread_id IS NOT NULL
46014603
)
46024604
AND (
46034605
provider_thread.status = 'active'

0 commit comments

Comments
 (0)