Skip to content

Commit cf8479e

Browse files
only21milclaude
andauthored
perf(server): drive shell last-error lookup from thread bindings (pingdotgg#17842)
Co-authored-by: Victor Vogel <[email protected]> Co-authored-by: Claude Opus 5.5 (1M context) <[email protected]>
1 parent 72fbad7 commit cf8479e

2 files changed

Lines changed: 182 additions & 6 deletions

File tree

‎apps/server/src/orchestration-v2/ProjectionStore.test.ts‎

Lines changed: 174 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3094,6 +3094,180 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => {
30943094
}),
30953095
);
30963096

3097+
it.effect("looks up a thread's last provider error through its own session bindings", () =>
3098+
Effect.gen(function* () {
3099+
const projectionStore = yield* ProjectionStore.ProjectionStoreV2;
3100+
const sql = yield* SqlClient.SqlClient;
3101+
const now = yield* DateTime.now;
3102+
const later = DateTime.add(now, { minutes: 1 });
3103+
const projectId = ProjectId.make("project:projection-session-error-lookup");
3104+
const attachErroredSession = Effect.fnUntraced(function* (
3105+
name: string,
3106+
lastError: string,
3107+
updatedAt: DateTime.Utc,
3108+
) {
3109+
const threadId = ThreadId.make(`thread:projection-session-error-lookup:${name}`);
3110+
yield* projectionStore.apply({
3111+
id: EventId.make(`event:projection-session-error-lookup:${name}:thread`),
3112+
type: "thread.created",
3113+
threadId,
3114+
occurredAt: now,
3115+
payload: {
3116+
createdBy: "user",
3117+
creationSource: "web",
3118+
id: threadId,
3119+
projectId,
3120+
title: name,
3121+
providerInstanceId,
3122+
modelSelection,
3123+
runtimeMode: "full-access",
3124+
interactionMode: "default",
3125+
branch: null,
3126+
worktreePath: null,
3127+
activeProviderThreadId: null,
3128+
lineage: { parentThreadId: null, relationshipToParent: null, rootThreadId: threadId },
3129+
forkedFrom: null,
3130+
createdAt: now,
3131+
updatedAt: now,
3132+
archivedAt: null,
3133+
settledOverride: null,
3134+
settledAt: null,
3135+
lastVisitedAt: null,
3136+
deletedAt: null,
3137+
},
3138+
});
3139+
yield* projectionStore.apply({
3140+
id: EventId.make(`event:projection-session-error-lookup:${name}:session`),
3141+
type: "provider-session.attached",
3142+
threadId,
3143+
driver,
3144+
providerInstanceId,
3145+
occurredAt: updatedAt,
3146+
payload: {
3147+
id: ProviderSessionId.make(`provider-session:projection-session-error-lookup:${name}`),
3148+
driver,
3149+
providerInstanceId,
3150+
status: "error",
3151+
cwd: "/workspace",
3152+
model: modelSelection.model,
3153+
capabilities: CodexProviderCapabilitiesV2,
3154+
createdAt: updatedAt,
3155+
updatedAt,
3156+
lastError,
3157+
},
3158+
});
3159+
return threadId;
3160+
});
3161+
3162+
const failure = {
3163+
class: "usage_limit" as const,
3164+
message: "Plan limit reached.",
3165+
resetAt: "2099-01-01T00:00:00.000Z",
3166+
code: "usageLimitExceeded",
3167+
retryable: null,
3168+
};
3169+
const threadId = yield* attachErroredSession("own", failure.message, now);
3170+
// The own session repeats the root turn's limit, so the thread stays a
3171+
// recovery candidate only while the lookup reads that session.
3172+
const runId = RunId.make("run:projection-session-error-lookup:own");
3173+
const rootNodeId = NodeId.make("node:projection-session-error-lookup:own");
3174+
yield* projectionStore.apply({
3175+
id: EventId.make("event:projection-session-error-lookup:own:run"),
3176+
type: "run.created",
3177+
threadId,
3178+
runId,
3179+
nodeId: rootNodeId,
3180+
driver,
3181+
providerInstanceId,
3182+
occurredAt: now,
3183+
payload: {
3184+
id: runId,
3185+
threadId,
3186+
ordinal: 1,
3187+
providerInstanceId,
3188+
modelSelection,
3189+
providerThreadId: null,
3190+
userMessageId: MessageId.make("message:projection-session-error-lookup:own"),
3191+
rootNodeId,
3192+
activeAttemptId: null,
3193+
status: "failed",
3194+
requestedAt: now,
3195+
startedAt: now,
3196+
completedAt: now,
3197+
checkpointId: null,
3198+
contextHandoffId: null,
3199+
},
3200+
});
3201+
yield* projectionStore.apply({
3202+
id: EventId.make("event:projection-session-error-lookup:own:error"),
3203+
type: "turn-item.updated",
3204+
threadId,
3205+
occurredAt: now,
3206+
payload: {
3207+
id: TurnItemId.make("projection-session-error-lookup:own:error"),
3208+
threadId,
3209+
runId,
3210+
nodeId: rootNodeId,
3211+
providerThreadId: null,
3212+
providerTurnId: null,
3213+
nativeItemRef: null,
3214+
parentItemId: null,
3215+
ordinal: 1,
3216+
status: "failed",
3217+
title: "Usage limit reached",
3218+
startedAt: now,
3219+
completedAt: now,
3220+
updatedAt: now,
3221+
type: "error",
3222+
failure,
3223+
},
3224+
});
3225+
// Every thread on the provider instance has its own session, and these
3226+
// are newer. They must neither leak into the thread nor be walked for it.
3227+
for (let index = 0; index < 24; index++) {
3228+
yield* attachErroredSession(`other-${index}`, "other session failed", later);
3229+
}
3230+
3231+
const { statements, tracer } = traceSqlStatements();
3232+
const shell = yield* projectionStore.getShellSnapshot().pipe(Effect.withTracer(tracer));
3233+
assert.equal(
3234+
shell.threads.find((thread) => thread.id === threadId)?.lastError,
3235+
failure.message,
3236+
);
3237+
const candidates = yield* projectionStore
3238+
.getLimitRecoveryCandidates({ now, autoResume: true, snooze: false })
3239+
.pipe(Effect.withTracer(tracer));
3240+
// Another thread's newer error would replace the limit and drop the candidate.
3241+
const candidate = candidates.find((row) => row.id === threadId);
3242+
assert.isDefined(candidate);
3243+
assert.include(candidate, {
3244+
status: "failed",
3245+
lastErrorClass: "usage_limit",
3246+
usageLimitResetAt: failure.resetAt,
3247+
latestRunId: runId,
3248+
});
3249+
3250+
// One provider instance usually holds most sessions. Driving the lookup
3251+
// from sessions walks all of them for every thread in the snapshot.
3252+
for (const marker of ["AS blocking_failure_payload_json", "AS failure_payload_json"]) {
3253+
const statement = statements.find((query) => query.includes(marker));
3254+
assert.isDefined(statement, marker);
3255+
const plan = (yield* sql.unsafe<{ readonly detail: string }>(
3256+
`EXPLAIN QUERY PLAN ${statement}`,
3257+
)).map((row) => row.detail);
3258+
const bindingLookup = plan.findIndex((detail) => detail.startsWith("SEARCH binding "));
3259+
const sessionLookup = plan.findIndex((detail) => detail.startsWith("SEARCH session "));
3260+
assert.isAtLeast(bindingLookup, 0, marker);
3261+
assert.include(
3262+
plan[bindingLookup],
3263+
"orchestration_v2_projection_provider_session_bindings_thread_idx (thread_id=?)",
3264+
);
3265+
assert.isAbove(sessionLookup, bindingLookup, marker);
3266+
assert.include(plan[sessionLookup], "(provider_session_id=?)");
3267+
}
3268+
}),
3269+
);
3270+
30973271
it.effect(
30983272
"reads checkpoint context without decoding transcript or checkpoint file payloads",
30993273
() =>

‎apps/server/src/orchestration-v2/ProjectionStore.ts‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -3479,9 +3479,10 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
34793479
item.payload_json AS failure_payload_json,
34803480
(
34813481
SELECT json_extract(session.payload_json, '$.lastError')
3482-
FROM orchestration_v2_projection_provider_sessions session
3483-
INNER JOIN orchestration_v2_projection_provider_session_bindings binding
3484-
ON binding.provider_session_id = session.provider_session_id
3482+
-- Keep bindings outermost: one provider instance can hold every session.
3483+
FROM orchestration_v2_projection_provider_session_bindings binding
3484+
CROSS JOIN orchestration_v2_projection_provider_sessions session
3485+
ON session.provider_session_id = binding.provider_session_id
34853486
WHERE binding.thread_id = t.thread_id
34863487
AND session.provider_instance_id = t.provider_instance_id
34873488
ORDER BY session.updated_at DESC, session.provider_session_id DESC
@@ -5201,9 +5202,10 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
52015202
) AS activity_run_started_at,
52025203
(
52035204
SELECT json_extract(session.payload_json, '$.lastError')
5204-
FROM orchestration_v2_projection_provider_sessions session
5205-
INNER JOIN orchestration_v2_projection_provider_session_bindings binding
5206-
ON binding.provider_session_id = session.provider_session_id
5205+
-- Keep bindings outermost: one provider instance can hold every session.
5206+
FROM orchestration_v2_projection_provider_session_bindings binding
5207+
CROSS JOIN orchestration_v2_projection_provider_sessions session
5208+
ON session.provider_session_id = binding.provider_session_id
52075209
WHERE binding.thread_id = t.thread_id
52085210
AND session.provider_instance_id = t.provider_instance_id
52095211
ORDER BY session.updated_at DESC, session.provider_session_id DESC

0 commit comments

Comments
 (0)