Skip to content

Commit 37e0fd3

Browse files
fix(server): a run whose recovery state can't be read fails instead of staying starting (pingdotgg#35013)
1 parent 1a5d373 commit 37e0fd3

2 files changed

Lines changed: 49 additions & 12 deletions

File tree

‎apps/server/src/orchestration-v2/ProviderTurnStartService.test.ts‎

Lines changed: 36 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -201,7 +201,7 @@ function startWithMissingWorktree(
201201

202202
const start = Effect.gen(function* () {
203203
return yield* (yield* ProviderTurnStart.ProviderTurnStartServiceV2)
204-
.start({ threadId, runId })
204+
.start({ threadId, runId, willRetry: true })
205205
.pipe(Effect.flip);
206206
}).pipe(Effect.provide(layer), Effect.runPromise);
207207
return {
@@ -288,6 +288,7 @@ function makeLocalCommandHarness(input: {
288288
readonly interruptRunBeforeOpenFailure?: boolean;
289289
readonly writeFailure?: unknown;
290290
readonly failReadsAfterRunning?: boolean;
291+
readonly recoveryReadFailure?: string;
291292
readonly previousTurn?: {
292293
readonly runStatus: "failed" | "completed";
293294
readonly turnStatus: "running" | "completed" | "interrupted";
@@ -924,14 +925,21 @@ function makeLocalCommandHarness(input: {
924925
const getRuntimeRecoveryProjection = vi.fn<
925926
ProjectionStore.ProjectionStoreV2Shape["getRuntimeRecoveryProjection"]
926927
>(() =>
927-
Effect.as(failReadIfRunning, {
928-
...projection,
929-
hasConversation: projection.messages.some(
930-
(m) =>
931-
m.role === "user" &&
932-
(m.text.trim().toLowerCase() !== "/compact" || m.attachments.length > 0),
933-
),
934-
}),
928+
input.recoveryReadFailure !== undefined
929+
? Effect.fail(
930+
new ProjectionStore.ProjectionStoreReadError({
931+
threadId,
932+
cause: input.recoveryReadFailure,
933+
}),
934+
)
935+
: Effect.as(failReadIfRunning, {
936+
...projection,
937+
hasConversation: projection.messages.some(
938+
(m) =>
939+
m.role === "user" &&
940+
(m.text.trim().toLowerCase() !== "/compact" || m.attachments.length > 0),
941+
),
942+
}),
935943
);
936944
const layer = ProviderTurnStart.layer.pipe(
937945
Layer.provide(
@@ -1660,6 +1668,25 @@ effectIt.effect("starts another provider in the same checkout while a turn is pe
16601668
}),
16611669
);
16621670

1671+
effectIt.effect("fails a starting run when its recovery state cannot be read", () =>
1672+
Effect.gen(function* () {
1673+
const harness = makeLocalCommandHarness({
1674+
text: "Continue",
1675+
recoveryReadFailure: "stored turn item no longer decodes",
1676+
});
1677+
1678+
yield* harness.start.pipe(Effect.ignore);
1679+
1680+
expect(harness.open).not.toHaveBeenCalled();
1681+
expect(harness.startRootRun).not.toHaveBeenCalled();
1682+
const projection = harness.projection();
1683+
expect(projection.runs.at(-1)).toMatchObject({ status: "failed", startedAt: null });
1684+
expect(projection.attempts[0]).toMatchObject({ status: "failed", startedAt: null });
1685+
expect(projection.nodes[0]).toMatchObject({ status: "failed", startedAt: null });
1686+
expect(projection.turnItems).toMatchObject([{ type: "error", status: "failed" }]);
1687+
}),
1688+
);
1689+
16631690
effectIt.effect("terminalizes a starting run when its provider session cannot open", () =>
16641691
Effect.gen(function* () {
16651692
const harness = makeLocalCommandHarness({

‎apps/server/src/orchestration-v2/ProviderTurnStartService.ts‎

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -702,9 +702,19 @@ export const layer: Layer.Layer<
702702
runs: current.runs,
703703
turnItems: current.turnItems,
704704
});
705-
const inheritedBackgroundTurnItems = yield* projectionStore
706-
.getRuntimeRecoveryProjection(projection.thread.id)
707-
.pipe(Effect.map(selectInheritedBackgroundItems));
705+
const recoveryResult = yield* Effect.result(
706+
projectionStore.getRuntimeRecoveryProjection(projection.thread.id),
707+
);
708+
if (recoveryResult._tag === "Failure") {
709+
if (input.willRetry === true) return yield* recoveryResult.failure;
710+
yield* settleStartFailure({
711+
signal: "runtime-recovery-read-failure",
712+
title: "Couldn't read thread state",
713+
error: recoveryResult.failure,
714+
});
715+
return;
716+
}
717+
const inheritedBackgroundTurnItems = selectInheritedBackgroundItems(recoveryResult.success);
708718
const providerSessionId = providerThread.providerSessionId;
709719
const runControls = makeRunControls({
710720
threadId: projection.thread.id,

0 commit comments

Comments
 (0)