Skip to content

Commit 21baef5

Browse files
authored
fix(server): stop retrying saved checkpoints after refresh failures (pingdotgg#20195)
1 parent 976eb02 commit 21baef5

3 files changed

Lines changed: 253 additions & 8 deletions

File tree

Lines changed: 132 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,132 @@
1+
import { assert, it } from "@effect/vitest";
2+
import { CheckpointScopeId, CommandId, GitCommandError, RunId, ThreadId } from "@t3tools/contracts";
3+
import * as Effect from "effect/Effect";
4+
import * as Layer from "effect/Layer";
5+
import * as Option from "effect/Option";
6+
7+
import * as SqlitePersistence from "../persistence/Sqlite.ts";
8+
import * as PullRequestService from "../pullRequest/PullRequestService.ts";
9+
import * as ServerSettings from "../serverSettings.ts";
10+
import * as VcsStatusBroadcaster from "../vcs/VcsStatusBroadcaster.ts";
11+
import * as WorkspaceEntries from "../workspace/WorkspaceEntries.ts";
12+
import * as CheckpointCapture from "./CheckpointCaptureService.ts";
13+
import * as CheckpointRollback from "./CheckpointRollbackService.ts";
14+
import * as EffectOutbox from "./EffectOutbox.ts";
15+
import * as EffectWorker from "./EffectWorker.ts";
16+
import * as ProjectionStore from "./ProjectionStore.ts";
17+
import * as ProviderSessions from "./ProviderSessionManager.ts";
18+
import * as ProviderTurnControl from "./ProviderTurnControlService.ts";
19+
import * as ProviderTurnStart from "./ProviderTurnStartService.ts";
20+
import * as RunFinalization from "./RunFinalizationService.ts";
21+
import * as RuntimeRequests from "./RuntimeRequestService.ts";
22+
import * as ThreadManagement from "./ThreadManagementService.ts";
23+
import * as ThreadTitles from "./ThreadTitleRegenerationService.ts";
24+
25+
it.effect.each([
26+
["starts the next turn after a post-capture Git status failure", false],
27+
["keeps the next turn queued when checkpoint capture fails", true],
28+
] as const)("%s", ([, captureFails]) => {
29+
const threadId = ThreadId.make("thread:finalization-queue");
30+
const runId = RunId.make("run:finalization-queue");
31+
const scopeId = CheckpointScopeId.make("scope:finalization-queue");
32+
const calls: string[] = [];
33+
const checkpointFailure = new CheckpointCapture.CheckpointCaptureExecutionError({
34+
threadId,
35+
runId,
36+
scopeId,
37+
cause: "checkpoint storage unavailable",
38+
});
39+
const projections = Layer.mock(ProjectionStore.ProjectionStoreV2)({
40+
getCheckpointContext: () =>
41+
Effect.succeed({
42+
runs: [],
43+
checkpointScopes: [{ id: scopeId, runId, kind: "root_run" as const, cwd: "/repo" }],
44+
checkpoints: [],
45+
}),
46+
});
47+
const observer = RunFinalization.layerObserver.pipe(
48+
Layer.provide(
49+
Layer.mergeAll(
50+
projections,
51+
Layer.mock(WorkspaceEntries.WorkspaceEntries)({ refresh: () => Effect.void }),
52+
Layer.mock(PullRequestService.PullRequestService)({ refreshAfterTurn: () => Effect.void }),
53+
Layer.mock(VcsStatusBroadcaster.VcsStatusBroadcaster)({
54+
refreshLocalStatus: () =>
55+
Effect.gen(function* () {
56+
calls.push("refresh");
57+
return yield* new GitCommandError({
58+
operation: "statusDetailsLocal",
59+
command: "git",
60+
cwd: "/repo",
61+
detail: "VCS process timed out after 30000ms",
62+
});
63+
}),
64+
}),
65+
),
66+
),
67+
);
68+
const finalization = RunFinalization.layer.pipe(
69+
Layer.provide(
70+
Layer.mergeAll(
71+
projections,
72+
observer,
73+
Layer.mock(CheckpointCapture.CheckpointCaptureServiceV2)({
74+
execute: () =>
75+
Effect.gen(function* () {
76+
calls.push("capture");
77+
if (captureFails) return yield* checkpointFailure;
78+
}),
79+
}),
80+
),
81+
),
82+
);
83+
const executor = EffectWorker.layerExecutor.pipe(
84+
Layer.provide(
85+
Layer.mergeAll(
86+
finalization,
87+
Layer.mock(ProviderSessions.ProviderSessionManagerV2)({}),
88+
Layer.mock(ProviderTurnControl.ProviderTurnControlServiceV2)({}),
89+
Layer.mock(CheckpointRollback.CheckpointRollbackServiceV2)({}),
90+
Layer.mock(RuntimeRequests.RuntimeRequestServiceV2)({}),
91+
Layer.mock(ThreadTitles.ThreadTitleRegenerationService)({}),
92+
Layer.mock(ThreadManagement.ThreadManagementService)({}),
93+
ServerSettings.layerTest(),
94+
Layer.mock(ProviderTurnStart.ProviderTurnStartServiceV2)({
95+
start: () => Effect.sync(() => calls.push("start")).pipe(Effect.asVoid),
96+
}),
97+
),
98+
),
99+
);
100+
const outbox = EffectOutbox.layer.pipe(Layer.provide(SqlitePersistence.layerMemory));
101+
const worker = EffectWorker.layer.pipe(Layer.provideMerge(Layer.merge(executor, outbox)));
102+
103+
return Effect.gen(function* () {
104+
const queue = yield* EffectOutbox.EffectOutboxV2;
105+
const effectWorker = yield* EffectWorker.OrchestrationEffectWorkerV2;
106+
yield* queue.enqueue([
107+
{
108+
id: "effect:finalization-queue:capture",
109+
commandId: CommandId.make("command:finalization-queue:capture"),
110+
threadId,
111+
request: { type: "checkpoint.capture", runId, scopeId },
112+
},
113+
{
114+
id: "effect:finalization-queue:start",
115+
commandId: CommandId.make("command:finalization-queue:start"),
116+
threadId,
117+
request: { type: "provider-turn.start", runId: RunId.make("run:next") },
118+
},
119+
]);
120+
121+
assert.equal(yield* effectWorker.drain(2), captureFails ? 1 : 2);
122+
const captured = Option.getOrThrow(yield* queue.get("effect:finalization-queue:capture"));
123+
const started = Option.getOrThrow(yield* queue.get("effect:finalization-queue:start"));
124+
assert.equal(captured.status, captureFails ? "pending" : "succeeded");
125+
assert.equal(captured.attemptCount, 1);
126+
assert.equal(started.status, captureFails ? "pending" : "succeeded");
127+
assert.equal(started.attemptCount, captureFails ? 0 : 1);
128+
assert.deepEqual(calls, captureFails ? ["capture"] : ["capture", "refresh", "start"]);
129+
if (captureFails) assert.include(captured.lastError ?? "", "checkpoint storage unavailable");
130+
else assert.equal(captured.lastError, null);
131+
}).pipe(Effect.provide(Layer.fresh(worker)));
132+
});

‎apps/server/src/orchestration-v2/RunFinalizationService.test.ts‎

Lines changed: 111 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,117 @@ it.effect("refreshes workspace after checkpoint capture without reading history"
5050
}).pipe(Effect.provide(layer));
5151
});
5252

53+
it.effect("completes finalization when the post-checkpoint workspace refresh fails", () => {
54+
const threadId = ThreadId.make("thread_refresh_failure");
55+
const runId = RunId.make("run_refresh_failure");
56+
const scopeId = CheckpointScopeId.make("scope_refresh_failure");
57+
const captured: string[] = [];
58+
const layer = RunFinalization.layer.pipe(
59+
Layer.provide(
60+
Layer.mergeAll(
61+
Layer.mock(CheckpointCapture.CheckpointCaptureServiceV2)({
62+
execute: () =>
63+
Effect.sync(() => {
64+
captured.push("checkpoint");
65+
}),
66+
}),
67+
Layer.mock(ProjectionStore.ProjectionStoreV2)({
68+
getCheckpointContext: () =>
69+
Effect.succeed({
70+
runs: [],
71+
checkpointScopes: [{ id: scopeId, runId, kind: "root_run" as const, cwd: "/repo" }],
72+
checkpoints: [],
73+
}),
74+
}),
75+
Layer.succeed(RunFinalization.RunFinalizationObserver, {
76+
refresh: () =>
77+
Effect.gen(function* () {
78+
captured.push("refresh");
79+
return yield* new RunFinalization.RunFinalizationRefreshError({
80+
cwd: "/repo",
81+
cause: new Error("Git command timed out"),
82+
});
83+
}),
84+
refreshAfterTurn: () => Effect.void,
85+
}),
86+
),
87+
),
88+
);
89+
return Effect.gen(function* () {
90+
const service = yield* RunFinalization.RunFinalizationService;
91+
yield* service.finalize({ threadId, runId, scopeId });
92+
assert.deepEqual(captured, ["checkpoint", "refresh"]);
93+
}).pipe(Effect.provide(layer));
94+
});
95+
96+
it.effect("preserves checkpoint failures without refreshing the workspace", () => {
97+
const threadId = ThreadId.make("thread_capture_failure");
98+
const runId = RunId.make("run_capture_failure");
99+
const scopeId = CheckpointScopeId.make("scope_capture_failure");
100+
const captureError = new CheckpointCapture.CheckpointCaptureExecutionError({
101+
threadId,
102+
runId,
103+
scopeId,
104+
cause: new Error("Checkpoint capture failed"),
105+
});
106+
const refresh = vi.fn(() => Effect.void);
107+
const layer = RunFinalization.layer.pipe(
108+
Layer.provide(
109+
Layer.mergeAll(
110+
Layer.mock(CheckpointCapture.CheckpointCaptureServiceV2)({
111+
execute: () => Effect.fail(captureError),
112+
}),
113+
Layer.mock(ProjectionStore.ProjectionStoreV2)({
114+
getCheckpointContext: () => Effect.die("capture failed before context lookup"),
115+
}),
116+
Layer.succeed(RunFinalization.RunFinalizationObserver, {
117+
refresh,
118+
refreshAfterTurn: () => Effect.void,
119+
}),
120+
),
121+
),
122+
);
123+
return Effect.gen(function* () {
124+
const service = yield* RunFinalization.RunFinalizationService;
125+
const error = yield* service.finalize({ threadId, runId, scopeId }).pipe(Effect.flip);
126+
assert.equal(error.operation, "capture-checkpoint");
127+
assert.strictEqual(error.cause, captureError);
128+
assert.equal(refresh.mock.calls.length, 0);
129+
}).pipe(Effect.provide(layer));
130+
});
131+
132+
it.effect("preserves checkpoint context read failures without refreshing the workspace", () => {
133+
const threadId = ThreadId.make("thread_context_failure");
134+
const runId = RunId.make("run_context_failure");
135+
const scopeId = CheckpointScopeId.make("scope_context_failure");
136+
const contextError = new ProjectionStore.ProjectionStoreReadError({
137+
threadId,
138+
cause: new Error("Checkpoint context read failed"),
139+
});
140+
const refresh = vi.fn(() => Effect.void);
141+
const layer = RunFinalization.layer.pipe(
142+
Layer.provide(
143+
Layer.mergeAll(
144+
Layer.mock(CheckpointCapture.CheckpointCaptureServiceV2)({ execute: () => Effect.void }),
145+
Layer.mock(ProjectionStore.ProjectionStoreV2)({
146+
getCheckpointContext: () => Effect.fail(contextError),
147+
}),
148+
Layer.succeed(RunFinalization.RunFinalizationObserver, {
149+
refresh,
150+
refreshAfterTurn: () => Effect.void,
151+
}),
152+
),
153+
),
154+
);
155+
return Effect.gen(function* () {
156+
const service = yield* RunFinalization.RunFinalizationService;
157+
const error = yield* service.finalize({ threadId, runId, scopeId }).pipe(Effect.flip);
158+
assert.equal(error.operation, "refresh-workspace");
159+
assert.strictEqual(error.cause, contextError);
160+
assert.equal(refresh.mock.calls.length, 0);
161+
}).pipe(Effect.provide(layer));
162+
});
163+
53164
it.effect.each(
54165
(
55166
[

‎apps/server/src/orchestration-v2/RunFinalizationService.ts‎

Lines changed: 10 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -72,14 +72,16 @@ const make = Effect.gen(function* () {
7272
);
7373
const cwd = projection.checkpointScopes.find((scope) => scope.id === input.scopeId)?.cwd;
7474
if (cwd !== undefined) {
75-
yield* observer
76-
.refresh({ cwd, threadId: input.threadId, runId: input.runId })
77-
.pipe(
78-
Effect.mapError(
79-
(cause) =>
80-
new RunFinalizationError({ ...input, operation: "refresh-workspace", cause }),
81-
),
82-
);
75+
// Retrying a display refresh after the checkpoint is durable blocks the next turn.
76+
yield* observer.refresh({ cwd, threadId: input.threadId, runId: input.runId }).pipe(
77+
Effect.catchTags({
78+
RunFinalizationRefreshError: (cause) =>
79+
Effect.logWarning("Failed to refresh workspace after checkpoint capture", {
80+
...input,
81+
cause,
82+
}),
83+
}),
84+
);
8385
}
8486
});
8587
return RunFinalizationService.of({ finalize });

0 commit comments

Comments
 (0)