Skip to content

Commit bfad1fa

Browse files
radroidclaude
andcommitted
fix(server): reconcile crash-frozen running turns to interrupted on boot
Fixes #5. When the backend exits ungracefully (SIGKILL/crash/OOM, no SIGTERM), the Effect finalizer a clean quit runs (ProviderService.runStopAll -> adapter.stopAll() -> a session-set that settles running turns to "interrupted") never fires. In-flight turns are left frozen at projection_turns.state="running" with active_turn_id still set; on a later resume the provider reports the cut-off turn as cancelled/aborted, and auto-resume never recovers them. At fresh boot no turn is actually live (adapter session maps start empty; runtime rows carry no pid/epoch), so every leftover in-flight turn is a crash remnant. Add a one-shot boot reconciler that mirrors the graceful path: for each such thread it dispatches a stopped `thread.session.set` command (through the event log, not a direct projection write), settling the running turn to the resumable "interrupted" state and clearing active_turn_id. Wired as a startup phase before reactors start and before commands are accepted. Scope: reconciling to a resumable interrupted state satisfies the issue's primary acceptance. Auto-resume RE-ARMING (a crash marker + boot producer to auto-continue the turn) is left as a documented follow-up. Adds CrashRecoveryReconciler.test.ts (precondition, reconcile, idempotency, multi-thread selectivity, event-log backing) and extends serverRuntimeStartup and autoResume/guards tests. Co-Authored-By: Claude Opus 4.8 <[email protected]>
1 parent c3ed80e commit bfad1fa

5 files changed

Lines changed: 625 additions & 1 deletion

File tree

Lines changed: 296 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,296 @@
1+
import {
2+
CommandId,
3+
DEFAULT_PROVIDER_INTERACTION_MODE,
4+
ProjectId,
5+
ProviderInstanceId,
6+
ThreadId,
7+
TurnId,
8+
type OrchestrationCommand,
9+
type OrchestrationEvent,
10+
} from "@t3tools/contracts";
11+
import * as NodeServices from "@effect/platform-node/NodeServices";
12+
import * as Effect from "effect/Effect";
13+
import * as Layer from "effect/Layer";
14+
import * as ManagedRuntime from "effect/ManagedRuntime";
15+
import * as Stream from "effect/Stream";
16+
import { describe, expect, it } from "vite-plus/test";
17+
18+
import { OrchestrationCommandReceiptRepositoryLive } from "../../persistence/Layers/OrchestrationCommandReceipts.ts";
19+
import { OrchestrationEventStoreLive } from "../../persistence/Layers/OrchestrationEventStore.ts";
20+
import { ProjectionTurnRepositoryLive } from "../../persistence/Layers/ProjectionTurns.ts";
21+
import { SqlitePersistenceMemory } from "../../persistence/Layers/Sqlite.ts";
22+
import { ProjectionTurnRepository } from "../../persistence/Services/ProjectionTurns.ts";
23+
import * as RepositoryIdentityResolver from "../../project/RepositoryIdentityResolver.ts";
24+
import { ServerConfig } from "../../config.ts";
25+
import { OrchestrationEngineLive } from "./OrchestrationEngine.ts";
26+
import { OrchestrationProjectionPipelineLive } from "./ProjectionPipeline.ts";
27+
import { OrchestrationProjectionSnapshotQueryLive } from "./ProjectionSnapshotQuery.ts";
28+
import { OrchestrationEngineService } from "../Services/OrchestrationEngine.ts";
29+
import { ProjectionSnapshotQuery } from "../Services/ProjectionSnapshotQuery.ts";
30+
import { reconcileInterruptedTurnsOnBoot } from "./CrashRecoveryReconciler.ts";
31+
32+
const MODEL_SELECTION = {
33+
instanceId: ProviderInstanceId.make("codex"),
34+
model: "gpt-5-codex",
35+
};
36+
37+
async function createReconcilerSystem() {
38+
const ServerConfigLayer = ServerConfig.layerTest(process.cwd(), {
39+
prefix: "t3-crash-recovery-reconciler-test-",
40+
});
41+
const orchestrationLayer = Layer.mergeAll(
42+
OrchestrationEngineLive.pipe(
43+
Layer.provide(OrchestrationProjectionSnapshotQueryLive),
44+
Layer.provide(OrchestrationProjectionPipelineLive),
45+
),
46+
OrchestrationProjectionSnapshotQueryLive,
47+
ProjectionTurnRepositoryLive,
48+
).pipe(
49+
Layer.provide(OrchestrationEventStoreLive),
50+
Layer.provide(OrchestrationCommandReceiptRepositoryLive),
51+
Layer.provide(RepositoryIdentityResolver.layer),
52+
Layer.provide(SqlitePersistenceMemory),
53+
Layer.provideMerge(ServerConfigLayer),
54+
Layer.provideMerge(NodeServices.layer),
55+
);
56+
const runtime = ManagedRuntime.make(orchestrationLayer);
57+
const engine = await runtime.runPromise(Effect.service(OrchestrationEngineService));
58+
const snapshotQuery = await runtime.runPromise(Effect.service(ProjectionSnapshotQuery));
59+
const turnRepository = await runtime.runPromise(Effect.service(ProjectionTurnRepository));
60+
61+
return {
62+
dispatch: (command: OrchestrationCommand) => runtime.runPromise(engine.dispatch(command)),
63+
readModel: () => runtime.runPromise(snapshotQuery.getSnapshot()),
64+
reconcile: () => runtime.runPromise(reconcileInterruptedTurnsOnBoot),
65+
getTurn: (threadId: string, turnId: string) =>
66+
runtime.runPromise(
67+
turnRepository.getByTurnId({
68+
threadId: ThreadId.make(threadId),
69+
turnId: TurnId.make(turnId),
70+
}),
71+
),
72+
readEvents: () =>
73+
runtime.runPromise(
74+
Stream.runCollect(engine.readEvents(0)).pipe(
75+
Effect.map((chunk): OrchestrationEvent[] => Array.from(chunk)),
76+
),
77+
),
78+
dispose: () => runtime.dispose(),
79+
};
80+
}
81+
82+
type ReconcilerSystem = Awaited<ReturnType<typeof createReconcilerSystem>>;
83+
84+
async function seedProject(system: ReconcilerSystem, projectId: string) {
85+
await system.dispatch({
86+
type: "project.create",
87+
commandId: CommandId.make(`cmd-project-${projectId}`),
88+
projectId: ProjectId.make(projectId),
89+
title: `Project ${projectId}`,
90+
workspaceRoot: `/tmp/${projectId}`,
91+
defaultModelSelection: MODEL_SELECTION,
92+
createdAt: "2026-01-01T00:00:00.000Z",
93+
});
94+
}
95+
96+
async function seedThread(system: ReconcilerSystem, projectId: string, threadId: string) {
97+
await system.dispatch({
98+
type: "thread.create",
99+
commandId: CommandId.make(`cmd-thread-${threadId}`),
100+
threadId: ThreadId.make(threadId),
101+
projectId: ProjectId.make(projectId),
102+
title: `Thread ${threadId}`,
103+
modelSelection: MODEL_SELECTION,
104+
interactionMode: DEFAULT_PROVIDER_INTERACTION_MODE,
105+
runtimeMode: "full-access",
106+
branch: null,
107+
worktreePath: null,
108+
createdAt: "2026-01-01T00:00:01.000Z",
109+
});
110+
}
111+
112+
async function setSession(
113+
system: ReconcilerSystem,
114+
threadId: string,
115+
status: "running" | "stopped",
116+
activeTurnId: string | null,
117+
updatedAt: string,
118+
) {
119+
await system.dispatch({
120+
type: "thread.session.set",
121+
commandId: CommandId.make(`cmd-session-${threadId}-${status}-${updatedAt}`),
122+
threadId: ThreadId.make(threadId),
123+
session: {
124+
threadId: ThreadId.make(threadId),
125+
status,
126+
providerName: "claudeAgent",
127+
providerInstanceId: ProviderInstanceId.make("codex"),
128+
runtimeMode: "full-access",
129+
activeTurnId: activeTurnId === null ? null : TurnId.make(activeTurnId),
130+
lastError: null,
131+
updatedAt,
132+
},
133+
createdAt: updatedAt,
134+
});
135+
}
136+
137+
/** Seed a thread frozen mid-turn the way an ungraceful crash leaves it. */
138+
async function seedCrashedThread(
139+
system: ReconcilerSystem,
140+
projectId: string,
141+
threadId: string,
142+
turnId: string,
143+
) {
144+
await seedThread(system, projectId, threadId);
145+
await setSession(system, threadId, "running", turnId, "2026-01-01T00:00:02.000Z");
146+
}
147+
148+
describe("reconcileInterruptedTurnsOnBoot", () => {
149+
it("leaves a crashed thread with a running turn / active session (precondition)", async () => {
150+
const system = await createReconcilerSystem();
151+
try {
152+
await seedProject(system, "project-1");
153+
await seedCrashedThread(system, "project-1", "thread-1", "turn-1");
154+
155+
const snapshot = await system.readModel();
156+
const thread = snapshot.threads.find((entry) => entry.id === "thread-1");
157+
expect(thread?.latestTurn?.state).toBe("running");
158+
expect(thread?.session?.status).toBe("running");
159+
expect(thread?.session?.activeTurnId).toBe("turn-1");
160+
161+
const turn = await system.getTurn("thread-1", "turn-1");
162+
expect(turn._tag).toBe("Some");
163+
if (turn._tag === "Some") {
164+
expect(turn.value.state).toBe("running");
165+
expect(turn.value.completedAt).toBeNull();
166+
}
167+
} finally {
168+
await system.dispose();
169+
}
170+
});
171+
172+
it("settles a crashed thread to interrupted and stops the session", async () => {
173+
const system = await createReconcilerSystem();
174+
try {
175+
await seedProject(system, "project-1");
176+
await seedCrashedThread(system, "project-1", "thread-1", "turn-1");
177+
178+
const result = await system.reconcile();
179+
expect(result.reconciledCount).toBe(1);
180+
181+
const snapshot = await system.readModel();
182+
const thread = snapshot.threads.find((entry) => entry.id === "thread-1");
183+
expect(thread?.session?.status).toBe("stopped");
184+
expect(thread?.session?.activeTurnId).toBeNull();
185+
186+
const turn = await system.getTurn("thread-1", "turn-1");
187+
expect(turn._tag).toBe("Some");
188+
if (turn._tag === "Some") {
189+
expect(turn.value.state).toBe("interrupted");
190+
expect(turn.value.completedAt).not.toBeNull();
191+
}
192+
} finally {
193+
await system.dispose();
194+
}
195+
});
196+
197+
it("is idempotent: a second boot reconciles nothing and leaves projections unchanged", async () => {
198+
const system = await createReconcilerSystem();
199+
try {
200+
await seedProject(system, "project-1");
201+
await seedCrashedThread(system, "project-1", "thread-1", "turn-1");
202+
203+
const first = await system.reconcile();
204+
expect(first.reconciledCount).toBe(1);
205+
206+
const eventsAfterFirst = await system.readEvents();
207+
const snapshotAfterFirst = await system.readModel();
208+
209+
const second = await system.reconcile();
210+
expect(second.reconciledCount).toBe(0);
211+
212+
const eventsAfterSecond = await system.readEvents();
213+
expect(eventsAfterSecond.length).toBe(eventsAfterFirst.length);
214+
215+
const snapshotAfterSecond = await system.readModel();
216+
const threadFirst = snapshotAfterFirst.threads.find((entry) => entry.id === "thread-1");
217+
const threadSecond = snapshotAfterSecond.threads.find((entry) => entry.id === "thread-1");
218+
expect(threadSecond?.session?.status).toBe(threadFirst?.session?.status);
219+
expect(threadSecond?.session?.activeTurnId).toBe(threadFirst?.session?.activeTurnId ?? null);
220+
221+
const turn = await system.getTurn("thread-1", "turn-1");
222+
if (turn._tag === "Some") {
223+
expect(turn.value.state).toBe("interrupted");
224+
}
225+
} finally {
226+
await system.dispose();
227+
}
228+
});
229+
230+
it("only reconciles crashed threads, not already-settled ones (multi-thread selectivity)", async () => {
231+
const system = await createReconcilerSystem();
232+
try {
233+
await seedProject(system, "project-1");
234+
await seedCrashedThread(system, "project-1", "thread-a", "turn-a");
235+
await seedCrashedThread(system, "project-1", "thread-b", "turn-b");
236+
237+
// An already-settled thread: it ran a turn and the session was cleanly stopped.
238+
await seedThread(system, "project-1", "thread-settled");
239+
await setSession(system, "thread-settled", "running", "turn-s", "2026-01-01T00:00:03.000Z");
240+
await setSession(system, "thread-settled", "stopped", null, "2026-01-01T00:00:04.000Z");
241+
242+
const settledTurnBefore = await system.getTurn("thread-settled", "turn-s");
243+
expect(settledTurnBefore._tag).toBe("Some");
244+
245+
const result = await system.reconcile();
246+
expect(result.reconciledCount).toBe(2);
247+
248+
const crashedTurns: ReadonlyArray<readonly [string, string]> = [
249+
["thread-a", "turn-a"],
250+
["thread-b", "turn-b"],
251+
];
252+
for (const [threadId, turnId] of crashedTurns) {
253+
const turn = await system.getTurn(threadId, turnId);
254+
if (turn._tag === "Some") {
255+
expect(turn.value.state).toBe("interrupted");
256+
expect(turn.value.completedAt).not.toBeNull();
257+
}
258+
}
259+
260+
// The already-settled thread must be untouched (same completedAt).
261+
const settledTurnAfter = await system.getTurn("thread-settled", "turn-s");
262+
if (settledTurnBefore._tag === "Some" && settledTurnAfter._tag === "Some") {
263+
expect(settledTurnAfter.value.state).toBe("interrupted");
264+
expect(settledTurnAfter.value.completedAt).toBe(settledTurnBefore.value.completedAt);
265+
}
266+
} finally {
267+
await system.dispose();
268+
}
269+
});
270+
271+
it("appends a durable thread.session-set event that survives a projection rebuild", async () => {
272+
const system = await createReconcilerSystem();
273+
try {
274+
await seedProject(system, "project-1");
275+
await seedCrashedThread(system, "project-1", "thread-1", "turn-1");
276+
277+
await system.reconcile();
278+
279+
const events = await system.readEvents();
280+
const stopEvents = events.filter(
281+
(event): event is Extract<OrchestrationEvent, { type: "thread.session-set" }> =>
282+
event.type === "thread.session-set" &&
283+
event.payload.threadId === "thread-1" &&
284+
event.payload.session.status === "stopped",
285+
);
286+
expect(stopEvents.length).toBe(1);
287+
const stopEvent = stopEvents[0];
288+
if (stopEvent) {
289+
expect(stopEvent.payload.session.activeTurnId).toBeNull();
290+
expect(String(stopEvent.commandId).startsWith("t3-crash-recovery:")).toBe(true);
291+
}
292+
} finally {
293+
await system.dispose();
294+
}
295+
});
296+
});

0 commit comments

Comments
 (0)