Skip to content

Commit 16271a3

Browse files
committed
fix(server): reconcile active sessions after restart
1 parent f708f63 commit 16271a3

4 files changed

Lines changed: 300 additions & 3 deletions

File tree

‎apps/server/src/orchestration/Layers/ProviderCommandReactor.test.ts‎

Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2653,6 +2653,63 @@ describe("ProviderCommandReactor", () => {
26532653
});
26542654
});
26552655

2656+
it("marks approval responses stale when the provider session is stopped", async () => {
2657+
const harness = await createHarness();
2658+
const now = "2026-01-01T00:00:00.000Z";
2659+
2660+
await harness.runEffect(
2661+
harness.engine.dispatch({
2662+
type: "thread.session.set",
2663+
commandId: CommandId.make("cmd-session-set-for-stale-approval"),
2664+
threadId: ThreadId.make("thread-1"),
2665+
session: {
2666+
threadId: ThreadId.make("thread-1"),
2667+
status: "stopped",
2668+
providerName: "codex",
2669+
runtimeMode: "approval-required",
2670+
activeTurnId: null,
2671+
lastError: null,
2672+
updatedAt: now,
2673+
},
2674+
createdAt: now,
2675+
}),
2676+
);
2677+
await harness.runEffect(
2678+
harness.engine.dispatch({
2679+
type: "thread.approval.respond",
2680+
commandId: CommandId.make("cmd-stale-approval-response"),
2681+
threadId: ThreadId.make("thread-1"),
2682+
requestId: asApprovalRequestId("approval-request-stopped"),
2683+
decision: "accept",
2684+
createdAt: now,
2685+
}),
2686+
);
2687+
2688+
await waitFor(async () => {
2689+
const thread = (await harness.readModel()).threads.find(
2690+
(entry) => entry.id === ThreadId.make("thread-1"),
2691+
);
2692+
return (
2693+
thread?.activities.some(
2694+
(activity) => activity.kind === "provider.approval.respond.failed",
2695+
) ?? false
2696+
);
2697+
});
2698+
expect(harness.respondToRequest).not.toHaveBeenCalled();
2699+
const thread = (await harness.readModel()).threads.find(
2700+
(entry) => entry.id === ThreadId.make("thread-1"),
2701+
);
2702+
expect(thread?.session?.status).toBe("stopped");
2703+
expect(
2704+
thread?.activities.find((activity) => activity.kind === "provider.approval.respond.failed"),
2705+
).toMatchObject({
2706+
payload: {
2707+
requestId: "approval-request-stopped",
2708+
detail: expect.stringContaining("Stale pending approval request: approval-request-stopped"),
2709+
},
2710+
});
2711+
});
2712+
26562713
it("reacts to thread.user-input.respond by forwarding structured user input answers", async () => {
26572714
const harness = await createHarness();
26582715
const now = "2026-01-01T00:00:00.000Z";
@@ -2698,6 +2755,65 @@ describe("ProviderCommandReactor", () => {
26982755
});
26992756
});
27002757

2758+
it("marks user input responses stale when the provider session is stopped", async () => {
2759+
const harness = await createHarness();
2760+
const now = "2026-01-01T00:00:00.000Z";
2761+
2762+
await harness.runEffect(
2763+
harness.engine.dispatch({
2764+
type: "thread.session.set",
2765+
commandId: CommandId.make("cmd-session-set-for-stale-user-input"),
2766+
threadId: ThreadId.make("thread-1"),
2767+
session: {
2768+
threadId: ThreadId.make("thread-1"),
2769+
status: "stopped",
2770+
providerName: "codex",
2771+
runtimeMode: "approval-required",
2772+
activeTurnId: null,
2773+
lastError: null,
2774+
updatedAt: now,
2775+
},
2776+
createdAt: now,
2777+
}),
2778+
);
2779+
await harness.runEffect(
2780+
harness.engine.dispatch({
2781+
type: "thread.user-input.respond",
2782+
commandId: CommandId.make("cmd-stale-user-input-response"),
2783+
threadId: ThreadId.make("thread-1"),
2784+
requestId: asApprovalRequestId("user-input-request-stopped"),
2785+
answers: { sandbox_mode: "workspace-write" },
2786+
createdAt: now,
2787+
}),
2788+
);
2789+
2790+
await waitFor(async () => {
2791+
const thread = (await harness.readModel()).threads.find(
2792+
(entry) => entry.id === ThreadId.make("thread-1"),
2793+
);
2794+
return (
2795+
thread?.activities.some(
2796+
(activity) => activity.kind === "provider.user-input.respond.failed",
2797+
) ?? false
2798+
);
2799+
});
2800+
expect(harness.respondToUserInput).not.toHaveBeenCalled();
2801+
const thread = (await harness.readModel()).threads.find(
2802+
(entry) => entry.id === ThreadId.make("thread-1"),
2803+
);
2804+
expect(thread?.session?.status).toBe("stopped");
2805+
expect(
2806+
thread?.activities.find((activity) => activity.kind === "provider.user-input.respond.failed"),
2807+
).toMatchObject({
2808+
payload: {
2809+
requestId: "user-input-request-stopped",
2810+
detail: expect.stringContaining(
2811+
"Stale pending user-input request: user-input-request-stopped",
2812+
),
2813+
},
2814+
});
2815+
});
2816+
27012817
it("surfaces stale provider approval request failures without faking approval resolution", async () => {
27022818
const harness = await createHarness();
27032819
const now = "2026-01-01T00:00:00.000Z";

‎apps/server/src/orchestration/Layers/ProviderCommandReactor.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1209,7 +1209,7 @@ const make = Effect.gen(function* () {
12091209
threadId: event.payload.threadId,
12101210
kind: "provider.approval.respond.failed",
12111211
summary: "Provider approval response failed",
1212-
detail: "No active provider session is bound to this thread.",
1212+
detail: stalePendingRequestDetail("approval", event.payload.requestId),
12131213
turnId: null,
12141214
createdAt: event.payload.createdAt,
12151215
requestId: event.payload.requestId,
@@ -1253,7 +1253,7 @@ const make = Effect.gen(function* () {
12531253
threadId: event.payload.threadId,
12541254
kind: "provider.user-input.respond.failed",
12551255
summary: "Provider user input response failed",
1256-
detail: "No active provider session is bound to this thread.",
1256+
detail: stalePendingRequestDetail("user-input", event.payload.requestId),
12571257
turnId: null,
12581258
createdAt: event.payload.createdAt,
12591259
requestId: event.payload.requestId,

‎apps/server/src/serverRuntimeStartup.test.ts‎

Lines changed: 143 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,15 @@
11
import * as NodeServices from "@effect/platform-node/NodeServices";
2-
import { DEFAULT_MODEL, ProjectId, ProviderInstanceId, ThreadId } from "@t3tools/contracts";
2+
import {
3+
DEFAULT_MODEL,
4+
type OrchestrationCommand,
5+
type OrchestrationSession,
6+
type OrchestrationThread,
7+
ProjectId,
8+
ProviderDriverKind,
9+
ProviderInstanceId,
10+
ThreadId,
11+
TurnId,
12+
} from "@t3tools/contracts";
313
import { assert, it } from "@effect/vitest";
414
import * as Crypto from "effect/Crypto";
515
import * as Deferred from "effect/Deferred";
@@ -11,11 +21,40 @@ import * as Ref from "effect/Ref";
1121
import * as Stream from "effect/Stream";
1222

1323
import * as ServerConfig from "./config.ts";
24+
import { OrchestrationCommandInvariantError } from "./orchestration/Errors.ts";
1425
import * as OrchestrationEngine from "./orchestration/Services/OrchestrationEngine.ts";
1526
import * as ProjectionSnapshotQuery from "./orchestration/Services/ProjectionSnapshotQuery.ts";
27+
import * as ProviderSessionDirectory from "./provider/Services/ProviderSessionDirectory.ts";
28+
import { ProviderSessionDirectoryPersistenceError } from "./provider/Errors.ts";
1629
import * as AnalyticsService from "./telemetry/AnalyticsService.ts";
1730
import * as ServerRuntimeStartup from "./serverRuntimeStartup.ts";
1831

32+
const sessionUpdatedAt = "2026-01-01T00:00:00.000Z";
33+
34+
function sessionThread(
35+
threadId: ThreadId,
36+
status: OrchestrationSession["status"],
37+
activeTurnId: TurnId | null = null,
38+
): Pick<OrchestrationThread, "id" | "session" | "deletedAt"> {
39+
return {
40+
id: threadId,
41+
deletedAt: null,
42+
session: {
43+
threadId,
44+
status,
45+
providerName: "codex" as const,
46+
runtimeMode: "full-access" as const,
47+
activeTurnId,
48+
lastError: null,
49+
updatedAt: sessionUpdatedAt,
50+
},
51+
};
52+
}
53+
54+
const queryWithThreads = (
55+
threads: ReadonlyArray<Pick<OrchestrationThread, "id" | "session" | "deletedAt">>,
56+
) => ({ getCommandReadModel: () => Effect.succeed({ threads }) }) as never;
57+
1958
it("uses the canonical Codex default for auto-bootstrapped model selection", () => {
2059
assert.deepStrictEqual(ServerRuntimeStartup.getAutoBootstrapDefaultModelSelection(), {
2160
instanceId: ProviderInstanceId.make("codex"),
@@ -70,6 +109,109 @@ it.effect("enqueueCommand fails queued work when readiness fails", () =>
70109
),
71110
);
72111

112+
it.effect("reconciles provider sessions inherited from the previous server", () => {
113+
const threadId = ThreadId.make("thread-startup-reconcile");
114+
const startingThreadId = ThreadId.make("thread-startup-starting");
115+
const resumeCursor = { schemaVersion: 1, sessionId: "session-startup-reconcile" };
116+
const bindingFailure = new ProviderSessionDirectoryPersistenceError({
117+
operation: "ProviderSessionDirectory.getBinding",
118+
detail: "simulated invalid binding",
119+
});
120+
const bindingWriteFailure = new ProviderSessionDirectoryPersistenceError({
121+
operation: "ProviderSessionDirectory.upsert",
122+
detail: "simulated binding write failure",
123+
});
124+
const upserts: Array<ProviderSessionDirectory.ProviderRuntimeBinding> = [];
125+
const dispatches: Array<OrchestrationCommand> = [];
126+
127+
return ServerRuntimeStartup.reconcileProviderSessions.pipe(
128+
Effect.provideService(
129+
ProjectionSnapshotQuery.ProjectionSnapshotQuery,
130+
queryWithThreads([
131+
{
132+
...sessionThread(threadId, "running", TurnId.make("turn-startup-reconcile")),
133+
deletedAt: "2026-01-02T00:00:00.000Z",
134+
},
135+
sessionThread(startingThreadId, "starting"),
136+
sessionThread(ThreadId.make("thread-startup-ready"), "ready"),
137+
]),
138+
),
139+
Effect.provideService(ProviderSessionDirectory.ProviderSessionDirectory, {
140+
getBinding: (candidate: ThreadId) =>
141+
candidate === threadId
142+
? Effect.succeed(
143+
Option.some({
144+
threadId,
145+
provider: ProviderDriverKind.make("codex"),
146+
providerInstanceId: ProviderInstanceId.make("codex"),
147+
status: "running",
148+
resumeCursor,
149+
}),
150+
)
151+
: Effect.fail(bindingFailure),
152+
upsert: (binding: ProviderSessionDirectory.ProviderRuntimeBinding) =>
153+
Effect.sync(() => upserts.push(binding)).pipe(
154+
Effect.andThen(Effect.fail(bindingWriteFailure)),
155+
),
156+
} satisfies Partial<ProviderSessionDirectory.ProviderSessionDirectory["Service"]> as never),
157+
Effect.provideService(OrchestrationEngine.OrchestrationEngineService, {
158+
dispatch: (command: OrchestrationCommand) =>
159+
Effect.sync(() => dispatches.push(command)).pipe(Effect.as({ sequence: 1 })),
160+
} satisfies Partial<OrchestrationEngine.OrchestrationEngineService["Service"]> as never),
161+
Effect.provide(NodeServices.layer),
162+
Effect.tap(() =>
163+
Effect.sync(() => {
164+
assert.equal(upserts.length, 1);
165+
assert.equal(upserts[0]?.status, "stopped");
166+
assert.deepStrictEqual(upserts[0]?.resumeCursor, resumeCursor);
167+
assert.deepStrictEqual(upserts[0]?.runtimePayload, { activeTurnId: null });
168+
169+
assert.equal(dispatches.length, 2);
170+
const command = dispatches[0];
171+
assert.equal(command?.type, "thread.session.set");
172+
if (command?.type === "thread.session.set") {
173+
assert.equal(command.threadId, threadId);
174+
assert.equal(command.session.status, "stopped");
175+
assert.equal(command.session.activeTurnId, null);
176+
assert.notEqual(command.session.updatedAt, sessionUpdatedAt);
177+
assert.equal(command.session.updatedAt, command.createdAt);
178+
}
179+
const startingCommand = dispatches[1];
180+
assert.equal(startingCommand?.type, "thread.session.set");
181+
if (startingCommand?.type === "thread.session.set") {
182+
assert.equal(startingCommand.threadId, startingThreadId);
183+
}
184+
}),
185+
),
186+
);
187+
});
188+
189+
it.effect("fails startup reconciliation when a session cannot be settled", () => {
190+
const threadId = ThreadId.make("thread-startup-reconcile-failure");
191+
const failure = new OrchestrationCommandInvariantError({
192+
commandType: "thread.session.set",
193+
detail: "simulated reconciliation failure",
194+
});
195+
196+
return ServerRuntimeStartup.reconcileProviderSessions.pipe(
197+
Effect.provideService(
198+
ProjectionSnapshotQuery.ProjectionSnapshotQuery,
199+
queryWithThreads([
200+
sessionThread(threadId, "running", TurnId.make("turn-startup-reconcile-failure")),
201+
]),
202+
),
203+
Effect.provideService(ProviderSessionDirectory.ProviderSessionDirectory, {
204+
getBinding: () => Effect.succeed(Option.none()),
205+
} satisfies Partial<ProviderSessionDirectory.ProviderSessionDirectory["Service"]> as never),
206+
Effect.provideService(OrchestrationEngine.OrchestrationEngineService, {
207+
dispatch: () => Effect.fail(failure),
208+
} satisfies Partial<OrchestrationEngine.OrchestrationEngineService["Service"]> as never),
209+
Effect.provide(NodeServices.layer),
210+
Effect.flip,
211+
Effect.tap((error) => Effect.sync(() => assert.equal(error, failure))),
212+
);
213+
});
214+
73215
it.effect("launchStartupHeartbeat does not block the caller while counts are loading", () =>
74216
Effect.scoped(
75217
Effect.gen(function* () {

‎apps/server/src/serverRuntimeStartup.ts‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ import * as ServerSettings from "./serverSettings.ts";
3333
import * as AnalyticsService from "./telemetry/AnalyticsService.ts";
3434
import * as ServerEnvironment from "./environment/ServerEnvironment.ts";
3535
import * as EnvironmentAuth from "./auth/EnvironmentAuth.ts";
36+
import * as ProviderSessionDirectory from "./provider/Services/ProviderSessionDirectory.ts";
3637
import * as ProviderSessionReaper from "./provider/Services/ProviderSessionReaper.ts";
3738
import { forkParked } from "./serverActivation.ts";
3839
import * as ServiceLauncherClient from "./cloud/serviceLauncherClient.ts";
@@ -290,6 +291,42 @@ const runStartupPhase = <A, E, R>(phase: string, effect: Effect.Effect<A, E, R>)
290291
Effect.withSpan(`server.startup.${phase}`),
291292
);
292293

294+
export const reconcileProviderSessions = Effect.gen(function* () {
295+
const directory = yield* ProviderSessionDirectory.ProviderSessionDirectory;
296+
const dispatch = (yield* OrchestrationEngine.OrchestrationEngineService).dispatch;
297+
const query = yield* ProjectionSnapshotQuery.ProjectionSnapshotQuery;
298+
for (const { id, session } of (yield* query.getCommandReadModel()).threads) {
299+
if (session?.status !== "running" && session?.status !== "starting") continue;
300+
301+
yield* Effect.gen(function* () {
302+
const binding = Option.getOrUndefined(yield* directory.getBinding(id));
303+
if (binding) {
304+
yield* directory.upsert({
305+
...binding,
306+
status: "stopped",
307+
runtimePayload: { activeTurnId: null },
308+
});
309+
}
310+
}).pipe(
311+
Effect.catch((error) =>
312+
Effect.logWarning("provider session startup reconciliation failed to reconcile binding", {
313+
error,
314+
threadId: id,
315+
}),
316+
),
317+
);
318+
319+
const reconciledAt = DateTime.formatIso(yield* DateTime.now);
320+
yield* dispatch({
321+
type: "thread.session.set",
322+
commandId: CommandId.make(yield* (yield* Crypto.Crypto).randomUUIDv4),
323+
threadId: id,
324+
session: { ...session, status: "stopped", activeTurnId: null, updatedAt: reconciledAt },
325+
createdAt: reconciledAt,
326+
});
327+
}
328+
});
329+
293330
interface StartupOptions {
294331
readonly activate?: Effect.Effect<void>;
295332
readonly awaitAuxiliaryParked?: Effect.Effect<void>;
@@ -432,6 +469,8 @@ export const make = (options?: StartupOptions) =>
432469

433470
// This is the prepared boundary. Every dependency has been acquired and
434471
// every runtime root has confirmed that it is parked before this request.
472+
// Reconcile before prepareTrial commits the launcher update
473+
yield* reconcileProviderSessions;
435474
const updateOutcome = yield* launcher.prepareTrial;
436475
yield* runStartupPhase(
437476
"welcome.publish",

0 commit comments

Comments
 (0)