Skip to content

Commit 0929907

Browse files
fix(server): reconcile orphaned provider sessions (pingdotgg#7719)
Co-authored-by: t3-code[bot] <269035359+t3-code[bot]@users.noreply.github.com> Co-authored-by: maria <[email protected]>
1 parent aa17ec6 commit 0929907

4 files changed

Lines changed: 743 additions & 0 deletions

File tree

‎.macroscope/check-run-agents/effect-service-conventions.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,7 @@ Review changed TypeScript and directly affected call sites for the conventions b
6868
- Export direct schema predicates such as `export const isFoo = Schema.is(Foo)`. Flag a private `Schema.is` constant wrapped by a redundant function with the same signature.
6969
- Do not introduce a large `switch` or lookup table in an error's `message` getter to model failures that deserve separate error classes.
7070
- Catch statically known tagged failures with `Effect.catchTags({ ... })`, including when handling only one tag. Do not use `catchIf` with a schema predicate merely to recover one or more known `_tag` variants, and do not use `catchTag`. `Effect.catch` is appropriate when the entire error channel is intentionally handled; `catchIf` remains appropriate for genuinely structural predicates such as inspecting an underlying platform error code.
71+
- For startup reconciliation that repairs multiple independent entities, preserve interruption rather than reducing it to a warning. Retry a transient per-entity repair before readiness, then isolate a persistent failure so one bad entity cannot abort global startup or prevent later entities from being repaired. Require tests for both the retry-success path and persistent-failure continuation.
7172
- Do not add a helper whose only behavior is `(...args) => new SomeError({ ...args })`, including curried aliases used once with `mapError`. Construct the error at the failure boundary so its attributes and cause remain visible. Keep a mapper only when it performs real normalization, passes through existing domain errors, or adds reusable context/control flow.
7273
- When a reusable error-to-error translation clearly belongs to the target error type, prefer a descriptive static factory on that error class over a detached production-side switch. Do not force a static method for one-off inline mappings.
7374

Lines changed: 358 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,358 @@
1+
import * as NodeServices from "@effect/platform-node/NodeServices";
2+
import {
3+
CommandId,
4+
DEFAULT_PROVIDER_INTERACTION_MODE,
5+
EnvironmentId,
6+
MessageId,
7+
ProjectId,
8+
ProviderDriverKind,
9+
ProviderInstanceId,
10+
ThreadId,
11+
} from "@t3tools/contracts";
12+
import { assert, it } from "@effect/vitest";
13+
import * as DateTime from "effect/DateTime";
14+
import * as Effect from "effect/Effect";
15+
import * as Exit from "effect/Exit";
16+
import * as Layer from "effect/Layer";
17+
import * as Option from "effect/Option";
18+
import * as Stream from "effect/Stream";
19+
import * as SqlClient from "effect/unstable/sql/SqlClient";
20+
import { HttpServer } from "effect/unstable/http";
21+
22+
import * as EnvironmentAuth from "../src/auth/EnvironmentAuth.ts";
23+
import * as ServiceLauncherClient from "../src/cloud/serviceLauncherClient.ts";
24+
import * as ServerConfig from "../src/config.ts";
25+
import * as ServerEnvironment from "../src/environment/ServerEnvironment.ts";
26+
import * as Keybindings from "../src/keybindings.ts";
27+
import { OrchestrationLayerLive } from "../src/orchestration/runtimeLayer.ts";
28+
import * as OrchestrationEngine from "../src/orchestration/Services/OrchestrationEngine.ts";
29+
import * as OrchestrationReactor from "../src/orchestration/Services/OrchestrationReactor.ts";
30+
import * as ProjectionSnapshotQuery from "../src/orchestration/Services/ProjectionSnapshotQuery.ts";
31+
import { makeSqlitePersistenceLive } from "../src/persistence/Layers/Sqlite.ts";
32+
import * as ProviderSessionRuntime from "../src/persistence/ProviderSessionRuntime.ts";
33+
import * as ExternalLauncher from "../src/process/externalLauncher.ts";
34+
import { ProviderSessionDirectoryLive } from "../src/provider/Layers/ProviderSessionDirectory.ts";
35+
import * as ProviderService from "../src/provider/Services/ProviderService.ts";
36+
import * as ProviderSessionDirectory from "../src/provider/Services/ProviderSessionDirectory.ts";
37+
import * as ProviderSessionReaper from "../src/provider/Services/ProviderSessionReaper.ts";
38+
import * as RepositoryIdentityResolver from "../src/project/RepositoryIdentityResolver.ts";
39+
import * as ServerLifecycleEvents from "../src/serverLifecycleEvents.ts";
40+
import * as ServerRuntimeStartup from "../src/serverRuntimeStartup.ts";
41+
import * as ServerSettings from "../src/serverSettings.ts";
42+
import * as AnalyticsService from "../src/telemetry/AnalyticsService.ts";
43+
44+
const providerInstanceId = ProviderInstanceId.make("codex");
45+
const projectId = ProjectId.make("project-startup-orphan");
46+
const threadId = ThreadId.make("thread-startup-orphan");
47+
const stoppedBindingThreadId = ThreadId.make("thread-startup-orphan-stopped-binding");
48+
const resumeCursor = { schemaVersion: 1, sessionId: "provider-session-before-restart" };
49+
const stoppedBindingResumeCursor = {
50+
schemaVersion: 1,
51+
sessionId: "provider-session-stopped-before-restart",
52+
};
53+
54+
const makePersistedRuntimeLayer = (dbPath: string) => {
55+
const persistence = makeSqlitePersistenceLive(dbPath);
56+
const orchestration = OrchestrationLayerLive.pipe(
57+
Layer.provideMerge(RepositoryIdentityResolver.layer),
58+
Layer.provideMerge(persistence),
59+
);
60+
const directory = ProviderSessionDirectoryLive.pipe(
61+
Layer.provide(ProviderSessionRuntime.layer),
62+
Layer.provide(persistence),
63+
);
64+
return Layer.mergeAll(orchestration, directory);
65+
};
66+
67+
const startupDependencies = Layer.mergeAll(
68+
Layer.mock(Keybindings.Keybindings)({
69+
start: Effect.void,
70+
}),
71+
ServerSettings.layerTest(),
72+
Layer.succeed(OrchestrationReactor.OrchestrationReactor, {
73+
start: () => Effect.void,
74+
}),
75+
Layer.succeed(ProviderSessionReaper.ProviderSessionReaper, {
76+
start: () => Effect.void,
77+
}),
78+
ServerLifecycleEvents.layer,
79+
Layer.succeed(ServerEnvironment.ServerEnvironment, {
80+
getEnvironmentId: Effect.succeed(EnvironmentId.make("environment-startup-orphan")),
81+
getDescriptor: Effect.succeed({
82+
environmentId: EnvironmentId.make("environment-startup-orphan"),
83+
label: "Startup orphan test",
84+
version: "test",
85+
platform: { os: "linux", arch: "x64" },
86+
capabilities: {},
87+
} as never),
88+
}),
89+
Layer.mock(EnvironmentAuth.EnvironmentAuth)({
90+
issueStartupPairingUrl: (baseUrl: string) => Effect.succeed(`${baseUrl}/pair`),
91+
}),
92+
Layer.mock(ExternalLauncher.ExternalLauncher)({
93+
launchBrowser: () => Effect.void,
94+
}),
95+
Layer.succeed(ServiceLauncherClient.ServiceLauncherClient, {
96+
managed: false,
97+
requestUpdate: () => Effect.die("unused"),
98+
prepareTrial: Effect.sync(() => undefined),
99+
}),
100+
Layer.succeed(
101+
HttpServer.HttpServer,
102+
HttpServer.HttpServer.of({
103+
address: { _tag: "TcpAddress", hostname: "127.0.0.1", port: 3773 },
104+
serve: (() => Effect.void) as HttpServer.HttpServer["Service"]["serve"],
105+
}),
106+
),
107+
AnalyticsService.layerTest,
108+
Layer.succeed(ProviderService.ProviderService, {
109+
startSession: () => Effect.die("unused"),
110+
sendTurn: () => Effect.die("unused"),
111+
interruptTurn: () => Effect.die("unused"),
112+
respondToRequest: () => Effect.die("unused"),
113+
respondToUserInput: () => Effect.die("unused"),
114+
stopSession: () => Effect.die("unused"),
115+
listSessions: () => Effect.succeed([]),
116+
getCapabilities: () => Effect.die("unused"),
117+
getInstanceInfo: () => Effect.die("unused"),
118+
rollbackConversation: () => Effect.die("unused"),
119+
streamEvents: Stream.empty,
120+
}),
121+
);
122+
123+
it.effect(
124+
"recovers a persisted starting session before opening the command gate after restart",
125+
() =>
126+
Effect.gen(function* () {
127+
const config = yield* ServerConfig.ServerConfig;
128+
const firstRuntime = makePersistedRuntimeLayer(config.dbPath);
129+
const now = yield* DateTime.now;
130+
const createdAt = DateTime.formatIso(now);
131+
132+
yield* Effect.gen(function* () {
133+
const engine = yield* OrchestrationEngine.OrchestrationEngineService;
134+
const directory = yield* ProviderSessionDirectory.ProviderSessionDirectory;
135+
136+
yield* engine.dispatch({
137+
type: "project.create",
138+
commandId: CommandId.make("command-create-project"),
139+
projectId,
140+
title: "Startup orphan project",
141+
workspaceRoot: "/tmp/startup-orphan-project",
142+
defaultModelSelection: { instanceId: providerInstanceId, model: "gpt-5" },
143+
createdAt,
144+
});
145+
yield* engine.dispatch({
146+
type: "thread.create",
147+
commandId: CommandId.make("command-create-thread"),
148+
threadId,
149+
projectId,
150+
title: "Startup orphan thread",
151+
modelSelection: { instanceId: providerInstanceId, model: "gpt-5" },
152+
interactionMode: DEFAULT_PROVIDER_INTERACTION_MODE,
153+
runtimeMode: "full-access",
154+
branch: null,
155+
worktreePath: null,
156+
createdAt,
157+
});
158+
yield* engine.dispatch({
159+
type: "thread.turn.start",
160+
commandId: CommandId.make("command-start-pending-turn"),
161+
threadId,
162+
message: {
163+
messageId: MessageId.make("message-pending-before-restart"),
164+
role: "user",
165+
text: "Persist this queued turn before restart",
166+
attachments: [],
167+
},
168+
interactionMode: DEFAULT_PROVIDER_INTERACTION_MODE,
169+
runtimeMode: "full-access",
170+
createdAt,
171+
});
172+
yield* engine.dispatch({
173+
type: "thread.session.set",
174+
commandId: CommandId.make("command-mark-session-starting"),
175+
threadId,
176+
session: {
177+
threadId,
178+
status: "starting",
179+
providerName: "codex",
180+
providerInstanceId,
181+
runtimeMode: "full-access",
182+
activeTurnId: null,
183+
lastError: null,
184+
updatedAt: createdAt,
185+
},
186+
createdAt,
187+
});
188+
yield* directory.upsert({
189+
threadId,
190+
provider: ProviderDriverKind.make("codex"),
191+
providerInstanceId,
192+
status: "running",
193+
resumeCursor,
194+
runtimePayload: { activeTurnId: null, unrelated: "preserve-me" },
195+
runtimeMode: "full-access",
196+
});
197+
yield* engine.dispatch({
198+
type: "thread.create",
199+
commandId: CommandId.make("command-create-stopped-binding-thread"),
200+
threadId: stoppedBindingThreadId,
201+
projectId,
202+
title: "Startup orphan with stopped binding",
203+
modelSelection: { instanceId: providerInstanceId, model: "gpt-5" },
204+
interactionMode: DEFAULT_PROVIDER_INTERACTION_MODE,
205+
runtimeMode: "full-access",
206+
branch: null,
207+
worktreePath: null,
208+
createdAt,
209+
});
210+
yield* engine.dispatch({
211+
type: "thread.turn.start",
212+
commandId: CommandId.make("command-start-stopped-binding-pending-turn"),
213+
threadId: stoppedBindingThreadId,
214+
message: {
215+
messageId: MessageId.make("message-stopped-binding-pending-before-restart"),
216+
role: "user",
217+
text: "Persist another queued turn before restart",
218+
attachments: [],
219+
},
220+
interactionMode: DEFAULT_PROVIDER_INTERACTION_MODE,
221+
runtimeMode: "full-access",
222+
createdAt,
223+
});
224+
yield* engine.dispatch({
225+
type: "thread.session.set",
226+
commandId: CommandId.make("command-mark-stopped-binding-session-starting"),
227+
threadId: stoppedBindingThreadId,
228+
session: {
229+
threadId: stoppedBindingThreadId,
230+
status: "starting",
231+
providerName: "codex",
232+
providerInstanceId,
233+
runtimeMode: "full-access",
234+
activeTurnId: null,
235+
lastError: null,
236+
updatedAt: createdAt,
237+
},
238+
createdAt,
239+
});
240+
yield* directory.upsert({
241+
threadId: stoppedBindingThreadId,
242+
provider: ProviderDriverKind.make("codex"),
243+
providerInstanceId,
244+
status: "stopped",
245+
resumeCursor: stoppedBindingResumeCursor,
246+
runtimePayload: { activeTurnId: "stale", unrelated: "also-preserve-me" },
247+
runtimeMode: "full-access",
248+
});
249+
}).pipe(Effect.provide(firstRuntime));
250+
251+
const secondRuntime = makePersistedRuntimeLayer(config.dbPath);
252+
const startupLayer = ServerRuntimeStartup.layer.pipe(
253+
Layer.provideMerge(secondRuntime),
254+
Layer.provideMerge(startupDependencies),
255+
);
256+
257+
const result = yield* Effect.gen(function* () {
258+
const startup = yield* ServerRuntimeStartup.ServerRuntimeStartup;
259+
const engine = yield* OrchestrationEngine.OrchestrationEngineService;
260+
const query = yield* ProjectionSnapshotQuery.ProjectionSnapshotQuery;
261+
const directory = yield* ProviderSessionDirectory.ProviderSessionDirectory;
262+
const sql = yield* SqlClient.SqlClient;
263+
264+
yield* startup.markHttpListening;
265+
yield* startup.awaitCommandReady;
266+
267+
const restartedThread = Option.getOrThrow(yield* query.getThreadDetailById(threadId));
268+
const restartedStoppedBindingThread = Option.getOrThrow(
269+
yield* query.getThreadDetailById(stoppedBindingThreadId),
270+
);
271+
const pendingRows = yield* sql<{ readonly threadId: string }>`
272+
SELECT thread_id AS "threadId"
273+
FROM projection_turns
274+
WHERE thread_id IN (${threadId}, ${stoppedBindingThreadId})
275+
AND turn_id IS NULL
276+
AND state = 'pending'
277+
`;
278+
const settleExit = yield* Effect.exit(
279+
engine.dispatch({
280+
type: "thread.settle",
281+
commandId: CommandId.make("command-settle-after-restart"),
282+
threadId,
283+
}),
284+
);
285+
const snoozeExit = yield* Effect.exit(
286+
engine.dispatch({
287+
type: "thread.snooze",
288+
commandId: CommandId.make("command-snooze-after-restart"),
289+
threadId,
290+
snoozedUntil: DateTime.formatIso(DateTime.add(now, { hours: 1 })),
291+
}),
292+
);
293+
const newTurnExit = yield* Effect.exit(
294+
engine.dispatch({
295+
type: "thread.turn.start",
296+
commandId: CommandId.make("command-new-turn-after-restart"),
297+
threadId,
298+
message: {
299+
messageId: MessageId.make("message-new-turn-after-restart"),
300+
role: "user",
301+
text: "Continue immediately after restart",
302+
attachments: [],
303+
},
304+
interactionMode: DEFAULT_PROVIDER_INTERACTION_MODE,
305+
runtimeMode: "full-access",
306+
createdAt,
307+
}),
308+
);
309+
const binding = Option.getOrThrow(yield* directory.getBinding(threadId));
310+
const stoppedBinding = Option.getOrThrow(
311+
yield* directory.getBinding(stoppedBindingThreadId),
312+
);
313+
314+
return {
315+
sessionStatus: restartedThread.session?.status,
316+
activeTurnId: restartedThread.session?.activeTurnId,
317+
latestTurn: restartedThread.latestTurn,
318+
pendingTurnCount: pendingRows.length,
319+
settleSucceeded: Exit.isSuccess(settleExit),
320+
snoozeSucceeded: Exit.isSuccess(snoozeExit),
321+
newTurnSucceeded: Exit.isSuccess(newTurnExit),
322+
bindingStatus: binding.status,
323+
resumeCursor: binding.resumeCursor,
324+
runtimePayload: binding.runtimePayload,
325+
stoppedBindingSessionStatus: restartedStoppedBindingThread.session?.status,
326+
stoppedBindingStatus: stoppedBinding.status,
327+
stoppedBindingResumeCursor: stoppedBinding.resumeCursor,
328+
stoppedBindingRuntimePayload: stoppedBinding.runtimePayload,
329+
};
330+
}).pipe(Effect.provide(startupLayer));
331+
332+
assert.deepStrictEqual(result, {
333+
sessionStatus: "error",
334+
activeTurnId: null,
335+
latestTurn: null,
336+
pendingTurnCount: 0,
337+
settleSucceeded: true,
338+
snoozeSucceeded: true,
339+
newTurnSucceeded: true,
340+
bindingStatus: "stopped",
341+
resumeCursor,
342+
runtimePayload: { activeTurnId: null, unrelated: "preserve-me" },
343+
stoppedBindingSessionStatus: "error",
344+
stoppedBindingStatus: "stopped",
345+
stoppedBindingResumeCursor,
346+
stoppedBindingRuntimePayload: {
347+
activeTurnId: null,
348+
unrelated: "also-preserve-me",
349+
},
350+
});
351+
}).pipe(
352+
Effect.provide(
353+
ServerConfig.layerTest(process.cwd(), {
354+
prefix: "t3-orphaned-provider-session-startup-",
355+
}).pipe(Layer.provideMerge(NodeServices.layer)),
356+
),
357+
),
358+
);

0 commit comments

Comments
 (0)