Skip to content

Commit 44eae6a

Browse files
committed
fix(codex): keep async replies working after restarts
1 parent 551eccf commit 44eae6a

12 files changed

Lines changed: 145 additions & 34 deletions

‎apps/server/src/checkpointing/CheckpointDiffQuery.test.ts‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,7 @@ describe("CheckpointDiffQuery.layer", () => {
7575
Layer.provideMerge(Layer.succeed(CheckpointStore.CheckpointStore, checkpointStore)),
7676
Layer.provideMerge(
7777
Layer.succeed(ProjectionSnapshotQuery.ProjectionSnapshotQuery, {
78+
getUserInputActivity: () => Effect.die("unused"),
7879
getCommandReadModel: () =>
7980
Effect.die("CheckpointDiffQuery should not request the command read model"),
8081
getSnapshot: () =>
@@ -185,6 +186,7 @@ describe("CheckpointDiffQuery.layer", () => {
185186
Layer.provideMerge(Layer.succeed(CheckpointStore.CheckpointStore, checkpointStore)),
186187
Layer.provideMerge(
187188
Layer.succeed(ProjectionSnapshotQuery.ProjectionSnapshotQuery, {
189+
getUserInputActivity: () => Effect.die("unused"),
188190
getCommandReadModel: () =>
189191
Effect.die("CheckpointDiffQuery should not request the command read model"),
190192
getSnapshot: () =>
@@ -270,6 +272,7 @@ describe("CheckpointDiffQuery.layer", () => {
270272
Layer.provideMerge(Layer.succeed(CheckpointStore.CheckpointStore, checkpointStore)),
271273
Layer.provideMerge(
272274
Layer.succeed(ProjectionSnapshotQuery.ProjectionSnapshotQuery, {
275+
getUserInputActivity: () => Effect.die("unused"),
273276
getCommandReadModel: () =>
274277
Effect.die("CheckpointDiffQuery should not request the command read model"),
275278
getSnapshot: () =>
@@ -340,6 +343,7 @@ describe("CheckpointDiffQuery.layer", () => {
340343
Layer.provideMerge(Layer.succeed(CheckpointStore.CheckpointStore, checkpointStore)),
341344
Layer.provideMerge(
342345
Layer.succeed(ProjectionSnapshotQuery.ProjectionSnapshotQuery, {
346+
getUserInputActivity: () => Effect.die("unused"),
343347
getCommandReadModel: () =>
344348
Effect.die("CheckpointDiffQuery should not request the command read model"),
345349
getSnapshot: () =>
@@ -395,6 +399,7 @@ describe("CheckpointDiffQuery.layer", () => {
395399
Layer.provideMerge(Layer.succeed(CheckpointStore.CheckpointStore, checkpointStore)),
396400
Layer.provideMerge(
397401
Layer.succeed(ProjectionSnapshotQuery.ProjectionSnapshotQuery, {
402+
getUserInputActivity: () => Effect.die("unused"),
398403
getCommandReadModel: () =>
399404
Effect.die("CheckpointDiffQuery should not request the command read model"),
400405
getSnapshot: () =>

‎apps/server/src/orchestration/Layers/OrchestrationEngine.test.ts‎

Lines changed: 69 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,8 @@
1+
// @effect-diagnostics nodeBuiltinImport:off
2+
import * as NodeFSP from "node:fs/promises";
3+
import * as NodeOS from "node:os";
4+
import * as NodePath from "node:path";
5+
16
import {
27
ApprovalRequestId,
38
EventId,
@@ -27,7 +32,10 @@ import { PersistenceSqlError } from "../../persistence/Errors.ts";
2732
import { OrchestrationCommandReceiptRepositoryLive } from "../../persistence/Layers/OrchestrationCommandReceipts.ts";
2833
import * as OrchestrationCommandReceipts from "../../persistence/Services/OrchestrationCommandReceipts.ts";
2934
import { OrchestrationEventStoreLive } from "../../persistence/Layers/OrchestrationEventStore.ts";
30-
import { SqlitePersistenceMemory } from "../../persistence/Layers/Sqlite.ts";
35+
import {
36+
makeSqlitePersistenceLive,
37+
SqlitePersistenceMemory,
38+
} from "../../persistence/Layers/Sqlite.ts";
3139
import {
3240
OrchestrationEventStore,
3341
type OrchestrationEventStoreShape,
@@ -51,7 +59,10 @@ const asMessageId = (value: string): MessageId => MessageId.make(value);
5159
const asTurnId = (value: string): TurnId => TurnId.make(value);
5260
const asCheckpointRef = (value: string): CheckpointRef => CheckpointRef.make(value);
5361

54-
function makeOrchestrationLayer() {
62+
function makeOrchestrationLayer(databasePath?: string) {
63+
const persistence = databasePath
64+
? makeSqlitePersistenceLive(databasePath)
65+
: SqlitePersistenceMemory;
5566
const ServerConfigLayer = ServerConfig.layerTest(process.cwd(), {
5667
prefix: "t3-orchestration-engine-test-",
5768
});
@@ -67,19 +78,21 @@ function makeOrchestrationLayer() {
6778
Layer.provide(OrchestrationEventStoreLive),
6879
Layer.provideMerge(OrchestrationCommandReceiptRepositoryLive),
6980
Layer.provide(RepositoryIdentityResolver.layer),
70-
Layer.provide(SqlitePersistenceMemory),
81+
Layer.provide(persistence),
7182
Layer.provideMerge(ServerConfigLayer),
7283
Layer.provideMerge(NodeServices.layer),
7384
);
7485
}
7586

76-
async function createOrchestrationSystem() {
77-
const runtime = ManagedRuntime.make(makeOrchestrationLayer());
87+
async function createOrchestrationSystem(databasePath?: string) {
88+
const runtime = ManagedRuntime.make(makeOrchestrationLayer(databasePath));
7889
const engine = await runtime.runPromise(Effect.service(OrchestrationEngineService));
7990
const snapshotQuery = await runtime.runPromise(Effect.service(ProjectionSnapshotQuery));
8091
return {
8192
engine,
8293
readModel: () => runtime.runPromise(snapshotQuery.getSnapshot()),
94+
readThread: (threadId: ThreadId) =>
95+
runtime.runPromise(snapshotQuery.getThreadDetailById(threadId)),
8396
run: <A, E>(effect: Effect.Effect<A, E>) => runtime.runPromise(effect),
8497
dispose: () => runtime.dispose(),
8598
};
@@ -102,9 +115,13 @@ const hasMetricSnapshot = (
102115

103116
describe("OrchestrationEngine", () => {
104117
it.each(["running", "stopped"] as const)(
105-
"sends async answers as messages with a %s session and rejects duplicate replies",
118+
"sends async answers with a %s session and rejects old duplicate replies",
106119
async (status) => {
107-
const system = await createOrchestrationSystem();
120+
const directory = await NodeFSP.mkdtemp(
121+
NodePath.join(NodeOS.tmpdir(), "t3-async-questions-"),
122+
);
123+
const databasePath = NodePath.join(directory, "state.sqlite");
124+
let system = await createOrchestrationSystem(databasePath);
108125
const threadId = ThreadId.make("async-thread");
109126
const projectId = ProjectId.make("async-project");
110127
const requestId = ApprovalRequestId.make("codex-async:question-1");
@@ -185,30 +202,36 @@ describe("OrchestrationEngine", () => {
185202
},
186203
}),
187204
);
188-
// The question must stay available when later work fills the activity window.
189-
for (let index = 0; index < 501; index += 1) {
190-
await system.run(
191-
system.engine.dispatch({
192-
type: "thread.activity.append",
193-
commandId: CommandId.make(`work-${index}`),
194-
threadId,
195-
createdAt: "2026-01-01T00:00:01.000Z",
196-
activity: {
197-
id: EventId.make(`work-${index}`),
198-
kind: "tool.completed",
199-
summary: "Work continued",
200-
payload: {},
201-
tone: "info",
202-
turnId: TurnId.make("turn-1"),
203-
createdAt: "2026-01-01T00:00:01.000Z",
204-
},
205-
}),
206-
);
207-
}
205+
const appendWork = async (prefix: string, createdAt: string) => {
206+
for (let index = 0; index < 501; index += 1) {
207+
await system.run(
208+
system.engine.dispatch({
209+
type: "thread.activity.append",
210+
commandId: CommandId.make(`${prefix}-${index}`),
211+
threadId,
212+
createdAt,
213+
activity: {
214+
id: EventId.make(`${prefix}-${index}`),
215+
kind: "tool.completed",
216+
summary: "Work continued",
217+
payload: {},
218+
tone: "info",
219+
turnId: TurnId.make("turn-1"),
220+
createdAt,
221+
},
222+
}),
223+
);
224+
}
225+
};
226+
await appendWork("work", "2026-01-01T00:00:01.000Z");
208227
const before = await system.readModel();
209228
expect(
210229
before.threads[0]?.activities.some((activity) => activity.id === "async-question"),
211230
).toBe(true);
231+
if (status === "stopped") {
232+
await system.dispose();
233+
system = await createOrchestrationSystem(databasePath);
234+
}
212235
const response = {
213236
type: "thread.user-input.respond" as const,
214237
commandId: CommandId.make("async-response"),
@@ -257,8 +280,26 @@ describe("OrchestrationEngine", () => {
257280
}),
258281
),
259282
).rejects.toThrow("This question has already been answered.");
283+
await appendWork("later-work", "2026-01-01T00:00:03.000Z");
284+
const afterEviction = Option.getOrThrow(await system.readThread(threadId));
285+
expect(
286+
afterEviction.activities.some((activity) => activity.kind === "user-input.resolved"),
287+
).toBe(false);
288+
if (status === "stopped") {
289+
await system.dispose();
290+
system = await createOrchestrationSystem(databasePath);
291+
}
292+
await expect(
293+
system.run(
294+
system.engine.dispatch({
295+
...response,
296+
commandId: CommandId.make("reply-after-eviction"),
297+
}),
298+
),
299+
).rejects.toThrow("This question has already been answered.");
260300
} finally {
261301
await system.dispose();
302+
await NodeFSP.rm(directory, { recursive: true, force: true });
262303
}
263304
},
264305
);
@@ -347,6 +388,7 @@ describe("OrchestrationEngine", () => {
347388
const layer = OrchestrationEngineLive.pipe(
348389
Layer.provide(
349390
Layer.succeed(ProjectionSnapshotQuery, {
391+
getUserInputActivity: () => Effect.die("unused"),
350392
getCommandReadModel: () => Effect.succeed(commandReadModel),
351393
getSnapshot: () =>
352394
Effect.sync(() => {

‎apps/server/src/orchestration/Layers/OrchestrationEngine.ts‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -195,9 +195,18 @@ const makeOrchestrationEngine = Effect.gen(function* () {
195195
});
196196
}
197197

198+
// Command snapshots omit activities at startup and cap them while running.
199+
// Read this request's durable state before deciding how to send the answer.
200+
const userInputActivity =
201+
envelope.command.type === "thread.user-input.respond"
202+
? yield* projectionSnapshotQuery.getUserInputActivity(envelope.command)
203+
: Option.none();
198204
const eventBase = yield* decideOrchestrationCommand({
199205
command: envelope.command,
200206
readModel: commandReadModel,
207+
...(Option.isSome(userInputActivity)
208+
? { userInputActivity: userInputActivity.value }
209+
: {}),
201210
}).pipe(
202211
Effect.provideService(Crypto.Crypto, crypto),
203212
Effect.mapError((cause) =>

‎apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.ts‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import {
2+
ApprovalRequestId,
23
ChatAttachment,
34
CheckpointRef,
45
IsoDateTime,
@@ -1119,6 +1120,40 @@ const makeProjectionSnapshotQuery = Effect.gen(function* () {
11191120
`,
11201121
});
11211122

1123+
const getUserInputActivityRow = SqlSchema.findOneOption({
1124+
Request: Schema.Struct({ threadId: ThreadId, requestId: ApprovalRequestId }),
1125+
Result: ProjectionThreadActivityDbRowSchema,
1126+
execute: ({ threadId, requestId }) => sql`
1127+
SELECT
1128+
activity_id AS "activityId",
1129+
thread_id AS "threadId",
1130+
turn_id AS "turnId",
1131+
tone,
1132+
kind,
1133+
summary,
1134+
payload_json AS "payload",
1135+
sequence,
1136+
created_at AS "createdAt"
1137+
FROM projection_thread_activities
1138+
WHERE thread_id = ${threadId}
1139+
AND kind IN ('user-input.requested', 'user-input.resolved')
1140+
AND json_extract(payload_json, '$.requestId') = ${requestId}
1141+
ORDER BY sequence DESC, created_at DESC, activity_id DESC
1142+
LIMIT 1
1143+
`,
1144+
});
1145+
1146+
const getUserInputActivity: ProjectionSnapshotQueryShape["getUserInputActivity"] = (input) =>
1147+
getUserInputActivityRow(input).pipe(
1148+
Effect.map(Option.map(mapThreadActivityRow)),
1149+
Effect.mapError(
1150+
toPersistenceSqlOrDecodeError(
1151+
"ProjectionSnapshotQuery.getUserInputActivity:query",
1152+
"ProjectionSnapshotQuery.getUserInputActivity:decodeRow",
1153+
),
1154+
),
1155+
);
1156+
11221157
const listThreadActivityIdsByThread = SqlSchema.findAll({
11231158
Request: ThreadIdLookupInput,
11241159
Result: ProjectionThreadActivityIdRowSchema,
@@ -3151,6 +3186,7 @@ pending_approval_requests AS (
31513186

31523187
return {
31533188
getCommandReadModel,
3189+
getUserInputActivity,
31543190
getSnapshot,
31553191
getShellSnapshot,
31563192
getArchivedShellSnapshot,

‎apps/server/src/orchestration/Services/ProjectionSnapshotQuery.ts‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
* @module ProjectionSnapshotQuery
88
*/
99
import type {
10+
ApprovalRequestId,
1011
CheckpointRef,
1112
OrchestrationCheckpointSummary,
1213
OrchestrationProject,
@@ -16,6 +17,7 @@ import type {
1617
OrchestrationSearchThreadsResult,
1718
OrchestrationShellSnapshot,
1819
OrchestrationThread,
20+
OrchestrationThreadActivity,
1921
OrchestrationThreadDetailSnapshot,
2022
OrchestrationThreadDetailWindow,
2123
OrchestrationThreadShell,
@@ -72,6 +74,12 @@ export interface ProjectionThreadDetailQuery {
7274
* ProjectionSnapshotQueryShape - Service API for read-model snapshots.
7375
*/
7476
export interface ProjectionSnapshotQueryShape {
77+
/** Read the latest request or resolution without loading the thread history. */
78+
readonly getUserInputActivity: (input: {
79+
readonly threadId: ThreadId;
80+
readonly requestId: ApprovalRequestId;
81+
}) => Effect.Effect<Option.Option<OrchestrationThreadActivity>, ProjectionRepositoryError>;
82+
7583
/**
7684
* Read the lightweight command snapshot used to bootstrap the in-memory
7785
* orchestration engine without hydrating message/activity/checkpoint bodies.

‎apps/server/src/orchestration/decider.ts‎

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import {
66
type OrchestrationEvent,
77
type OrchestrationReadModel,
88
type OrchestrationThread,
9+
type OrchestrationThreadActivity,
910
} from "@t3tools/contracts";
1011
import * as DateTime from "effect/DateTime";
1112
import * as Crypto from "effect/Crypto";
@@ -187,9 +188,11 @@ const decideCommandSequence = Effect.fn("decideCommandSequence")(function* ({
187188
export const decideOrchestrationCommand = Effect.fn("decideOrchestrationCommand")(function* ({
188189
command,
189190
readModel,
191+
userInputActivity,
190192
}: {
191193
readonly command: OrchestrationCommand;
192194
readonly readModel: OrchestrationReadModel;
195+
readonly userInputActivity?: OrchestrationThreadActivity;
193196
}): Effect.fn.Return<
194197
DecideOrchestrationCommandResult,
195198
OrchestrationCommandRejection | PlatformError.PlatformError,
@@ -1068,12 +1071,7 @@ export const decideOrchestrationCommand = Effect.fn("decideOrchestrationCommand"
10681071
command,
10691072
threadId: command.threadId,
10701073
});
1071-
const request = thread.activities.findLast(
1072-
(activity) =>
1073-
(activity.kind === "user-input.requested" || activity.kind === "user-input.resolved") &&
1074-
Predicate.isObject(activity.payload) &&
1075-
activity.payload.requestId === command.requestId,
1076-
);
1074+
const request = userInputActivity;
10771075
if (
10781076
request &&
10791077
Predicate.isObject(request.payload) &&

‎apps/server/src/project/ProjectSetupScriptRunner.test.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ const makeProject = (scripts: OrchestrationProject["scripts"]): OrchestrationPro
2626

2727
const makeProjectionSnapshotQueryLayer = (project: OrchestrationProject) =>
2828
Layer.succeed(ProjectionSnapshotQuery.ProjectionSnapshotQuery, {
29+
getUserInputActivity: () => Effect.die("unused"),
2930
getCommandReadModel: () => Effect.die("unused"),
3031
getSnapshot: () => Effect.die("unused"),
3132
getShellSnapshot: () => Effect.die("unused"),

‎apps/server/src/provider/Layers/ProviderSessionReaper.test.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -204,6 +204,7 @@ describe("ProviderSessionReaper", () => {
204204
Layer.provideMerge(Layer.succeed(ProviderService, providerService)),
205205
Layer.provideMerge(
206206
Layer.succeed(ProjectionSnapshotQuery, {
207+
getUserInputActivity: () => Effect.die("unused"),
207208
getCommandReadModel: () => Effect.die("unused"),
208209
getSnapshot: () => Effect.die("unused"),
209210
getShellSnapshot: () => Effect.die("unused"),

‎apps/server/src/server.test.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -926,6 +926,7 @@ const buildAppUnderTest = (options?: {
926926
),
927927
Layer.provide(
928928
Layer.mock(ProjectionSnapshotQuery.ProjectionSnapshotQuery)({
929+
getUserInputActivity: () => Effect.die("unused"),
929930
getCommandReadModel: () => Effect.succeed(makeDefaultOrchestrationReadModel()),
930931
getSnapshot: () => Effect.succeed(makeDefaultOrchestrationReadModel()),
931932
getShellSnapshot: () =>

‎apps/server/src/serverRuntimeStartup.reconcile.test.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,7 @@ const makeProviderService = (liveThreadIds: ReadonlyArray<ThreadId> = []) =>
6666

6767
const queryWithThreads = (threads: ReadonlyArray<ReturnType<typeof makeThread>>) =>
6868
({
69+
getUserInputActivity: () => Effect.die("unused"),
6970
getCommandReadModel: () => Effect.succeed({ threads } as never),
7071
}) as unknown as ProjectionSnapshotQuery.ProjectionSnapshotQuery["Service"];
7172

@@ -631,6 +632,7 @@ it.effect("does not fail startup when the live provider session inventory cannot
631632
let queried = false;
632633
return ServerRuntimeStartup.reconcileProviderSessions.pipe(
633634
Effect.provideService(ProjectionSnapshotQuery.ProjectionSnapshotQuery, {
635+
getUserInputActivity: () => Effect.die("unused"),
634636
getCommandReadModel: () =>
635637
Effect.sync(() => {
636638
queried = true;

0 commit comments

Comments
 (0)