Skip to content

Commit a40aef4

Browse files
shivamhwpclaude
andauthored
fix(server): a draft can retry its first send after a failed bootstrap (pingdotgg#8226)
Co-authored-by: Claude Fable 5 <[email protected]>
1 parent 48c176b commit a40aef4

16 files changed

Lines changed: 690 additions & 25 deletions

‎apps/server/integration/OrchestrationEngineHarness.integration.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -373,7 +373,7 @@ export const makeOrchestrationIntegrationHarness = (
373373
Layer.provideMerge(
374374
Layer.succeed(ThreadDeletionReactor, {
375375
start: () => Effect.void,
376-
drain: Effect.void,
376+
drainThrough: () => Effect.void,
377377
}),
378378
),
379379
Layer.provideMerge(

‎apps/server/src/orchestration/Layers/OrchestrationEngine.test.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -113,6 +113,7 @@ describe("OrchestrationEngine", () => {
113113
detail: "historical replay should not be used during bootstrap",
114114
}),
115115
),
116+
hasEventAfter: () => Effect.succeed(false),
116117
};
117118

118119
const projectionSnapshot = {
@@ -812,6 +813,7 @@ describe("OrchestrationEngine", () => {
812813
readAll() {
813814
return Stream.fromIterable(events);
814815
},
816+
hasEventAfter: () => Effect.succeed(false),
815817
};
816818

817819
const ServerConfigLayer = ServerConfig.layerTest(process.cwd(), {
@@ -1048,6 +1050,7 @@ describe("OrchestrationEngine", () => {
10481050
readAll() {
10491051
return Stream.fromIterable(events);
10501052
},
1053+
hasEventAfter: () => Effect.succeed(false),
10511054
};
10521055

10531056
let shouldFailProjection = true;

‎apps/server/src/orchestration/Layers/OrchestrationReactor.test.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,7 @@ describe("OrchestrationReactor", () => {
6161
started.push("thread-deletion-reactor");
6262
return Effect.void;
6363
},
64-
drain: Effect.void,
64+
drainThrough: () => Effect.void,
6565
}),
6666
),
6767
Layer.provideMerge(

‎apps/server/src/orchestration/Layers/ProjectionPipeline.test.ts‎

Lines changed: 251 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ import {
99
TurnId,
1010
ProviderInstanceId,
1111
} from "@t3tools/contracts";
12+
import * as Option from "effect/Option";
1213
import * as NodeServices from "@effect/platform-node/NodeServices";
1314
import { assert, it } from "@effect/vitest";
1415
import * as Effect from "effect/Effect";
@@ -31,6 +32,7 @@ import {
3132
OrchestrationProjectionPipelineLive,
3233
} from "./ProjectionPipeline.ts";
3334
import { OrchestrationProjectionSnapshotQueryLive } from "./ProjectionSnapshotQuery.ts";
35+
import { ProjectionSnapshotQuery } from "../Services/ProjectionSnapshotQuery.ts";
3436
import * as ThreadBackgroundLiveness from "../ThreadBackgroundLiveness.ts";
3537
import * as ThreadPlanProgress from "../ThreadPlanProgress.ts";
3638
import { OrchestrationEngineService } from "../Services/OrchestrationEngine.ts";
@@ -1277,6 +1279,111 @@ it.layer(Layer.fresh(makeProjectionPipelinePrefixedTestLayer("t3-projection-atta
12771279
},
12781280
);
12791281

1282+
it.layer(Layer.fresh(makeProjectionPipelinePrefixedTestLayer("t3-projection-attachments-replay-")))(
1283+
"OrchestrationProjectionPipeline",
1284+
(it) => {
1285+
it.effect("replaying a superseded thread.deleted keeps the re-created thread's files", () =>
1286+
Effect.gen(function* () {
1287+
const fileSystem = yield* FileSystem.FileSystem;
1288+
const path = yield* Path.Path;
1289+
const projectionPipeline = yield* OrchestrationProjectionPipeline;
1290+
const eventStore = yield* OrchestrationEventStore;
1291+
const { attachmentsDir } = yield* ServerConfig;
1292+
const now = "2026-01-01T00:00:00.000Z";
1293+
const projectId = ProjectId.make("project-replay");
1294+
const retriedThreadId = ThreadId.make("thread-replay-retried");
1295+
const goneThreadId = ThreadId.make("thread-replay-gone");
1296+
const retriedAttachmentPath = path.join(
1297+
attachmentsDir,
1298+
"thread-replay-retried-00000000-0000-4000-8000-000000000001.png",
1299+
);
1300+
const goneAttachmentPath = path.join(
1301+
attachmentsDir,
1302+
"thread-replay-gone-00000000-0000-4000-8000-000000000002.png",
1303+
);
1304+
const threadCreated = (threadId: ThreadId, suffix: string) =>
1305+
eventStore.append({
1306+
type: "thread.created",
1307+
eventId: EventId.make(`evt-replay-create-${suffix}`),
1308+
aggregateKind: "thread",
1309+
aggregateId: threadId,
1310+
occurredAt: now,
1311+
commandId: CommandId.make(`cmd-replay-create-${suffix}`),
1312+
causationEventId: null,
1313+
correlationId: CorrelationId.make(`cmd-replay-create-${suffix}`),
1314+
metadata: {},
1315+
payload: {
1316+
threadId,
1317+
projectId,
1318+
title: `Thread ${suffix}`,
1319+
modelSelection: {
1320+
instanceId: ProviderInstanceId.make("codex"),
1321+
model: "gpt-5-codex",
1322+
},
1323+
runtimeMode: "full-access",
1324+
branch: null,
1325+
worktreePath: null,
1326+
createdAt: now,
1327+
updatedAt: now,
1328+
},
1329+
});
1330+
const threadDeleted = (threadId: ThreadId, suffix: string) =>
1331+
eventStore.append({
1332+
type: "thread.deleted",
1333+
eventId: EventId.make(`evt-replay-delete-${suffix}`),
1334+
aggregateKind: "thread",
1335+
aggregateId: threadId,
1336+
occurredAt: now,
1337+
commandId: CommandId.make(`cmd-replay-delete-${suffix}`),
1338+
causationEventId: null,
1339+
correlationId: CorrelationId.make(`cmd-replay-delete-${suffix}`),
1340+
metadata: {},
1341+
payload: { threadId, deletedAt: now },
1342+
});
1343+
1344+
yield* eventStore.append({
1345+
type: "project.created",
1346+
eventId: EventId.make("evt-replay-project"),
1347+
aggregateKind: "project",
1348+
aggregateId: projectId,
1349+
occurredAt: now,
1350+
commandId: CommandId.make("cmd-replay-project"),
1351+
causationEventId: null,
1352+
correlationId: CorrelationId.make("cmd-replay-project"),
1353+
metadata: {},
1354+
payload: {
1355+
projectId,
1356+
title: "Replay",
1357+
workspaceRoot: "/tmp/project-replay",
1358+
defaultModelSelection: null,
1359+
scripts: [],
1360+
createdAt: now,
1361+
updatedAt: now,
1362+
},
1363+
});
1364+
// A failed first send: create, roll back, then the draft retries the id.
1365+
yield* threadCreated(retriedThreadId, "retried-1");
1366+
yield* threadDeleted(retriedThreadId, "retried");
1367+
yield* threadCreated(retriedThreadId, "retried-2");
1368+
// A thread that was deleted for good.
1369+
yield* threadCreated(goneThreadId, "gone");
1370+
yield* threadDeleted(goneThreadId, "gone");
1371+
1372+
// Files on disk are not event-sourced: by the time anything replays,
1373+
// the retried thread's attachments already belong to its second life.
1374+
yield* fileSystem.makeDirectory(attachmentsDir, { recursive: true });
1375+
yield* fileSystem.writeFileString(retriedAttachmentPath, "second incarnation");
1376+
yield* fileSystem.writeFileString(goneAttachmentPath, "gone");
1377+
1378+
yield* projectionPipeline.bootstrap;
1379+
1380+
assert.isTrue(yield* exists(retriedAttachmentPath));
1381+
assert.isFalse(yield* exists(goneAttachmentPath));
1382+
}),
1383+
);
1384+
},
1385+
);
1386+
12801387
it.layer(BaseTestLayer)("OrchestrationProjectionPipeline", (it) => {
12811388
it.effect("replays a bootstrap backlog larger than the event store default limit", () =>
12821389
Effect.gen(function* () {
@@ -2850,7 +2957,7 @@ it.effect("restores pending turn-start metadata across projection pipeline resta
28502957

28512958
const engineLayer = it.layer(
28522959
OrchestrationEngineLive.pipe(
2853-
Layer.provide(OrchestrationProjectionSnapshotQueryLive),
2960+
Layer.provideMerge(OrchestrationProjectionSnapshotQueryLive),
28542961
Layer.provide(ThreadBackgroundLiveness.layer),
28552962
Layer.provide(ThreadPlanProgress.layer),
28562963
Layer.provide(OrchestrationProjectionPipelineLive),
@@ -2967,4 +3074,147 @@ engineLayer("OrchestrationProjectionPipeline via engine dispatch", (it) => {
29673074
]);
29683075
}),
29693076
);
3077+
3078+
it.effect("re-creating a deleted thread id starts from an empty projection", () =>
3079+
Effect.gen(function* () {
3080+
const engine = yield* OrchestrationEngineService;
3081+
const snapshotQuery = yield* ProjectionSnapshotQuery;
3082+
const sql = yield* SqlClient.SqlClient;
3083+
const createdAt = "2026-01-01T00:00:00.000Z";
3084+
const projectId = ProjectId.make("project-retry");
3085+
const threadId = ThreadId.make("thread-retry");
3086+
const modelSelection = {
3087+
instanceId: ProviderInstanceId.make("codex"),
3088+
model: "gpt-5-codex",
3089+
};
3090+
const createThread = (commandId: string, title: string) =>
3091+
engine.dispatch({
3092+
type: "thread.create",
3093+
commandId: CommandId.make(commandId),
3094+
threadId,
3095+
projectId,
3096+
title,
3097+
modelSelection,
3098+
runtimeMode: "full-access",
3099+
interactionMode: "default",
3100+
branch: null,
3101+
worktreePath: null,
3102+
createdAt,
3103+
});
3104+
const countRowsForThread = (table: string) =>
3105+
sql<{ readonly count: number }>`
3106+
SELECT COUNT(*) AS count FROM ${sql(table)} WHERE thread_id = ${threadId}
3107+
`.pipe(Effect.map((rows) => rows[0]?.count ?? 0));
3108+
const perThreadTables = [
3109+
"projection_thread_messages",
3110+
"projection_thread_activities",
3111+
"projection_thread_sessions",
3112+
"projection_turns",
3113+
"projection_thread_proposed_plans",
3114+
"projection_pending_approvals",
3115+
];
3116+
3117+
yield* engine.dispatch({
3118+
type: "project.create",
3119+
commandId: CommandId.make("cmd-retry-project"),
3120+
projectId,
3121+
title: "Retry Project",
3122+
workspaceRoot: "/tmp/project-retry",
3123+
defaultModelSelection: modelSelection,
3124+
createdAt,
3125+
});
3126+
3127+
// First attempt: the thread gets a turn, a message, an activity, and a
3128+
// running session before its bootstrap fails and the server rolls back.
3129+
yield* createThread("cmd-retry-create-1", "First attempt");
3130+
yield* engine.dispatch({
3131+
type: "thread.turn.start",
3132+
commandId: CommandId.make("cmd-retry-turn-1"),
3133+
threadId,
3134+
message: {
3135+
messageId: MessageId.make("message-retry-1"),
3136+
role: "user",
3137+
text: "first attempt",
3138+
attachments: [],
3139+
},
3140+
runtimeMode: "full-access",
3141+
interactionMode: "default",
3142+
createdAt,
3143+
});
3144+
yield* engine.dispatch({
3145+
type: "thread.activity.append",
3146+
commandId: CommandId.make("cmd-retry-activity-1"),
3147+
threadId,
3148+
activity: {
3149+
id: EventId.make("activity-retry-1"),
3150+
tone: "info",
3151+
kind: "approval.requested",
3152+
summary: "approval requested",
3153+
payload: { requestId: "request-retry-1" },
3154+
turnId: null,
3155+
createdAt,
3156+
},
3157+
createdAt,
3158+
});
3159+
yield* engine.dispatch({
3160+
type: "thread.proposed-plan.upsert",
3161+
commandId: CommandId.make("cmd-retry-plan-1"),
3162+
threadId,
3163+
proposedPlan: {
3164+
id: "plan-retry-1",
3165+
turnId: null,
3166+
planMarkdown: "# Plan",
3167+
implementedAt: null,
3168+
implementationThreadId: null,
3169+
createdAt,
3170+
updatedAt: createdAt,
3171+
},
3172+
createdAt,
3173+
});
3174+
yield* engine.dispatch({
3175+
type: "thread.session.set",
3176+
commandId: CommandId.make("cmd-retry-session-1"),
3177+
threadId,
3178+
session: {
3179+
threadId,
3180+
status: "running",
3181+
providerName: "codex",
3182+
runtimeMode: "full-access",
3183+
activeTurnId: TurnId.make("turn-retry-1"),
3184+
lastError: null,
3185+
updatedAt: createdAt,
3186+
},
3187+
createdAt,
3188+
});
3189+
for (const table of perThreadTables) {
3190+
assert.isAbove(yield* countRowsForThread(table), 0, `${table} should be populated`);
3191+
}
3192+
const populatedShell = Option.getOrThrow(yield* snapshotQuery.getThreadShellById(threadId));
3193+
assert.isTrue(populatedShell.hasPendingApprovals);
3194+
assert.isTrue(populatedShell.hasActionableProposedPlan);
3195+
3196+
yield* engine.dispatch({
3197+
type: "thread.delete",
3198+
commandId: CommandId.make("cmd-retry-delete"),
3199+
threadId,
3200+
});
3201+
assert.isTrue(Option.isNone(yield* snapshotQuery.getThreadShellById(threadId)));
3202+
3203+
// Retry from the same draft reuses the thread id.
3204+
yield* createThread("cmd-retry-create-2", "Second attempt");
3205+
3206+
const shell = Option.getOrThrow(yield* snapshotQuery.getThreadShellById(threadId));
3207+
assert.strictEqual(shell.title, "Second attempt");
3208+
assert.isFalse(shell.hasPendingApprovals);
3209+
assert.isFalse(shell.hasActionableProposedPlan);
3210+
for (const table of perThreadTables) {
3211+
assert.strictEqual(yield* countRowsForThread(table), 0, `${table} should be empty`);
3212+
}
3213+
const detail = Option.getOrThrow(yield* snapshotQuery.getThreadDetailById(threadId));
3214+
assert.deepEqual(detail.messages, []);
3215+
assert.deepEqual(detail.activities, []);
3216+
assert.isNull(detail.latestTurn);
3217+
assert.isNull(detail.session);
3218+
}),
3219+
);
29703220
});

0 commit comments

Comments
 (0)