Skip to content

Commit 64c33af

Browse files
committed
fix(orchestration): harden durable turn queue recovery
1 parent 9c554e9 commit 64c33af

17 files changed

Lines changed: 504 additions & 179 deletions

‎apps/mobile/src/state/use-thread-composer-state.ts‎

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -133,7 +133,7 @@ export function useThreadComposerState() {
133133
}, [selectedThreadDetail, selectedThreadSessionActivity, selectedThreadShell]);
134134

135135
const onSendMessage = useCallback(async () => {
136-
if (!selectedThreadShell) {
136+
if (!selectedThreadShell || !AsyncResult.isSuccess(preferences)) {
137137
return null;
138138
}
139139

@@ -163,10 +163,7 @@ export function useThreadComposerState() {
163163
modelSelection: draft.modelSelection ?? thread.modelSelection,
164164
runtimeMode: draft.runtimeMode ?? thread.runtimeMode,
165165
interactionMode: draft.interactionMode ?? thread.interactionMode,
166-
deliveryMode:
167-
AsyncResult.isSuccess(preferences) && preferences.value.steerActiveTurns === false
168-
? "after-current"
169-
: "immediate",
166+
deliveryMode: preferences.value.steerActiveTurns === false ? "after-current" : "immediate",
170167
createdAt: metadata.createdAt,
171168
});
172169
clearComposerDraftContent(threadKey);

‎apps/server/src/orchestration/Layers/ProjectionPipeline.test.ts‎

Lines changed: 130 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -389,6 +389,136 @@ it.layer(Layer.fresh(makeProjectionPipelinePrefixedTestLayer("t3-projection-atta
389389
},
390390
);
391391

392+
it.layer(Layer.fresh(makeProjectionPipelinePrefixedTestLayer("t3-projection-queue-revert-")))(
393+
"OrchestrationProjectionPipeline",
394+
(it) => {
395+
it.effect("cancels durable queued messages when a thread is reverted", () =>
396+
Effect.gen(function* () {
397+
const projectionPipeline = yield* OrchestrationProjectionPipeline;
398+
const eventStore = yield* OrchestrationEventStore;
399+
const sql = yield* SqlClient.SqlClient;
400+
const now = "2026-08-16T12:00:00.000Z";
401+
const threadId = ThreadId.make("thread-queue-revert");
402+
const messageId = MessageId.make("message-queue-revert");
403+
const appendAndProject = (event: Parameters<typeof eventStore.append>[0]) =>
404+
eventStore
405+
.append(event)
406+
.pipe(Effect.flatMap((savedEvent) => projectionPipeline.projectEvent(savedEvent)));
407+
408+
yield* appendAndProject({
409+
type: "project.created",
410+
eventId: EventId.make("event-queue-revert-project"),
411+
aggregateKind: "project",
412+
aggregateId: ProjectId.make("project-queue-revert"),
413+
occurredAt: now,
414+
commandId: CommandId.make("command-queue-revert-project"),
415+
causationEventId: null,
416+
correlationId: CorrelationId.make("command-queue-revert-project"),
417+
metadata: {},
418+
payload: {
419+
projectId: ProjectId.make("project-queue-revert"),
420+
title: "Queue revert",
421+
workspaceRoot: "/tmp/queue-revert",
422+
defaultModelSelection: null,
423+
scripts: [],
424+
createdAt: now,
425+
updatedAt: now,
426+
},
427+
});
428+
yield* appendAndProject({
429+
type: "thread.created",
430+
eventId: EventId.make("event-queue-revert-thread"),
431+
aggregateKind: "thread",
432+
aggregateId: threadId,
433+
occurredAt: now,
434+
commandId: CommandId.make("command-queue-revert-thread"),
435+
causationEventId: null,
436+
correlationId: CorrelationId.make("command-queue-revert-thread"),
437+
metadata: {},
438+
payload: {
439+
threadId,
440+
projectId: ProjectId.make("project-queue-revert"),
441+
title: "Queue revert",
442+
modelSelection: {
443+
instanceId: ProviderInstanceId.make("codex"),
444+
model: "gpt-5-codex",
445+
},
446+
runtimeMode: "full-access",
447+
branch: null,
448+
worktreePath: null,
449+
createdAt: now,
450+
updatedAt: now,
451+
},
452+
});
453+
yield* appendAndProject({
454+
type: "thread.message-sent",
455+
eventId: EventId.make("event-queue-revert-message"),
456+
aggregateKind: "thread",
457+
aggregateId: threadId,
458+
occurredAt: now,
459+
commandId: CommandId.make("command-queue-revert-message"),
460+
causationEventId: null,
461+
correlationId: CorrelationId.make("command-queue-revert-message"),
462+
metadata: {},
463+
payload: {
464+
threadId,
465+
messageId,
466+
role: "user",
467+
text: "Run this next",
468+
turnId: null,
469+
streaming: false,
470+
createdAt: now,
471+
updatedAt: now,
472+
},
473+
});
474+
yield* appendAndProject({
475+
type: "thread.turn-queued",
476+
eventId: EventId.make("event-queue-revert-queued"),
477+
aggregateKind: "thread",
478+
aggregateId: threadId,
479+
occurredAt: now,
480+
commandId: CommandId.make("command-queue-revert-queued"),
481+
causationEventId: EventId.make("event-queue-revert-message"),
482+
correlationId: CorrelationId.make("command-queue-revert-message"),
483+
metadata: {},
484+
payload: {
485+
threadId,
486+
messageId,
487+
runtimeMode: "full-access",
488+
interactionMode: "default",
489+
createdAt: now,
490+
},
491+
});
492+
yield* appendAndProject({
493+
type: "thread.reverted",
494+
eventId: EventId.make("event-queue-revert-reverted"),
495+
aggregateKind: "thread",
496+
aggregateId: threadId,
497+
occurredAt: now,
498+
commandId: CommandId.make("command-queue-revert-reverted"),
499+
causationEventId: null,
500+
correlationId: CorrelationId.make("command-queue-revert-reverted"),
501+
metadata: {},
502+
payload: { threadId, turnCount: 0 },
503+
});
504+
505+
const messageRows = yield* sql<{ readonly messageId: string }>`
506+
SELECT message_id AS "messageId"
507+
FROM projection_thread_messages
508+
WHERE thread_id = ${threadId}
509+
`;
510+
const queueRows = yield* sql<{ readonly messageId: string }>`
511+
SELECT message_id AS "messageId"
512+
FROM projection_thread_turn_queue
513+
WHERE thread_id = ${threadId}
514+
`;
515+
assert.deepEqual(messageRows, []);
516+
assert.deepEqual(queueRows, []);
517+
}),
518+
);
519+
},
520+
);
521+
392522
it.layer(BaseTestLayer)("OrchestrationProjectionPipeline", (it) => {
393523
it.effect(
394524
"passes explicit empty attachment arrays through the projection pipeline to clear attachments",

‎apps/server/src/orchestration/Layers/ProjectionPipeline.ts‎

Lines changed: 9 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ import {
3535
ProjectionTurnRepository,
3636
} from "../../persistence/Services/ProjectionTurns.ts";
3737
import { ProjectionThreadRepository } from "../../persistence/Services/ProjectionThreads.ts";
38-
import { ProjectionQueuedTurnRepository } from "../../persistence/Services/ProjectionQueuedTurns.ts";
38+
import * as ProjectionQueuedTurns from "../../persistence/ProjectionQueuedTurns.ts";
3939
import { ProjectionPendingApprovalRepositoryLive } from "../../persistence/Layers/ProjectionPendingApprovals.ts";
4040
import { ProjectionProjectRepositoryLive } from "../../persistence/Layers/ProjectionProjects.ts";
4141
import { ProjectionStateRepositoryLive } from "../../persistence/Layers/ProjectionState.ts";
@@ -45,7 +45,6 @@ import { ProjectionThreadProposedPlanRepositoryLive } from "../../persistence/La
4545
import { ProjectionThreadSessionRepositoryLive } from "../../persistence/Layers/ProjectionThreadSessions.ts";
4646
import { ProjectionTurnRepositoryLive } from "../../persistence/Layers/ProjectionTurns.ts";
4747
import { ProjectionThreadRepositoryLive } from "../../persistence/Layers/ProjectionThreads.ts";
48-
import { ProjectionQueuedTurnRepositoryLive } from "../../persistence/Layers/ProjectionQueuedTurns.ts";
4948
import { ServerConfig } from "../../config.ts";
5049
import {
5150
OrchestrationProjectionPipeline,
@@ -288,7 +287,9 @@ function retainProjectionMessagesAfterRevert(
288287
}
289288
}
290289

291-
return messages.filter((message) => retainedMessageIds.has(message.messageId));
290+
return messages.filter(
291+
(message) => message.deliveryState !== "queued" && retainedMessageIds.has(message.messageId),
292+
);
292293
}
293294

294295
function retainProjectionActivitiesAfterRevert(
@@ -483,7 +484,8 @@ const makeOrchestrationProjectionPipeline = Effect.fn("makeOrchestrationProjecti
483484
const projectionThreadActivityRepository = yield* ProjectionThreadActivityRepository;
484485
const projectionThreadSessionRepository = yield* ProjectionThreadSessionRepository;
485486
const projectionTurnRepository = yield* ProjectionTurnRepository;
486-
const projectionQueuedTurnRepository = yield* ProjectionQueuedTurnRepository;
487+
const projectionQueuedTurnRepository =
488+
yield* ProjectionQueuedTurns.ProjectionQueuedTurnRepository;
487489
const projectionPendingApprovalRepository = yield* ProjectionPendingApprovalRepository;
488490

489491
const fileSystem = yield* FileSystem.FileSystem;
@@ -1097,7 +1099,7 @@ const makeOrchestrationProjectionPipeline = Effect.fn("makeOrchestrationProjecti
10971099
return;
10981100

10991101
case "thread.queued-turn-dispatched":
1100-
yield* projectionQueuedTurnRepository.markReady({
1102+
yield* projectionQueuedTurnRepository.markHandoff({
11011103
messageId: event.payload.messageId,
11021104
});
11031105
return;
@@ -1115,7 +1117,7 @@ const makeOrchestrationProjectionPipeline = Effect.fn("makeOrchestrationProjecti
11151117
event.payload.session.status === "stopped" ||
11161118
event.payload.session.status === "interrupted"
11171119
) {
1118-
yield* projectionQueuedTurnRepository.deleteReadyByThreadId({
1120+
yield* projectionQueuedTurnRepository.deleteHandoffByThreadId({
11191121
threadId: event.payload.threadId,
11201122
});
11211123
}
@@ -1857,7 +1859,7 @@ export const OrchestrationProjectionPipelineLive = Layer.effect(
18571859
Layer.provideMerge(ProjectionThreadActivityRepositoryLive),
18581860
Layer.provideMerge(ProjectionThreadSessionRepositoryLive),
18591861
Layer.provideMerge(ProjectionTurnRepositoryLive),
1860-
Layer.provideMerge(ProjectionQueuedTurnRepositoryLive),
1862+
Layer.provideMerge(ProjectionQueuedTurns.layer),
18611863
Layer.provideMerge(ProjectionPendingApprovalRepositoryLive),
18621864
Layer.provideMerge(ProjectionStateRepositoryLive),
18631865
);

‎apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.test.ts‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -455,6 +455,7 @@ projectionSnapshotLayer("ProjectionSnapshotQuery", (it) => {
455455
updatedAt: "2026-02-24T00:00:07.000Z",
456456
},
457457
latestUserMessageAt: "2026-02-24T00:00:04.000Z",
458+
hasQueuedTurns: false,
458459
hasPendingApprovals: true,
459460
hasPendingUserInput: false,
460461
hasActionableProposedPlan: false,
@@ -2405,6 +2406,73 @@ projectionSnapshotLayer("ProjectionSnapshotQuery windowed thread detail", (it) =
24052406
}),
24062407
);
24072408

2409+
it.effect("hydrates durable queued messages into command and shell reads", () =>
2410+
Effect.gen(function* () {
2411+
const sql = yield* SqlClient.SqlClient;
2412+
const snapshotQuery = yield* ProjectionSnapshotQuery;
2413+
2414+
yield* sql`DELETE FROM projection_thread_turn_queue`;
2415+
yield* sql`DELETE FROM projection_thread_messages`;
2416+
yield* sql`DELETE FROM projection_thread_sessions`;
2417+
yield* sql`DELETE FROM projection_thread_proposed_plans`;
2418+
yield* sql`DELETE FROM projection_turns`;
2419+
yield* sql`DELETE FROM projection_threads`;
2420+
yield* sql`DELETE FROM projection_projects`;
2421+
yield* sql`DELETE FROM projection_state`;
2422+
2423+
yield* sql`
2424+
INSERT INTO projection_projects (
2425+
project_id, title, workspace_root, scripts_json, created_at, updated_at, deleted_at
2426+
) VALUES (
2427+
'project-queue-read', 'Queue read', '/tmp/queue-read', '[]',
2428+
'2026-08-16T12:00:00.000Z', '2026-08-16T12:00:00.000Z', NULL
2429+
)
2430+
`;
2431+
yield* sql`
2432+
INSERT INTO projection_threads (
2433+
thread_id, project_id, title, model_selection_json, runtime_mode, interaction_mode,
2434+
pending_approval_count, pending_user_input_count, has_actionable_proposed_plan,
2435+
created_at, updated_at, deleted_at
2436+
) VALUES (
2437+
'thread-queue-read', 'project-queue-read', 'Queue read',
2438+
'{"provider":"codex","model":"gpt-5-codex"}', 'full-access', 'default',
2439+
0, 0, 0, '2026-08-16T12:00:00.000Z', '2026-08-16T12:01:00.000Z', NULL
2440+
)
2441+
`;
2442+
yield* sql`
2443+
INSERT INTO projection_thread_messages (
2444+
message_id, thread_id, turn_id, role, text, is_streaming,
2445+
created_at, updated_at, delivery_state
2446+
) VALUES
2447+
('message-history', 'thread-queue-read', NULL, 'user', 'Already delivered', 0,
2448+
'2026-08-16T12:00:00.000Z', '2026-08-16T12:00:00.000Z', NULL),
2449+
('message-queued', 'thread-queue-read', NULL, 'user', 'Run next', 0,
2450+
'2026-08-16T12:01:00.000Z', '2026-08-16T12:01:00.000Z', 'queued')
2451+
`;
2452+
yield* sql`
2453+
INSERT INTO projection_thread_turn_queue (
2454+
message_id, thread_id, event_id, command_id, model_selection_json, title_seed,
2455+
runtime_mode, interaction_mode, source_proposed_plan_thread_id,
2456+
source_proposed_plan_id, queued_at, event_sequence, status
2457+
) VALUES (
2458+
'message-queued', 'thread-queue-read', 'event-queue-read', 'command-queue-read',
2459+
NULL, NULL, 'full-access', 'default', NULL, NULL,
2460+
'2026-08-16T12:01:00.000Z', 1, 'queued'
2461+
)
2462+
`;
2463+
2464+
const commandRead = yield* snapshotQuery.getCommandReadModel();
2465+
assert.deepEqual(
2466+
commandRead.threads[0]?.messages.map((message) => message.id),
2467+
[asMessageId("message-queued")],
2468+
);
2469+
assert.equal(commandRead.threads[0]?.messages[0]?.deliveryState, "queued");
2470+
2471+
const shellRead = yield* snapshotQuery.getShellSnapshot();
2472+
assert.equal(shellRead.threads[0]?.hasQueuedTurns, true);
2473+
}),
2474+
);
2475+
24082476
it.effect("a thread with no turns returns its content unwindowed on the first page", () =>
24092477
Effect.gen(function* () {
24102478
const sql = yield* SqlClient.SqlClient;

0 commit comments

Comments
 (0)