Skip to content

Commit 6c0a0a4

Browse files
committed
fix(server): avoid rereading held queues during recovery
- Read only queued message records when checking buffered wakes - Keep held queues out of recovery scans
1 parent a13975b commit 6c0a0a4

5 files changed

Lines changed: 40 additions & 64 deletions

File tree

‎apps/server/src/orchestration-v2/ProjectionRecovery.test.ts‎

Lines changed: 5 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -134,29 +134,22 @@ const createRun = Effect.fn(function* (
134134
});
135135

136136
it.effect.each(["startup", "shutdown"] as const)(
137-
"%s cancels buffered provider wakes left behind by completed turns",
137+
"%s cancels queued buffered wakes and holds durable messages",
138138
(trigger) =>
139139
Effect.gen(function* () {
140140
const projections = yield* ProjectionStore.ProjectionStoreV2;
141141
const recovery = yield* ProviderRuntimeRecovery.ProviderRuntimeRecoveryService;
142142
const now = yield* DateTime.now;
143143
const bufferedThreads = [] as ThreadId[];
144144
const durableThreads = [] as ThreadId[];
145-
for (const kind of [
146-
"buffered",
147-
"held-buffered",
148-
"user",
149-
"notification",
150-
"delegated",
151-
] as const) {
145+
for (const kind of ["buffered", "user", "notification", "delegated"] as const) {
152146
const threadId = yield* createThread(`wake:${trigger}:${kind}`);
153-
const buffered = kind === "buffered" || kind === "held-buffered";
147+
const buffered = kind === "buffered";
154148
(buffered ? bufferedThreads : durableThreads).push(threadId);
155149
const messageId = MessageId.make(`message:${threadId}`);
156150
const runId = yield* createRun(threadId, "queued", {
157151
userMessageId: messageId,
158152
startedAt: null,
159-
queueHeld: kind !== "buffered",
160153
queuePosition: 1,
161154
});
162155
yield* createRun(threadId, "completed", { ordinal: 2 });
@@ -196,10 +189,10 @@ it.effect.each(["startup", "shutdown"] as const)(
196189
}
197190
assert.deepEqual(
198191
new Set(yield* projections.getRecoveryThreadIds("runtime")),
199-
new Set(bufferedThreads),
192+
new Set([...bufferedThreads, ...durableThreads]),
200193
);
201194
const summary = yield* recovery.reconcile(trigger);
202-
assert.equal(summary.terminalizedRuns, 2);
195+
assert.equal(summary.terminalizedRuns, 1);
203196
for (const threadId of bufferedThreads) {
204197
const projection = yield* projections.getThreadProjection(threadId);
205198
assert.equal(projection.runs[0]?.status, "cancelled");

‎apps/server/src/orchestration-v2/ProjectionStore.ts‎

Lines changed: 13 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,6 @@ import {
6969
isOrchestrationV2SupersededInterrupt,
7070
} from "@t3tools/shared/orchestrationV2Timeline";
7171
import { derivePendingBackgroundWork } from "@t3tools/shared/orchestrationV2PendingBackgroundWork";
72-
import { isBufferedProviderNotification } from "./NotificationMailbox.ts";
7372
import * as Context from "effect/Context";
7473
import * as DateTime from "effect/DateTime";
7574
import * as Effect from "effect/Effect";
@@ -593,12 +592,7 @@ function needsRecovery(
593592
projection.runs.some(
594593
(run) =>
595594
["preparing", "starting", "running", "waiting"].includes(run.status) ||
596-
(run.status === "queued" &&
597-
(run.queueHeld !== true ||
598-
projection.messages.some(
599-
(message) =>
600-
message.id === run.userMessageId && isBufferedProviderNotification(message),
601-
))),
595+
(run.status === "queued" && run.queueHeld !== true),
602596
) ||
603597
projection.runtimeRequests.some((request) => request.status === "pending") ||
604598
projection.subagents.some(
@@ -3645,23 +3639,13 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
36453639
SELECT thread_id FROM orchestration_v2_projection_runs
36463640
WHERE status IN ('preparing', 'starting', 'running', 'waiting')
36473641
UNION
3648-
-- Held durable queues need no reread. Old buffered wakes still
3649-
-- need cancellation even if earlier recovery held them.
3650-
SELECT queued.thread_id FROM orchestration_v2_projection_runs AS queued
3651-
WHERE queued.status = 'queued'
3652-
AND (CASE WHEN json_valid(queued.payload_json)
3653-
THEN json_extract(queued.payload_json, '$.queueHeld') IS NOT 1
3654-
ELSE 1 END OR EXISTS (
3655-
SELECT 1 FROM orchestration_v2_projection_messages AS message
3656-
WHERE message.message_id = CASE WHEN json_valid(queued.payload_json)
3657-
THEN json_extract(queued.payload_json, '$.userMessageId') END
3658-
AND CASE WHEN json_valid(message.payload_json) THEN
3659-
json_extract(message.payload_json, '$.createdBy') = 'agent'
3660-
AND json_extract(message.payload_json, '$.creationSource') = 'provider'
3661-
AND json_type(message.payload_json, '$.notification') = 'object'
3662-
AND json_type(message.payload_json, '$.delegatedCompletion') IS NULL
3663-
ELSE 0 END
3664-
))
3642+
-- A held queue already went through recovery; rereading it on
3643+
-- every boot costs a projection read per held thread.
3644+
SELECT thread_id FROM orchestration_v2_projection_runs
3645+
WHERE status = 'queued'
3646+
AND CASE WHEN json_valid(payload_json)
3647+
THEN json_extract(payload_json, '$.queueHeld') IS NOT 1
3648+
ELSE 1 END
36653649
UNION
36663650
SELECT thread_id FROM orchestration_v2_projection_runtime_requests
36673651
WHERE status = 'pending'
@@ -4207,18 +4191,12 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
42074191
`,
42084192
sql<PayloadRow>`
42094193
SELECT message.payload_json FROM orchestration_v2_projection_messages AS message
4210-
WHERE message.thread_id = ${threadId} AND (
4211-
(message.streaming = 1 AND message.run_id IN (
4212-
SELECT run_id FROM orchestration_v2_projection_runs
4213-
WHERE thread_id = ${threadId}
4214-
AND status IN ('queued', 'preparing', 'starting', 'running', 'waiting')
4215-
))
4216-
OR message.message_id IN (
4217-
SELECT json_extract(payload_json, '$.userMessageId')
4218-
FROM orchestration_v2_projection_runs
4219-
WHERE thread_id = ${threadId} AND status = 'queued'
4194+
WHERE message.thread_id = ${threadId} AND message.streaming = 1
4195+
AND message.run_id IN (
4196+
SELECT run_id FROM orchestration_v2_projection_runs
4197+
WHERE thread_id = ${threadId}
4198+
AND status IN ('queued', 'preparing', 'starting', 'running', 'waiting')
42204199
)
4221-
)
42224200
ORDER BY message.created_at ASC, message.message_id ASC
42234201
`,
42244202
sql<PayloadRow>`

‎apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.test.ts‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -147,7 +147,6 @@ it.effect("expires orphaned runtime requests before command readiness", () => {
147147
providerThreads: [],
148148
runs: [],
149149
nodes: [],
150-
messages: [],
151150
} as unknown as OrchestrationV2ThreadProjection;
152151
const layer = ProviderRuntimeRecovery.layer.pipe(
153152
Layer.provide(ServerSettings.layerTest()),
@@ -198,7 +197,6 @@ it.effect("preserves async questions across startup and shutdown", () => {
198197
providerThreads: [],
199198
runs: [],
200199
nodes: [],
201-
messages: [],
202200
turnItems: [],
203201
} as unknown as OrchestrationV2ThreadProjection;
204202
const commitCommand = vi.fn(() => Effect.die("an async question needs no process-loss write"));
@@ -249,7 +247,6 @@ it.effect("uses the same reconciliation path to cancel runtime requests during s
249247
providerThreads: [],
250248
runs: [],
251249
nodes: [],
252-
messages: [],
253250
} as unknown as OrchestrationV2ThreadProjection;
254251
const layer = ProviderRuntimeRecovery.layer.pipe(
255252
Layer.provide(ServerSettings.layerTest()),
@@ -693,6 +690,7 @@ it.effect("holds accepted queued work without cancelling its execution state aft
693690
Layer.mock(ProjectionStore.ProjectionStoreV2)({
694691
getRecoveryThreadIds: () => Effect.succeed([threadId]),
695692
getRuntimeRecoveryProjection: () => Effect.succeed(projection),
693+
getThreadRecords: () => Effect.succeed(projection),
696694
}),
697695
Layer.mock(EventSink.EventSinkV2)({
698696
commitCommand: (input) => {

‎apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts‎

Lines changed: 20 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -285,13 +285,30 @@ export const make = Effect.gen(function* () {
285285
cancelledBackgroundNativeIds.add(item.nativeItemRef.nativeId);
286286
}
287287
};
288+
const queuedRuns = projection.runs.filter((run) => run.status === "queued");
289+
const queuedMessages =
290+
queuedRuns.length === 0
291+
? []
292+
: (yield* projections
293+
.getThreadRecords(projection.thread.id, ["messages"], {
294+
messageIds: queuedRuns.map((run) => run.userMessageId),
295+
})
296+
.pipe(
297+
Effect.mapError(
298+
(cause) =>
299+
new ProviderRuntimeRecoveryError({
300+
operation: "read-projections",
301+
threadId: projection.thread.id,
302+
cause,
303+
}),
304+
),
305+
)).messages;
288306
const bufferedMessageIds = new Set(
289-
projection.messages.filter(isBufferedProviderNotification).map((message) => message.id),
307+
queuedMessages.filter(isBufferedProviderNotification).map((message) => message.id),
290308
);
291309
// Buffered wakes lost their output with the process. Durable queued messages
292310
// keep their execution identities and order until the user resumes them.
293-
for (const run of projection.runs) {
294-
if (run.status !== "queued") continue;
311+
for (const run of queuedRuns) {
295312
if (bufferedMessageIds.has(run.userMessageId)) {
296313
runs.push(run);
297314
continue;

‎apps/server/src/orchestration-v2/ThreadStop.test.ts‎

Lines changed: 1 addition & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -226,11 +226,10 @@ it.effect("Stop ends watches, holds queues, and stops the delegated tasks under
226226
);
227227

228228
it.effect.each(["thread.stop", "run.interrupt"] as const)(
229-
"%s cancels buffered wakes, including held ones, and preserves durable queued messages",
229+
"%s cancels buffered wakes and preserves durable queued messages",
230230
(type) =>
231231
Effect.gen(function* () {
232232
const orchestrator = yield* Orchestrator.OrchestratorV2;
233-
const projections = yield* ProjectionStore.ProjectionStoreV2;
234233
const threadId = ThreadId.make(`thread:stop-buffered:${type}`);
235234
yield* createWatchingThread(threadId, 5);
236235
yield* send(threadId, "active", "start_immediately");
@@ -259,15 +258,6 @@ it.effect.each(["thread.stop", "run.interrupt"] as const)(
259258
(message) => message.id === run.userMessageId && message.creationSource === "provider",
260259
),
261260
)!;
262-
// Also clean a wake that an earlier Stop or recovery already held.
263-
yield* projections.apply({
264-
id: EventId.make(`held:${type}`),
265-
type: "run.updated",
266-
threadId,
267-
runId: buffered.id,
268-
occurredAt: yield* DateTime.now,
269-
payload: { ...buffered, queueHeld: true },
270-
});
271261
const stop = { commandId: CommandId.make(`stop:${type}`), threadId };
272262
yield* orchestrator.dispatch(
273263
type === "run.interrupt"

0 commit comments

Comments
 (0)