Skip to content

Commit a13975b

Browse files
committed
fix(server): snoozed threads allow queued automatic wakes
- Keep notification and delegated completion runs queued when snoozing. - Cancel buffered provider wakes on Stop and runtime recovery.
1 parent 42160f2 commit a13975b

8 files changed

Lines changed: 431 additions & 22 deletions

‎apps/server/src/orchestration-v2/NotificationMailbox.ts‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,22 @@
11
import {
22
latestProviderTurnForAttempt,
33
type MessageId,
4+
type OrchestrationV2ConversationMessage,
45
type OrchestrationV2ThreadProjection,
56
} from "@t3tools/contracts";
67

8+
/** A wake attaches to output held in provider memory, which Stop or process loss discards. */
9+
export function isBufferedProviderNotification(
10+
message: OrchestrationV2ConversationMessage,
11+
): boolean {
12+
return (
13+
message.createdBy === "agent" &&
14+
message.creationSource === "provider" &&
15+
message.notification !== undefined &&
16+
message.delegatedCompletion === undefined
17+
);
18+
}
19+
720
/**
821
* A persisted steer without an acceptance receipt can be retried as a continuation.
922
* Delivery is at least once: provider acceptance and our receipt cannot commit

‎apps/server/src/orchestration-v2/Orchestrator.ts‎

Lines changed: 56 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -89,7 +89,10 @@ import { CommandReceiptStoreV2 } from "./CommandReceiptStore.ts";
8989
import { ContextHandoffServiceV2 } from "./ContextHandoffService.ts";
9090
import { notificationTurnItem } from "@t3tools/provider-core/server/notification";
9191
import { isRestartNoteSource } from "./RestartBackgroundNote.ts";
92-
import { isUndeliveredMailboxSteer } from "./NotificationMailbox.ts";
92+
import {
93+
isBufferedProviderNotification,
94+
isUndeliveredMailboxSteer,
95+
} from "./NotificationMailbox.ts";
9396
import { EventSinkV2 } from "./EventSink.ts";
9497
import * as EffectOutbox from "./EffectOutbox.ts";
9598
import type { OrchestrationEffectRequestV2, PendingOrchestrationEffectV2 } from "./EffectOutbox.ts";
@@ -2798,7 +2801,24 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
27982801
cause: `Thread ${command.threadId} has a pending approval or user-input request and cannot be snoozed.`,
27992802
});
28002803
}
2801-
if (projection.runs.some((run) => run.status === "queued")) {
2804+
const queuedRuns = projection.runs.filter((run) => run.status === "queued");
2805+
const queuedMessages =
2806+
queuedRuns.length === 0
2807+
? []
2808+
: (yield* loadProjectionForCommand(command, ["messages"], {
2809+
messageIds: queuedRuns.map((run) => run.userMessageId),
2810+
})).messages;
2811+
// Automatic wakes are hidden from the user queue and must not block snooze.
2812+
// Snooze only changes visibility, so keep those deliveries intact.
2813+
const automaticMessageIds = new Set(
2814+
queuedMessages
2815+
.filter(
2816+
(message) =>
2817+
message.notification !== undefined || message.delegatedCompletion !== undefined,
2818+
)
2819+
.map((message) => message.id),
2820+
);
2821+
if (queuedRuns.some((run) => !automaticMessageIds.has(run.userMessageId))) {
28022822
return yield* new OrchestratorDispatchError({
28032823
commandId: command.commandId,
28042824
commandType: command.type,
@@ -8797,9 +8817,9 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
87978817
});
87988818

87998819
/**
8800-
* What Stop holds besides the run it interrupts: queued runs wait for the user, the
8801-
* thread's pull request watches end, and every wake its delegated tasks still owe is
8802-
* dropped, since Stop stops those tasks too. Nothing automatic starts the thread again.
8820+
* What Stop holds besides the run it interrupts: durable queued messages wait for the
8821+
* user, buffered provider wakes are cancelled, and the thread's pull request watches end.
8822+
* Every wake its delegated tasks still owe is dropped, since Stop stops those tasks too.
88038823
* `cohortRunIds` are dropped even when they owe nothing, like a plain interrupt's.
88048824
*/
88058825
const holdStoppedThread = (input: {
@@ -8815,6 +8835,22 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
88158835
Effect.gen(function* () {
88168836
const emitEvent = emit(input.events, input.command);
88178837
const { thread, runs, subagents } = input.projection;
8838+
const queuedRuns = runs.filter((run) => run.status === "queued");
8839+
const queuedMessages =
8840+
queuedRuns.length === 0
8841+
? []
8842+
: (yield* projectionStore
8843+
.getThreadRecords(thread.id, ["messages"], {
8844+
messageIds: queuedRuns.map((run) => run.userMessageId),
8845+
})
8846+
.pipe(
8847+
Effect.mapError(
8848+
(cause) => new OrchestratorProjectionError({ threadId: thread.id, cause }),
8849+
),
8850+
)).messages;
8851+
const bufferedMessageIds = new Set(
8852+
queuedMessages.filter(isBufferedProviderNotification).map((message) => message.id),
8853+
);
88188854
for (const run of runs) {
88198855
// A resumed queue can have a lower ordinal than a pending continuation source.
88208856
// Stop must reach those sources even when interrupting the resumed run succeeds.
@@ -8824,7 +8860,21 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
88248860
) {
88258861
yield* markStoppedRun({ ...input, thread, run });
88268862
}
8827-
if (run.status !== "queued" || run.queueHeld === true) continue;
8863+
if (run.status !== "queued") continue;
8864+
// Stop discards the provider's wake buffer, so its queued placeholders cannot resume.
8865+
if (bufferedMessageIds.has(run.userMessageId)) {
8866+
yield* dispatchQueuedRunCancel(
8867+
{
8868+
type: "queued-run.cancel",
8869+
commandId: input.command.commandId,
8870+
threadId: thread.id,
8871+
runId: run.id,
8872+
},
8873+
input.events,
8874+
);
8875+
continue;
8876+
}
8877+
if (run.queueHeld === true) continue;
88288878
yield* emitEvent({
88298879
type: "run.updated",
88308880
threadId: thread.id,

‎apps/server/src/orchestration-v2/ProjectionRecovery.test.ts‎

Lines changed: 103 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,14 +23,34 @@ import * as Layer from "effect/Layer";
2323
import * as SqlClient from "effect/sql/SqlClient";
2424

2525
import * as SqlitePersistence from "../persistence/Sqlite.ts";
26+
import * as ServerSettings from "../serverSettings.ts";
2627
import { CodexProviderCapabilitiesV2 } from "./Adapters/CodexAdapterV2.ts";
2728
import * as EffectOutbox from "./EffectOutbox.ts";
2829
import * as ProjectionStore from "./ProjectionStore.ts";
30+
import * as EventSink from "./EventSink.ts";
31+
import * as EventStore from "./EventStore.ts";
32+
import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator";
33+
import * as ProviderRuntimeRecovery from "./ProviderRuntimeRecoveryService.ts";
2934
import { restartContinuationRun } from "./RestartContinuation.ts";
3035

3136
const layerTest = Layer.mergeAll(ProjectionStore.layer, EffectOutbox.layer).pipe(
3237
Layer.provideMerge(SqlitePersistence.layerMemory),
3338
);
39+
const layerRecoveryTest = ProviderRuntimeRecovery.layer.pipe(
40+
Layer.provideMerge(
41+
EventSink.layer.pipe(
42+
Layer.provideMerge(
43+
Layer.mergeAll(
44+
EventStore.layer,
45+
ProjectionStore.layer,
46+
EffectOutbox.layer,
47+
IdAllocator.layer,
48+
).pipe(Layer.provideMerge(SqlitePersistence.layerMemory)),
49+
),
50+
),
51+
),
52+
Layer.provide(ServerSettings.layerTest()),
53+
);
3454
const providerInstanceId = ProviderInstanceId.make("codex");
3555
const modelSelection = { instanceId: providerInstanceId, model: "gpt-5.4" };
3656
const driver = ProviderDriverKind.make("codex");
@@ -113,6 +133,89 @@ const createRun = Effect.fn(function* (
113133
return runId;
114134
});
115135

136+
it.effect.each(["startup", "shutdown"] as const)(
137+
"%s cancels buffered provider wakes left behind by completed turns",
138+
(trigger) =>
139+
Effect.gen(function* () {
140+
const projections = yield* ProjectionStore.ProjectionStoreV2;
141+
const recovery = yield* ProviderRuntimeRecovery.ProviderRuntimeRecoveryService;
142+
const now = yield* DateTime.now;
143+
const bufferedThreads = [] as ThreadId[];
144+
const durableThreads = [] as ThreadId[];
145+
for (const kind of [
146+
"buffered",
147+
"held-buffered",
148+
"user",
149+
"notification",
150+
"delegated",
151+
] as const) {
152+
const threadId = yield* createThread(`wake:${trigger}:${kind}`);
153+
const buffered = kind === "buffered" || kind === "held-buffered";
154+
(buffered ? bufferedThreads : durableThreads).push(threadId);
155+
const messageId = MessageId.make(`message:${threadId}`);
156+
const runId = yield* createRun(threadId, "queued", {
157+
userMessageId: messageId,
158+
startedAt: null,
159+
queueHeld: kind !== "buffered",
160+
queuePosition: 1,
161+
});
162+
yield* createRun(threadId, "completed", { ordinal: 2 });
163+
yield* projections.apply({
164+
id: EventId.make(`message:${threadId}`),
165+
type: "message.updated",
166+
threadId,
167+
runId,
168+
occurredAt: now,
169+
payload: {
170+
id: messageId,
171+
threadId,
172+
runId,
173+
nodeId: null,
174+
role: "user",
175+
text: "Queued work",
176+
attachments: [],
177+
streaming: false,
178+
createdAt: now,
179+
updatedAt: now,
180+
createdBy: kind === "user" ? "user" : "agent",
181+
creationSource: buffered ? "provider" : "server",
182+
...(kind === "user" || kind === "delegated"
183+
? {}
184+
: {
185+
notification: {
186+
source: { kind: "background_task" as const },
187+
outcome: "updated" as const,
188+
summary: "Background activity updated",
189+
},
190+
}),
191+
...(kind === "delegated"
192+
? { delegatedCompletion: { parentRunId: runId, generation: 1, taskIds: [] } }
193+
: {}),
194+
},
195+
});
196+
}
197+
assert.deepEqual(
198+
new Set(yield* projections.getRecoveryThreadIds("runtime")),
199+
new Set(bufferedThreads),
200+
);
201+
const summary = yield* recovery.reconcile(trigger);
202+
assert.equal(summary.terminalizedRuns, 2);
203+
for (const threadId of bufferedThreads) {
204+
const projection = yield* projections.getThreadProjection(threadId);
205+
assert.equal(projection.runs[0]?.status, "cancelled");
206+
assert.isNotNull(projection.runs[0]?.completedAt);
207+
assert.isNull(projection.runs[0]?.queuePosition);
208+
assert.equal(projection.runs[1]?.status, "completed");
209+
}
210+
for (const threadId of durableThreads) {
211+
const projection = yield* projections.getThreadProjection(threadId);
212+
assert.equal(projection.runs[0]?.status, "queued");
213+
assert.isTrue(projection.runs[0]?.queueHeld);
214+
}
215+
assert.deepEqual(yield* projections.getRecoveryThreadIds("runtime"), []);
216+
}).pipe(Effect.provide(layerRecoveryTest)),
217+
);
218+
116219
it.effect("selects unfinished recovery work without reading settled thread histories", () =>
117220
Effect.gen(function* () {
118221
const projections = yield* ProjectionStore.ProjectionStoreV2;

‎apps/server/src/orchestration-v2/ProjectionStore.ts‎

Lines changed: 35 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,7 @@ import {
6969
isOrchestrationV2SupersededInterrupt,
7070
} from "@t3tools/shared/orchestrationV2Timeline";
7171
import { derivePendingBackgroundWork } from "@t3tools/shared/orchestrationV2PendingBackgroundWork";
72+
import { isBufferedProviderNotification } from "./NotificationMailbox.ts";
7273
import * as Context from "effect/Context";
7374
import * as DateTime from "effect/DateTime";
7475
import * as Effect from "effect/Effect";
@@ -592,7 +593,12 @@ function needsRecovery(
592593
projection.runs.some(
593594
(run) =>
594595
["preparing", "starting", "running", "waiting"].includes(run.status) ||
595-
(run.status === "queued" && run.queueHeld !== true),
596+
(run.status === "queued" &&
597+
(run.queueHeld !== true ||
598+
projection.messages.some(
599+
(message) =>
600+
message.id === run.userMessageId && isBufferedProviderNotification(message),
601+
))),
596602
) ||
597603
projection.runtimeRequests.some((request) => request.status === "pending") ||
598604
projection.subagents.some(
@@ -3639,13 +3645,23 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
36393645
SELECT thread_id FROM orchestration_v2_projection_runs
36403646
WHERE status IN ('preparing', 'starting', 'running', 'waiting')
36413647
UNION
3642-
-- A held queue already went through recovery; rereading it on
3643-
-- every boot costs a projection read per held thread.
3644-
SELECT thread_id FROM orchestration_v2_projection_runs
3645-
WHERE status = 'queued'
3646-
AND CASE WHEN json_valid(payload_json)
3647-
THEN json_extract(payload_json, '$.queueHeld') IS NOT 1
3648-
ELSE 1 END
3648+
-- Held durable queues need no reread. Old buffered wakes still
3649+
-- need cancellation even if earlier recovery held them.
3650+
SELECT queued.thread_id FROM orchestration_v2_projection_runs AS queued
3651+
WHERE queued.status = 'queued'
3652+
AND (CASE WHEN json_valid(queued.payload_json)
3653+
THEN json_extract(queued.payload_json, '$.queueHeld') IS NOT 1
3654+
ELSE 1 END OR EXISTS (
3655+
SELECT 1 FROM orchestration_v2_projection_messages AS message
3656+
WHERE message.message_id = CASE WHEN json_valid(queued.payload_json)
3657+
THEN json_extract(queued.payload_json, '$.userMessageId') END
3658+
AND CASE WHEN json_valid(message.payload_json) THEN
3659+
json_extract(message.payload_json, '$.createdBy') = 'agent'
3660+
AND json_extract(message.payload_json, '$.creationSource') = 'provider'
3661+
AND json_type(message.payload_json, '$.notification') = 'object'
3662+
AND json_type(message.payload_json, '$.delegatedCompletion') IS NULL
3663+
ELSE 0 END
3664+
))
36493665
UNION
36503666
SELECT thread_id FROM orchestration_v2_projection_runtime_requests
36513667
WHERE status = 'pending'
@@ -4191,12 +4207,18 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
41914207
`,
41924208
sql<PayloadRow>`
41934209
SELECT message.payload_json FROM orchestration_v2_projection_messages AS message
4194-
WHERE message.thread_id = ${threadId} AND message.streaming = 1
4195-
AND message.run_id IN (
4196-
SELECT run_id FROM orchestration_v2_projection_runs
4197-
WHERE thread_id = ${threadId}
4198-
AND status IN ('queued', 'preparing', 'starting', 'running', 'waiting')
4210+
WHERE message.thread_id = ${threadId} AND (
4211+
(message.streaming = 1 AND message.run_id IN (
4212+
SELECT run_id FROM orchestration_v2_projection_runs
4213+
WHERE thread_id = ${threadId}
4214+
AND status IN ('queued', 'preparing', 'starting', 'running', 'waiting')
4215+
))
4216+
OR message.message_id IN (
4217+
SELECT json_extract(payload_json, '$.userMessageId')
4218+
FROM orchestration_v2_projection_runs
4219+
WHERE thread_id = ${threadId} AND status = 'queued'
41994220
)
4221+
)
42004222
ORDER BY message.created_at ASC, message.message_id ASC
42014223
`,
42024224
sql<PayloadRow>`

‎apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.test.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -147,6 +147,7 @@ it.effect("expires orphaned runtime requests before command readiness", () => {
147147
providerThreads: [],
148148
runs: [],
149149
nodes: [],
150+
messages: [],
150151
} as unknown as OrchestrationV2ThreadProjection;
151152
const layer = ProviderRuntimeRecovery.layer.pipe(
152153
Layer.provide(ServerSettings.layerTest()),
@@ -197,6 +198,7 @@ it.effect("preserves async questions across startup and shutdown", () => {
197198
providerThreads: [],
198199
runs: [],
199200
nodes: [],
201+
messages: [],
200202
turnItems: [],
201203
} as unknown as OrchestrationV2ThreadProjection;
202204
const commitCommand = vi.fn(() => Effect.die("an async question needs no process-loss write"));
@@ -247,6 +249,7 @@ it.effect("uses the same reconciliation path to cancel runtime requests during s
247249
providerThreads: [],
248250
runs: [],
249251
nodes: [],
252+
messages: [],
250253
} as unknown as OrchestrationV2ThreadProjection;
251254
const layer = ProviderRuntimeRecovery.layer.pipe(
252255
Layer.provide(ServerSettings.layerTest()),

‎apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts‎

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator";
2222
import * as ProjectionStore from "./ProjectionStore.ts";
2323
import * as ServerSettings from "../serverSettings.ts";
2424
import { restartContinuationRun } from "./RestartContinuation.ts";
25+
import { isBufferedProviderNotification } from "./NotificationMailbox.ts";
2526
import {
2627
endOrphanedNativeSubagent,
2728
endRunlessRootTurns,
@@ -284,10 +285,18 @@ export const make = Effect.gen(function* () {
284285
cancelledBackgroundNativeIds.add(item.nativeItemRef.nativeId);
285286
}
286287
};
287-
// Queued runs have not started provider work. Preserve their execution
288-
// identities and order, but require explicit consent before draining them.
288+
const bufferedMessageIds = new Set(
289+
projection.messages.filter(isBufferedProviderNotification).map((message) => message.id),
290+
);
291+
// Buffered wakes lost their output with the process. Durable queued messages
292+
// keep their execution identities and order until the user resumes them.
289293
for (const run of projection.runs) {
290-
if (run.status !== "queued" || run.queueHeld === true) continue;
294+
if (run.status !== "queued") continue;
295+
if (bufferedMessageIds.has(run.userMessageId)) {
296+
runs.push(run);
297+
continue;
298+
}
299+
if (run.queueHeld === true) continue;
291300
events.push({
292301
id: yield* allocateEventId(),
293302
type: "run.updated",

0 commit comments

Comments
 (0)