Skip to content

Commit 307e0ff

Browse files
fix(server): held queues release agent and scheduled messages once the thread completes a run
A provider failure, a restart or a stop holds a thread's queue until someone resumes it or sends a client message. Threads fed by agents, MCP tools or scheduled tasks never get either, so every message that arrived while they were busy stayed held forever (the issue triager held 165 issue dispatches, oldest a day old) while messages that found the thread idle ran past them. A hold now waits only for messages a person wrote. When a run on the thread completes, held messages from agents, the server or scheduled tasks are released and run in order. A run that completes after a stop request releases nothing, and held user messages still wait for the user. Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent dd14848 commit 307e0ff

4 files changed

Lines changed: 334 additions & 15 deletions

File tree

‎apps/server/src/orchestration-v2/Orchestrator.ts‎

Lines changed: 73 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -741,6 +741,20 @@ function delegatedTaskTerminalStatus(
741741
}
742742
}
743743

744+
/** Failed worktree setup keeps the queue held so queued messages cannot start in the project checkout. */
745+
function worktreeSetupPending(
746+
projection: Pick<OrchestrationV2ThreadProjection, "thread" | "runs">,
747+
): boolean {
748+
return (
749+
projection.thread.worktreePath === null &&
750+
projection.runs.some(
751+
(run) =>
752+
(run.status === "failed" || run.status === "interrupted") &&
753+
run.workspacePreparation?.type === "worktree",
754+
)
755+
);
756+
}
757+
744758
function nextQueuedRun(
745759
projection: Pick<OrchestrationV2ThreadProjection, "runs" | "messages">,
746760
): OrchestrationV2Run | undefined {
@@ -2195,6 +2209,52 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
21952209
),
21962210
);
21972211

2212+
// A hold waits for a person to review what they queued. Nobody reviews messages from agents,
2213+
// the server or scheduled tasks, so a later completed run releases those unless it was stopped.
2214+
const releaseUnattendedHolds = Effect.fn("Orchestrator.releaseUnattendedHolds")(function* (
2215+
threadId: ThreadId,
2216+
completedRunId: RunId,
2217+
) {
2218+
const held = yield* projectionStore.getHeldQueuedRuns(threadId);
2219+
if (held.length === 0) return;
2220+
const projection = yield* projectionStore.getThreadRecords(
2221+
threadId,
2222+
["runs", "messages", "turnItems"],
2223+
{
2224+
messageIds: held.map((run) => run.userMessageId),
2225+
turnItemTypes: ["run_interrupt_request"],
2226+
turnItemRunIds: [completedRunId],
2227+
},
2228+
);
2229+
// Only the newest run completing shows the thread moved past the hold; a late or
2230+
// bookkeeping update to an older run, overtaken by a failure or Stop, does not.
2231+
if (
2232+
latestExecutedRun(projection.runs)?.id !== completedRunId ||
2233+
projection.turnItems.length > 0 ||
2234+
worktreeSetupPending(projection)
2235+
)
2236+
return;
2237+
const { messages } = projection;
2238+
const unattended = new Set(
2239+
messages
2240+
.filter((message) => message.createdBy !== "user" || message.scheduledTaskId !== undefined)
2241+
.map((message) => message.id),
2242+
);
2243+
const now = yield* DateTime.now;
2244+
const released = held.filter((run) => unattended.has(run.userMessageId));
2245+
if (released.length === 0) return;
2246+
yield* writeSystemEvents(
2247+
released.map((run) => ({
2248+
type: "run.updated" as const,
2249+
threadId,
2250+
runId: run.id,
2251+
providerInstanceId: run.providerInstanceId,
2252+
occurredAt: now,
2253+
payload: { ...run, queueHeld: false },
2254+
})),
2255+
);
2256+
});
2257+
21982258
const resumeQueuedRuns = Effect.gen(function* () {
21992259
const threadIds = yield* projectionStore.getRecoveryThreadIds("queued-runs");
22002260
let resumed = 0;
@@ -5539,15 +5599,8 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
55395599
dispatchMode.type === "queue_after_active";
55405600
const shouldQueue = queueBehindRun !== undefined && startsOrQueuesRun;
55415601
// Client messages resume held queues; server continuations and scheduled tasks do not.
5542-
// Provider failure or usage limits re-hold the queue. Failed worktree setup keeps it held
5543-
// until a worktree exists, so queued messages cannot start in the project checkout.
5544-
const worktreeSetupUnresolved =
5545-
projection.thread.worktreePath === null &&
5546-
projection.runs.some(
5547-
(run) =>
5548-
(run.status === "failed" || run.status === "interrupted") &&
5549-
run.workspacePreparation?.type === "worktree",
5550-
);
5602+
// Provider failure or usage limits re-hold the queue.
5603+
const worktreeSetupUnresolved = worktreeSetupPending(projection);
55515604
const sentByUserFromClient =
55525605
command.createdBy === "user" &&
55535606
(command.creationSource === "web" || command.creationSource === "mobile") &&
@@ -13083,10 +13136,17 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
1308313136
yield* threadDispatch
1308413137
.withLock(
1308513138
threadId,
13086-
startNextQueuedRun(threadId, {
13087-
endedRunId: run.id,
13088-
...(run.status === "failed" ? { failedRunId: run.id } : {}),
13089-
}),
13139+
(run.status === "completed"
13140+
? releaseUnattendedHolds(threadId, run.id)
13141+
: Effect.void
13142+
).pipe(
13143+
Effect.andThen(
13144+
startNextQueuedRun(threadId, {
13145+
endedRunId: run.id,
13146+
...(run.status === "failed" ? { failedRunId: run.id } : {}),
13147+
}),
13148+
),
13149+
),
1309013150
)
1309113151
.pipe(
1309213152
Effect.catchCause((cause) =>

‎apps/server/src/orchestration-v2/ProjectionStore.ts‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -612,6 +612,9 @@ export interface ProjectionStoreV2Shape {
612612
readonly canStartQueuedRun: (
613613
threadId: ThreadId,
614614
) => Effect.Effect<boolean, ProjectionStoreV2Error>;
615+
readonly getHeldQueuedRuns: (
616+
threadId: ThreadId,
617+
) => Effect.Effect<ReadonlyArray<OrchestrationV2Run>, ProjectionStoreV2Error>;
615618
readonly getRecoveryThreadIds: (
616619
kind: ProjectionRecoveryKind,
617620
) => Effect.Effect<ReadonlyArray<ThreadId>, ProjectionStoreV2Error>;
@@ -5183,6 +5186,17 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
51835186
)
51845187
.pipe(Effect.mapError(controlReadError(threadId)));
51855188

5189+
const getHeldQueuedRuns: ProjectionStoreV2Shape["getHeldQueuedRuns"] = (threadId) =>
5190+
sql<PayloadRow>`
5191+
SELECT payload_json FROM orchestration_v2_projection_runs
5192+
WHERE thread_id = ${threadId} AND status = 'queued'
5193+
AND json_extract(payload_json, '$.queueHeld') IS 1
5194+
ORDER BY ordinal ASC
5195+
`.pipe(
5196+
Effect.flatMap(decodeRows(decodeRunPayload, threadId)),
5197+
Effect.mapError(controlReadError(threadId)),
5198+
);
5199+
51865200
const getMessageCount: ProjectionStoreV2Shape["getMessageCount"] = (threadId) =>
51875201
sql<{
51885202
count: number;
@@ -6899,6 +6913,7 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
68996913
getCheckpointRollbackContext: readOnReader(getCheckpointRollbackContext),
69006914
getRunMessage: readOnReader(getRunMessage),
69016915
canStartQueuedRun: readOnReader(canStartQueuedRun),
6916+
getHeldQueuedRuns: readOnReader(getHeldQueuedRuns),
69026917
getPendingNativeUserInputs: readOnReader(getPendingNativeUserInputs),
69036918
hasUnpairedRunInterruptRequest: readOnReader(hasUnpairedRunInterruptRequest),
69046919
isNodeStored: readOnReader(isNodeStored),

‎apps/server/src/orchestration-v2/ProviderTurnControlService.test.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -263,6 +263,7 @@ it.effect(
263263
getRunMessage: () => Effect.die("not used"),
264264
isNodeStored: () => Effect.die("not used"),
265265
canStartQueuedRun: () => Effect.die("not used"),
266+
getHeldQueuedRuns: () => Effect.die("not used"),
266267
getRecoveryThreadIds: () => Effect.die("unused getRecoveryThreadIds"),
267268
getUnreadableThreadIds: () => Effect.die("unused getUnreadableThreadIds"),
268269
getThreadSnapshot: () => Effect.die("unused getThreadSnapshot"),

0 commit comments

Comments
 (0)