Skip to content

Commit 8ab098a

Browse files
fix(server): held queues release agent and scheduled messages once the thread completes a run (pingdotgg#27221)
Co-authored-by: Claude Opus 5.5 <[email protected]>
1 parent 44d508f commit 8ab098a

4 files changed

Lines changed: 351 additions & 15 deletions

File tree

‎apps/server/src/orchestration-v2/Orchestrator.ts‎

Lines changed: 73 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -741,6 +741,20 @@ function delegatedTaskTerminalStatus(
741741
}
742742
}
743743

744+
/** Failed worktree setup keeps the queue held so queued messages cannot start in the project checkout. */
745+
function worktreeSetupPending(
746+
projection: Pick<OrchestrationV2ThreadProjection, "thread" | "runs">,
747+
): boolean {
748+
return (
749+
projection.thread.worktreePath === null &&
750+
projection.runs.some(
751+
(run) =>
752+
(run.status === "failed" || run.status === "interrupted") &&
753+
run.workspacePreparation?.type === "worktree",
754+
)
755+
);
756+
}
757+
744758
function nextQueuedRun(
745759
projection: Pick<OrchestrationV2ThreadProjection, "runs" | "messages">,
746760
): OrchestrationV2Run | undefined {
@@ -2195,6 +2209,52 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
21952209
),
21962210
);
21972211

2212+
// A hold waits for a person to review what they queued. Nobody reviews messages from agents,
2213+
// the server or scheduled tasks, so a later completed run releases those unless it was stopped.
2214+
const releaseUnattendedHolds = Effect.fn("Orchestrator.releaseUnattendedHolds")(function* (
2215+
threadId: ThreadId,
2216+
completedRunId: RunId,
2217+
) {
2218+
const held = yield* projectionStore.getHeldQueuedRuns(threadId);
2219+
if (held.length === 0) return;
2220+
const projection = yield* projectionStore.getThreadRecords(
2221+
threadId,
2222+
["runs", "messages", "turnItems"],
2223+
{
2224+
messageIds: held.map((run) => run.userMessageId),
2225+
turnItemTypes: ["run_interrupt_request"],
2226+
turnItemRunIds: [completedRunId],
2227+
},
2228+
);
2229+
// Only the newest run completing shows the thread moved past the hold; a late or
2230+
// bookkeeping update to an older run, overtaken by a failure or Stop, does not.
2231+
if (
2232+
latestExecutedRun(projection.runs)?.id !== completedRunId ||
2233+
projection.turnItems.length > 0 ||
2234+
worktreeSetupPending(projection)
2235+
)
2236+
return;
2237+
const { messages } = projection;
2238+
const unattended = new Set(
2239+
messages
2240+
.filter((message) => message.createdBy !== "user" || message.scheduledTaskId !== undefined)
2241+
.map((message) => message.id),
2242+
);
2243+
const now = yield* DateTime.now;
2244+
const released = held.filter((run) => unattended.has(run.userMessageId));
2245+
if (released.length === 0) return;
2246+
yield* writeSystemEvents(
2247+
released.map((run) => ({
2248+
type: "run.updated" as const,
2249+
threadId,
2250+
runId: run.id,
2251+
providerInstanceId: run.providerInstanceId,
2252+
occurredAt: now,
2253+
payload: { ...run, queueHeld: false },
2254+
})),
2255+
);
2256+
});
2257+
21982258
const resumeQueuedRuns = Effect.gen(function* () {
21992259
const threadIds = yield* projectionStore.getRecoveryThreadIds("queued-runs");
22002260
let resumed = 0;
@@ -5539,15 +5599,8 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
55395599
dispatchMode.type === "queue_after_active";
55405600
const shouldQueue = queueBehindRun !== undefined && startsOrQueuesRun;
55415601
// Client messages resume held queues; server continuations and scheduled tasks do not.
5542-
// Provider failure or usage limits re-hold the queue. Failed worktree setup keeps it held
5543-
// until a worktree exists, so queued messages cannot start in the project checkout.
5544-
const worktreeSetupUnresolved =
5545-
projection.thread.worktreePath === null &&
5546-
projection.runs.some(
5547-
(run) =>
5548-
(run.status === "failed" || run.status === "interrupted") &&
5549-
run.workspacePreparation?.type === "worktree",
5550-
);
5602+
// Provider failure or usage limits re-hold the queue.
5603+
const worktreeSetupUnresolved = worktreeSetupPending(projection);
55515604
const sentByUserFromClient =
55525605
command.createdBy === "user" &&
55535606
(command.creationSource === "web" || command.creationSource === "mobile") &&
@@ -13083,10 +13136,17 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
1308313136
yield* threadDispatch
1308413137
.withLock(
1308513138
threadId,
13086-
startNextQueuedRun(threadId, {
13087-
endedRunId: run.id,
13088-
...(run.status === "failed" ? { failedRunId: run.id } : {}),
13089-
}),
13139+
(run.status === "completed"
13140+
? releaseUnattendedHolds(threadId, run.id)
13141+
: Effect.void
13142+
).pipe(
13143+
Effect.andThen(
13144+
startNextQueuedRun(threadId, {
13145+
endedRunId: run.id,
13146+
...(run.status === "failed" ? { failedRunId: run.id } : {}),
13147+
}),
13148+
),
13149+
),
1309013150
)
1309113151
.pipe(
1309213152
Effect.catchCause((cause) =>

‎apps/server/src/orchestration-v2/ProjectionStore.ts‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -612,6 +612,9 @@ export interface ProjectionStoreV2Shape {
612612
readonly canStartQueuedRun: (
613613
threadId: ThreadId,
614614
) => Effect.Effect<boolean, ProjectionStoreV2Error>;
615+
readonly getHeldQueuedRuns: (
616+
threadId: ThreadId,
617+
) => Effect.Effect<ReadonlyArray<OrchestrationV2Run>, ProjectionStoreV2Error>;
615618
readonly getRecoveryThreadIds: (
616619
kind: ProjectionRecoveryKind,
617620
) => Effect.Effect<ReadonlyArray<ThreadId>, ProjectionStoreV2Error>;
@@ -5183,6 +5186,17 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
51835186
)
51845187
.pipe(Effect.mapError(controlReadError(threadId)));
51855188

5189+
const getHeldQueuedRuns: ProjectionStoreV2Shape["getHeldQueuedRuns"] = (threadId) =>
5190+
sql<PayloadRow>`
5191+
SELECT payload_json FROM orchestration_v2_projection_runs
5192+
WHERE thread_id = ${threadId} AND status = 'queued'
5193+
AND json_extract(payload_json, '$.queueHeld') IS 1
5194+
ORDER BY ordinal ASC
5195+
`.pipe(
5196+
Effect.flatMap(decodeRows(decodeRunPayload, threadId)),
5197+
Effect.mapError(controlReadError(threadId)),
5198+
);
5199+
51865200
const getMessageCount: ProjectionStoreV2Shape["getMessageCount"] = (threadId) =>
51875201
sql<{
51885202
count: number;
@@ -6899,6 +6913,7 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
68996913
getCheckpointRollbackContext: readOnReader(getCheckpointRollbackContext),
69006914
getRunMessage: readOnReader(getRunMessage),
69016915
canStartQueuedRun: readOnReader(canStartQueuedRun),
6916+
getHeldQueuedRuns: readOnReader(getHeldQueuedRuns),
69026917
getPendingNativeUserInputs: readOnReader(getPendingNativeUserInputs),
69036918
hasUnpairedRunInterruptRequest: readOnReader(hasUnpairedRunInterruptRequest),
69046919
isNodeStored: readOnReader(isNodeStored),

‎apps/server/src/orchestration-v2/ProviderTurnControlService.test.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -263,6 +263,7 @@ it.effect(
263263
getRunMessage: () => Effect.die("not used"),
264264
isNodeStored: () => Effect.die("not used"),
265265
canStartQueuedRun: () => Effect.die("not used"),
266+
getHeldQueuedRuns: () => Effect.die("not used"),
266267
getRecoveryThreadIds: () => Effect.die("unused getRecoveryThreadIds"),
267268
getUnreadableThreadIds: () => Effect.die("unused getUnreadableThreadIds"),
268269
getThreadSnapshot: () => Effect.die("unused getThreadSnapshot"),

0 commit comments

Comments
 (0)