Skip to content

Commit 37859da

Browse files
dev-ankitclaude
andcommitted
fix(server): background wakes no longer block usage-limit auto-resume
When a thread hit a usage limit, notices from background subagents that died at the same limit started their own turns. Those turns completed, so recovery no longer saw the limited run as the latest and the scheduled resume never fired. They also cleared the thread's snooze. Work dispatched to run after the active run now also waits out a usage limit: it queues behind the limited run, leaves the thread snoozed or settled, and drains after the resume like any queued message. Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent 144aae4 commit 37859da

2 files changed

Lines changed: 175 additions & 26 deletions

File tree

‎apps/server/src/orchestration-v2/Orchestrator.ts‎

Lines changed: 35 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -519,6 +519,26 @@ function isBlockingRun(run: OrchestrationV2Run): boolean {
519519
);
520520
}
521521

522+
/**
523+
* The usage-limited run the thread is waiting out, or null. Queued work waits
524+
* behind it like it waits behind an active run, until the thread is continued.
525+
*/
526+
function limitBlockingRun(
527+
projection: Pick<
528+
OrchestrationV2ThreadProjection,
529+
"thread" | "runs" | "turnItems" | "providerSessions"
530+
>,
531+
): OrchestrationV2Run | null {
532+
const sessionError =
533+
projection.providerSessions
534+
.filter((session) => session.providerInstanceId === projection.thread.providerInstanceId)
535+
.toSorted(
536+
(left, right) =>
537+
DateTime.toEpochMillis(right.updatedAt) - DateTime.toEpochMillis(left.updatedAt),
538+
)[0]?.lastError ?? null;
539+
return usageLimitBlockedRun(projection.runs, projection.turnItems, sessionError);
540+
}
541+
522542
/**
523543
* A parent thread is "live" for wake purposes while a run is still producing
524544
* agent output. A run parked at "waiting" is post-terminal drain, so its agent
@@ -1279,14 +1299,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
12791299

12801300
// The limit already stopped this thread. Starting the queue would send
12811301
// every waiting message and drop it from the queue as each one fails.
1282-
const sessionError =
1283-
projection.providerSessions
1284-
.filter((session) => session.providerInstanceId === projection.thread.providerInstanceId)
1285-
.toSorted(
1286-
(left, right) =>
1287-
DateTime.toEpochMillis(right.updatedAt) - DateTime.toEpochMillis(left.updatedAt),
1288-
)[0]?.lastError ?? null;
1289-
if (usageLimitBlockedRun(projection.runs, projection.turnItems, sessionError) !== null) {
1302+
if (limitBlockingRun(projection) !== null) {
12901303
return;
12911304
}
12921305
const queuedRun = nextQueuedRun(projection);
@@ -4600,7 +4613,17 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
46004613
(run) => run.id === command.restartContinuationOfRunId,
46014614
)?.delegatedTaskId;
46024615

4603-
if (projection.thread.settledOverride !== null) {
4616+
let dispatchMode = resolveMessageDispatchIntent(
4617+
projection,
4618+
command.dispatchMode,
4619+
command.deliveryIntent,
4620+
);
4621+
// Work that waits for the active run also waits out a usage limit, so a
4622+
// background wake cannot start a turn that hides the limit from recovery.
4623+
// It starts nothing yet, so it leaves the thread settled or snoozed.
4624+
const limitedRun =
4625+
dispatchMode.type === "queue_after_active" ? limitBlockingRun(projection) : null;
4626+
if (projection.thread.settledOverride !== null && limitedRun === null) {
46044627
const now = yield* DateTime.now;
46054628
const thread: OrchestrationV2AppThread = {
46064629
...projection.thread,
@@ -4624,7 +4647,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
46244647
});
46254648
projection = yield* getProjectionWithPendingEvents(command.threadId, events);
46264649
}
4627-
if (projection.thread.snoozedUntil != null) {
4650+
if (projection.thread.snoozedUntil != null && limitedRun === null) {
46284651
const now = yield* DateTime.now;
46294652
const thread: OrchestrationV2AppThread = {
46304653
...projection.thread,
@@ -4682,11 +4705,6 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
46824705
projection = yield* getProjectionWithPendingEvents(command.threadId, events);
46834706
}
46844707
const modelSelection = command.modelSelection ?? projection.thread.modelSelection;
4685-
let dispatchMode = resolveMessageDispatchIntent(
4686-
projection,
4687-
command.dispatchMode,
4688-
command.deliveryIntent,
4689-
);
46904708
if (dispatchMode.type === "steer_active") {
46914709
const targetRunId = dispatchMode.targetRunId;
46924710
const target = projection.runs.find((run) => run.id === targetRunId);
@@ -4899,7 +4917,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
48994917
projection.thread.worktreePath === null
49004918
? latestRun
49014919
: undefined;
4902-
const queueBehindRun = activeRun ?? worktreeMissingRun;
4920+
const queueBehindRun = activeRun ?? worktreeMissingRun ?? limitedRun ?? undefined;
49034921
const pendingMergeBackTransfers = pendingMergeBackTransfersForThread(projection);
49044922
const shouldQueue =
49054923
queueBehindRun !== undefined &&
@@ -10827,16 +10845,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
1082710845
cause: `Thread ${command.threadId} is not active.`,
1082810846
});
1082910847
}
10830-
const sessionError =
10831-
projection.providerSessions
10832-
.filter(
10833-
(session) => session.providerInstanceId === projection.thread.providerInstanceId,
10834-
)
10835-
.toSorted(
10836-
(left, right) =>
10837-
DateTime.toEpochMillis(right.updatedAt) - DateTime.toEpochMillis(left.updatedAt),
10838-
)[0]?.lastError ?? null;
10839-
if (usageLimitBlockedRun(projection.runs, projection.turnItems, sessionError) !== null) {
10848+
if (limitBlockingRun(projection) !== null) {
1084010849
return yield* new OrchestratorDispatchError({
1084110850
commandId: command.commandId,
1084210851
commandType: command.type,

‎apps/server/src/orchestration-v2/runtimeLayer.test.ts‎

Lines changed: 140 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5725,6 +5725,146 @@ it.layer(layerTest)("usage-limit recovery", (it) => {
57255725
}
57265726
}),
57275727
);
5728+
5729+
it.effect.each(["snoozed", "settled"] as const)(
5730+
"holds background wakes behind a usage limit on a %s thread",
5731+
(scenario) =>
5732+
Effect.gen(function* () {
5733+
const orchestrator = yield* Orchestrator.OrchestratorV2;
5734+
const events = yield* EventSink.EventSinkV2;
5735+
const threadId = ThreadId.make(`recovery:held-wake:${scenario}`);
5736+
const projectId = ProjectId.make(`recovery:project:held-wake:${scenario}`);
5737+
yield* seedProject({
5738+
projectId,
5739+
title: "Recovery project",
5740+
workspaceRoot: process.cwd(),
5741+
defaultModelSelection: modelSelection,
5742+
createdAt: DateTime.formatIso(yield* DateTime.now),
5743+
});
5744+
yield* orchestrator.dispatch({
5745+
type: "thread.create",
5746+
commandId: CommandId.make(`recovery:create:held-wake:${scenario}`),
5747+
threadId,
5748+
projectId,
5749+
title: "Limited thread",
5750+
modelSelection,
5751+
runtimeMode: "full-access",
5752+
interactionMode: "default",
5753+
branch: null,
5754+
worktreePath: null,
5755+
createdBy: "user",
5756+
creationSource: "web",
5757+
});
5758+
yield* orchestrator.dispatch({
5759+
type: "message.dispatch",
5760+
commandId: CommandId.make(`recovery:message:held-wake:${scenario}`),
5761+
threadId,
5762+
messageId: MessageId.make(`recovery:message:held-wake:${scenario}`),
5763+
text: "Work on this.",
5764+
attachments: [],
5765+
dispatchMode: { type: "defer_start" },
5766+
createdBy: "user",
5767+
creationSource: "web",
5768+
});
5769+
const run = (yield* orchestrator.getThreadProjection(threadId)).runs[0]!;
5770+
const now = yield* DateTime.now;
5771+
const resetAt = DateTime.formatIso(DateTime.add(now, { minutes: 1 }));
5772+
yield* events.write({
5773+
commandId: CommandId.make(`recovery:failure:held-wake:${scenario}`),
5774+
events: [
5775+
{
5776+
id: EventId.make(`recovery:run:held-wake:${scenario}`),
5777+
type: "run.updated",
5778+
threadId,
5779+
occurredAt: now,
5780+
payload: { ...run, status: "failed", completedAt: now },
5781+
},
5782+
{
5783+
id: EventId.make(`recovery:error:held-wake:${scenario}`),
5784+
type: "turn-item.updated",
5785+
threadId,
5786+
occurredAt: now,
5787+
payload: {
5788+
id: TurnItemId.make(`recovery:error:held-wake:${scenario}`),
5789+
type: "error",
5790+
threadId,
5791+
runId: run.id,
5792+
nodeId: run.rootNodeId,
5793+
providerThreadId: null,
5794+
providerTurnId: null,
5795+
nativeItemRef: null,
5796+
parentItemId: null,
5797+
ordinal: 2,
5798+
status: "failed",
5799+
title: "Usage limit reached",
5800+
startedAt: now,
5801+
completedAt: now,
5802+
updatedAt: now,
5803+
failure: {
5804+
class: "usage_limit",
5805+
message: "Plan limit reached.",
5806+
code: "usageLimitExceeded",
5807+
retryable: null,
5808+
resetAt,
5809+
},
5810+
},
5811+
},
5812+
],
5813+
});
5814+
const shellOf = Effect.map(orchestrator.getShellSnapshot(), (snapshot) =>
5815+
snapshot.threads.find((thread) => thread.id === threadId),
5816+
);
5817+
yield* orchestrator.dispatch(
5818+
limitRecoveryCommand((yield* shellOf)!, true, DateTime.toEpochMillis(now), true)!,
5819+
);
5820+
if (scenario === "settled")
5821+
yield* orchestrator.dispatch({
5822+
type: "thread.settle",
5823+
commandId: CommandId.make(`recovery:settle:held-wake:${scenario}`),
5824+
threadId,
5825+
});
5826+
5827+
// A subagent that died at the same limit reports back while the thread waits.
5828+
yield* TestClock.adjust("10 seconds");
5829+
yield* orchestrator.dispatch({
5830+
type: "message.dispatch",
5831+
commandId: CommandId.make(`recovery:wake:held-wake:${scenario}`),
5832+
threadId,
5833+
messageId: MessageId.make(`recovery:wake:held-wake:${scenario}`),
5834+
text: "Agent terminated early due to an API error.",
5835+
notification: {
5836+
source: { kind: "background_task" },
5837+
outcome: "failed",
5838+
summary: 'Subagent "Step 4" failed',
5839+
},
5840+
attachments: [],
5841+
dispatchMode: { type: "queue_after_active" },
5842+
createdBy: "agent",
5843+
creationSource: "provider",
5844+
});
5845+
const held = yield* orchestrator.getThreadProjection(threadId);
5846+
assert.equal(held.runs[1]?.status, "queued");
5847+
if (scenario === "settled") assert.equal(held.thread.settledOverride, "settled");
5848+
else assert.equal(DateTime.toEpochMillis(held.thread.snoozedUntil!), Date.parse(resetAt));
5849+
5850+
yield* TestClock.adjust("1 minute");
5851+
const resume = limitRecoveryCommand(
5852+
(yield* shellOf)!,
5853+
true,
5854+
DateTime.toEpochMillis(yield* DateTime.now),
5855+
);
5856+
// Settling the thread opted out of the resume; a held wake keeps it that way.
5857+
if (scenario === "settled") {
5858+
assert.isNull(resume);
5859+
return;
5860+
}
5861+
assert.equal(resume?.type, "message.dispatch");
5862+
yield* orchestrator.dispatch(resume!);
5863+
const resumed = yield* orchestrator.getThreadProjection(threadId);
5864+
assert.equal(resumed.runs[2]?.status, "starting");
5865+
assert.equal(resumed.runs[1]?.status, "queued");
5866+
}),
5867+
);
57285868
});
57295869

57305870
const queueResumeDisabledInstanceProjectId = ProjectId.make(

0 commit comments

Comments
 (0)