Skip to content

Commit 6f5caae

Browse files
committed
fix(server): a limited provider's queue stays held through another provider's turn until its reset
A usage-limit failure held queued work only while it was the thread's latest run, so a turn that completed on another provider started the limited provider's queued messages against its limit. The hold now reads the queued provider's own latest run. It still holds when that run is the thread's latest, and otherwise holds only while the reported reset time is ahead, so a limit that has reset no longer pins messages queued for it.
1 parent 5d4c202 commit 6f5caae

2 files changed

Lines changed: 150 additions & 5 deletions

File tree

‎apps/server/src/orchestration-v2/Orchestrator.ts‎

Lines changed: 17 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -674,9 +674,15 @@ function queuedRunModelSelection(
674674
: run.modelSelection;
675675
}
676676

677+
/**
678+
* The provider's latest run stopped on its usage limit, and either nothing
679+
* ran after it or the limit has not reset yet. A run that completed on
680+
* another provider since does not lift the limit.
681+
*/
677682
function usageLimitHoldsQueue(
678683
projection: Pick<OrchestrationV2ThreadProjection, "runs" | "turnItems" | "providerSessions">,
679684
providerInstanceId: ModelSelection["instanceId"],
685+
now: DateTime.Utc,
680686
): boolean {
681687
let latestSession: OrchestrationV2ThreadProjection["providerSessions"][number] | undefined;
682688
for (const session of projection.providerSessions) {
@@ -688,10 +694,15 @@ function usageLimitHoldsQueue(
688694
latestSession = session;
689695
}
690696
}
691-
return (
692-
usageLimitBlockedRun(projection.runs, projection.turnItems, latestSession?.lastError ?? null)
693-
?.providerInstanceId === providerInstanceId
697+
const limited = usageLimitBlockedRun(
698+
projection.runs.filter((run) => run.providerInstanceId === providerInstanceId),
699+
projection.turnItems,
700+
latestSession?.lastError ?? null,
694701
);
702+
if (limited === null) return false;
703+
if (latestExecutedRun(projection.runs)?.id === limited.id) return true;
704+
const resetAt = latestRootProviderFailure(limited, projection.turnItems)?.resetAt;
705+
return resetAt != null && Date.parse(resetAt) > DateTime.toEpochMillis(now);
695706
}
696707

697708
function latestStableRun(
@@ -1398,7 +1409,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
13981409
providerInstanceId: modelSelection.instanceId,
13991410
};
14001411
// A limited provider would fail and consume every waiting message.
1401-
if (usageLimitHoldsQueue(projection, queuedRun.providerInstanceId)) {
1412+
if (usageLimitHoldsQueue(projection, queuedRun.providerInstanceId, yield* DateTime.now)) {
14021413
return;
14031414
}
14041415
// A message queued behind a worktree that was cancelled or never created
@@ -11333,12 +11344,14 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
1133311344
});
1133411345
}
1133511346
const nextRun = nextQueuedRun(projection);
11347+
const now = yield* DateTime.now;
1133611348
if (
1133711349
usageLimitHoldsQueue(
1133811350
projection,
1133911351
nextRun === undefined
1134011352
? projection.thread.providerInstanceId
1134111353
: queuedRunModelSelection(projection, nextRun).instanceId,
11354+
now,
1134211355
)
1134311356
) {
1134411357
return yield* new OrchestratorDispatchError({
@@ -11347,7 +11360,6 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
1134711360
cause: "Continue the limited thread before resuming its queue.",
1134811361
});
1134911362
}
11350-
const now = yield* DateTime.now;
1135111363
const queued = projection.runs.filter((run) => run.status === "queued");
1135211364
for (const run of queued) {
1135311365
yield* emit(

‎apps/server/src/orchestration-v2/runtimeLayer.test.ts‎

Lines changed: 133 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5069,6 +5069,139 @@ it.layer(layerTest)("OrchestrationV2LayerLive lifecycle", (it) => {
50695069
}),
50705070
);
50715071

5072+
it.effect("holds a limited provider's queue through another provider's turn until reset", () =>
5073+
Effect.gen(function* () {
5074+
const orchestrator = yield* Orchestrator.OrchestratorV2;
5075+
const eventSink = yield* EventSink.EventSinkV2;
5076+
const threadId = ThreadId.make("runtime-layer-queue-limit-switch");
5077+
const switchedSelection = { instanceId: alternateInstanceId, model: "gpt-5.5" };
5078+
yield* orchestrator.dispatch({
5079+
type: "thread.create",
5080+
commandId: CommandId.make(`${threadId}:create`),
5081+
threadId,
5082+
createdBy: "user",
5083+
creationSource: "web",
5084+
projectId: ProjectId.make(`${threadId}:project`),
5085+
title: "Limit switch",
5086+
modelSelection,
5087+
runtimeMode: "full-access",
5088+
interactionMode: "default",
5089+
branch: null,
5090+
worktreePath: process.cwd(),
5091+
});
5092+
const send = (
5093+
name: string,
5094+
selection: typeof modelSelection | typeof switchedSelection,
5095+
type: "start_immediately" | "queue_after_active",
5096+
) =>
5097+
orchestrator.dispatch({
5098+
type: "message.dispatch",
5099+
commandId: CommandId.make(`${threadId}:${name}`),
5100+
threadId,
5101+
messageId: MessageId.make(`${threadId}:${name}`),
5102+
text: name,
5103+
attachments: [],
5104+
createdBy: "user",
5105+
creationSource: "web",
5106+
modelSelection: selection,
5107+
dispatchMode: { type },
5108+
});
5109+
yield* send("limited", modelSelection, "start_immediately");
5110+
yield* send("queued", modelSelection, "queue_after_active");
5111+
const [limited, queued] = (yield* orchestrator.getThreadProjection(threadId)).runs;
5112+
assert.isDefined(limited);
5113+
assert.isDefined(queued);
5114+
const now = yield* DateTime.now;
5115+
yield* eventSink.write({
5116+
events: [
5117+
{
5118+
id: EventId.make(`${threadId}:error`),
5119+
type: "turn-item.updated",
5120+
threadId,
5121+
runId: limited.id,
5122+
nodeId: limited.rootNodeId!,
5123+
providerInstanceId: limited.providerInstanceId,
5124+
occurredAt: now,
5125+
payload: {
5126+
id: TurnItemId.make(`${threadId}:error`),
5127+
type: "error",
5128+
threadId,
5129+
runId: limited.id,
5130+
nodeId: limited.rootNodeId!,
5131+
providerThreadId: limited.providerThreadId,
5132+
providerTurnId: null,
5133+
nativeItemRef: null,
5134+
parentItemId: null,
5135+
ordinal: 2,
5136+
status: "failed",
5137+
title: "Usage limit",
5138+
startedAt: now,
5139+
completedAt: now,
5140+
updatedAt: now,
5141+
failure: {
5142+
class: "usage_limit",
5143+
message: "Usage limit reached",
5144+
code: "usage_limit",
5145+
retryable: null,
5146+
resetAt: DateTime.formatIso(DateTime.add(now, { hours: 1 })),
5147+
},
5148+
},
5149+
},
5150+
{
5151+
id: EventId.make(`${threadId}:failed`),
5152+
type: "run.updated",
5153+
threadId,
5154+
runId: limited.id,
5155+
providerInstanceId: limited.providerInstanceId,
5156+
occurredAt: now,
5157+
payload: { ...limited, status: "failed", completedAt: now },
5158+
},
5159+
],
5160+
});
5161+
5162+
// A message on another provider starts ahead of the held queue and
5163+
// completes there.
5164+
yield* send("switch", switchedSelection, "start_immediately");
5165+
const switched = (yield* orchestrator.getThreadProjection(threadId)).runs.find(
5166+
(run) => run.userMessageId === MessageId.make(`${threadId}:switch`),
5167+
);
5168+
assert.equal(switched?.status, "starting");
5169+
yield* eventSink.write({
5170+
events: [
5171+
{
5172+
id: EventId.make(`${threadId}:switch:completed`),
5173+
type: "run.updated",
5174+
threadId,
5175+
runId: switched!.id,
5176+
providerInstanceId: switched!.providerInstanceId,
5177+
occurredAt: now,
5178+
payload: { ...switched!, status: "completed", completedAt: now },
5179+
},
5180+
],
5181+
});
5182+
5183+
yield* orchestrator.resumeQueuedRuns;
5184+
const held = yield* orchestrator.getThreadProjection(threadId);
5185+
assert.equal(held.runs.find((run) => run.id === queued.id)?.status, "queued");
5186+
const blocked = yield* orchestrator
5187+
.dispatch({
5188+
type: "queue.resume",
5189+
commandId: CommandId.make(`${threadId}:blocked`),
5190+
threadId,
5191+
})
5192+
.pipe(Effect.flip);
5193+
assert.equal(blocked._tag, "OrchestratorDispatchError");
5194+
5195+
// Once the reported reset passes, the queued message starts on its own provider.
5196+
yield* TestClock.adjust("61 minutes");
5197+
yield* orchestrator.resumeQueuedRuns;
5198+
const reset = yield* orchestrator.getThreadProjection(threadId);
5199+
const resumed = reset.runs.find((run) => run.id === queued.id);
5200+
assert.equal(resumed?.status, "starting");
5201+
assert.deepEqual(resumed?.modelSelection, modelSelection);
5202+
}),
5203+
);
5204+
50725205
it.effect.each([
50735206
["server", modelSelection],
50745207
["provider", modelSelection],

0 commit comments

Comments
 (0)