Skip to content

Commit 8d23699

Browse files
Marve10smkantautasmaria-rcksjuliusmarmingeclaude
authored
fix(server): a limited provider's queue stays held after another provider's turn succeeds (pingdotgg#22425)
Co-authored-by: Mantas Kantautas <[email protected]> Co-authored-by: maria-rcks <[email protected]> Co-authored-by: Julius Marminge <[email protected]> Co-authored-by: Claude Opus 5.5 <[email protected]>
1 parent d0d0608 commit 8d23699

2 files changed

Lines changed: 165 additions & 12 deletions

File tree

‎apps/server/src/orchestration-v2/Orchestrator.ts‎

Lines changed: 31 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -899,9 +899,15 @@ function queueTailModelSelection(
899899
);
900900
}
901901

902+
/**
903+
* The provider's latest run stopped on its usage limit and nothing ran after it.
904+
* With `now` (automatic starts), a run on another provider since does not lift
905+
* a reported reset still in the future; an unknown reset is lifted, as transient 429s are.
906+
*/
902907
function usageLimitHoldsQueue(
903908
projection: Pick<OrchestrationV2ThreadProjection, "runs" | "turnItems" | "providerSessions">,
904909
providerInstanceId: ModelSelection["instanceId"],
910+
now?: DateTime.Utc,
905911
): boolean {
906912
let latestSession: OrchestrationV2ThreadProjection["providerSessions"][number] | undefined;
907913
for (const session of projection.providerSessions) {
@@ -913,10 +919,16 @@ function usageLimitHoldsQueue(
913919
latestSession = session;
914920
}
915921
}
916-
return (
917-
usageLimitBlockedRun(projection.runs, projection.turnItems, latestSession?.lastError ?? null)
918-
?.providerInstanceId === providerInstanceId
922+
const limited = usageLimitBlockedRun(
923+
projection.runs.filter((run) => run.providerInstanceId === providerInstanceId),
924+
projection.turnItems,
925+
latestSession?.lastError ?? null,
919926
);
927+
if (limited === null) return false;
928+
if (latestExecutedRun(projection.runs)?.id === limited.id) return true;
929+
if (now === undefined) return false;
930+
const resetAt = latestRootProviderFailure(limited, projection.turnItems)?.resetAt;
931+
return resetAt != null && Date.parse(resetAt) > DateTime.toEpochMillis(now);
920932
}
921933

922934
function latestStableRun(
@@ -1920,6 +1932,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
19201932
readonly failedRunId?: RunId;
19211933
readonly endedRunId?: RunId;
19221934
readonly queuedRunId?: RunId;
1935+
readonly userStarted?: boolean;
19231936
},
19241937
) {
19251938
if (!(yield* projectionStore.canStartQueuedRun(threadId))) return;
@@ -1951,7 +1964,13 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
19511964
providerInstanceId: modelSelection.instanceId,
19521965
};
19531966
// A limited provider would fail and consume every waiting message.
1954-
if (usageLimitHoldsQueue(projection, queuedRun.providerInstanceId)) {
1967+
if (
1968+
usageLimitHoldsQueue(
1969+
projection,
1970+
queuedRun.providerInstanceId,
1971+
options?.userStarted === true ? undefined : yield* DateTime.now,
1972+
)
1973+
) {
19551974
return;
19561975
}
19571976
// Waits without a hold: lowering the thread's mode starts the queue again.
@@ -14009,10 +14028,10 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
1400914028
);
1401014029
if (command.type === "queue.resume" || command.type === "queued-run.start") {
1401114030
yield* mapDispatchError(command)(
14012-
startNextQueuedRun(
14013-
command.threadId,
14014-
command.type === "queued-run.start" ? { queuedRunId: command.runId } : undefined,
14015-
),
14031+
startNextQueuedRun(command.threadId, {
14032+
userStarted: true,
14033+
...(command.type === "queued-run.start" ? { queuedRunId: command.runId } : {}),
14034+
}),
1401614035
);
1401714036
}
1401814037
return {
@@ -14229,10 +14248,10 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
1422914248
(command.type === "thread.runtime-mode.set" && runtimeModeCeiling !== undefined)
1423014249
) {
1423114250
yield* mapDispatchError(command)(
14232-
startNextQueuedRun(
14233-
command.threadId,
14234-
command.type === "queued-run.start" ? { queuedRunId: command.runId } : undefined,
14235-
),
14251+
startNextQueuedRun(command.threadId, {
14252+
userStarted: command.type !== "thread.runtime-mode.set",
14253+
...(command.type === "queued-run.start" ? { queuedRunId: command.runId } : {}),
14254+
}),
1423614255
);
1423714256
}
1423814257
if (

‎apps/server/src/orchestration-v2/runtimeLayer.test.ts‎

Lines changed: 134 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6284,6 +6284,140 @@ it.layer(layerTest)("OrchestrationV2LayerLive lifecycle", (it) => {
62846284
}),
62856285
);
62866286

6287+
it.effect.each(["the reset passes", "the user resumes"] as const)(
6288+
"holds a limited provider's queue through another provider's turn until %s",
6289+
(release) =>
6290+
Effect.gen(function* () {
6291+
const orchestrator = yield* Orchestrator.OrchestratorV2;
6292+
const eventSink = yield* EventSink.EventSinkV2;
6293+
const threadId = ThreadId.make(`runtime-layer-queue-limit-switch-${release}`);
6294+
const switchedSelection = { instanceId: alternateInstanceId, model: "gpt-5.5" };
6295+
yield* orchestrator.dispatch({
6296+
type: "thread.create",
6297+
commandId: CommandId.make(`${threadId}:create`),
6298+
threadId,
6299+
createdBy: "user",
6300+
creationSource: "web",
6301+
projectId: ProjectId.make(`${threadId}:project`),
6302+
title: "Limit switch",
6303+
modelSelection,
6304+
runtimeMode: "full-access",
6305+
interactionMode: "default",
6306+
branch: null,
6307+
worktreePath: process.cwd(),
6308+
});
6309+
const send = (
6310+
name: string,
6311+
selection: typeof modelSelection | typeof switchedSelection,
6312+
type: "start_immediately" | "queue_after_active",
6313+
) =>
6314+
orchestrator.dispatch({
6315+
type: "message.dispatch",
6316+
commandId: CommandId.make(`${threadId}:${name}`),
6317+
threadId,
6318+
messageId: MessageId.make(`${threadId}:${name}`),
6319+
text: name,
6320+
attachments: [],
6321+
createdBy: "user",
6322+
creationSource: "web",
6323+
modelSelection: selection,
6324+
dispatchMode: { type },
6325+
});
6326+
yield* send("limited", modelSelection, "start_immediately");
6327+
yield* send("queued", modelSelection, "queue_after_active");
6328+
const [limited, queued] = (yield* orchestrator.getThreadProjection(threadId)).runs;
6329+
assert.isDefined(limited);
6330+
assert.isDefined(queued);
6331+
const now = yield* DateTime.now;
6332+
yield* eventSink.write({
6333+
events: [
6334+
{
6335+
id: EventId.make(`${threadId}:error`),
6336+
type: "turn-item.updated",
6337+
threadId,
6338+
runId: limited.id,
6339+
nodeId: limited.rootNodeId!,
6340+
providerInstanceId: limited.providerInstanceId,
6341+
occurredAt: now,
6342+
payload: {
6343+
id: TurnItemId.make(`${threadId}:error`),
6344+
type: "error",
6345+
threadId,
6346+
runId: limited.id,
6347+
nodeId: limited.rootNodeId!,
6348+
providerThreadId: limited.providerThreadId,
6349+
providerTurnId: null,
6350+
nativeItemRef: null,
6351+
parentItemId: null,
6352+
ordinal: 2,
6353+
status: "failed",
6354+
title: "Usage limit",
6355+
startedAt: now,
6356+
completedAt: now,
6357+
updatedAt: now,
6358+
failure: {
6359+
class: "usage_limit",
6360+
message: "Usage limit reached",
6361+
code: "usage_limit",
6362+
retryable: null,
6363+
resetAt: DateTime.formatIso(DateTime.add(now, { hours: 1 })),
6364+
},
6365+
},
6366+
},
6367+
{
6368+
id: EventId.make(`${threadId}:failed`),
6369+
type: "run.updated",
6370+
threadId,
6371+
runId: limited.id,
6372+
providerInstanceId: limited.providerInstanceId,
6373+
occurredAt: now,
6374+
payload: { ...limited, status: "failed", completedAt: now },
6375+
},
6376+
],
6377+
});
6378+
6379+
// A message on another provider starts ahead of the held queue and
6380+
// completes there.
6381+
yield* send("switch", switchedSelection, "start_immediately");
6382+
const switched = (yield* orchestrator.getThreadProjection(threadId)).runs.find(
6383+
(run) => run.userMessageId === MessageId.make(`${threadId}:switch`),
6384+
);
6385+
assert.equal(switched?.status, "starting");
6386+
yield* eventSink.write({
6387+
events: [
6388+
{
6389+
id: EventId.make(`${threadId}:switch:completed`),
6390+
type: "run.updated",
6391+
threadId,
6392+
runId: switched!.id,
6393+
providerInstanceId: switched!.providerInstanceId,
6394+
occurredAt: now,
6395+
payload: { ...switched!, status: "completed", completedAt: now },
6396+
},
6397+
],
6398+
});
6399+
6400+
yield* orchestrator.resumeQueuedRuns;
6401+
const held = yield* orchestrator.getThreadProjection(threadId);
6402+
assert.equal(held.runs.find((run) => run.id === queued.id)?.status, "queued");
6403+
6404+
if (release === "the reset passes") {
6405+
yield* TestClock.adjust("61 minutes");
6406+
yield* orchestrator.resumeQueuedRuns;
6407+
} else {
6408+
yield* orchestrator.dispatch({
6409+
type: "queue.resume",
6410+
commandId: CommandId.make(`${threadId}:resume`),
6411+
threadId,
6412+
});
6413+
}
6414+
const reset = yield* orchestrator.getThreadProjection(threadId);
6415+
const resumed = reset.runs.find((run) => run.id === queued.id);
6416+
assert.equal(resumed?.status, "starting");
6417+
assert.deepEqual(resumed?.modelSelection, modelSelection);
6418+
}),
6419+
);
6420+
62876421
it.effect.each([
62886422
["server", modelSelection],
62896423
["provider", modelSelection],

0 commit comments

Comments
 (0)