Skip to content

Commit 20cdc0d

Browse files
authored
fix: scope Spark cooldowns to its native quota (#599)
* Merge remote-tracking branch 'origin/dev' into fix/codex-spark-quota-scope-dev * fix: retain affinity per Codex quota scope * fix: address scoped affinity review feedback * fix: restore dev version and harden compact test
1 parent 257c626 commit 20cdc0d

11 files changed

Lines changed: 1109 additions & 130 deletions

File tree

‎docs-site/src/content/docs/guides/providers.md‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -127,7 +127,10 @@ Terminal refresh failures mark the account as needing reauthentication instead o
127127
**Cooldowns (Codex pool).** Upstream `429` / quota responses set a hard cooldown from
128128
`Retry-After`, quota `reset` headers (capped), or a short default backoff. Accounts on an explicit
129129
`Retry-After` cooldown are not probed early; reset-derived cooldowns may receive a paced probe lease
130-
so recovery can be detected without flooding the provider.
130+
so recovery can be detected without flooding the provider. Reset-derived native-model cooldowns
131+
also preserve known independent quota groups: `gpt-5.3-codex-spark` does not prevent the same account
132+
from trying the shared GPT-5.6 Terra/Luna quota, while models in that shared group still protect one
133+
another. Explicit `Retry-After` and default cooldowns always remain account-wide.
131134

132135
**Session affinity.** Codex thread→account affinity is process-local (in-memory only; not persisted
133136
across proxy restarts). On credential failures (`401` / `403`) the account is quarantined for

‎src/codex/auth-context.ts‎

Lines changed: 59 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -9,13 +9,16 @@ import { isCodexAccountUsable } from "./account-usability";
99
import { reconcileMainCodexAccountRuntimeState } from "./account-lifecycle";
1010
import { MAIN_CODEX_ACCOUNT_ID, getMainAccountToken } from "./main-account";
1111
import {
12-
getCodexAccountHealthSnapshot,
12+
codexQuotaScopeForModel,
13+
getCodexQuotaHealthSnapshot,
1314
releaseCodexQuotaProbeLease,
15+
releaseCodexQuotaScopeProbeLease,
1416
tryAcquireCodexQuotaProbeLease,
17+
tryAcquireCodexQuotaScopeProbeLease,
1518
pickAlternateCodexAccount,
1619
resolveCodexAccountForThreadDetailed,
1720
} from "./routing";
18-
import type { CodexCooldownSource } from "./routing";
21+
import type { CodexCooldownSource, CodexQuotaScope } from "./routing";
1922
import { maskAccountId } from "../lib/privacy";
2023
import { formatErrorResponse } from "../bridge";
2124
import { getAccountQuota } from "./quota";
@@ -34,8 +37,12 @@ export type CodexAuthContext =
3437
* Set when this request was admitted through an active quota cooldown as
3538
* the account's single probe. Must be echoed into the upstream outcome so
3639
* only this request can clear the cooldown (#433).
37-
*/
40+
*/
3841
probeLeaseId?: string;
42+
/** Native model quota group selected for this request, when known. */
43+
quotaScope?: CodexQuotaScope;
44+
/** Scope that owns `probeLeaseId`, when it is a scoped recovery probe. */
45+
probeQuotaScope?: CodexQuotaScope;
3946
}
4047
| {
4148
// Main Codex account participating in rotation: token injected from ~/.codex/auth.json
@@ -46,20 +53,29 @@ export type CodexAuthContext =
4653
chatgptAccountId: string;
4754
/** See `pool.probeLeaseId`. */
4855
probeLeaseId?: string;
56+
quotaScope?: CodexQuotaScope;
57+
probeQuotaScope?: CodexQuotaScope;
4958
};
5059

5160
/** Probe lease carried by this context, when it holds one. */
5261
export function codexProbeLeaseId(ctx: CodexAuthContext | undefined): string | undefined {
5362
return ctx?.kind === "pool" || ctx?.kind === "main-pool" ? ctx.probeLeaseId : undefined;
5463
}
5564

65+
/** Scope of a lease carried by this context, when it probes a model-specific quota. */
66+
export function codexProbeQuotaScope(ctx: CodexAuthContext | undefined): CodexQuotaScope | undefined {
67+
return ctx?.kind === "pool" || ctx?.kind === "main-pool" ? ctx.probeQuotaScope : undefined;
68+
}
69+
5670
/**
5771
* Hand back a probe lease for a request that will not reach upstream. Safe to
5872
* call with a context that holds no lease.
5973
*/
6074
export function releaseCodexAuthContextProbeLease(ctx: CodexAuthContext | undefined): void {
6175
const leaseId = codexProbeLeaseId(ctx);
62-
if (ctx && leaseId) releaseCodexQuotaProbeLease(ctx.accountId!, leaseId);
76+
if (!ctx || ctx.kind === "main" || !leaseId) return;
77+
if (ctx.probeQuotaScope) releaseCodexQuotaScopeProbeLease(ctx.accountId!, ctx.probeQuotaScope, leaseId);
78+
else releaseCodexQuotaProbeLease(ctx.accountId!, leaseId);
6379
}
6480

6581
export type OcxRuntimeProviderConfig = OcxProviderConfig & {
@@ -99,13 +115,20 @@ export class CodexAccountCooldownError extends Error {
99115
accountId: string;
100116
cooldownUntil: number;
101117
cooldownSource?: CodexCooldownSource;
118+
quotaScope?: CodexQuotaScope;
102119

103-
constructor(accountId: string, cooldownUntil: number, cooldownSource?: CodexCooldownSource) {
120+
constructor(
121+
accountId: string,
122+
cooldownUntil: number,
123+
cooldownSource?: CodexCooldownSource,
124+
quotaScope?: CodexQuotaScope,
125+
) {
104126
super("Selected Codex account is cooling down");
105127
this.name = "CodexAccountCooldownError";
106128
this.accountId = accountId;
107129
this.cooldownUntil = cooldownUntil;
108130
this.cooldownSource = cooldownSource;
131+
this.quotaScope = quotaScope;
109132
}
110133
}
111134

@@ -127,7 +150,12 @@ export function cooldownAccountLabel(accountId: string): string {
127150
*/
128151
export function cooldownErrorMessage(err: CodexAccountCooldownError): string {
129152
const until = new Date(err.cooldownUntil).toISOString();
130-
return `Selected Codex account (${cooldownAccountLabel(err.accountId)}) is cooling down until ${until}`
153+
const scope = err.quotaScope === "spark"
154+
? "Spark quota"
155+
: err.quotaScope === "shared"
156+
? "shared native quota"
157+
: null;
158+
return `Selected Codex account (${cooldownAccountLabel(err.accountId)})${scope ? ` ${scope} is` : " is"} cooling down until ${until}`
131159
+ ` (source: ${err.cooldownSource ?? "default"}).`
132160
+ ` Run 'ocx account list openai' to find the id, then`
133161
+ ` 'ocx account clear-cooldown openai <id>' to lift it, or switch accounts with 'ocx account use openai <id>'.`;
@@ -157,6 +185,8 @@ export function shouldMarkAccountNeedsReauthForCodexAuthFailure(cause: unknown):
157185

158186
export interface ResolveCodexAuthContextOptions {
159187
excludeAccountId?: string;
188+
/** Final native model selected for this request, used to select its quota group. */
189+
modelId?: string;
160190
}
161191

162192
export async function resolveCodexAuthContext(
@@ -171,16 +201,17 @@ export async function resolveCodexAuthContext(
171201
}
172202
reconcileMainCodexAccountRuntimeState();
173203
const threadId = headers.get("x-codex-parent-thread-id");
204+
const quotaScope = codexQuotaScopeForModel(options.modelId);
174205
const resolution = options.excludeAccountId
175206
? (() => {
176-
const accountId = pickAlternateCodexAccount(config, options.excludeAccountId!);
207+
const accountId = pickAlternateCodexAccount(config, options.excludeAccountId!, Date.now(), quotaScope);
177208
return accountId
178209
? { status: "selected" as const, accountId }
179210
: { status: "none" as const };
180211
})()
181-
: resolveCodexAccountForThreadDetailed(threadId, config);
212+
: resolveCodexAccountForThreadDetailed(threadId, config, Date.now(), quotaScope);
182213
if (resolution.status === "expired") throw new CodexThreadAffinityExpiredError(resolution.accountId);
183-
const accountId = resolution.status === "selected" ? resolution.accountId : null;
214+
let accountId = resolution.status === "selected" ? resolution.accountId : null;
184215
if (!accountId) throw new CodexPoolAuthenticationError();
185216
// Lazy prime: if the selected account has no quota yet, the pool is likely
186217
// unprimed (dashboard never opened, or startup prime was blocked). Kick a
@@ -194,31 +225,40 @@ export async function resolveCodexAuthContext(
194225
}
195226
// Snapshot (not just the deadline) so a refused request can report WHY it is cooled:
196227
// a literal Retry-After reads very differently to a user than a reset-derived guess.
197-
const cooldown = getCodexAccountHealthSnapshot(accountId);
228+
const cooldown = getCodexQuotaHealthSnapshot(accountId, quotaScope);
198229
const cooldownUntil = cooldown?.cooldownUntil;
199230
// A cooled-down account never sends traffic, so upstream recovery can never be
200231
// observed and the cooldown outlives the real limit. Admit one probe per
201232
// interval; its outcome decides whether the cooldown ends (#433).
202233
let probeLeaseId: string | undefined;
234+
let probeQuotaScope: CodexQuotaScope | undefined;
203235
if (cooldownUntil) {
204-
probeLeaseId = tryAcquireCodexQuotaProbeLease(accountId) ?? undefined;
205-
if (!probeLeaseId) throw new CodexAccountCooldownError(accountId, cooldownUntil, cooldown?.cooldownSource);
236+
probeQuotaScope = cooldown?.quotaScope;
237+
probeLeaseId = probeQuotaScope
238+
? tryAcquireCodexQuotaScopeProbeLease(accountId, probeQuotaScope) ?? undefined
239+
: tryAcquireCodexQuotaProbeLease(accountId) ?? undefined;
240+
if (!probeLeaseId) {
241+
throw new CodexAccountCooldownError(accountId, cooldownUntil, cooldown?.cooldownSource, cooldown?.quotaScope);
242+
}
206243
}
207244

208245
if (accountId === MAIN_CODEX_ACCOUNT_ID) {
209246
// Main account in rotation: inject the read-only auth.json token and fail closed if it vanished.
210247
const token = getMainAccountToken();
211248
if (!token) {
212249
// Nothing will reach upstream, so give the probe back instead of burning it.
213-
if (probeLeaseId) releaseCodexQuotaProbeLease(accountId, probeLeaseId);
250+
if (probeLeaseId && probeQuotaScope) releaseCodexQuotaScopeProbeLease(accountId, probeQuotaScope, probeLeaseId);
251+
else if (probeLeaseId) releaseCodexQuotaProbeLease(accountId, probeLeaseId);
214252
throw new CodexPoolAuthenticationError();
215253
}
216254
return {
217255
kind: "main-pool",
218256
accountId,
219257
accessToken: token.accessToken,
220258
chatgptAccountId: token.chatgptAccountId,
259+
...(quotaScope ? { quotaScope } : {}),
221260
...(probeLeaseId ? { probeLeaseId } : {}),
261+
...(probeQuotaScope ? { probeQuotaScope } : {}),
222262
};
223263
}
224264

@@ -230,10 +270,13 @@ export async function resolveCodexAuthContext(
230270
generation: token.generation,
231271
accessToken: token.accessToken,
232272
chatgptAccountId: token.chatgptAccountId,
273+
...(quotaScope ? { quotaScope } : {}),
233274
...(probeLeaseId ? { probeLeaseId } : {}),
275+
...(probeQuotaScope ? { probeQuotaScope } : {}),
234276
};
235277
} catch (cause) {
236-
if (probeLeaseId) releaseCodexQuotaProbeLease(accountId, probeLeaseId);
278+
if (probeLeaseId && probeQuotaScope) releaseCodexQuotaScopeProbeLease(accountId, probeQuotaScope, probeLeaseId);
279+
else if (probeLeaseId) releaseCodexQuotaProbeLease(accountId, probeLeaseId);
237280
if (shouldMarkAccountNeedsReauthForCodexAuthFailure(cause)) {
238281
markAccountNeedsReauth(accountId);
239282
}
@@ -245,9 +288,9 @@ export function assertCodexAuthContextNotCooled(ctx: CodexAuthContext | undefine
245288
if (ctx?.kind !== "pool" && ctx?.kind !== "main-pool") return;
246289
// A context holding the probe lease was deliberately admitted through the cooldown.
247290
if (ctx.probeLeaseId) return;
248-
const cooldown = getCodexAccountHealthSnapshot(ctx.accountId);
291+
const cooldown = getCodexQuotaHealthSnapshot(ctx.accountId, ctx.quotaScope);
249292
if (cooldown?.cooldownUntil) {
250-
throw new CodexAccountCooldownError(ctx.accountId, cooldown.cooldownUntil, cooldown.cooldownSource);
293+
throw new CodexAccountCooldownError(ctx.accountId, cooldown.cooldownUntil, cooldown.cooldownSource, cooldown.quotaScope);
251294
}
252295
}
253296

0 commit comments

Comments
 (0)