Skip to content

Commit c2af95b

Browse files
committed
fix: address scoped cooldown review feedback
1 parent a2b2b06 commit c2af95b

5 files changed

Lines changed: 126 additions & 24 deletions

File tree

‎src/codex/auth-context.ts‎

Lines changed: 32 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -115,13 +115,20 @@ export class CodexAccountCooldownError extends Error {
115115
accountId: string;
116116
cooldownUntil: number;
117117
cooldownSource?: CodexCooldownSource;
118+
quotaScope?: CodexQuotaScope;
118119

119-
constructor(accountId: string, cooldownUntil: number, cooldownSource?: CodexCooldownSource) {
120+
constructor(
121+
accountId: string,
122+
cooldownUntil: number,
123+
cooldownSource?: CodexCooldownSource,
124+
quotaScope?: CodexQuotaScope,
125+
) {
120126
super("Selected Codex account is cooling down");
121127
this.name = "CodexAccountCooldownError";
122128
this.accountId = accountId;
123129
this.cooldownUntil = cooldownUntil;
124130
this.cooldownSource = cooldownSource;
131+
this.quotaScope = quotaScope;
125132
}
126133
}
127134

@@ -143,7 +150,12 @@ export function cooldownAccountLabel(accountId: string): string {
143150
*/
144151
export function cooldownErrorMessage(err: CodexAccountCooldownError): string {
145152
const until = new Date(err.cooldownUntil).toISOString();
146-
return `Selected Codex account (${cooldownAccountLabel(err.accountId)}) is cooling down until ${until}`
153+
const scope = err.quotaScope === "spark"
154+
? "Spark quota"
155+
: err.quotaScope === "shared"
156+
? "shared native quota"
157+
: null;
158+
return `Selected Codex account (${cooldownAccountLabel(err.accountId)})${scope ? ` ${scope} is` : " is"} cooling down until ${until}`
147159
+ ` (source: ${err.cooldownSource ?? "default"}).`
148160
+ ` Run 'ocx account list openai' to find the id, then`
149161
+ ` 'ocx account clear-cooldown openai <id>' to lift it, or switch accounts with 'ocx account use openai <id>'.`;
@@ -189,16 +201,17 @@ export async function resolveCodexAuthContext(
189201
}
190202
reconcileMainCodexAccountRuntimeState();
191203
const threadId = headers.get("x-codex-parent-thread-id");
204+
const quotaScope = codexQuotaScopeForModel(options.modelId);
192205
const resolution = options.excludeAccountId
193206
? (() => {
194-
const accountId = pickLowestUsageCodexAccount(config, options.excludeAccountId);
207+
const accountId = pickLowestUsageCodexAccount(config, options.excludeAccountId, Date.now(), quotaScope);
195208
return accountId
196209
? { status: "selected" as const, accountId }
197210
: { status: "none" as const };
198211
})()
199212
: resolveCodexAccountForThreadDetailed(threadId, config);
200213
if (resolution.status === "expired") throw new CodexThreadAffinityExpiredError(resolution.accountId);
201-
const accountId = resolution.status === "selected" ? resolution.accountId : null;
214+
let accountId = resolution.status === "selected" ? resolution.accountId : null;
202215
if (!accountId) throw new CodexPoolAuthenticationError();
203216
// Lazy prime: if the selected account has no quota yet, the pool is likely
204217
// unprimed (dashboard never opened, or startup prime was blocked). Kick a
@@ -212,8 +225,17 @@ export async function resolveCodexAuthContext(
212225
}
213226
// Snapshot (not just the deadline) so a refused request can report WHY it is cooled:
214227
// a literal Retry-After reads very differently to a user than a reset-derived guess.
215-
const quotaScope = codexQuotaScopeForModel(options.modelId);
216-
const cooldown = getCodexQuotaHealthSnapshot(accountId, quotaScope);
228+
let cooldown = getCodexQuotaHealthSnapshot(accountId, quotaScope);
229+
// Keep session affinity and the active account intact for independent quota
230+
// groups, but use an eligible peer for this model when one exists. This lets
231+
// Spark fail over to another account without moving Terra/Luna off accountId.
232+
if (cooldown?.quotaScope) {
233+
const alternate = pickLowestUsageCodexAccount(config, accountId, Date.now(), quotaScope);
234+
if (alternate) {
235+
accountId = alternate;
236+
cooldown = getCodexQuotaHealthSnapshot(accountId, quotaScope);
237+
}
238+
}
217239
const cooldownUntil = cooldown?.cooldownUntil;
218240
// A cooled-down account never sends traffic, so upstream recovery can never be
219241
// observed and the cooldown outlives the real limit. Admit one probe per
@@ -225,7 +247,9 @@ export async function resolveCodexAuthContext(
225247
probeLeaseId = probeQuotaScope
226248
? tryAcquireCodexQuotaScopeProbeLease(accountId, probeQuotaScope) ?? undefined
227249
: tryAcquireCodexQuotaProbeLease(accountId) ?? undefined;
228-
if (!probeLeaseId) throw new CodexAccountCooldownError(accountId, cooldownUntil, cooldown?.cooldownSource);
250+
if (!probeLeaseId) {
251+
throw new CodexAccountCooldownError(accountId, cooldownUntil, cooldown?.cooldownSource, cooldown?.quotaScope);
252+
}
229253
}
230254

231255
if (accountId === MAIN_CODEX_ACCOUNT_ID) {
@@ -276,7 +300,7 @@ export function assertCodexAuthContextNotCooled(ctx: CodexAuthContext | undefine
276300
if (ctx.probeLeaseId) return;
277301
const cooldown = getCodexQuotaHealthSnapshot(ctx.accountId, ctx.quotaScope);
278302
if (cooldown?.cooldownUntil) {
279-
throw new CodexAccountCooldownError(ctx.accountId, cooldown.cooldownUntil, cooldown.cooldownSource);
303+
throw new CodexAccountCooldownError(ctx.accountId, cooldown.cooldownUntil, cooldown.cooldownSource, cooldown.quotaScope);
280304
}
281305
}
282306

‎src/codex/routing.ts‎

Lines changed: 26 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -567,10 +567,15 @@ function bindThreadAffinity(threadId: string, accountId: string, now: number): v
567567
pruneLruThreadAffinities();
568568
}
569569

570-
function getEligiblePoolAccounts(config: OcxConfig, excludeId?: string, now = Date.now()): string[] {
570+
function getEligiblePoolAccounts(
571+
config: OcxConfig,
572+
excludeId?: string,
573+
now = Date.now(),
574+
quotaScope?: CodexQuotaScope,
575+
): string[] {
571576
const ids = (config.codexAccounts ?? [])
572577
.filter(account => !account.isMain && account.id !== excludeId && !isAccountNeedsReauth(account.id))
573-
.filter(account => !isCodexAccountInCooldown(account.id, now))
578+
.filter(account => getCodexQuotaHealthSnapshot(account.id, quotaScope, now) === null)
574579
.filter(account => !isCodexAccountSoftAvoided(account.id, now))
575580
.filter(account => isCodexAccountUsable(config, account.id))
576581
.map(account => account.id);
@@ -579,7 +584,7 @@ function getEligiblePoolAccounts(config: OcxConfig, excludeId?: string, now = Da
579584
if (
580585
excludeId !== MAIN_CODEX_ACCOUNT_ID
581586
&& !isAccountNeedsReauth(MAIN_CODEX_ACCOUNT_ID)
582-
&& !isCodexAccountInCooldown(MAIN_CODEX_ACCOUNT_ID, now)
587+
&& getCodexQuotaHealthSnapshot(MAIN_CODEX_ACCOUNT_ID, quotaScope, now) === null
583588
&& !isCodexAccountSoftAvoided(MAIN_CODEX_ACCOUNT_ID, now)
584589
&& isCodexAccountUsable(config, MAIN_CODEX_ACCOUNT_ID)
585590
) {
@@ -606,10 +611,15 @@ function pickLowerUsageAccount(config: OcxConfig, active: string, activeUsage: n
606611
return best;
607612
}
608613

609-
export function pickLowestUsageCodexAccount(config: OcxConfig, excludeId?: string, now = Date.now()): string | null {
614+
export function pickLowestUsageCodexAccount(
615+
config: OcxConfig,
616+
excludeId?: string,
617+
now = Date.now(),
618+
quotaScope?: CodexQuotaScope,
619+
): string | null {
610620
let best: string | null = null;
611621
let bestUsage = Number.POSITIVE_INFINITY;
612-
for (const id of getEligiblePoolAccounts(config, excludeId, now)) {
622+
for (const id of getEligiblePoolAccounts(config, excludeId, now, quotaScope)) {
613623
const usage = computeCodexUsageScore(getAccountQuota(id), getPoolAccountPlan(config, id));
614624
if (usage < bestUsage) {
615625
best = id;
@@ -927,8 +937,18 @@ export function recordCodexUpstreamOutcome(
927937
...(prior?.probeLeaseId !== undefined ? { probeLeaseId: prior.probeLeaseId } : {}),
928938
...(prior?.probeLeaseGeneration !== undefined ? { probeLeaseGeneration: prior.probeLeaseGeneration } : {}),
929939
...(prior?.lastProbeAt !== undefined ? { lastProbeAt: prior.lastProbeAt } : {}),
930-
}),
940+
}),
931941
});
942+
// The shared native scope is the existing account-wide native behavior:
943+
// threads must leave it and new requests should prefer an eligible account.
944+
// Spark remains isolated so a same-account Terra/Luna combo fallback can run.
945+
if (quotaScope === "shared") {
946+
clearThreadAccountMapForAccount(accountId);
947+
if (config.activeCodexAccountId === accountId) {
948+
const fallback = pickLowestUsageCodexAccount(config, accountId, now, quotaScope);
949+
if (fallback) setActiveCodexAccount(config, fallback);
950+
}
951+
}
932952
return;
933953
}
934954

‎src/server/responses/compact.ts‎

Lines changed: 3 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -306,16 +306,9 @@ export async function handleResponsesCompact(
306306
if (buffered.status === 499) {
307307
return buffered;
308308
}
309-
if (upstream.ok && buffered.status >= 500) {
310-
// The upstream account returned 200 — it is healthy. The buffering failure
311-
// (oversized body exceeding COMPACT_RESPONSE_MAX_BYTES, or a rare mid-read
312-
// reset on a small JSON payload) is a local proxy issue, not account flakiness.
313-
// Record the upstream status so a deterministic payload-size limit does not
314-
// soft-avoid a healthy account and rotate a thread for 30s.
315-
recordCompactPoolOutcome(upstream.status, { retryAfter, resetAt });
316-
} else {
317-
recordCompactPoolOutcome(upstream.status, { retryAfter, resetAt });
318-
}
309+
// Always record the real upstream status: a local buffering failure after a
310+
// 200 upstream response must not soft-avoid a healthy account or rotate a thread.
311+
recordCompactPoolOutcome(upstream.status, { retryAfter, resetAt });
319312
return buffered;
320313
}
321314

‎tests/codex-auth-context.test.ts‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -310,6 +310,43 @@ describe("Codex auth context", () => {
310310
}
311311
});
312312

313+
test("a scoped cooldown uses another account without moving the independent native scope", async () => {
314+
const originalNow = Date.now;
315+
const now = 1_800_000_000_000;
316+
const cfg = config();
317+
cfg.codexAccounts?.push({
318+
id: "pool-b",
319+
320+
isMain: false,
321+
chatgptAccountId: "pool_b_acc",
322+
});
323+
const headers = new Headers({ authorization: "Bearer main_token" });
324+
for (const accountId of ["pool-a", "pool-b"]) {
325+
saveCodexAccountCredential(accountId, {
326+
accessToken: `${accountId}-token`,
327+
refreshToken: `${accountId}-refresh`,
328+
expiresAt: now + 24 * 60 * 60_000,
329+
chatgptAccountId: `${accountId}-acc`,
330+
});
331+
}
332+
try {
333+
Date.now = () => now;
334+
recordCodexUpstreamOutcome(cfg, "pool-a", 429, {
335+
now,
336+
resetAt: Math.floor((now + 4 * 24 * 60 * 60_000) / 1_000),
337+
modelId: "gpt-5.3-codex-spark",
338+
});
339+
340+
await expect(resolveCodexAuthContext(headers, cfg, "pool", { modelId: "gpt-5.3-codex-spark" }))
341+
.resolves.toMatchObject({ kind: "pool", accountId: "pool-b" });
342+
expect(cfg.activeCodexAccountId).toBe("pool-a");
343+
await expect(resolveCodexAuthContext(headers, cfg, "pool", { modelId: "gpt-5.6-terra" }))
344+
.resolves.toMatchObject({ kind: "pool", accountId: "pool-a" });
345+
} finally {
346+
Date.now = originalNow;
347+
}
348+
});
349+
313350
test("a successful Spark recovery probe leaves the shared native cooldown intact", async () => {
314351
const originalNow = Date.now;
315352
const now = 1_800_000_000_000;
@@ -529,6 +566,17 @@ describe("cooldown error surface", () => {
529566
expect(message).toContain("account-…3c21");
530567
});
531568

569+
test("message identifies a model-scoped cooldown without implying an account-wide block", () => {
570+
const err = new CodexAccountCooldownError(
571+
"acct_9f3c21",
572+
Date.parse("2026-07-26T10:00:00.000Z"),
573+
"reset-derived",
574+
"spark",
575+
);
576+
577+
expect(cooldownErrorMessage(err)).toContain("Spark quota is cooling down");
578+
});
579+
532580
test("the main login renders as the alias users actually type", () => {
533581
const err = new CodexAccountCooldownError(MAIN_CODEX_ACCOUNT_ID, Date.now() + 60_000);
534582

‎tests/codex-routing.test.ts‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -315,6 +315,23 @@ describe("codex routing", () => {
315315
expect(resolveCodexAccountForThread("quota-next", config)).toBe("b");
316316
});
317317

318+
test("shared native reset cooldown clears affinity and rotates the active account", () => {
319+
const config = makeConfig();
320+
const now = 1_800_000_000_000;
321+
updateAccountQuota("a", 10);
322+
updateAccountQuota("b", 20);
323+
expect(resolveCodexAccountForThread("shared-quota-existing", config, now)).toBe("a");
324+
325+
recordCodexUpstreamOutcome(config, "a", 429, {
326+
now,
327+
resetAt: Math.floor((now + 4 * 24 * 60 * 60_000) / 1_000),
328+
modelId: "gpt-5.6-terra",
329+
});
330+
331+
expect(config.activeCodexAccountId).toBe("b");
332+
expect(resolveCodexAccountForThread("shared-quota-existing", config, now + 1)).toBe("b");
333+
});
334+
318335
test("2xx responses clear transient failures without clearing an unexpired cooldown", () => {
319336
const config = makeConfig();
320337
const now = 1_800_000_000_000;

0 commit comments

Comments
 (0)