Skip to content

Commit 48338a1

Browse files
authored
Merge pull request #955 from lidge-jun/codex/915-cooldown-recovery-probe
stack 5/7: probe reset-derived cooldowns without waiting to be selected (#915)
2 parents ec8ba11 + 8b69769 commit 48338a1

5 files changed

Lines changed: 579 additions & 4 deletions

File tree

‎src/codex/auth-api.ts‎

Lines changed: 52 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,11 +16,13 @@ import {
1616
import { deleteCodexAccount, reconcileMainCodexAccountRuntimeState } from "./account-lifecycle";
1717
import { isCodexAccountPaused, setCodexAccountPaused } from "./account-pause";
1818
import {
19+
claimDueCodexQuotaRecoveryProbes,
1920
clearCodexAccountCooldown,
2021
clearThreadAccountMapForAccount,
2122
getEffectiveActiveCodexAccountId,
2223
reconcileCodexActiveAfterExclusion,
2324
resetCodexRoutingForManualSelection,
25+
settleCodexQuotaRecoveryProbe,
2426
} from "./routing";
2527
import {
2628
normalizeAccountPoolStickyLimit,
@@ -35,6 +37,7 @@ import { clearAccountNeedsReauth, isAccountNeedsReauth, markAccountNeedsReauth }
3537
import {
3638
clearAccountQuota,
3739
getAccountQuota,
40+
isCompleteCodexQuotaRecoverySnapshot,
3841
isCodexQuotaExhausted,
3942
listAccountQuotas,
4043
parseUsageQuota,
@@ -53,7 +56,7 @@ export {
5356
} from "./quota";
5457
import { extractAccountId, decodeJwtPayload } from "../oauth/chatgpt";
5558
import { getMainAccountPlan, MAIN_CODEX_ACCOUNT_ID, setMainAccountPlan } from "./main-account";
56-
import { captureConfigGeneration } from "../lib/state-store-sweeper";
59+
import { captureConfigGeneration, registerStateSweepAfterTick } from "../lib/state-store-sweeper";
5760
import { reconcileLiveStateStores } from "../lib/state-store-registrations";
5861
import {
5962
captureMainAccountIdentityGeneration,
@@ -816,6 +819,49 @@ async function fetchPoolAccountQuota(accountId: string, forceRefresh = false, co
816819
}
817820

818821
let primeInFlight: Promise<void> | null = null;
822+
let cooldownRecoveryInFlight: Promise<void> | null = null;
823+
824+
export async function runCodexCooldownRecoveryProbes(config: OcxConfig, now = Date.now()): Promise<void> {
825+
const openai = config.providers[OPENAI_CODEX_PROVIDER_ID];
826+
if (!openai
827+
|| openai.disabled === true
828+
|| !isCanonicalOpenAiForwardProvider(openai)
829+
|| providerCodexAccountMode(OPENAI_CODEX_PROVIDER_ID, openai) !== "pool") return;
830+
if (cooldownRecoveryInFlight) return cooldownRecoveryInFlight;
831+
cooldownRecoveryInFlight = (async () => {
832+
const claims = claimDueCodexQuotaRecoveryProbes(config, POOL_QUOTA_REFRESH_CONCURRENCY, now);
833+
await mapWithConcurrency(claims, POOL_QUOTA_REFRESH_CONCURRENCY, async claim => {
834+
const account = configuredPoolAccount(config, claim.accountId);
835+
if (!account) {
836+
settleCodexQuotaRecoveryProbe(claim, false, {}, now);
837+
return;
838+
}
839+
try {
840+
const result = await fetchPoolAccountQuota(claim.accountId, true, account.plan);
841+
// Defence in depth: `spark` is already excluded at the claim site, since generic WHAM
842+
// cannot prove a spark recovery. Keep the settle-side guard so a future claim change
843+
// cannot silently start clearing spark on generic evidence.
844+
const recovered = claim.scope !== "spark"
845+
&& isCompleteCodexQuotaRecoverySnapshot(result.freshQuota ?? null, result.freshPlan ?? account.plan);
846+
settleCodexQuotaRecoveryProbe(claim, recovered, {
847+
credentialGeneration: result.freshCredentialGeneration,
848+
}, now);
849+
} catch {
850+
settleCodexQuotaRecoveryProbe(claim, false, {}, now);
851+
}
852+
});
853+
})().catch(() => {
854+
// Background recovery is best-effort; routing keeps the cooldown on failure.
855+
}).finally(() => { cooldownRecoveryInFlight = null; });
856+
return cooldownRecoveryInFlight;
857+
}
858+
859+
export function registerCodexCooldownRecoveryProbeWorker(config: OcxConfig): void {
860+
registerStateSweepAfterTick({
861+
name: "codex-cooldown-recovery",
862+
afterTick: () => { void runCodexCooldownRecoveryProbes(config); },
863+
});
864+
}
819865

820866
export interface PrimeCodexPoolQuotasOptions {
821867
/** Test seams for proving fenced/recovery priming performs no native-main work. */
@@ -908,6 +954,11 @@ export function clearCodexQuotaPrimeState(): void {
908954
primeInFlight = null;
909955
}
910956

957+
/** Test-only reset for the worker-level single-flight. */
958+
export function clearCodexCooldownRecoveryProbeState(): void {
959+
cooldownRecoveryInFlight = null;
960+
}
961+
911962
export function effectiveCodexAuthAccountId(config: OcxConfig): string {
912963
return getEffectiveActiveCodexAccountId(config) ?? MAIN_CODEX_ACCOUNT_ID;
913964
}

‎src/codex/quota.ts‎

Lines changed: 36 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -67,15 +67,48 @@ export function isCodexQuotaExhausted(
6767
plan?: string | null,
6868
): boolean {
6969
if (!quota) return false;
70-
const normalizedPlan = plan?.trim().toLowerCase();
71-
const values = normalizedPlan === "go" || normalizedPlan === "free"
70+
const values = codexQuotaWindowForPlan(plan) === "monthly"
7271
? [quota.monthlyPercent]
7372
: [quota.weeklyPercent, quota.monthlyPercent];
7473
return values.some(value => typeof value === "number"
7574
&& Number.isFinite(value)
7675
&& value >= CODEX_EXHAUSTED_USAGE_PERCENT);
7776
}
7877

78+
/**
79+
* Which usage window a plan reports in. This is the SINGLE rule shared by quota
80+
* parsing, exhaustion, and recovery — they must not diverge.
81+
*
82+
* An allowlist of "known" plans was tried here and was wrong: the upstream model
83+
* snapshot alone carries 21 distinct plan strings (`edu_plus`, `finserv`, `k12`,
84+
* `quorum`, `self_serve_business_usage_based`, ...), and `CodexAccount.plan` is an
85+
* unrestricted string, so any list is a list of the plans someone remembered.
86+
* Twelve real plans would have been refused recovery and stayed cooled forever —
87+
* the very defect this unit exists to fix, reintroduced as a typo-shaped hole.
88+
*
89+
* The honest rule is the parser's own: Go and Free report a 30-day window,
90+
* everything else (including an absent plan) reports weekly. Recovery reads the
91+
* window the parser actually wrote rather than second-guessing it.
92+
*/
93+
export function codexQuotaWindowForPlan(plan?: string | null): "monthly" | "weekly" {
94+
const normalized = plan?.trim().toLowerCase();
95+
return normalized === "go" || normalized === "free" ? "monthly" : "weekly";
96+
}
97+
98+
export function isCompleteCodexQuotaRecoverySnapshot(
99+
quota: Pick<StoredAccountQuota, "weeklyPercent" | "monthlyPercent"> | null,
100+
plan?: string | null,
101+
): boolean {
102+
if (!quota || isCodexQuotaExhausted(quota, plan)) return false;
103+
// Recovery still fails closed on MISSING EVIDENCE — a credits-only or windowless payload
104+
// carries no usage reading at all and must never clear a cooldown. What it does not do is
105+
// fail closed on an unfamiliar plan NAME, which only ever meant "cooled forever".
106+
const required = codexQuotaWindowForPlan(plan) === "monthly"
107+
? quota.monthlyPercent
108+
: quota.weeklyPercent;
109+
return typeof required === "number" && Number.isFinite(required);
110+
}
111+
79112
export function normalizeUsagePercent(value: unknown): number | undefined {
80113
const numeric = typeof value === "number"
81114
? value
@@ -367,7 +400,7 @@ export function parseUsageQuota(data: WhamUsageResponse): Omit<StoredAccountQuot
367400
}
368401

369402
const quota: Omit<StoredAccountQuota, "updatedAt"> = {};
370-
const thirtyDayOnly = data.plan_type?.trim().toLowerCase() === "go" || data.plan_type?.trim().toLowerCase() === "free";
403+
const thirtyDayOnly = codexQuotaWindowForPlan(data.plan_type) === "monthly";
371404
const primaryWindow = data.rate_limit.primary_window;
372405
const secondaryWindow = data.rate_limit.secondary_window;
373406
const tertiaryWindow = data.rate_limit.tertiary_window;

‎src/codex/routing.ts‎

Lines changed: 125 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -133,6 +133,18 @@ export type CodexCooldownSource = "retry-after" | "reset-derived" | "default";
133133
*/
134134
export type CodexQuotaScope = "shared" | "spark";
135135

136+
export type CodexQuotaRecoveryProbeClaim = {
137+
accountId: string;
138+
scope?: CodexQuotaScope;
139+
leaseId: string;
140+
cooldownGeneration: number;
141+
credentialGeneration: number;
142+
};
143+
144+
export type CodexQuotaRecoveryProbeProof = {
145+
credentialGeneration?: number;
146+
};
147+
136148
/**
137149
* Requests without a resolved native model retain the historic one-account-per-
138150
* thread behavior. Requests with a known quota scope get an independent
@@ -417,6 +429,119 @@ function canAcquireQuotaProbeLease(health: CodexUpstreamHealth | undefined, now:
417429
return now - origin >= CODEX_QUOTA_PROBE_INTERVAL_MS;
418430
}
419431

432+
/**
433+
* Claim due reset-derived cooldown probes without consulting account selection.
434+
* Pool credentials only: the main account has no quota-refresh single-flight.
435+
*/
436+
export function claimDueCodexQuotaRecoveryProbes(
437+
config: OcxConfig,
438+
limit: number,
439+
now = Date.now(),
440+
): CodexQuotaRecoveryProbeClaim[] {
441+
const boundedLimit = Math.max(0, Math.floor(limit));
442+
if (boundedLimit === 0) return [];
443+
const candidates: Array<{
444+
accountId: string;
445+
scope?: CodexQuotaScope;
446+
health: CodexUpstreamHealth;
447+
credentialGeneration: number;
448+
order: number;
449+
}> = [];
450+
for (const [order, account] of (config.codexAccounts ?? []).entries()) {
451+
if (!isSelectableCodexPoolAccount(account)
452+
|| isCodexAccountPaused(config, account.id)
453+
|| isAccountNeedsReauth(account.id)) continue;
454+
const record = readCodexAccountRecord(account.id);
455+
if (!record?.credential || record.deletedAt != null) continue;
456+
const due = [
457+
{ scope: undefined, health: upstreamHealth.get(account.id) },
458+
...[...(quotaScopedHealth.get(account.id) ?? [])].map(([scope, health]) => ({ scope, health })),
459+
].filter((entry): entry is { scope?: CodexQuotaScope; health: CodexUpstreamHealth } =>
460+
// `spark` is deliberately never claimed. `GET /backend-api/wham/usage` takes no scope
461+
// parameter and returns generic weekly/monthly windows, so its result can never prove a
462+
// spark recovery — a claim here would spend an upstream call to settle `false` every
463+
// time, and (with one claim per account per pass) delay the shared scope that CAN recover.
464+
entry.scope !== "spark"
465+
&& entry.health?.cooldownSource === "reset-derived"
466+
&& canAcquireQuotaProbeLease(entry.health, now))
467+
.sort((a, b) =>
468+
(a.health.lastProbeAt ?? a.health.cooldownSince ?? 0)
469+
- (b.health.lastProbeAt ?? b.health.cooldownSince ?? 0));
470+
const candidate = due[0];
471+
if (candidate) candidates.push({
472+
accountId: account.id,
473+
...(candidate.scope ? { scope: candidate.scope } : {}),
474+
health: candidate.health,
475+
credentialGeneration: record.generation,
476+
order,
477+
});
478+
}
479+
candidates.sort((a, b) => {
480+
const age = (a.health.lastProbeAt ?? a.health.cooldownSince ?? 0)
481+
- (b.health.lastProbeAt ?? b.health.cooldownSince ?? 0);
482+
return age || a.order - b.order;
483+
});
484+
return candidates.slice(0, boundedLimit).map(candidate => {
485+
const leaseId = randomUUID();
486+
const next = {
487+
...candidate.health,
488+
probeLeaseId: leaseId,
489+
probeLeaseGeneration: candidate.health.cooldownGeneration ?? 0,
490+
lastProbeAt: now,
491+
};
492+
if (candidate.scope) setScopedHealth(candidate.accountId, candidate.scope, next);
493+
else upstreamHealth.set(candidate.accountId, next);
494+
return {
495+
accountId: candidate.accountId,
496+
...(candidate.scope ? { scope: candidate.scope } : {}),
497+
leaseId,
498+
cooldownGeneration: candidate.health.cooldownGeneration ?? 0,
499+
credentialGeneration: candidate.credentialGeneration,
500+
};
501+
});
502+
}
503+
504+
/** Settle one background recovery claim without mutating account-wide outcome state. */
505+
export function settleCodexQuotaRecoveryProbe(
506+
claim: CodexQuotaRecoveryProbeClaim,
507+
recovered: boolean,
508+
proof: CodexQuotaRecoveryProbeProof,
509+
now = Date.now(),
510+
): boolean {
511+
const health = claim.scope
512+
? scopedHealthFor(claim.accountId, claim.scope)
513+
: upstreamHealth.get(claim.accountId);
514+
if (!health || health.probeLeaseId !== claim.leaseId) return false;
515+
const fenced = (health.cooldownGeneration ?? 0) === claim.cooldownGeneration
516+
&& (health.probeLeaseGeneration ?? 0) === claim.cooldownGeneration
517+
&& claim.credentialGeneration === proof.credentialGeneration
518+
&& isCodexAccountGenerationLive(claim.accountId, claim.credentialGeneration);
519+
if (!recovered || !fenced) {
520+
const released = withProbeLeaseReleased(health, now);
521+
if (claim.scope) setScopedHealth(claim.accountId, claim.scope, released);
522+
else upstreamHealth.set(claim.accountId, released);
523+
return false;
524+
}
525+
if (claim.scope) {
526+
deleteScopedHealth(claim.accountId, claim.scope);
527+
} else {
528+
const {
529+
cooldownUntil: _until,
530+
cooldownSince: _since,
531+
cooldownSource: _source,
532+
probeLeaseId: _leaseId,
533+
probeLeaseGeneration: _leaseGeneration,
534+
...rest
535+
} = health;
536+
upstreamHealth.set(claim.accountId, {
537+
...rest,
538+
cooldownGeneration: claim.cooldownGeneration + 1,
539+
lastProbeAt: now,
540+
});
541+
}
542+
return true;
543+
}
544+
420545
/** Acquire the recovery probe for one confirmed model-specific quota group. */
421546
export function tryAcquireCodexQuotaScopeProbeLease(
422547
accountId: string,

‎src/server/index.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ import {
2020
} from "../config";
2121
import { reconcileOAuthProviders } from "../oauth";
2222
import { invalidateCodexModelsCache } from "../codex/catalog";
23+
import { registerCodexCooldownRecoveryProbeWorker } from "../codex/auth-api";
2324
import { startMemoryWatchdog } from "./memory-watchdog";
2425
import {
2526
reconcileLiveStateStores,
@@ -416,6 +417,7 @@ export function startServer(port?: number, deps: StartServerDeps = {}) {
416417
registerAppOwnedMemorySweepFallback();
417418
configureAppOwnedMemoryBudget(resolveAppOwnedMemoryBudgetBytes(config.appOwnedMemoryBudgetMb));
418419
enforceAppOwnedMemoryBudget();
420+
registerCodexCooldownRecoveryProbeWorker(config);
419421
startStateStoreSweeper();
420422
// Issue #42 Phase 3: opt-in archived auto-cleanup (default OFF). Unref'd hourly
421423
// tick for daily/weekly; startup evaluation is fire-and-forget after listen.

0 commit comments

Comments
 (0)