Skip to content

Commit 5c303a6

Browse files
committed
fix(codex): tolerate extended gated-model shard drift
1 parent aac1177 commit 5c303a6

3 files changed

Lines changed: 71 additions & 27 deletions

File tree

‎src/server/responses/core.ts‎

Lines changed: 43 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -592,7 +592,7 @@ async function retryCodexPoolOnAlternateAccount(
592592
invalidateCodexModelEntitlementsForAccount(firstAuthCtx.accountId);
593593
const refreshed = await resolveCodexModelEntitlements(config);
594594
if (entitledCodexAccountIdsForModel(refreshed, route.modelId)?.has(firstAuthCtx.accountId)) {
595-
// The authenticated roster still grants this exact model. Retry once on the same account:
595+
// The authenticated roster still grants this exact model. Retry on the same account:
596596
// upstream shards can briefly disagree during a gated-model rollout, but a pre-stream 400
597597
// proves no output was committed and keeps this replay bounded.
598598
retryAuthCtx = firstAuthCtx;
@@ -693,27 +693,50 @@ async function retryCodexPoolOnAlternateAccount(
693693
logCtx.accountLogLabel,
694694
);
695695

696-
noteAttemptSend(logCtx.activeAttempt, passthroughEstimate);
696+
const retrySameConfirmedAccount = outcomeStatus === 400
697+
&& ACCOUNT_GATED_NATIVE_OPENAI_MODELS.has(route.modelId)
698+
&& retryAuthCtx.accountId === firstAuthCtx.accountId;
699+
// Live Daybreak traffic has produced long runs of unsupported-model 400s from different
700+
// upstream shards even while the authenticated roster continues to grant the model. Permit
701+
// seven additional same-account sends (eight total including the original), re-checking the
702+
// exact allow-listed body and fresh entitlement before every later send. Alternate-account and
703+
// quota recovery retain their historical one-send bound.
704+
const maxRetrySends = retrySameConfirmedAccount ? 7 : 1;
705+
let retrySendCount = 0;
697706
let upstreamResponse: Response;
698707
try {
699-
upstreamResponse = await fetchWithHeaderTimeout(
700-
request.url,
701-
{
702-
method: request.method,
703-
headers: request.headers,
704-
body: request.body,
705-
},
706-
upstream.signal,
707-
connectMs,
708-
stream,
709-
providerFetch(route.provider, options.codexWsRuntimeIdentity, {
710-
providerName: route.providerName,
711-
modelId: route.modelId,
712-
}),
713-
// Credential-bearing forward send: never follow a redirect into a
714-
// dead-host rejection after the credential was seen (#914).
715-
route.provider.authMode === "forward",
716-
);
708+
while (true) {
709+
noteAttemptSend(logCtx.activeAttempt, passthroughEstimate);
710+
upstreamResponse = await fetchWithHeaderTimeout(
711+
request.url,
712+
{
713+
method: request.method,
714+
headers: request.headers,
715+
body: request.body,
716+
},
717+
upstream.signal,
718+
connectMs,
719+
stream,
720+
providerFetch(route.provider, options.codexWsRuntimeIdentity, {
721+
providerName: route.providerName,
722+
modelId: route.modelId,
723+
}),
724+
// Credential-bearing forward send: never follow a redirect into a
725+
// dead-host rejection after the credential was seen (#914).
726+
route.provider.authMode === "forward",
727+
);
728+
retrySendCount += 1;
729+
if (!retrySameConfirmedAccount || retrySendCount >= maxRetrySends) break;
730+
if (!await shouldRetryCodexPoolAccountModel400(
731+
upstreamResponse,
732+
route.modelId,
733+
options.abortSignal,
734+
)) break;
735+
invalidateCodexModelEntitlementsForAccount(retryAuthCtx.accountId);
736+
const refreshed = await resolveCodexModelEntitlements(config);
737+
if (!entitledCodexAccountIdsForModel(refreshed, route.modelId)?.has(retryAuthCtx.accountId)) break;
738+
await upstreamResponse.body?.cancel().catch(() => undefined);
739+
}
717740
} catch (error) {
718741
// Attribute the transport failure to the alternate account (already selected).
719742
return { kind: "transport", error, authCtx: retryAuthCtx };

‎structure/08_openai-provider-tiers.md‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -122,8 +122,9 @@ preserving a stale one would block every later migration.
122122
checks the forwarded caller credential, or stored main when an admission bearer is substituted.
123123
Discovery failures fail closed. If an
124124
entitled account still receives the exact pre-stream unsupported-model 400, opencodex invalidates
125-
that account's roster and retries once on the same account only when the refreshed roster still
126-
grants the model; otherwise ordinary eligible-account failover applies.
125+
that account's roster and permits at most seven additional same-account sends, re-confirming the
126+
exact rejection and fresh grant before each later send; otherwise ordinary eligible-account
127+
failover applies.
127128

128129
[Decision Log]
129130
- 목적과 의도: Prevent account-gated native models from being shown or dispatched through a

‎tests/server-auth.test.ts‎

Lines changed: 25 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2142,11 +2142,11 @@ describe("server local API auth", () => {
21422142
}
21432143
});
21442144

2145-
test("#2097: a confirmed entitled account retries one transient unsupported-model 400 in place", async () => {
2145+
test("#2097: a confirmed entitled account survives two transient unsupported-model 400s in place", async () => {
21462146
const model = "gpt-daybreak-blue-latest";
21472147
let attempts = 0;
21482148
const harness = await startPoolRetryHarness(
2149-
() => ++attempts === 1
2149+
() => ++attempts <= 2
21502150
? rejectionResponse(unsupportedModelBody(model))
21512151
: Response.json({ id: "same-account-success", status: "completed", output: [] }),
21522152
{
@@ -2158,7 +2158,7 @@ describe("server local API auth", () => {
21582158
const response = await harness.request({ model });
21592159
expect(response.status).toBe(200);
21602160
expect((await response.json() as { id: string }).id).toBe("same-account-success");
2161-
expect(harness.dispatches).toEqual(["acct-pool-a", "acct-pool-a"]);
2161+
expect(harness.dispatches).toEqual(["acct-pool-a", "acct-pool-a", "acct-pool-a"]);
21622162
} finally {
21632163
await stopPoolRetryHarness(harness);
21642164
}
@@ -2168,7 +2168,7 @@ describe("server local API auth", () => {
21682168
const model = "gpt-daybreak-blue-latest";
21692169
let attempts = 0;
21702170
const harness = await startPoolRetryHarness(
2171-
() => ++attempts === 1
2171+
() => ++attempts <= 2
21722172
? rejectionResponse(unsupportedModelBody(model))
21732173
: Response.json({ id: "same-exact-account-success", status: "completed", output: [] }),
21742174
{
@@ -2182,13 +2182,33 @@ describe("server local API auth", () => {
21822182
const response = await harness.request({ model: `side/${model}`, callerBearer: false });
21832183
expect(response.status).toBe(200);
21842184
expect((await response.json() as { id: string }).id).toBe("same-exact-account-success");
2185-
expect(harness.dispatches).toEqual(["acct-pool-a", "acct-pool-a"]);
2185+
expect(harness.dispatches).toEqual(["acct-pool-a", "acct-pool-a", "acct-pool-a"]);
21862186
expect(loadConfig().activeCodexAccountId).toBe("pool-b");
21872187
} finally {
21882188
await stopPoolRetryHarness(harness);
21892189
}
21902190
});
21912191

2192+
test("#2097: repeated gated-model rejection remains bounded at eight total sends", async () => {
2193+
const model = "gpt-daybreak-blue-latest";
2194+
const body = unsupportedModelBody(model);
2195+
const harness = await startPoolRetryHarness(
2196+
() => rejectionResponse(body),
2197+
{
2198+
secondAccount: false,
2199+
modelRosterByAccount: { "acct-pool-a": ["gpt-5.6-sol", model] },
2200+
},
2201+
);
2202+
try {
2203+
const response = await harness.request({ model });
2204+
expect(response.status).toBe(400);
2205+
expect(await response.text()).toBe(body);
2206+
expect(harness.dispatches).toEqual(Array(8).fill("acct-pool-a"));
2207+
} finally {
2208+
await stopPoolRetryHarness(harness);
2209+
}
2210+
});
2211+
21922212
test("#2097: an account-gated model with no confirmed grant fails before upstream dispatch", async () => {
21932213
const model = "gpt-daybreak-blue-latest";
21942214
const harness = await startPoolRetryHarness(

0 commit comments

Comments
 (0)