Skip to content

Commit beab5b3

Browse files
authored
Merge pull request #352 from lidge-jun/codex/260724-pool-retry
fix(codex): bounded single pool retry on account-specific unsupported-model 400 (#335)
2 parents affc477 + 8e2b247 commit beab5b3

4 files changed

Lines changed: 627 additions & 12 deletions

File tree

‎src/codex/auth-context.ts‎

Lines changed: 18 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,11 @@ import {
77
import { markAccountNeedsReauth } from "./account-runtime-state";
88
import { isCodexAccountUsable } from "./account-usability";
99
import { MAIN_CODEX_ACCOUNT_ID, getMainAccountToken } from "./main-account";
10-
import { getCodexAccountCooldownUntil, resolveCodexAccountForThreadDetailed } from "./routing";
10+
import {
11+
getCodexAccountCooldownUntil,
12+
pickLowestUsageCodexAccount,
13+
resolveCodexAccountForThreadDetailed,
14+
} from "./routing";
1115
import { getAccountQuota } from "./quota";
1216
import type { CodexAccountMode, OcxConfig, OcxProviderConfig } from "../types";
1317
import { FORWARD_HEADERS } from "../adapters/openai-responses";
@@ -89,17 +93,29 @@ export function shouldMarkAccountNeedsReauthForCodexAuthFailure(cause: unknown):
8993
return !(cause instanceof CodexCredentialGenerationConflictError) && !(cause instanceof CodexCredentialRefreshLockTimeoutError);
9094
}
9195

96+
export interface ResolveCodexAuthContextOptions {
97+
excludeAccountId?: string;
98+
}
99+
92100
export async function resolveCodexAuthContext(
93101
headers: Headers,
94102
config: OcxConfig,
95103
mode: CodexAccountMode,
104+
options: ResolveCodexAuthContextOptions = {},
96105
): Promise<CodexAuthContext> {
97106
if (mode === "direct") {
98107
if (!hasCallerCodexBearer(headers)) throw new CodexDirectAuthenticationError();
99108
return { kind: "main", accountId: null };
100109
}
101110
const threadId = headers.get("x-codex-parent-thread-id");
102-
const resolution = resolveCodexAccountForThreadDetailed(threadId, config);
111+
const resolution = options.excludeAccountId
112+
? (() => {
113+
const accountId = pickLowestUsageCodexAccount(config, options.excludeAccountId);
114+
return accountId
115+
? { status: "selected" as const, accountId }
116+
: { status: "none" as const };
117+
})()
118+
: resolveCodexAccountForThreadDetailed(threadId, config);
103119
if (resolution.status === "expired") throw new CodexThreadAffinityExpiredError(resolution.accountId);
104120
const accountId = resolution.status === "selected" ? resolution.accountId : null;
105121
if (!accountId) throw new CodexPoolAuthenticationError();

‎src/server/responses/core.ts‎

Lines changed: 126 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@ import {
4949
headersForCodexAuthContext,
5050
isCodexAuthContextUsable,
5151
resolveCodexAuthContext,
52+
stripCodexRuntimeProviderFields,
5253
type CodexAuthContext,
5354
} from "../../codex/auth-context";
5455
import {
@@ -150,6 +151,45 @@ export function usesCodexForwardPoolAuth(
150151
&& provider.authMode === "forward" && provider.adapter === "openai-responses";
151152
}
152153

154+
function normalizeCodexUnsupportedModelDetail(value: string): string {
155+
return value.trim().replace(/\s+/gu, " ").toLocaleLowerCase("en-US");
156+
}
157+
158+
function isAllowListedCodexAccountModel400(
159+
status: number,
160+
bodyText: string,
161+
modelId: string,
162+
): boolean {
163+
if (status !== 400) return false;
164+
try {
165+
const payload = JSON.parse(bodyText) as unknown;
166+
if (!payload || typeof payload !== "object" || Array.isArray(payload)) return false;
167+
const detail = (payload as { detail?: unknown }).detail;
168+
if (typeof detail !== "string") return false;
169+
const expected = `The '${modelId}' model is not supported when using Codex with a ChatGPT account.`;
170+
return normalizeCodexUnsupportedModelDetail(detail)
171+
=== normalizeCodexUnsupportedModelDetail(expected);
172+
} catch {
173+
return false;
174+
}
175+
}
176+
177+
async function shouldRetryCodexPoolAccountModel400(
178+
response: Response,
179+
modelId: string,
180+
signal?: AbortSignal,
181+
): Promise<boolean> {
182+
if (response.status !== 400) return false;
183+
try {
184+
const body = await readBoundedResponseBody(response.clone(), { signal });
185+
return body.displaySafe
186+
&& !body.truncated
187+
&& isAllowListedCodexAccountModel400(response.status, body.text, modelId);
188+
} catch {
189+
return false;
190+
}
191+
}
192+
153193

154194

155195
export function codexForwardTerminalOutcomeRecorder(
@@ -920,7 +960,7 @@ export async function handleResponses(
920960
+ `(model ${parsed.modelId}); forwarding without it — earlier turns may be missing from this request`,
921961
);
922962
}
923-
const request = await adapter.buildRequest(parsed, { headers: selectedForwardHeaders });
963+
let request = await adapter.buildRequest(parsed, { headers: selectedForwardHeaders });
924964
const passthroughEstimate = typeof request.usageLog?.inputTokens === "number"
925965
? request.usageLog.inputTokens
926966
: undefined;
@@ -934,6 +974,20 @@ export async function handleResponses(
934974
linkAbortSignal(upstream, options.abortSignal);
935975
const connectMs = config.connectTimeoutMs ?? 200_000;
936976
let upstreamResponse: Response;
977+
const transportFailureResponse = (err: unknown): Response => {
978+
upstream.abort();
979+
if (options.abortSignal?.aborted) return clientCancelledResponse();
980+
const outcome = err instanceof Error && err.name === "TimeoutError" ? "timeout" : "connect_error";
981+
if (usesCodexForwardPoolAuth(authCtx, route.provider)) {
982+
recordCodexUpstreamOutcome(config, authCtx.accountId, outcome, {
983+
threadId: req.headers.get("x-codex-parent-thread-id"),
984+
});
985+
}
986+
const msg = outcome === "timeout"
987+
? `Provider connect timeout after ${connectMs}ms`
988+
: `Provider unreachable: ${err instanceof Error ? err.message : String(err)}`;
989+
return formatErrorResponse(502, "upstream_error", msg);
990+
};
937991
try {
938992
// Transient-5xx pre-stream retry (devlog/_plan/260716_claudecode_hardening/010):
939993
// the ChatGPT backend emits transient 502/520s that an immediate retry absorbs.
@@ -950,18 +1004,80 @@ export async function handleResponses(
9501004
{ abortSignal: upstream.signal, label: safeHostLabel(request.url) },
9511005
);
9521006
} catch (err) {
953-
upstream.abort();
954-
if (options.abortSignal?.aborted) return clientCancelledResponse();
955-
const outcome = err instanceof Error && err.name === "TimeoutError" ? "timeout" : "connect_error";
956-
if (usesCodexForwardPoolAuth(authCtx, route.provider)) {
957-
recordCodexUpstreamOutcome(config, authCtx.accountId, outcome, {
1007+
return transportFailureResponse(err);
1008+
}
1009+
1010+
if (
1011+
usesCodexForwardPoolAuth(authCtx, route.provider)
1012+
&& await shouldRetryCodexPoolAccountModel400(
1013+
upstreamResponse,
1014+
route.modelId,
1015+
options.abortSignal,
1016+
)
1017+
) {
1018+
const firstAuthCtx = authCtx;
1019+
let retryAuthCtx: CodexAuthContext | undefined;
1020+
try {
1021+
retryAuthCtx = await resolveCodexAuthContext(
1022+
req.headers,
1023+
config,
1024+
"pool",
1025+
{ excludeAccountId: firstAuthCtx.accountId },
1026+
);
1027+
} catch (error) {
1028+
if (
1029+
!(error instanceof CodexPoolAuthenticationError)
1030+
&& !(error instanceof CodexAuthContextError)
1031+
&& !(error instanceof CodexAccountCooldownError)
1032+
) throw error;
1033+
}
1034+
1035+
if (retryAuthCtx?.kind === "pool" || retryAuthCtx?.kind === "main-pool") {
1036+
recordCodexUpstreamOutcome(config, firstAuthCtx.accountId, 400, {
9581037
threadId: req.headers.get("x-codex-parent-thread-id"),
9591038
});
1039+
1040+
const retryHeaders = headersForCodexAuthContext(req.headers, retryAuthCtx);
1041+
const retryProvider = applyCodexAuthContextToProvider(
1042+
stripCodexRuntimeProviderFields(route.provider),
1043+
retryAuthCtx,
1044+
"pool",
1045+
);
1046+
const retryAdapter = resolveAdapter(
1047+
resolveWireProtocolOverride(route.providerName, route.modelId, retryProvider),
1048+
config.cacheRetention,
1049+
);
1050+
request = await retryAdapter.buildRequest(parsed, { headers: retryHeaders });
1051+
1052+
await upstreamResponse.body?.cancel().catch(() => undefined);
1053+
authCtx = retryAuthCtx;
1054+
options.onCodexAuthContextResolved?.(retryAuthCtx);
1055+
selectedForwardHeaders = retryHeaders;
1056+
route.provider = retryProvider;
1057+
logCtx.provider = formatCodexProviderForLog(
1058+
route.providerName,
1059+
retryAuthCtx.accountId,
1060+
config,
1061+
);
1062+
1063+
noteAttemptSend(logCtx.activeAttempt, passthroughEstimate);
1064+
try {
1065+
upstreamResponse = await fetchWithHeaderTimeout(
1066+
request.url,
1067+
{
1068+
method: request.method,
1069+
headers: request.headers,
1070+
body: request.body,
1071+
},
1072+
upstream.signal,
1073+
connectMs,
1074+
parsed.stream,
1075+
providerFetch(route.provider),
1076+
);
1077+
} catch (err) {
1078+
return transportFailureResponse(err);
1079+
}
9601080
}
961-
const msg = outcome === "timeout"
962-
? `Provider connect timeout after ${connectMs}ms`
963-
: `Provider unreachable: ${err instanceof Error ? err.message : String(err)}`;
964-
return formatErrorResponse(502, "upstream_error", msg);
9651081
}
9661082
const headers = sanitizePassthroughHeaders(upstreamResponse.headers);
9671083
const resolvedModel = headers.get("openai-model")?.trim();

‎tests/codex-auth-context.test.ts‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import {
88
CodexAccountCooldownError,
99
CodexAuthContextError,
1010
CodexDirectAuthenticationError,
11+
CodexPoolAuthenticationError,
1112
CodexThreadAffinityExpiredError,
1213
headersForCodexAuthContext,
1314
isCodexAuthContextUsable,
@@ -46,13 +47,15 @@ beforeEach(() => {
4647
clearThreadAccountMap();
4748
clearCodexUpstreamHealth();
4849
clearAccountNeedsReauth("pool-a");
50+
clearAccountNeedsReauth("pool-b");
4951
});
5052

5153
afterEach(() => {
5254
rmSync(testDir, { recursive: true, force: true });
5355
clearThreadAccountMap();
5456
clearCodexUpstreamHealth();
5557
clearAccountNeedsReauth("pool-a");
58+
clearAccountNeedsReauth("pool-b");
5659
if (previousOpencodexHome === undefined) delete process.env.OPENCODEX_HOME;
5760
else process.env.OPENCODEX_HOME = previousOpencodexHome;
5861
if (previousCodexHome === undefined) delete process.env.CODEX_HOME;
@@ -112,6 +115,54 @@ describe("Codex auth context", () => {
112115
});
113116
});
114117

118+
test("exclusion selects another eligible account without changing the active account", async () => {
119+
const cfg = config();
120+
cfg.codexAccounts?.push({
121+
id: "pool-b",
122+
123+
isMain: false,
124+
chatgptAccountId: "pool_b_acc",
125+
});
126+
saveCodexAccountCredential("pool-a", {
127+
accessToken: "pool_a_token",
128+
refreshToken: "pool_a_refresh",
129+
expiresAt: Date.now() + 5 * 60_000,
130+
chatgptAccountId: "pool_a_acc",
131+
});
132+
saveCodexAccountCredential("pool-b", {
133+
accessToken: "pool_b_token",
134+
refreshToken: "pool_b_refresh",
135+
expiresAt: Date.now() + 5 * 60_000,
136+
chatgptAccountId: "pool_b_acc",
137+
});
138+
139+
await expect(resolveCodexAuthContext(new Headers(), cfg, "pool", {
140+
excludeAccountId: "pool-a",
141+
})).resolves.toMatchObject({
142+
kind: "pool",
143+
accountId: "pool-b",
144+
accessToken: "pool_b_token",
145+
chatgptAccountId: "pool_b_acc",
146+
});
147+
expect(cfg.activeCodexAccountId).toBe("pool-a");
148+
});
149+
150+
test("exclusion fails closed when no alternate account is eligible", async () => {
151+
saveCodexAccountCredential("pool-a", {
152+
accessToken: "pool_a_token",
153+
refreshToken: "pool_a_refresh",
154+
expiresAt: Date.now() + 5 * 60_000,
155+
chatgptAccountId: "pool_a_acc",
156+
});
157+
158+
await expect(resolveCodexAuthContext(
159+
new Headers({ authorization: "Bearer caller_token" }),
160+
config(),
161+
"pool",
162+
{ excludeAccountId: "pool-a" },
163+
)).rejects.toBeInstanceOf(CodexPoolAuthenticationError);
164+
});
165+
115166
test("selected pool headers replace inbound main auth", () => {
116167
const headers = headersForCodexAuthContext(
117168
new Headers({ authorization: "Bearer main_token", "chatgpt-account-id": "main_acc", "openai-beta": "responses=experimental" }),

0 commit comments

Comments
 (0)