Skip to content

Commit 80cd0ce

Browse files
committed
fix(codex): address #914 review blockers (5xx evidence, sidecar redirects)
Keep prior transient 5xx statuses attached when fetchWithTransientRetry rejects, so a mixed 503 -> rejection stays account-attributed instead of being downgraded to the pre-connection neutral class (review blocker 1). Extend redirect:manual to the search/images/live/web-search/vision sidecar sends so a credential-bearing 3xx to a dead host is relayed as the neutral 3xx class (review blocker 2). Exact-account sidecar recorders now forward host/lastFailureCode into the (provider, host) ledger (Codex review P2). Tests: evidence-wrapper classification units, mixed-503 and redirect activation e2e, exact-account host-ledger coverage.
1 parent fe693ae commit 80cd0ce

11 files changed

Lines changed: 220 additions & 4 deletions

‎src/lib/upstream-reachability.ts‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,8 @@
2121
* MUST stay a leaf module: imports nothing from server.ts or adapters.
2222
*/
2323

24+
import { TransientRetryEvidenceError } from "./upstream-retry";
25+
2426
export const PRE_CONNECT_REACHABILITY_CODES = new Set([
2527
// Bun: DNS failure and TCP refusal share this class.
2628
"ConnectionRefused",
@@ -64,8 +66,16 @@ export type TransportFailureKind = "timeout" | "connect_neutral" | "connect_erro
6466
* account-attributed behavior.
6567
*/
6668
export function classifyTransportFailureKind(err: unknown): TransportFailureKind {
67-
if (err instanceof Error && err.name === "TimeoutError") return "timeout";
68-
if (isPreConnectReachabilityError(err)) return "connect_neutral";
69+
const evidence = err instanceof TransientRetryEvidenceError ? err : undefined;
70+
const rejection = evidence ? evidence.cause : err;
71+
if (rejection instanceof Error && rejection.name === "TimeoutError") return "timeout";
72+
if (isPreConnectReachabilityError(rejection)) {
73+
// A transient upstream response (5xx) before the rejection proves the host
74+
// and credential path were reached: the failure is account-attributable,
75+
// never the pre-connection neutral class (issue #914 review).
76+
if (evidence && evidence.transientStatuses.length > 0) return "connect_error";
77+
return "connect_neutral";
78+
}
6979
return "connect_error";
7080
}
7181

‎src/lib/upstream-retry.ts‎

Lines changed: 30 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -148,6 +148,27 @@ export interface TransientRetryOptions extends ResetRetryOptions {
148148
export type UpstreamSendRecovery = "connection-reset" | "transient-5xx";
149149
type ReplayableFetch = (recovery?: UpstreamSendRecovery) => Promise<Response>;
150150

151+
/**
152+
* Rejection thrown by {@link fetchWithTransientRetry} when the final attempt
153+
* rejects after earlier attempts already returned transient 5xx responses.
154+
*
155+
* A transient 5xx proves the host and credential path were reached, so the
156+
* failure must stay account-attributed even though the terminal promise looks
157+
* like a transport rejection (issue #914 review: mixed 5xx -> rejection must
158+
* not be downgraded to the account-neutral pre-connection class). The original
159+
* rejection is preserved as `cause` so its code and message stay inspectable.
160+
*/
161+
export class TransientRetryEvidenceError extends Error {
162+
constructor(
163+
public readonly transientStatuses: readonly number[],
164+
cause: unknown,
165+
) {
166+
const detail = cause instanceof Error ? cause.message : String(cause);
167+
super(`upstream fetch failed after transient 5xx response(s): ${detail}`, { cause });
168+
this.name = "TransientRetryEvidenceError";
169+
}
170+
}
171+
151172
/**
152173
* Opt out of Bun's keep-alive pool after a connection-reset retry.
153174
*
@@ -216,6 +237,7 @@ export async function fetchWithTransientRetry(
216237
): Promise<Response> {
217238
const attempts = Math.max(1, opts.attempts ?? TRANSIENT_RETRY_MAX_ATTEMPTS);
218239
const slowAttemptMs = opts.slowAttemptMs ?? TRANSIENT_RETRY_SLOW_ATTEMPT_MS;
240+
const transientStatuses: number[] = [];
219241
let attemptStart = Date.now();
220242
let res = await fetchWithResetRetry(doFetch, opts);
221243
for (let attempt = 0; attempt < attempts - 1; attempt++) {
@@ -233,7 +255,14 @@ export async function fetchWithTransientRetry(
233255
cancelResponseBodyBestEffort(res);
234256
await sleepWithAbort(delay, opts.abortSignal);
235257
attemptStart = Date.now();
236-
res = await fetchWithResetRetry(doFetch, opts, "transient-5xx");
258+
transientStatuses.push(res.status);
259+
try {
260+
res = await fetchWithResetRetry(doFetch, opts, "transient-5xx");
261+
} catch (err) {
262+
// Keep the prior 5xx evidence attached: the origin already responded, so
263+
// this rejection is not pre-connection and must not classify as neutral.
264+
throw new TransientRetryEvidenceError(transientStatuses, err);
265+
}
237266
}
238267
return res;
239268
}

‎src/providers/openai-sidecar.ts‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -124,7 +124,7 @@ export async function resolveFirstUsableOpenAiSidecar(
124124
...candidate,
125125
authContext,
126126
headers: headersForCodexAuthContext(incomingHeaders, authContext),
127-
recordOutcome: (outcome: CodexUpstreamOutcome) => recordCodexUpstreamOutcome(
127+
recordOutcome: (outcome: CodexUpstreamOutcome, meta: SidecarOutcomeMeta = {}) => recordCodexUpstreamOutcome(
128128
config,
129129
authContext.accountId,
130130
outcome,
@@ -134,6 +134,11 @@ export async function resolveFirstUsableOpenAiSidecar(
134134
probeLeaseId: authContext.probeLeaseId,
135135
probeQuotaScope: authContext.probeQuotaScope,
136136
writerGeneration: authContext.writerGeneration,
137+
// Exact-account sends must feed the (provider, host) ledger exactly
138+
// like pool sends do, or fixed-account DNS/TCP failures undercount
139+
// hostConnectHealth (Codex review P2).
140+
...(meta.host ? { hostKey: hostConnectHealthKey(candidate.providerName, meta.host) } : {}),
141+
...(meta.lastFailureCode ? { lastFailureCode: meta.lastFailureCode } : {}),
137142
},
138143
),
139144
};

‎src/server/images.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -454,6 +454,8 @@ export async function handleImages(
454454
headers,
455455
body: JSON.stringify(body),
456456
signal: linkedSignal.signal,
457+
// #914: never follow a redirect into a dead-host rejection after the credential was seen.
458+
redirect: "manual",
457459
});
458460
// Buffer rather than stream: the payload is one JSON document (base64 image, typically a few
459461
// MB), and buffering keeps the timeout window covering the whole exchange. Cap the size to

‎src/server/live.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -561,6 +561,8 @@ export async function handleLive(
561561
headers,
562562
body: outboundBody,
563563
signal: linkedSignal.signal,
564+
// #914: never follow a redirect into a dead-host rejection after the credential was seen.
565+
redirect: "manual",
564566
});
565567
// Record every completed upstream response before body size handling so account health /
566568
// cooldown still updates when we reject an oversized payload.

‎src/server/search.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -154,6 +154,8 @@ export async function handleSearch(
154154
headers,
155155
body: JSON.stringify(relayBody),
156156
signal: linkedSignal.signal,
157+
// #914: never follow a redirect into a dead-host rejection after the credential was seen.
158+
redirect: "manual",
157159
});
158160
const payload = await upstreamResponse.arrayBuffer();
159161
if (payload.byteLength > SEARCH_RESPONSE_MAX_BYTES) {

‎src/vision/describe.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -98,6 +98,8 @@ export async function describeImage(
9898
headers,
9999
body: JSON.stringify(body),
100100
signal: linkedSignal.signal,
101+
// #914: never follow a redirect into a dead-host rejection after the credential was seen.
102+
redirect: "manual",
101103
}),
102104
{ abortSignal: linkedSignal.signal, label: "vision-sidecar" },
103105
);

‎src/web-search/executor.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,8 @@ export async function runWebSearch(
8888
headers,
8989
body: JSON.stringify(body),
9090
signal: linkedSignal.signal,
91+
// #914: never follow a redirect into a dead-host rejection after the credential was seen.
92+
redirect: "manual",
9193
}),
9294
{ abortSignal: linkedSignal.signal, label: "web-search-sidecar" },
9395
);

‎tests/issue-914-transport-attribution.test.ts‎

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -283,4 +283,90 @@ describe("issue #914: pre-connection reachability failures are account-neutral",
283283
await server.stop(true);
284284
}
285285
});
286+
287+
test("mixed transient 5xx then rejection stays account-attributed (#914 review)", async () => {
288+
const config = makePoolConfig();
289+
saveConfig(config);
290+
let upstreamCalls = 0;
291+
globalThis.fetch = (async (input, init) => {
292+
const requestUrl = input instanceof Request
293+
? input.url
294+
: input instanceof URL
295+
? input.toString()
296+
: String(input);
297+
const url = new URL(requestUrl);
298+
if (url.hostname === "chatgpt.com" && url.pathname.startsWith("/backend-api/codex")) {
299+
upstreamCalls += 1;
300+
if (upstreamCalls === 1) {
301+
return new Response("Service Unavailable", { status: 503, headers: { "content-type": "text/plain" } });
302+
}
303+
throw Object.assign(new Error("Unable to connect. Is the computer able to access the url?"), {
304+
code: "ConnectionRefused",
305+
errno: 0,
306+
});
307+
}
308+
return ORIGINAL_FETCH(input, init);
309+
}) as typeof fetch;
310+
const server = startServer(0);
311+
try {
312+
const res = await responsesRequest(server.url.toString(), "thread-mixed-503");
313+
expect(res.status).toBe(502);
314+
315+
// The upstream already answered 503 before the final rejection, so the
316+
// failure is account evidence — the pre-connection ledger must NOT swallow it.
317+
expect(getCodexUpstreamHealth("a")).toMatchObject({ consecutiveFailures: 1, lastFailureStatus: 0 });
318+
expect(isCodexAccountSoftAvoided("a")).toBe(false);
319+
expect(getHostConnectHealth(hostConnectHealthKey("openai", "chatgpt.com"))).toBeNull();
320+
} finally {
321+
await server.stop(true);
322+
}
323+
});
324+
325+
test("pool sends relay 3xx instead of following a redirect into a dead host (#914 review)", async () => {
326+
const config = makePoolConfig();
327+
saveConfig(config);
328+
globalThis.fetch = (async (input, init) => {
329+
const requestUrl = input instanceof Request
330+
? input.url
331+
: input instanceof URL
332+
? input.toString()
333+
: String(input);
334+
const url = new URL(requestUrl);
335+
if (url.hostname === "chatgpt.com" && url.pathname.startsWith("/backend-api/codex")) {
336+
if (init?.redirect !== "manual") {
337+
// What default-follow fetch would do: chase the 307 into a dead host and
338+
// reject with a pre-connection shape AFTER the credential was seen.
339+
throw Object.assign(new Error("Unable to connect. Is the computer able to access the url?"), {
340+
code: "ConnectionRefused",
341+
errno: 0,
342+
});
343+
}
344+
return new Response(JSON.stringify({ location: "https://dead.invalid/x" }), {
345+
status: 307,
346+
headers: { "content-type": "application/json", location: "https://dead.invalid/x" },
347+
});
348+
}
349+
return ORIGINAL_FETCH(input, init);
350+
}) as typeof fetch;
351+
const server = startServer(0);
352+
try {
353+
// The relayed 307 carries a Location header, so the test client must not
354+
// follow it (default-follow would chase it into the dead host).
355+
const res = await ORIGINAL_FETCH(new URL("/v1/responses", server.url), {
356+
method: "POST",
357+
headers: { "content-type": "application/json", "x-codex-parent-thread-id": "thread-redirect" },
358+
body: JSON.stringify({ model: "gpt-5.6-sol", input: "hello", stream: false }),
359+
redirect: "manual",
360+
});
361+
expect(res.status).toBe(307);
362+
363+
// 3xx is the neutral class: no account evidence, no host evidence.
364+
expect(getCodexUpstreamHealth("a")).toBeNull();
365+
expect(isCodexAccountSoftAvoided("a")).toBe(false);
366+
expect(getEffectiveActiveCodexAccountId(config)).toBe("a");
367+
expect(getHostConnectHealth(hostConnectHealthKey("openai", "chatgpt.com"))).toBeNull();
368+
} finally {
369+
await server.stop(true);
370+
}
371+
});
286372
});

‎tests/server-search.test.ts‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,8 @@ import {
1313
clearCodexUpstreamHealth,
1414
clearThreadAccountMap,
1515
getCodexUpstreamHealth,
16+
getHostConnectHealth,
17+
hostConnectHealthKey,
1618
recordCodexUpstreamOutcome,
1719
} from "../src/codex/routing";
1820
import { loadConfig, saveConfig } from "../src/config";
@@ -352,6 +354,46 @@ test("an exact search account needing reauthentication fails closed with an acti
352354
}
353355
});
354356

357+
test("an exact-account DNS/TCP failure feeds the (provider, host) ledger (#914 review P2)", async () => {
358+
const config = exactSearchConfig();
359+
saveConfig(config);
360+
saveExactSearchCredentials();
361+
globalThis.fetch = ((input: RequestInfo | URL, init?: RequestInit) => {
362+
const requestUrl = typeof input === "string" ? input : input instanceof URL ? input.toString() : input.url;
363+
const url = new URL(requestUrl);
364+
if (url.hostname === "chatgpt.com" && url.pathname.startsWith("/backend-api/codex")) {
365+
throw Object.assign(new Error("Unable to connect. Is the computer able to access the url?"), {
366+
code: "ConnectionRefused",
367+
errno: 0,
368+
});
369+
}
370+
return originalFetch(input, init);
371+
}) as typeof fetch;
372+
373+
const server = startServer(0);
374+
try {
375+
const response = await fetch(new URL("/v1/alpha/search", server.url), {
376+
method: "POST",
377+
headers: { "content-type": "application/json" },
378+
body: JSON.stringify({ id: "search-session", model: "side/gpt-test" }),
379+
});
380+
expect(response.status).toBe(502);
381+
382+
// The fixed account keeps no streak, and the active Pool account is untouched.
383+
expect(getCodexUpstreamHealth("pool-a")).toBeNull();
384+
expect(getCodexUpstreamHealth("pool-b")).toBeNull();
385+
expect(loadConfig().activeCodexAccountId).toBe("pool-b");
386+
// But the (provider, host) ledger must see the failure exactly like pool sends.
387+
const hostKey = hostConnectHealthKey("openai", "chatgpt.com");
388+
expect(getHostConnectHealth(hostKey)).toMatchObject({
389+
consecutiveFailures: 1,
390+
lastFailureCode: "ConnectionRefused",
391+
});
392+
} finally {
393+
await server.stop(true);
394+
}
395+
});
396+
355397
test("zstd-compressed search request bodies are decoded before the relay", async () => {
356398
const captured: CapturedRequest[] = [];
357399
const upstream = fakeSearchUpstream(captured);

0 commit comments

Comments
 (0)