Skip to content

Commit 58bf694

Browse files
committed
fix(proxy): local audit round (shared request budget, awaited body cancel, post-sleep abort re-checks, pinned xAI req-id)
1 parent d4e1978 commit 58bf694

6 files changed

Lines changed: 192 additions & 16 deletions

File tree

‎src/providers/xai-transport.ts‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -128,9 +128,14 @@ export function resolveProviderTransport(
128128
provider.headers,
129129
XAI_GROK_COMPATIBILITY.headers.requestId,
130130
);
131+
// Pin the request id per resolved transport (= per logical request until key rotation):
132+
// same-target 429 replays must carry the SAME x-grok-req-id as the original dispatch, and
133+
// transient retries reuse one id so the upstream can dedupe them. A rotated key resolves a
134+
// fresh transport, which gets its own id.
135+
const requestId = configuredRequestId ?? randomUUID();
131136
const baseFetch = provider.fetch ?? globalThis.fetch;
132137
const attemptFetch = ((input, init) =>
133-
baseFetch(input, withGeneratedRequestId(init, configuredRequestId, stableHeaders))) as typeof globalThis.fetch;
138+
baseFetch(input, withGeneratedRequestId(init, requestId, stableHeaders))) as typeof globalThis.fetch;
134139

135140
return {
136141
...provider,

‎src/server/responses/core.ts‎

Lines changed: 31 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -87,7 +87,6 @@ import {
8787
} from "../../codex/routing";
8888
import {
8989
applyUpstreamRecoveryInit,
90-
cancelResponseBodyBestEffort,
9190
fetchWithResetRetry,
9291
fetchWithTransientRetry,
9392
sleepWithAbort,
@@ -1656,7 +1655,8 @@ async function handleResponsesInner(
16561655
rateLimitRetries += 1;
16571656
// Release the unread 429 body before the backoff (only the header is needed for the wait).
16581657
const retryAfterHeader = upstreamResponse.headers.get("retry-after");
1659-
cancelResponseBodyBestEffort(upstreamResponse);
1658+
// AWAIT the cancellation so the resource-release guarantee is real, not best-effort.
1659+
try { await upstreamResponse.body?.cancel().catch(() => {}); } catch { /* already closed */ }
16601660
try {
16611661
await sleepWithAbort(
16621662
rateLimitRetryDelayMs(rateLimitPolicy, retryAfterHeader, Date.now()),
@@ -1666,6 +1666,12 @@ async function handleResponsesInner(
16661666
upstream.abort();
16671667
return clientCancelledResponse();
16681668
}
1669+
// Client cancellation wins over any stale timer edge: re-check before dispatching the
1670+
// replay so the wire never starts work for a request the client already abandoned.
1671+
if (options.abortSignal?.aborted || upstream.signal.aborted) {
1672+
upstream.abort();
1673+
return clientCancelledResponse();
1674+
}
16691675
try {
16701676
upstreamResponse = await fetchWithTransientRetry(
16711677
recovery => {
@@ -2355,6 +2361,12 @@ async function handleResponsesInner(
23552361
request.releaseBodyObservation?.();
23562362
}
23572363

2364+
// Same-target 429 retry budget is per REQUEST: it lives OUTSIDE the recovery loop (so a 413/401
2365+
// replay that comes back 429 cannot silently re-arm a fresh budget) and is SHARED with the
2366+
// terminal-guard continuation below, so the main loop + one continuation can never exceed
2367+
// `attempts` same-key replays in total (bounded per request).
2368+
const rateLimitPolicy = rateLimitRetryPolicyFor(route.provider);
2369+
let rateLimitRetries = 0;
23582370
if (!upstreamResponse.ok) {
23592371
// Recovery loop: multi-key 429 failover + at most ONE anthropic 413 tightened retry
23602372
// (devlog/260714_image_normalization_pipeline/030). One mutable activeAdapter serves
@@ -2364,10 +2376,6 @@ async function handleResponsesInner(
23642376
let imageTierBias = 0;
23652377
let imageRetryAttempted = false;
23662378
let oauth401ReplayAttempted = false;
2367-
// Same-target 429 retry budget lives OUTSIDE the recovery loop so a 413/401 replay that
2368-
// comes back 429 cannot silently re-arm a fresh budget (bounded to `attempts` per request).
2369-
const rateLimitPolicy = rateLimitRetryPolicyFor(route.provider);
2370-
let rateLimitRetries = 0;
23712379
/**
23722380
* Rebuild the request from the current parsed input (and any image-tier bias) and refetch
23732381
* it once, tagging the attempt with the given recovery kind. Rebuilds are deterministic
@@ -2462,7 +2470,8 @@ async function handleResponsesInner(
24622470
// wait, and under a 429 storm the sockets would otherwise accumulate for the whole
24632471
// configured interval (same pattern as the key-failover branch below).
24642472
const retryAfterHeader = upstreamResponse.headers.get("retry-after");
2465-
cancelResponseBodyBestEffort(upstreamResponse);
2473+
// AWAIT the cancellation so the resource-release guarantee is real, not best-effort.
2474+
try { await upstreamResponse.body?.cancel().catch(() => {}); } catch { /* already closed */ }
24662475
try {
24672476
await sleepWithAbort(
24682477
rateLimitRetryDelayMs(rateLimitPolicy, retryAfterHeader, Date.now()),
@@ -2473,6 +2482,13 @@ async function handleResponsesInner(
24732482
upstream.abort();
24742483
return clientCancelledResponse();
24752484
}
2485+
// Client cancellation wins over any stale timer edge: re-check before dispatching the
2486+
// replay so an adapter never starts work for a request the client already abandoned.
2487+
if (options.abortSignal?.aborted || upstream.signal.aborted) {
2488+
cleanupUpstreamAbort();
2489+
upstream.abort();
2490+
return clientCancelledResponse();
2491+
}
24762492
const result = await rebuildAndRefetch("rate-limit-429");
24772493
if ("failed" in result) return result.failed;
24782494
upstreamResponse = result;
@@ -2601,11 +2617,6 @@ async function handleResponsesInner(
26012617
const fetchTerminalGuardContinuation = async function* (nextParsed: OcxParsedRequest): AsyncGenerator<AdapterEvent> {
26022618
let imageTierBias = 0;
26032619
let response: Response | undefined;
2604-
// Same-target 429 budget is per REQUEST, not per failover iteration: a key/account rotation
2605-
// or a 413 tier retry that comes back 429 must not re-arm a fresh budget (parity with the
2606-
// main recovery loop above).
2607-
const rateLimitPolicy = rateLimitRetryPolicyFor(route.provider);
2608-
let rateLimitRetries = 0;
26092620
/**
26102621
* Build and fetch one terminal-guard continuation. `replay` marks a same-target 429
26112622
* replay so its send records the `rate-limit-429` recovery kind; the adapter rebuild is
@@ -2676,7 +2687,8 @@ async function handleResponsesInner(
26762687
rateLimitRetries += 1;
26772688
// Release the unread 429 body before the backoff (only the header is needed for the wait).
26782689
const retryAfterHeader = response.headers.get("retry-after");
2679-
try { void response.body?.cancel().catch(() => {}); } catch { /* already closed */ }
2690+
// AWAIT the cancellation so the resource-release guarantee is real, not best-effort.
2691+
try { await response.body?.cancel().catch(() => {}); } catch { /* already closed */ }
26802692
try {
26812693
await sleepWithAbort(
26822694
rateLimitRetryDelayMs(rateLimitPolicy, retryAfterHeader, Date.now()),
@@ -2693,6 +2705,12 @@ async function handleResponsesInner(
26932705
}
26942706
return;
26952707
}
2708+
// Client cancellation wins over any stale timer edge: re-check before dispatching the
2709+
// replay so the continuation never starts work for a request the client abandoned.
2710+
if (options.abortSignal?.aborted || upstream.signal.aborted) {
2711+
yield { type: "error", message: "client closed request during terminal continuation", status: 499 };
2712+
return;
2713+
}
26962714
try {
26972715
response = await fetchContinuation(true);
26982716
} catch (error) {

‎tests/images/loop.test.ts‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -261,6 +261,52 @@ describe("runWithImageBridge", () => {
261261
expect(sse).toContain("Provider error 429");
262262
});
263263

264+
test("retryOn429 budget is not re-armed after on429 rotation returns a new adapter", async () => {
265+
let sends = 0;
266+
let retrySends = 0;
267+
let rotations = 0;
268+
const retryingAdapter: ProviderAdapter = {
269+
...mockAdapter,
270+
fetchResponse: async () => {
271+
sends += 1;
272+
return new Response(JSON.stringify({ error: { message: "rate limited" } }), {
273+
status: 429,
274+
headers: { "content-type": "application/json" },
275+
});
276+
},
277+
};
278+
streamQueue = [[{ type: "text_delta" as const, text: "unused" }, { type: "done" as const }]];
279+
const response = await runWithImageBridge({
280+
parsed: makeParsed(),
281+
adapter: retryingAdapter,
282+
plan,
283+
retryOn429Policy: { enabled: true, attempts: 1, intervalMs: 50, maxIntervalMs: 60_000, respectRetryAfter: false },
284+
on429: () => {
285+
rotations += 1;
286+
// First rotation returns a new adapter that also 429s; the exhausted budget must not
287+
// re-arm for it. Second call returns null to terminate the pool.
288+
return rotations === 1
289+
? ({
290+
...mockAdapter,
291+
fetchResponse: async () => {
292+
sends += 1;
293+
return new Response("{}", { status: 429 });
294+
},
295+
} as ProviderAdapter)
296+
: null;
297+
},
298+
onAttemptSend: recovery => {
299+
if (recovery === "rate-limit-429") retrySends += 1;
300+
},
301+
});
302+
const sse = await response.text();
303+
// initial 429 + 1 same-key replay + 1 rotated send (no replay on the rotated adapter) = 3.
304+
expect(sends).toBe(3);
305+
expect(retrySends).toBe(1);
306+
expect(rotations).toBe(2);
307+
expect(sse).toContain("Provider error 429");
308+
});
309+
264310
test("forced-final clears named image tool_choice", async () => {
265311
streamQueue = [
266312
[{ type: "text_delta" as const, text: "done" }, { type: "done" as const }],

‎tests/rate-limit-retry.test.ts‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,11 @@ describe("rateLimitRetryDelayMs", () => {
8282
expect(rateLimitRetryDelayMs(policy, "Wed, 21 Oct 2026 07:28:00 GMT", now)).toBe(1);
8383
});
8484

85+
test("a far-future HTTP-date Retry-After is capped at maxIntervalMs", () => {
86+
const now = Date.parse("2026-10-21T07:27:30Z");
87+
expect(rateLimitRetryDelayMs(policy, "Wed, 21 Oct 2027 07:28:00 GMT", now)).toBe(60_000);
88+
});
89+
8590
test("Retry-After 0 retries immediately instead of falling back to the interval", () => {
8691
expect(rateLimitRetryDelayMs(policy, "0", 1_000_000)).toBe(1);
8792
});

‎tests/terminal-guard-server.test.ts‎

Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -172,4 +172,102 @@ describe("server terminal guard integration", () => {
172172
expect(sends).toBe(4);
173173
});
174174

175+
test("terminal-guard continuation shares the request-wide 429 budget with the main loop", async () => {
176+
const budgetConfig = {
177+
...config,
178+
providers: {
179+
"claude-se": {
180+
adapter: "anthropic",
181+
baseUrl: "https://example.test",
182+
apiKey: "sk-test",
183+
retryOn429: { attempts: 1, intervalMs: 120, respectRetryAfter: false },
184+
},
185+
},
186+
} as unknown as OcxConfig;
187+
let sends = 0;
188+
globalThis.fetch = (async () => {
189+
sends += 1;
190+
if (sends === 1) {
191+
// The main recovery loop consumes the only same-key replay...
192+
return new Response(JSON.stringify({ error: { message: "rate limited" } }), {
193+
status: 429,
194+
headers: { "content-type": "application/json" },
195+
});
196+
}
197+
if (sends === 2) {
198+
// ...the replay succeeds and the terminal-guard continuation starts.
199+
return anthropicSse(firstTurn);
200+
}
201+
// Continuation 429 with the request budget already spent: surfaces, never replays.
202+
return new Response(JSON.stringify({ error: { message: "rate limited" } }), {
203+
status: 429,
204+
headers: { "content-type": "application/json" },
205+
});
206+
}) as typeof fetch;
207+
208+
const response = await handleResponses(new Request("http://localhost/v1/responses", {
209+
method: "POST",
210+
headers: { "content-type": "application/json" },
211+
body: JSON.stringify({
212+
model: "se-claude-opus-4.8",
213+
input: "请检查这个问题并修复代码",
214+
stream: true,
215+
tools: [{ type: "function", name: "exec_command", description: "run a command", parameters: { type: "object" } }],
216+
}),
217+
}), budgetConfig, { model: "", provider: "" });
218+
219+
const text = await response.text();
220+
// main 429 -> same-key replay -> continuation 429 (no budget left) = exactly 3 sends.
221+
expect(sends).toBe(3);
222+
expect(text).toContain("Provider continuation error 429");
223+
});
224+
225+
test("terminal-guard continuation abort during the 429 wait yields 499 without replaying", async () => {
226+
const abortConfig = {
227+
...config,
228+
providers: {
229+
"claude-se": {
230+
adapter: "anthropic",
231+
baseUrl: "https://example.test",
232+
apiKey: "sk-test",
233+
retryOn429: { attempts: 3, intervalMs: 30_000, respectRetryAfter: false },
234+
},
235+
},
236+
} as unknown as OcxConfig;
237+
let sends = 0;
238+
globalThis.fetch = (async () => {
239+
sends += 1;
240+
if (sends === 1) {
241+
return anthropicSse(firstTurn);
242+
}
243+
return new Response(JSON.stringify({ error: { message: "rate limited" } }), {
244+
status: 429,
245+
headers: { "content-type": "application/json" },
246+
});
247+
}) as typeof fetch;
248+
249+
const abort = new AbortController();
250+
const pending = handleResponses(new Request("http://localhost/v1/responses", {
251+
method: "POST",
252+
headers: { "content-type": "application/json" },
253+
body: JSON.stringify({
254+
model: "se-claude-opus-4.8",
255+
input: "请检查这个问题并修复代码",
256+
stream: true,
257+
tools: [{ type: "function", name: "exec_command", description: "run a command", parameters: { type: "object" } }],
258+
}),
259+
}), abortConfig, { model: "", provider: "" }, { abortSignal: abort.signal });
260+
261+
// The terminal-guard continuation runs inside the SSE producer, so consume the body to
262+
// drive it, then wait until the continuation's 429 lands and its retry sleep is running.
263+
const response = await pending;
264+
const textPromise = response.text();
265+
for (let i = 0; i < 100 && sends < 2; i += 1) await Bun.sleep(10);
266+
expect(sends).toBe(2);
267+
abort.abort(new DOMException("client disconnected", "AbortError"));
268+
const text = await textPromise;
269+
expect(text).toContain("client closed request during terminal continuation");
270+
expect(sends).toBe(2);
271+
});
272+
175273
});

‎tests/xai-transport.test.ts‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -294,12 +294,16 @@ describe("xAI outbound compatibility headers", () => {
294294
]) expect(seen[0].has(name)).toBe(false);
295295
});
296296

297-
test("same resolved transport refreshes req-id but keeps conv-id stable", async () => {
297+
test("same resolved transport pins one req-id per logical request (identical replays) and keeps conv-id stable", async () => {
298298
const { seen } = await capture("oauth", 2);
299299
expect(seen).toHaveLength(2);
300300
expect(seen[0].get("x-grok-req-id")).toMatch(UUID_V4);
301301
expect(seen[1].get("x-grok-req-id")).toMatch(UUID_V4);
302-
expect(seen[1].get("x-grok-req-id")).not.toBe(seen[0].get("x-grok-req-id"));
302+
// A same-target 429 replay must be byte-identical, including x-grok-req-id; a new resolve
303+
// (e.g. after key rotation) produces a fresh transport and therefore a fresh id.
304+
expect(seen[1].get("x-grok-req-id")).toBe(seen[0].get("x-grok-req-id"));
305+
const freshTransport = await capture("oauth", 1);
306+
expect(freshTransport.seen[0].get("x-grok-req-id")).not.toBe(seen[0].get("x-grok-req-id"));
303307
expect(seen[0].get("x-grok-conv-id")).toBe(deriveXaiConvId("codex-session-abc"));
304308
expect(seen[1].get("x-grok-conv-id")).toBe(seen[0].get("x-grok-conv-id"));
305309
expect(seen[1].get("x-grok-session-id")).toBe(seen[0].get("x-grok-session-id"));

0 commit comments

Comments
 (0)