Skip to content

Commit 3d7b324

Browse files
committed
merge: integrate dev into the Chat inbound developer-position lane
2 parents f56cd55 + 556b670 commit 3d7b324

23 files changed

Lines changed: 1432 additions & 54 deletions

‎docs-site/src/content/docs/reference/configuration/server.md‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -64,10 +64,14 @@ arrives, the proxy cannot tell whether the model already processed the request,
6464
to send it again and answers HTTP 429 with `upstream_reset_replay_refused`. The status is
6565
deliberate: a 5xx here is an instruction to most clients, including Codex, to send the whole
6666
turn again, which is the duplicate the refusal exists to prevent. No `Retry-After` is
67-
attached, and the proxy performs no key rotation, account failover or same-target replay on
68-
it, nor does it record the refusal as rate-limit or quota evidence against the credential it
69-
was holding. Tool-call side requests such as vision and web search are replayed normally, because
70-
repeating them cannot duplicate a turn.
67+
attached, and the response carries `x-should-retry: false`, which the official OpenAI and
68+
Anthropic SDKs read before their own status rules — without it those clients retry a 429 on
69+
their own schedule and resend the turn anyway. The answer is identical on `/v1/responses` and
70+
on `/v1/chat/completions`, whether the request is forwarded natively or translated. The proxy
71+
performs no key rotation, account failover or same-target replay on it, nor does it record the
72+
refusal as rate-limit or quota evidence against the credential it was holding. Tool-call side
73+
requests such as vision and web search are replayed normally, because repeating them cannot
74+
duplicate a turn.
7175

7276
A native Responses provider can opt into replacing that send with
7377
[`retryOnReset`](providers.md#provider-entries-ocxproviderconfig). The same grant covers the

‎scripts/test-layout/layout.json‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1228,6 +1228,7 @@
12281228
"remote-workspace-protocol.test.ts": "clients",
12291229
"remote-workspace-rpc-framing.test.ts": "clients",
12301230
"remove-tree-helper.test.ts": "lib",
1231+
"replay-refusal-parity.test.ts": "server",
12311232
"repo-hygiene.test.ts": "ci-workflows",
12321233
"request-decompress.test.ts": "usage",
12331234
"request-evidence.test.ts": "usage",
@@ -1297,6 +1298,7 @@
12971298
"responses-show-thinking-summary.test.ts": "responses",
12981299
"responses-snapshot-repair-server.test.ts": "responses",
12991300
"responses-snapshot-repair.test.ts": "responses",
1301+
"responses-sparse-terminal-tool-scope.test.ts": "responses",
13001302
"responses-spill-shutdown-clock.test.ts": "responses",
13011303
"responses-state-write-amplification.test.ts": "responses",
13021304
"responses-state.test.ts": "responses",
@@ -1557,6 +1559,7 @@
15571559
"ws-upstream.test.ts": "responses",
15581560
"ws-upstream-socks5.test.ts": "responses",
15591561
"xai-client.test.ts": "images",
1562+
"xai-empty-catalog-tool-choice.test.ts": "providers/xai",
15601563
"xai-oauth-retry.test.ts": "providers/xai",
15611564
"xai-refresh-lock.test.ts": "providers/xai",
15621565
"xai-responses-adjacency.test.ts": "providers/xai",
@@ -1627,6 +1630,7 @@
16271630
"responses-reset-replay.test.ts": "responses",
16281631
"ambiguous-resend-gate.test.ts": "lib",
16291632
"docs-developer-role-policy.test.ts": "ci-workflows",
1633+
"ambiguous-resend-composition.test.ts": "lib",
16301634
"chat-inbound-developer-position.test.ts": "responses"
16311635
},
16321636
"migrated": [

‎src/adapters/xai-web-search.ts‎

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import type { OcxProviderConfig } from "../types";
2+
import { debugProviderDiagnostic } from "../lib/debug";
23
import { isXaiResponsesDestination } from "../providers/xai-transport";
34

45
const CODEX_WEB_SEARCH_TOOL = "web_search";
@@ -192,7 +193,21 @@ export function normalizeXaiResponsesWebSearch(
192193
if (inputChanged) next = { ...next, input };
193194
}
194195

195-
return normalizeToolChoice(next);
196+
const normalized = normalizeToolChoice(next);
197+
const choice = normalized.tool_choice;
198+
if ((choice === "auto" || choice === "none") && !hasAnyDeclaredTool(normalized)) {
199+
debugProviderDiagnostic("xai", "tool-choice-omitted", { choice });
200+
const { tool_choice: _toolChoice, ...rest } = normalized;
201+
// `auto` selects from the catalog, so a catalog with nothing in it makes it meaningless and
202+
// the omission says nothing the request did not already say. `none` is the opposite: it is a
203+
// prohibition, and on a request whose catalog this normalizer just emptied it is the only
204+
// place the turn's client-call boundary is written down. Downstream repair reads that
205+
// boundary off the final outbound body, so omitting the word alone would hand back a call the
206+
// caller ruled out. Restate it as the explicit empty catalog, which carries the same deny-all
207+
// and which this destination already receives whenever a caller sends one itself.
208+
return choice === "none" && !Array.isArray(rest.tools) ? { ...rest, tools: [] } : rest;
209+
}
210+
return normalized;
196211
}
197212

198213
function isLiveWebSearchTool(tool: unknown): boolean {

‎src/bridge/errors.ts‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,10 @@
11
import {
2+
applyReplayRefusalClientHeaders,
23
isNonReplayableUpstreamCode,
34
isReplayRefusalCode,
4-
markReplayRefusalResponse,
55
markResponseNonReplayable,
66
REPLAY_REFUSED_STATUS,
7+
retainReplayRefusal,
78
} from "../lib/upstream-retry";
89
import {
910
adapterFailureFromMessage,
@@ -45,6 +46,11 @@ export function formatErrorResponse(
4546
&& retryAfter.length <= 128) {
4647
headers.set("Retry-After", retryAfter);
4748
}
49+
// The refusal's client policy, restated here for the same reason its status is: this
50+
// formatter is the last thing several adapter and combo paths touch before the client,
51+
// and no wait of its own does not stop a client that retries every 429 by default.
52+
const refusal = isReplayRefusalCode(error.code) && replayBlocked;
53+
if (refusal) applyReplayRefusalClientHeaders(headers);
4854
const response = new Response(JSON.stringify({ error }), {
4955
status: finalStatus,
5056
headers,
@@ -53,6 +59,6 @@ export function formatErrorResponse(
5359
// Re-wrapping is where the refusal loses its provenance: combo failure consumption parses
5460
// the JSON and builds a new Response, and the code alone does not tell a later quota
5561
// recorder that no upstream produced this status. Carry the narrower marker across too.
56-
if (replayBlocked && isReplayRefusalCode(error.code)) markReplayRefusalResponse(response);
62+
if (refusal) retainReplayRefusal(response);
5763
return response;
5864
}

‎src/lib/request-execution-budget.ts‎

Lines changed: 51 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -176,6 +176,13 @@ export interface RequestExecutionBudget extends TransientSendBudget {
176176
* twice. It is NOT a send budget: an authorised replacement still has to fit inside
177177
* `remainingBaseSends` like every other send.
178178
*
179+
* `limit` is the ceiling the ASKING leg is authorised to present, and the request keeps the
180+
* smallest one any leg has presented. A leg reads it from `route.provider`, which credential
181+
* rotation, OAuth refresh, transport resolution and a combo target all reassign mid-request,
182+
* so a per-call ceiling meant the number of duplicate inferences a request could make
183+
* depended on which row happened to ask last: a row granting one, then a row granting two,
184+
* bought a second replacement of a turn that may already have run.
185+
*
179186
* Optional so a hand-written stub that satisfies the shape test keeps typechecking; a caller
180187
* that cannot reach it has no operator override, which is the fail-closed answer.
181188
*/
@@ -208,16 +215,48 @@ interface SharedSendLedger {
208215
spent: number;
209216
pendingExternalSends: number;
210217
/**
211-
* Replacements this logical request has already spent on ambiguous failures. Beside `spent`
218+
* Spend one of this logical request's replacements for an ambiguous failure. Beside `spent`
212219
* for the same reason `pendingExternalSends` is: a derived scope that shared one without the
213220
* other would hand the request a second grant.
221+
*
222+
* A function rather than the raw count, because the count is not the whole state. The
223+
* ceiling belongs to the request too, and a bridged scope has no counter of its own to keep
224+
* it in -- it has to ask whoever holds the request's grant.
214225
*/
215-
ambiguousResendsClaimed: number;
226+
claimAmbiguousResend(limit: number): boolean;
216227
readonly observer?: RequestSendObserver;
217228
}
218229

219230
const sharedSendLedgers = new WeakMap<RequestExecutionBudget, SharedSendLedger>();
220231

232+
/**
233+
* One logical request's replacement grant: how many it has spent, and the ceiling it is held
234+
* to.
235+
*
236+
* The ceiling is the SMALLEST any leg has presented rather than whatever the current leg
237+
* presents. Each leg reads its number from the provider row it is running against, and that
238+
* row changes inside one request -- credential rotation, OAuth refresh, transport resolution
239+
* and each combo target reassign it. Taking the asking leg's number let a request that had
240+
* already spent the one replacement a strict row granted buy another as soon as a more
241+
* permissive row asked, which is a second duplicate inference of one turn.
242+
*/
243+
function createAmbiguousResendGrant(): (limit: number) => boolean {
244+
let claimed = 0;
245+
let ceiling: number | undefined;
246+
return (limit: number): boolean => {
247+
const presented = Number.isFinite(limit) ? Math.trunc(limit) : 0;
248+
// A zero or nonsense ceiling refuses on its own and leaves the request's alone. It is a
249+
// caller that cannot state a grant, not an operator narrowing this request: a leg with no
250+
// policy is refused before it ever claims, so binding the request to a malformed number
251+
// would only let such a caller cancel a grant an opted-in row really made.
252+
if (presented <= 0) return false;
253+
ceiling = ceiling === undefined ? presented : Math.min(ceiling, presented);
254+
if (claimed >= ceiling) return false;
255+
claimed += 1;
256+
return true;
257+
};
258+
}
259+
221260
function createRequestExecutionBudgetWithLedger(
222261
policy: RequestExecutionBudgetPolicy,
223262
logicalRequestId: string | undefined,
@@ -261,10 +300,7 @@ function createRequestExecutionBudgetWithLedger(
261300
return Math.max(0, Math.min(capped, policy.baseSendAllowance - counter.spent));
262301
},
263302
claimAmbiguousResend(limit: number): boolean {
264-
const ceiling = Number.isFinite(limit) ? Math.trunc(limit) : 0;
265-
if (counter.ambiguousResendsClaimed >= ceiling) return false;
266-
counter.ambiguousResendsClaimed += 1;
267-
return true;
303+
return counter.claimAmbiguousResend(limit);
268304
},
269305
reserveDispatch(intent: DispatchIntent): DispatchDecision {
270306
if (intent.replaySafe === false) return { allowed: false, reason: "not-replay-safe" };
@@ -363,7 +399,7 @@ export function createRequestExecutionBudget(
363399
return createRequestExecutionBudgetWithLedger(policy, logicalRequestId, {
364400
spent: 0,
365401
pendingExternalSends: 0,
366-
ambiguousResendsClaimed: 0,
402+
claimAmbiguousResend: createAmbiguousResendGrant(),
367403
...(observer ? { observer } : {}),
368404
});
369405
}
@@ -398,14 +434,19 @@ function ledgerFor(parent: RequestExecutionBudget): SharedSendLedger {
398434
const existing = sharedSendLedgers.get(parent);
399435
if (existing) return existing;
400436
let pendingExternalSends = 0;
401-
let ambiguousResendsClaimed = 0;
402437
return {
403438
get spent(): number { return parent.used; },
404439
set spent(next: number) { parent.used = next; },
405440
get pendingExternalSends(): number { return pendingExternalSends; },
406441
set pendingExternalSends(next: number) { pendingExternalSends = next; },
407-
get ambiguousResendsClaimed(): number { return ambiguousResendsClaimed; },
408-
set ambiguousResendsClaimed(next: number) { ambiguousResendsClaimed = next; },
442+
// Asked of the parent rather than counted here. A local counter is a SECOND grant: two
443+
// scopes derived from one bridged parent, or one scope beside the parent it was derived
444+
// from, each replaced an unknown-state send once. Pending bookings and the durable-spend
445+
// observer genuinely cannot cross this boundary because they are private to the factory,
446+
// but the grant can -- `claimAmbiguousResend` is public on the parent. A parent that does
447+
// not implement it grants nothing, which is the fail-closed answer for a send whose
448+
// upstream state is unknown.
449+
claimAmbiguousResend: (limit: number): boolean => parent.claimAmbiguousResend?.(limit) === true,
409450
};
410451
}
411452

‎src/lib/upstream-retry.ts‎

Lines changed: 54 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -98,6 +98,55 @@ export function isReplayRefusalCode(code: unknown): boolean {
9898
/** Client-facing status for {@link UPSTREAM_RESET_REPLAY_REFUSED_CODE}. */
9999
export const REPLAY_REFUSED_STATUS = 429;
100100

101+
/**
102+
* The header every surface attaches to a replay refusal, and its only accepted value.
103+
*
104+
* Dropping `Retry-After` is necessary and not sufficient. The Stainless-generated clients --
105+
* `openai` and `anthropic` for both Python and Node, which is what most callers of this proxy
106+
* actually are -- decide from a status table (408, 409, 429 and every 5xx) and compute their own
107+
* backoff when no wait is named, so a 429 with no header is still resent. `x-should-retry` is
108+
* the one signal each of them reads BEFORE that table, and `"false"` is the exact string they
109+
* compare against.
110+
*/
111+
export const REPLAY_REFUSAL_NO_RETRY_HEADER = "x-should-retry";
112+
export const REPLAY_REFUSAL_NO_RETRY_VALUE = "false";
113+
114+
/** Spreadable form for the surfaces that build their headers as an object literal. */
115+
export const REPLAY_REFUSAL_CLIENT_HEADERS: Readonly<Record<string, string>> = Object.freeze({
116+
[REPLAY_REFUSAL_NO_RETRY_HEADER]: REPLAY_REFUSAL_NO_RETRY_VALUE,
117+
});
118+
119+
/**
120+
* Apply the one client-facing retry policy a refusal carries: no wait, and no automatic resend.
121+
*
122+
* Kept as a single function rather than two rules each surface repeats, because the two halves
123+
* are only correct together -- a surface that removed the wait but not the suppression still
124+
* hands a retrying client a turn it may already have run.
125+
*/
126+
export function applyReplayRefusalClientHeaders(headers: Headers): void {
127+
headers.delete("retry-after");
128+
headers.set(REPLAY_REFUSAL_NO_RETRY_HEADER, REPLAY_REFUSAL_NO_RETRY_VALUE);
129+
}
130+
131+
/**
132+
* Mark a response that re-wraps a refusal as the same refusal.
133+
*
134+
* The verdict has to be a property of the result the surfaces pass around, because the thing it
135+
* would otherwise be read from is the status, and 429 is exactly what a refusal and a real rate
136+
* limit have in common. Every formatter between the helper that made the refusal and the client
137+
* builds a new Response, so each of them restates the verdict rather than dropping it.
138+
*/
139+
export function retainReplayRefusal<T extends Response>(response: T): T {
140+
markResponseNonReplayable(response);
141+
markReplayRefusalResponse(response);
142+
return response;
143+
}
144+
145+
/** Carry the verdict from a response onto the one that replaces it. */
146+
export function carryReplayRefusal<T extends Response>(source: Response, rewrapped: T): T {
147+
return isReplayRefusalResponse(source) ? retainReplayRefusal(rewrapped) : rewrapped;
148+
}
149+
101150
// 1 initial + 2 retries: the pool may hold more than one stale socket.
102151
const RESET_RETRY_MAX_ATTEMPTS = 3;
103152
const RESET_RETRY_BASE_DELAY_MS = 150;
@@ -521,10 +570,11 @@ export function replayRefusalResponse(): Response {
521570
type: "upstream_error",
522571
code: UPSTREAM_RESET_REPLAY_REFUSED_CODE,
523572
message: "The upstream connection closed before a response was received. The request may already have been processed; automatic replay was stopped.",
524-
} }), { status: REPLAY_REFUSED_STATUS, headers: { "content-type": "application/json" } });
525-
markResponseNonReplayable(response);
526-
markReplayRefusalResponse(response);
527-
return response;
573+
} }), {
574+
status: REPLAY_REFUSED_STATUS,
575+
headers: { "content-type": "application/json", ...REPLAY_REFUSAL_CLIENT_HEADERS },
576+
});
577+
return retainReplayRefusal(response);
528578
}
529579

530580
/**

‎src/server/chat-completions.ts‎

Lines changed: 40 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,15 @@ import {
2222
import { classifyError, cyberPolicyErrorType, CYBER_POLICY_ERROR_CODE, isCyberPolicyCode } from "../lib/errors";
2323
import { redactSecretString } from "../lib/redact";
2424
import { resolveClientRetryAfter } from "../lib/retry-after";
25+
import {
26+
applyReplayRefusalClientHeaders,
27+
isReplayRefusalCode,
28+
isReplayRefusalResponse,
29+
REPLAY_REFUSAL_CLIENT_HEADERS,
30+
REPLAY_REFUSED_STATUS,
31+
retainReplayRefusal,
32+
UPSTREAM_RESET_REPLAY_REFUSED_CODE,
33+
} from "../lib/upstream-retry";
2534
import { estimateTokens } from "../lib/token-estimate";
2635
import { captureRouteStaticPolicy, NoEligiblePolicyCandidateError, UnknownRoutingPolicyError, routeModel } from "../router";
2736
import { evidenceFromBody } from "../routing/request-evidence";
@@ -419,18 +428,30 @@ async function handleChatCompletionsWithBudget(
419428
: "invalid_request_error"),
420429
message,
421430
);
431+
// The same verdict the native Chat surface reads, from the same two places: the response
432+
// this wrapper still holds, and the code a body kept through an intermediate formatter.
433+
// Not the status -- a refusal and a real rate limit are both 429, which is the whole
434+
// reason this surface used to report one as the other.
435+
const replayRefusal = isReplayRefusalResponse(upstream) || isReplayRefusalCode(upstreamCode);
422436
if (isCyberPolicyCode(upstreamCode) || classified.code === CYBER_POLICY_ERROR_CODE) {
423437
classified.code = CYBER_POLICY_ERROR_CODE;
424438
classified.type = cyberPolicyErrorType(upstreamType);
439+
} else if (replayRefusal) {
440+
// 429 classifies as a rate limit, which already carries a code, so the empty-code branch
441+
// below could never restore this one -- the translated client was told the provider
442+
// throttled the turn, and handed a two-second wait to send it again.
443+
classified.code = UPSTREAM_RESET_REPLAY_REFUSED_CODE;
425444
} else if (upstreamCode === "model_not_found") {
426445
// Structured model_not_found must win over classifyError's generic remaps.
427446
classified.code = "model_not_found";
428447
classified.type = "invalid_request_error";
429448
} else if (upstreamCode !== undefined && upstreamCode !== null && classified.code == null) {
430449
classified.code = upstreamCode;
431450
}
432-
const status = isCyberPolicyCode(classified.code) ? 400 : upstream.status;
433-
const retryAfter = isCyberPolicyCode(classified.code)
451+
const status = isCyberPolicyCode(classified.code) ? 400
452+
: replayRefusal ? REPLAY_REFUSED_STATUS
453+
: upstream.status;
454+
const retryAfter = isCyberPolicyCode(classified.code) || replayRefusal
434455
? undefined
435456
: resolveClientRetryAfter({
436457
status: upstream.status,
@@ -449,9 +470,12 @@ async function handleChatCompletionsWithBudget(
449470
headers: {
450471
"Content-Type": "application/json",
451472
...(retryAfter ? { "Retry-After": retryAfter } : {}),
473+
...(replayRefusal ? REPLAY_REFUSAL_CLIENT_HEADERS : {}),
452474
},
453475
});
476+
if (replayRefusal) retainReplayRefusal(rewritten);
454477
return logIds
478+
// Deferred logging re-wraps this response and carries the verdict with it.
455479
? responseWithDeferredRequestLog(rewritten, logIds.requestId, logIds.start, logCtx)
456480
: rewritten;
457481
}
@@ -518,11 +542,25 @@ async function handleChatCompletionsWithBudget(
518542
} else if (isCyberPolicyCode(error?.code) || classified.code === CYBER_POLICY_ERROR_CODE) {
519543
classified.code = CYBER_POLICY_ERROR_CODE;
520544
classified.type = cyberPolicyErrorType(error?.type);
545+
} else if (isReplayRefusalCode(error?.code)) {
546+
// The refusal can also arrive as a failed Responses envelope rather than a non-2xx.
547+
// Reporting that as the 502 below would invite the four resends the refusal prevents.
548+
classified.code = UPSTREAM_RESET_REPLAY_REFUSED_CODE;
521549
} else if (error?.code === "model_not_found") {
522550
// Same deliberate preserve as the non-OK path: structured code beats generic classify.
523551
classified.code = "model_not_found";
524552
classified.type = "invalid_request_error";
525553
}
554+
if (isReplayRefusalCode(classified.code)) {
555+
const refusal = chatCompletionsErrorResponse(
556+
REPLAY_REFUSED_STATUS, message, classified.type, classified.code,
557+
);
558+
const headers = new Headers(refusal.headers);
559+
applyReplayRefusalClientHeaders(headers);
560+
return finishJson(retainReplayRefusal(
561+
new Response(refusal.body, { status: refusal.status, headers }),
562+
));
563+
}
526564
return finishJson(chatCompletionsErrorResponse(
527565
classified.code === "translation_buffer_limit"
528566
? 502

0 commit comments

Comments
 (0)