Skip to content

Commit eb9890e

Browse files
committed
fix(proxy): maintainer review round (watchdog-fed backoffs, one cached request per target, deadline/stall regression tests)
1 parent 89535fb commit eb9890e

8 files changed

Lines changed: 354 additions & 49 deletions

File tree

‎src/images/loop.ts‎

Lines changed: 20 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ import type { AttemptRecoveryKind } from "../usage/log";
2020
import { bridgeToResponsesSSE } from "../bridge";
2121
import { clearableDeadline, idleDeadline } from "../lib/abort";
2222
import { readBoundedResponseBody } from "../lib/bounded-body";
23-
import { fetchWithResetRetry, sleepWithAbort } from "../lib/upstream-retry";
23+
import { fetchWithResetRetry, sleepWithHeartbeats } from "../lib/upstream-retry";
2424
import { rateLimitRetryDelayMs } from "../providers/key-failover";
2525
import {
2626
isTranslatorBudgetExceededError,
@@ -442,14 +442,25 @@ export async function runWithImageBridge(deps: ImageBridgeDeps): Promise<Respons
442442
/**
443443
* Build and fetch one image-bridge iteration on the given adapter, under the iteration
444444
* header deadline. The caller owns same-target 429 replays and key rotation around it.
445+
* The outbound request is cached per adapter so a same-target replay reuses the EXACT
446+
* URL, serialized body, and headers (builder runs once per target sequence).
445447
*/
448+
let cachedRequest: AdapterRequest | undefined;
449+
let cachedAdapter: ProviderAdapter | undefined;
446450
const fetchOnce = async (requestAdapter: ProviderAdapter, recovery?: AttemptRecoveryKind): Promise<IterationResponse> => {
447-
const request = await requestAdapter.buildRequest(iterParsed, {
448-
headers: deps.forwardHeaders ? new Headers(deps.forwardHeaders) : new Headers(),
449-
abortSignal: headerDeadline.signal,
450-
translatorBudget,
451-
});
452-
try { deps.onRequestBuilt?.(request); } catch { /* diagnostics are best-effort */ }
451+
let request: AdapterRequest;
452+
if (cachedRequest !== undefined && cachedAdapter === requestAdapter) {
453+
request = cachedRequest;
454+
} else {
455+
request = await requestAdapter.buildRequest(iterParsed, {
456+
headers: deps.forwardHeaders ? new Headers(deps.forwardHeaders) : new Headers(),
457+
abortSignal: headerDeadline.signal,
458+
translatorBudget,
459+
});
460+
try { deps.onRequestBuilt?.(request); } catch { /* diagnostics are best-effort */ }
461+
cachedRequest = request;
462+
cachedAdapter = requestAdapter;
463+
}
453464
deps.onAttemptSend?.(recovery);
454465
let response: Response;
455466
try {
@@ -496,9 +507,10 @@ export async function runWithImageBridge(deps: ImageBridgeDeps): Promise<Respons
496507
// before sleeping so a stale expiry can never race the client-cancel path.
497508
headerDeadline.clear();
498509
try {
499-
await sleepWithAbort(
510+
yield* sleepWithHeartbeats(
500511
rateLimitRetryDelayMs(rateLimitRetryPolicy, retryAfterHeader, Date.now()),
501512
signal,
513+
Math.min(10_000, Math.max(250, stallTimeoutMs / 2)),
502514
);
503515
} catch {
504516
throw new LoopError(499, "client closed request during image-bridge");

‎src/lib/upstream-retry.ts‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,27 @@ export async function sleepWithAbort(ms: number, signal?: AbortSignal): Promise<
7171
});
7272
}
7373

74+
/**
75+
* Abort-aware sleep that yields an adapter `heartbeat` at least every `heartbeatIntervalMs`.
76+
* The Responses bridge treats a returned iterator event as upstream liveness and aborts turns
77+
* that stay silent past the stall budget (default 300s), while a retryOn429 wait may legally
78+
* reach 600s — so deliberate waits must keep the watchdog fed or a long backoff is killed
79+
* mid-turn. The final chunk always yields once, which doubles as the post-wait liveness beat.
80+
*/
81+
export async function* sleepWithHeartbeats(
82+
ms: number,
83+
signal?: AbortSignal,
84+
heartbeatIntervalMs = 10_000,
85+
): AsyncGenerator<{ type: "heartbeat" }> {
86+
let remaining = ms;
87+
while (remaining > 0) {
88+
const chunk = Math.min(remaining, heartbeatIntervalMs);
89+
await sleepWithAbort(chunk, signal);
90+
remaining -= chunk;
91+
yield { type: "heartbeat" };
92+
}
93+
}
94+
7495
export function isConnectionResetError(err: unknown): boolean {
7596
if (!(err instanceof Error)) return false;
7697
// Aborts and timeouts are caller decisions / honest failures — never retryable.

‎src/server/responses/core.ts‎

Lines changed: 66 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,7 @@ import {
8989
applyUpstreamRecoveryInit,
9090
fetchWithResetRetry,
9191
fetchWithTransientRetry,
92+
sleepWithHeartbeats,
9293
sleepWithAbort,
9394
} from "../../lib/upstream-retry";
9495
import { ForwardAdmissionCredentialError, validateForwardAdmissionCredential } from "../auth-cors";
@@ -101,6 +102,7 @@ import { isUsageDebugEnabled } from "../../usage/debug";
101102
import { readJsonRequestBody, DecompressedBodyTooLargeError, UnsupportedContentEncodingError } from "../request-decompress";
102103
import { resolveAdapter, resolveWireProtocolOverride } from "../adapter-resolve";
103104
import type { InboundWire } from "../../providers/registry";
105+
import type { AdapterRequest } from "../../adapters/base";
104106
import {
105107
hasKeyPoolFailover,
106108
rateLimitRetryDelayMs,
@@ -2321,19 +2323,33 @@ async function handleResponsesInner(
23212323
const upstream = new AbortController();
23222324
const cleanupUpstreamAbort = linkAbortSignal(upstream, options.abortSignal);
23232325
const connectMs = config.connectTimeoutMs ?? 200_000;
2326+
// Bridge stall budget (seconds of silence before upstream_stall_timeout); the retry backoff
2327+
// heartbeat interval is derived from it so the watchdog is always fed during deliberate waits.
2328+
const stallTimeoutMs = typeof config.stallTimeoutSec === "number" && Number.isFinite(config.stallTimeoutSec) && config.stallTimeoutSec > 0
2329+
? Math.floor(config.stallTimeoutSec * 1000)
2330+
: 300_000;
23242331
let activeAdapter = adapter;
23252332

2326-
const request = await activeAdapter.buildRequest(parsed, { headers: selectedForwardHeaders, translatorBudget });
2327-
recordAdapterReasoning(logCtx, request);
2328-
const inputTokenEstimate = typeof request.usageLog?.inputTokens === "number"
2329-
? request.usageLog.inputTokens
2333+
// One immutable, body-safe outbound request per same-target sequence (URL, serialized body,
2334+
// auth headers, generated compat headers). Same-target 429 replays reuse it verbatim; the
2335+
// builder runs again only after a key/account/adapter rotation, an oauth refresh, or an
2336+
// image-tier bias change (transportToken bump). `body` is always a serialized string, so
2337+
// reuse is safe, and releaseBodyObservation is idempotent per build.
2338+
const initialRequest = await activeAdapter.buildRequest(parsed, { headers: selectedForwardHeaders, translatorBudget });
2339+
recordAdapterReasoning(logCtx, initialRequest);
2340+
const inputTokenEstimate = typeof initialRequest.usageLog?.inputTokens === "number"
2341+
? initialRequest.usageLog.inputTokens
23302342
: undefined;
23312343
if (inputTokenEstimate !== undefined) logCtx.usageLogInputTokens = inputTokenEstimate;
2344+
let sameTargetRequest: AdapterRequest | undefined = initialRequest;
2345+
let sameTargetParsed: OcxParsedRequest | undefined = parsed;
2346+
let sameTargetToken = 0;
2347+
let transportToken = 0;
23322348
let upstreamResponse: Response;
23332349
try {
23342350
if (activeAdapter.fetchResponse) {
23352351
noteAttemptSend(logCtx.activeAttempt, inputTokenEstimate);
2336-
upstreamResponse = await activeAdapter.fetchResponse(request, {
2352+
upstreamResponse = await activeAdapter.fetchResponse(initialRequest, {
23372353
abortSignal: upstream.signal,
23382354
timeoutMs: connectMs,
23392355
stream: parsed.stream,
@@ -2342,13 +2358,13 @@ async function handleResponsesInner(
23422358
upstreamResponse = await fetchWithResetRetry(
23432359
recovery => {
23442360
noteAttemptSend(logCtx.activeAttempt, inputTokenEstimate, recovery);
2345-
return fetchWithHeaderTimeout(request.url, applyUpstreamRecoveryInit({
2346-
method: request.method,
2347-
headers: request.headers,
2348-
body: request.body,
2361+
return fetchWithHeaderTimeout(initialRequest.url, applyUpstreamRecoveryInit({
2362+
method: initialRequest.method,
2363+
headers: initialRequest.headers,
2364+
body: initialRequest.body,
23492365
}, recovery), upstream.signal, connectMs, parsed.stream, providerFetch(route.provider));
23502366
},
2351-
{ abortSignal: upstream.signal, label: safeHostLabel(request.url) },
2367+
{ abortSignal: upstream.signal, label: safeHostLabel(initialRequest.url) },
23522368
);
23532369
}
23542370
} catch (err) {
@@ -2358,7 +2374,7 @@ async function handleResponsesInner(
23582374
const msg = describeUpstreamConnectFailure(err, connectMs);
23592375
return formatErrorResponse(502, "upstream_error", msg);
23602376
} finally {
2361-
request.releaseBodyObservation?.();
2377+
initialRequest.releaseBodyObservation?.();
23622378
}
23632379

23642380
// Same-target 429 retry budget is per REQUEST: it lives OUTSIDE the recovery loop (so a 413/401
@@ -2384,12 +2400,21 @@ async function handleResponsesInner(
23842400
const rebuildAndRefetch = async (
23852401
recovery: AttemptRecoveryKind,
23862402
): Promise<Response | { failed: Response }> => {
2387-
const retryRequest = await activeAdapter.buildRequest(parsed, {
2388-
headers: selectedForwardHeaders,
2389-
translatorBudget,
2390-
...(imageTierBias > 0 ? { imageTierBias } : {}),
2391-
});
2392-
recordAdapterReasoning(logCtx, retryRequest);
2403+
let retryRequest: AdapterRequest;
2404+
if (sameTargetRequest !== undefined && sameTargetParsed === parsed && sameTargetToken === transportToken) {
2405+
// Same target (key/adapter/parsed/tier unchanged): replay the exact cached request.
2406+
retryRequest = sameTargetRequest;
2407+
} else {
2408+
retryRequest = await activeAdapter.buildRequest(parsed, {
2409+
headers: selectedForwardHeaders,
2410+
translatorBudget,
2411+
...(imageTierBias > 0 ? { imageTierBias } : {}),
2412+
});
2413+
recordAdapterReasoning(logCtx, retryRequest);
2414+
sameTargetRequest = retryRequest;
2415+
sameTargetParsed = parsed;
2416+
sameTargetToken = transportToken;
2417+
}
23932418
const retryEstimate = typeof retryRequest.usageLog?.inputTokens === "number"
23942419
? retryRequest.usageLog.inputTokens
23952420
: undefined;
@@ -2444,6 +2469,7 @@ async function handleResponsesInner(
24442469
route.providerName === "github-copilot" ? getOAuthCredentialApiBaseUrl(route.providerName) : undefined,
24452470
);
24462471
route.provider = refreshedProvider;
2472+
transportToken += 1;
24472473
activeAdapter = resolveAdapter(
24482474
resolveWireProtocolOverride(route.providerName, route.modelId, refreshedProvider, inboundWire),
24492475
config.cacheRetention,
@@ -2509,6 +2535,7 @@ async function handleResponsesInner(
25092535
// until runtime cleanup (one per rotated key under a rate-limit storm).
25102536
try { void upstreamResponse.body?.cancel().catch(() => {}); } catch { /* already consumed/closed */ }
25112537
route.provider = rotated;
2538+
transportToken += 1;
25122539
activeAdapter = resolveAdapter(
25132540
resolveWireProtocolOverride(route.providerName, route.modelId, route.provider, inboundWire),
25142541
config.cacheRetention,
@@ -2539,6 +2566,7 @@ async function handleResponsesInner(
25392566
anthropicPoolAccountId = nextAccountId;
25402567
anthropicPoolFailovers += 1;
25412568
route.provider = { ...route.provider, apiKey: accessToken };
2569+
transportToken += 1;
25422570
promoteAnthropicActiveAccount(nextAccountId);
25432571
logCtx.provider = formatAnthropicProviderForLog("anthropic", nextAccountId, config);
25442572
activeAdapter = resolveAdapter(
@@ -2563,6 +2591,7 @@ async function handleResponsesInner(
25632591
})) {
25642592
imageRetryAttempted = true;
25652593
imageTierBias = 1;
2594+
transportToken += 1;
25662595
try { void upstreamResponse.body?.cancel().catch(() => {}); } catch { /* already consumed/closed */ }
25672596
const result = await rebuildAndRefetch("image-413");
25682597
if ("failed" in result) return result.failed;
@@ -2623,12 +2652,21 @@ async function handleResponsesInner(
26232652
* deterministic for the same parsed request (tests assert byte-identical replays).
26242653
*/
26252654
const fetchContinuation = async (replay = false): Promise<Response> => {
2626-
const continuationRequest = await activeAdapter.buildRequest(nextParsed, {
2627-
headers: selectedForwardHeaders,
2628-
translatorBudget,
2629-
...(imageTierBias > 0 ? { imageTierBias } : {}),
2630-
});
2631-
recordAdapterReasoning(logCtx, continuationRequest);
2655+
let continuationRequest: AdapterRequest;
2656+
if (sameTargetRequest !== undefined && sameTargetParsed === nextParsed && sameTargetToken === transportToken) {
2657+
// Same target (key/adapter/parsed/tier unchanged): replay the exact cached request.
2658+
continuationRequest = sameTargetRequest;
2659+
} else {
2660+
continuationRequest = await activeAdapter.buildRequest(nextParsed, {
2661+
headers: selectedForwardHeaders,
2662+
translatorBudget,
2663+
...(imageTierBias > 0 ? { imageTierBias } : {}),
2664+
});
2665+
recordAdapterReasoning(logCtx, continuationRequest);
2666+
sameTargetRequest = continuationRequest;
2667+
sameTargetParsed = nextParsed;
2668+
sameTargetToken = transportToken;
2669+
}
26322670
const continuationEstimate = typeof continuationRequest.usageLog?.inputTokens === "number"
26332671
? continuationRequest.usageLog.inputTokens
26342672
: undefined;
@@ -2690,12 +2728,13 @@ async function handleResponsesInner(
26902728
// AWAIT the cancellation so the resource-release guarantee is real, not best-effort.
26912729
try { await response.body?.cancel().catch(() => {}); } catch { /* already closed */ }
26922730
try {
2693-
await sleepWithAbort(
2731+
yield* sleepWithHeartbeats(
26942732
rateLimitRetryDelayMs(rateLimitPolicy, retryAfterHeader, Date.now()),
26952733
// Listen on the upstream signal: once the SSE body is being streamed, a client
26962734
// cancel aborts `upstream` through the bridge, and upstream is also linked from
26972735
// options.abortSignal — so this covers both cancellation paths.
26982736
upstream.signal,
2737+
Math.min(10_000, Math.max(250, stallTimeoutMs / 2)),
26992738
);
27002739
} catch {
27012740
if (options.abortSignal?.aborted || upstream.signal.aborted) {
@@ -2733,6 +2772,7 @@ async function handleResponsesInner(
27332772
if (rotated) {
27342773
try { void response.body?.cancel().catch(() => {}); } catch { /* already closed */ }
27352774
route.provider = rotated;
2775+
transportToken += 1;
27362776
activeAdapter = resolveAdapter(
27372777
resolveWireProtocolOverride(route.providerName, route.modelId, route.provider, inboundWire),
27382778
config.cacheRetention,
@@ -2759,6 +2799,7 @@ async function handleResponsesInner(
27592799
anthropicPoolAccountId = nextAccountId;
27602800
anthropicPoolFailovers += 1;
27612801
route.provider = { ...route.provider, apiKey: accessToken };
2802+
transportToken += 1;
27622803
promoteAnthropicActiveAccount(nextAccountId);
27632804
logCtx.provider = formatAnthropicProviderForLog("anthropic", nextAccountId, config);
27642805
activeAdapter = resolveAdapter(
@@ -2779,6 +2820,7 @@ async function handleResponsesInner(
27792820
alreadyAttempted: imageTierBias > 0,
27802821
})) {
27812822
imageTierBias = 1;
2823+
transportToken += 1;
27822824
try { void response.body?.cancel().catch(() => {}); } catch { /* already closed */ }
27832825
continue;
27842826
}

‎src/web-search/loop.ts‎

Lines changed: 29 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ import { runAnthropicWebSearch } from "./anthropic-executor";
88
import { clearableDeadline } from "../lib/abort";
99
import { redactSecretString } from "../lib/redact";
1010
import { readBoundedResponseBody } from "../lib/bounded-body";
11-
import { fetchWithResetRetry, sleepWithAbort } from "../lib/upstream-retry";
11+
import { fetchWithResetRetry, sleepWithHeartbeats } from "../lib/upstream-retry";
1212
import { rateLimitRetryDelayMs } from "../providers/key-failover";
1313
import {
1414
isTranslatorBudgetExceededError,
@@ -266,6 +266,12 @@ export async function runWithWebSearch(deps: WebSearchLoopDeps): Promise<Respons
266266
// Mutable: 429 key-failover (deps.on429) can swap in a rebuilt adapter mid-loop.
267267
let adapter = deps.adapter;
268268

269+
// Bridge stall budget (seconds of silence before upstream_stall_timeout); the retry backoff
270+
// heartbeat interval is derived from it so the watchdog is always fed during deliberate waits.
271+
const stallTimeoutMs = typeof deps.stallTimeoutSec === "number" && Number.isFinite(deps.stallTimeoutSec) && deps.stallTimeoutSec > 0
272+
? Math.floor(deps.stallTimeoutSec * 1000)
273+
: 300_000;
274+
269275
const messages: OcxMessage[] = [...parsed.context.messages];
270276
const loopT0 = Date.now();
271277
const allTools = parsed.context.tools ?? [];
@@ -335,17 +341,28 @@ export async function runWithWebSearch(deps: WebSearchLoopDeps): Promise<Respons
335341
/**
336342
* Build and fetch one web-search iteration on the given adapter, under the iteration
337343
* header deadline. The caller owns same-target 429 replays and key rotation around it.
344+
* The outbound request is cached per adapter so a same-target replay reuses the EXACT
345+
* URL, serialized body, and headers (builder runs once per target sequence).
338346
*/
347+
let cachedRequest: AdapterRequest | undefined;
348+
let cachedAdapter: ProviderAdapter | undefined;
339349
const fetchOnce = async (requestAdapter: ProviderAdapter, recovery?: AttemptRecoveryKind): Promise<IterationResponse> => {
340-
const request = await requestAdapter.buildRequest(iterParsed, {
341-
headers: selectedForwardHeaders,
342-
abortSignal: headerDeadline.signal,
343-
translatorBudget,
344-
});
345-
try {
346-
deps.onRequestBuilt?.(request);
347-
} catch {
348-
// Diagnostics are best-effort and must never abort a web-search iteration.
350+
let request: AdapterRequest;
351+
if (cachedRequest !== undefined && cachedAdapter === requestAdapter) {
352+
request = cachedRequest;
353+
} else {
354+
request = await requestAdapter.buildRequest(iterParsed, {
355+
headers: selectedForwardHeaders,
356+
abortSignal: headerDeadline.signal,
357+
translatorBudget,
358+
});
359+
try {
360+
deps.onRequestBuilt?.(request);
361+
} catch {
362+
// Diagnostics are best-effort and must never abort a web-search iteration.
363+
}
364+
cachedRequest = request;
365+
cachedAdapter = requestAdapter;
349366
}
350367
deps.onAttemptSend?.(recovery);
351368
let response: Response;
@@ -393,9 +410,10 @@ export async function runWithWebSearch(deps: WebSearchLoopDeps): Promise<Respons
393410
// before sleeping so a stale expiry can never race the client-cancel path.
394411
headerDeadline.clear();
395412
try {
396-
await sleepWithAbort(
413+
yield* sleepWithHeartbeats(
397414
rateLimitRetryDelayMs(rateLimitRetryPolicy, retryAfterHeader, Date.now()),
398415
signal,
416+
Math.min(10_000, Math.max(250, stallTimeoutMs / 2)),
399417
);
400418
} catch {
401419
throw new LoopError(499, "client closed request during web-search");

0 commit comments

Comments
 (0)