Skip to content

Commit 32c3637

Browse files
authored
fix(console): stream zen request bodies (#44429)
1 parent 9d466cd commit 32c3637

3 files changed

Lines changed: 299 additions & 129 deletions

File tree

‎packages/console/app/src/routes/zen/util/handler.ts‎

Lines changed: 66 additions & 129 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,6 @@ import {
3030
} from "./error"
3131
import {
3232
buildCostChunk,
33-
createBodyConverter,
3433
createStreamPartConverter,
3534
createResponseConverter,
3635
UsageInfo,
@@ -53,12 +52,10 @@ import { createProviderBudgetTracker } from "./providerBudgetTracker"
5352
import { accumulateUsage, HOT_WORKSPACES } from "./usageBatcher"
5453
import { Workspace } from "@opencode-ai/console-core/workspace.js"
5554
import { countryFromRequest, isModelCountryRestricted } from "~/lib/request-country"
55+
import { prepareRequestBody } from "./requestBody"
5656

5757
type ZenData = Awaited<ReturnType<typeof ZenData.list>>
58-
type RetryOptions = {
59-
excludeProviders: string[]
60-
retryCount: number
61-
}
58+
type PreparedBody = Awaited<ReturnType<typeof prepareRequestBody>>
6259
type BillingSource = "anonymous" | "free" | "byok" | "subscription" | "lite" | "balance"
6360

6461
function resolve(text: string, params?: Record<string, string | number>) {
@@ -86,8 +83,6 @@ export async function handler(
8683
type ProviderInfo = Awaited<ReturnType<typeof selectProvider>>
8784
type CostInfo = ReturnType<typeof calculateCost>
8885

89-
const MAX_FAILOVER_RETRIES = 3
90-
const MAX_RETRYABLE_STATUS_RETRIES = 3
9186
const dict = i18n(localeFromRequest(input.request))
9287
const t = (key: Key, params?: Record<string, string | number>) => resolve(dict[key], params)
9388
const ADMIN_WORKSPACES = [
@@ -96,12 +91,15 @@ export async function handler(
9691
"wrk_01KKZDKDWCS1VTJF8QTX62DD50", // contributors
9792
]
9893

94+
let requestBody: PreparedBody | undefined
9995
try {
10096
const url = input.request.url
101-
const body = await input.request.text()
97+
const body = input.request.body
98+
if (!body) throw new Error("Missing request body")
99+
requestBody = opts.format === "google" ? undefined : await prepareRequestBody(body)
102100
const model =
103-
opts.format === "google" ? opts.parseModel(url, undefined) : (body.match(/"model"\s*:\s*"([^"]+)"/)?.[1] ?? "")
104-
const isStream = opts.format === "google" ? opts.parseIsStream(url, undefined) : /"stream"\s*:\s*true/.test(body)
101+
opts.format === "google" ? opts.parseModel(url, undefined) : (requestBody?.model ?? "")
102+
const googleStream = opts.format === "google" ? opts.parseIsStream(url, undefined) : undefined
105103
const rawIp = input.request.headers.get("x-real-ip") ?? ""
106104
const ip = rawIp.includes(":") ? rawIp.split(":").slice(0, 4).join(":") : rawIp
107105
const rawZenApiKey = opts.parseApiKey(input.request.headers)
@@ -112,7 +110,6 @@ export async function handler(
112110
const projectId = input.request.headers.get("x-opencode-project") ?? ""
113111
const userAgent = input.request.headers.get("user-agent") ?? ""
114112
logger.metric({
115-
is_stream: isStream,
116113
session: sessionId,
117114
request: requestId,
118115
client: ocClient,
@@ -174,15 +171,14 @@ export async function handler(
174171
)
175172
const providerBudget = await providerBudgetTracker?.check()
176173

177-
const retriableRequest = async (retry: RetryOptions = { excludeProviders: [], retryCount: 0 }) => {
174+
const providerRequest = async () => {
178175
const providerInfo = selectProvider(
179176
model,
180177
zenData,
181178
authInfo,
182179
modelInfo,
183180
stickyId,
184181
trialProviders,
185-
retry,
186182
stickyProvider,
187183
modelTpmLimits,
188184
modelTpsLimits,
@@ -198,95 +194,66 @@ export async function handler(
198194
})
199195

200196
const startTimestamp = Date.now()
201-
const reqUrl = providerInfo.modifyUrl(providerInfo.api, isStream)
202-
const directBody = (() => {
203-
const specialAnthropic =
204-
providerInfo.format === "anthropic" &&
205-
(providerInfo.model.startsWith("arn:aws:bedrock:") ||
206-
providerInfo.model.startsWith("global.anthropic.") ||
207-
providerInfo.model.startsWith("databricks-claude-"))
208-
if (providerInfo.format === opts.format && !providerInfo.payloadModifier && !specialAnthropic) {
209-
const patched = body.replace(/"model"\s*:\s*"[^"]+"/, `"model":${JSON.stringify(providerInfo.model)}`)
210-
if (providerInfo.format !== "oa-compat" || !isStream) return patched
211-
return patched.replace(/}\s*$/, ',"stream_options":{"include_usage":true}}')
212-
}
213-
return undefined
197+
const reqUrl = providerInfo.modifyUrl(providerInfo.api, googleStream ?? false)
198+
const specialAnthropic =
199+
providerInfo.format === "anthropic" &&
200+
(providerInfo.model.startsWith("arn:aws:bedrock:") ||
201+
providerInfo.model.startsWith("global.anthropic.") ||
202+
providerInfo.model.startsWith("databricks-claude-"))
203+
if (providerInfo.format !== opts.format) throw new Error("Zen provider format must match request format")
204+
if (providerInfo.payloadModifier) throw new Error("Zen provider payload modifiers are incompatible with streaming")
205+
if (specialAnthropic) throw new Error("Anthropic provider body modifiers are incompatible with streaming")
206+
const prepared = requestBody
207+
208+
const reqBody = (() => {
209+
if (opts.format === "google") return body
210+
if (!prepared) throw new Error("Missing prepared request body")
211+
return prepared.stream(providerInfo.model, providerInfo.format === "oa-compat")
214212
})()
215-
const reqBody =
216-
directBody ??
217-
JSON.stringify(
218-
providerInfo.modifyBody({
219-
...createBodyConverter(opts.format, providerInfo.format)(JSON.parse(body)),
220-
model: providerInfo.model,
221-
...(() => {
222-
const replacer = (obj: Record<string, any>): Record<string, any> =>
223-
Object.fromEntries(
224-
Object.entries(obj).flatMap(([k, v]) => {
225-
if (Array.isArray(v)) return [[k, v]]
226-
if (typeof v === "object") return [[k, replacer(v)]]
227-
if (typeof v === "string") {
228-
if (v === "$workspace") return authInfo?.workspaceID ? [[k, authInfo.workspaceID]] : []
229-
if (v === "$org")
230-
return authInfo?.workspaceID ? [[k, authInfo.workspaceID.replace("wrk_", "org_")]] : []
231-
if (v === "$user") return stickyId ? [[k, stickyId]] : []
232-
if (v.startsWith("$header.")) {
233-
const headerValue = input.request.headers.get(v.slice(8))
234-
return headerValue ? [[k, headerValue]] : []
235-
}
236-
}
237-
return [[k, v]]
238-
}),
239-
)
240-
return replacer(providerInfo.payloadModifier ?? {})
241-
})(),
242-
}),
243-
)
244213
logger.debug("REQUEST URL: " + reqUrl)
245-
logger.debug("REQUEST: " + reqBody.substring(0, 300) + "...")
214+
logger.debug("REQUEST: " + (requestBody?.preview ?? "") + "...")
246215
const isNewInference =
247216
providerInfo.id.startsWith("console.") ||
248217
providerInfo.id.startsWith("console-go.") ||
249218
providerInfo.id.startsWith("inf.") ||
250219
providerInfo.id.startsWith("inf-go.")
251-
const res = await fetchWithRetryableStatus(
252-
reqUrl,
253-
{
254-
method: "POST",
255-
headers: (() => {
256-
const headers = new Headers(input.request.headers)
257-
providerInfo.modifyHeaders(headers, providerInfo.apiKey, stickyId)
258-
Object.entries(providerInfo.headerModifier ?? {}).forEach(([k, v]) => {
259-
if (v === "$ip") return headers.set(k, ip)
260-
if (v === "$caller") return headers.set(k, stickyId)
261-
if (v === "$session") return headers.set(k, sessionId)
262-
if (v === "$model") return headers.set(k, model)
263-
if (v === "$request") return headers.set(k, requestId)
264-
if (v === "$project") return headers.set(k, projectId)
265-
if (v === "$workspace") {
266-
if (authInfo?.workspaceID) headers.set(k, authInfo.workspaceID)
267-
return
268-
}
269-
if (v === "$org") {
270-
if (authInfo?.workspaceID) headers.set(k, authInfo.workspaceID.replace("wrk_", "org_"))
271-
return
272-
}
273-
headers.set(k, v)
274-
})
275-
headers.delete("host")
276-
headers.delete("content-length")
277-
headers.delete("x-opencode-request")
278-
if (!isNewInference) headers.delete("x-opencode-session")
279-
headers.delete("x-opencode-project")
280-
headers.delete("x-opencode-client")
281-
return headers
282-
})(),
283-
body: reqBody,
284-
// Propagate caller disconnects to the upstream provider request so
285-
// abandoned Console requests do not leave orphaned inference work open.
286-
signal: input.request.signal,
287-
},
288-
{ count: isNewInference ? MAX_RETRYABLE_STATUS_RETRIES : 0 },
289-
)
220+
const res = await fetch(reqUrl, {
221+
method: "POST",
222+
headers: (() => {
223+
const headers = new Headers(input.request.headers)
224+
providerInfo.modifyHeaders(headers, providerInfo.apiKey, stickyId)
225+
Object.entries(providerInfo.headerModifier ?? {}).forEach(([k, v]) => {
226+
if (v === "$ip") return headers.set(k, ip)
227+
if (v === "$caller") return headers.set(k, stickyId)
228+
if (v === "$session") return headers.set(k, sessionId)
229+
if (v === "$model") return headers.set(k, model)
230+
if (v === "$request") return headers.set(k, requestId)
231+
if (v === "$project") return headers.set(k, projectId)
232+
if (v === "$workspace") {
233+
if (authInfo?.workspaceID) headers.set(k, authInfo.workspaceID)
234+
return
235+
}
236+
if (v === "$org") {
237+
if (authInfo?.workspaceID) headers.set(k, authInfo.workspaceID.replace("wrk_", "org_"))
238+
return
239+
}
240+
headers.set(k, v)
241+
})
242+
headers.delete("host")
243+
headers.delete("content-length")
244+
headers.delete("x-opencode-request")
245+
if (!isNewInference) headers.delete("x-opencode-session")
246+
headers.delete("x-opencode-project")
247+
headers.delete("x-opencode-client")
248+
return headers
249+
})(),
250+
body: reqBody,
251+
// Propagate caller disconnects to the upstream provider request so
252+
// abandoned Console requests do not leave orphaned inference work open.
253+
signal: input.request.signal,
254+
})
255+
const isStream = res.headers.get("content-type")?.toLowerCase().includes("text/event-stream") ?? false
256+
logger.metric({ is_stream: isStream })
290257

291258
if (isNewInference) {
292259
const resEndpointId = res.headers.get("x-opencode-endpoint-id")
@@ -305,29 +272,10 @@ export async function handler(
305272
})
306273
}
307274

308-
// Try another provider => stop retrying if using fallback provider
309-
if (
310-
//!isNewInference &&
311-
res.status !== 200 &&
312-
// ie. 400 error is usually provider error like malformed request
313-
res.status !== 400 &&
314-
// ie. openai 404 error: Item with id 'msg_0ead8b004a3b165d0069436a6b6834819896da85b63b196a3f' not found.
315-
!(modelInfo.id.startsWith("gpt-") && res.status === 404) &&
316-
// ie. cannot change codex model providers mid-session
317-
modelInfo.stickyProvider !== "strict" &&
318-
modelInfo.fallbackProvider &&
319-
providerInfo.id !== modelInfo.fallbackProvider
320-
) {
321-
return retriableRequest({
322-
excludeProviders: [...retry.excludeProviders, providerInfo.id],
323-
retryCount: retry.retryCount + 1,
324-
})
325-
}
326-
327-
return { providerInfo, reqBody, res, startTimestamp }
275+
return { providerInfo, res, startTimestamp, isStream }
328276
}
329277

330-
const { providerInfo, reqBody, res, startTimestamp } = await retriableRequest()
278+
const { providerInfo, res, startTimestamp, isStream } = await providerRequest()
331279

332280
// Store sticky provider
333281
if (res.status === 200) await stickyTracker?.set(providerInfo.id)
@@ -483,6 +431,8 @@ export async function handler(
483431
headers: resHeaders,
484432
})
485433
} catch (error: any) {
434+
if (requestBody) void requestBody.cancel().catch(() => {})
435+
else void input.request.body?.cancel().catch(() => {})
486436
// The caller disconnected before we finished. Because the outbound provider
487437
// request shares input.request.signal, an aborted caller surfaces here as an
488438
// AbortError. There is no client left to receive a body, so skip the error
@@ -607,7 +557,6 @@ export async function handler(
607557
modelInfo: ModelInfo,
608558
stickyId: string,
609559
trialProviders: string[] | undefined,
610-
retry: RetryOptions,
611560
stickyProviderId: string | undefined,
612561
modelTpmLimits: Record<string, number> | undefined,
613562
modelTpsLimits: Record<string, { qualify: number; unqualify: number }> | undefined,
@@ -634,14 +583,11 @@ export async function handler(
634583
}))
635584
}
636585

637-
// Use fallback provider if max retries reached
638586
const fallbackProvider = allProviders.find((provider) => provider.id === modelInfo.fallbackProvider)
639-
if (retry.retryCount === MAX_FAILOVER_RETRIES) return fallbackProvider
640587

641588
let topPriority = Infinity
642589
const providers = allProviders
643590
.filter((provider) => provider.weight !== 0)
644-
.filter((provider) => !retry.excludeProviders.includes(provider.id))
645591
.filter((provider) => {
646592
if (provider.budgetPriority === undefined) return true
647593
if (!providerBudget) return true
@@ -1049,15 +995,6 @@ export async function handler(
1049995
providerInfo.apiKey = authInfo.provider.credentials
1050996
}
1051997

1052-
async function fetchWithRetryableStatus(url: string, options: RequestInit, retry = { count: 0 }) {
1053-
const res = await fetch(url, options)
1054-
if ([429, 529].includes(res.status) && retry.count < MAX_RETRYABLE_STATUS_RETRIES) {
1055-
await new Promise((resolve) => setTimeout(resolve, Math.pow(2, retry.count) * 500))
1056-
return fetchWithRetryableStatus(url, options, { count: retry.count + 1 })
1057-
}
1058-
return res
1059-
}
1060-
1061998
function calculateCost(modelInfo: ModelInfo, usageInfo: UsageInfo) {
1062999
const { inputTokens, outputTokens, reasoningTokens, cacheReadTokens, cacheWrite5mTokens, cacheWrite1hTokens } =
10631000
usageInfo

0 commit comments

Comments
 (0)