@@ -30,7 +30,6 @@ import {
3030} from "./error"
3131import {
3232 buildCostChunk ,
33- createBodyConverter ,
3433 createStreamPartConverter ,
3534 createResponseConverter ,
3635 UsageInfo ,
@@ -53,12 +52,10 @@ import { createProviderBudgetTracker } from "./providerBudgetTracker"
5352import { accumulateUsage , HOT_WORKSPACES } from "./usageBatcher"
5453import { Workspace } from "@opencode-ai/console-core/workspace.js"
5554import { countryFromRequest , isModelCountryRestricted } from "~/lib/request-country"
55+ import { prepareRequestBody } from "./requestBody"
5656
5757type ZenData = Awaited < ReturnType < typeof ZenData . list > >
58- type RetryOptions = {
59- excludeProviders : string [ ]
60- retryCount : number
61- }
58+ type PreparedBody = Awaited < ReturnType < typeof prepareRequestBody > >
6259type BillingSource = "anonymous" | "free" | "byok" | "subscription" | "lite" | "balance"
6360
6461function resolve ( text : string , params ?: Record < string , string | number > ) {
@@ -86,8 +83,6 @@ export async function handler(
8683 type ProviderInfo = Awaited < ReturnType < typeof selectProvider > >
8784 type CostInfo = ReturnType < typeof calculateCost >
8885
89- const MAX_FAILOVER_RETRIES = 3
90- const MAX_RETRYABLE_STATUS_RETRIES = 3
9186 const dict = i18n ( localeFromRequest ( input . request ) )
9287 const t = ( key : Key , params ?: Record < string , string | number > ) => resolve ( dict [ key ] , params )
9388 const ADMIN_WORKSPACES = [
@@ -96,12 +91,15 @@ export async function handler(
9691 "wrk_01KKZDKDWCS1VTJF8QTX62DD50" , // contributors
9792 ]
9893
94+ let requestBody : PreparedBody | undefined
9995 try {
10096 const url = input . request . url
101- const body = await input . request . text ( )
97+ const body = input . request . body
98+ if ( ! body ) throw new Error ( "Missing request body" )
99+ requestBody = opts . format === "google" ? undefined : await prepareRequestBody ( body )
102100 const model =
103- opts . format === "google" ? opts . parseModel ( url , undefined ) : ( body . match ( / " m o d e l " \s * : \s * " ( [ ^ " ] + ) " / ) ?. [ 1 ] ?? "" )
104- const isStream = opts . format === "google" ? opts . parseIsStream ( url , undefined ) : / " s t r e a m " \s * : \s * t r u e / . test ( body )
101+ opts . format === "google" ? opts . parseModel ( url , undefined ) : ( requestBody ?. model ?? "" )
102+ const googleStream = opts . format === "google" ? opts . parseIsStream ( url , undefined ) : undefined
105103 const rawIp = input . request . headers . get ( "x-real-ip" ) ?? ""
106104 const ip = rawIp . includes ( ":" ) ? rawIp . split ( ":" ) . slice ( 0 , 4 ) . join ( ":" ) : rawIp
107105 const rawZenApiKey = opts . parseApiKey ( input . request . headers )
@@ -112,7 +110,6 @@ export async function handler(
112110 const projectId = input . request . headers . get ( "x-opencode-project" ) ?? ""
113111 const userAgent = input . request . headers . get ( "user-agent" ) ?? ""
114112 logger . metric ( {
115- is_stream : isStream ,
116113 session : sessionId ,
117114 request : requestId ,
118115 client : ocClient ,
@@ -174,15 +171,14 @@ export async function handler(
174171 )
175172 const providerBudget = await providerBudgetTracker ?. check ( )
176173
177- const retriableRequest = async ( retry : RetryOptions = { excludeProviders : [ ] , retryCount : 0 } ) => {
174+ const providerRequest = async ( ) => {
178175 const providerInfo = selectProvider (
179176 model ,
180177 zenData ,
181178 authInfo ,
182179 modelInfo ,
183180 stickyId ,
184181 trialProviders ,
185- retry ,
186182 stickyProvider ,
187183 modelTpmLimits ,
188184 modelTpsLimits ,
@@ -198,95 +194,66 @@ export async function handler(
198194 } )
199195
200196 const startTimestamp = Date . now ( )
201- const reqUrl = providerInfo . modifyUrl ( providerInfo . api , isStream )
202- const directBody = ( ( ) => {
203- const specialAnthropic =
204- providerInfo . format === "anthropic" &&
205- ( providerInfo . model . startsWith ( "arn:aws:bedrock:" ) ||
206- providerInfo . model . startsWith ( "global.anthropic." ) ||
207- providerInfo . model . startsWith ( "databricks-claude-" ) )
208- if ( providerInfo . format === opts . format && ! providerInfo . payloadModifier && ! specialAnthropic ) {
209- const patched = body . replace ( / " m o d e l " \s * : \s * " [ ^ " ] + " / , `"model":${ JSON . stringify ( providerInfo . model ) } ` )
210- if ( providerInfo . format !== "oa-compat" || ! isStream ) return patched
211- return patched . replace ( / } \s * $ / , ',"stream_options":{"include_usage":true}}' )
212- }
213- return undefined
197+ const reqUrl = providerInfo . modifyUrl ( providerInfo . api , googleStream ?? false )
198+ const specialAnthropic =
199+ providerInfo . format === "anthropic" &&
200+ ( providerInfo . model . startsWith ( "arn:aws:bedrock:" ) ||
201+ providerInfo . model . startsWith ( "global.anthropic." ) ||
202+ providerInfo . model . startsWith ( "databricks-claude-" ) )
203+ if ( providerInfo . format !== opts . format ) throw new Error ( "Zen provider format must match request format" )
204+ if ( providerInfo . payloadModifier ) throw new Error ( "Zen provider payload modifiers are incompatible with streaming" )
205+ if ( specialAnthropic ) throw new Error ( "Anthropic provider body modifiers are incompatible with streaming" )
206+ const prepared = requestBody
207+
208+ const reqBody = ( ( ) => {
209+ if ( opts . format === "google" ) return body
210+ if ( ! prepared ) throw new Error ( "Missing prepared request body" )
211+ return prepared . stream ( providerInfo . model , providerInfo . format === "oa-compat" )
214212 } ) ( )
215- const reqBody =
216- directBody ??
217- JSON . stringify (
218- providerInfo . modifyBody ( {
219- ...createBodyConverter ( opts . format , providerInfo . format ) ( JSON . parse ( body ) ) ,
220- model : providerInfo . model ,
221- ...( ( ) => {
222- const replacer = ( obj : Record < string , any > ) : Record < string , any > =>
223- Object . fromEntries (
224- Object . entries ( obj ) . flatMap ( ( [ k , v ] ) => {
225- if ( Array . isArray ( v ) ) return [ [ k , v ] ]
226- if ( typeof v === "object" ) return [ [ k , replacer ( v ) ] ]
227- if ( typeof v === "string" ) {
228- if ( v === "$workspace" ) return authInfo ?. workspaceID ? [ [ k , authInfo . workspaceID ] ] : [ ]
229- if ( v === "$org" )
230- return authInfo ?. workspaceID ? [ [ k , authInfo . workspaceID . replace ( "wrk_" , "org_" ) ] ] : [ ]
231- if ( v === "$user" ) return stickyId ? [ [ k , stickyId ] ] : [ ]
232- if ( v . startsWith ( "$header." ) ) {
233- const headerValue = input . request . headers . get ( v . slice ( 8 ) )
234- return headerValue ? [ [ k , headerValue ] ] : [ ]
235- }
236- }
237- return [ [ k , v ] ]
238- } ) ,
239- )
240- return replacer ( providerInfo . payloadModifier ?? { } )
241- } ) ( ) ,
242- } ) ,
243- )
244213 logger . debug ( "REQUEST URL: " + reqUrl )
245- logger . debug ( "REQUEST: " + reqBody . substring ( 0 , 300 ) + "..." )
214+ logger . debug ( "REQUEST: " + ( requestBody ?. preview ?? "" ) + "..." )
246215 const isNewInference =
247216 providerInfo . id . startsWith ( "console." ) ||
248217 providerInfo . id . startsWith ( "console-go." ) ||
249218 providerInfo . id . startsWith ( "inf." ) ||
250219 providerInfo . id . startsWith ( "inf-go." )
251- const res = await fetchWithRetryableStatus (
252- reqUrl ,
253- {
254- method : "POST" ,
255- headers : ( ( ) => {
256- const headers = new Headers ( input . request . headers )
257- providerInfo . modifyHeaders ( headers , providerInfo . apiKey , stickyId )
258- Object . entries ( providerInfo . headerModifier ?? { } ) . forEach ( ( [ k , v ] ) => {
259- if ( v === "$ip" ) return headers . set ( k , ip )
260- if ( v === "$caller" ) return headers . set ( k , stickyId )
261- if ( v === "$session" ) return headers . set ( k , sessionId )
262- if ( v === "$model" ) return headers . set ( k , model )
263- if ( v === "$request" ) return headers . set ( k , requestId )
264- if ( v === "$project" ) return headers . set ( k , projectId )
265- if ( v === "$workspace" ) {
266- if ( authInfo ?. workspaceID ) headers . set ( k , authInfo . workspaceID )
267- return
268- }
269- if ( v === "$org" ) {
270- if ( authInfo ?. workspaceID ) headers . set ( k , authInfo . workspaceID . replace ( "wrk_" , "org_" ) )
271- return
272- }
273- headers . set ( k , v )
274- } )
275- headers . delete ( "host" )
276- headers . delete ( "content-length" )
277- headers . delete ( "x-opencode-request" )
278- if ( ! isNewInference ) headers . delete ( "x-opencode-session" )
279- headers . delete ( "x-opencode-project" )
280- headers . delete ( "x-opencode-client" )
281- return headers
282- } ) ( ) ,
283- body : reqBody ,
284- // Propagate caller disconnects to the upstream provider request so
285- // abandoned Console requests do not leave orphaned inference work open.
286- signal : input . request . signal ,
287- } ,
288- { count : isNewInference ? MAX_RETRYABLE_STATUS_RETRIES : 0 } ,
289- )
220+ const res = await fetch ( reqUrl , {
221+ method : "POST" ,
222+ headers : ( ( ) => {
223+ const headers = new Headers ( input . request . headers )
224+ providerInfo . modifyHeaders ( headers , providerInfo . apiKey , stickyId )
225+ Object . entries ( providerInfo . headerModifier ?? { } ) . forEach ( ( [ k , v ] ) => {
226+ if ( v === "$ip" ) return headers . set ( k , ip )
227+ if ( v === "$caller" ) return headers . set ( k , stickyId )
228+ if ( v === "$session" ) return headers . set ( k , sessionId )
229+ if ( v === "$model" ) return headers . set ( k , model )
230+ if ( v === "$request" ) return headers . set ( k , requestId )
231+ if ( v === "$project" ) return headers . set ( k , projectId )
232+ if ( v === "$workspace" ) {
233+ if ( authInfo ?. workspaceID ) headers . set ( k , authInfo . workspaceID )
234+ return
235+ }
236+ if ( v === "$org" ) {
237+ if ( authInfo ?. workspaceID ) headers . set ( k , authInfo . workspaceID . replace ( "wrk_" , "org_" ) )
238+ return
239+ }
240+ headers . set ( k , v )
241+ } )
242+ headers . delete ( "host" )
243+ headers . delete ( "content-length" )
244+ headers . delete ( "x-opencode-request" )
245+ if ( ! isNewInference ) headers . delete ( "x-opencode-session" )
246+ headers . delete ( "x-opencode-project" )
247+ headers . delete ( "x-opencode-client" )
248+ return headers
249+ } ) ( ) ,
250+ body : reqBody ,
251+ // Propagate caller disconnects to the upstream provider request so
252+ // abandoned Console requests do not leave orphaned inference work open.
253+ signal : input . request . signal ,
254+ } )
255+ const isStream = res . headers . get ( "content-type" ) ?. toLowerCase ( ) . includes ( "text/event-stream" ) ?? false
256+ logger . metric ( { is_stream : isStream } )
290257
291258 if ( isNewInference ) {
292259 const resEndpointId = res . headers . get ( "x-opencode-endpoint-id" )
@@ -305,29 +272,10 @@ export async function handler(
305272 } )
306273 }
307274
308- // Try another provider => stop retrying if using fallback provider
309- if (
310- //!isNewInference &&
311- res . status !== 200 &&
312- // ie. 400 error is usually provider error like malformed request
313- res . status !== 400 &&
314- // ie. openai 404 error: Item with id 'msg_0ead8b004a3b165d0069436a6b6834819896da85b63b196a3f' not found.
315- ! ( modelInfo . id . startsWith ( "gpt-" ) && res . status === 404 ) &&
316- // ie. cannot change codex model providers mid-session
317- modelInfo . stickyProvider !== "strict" &&
318- modelInfo . fallbackProvider &&
319- providerInfo . id !== modelInfo . fallbackProvider
320- ) {
321- return retriableRequest ( {
322- excludeProviders : [ ...retry . excludeProviders , providerInfo . id ] ,
323- retryCount : retry . retryCount + 1 ,
324- } )
325- }
326-
327- return { providerInfo, reqBody, res, startTimestamp }
275+ return { providerInfo, res, startTimestamp, isStream }
328276 }
329277
330- const { providerInfo, reqBody , res, startTimestamp } = await retriableRequest ( )
278+ const { providerInfo, res, startTimestamp, isStream } = await providerRequest ( )
331279
332280 // Store sticky provider
333281 if ( res . status === 200 ) await stickyTracker ?. set ( providerInfo . id )
@@ -483,6 +431,8 @@ export async function handler(
483431 headers : resHeaders ,
484432 } )
485433 } catch ( error : any ) {
434+ if ( requestBody ) void requestBody . cancel ( ) . catch ( ( ) => { } )
435+ else void input . request . body ?. cancel ( ) . catch ( ( ) => { } )
486436 // The caller disconnected before we finished. Because the outbound provider
487437 // request shares input.request.signal, an aborted caller surfaces here as an
488438 // AbortError. There is no client left to receive a body, so skip the error
@@ -607,7 +557,6 @@ export async function handler(
607557 modelInfo : ModelInfo ,
608558 stickyId : string ,
609559 trialProviders : string [ ] | undefined ,
610- retry : RetryOptions ,
611560 stickyProviderId : string | undefined ,
612561 modelTpmLimits : Record < string , number > | undefined ,
613562 modelTpsLimits : Record < string , { qualify : number ; unqualify : number } > | undefined ,
@@ -634,14 +583,11 @@ export async function handler(
634583 } ) )
635584 }
636585
637- // Use fallback provider if max retries reached
638586 const fallbackProvider = allProviders . find ( ( provider ) => provider . id === modelInfo . fallbackProvider )
639- if ( retry . retryCount === MAX_FAILOVER_RETRIES ) return fallbackProvider
640587
641588 let topPriority = Infinity
642589 const providers = allProviders
643590 . filter ( ( provider ) => provider . weight !== 0 )
644- . filter ( ( provider ) => ! retry . excludeProviders . includes ( provider . id ) )
645591 . filter ( ( provider ) => {
646592 if ( provider . budgetPriority === undefined ) return true
647593 if ( ! providerBudget ) return true
@@ -1049,15 +995,6 @@ export async function handler(
1049995 providerInfo . apiKey = authInfo . provider . credentials
1050996 }
1051997
1052- async function fetchWithRetryableStatus ( url : string , options : RequestInit , retry = { count : 0 } ) {
1053- const res = await fetch ( url , options )
1054- if ( [ 429 , 529 ] . includes ( res . status ) && retry . count < MAX_RETRYABLE_STATUS_RETRIES ) {
1055- await new Promise ( ( resolve ) => setTimeout ( resolve , Math . pow ( 2 , retry . count ) * 500 ) )
1056- return fetchWithRetryableStatus ( url , options , { count : retry . count + 1 } )
1057- }
1058- return res
1059- }
1060-
1061998 function calculateCost ( modelInfo : ModelInfo , usageInfo : UsageInfo ) {
1062999 const { inputTokens, outputTokens, reasoningTokens, cacheReadTokens, cacheWrite5mTokens, cacheWrite1hTokens } =
10631000 usageInfo
0 commit comments