Skip to content

Commit 01c823e

Browse files
committed
fix(chat): align native passthrough tier policy
1 parent 71e90bf commit 01c823e

3 files changed

Lines changed: 140 additions & 13 deletions

File tree

‎src/adapters/openai-chat.ts‎

Lines changed: 15 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -14,11 +14,13 @@ import { buildNonOpenAIToolCatalogNudgeForTools, shouldInjectNonOpenAIToolCatalo
1414
import { openRouterProviderPayload, resolveOpenRouterRouting } from "../providers/openrouter-routing";
1515
import {
1616
canForwardForeignServiceTierForChatModel,
17+
fastPolicyForModel,
1718
supportsServiceTierForModel,
1819
} from "../providers/service-tier";
1920
import {
2021
canonicalFastTierMarker,
2122
createAdapterTierMetadata,
23+
decideTier,
2224
type ResolvedFastPolicy,
2325
} from "../providers/fastwire";
2426
import { openaiChatCompletionsUrl } from "./openai-chat-url";
@@ -96,7 +98,8 @@ export function buildOpenAIChatPassthroughRequest(
9698
rawBody: Record<string, unknown>,
9799
modelId: string,
98100
stream: boolean,
99-
fastPolicy: ResolvedFastPolicy,
101+
fastPolicy: ResolvedFastPolicy = fastPolicyForModel(provider, modelId, undefined, "chat"),
102+
fastMode?: boolean,
100103
): AdapterRequest {
101104
const { url, headers, hasCredential } = openAIChatTransport(provider);
102105

@@ -124,10 +127,16 @@ export function buildOpenAIChatPassthroughRequest(
124127
// `<listed>:<tag>` siblings the operator never opted out, silently returning prose.
125128
if (provider.noStructuredOutputModels?.includes(modelId)) delete body.response_format;
126129

127-
// Share the ordinary openai-chat adapter's capability gate, while preserving the
128-
// caller's exact wire representation promised by this passthrough contract. The
129-
// Responses surface may normalize canonical Fast spelling; this difference is intentional.
130-
if (canSerializeOpenAIChatServiceTier(provider, modelId, rawBody.service_tier, undefined, fastPolicy)) {
130+
// Run the same complete Fast policy as the translated Chat path, including explicit
131+
// fastMode and foreign-tier handling. On inherited canonical Fast, the passthrough still
132+
// retains the caller's exact spelling; forced Fast uses the policy-owned wire value.
133+
const callerTier = typeof rawBody.service_tier === "string" ? rawBody.service_tier : undefined;
134+
const tierDecision = decideTier(fastPolicy, fastMode, callerTier);
135+
if (tierDecision.kind === "set") {
136+
body.service_tier = fastMode === undefined && canonicalFastTierMarker(callerTier) !== undefined
137+
? callerTier
138+
: tierDecision.value;
139+
} else if (tierDecision.kind === "forward-caller" && rawBody.service_tier !== undefined) {
131140
body.service_tier = rawBody.service_tier;
132141
}
133142
if (provider.promptCacheKey && rawBody.prompt_cache_key !== undefined) {
@@ -1294,17 +1303,14 @@ function canSerializeOpenAIChatServiceTier(
12941303
modelId: string,
12951304
serviceTier: unknown,
12961305
tierDecision?: OcxParsedRequest["options"]["tierDecision"],
1297-
resolvedPolicy?: Pick<ResolvedFastPolicy, "capability">,
12981306
): boolean {
12991307
if (serviceTier === undefined) return false;
13001308
if (tierDecision !== undefined) {
13011309
return tierDecision.kind === "set" || tierDecision.kind === "forward-caller";
13021310
}
13031311
const callerTier = typeof serviceTier === "string" ? serviceTier : undefined;
13041312
const callerCanonicalFast = canonicalFastTierMarker(callerTier) !== undefined;
1305-
const capability = resolvedPolicy === undefined
1306-
? supportsServiceTierForModel(provider, modelId)
1307-
: resolvedPolicy.capability;
1313+
const capability = supportsServiceTierForModel(provider, modelId);
13081314
const callerTierForwardAllowed = canForwardForeignServiceTierForChatModel(provider, modelId);
13091315
return callerTierForwardAllowed || (callerCanonicalFast && capability === true);
13101316
}

‎src/server/chat-native.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -161,6 +161,7 @@ export async function handleNativeChatCompletions(options: HandleNativeChatOptio
161161
route.modelId,
162162
requestedStream,
163163
fastPolicyForModel(activeProvider, route.modelId, route.providerName, "chat"),
164+
config.fastMode,
164165
);
165166
try {
166167
activeRequest = buildActiveRequest();

‎tests/openai-chat-native-policy.test.ts‎

Lines changed: 124 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,7 @@
11
import { afterEach, describe, expect, test } from "bun:test";
2+
import { mkdtempSync, rmSync } from "node:fs";
3+
import { tmpdir } from "node:os";
4+
import { join } from "node:path";
25
import {
36
buildOpenAIChatPassthroughRequest,
47
createOpenAIChatAdapter,
@@ -7,6 +10,7 @@ import {
710
decideTier,
811
tierValueAfterDecision,
912
} from "../src/providers/fastwire";
13+
import { clearKeyCooldowns } from "../src/providers/key-failover";
1014
import { fastPolicyForModel } from "../src/providers/service-tier";
1115
import { handleChatCompletions } from "../src/server/chat-completions";
1216
import type { OcxConfig, OcxParsedRequest, OcxProviderConfig } from "../src/types";
@@ -17,6 +21,7 @@ const originalFetch = globalThis.fetch;
1721

1822
afterEach(() => {
1923
globalThis.fetch = originalFetch;
24+
clearKeyCooldowns(PROVIDER_NAME);
2025
});
2126

2227
function provider(overrides: Partial<OcxProviderConfig> = {}): OcxProviderConfig {
@@ -31,31 +36,34 @@ function provider(overrides: Partial<OcxProviderConfig> = {}): OcxProviderConfig
3136

3237
function nativeBody(
3338
target: OcxProviderConfig,
34-
callerTier: string,
39+
callerTier: string | undefined,
3540
modelId = MODEL_ID,
41+
fastMode?: boolean,
3642
): Record<string, unknown> {
3743
const policy = fastPolicyForModel(target, modelId, PROVIDER_NAME, "chat");
3844
const request = buildOpenAIChatPassthroughRequest(
3945
target,
4046
{
4147
model: modelId,
4248
messages: [{ role: "user", content: "ping" }],
43-
service_tier: callerTier,
49+
...(callerTier === undefined ? {} : { service_tier: callerTier }),
4450
},
4551
modelId,
4652
false,
4753
policy,
54+
fastMode,
4855
);
4956
return JSON.parse(request.body) as Record<string, unknown>;
5057
}
5158

5259
function mainPathBody(
5360
target: OcxProviderConfig,
54-
callerTier: string,
61+
callerTier: string | undefined,
5562
modelId = MODEL_ID,
63+
fastMode?: boolean,
5664
): Record<string, unknown> {
5765
const policy = fastPolicyForModel(target, modelId, PROVIDER_NAME, "chat");
58-
const tierDecision = decideTier(policy, undefined, callerTier);
66+
const tierDecision = decideTier(policy, fastMode, callerTier);
5967
const serviceTier = tierValueAfterDecision(tierDecision, callerTier);
6068
const parsed: OcxParsedRequest = {
6169
modelId,
@@ -116,6 +124,20 @@ describe("native Chat passthrough service-tier policy", () => {
116124
callerTier: "flex",
117125
expectedTier: "flex",
118126
},
127+
{
128+
name: "classified foreign-tier drop overrides CallerTierForward",
129+
config: {
130+
supportsServiceTier: true,
131+
chatServiceTier: true,
132+
fastWire: {
133+
kind: "service-tier",
134+
canonicalToWire: { priority: "priority" },
135+
foreignCallerTiers: "drop",
136+
},
137+
},
138+
callerTier: "flex",
139+
expectedTier: undefined,
140+
},
119141
] as const)("$name", ({ config, callerTier, expectedTier }) => {
120142
const body = nativeBody(provider(config), callerTier);
121143
if (expectedTier === undefined) expect(body).not.toHaveProperty("service_tier");
@@ -157,6 +179,79 @@ describe("native Chat passthrough service-tier policy", () => {
157179
expect(captured).toHaveLength(1);
158180
expect(captured[0]).not.toHaveProperty("service_tier");
159181
});
182+
183+
test("forced Fast injects the policy wire value and forced default drops the caller tier", () => {
184+
const target = provider({ supportsServiceTier: true, chatServiceTier: true });
185+
186+
expect(nativeBody(target, "flex", MODEL_ID, true).service_tier).toBe("priority");
187+
expect(nativeBody(target, undefined, MODEL_ID, true).service_tier).toBe("priority");
188+
expect(nativeBody(target, "priority", MODEL_ID, false)).not.toHaveProperty("service_tier");
189+
});
190+
191+
test("key failover rebuilds the request without reintroducing a dropped foreign tier", async () => {
192+
const previousHome = process.env.OPENCODEX_HOME;
193+
const home = mkdtempSync(join(tmpdir(), "ocx-native-tier-failover-"));
194+
process.env.OPENCODEX_HOME = home;
195+
const captured: Array<{ authorization: string | null; body: Record<string, unknown> }> = [];
196+
globalThis.fetch = (async (_input: RequestInfo | URL, init?: RequestInit) => {
197+
captured.push({
198+
authorization: new Headers(init?.headers).get("authorization"),
199+
body: JSON.parse(String(init?.body ?? "{}")) as Record<string, unknown>,
200+
});
201+
if (captured.length === 1) {
202+
return Response.json({ error: { message: "rate limited" } }, {
203+
status: 429,
204+
headers: { "retry-after": "0" },
205+
});
206+
}
207+
return Response.json({
208+
id: "chatcmpl_native_tier_failover",
209+
object: "chat.completion",
210+
choices: [{ index: 0, message: { role: "assistant", content: "ok" }, finish_reason: "stop" }],
211+
});
212+
}) as typeof fetch;
213+
const target = provider({
214+
supportsServiceTier: true,
215+
chatServiceTier: true,
216+
fastWire: {
217+
kind: "service-tier",
218+
canonicalToWire: { priority: "priority" },
219+
foreignCallerTiers: "drop",
220+
},
221+
apiKey: "key-one",
222+
apiKeyPool: [{ id: "one", key: "key-one" }, { id: "two", key: "key-two" }],
223+
});
224+
const config = {
225+
port: 0,
226+
defaultProvider: PROVIDER_NAME,
227+
providers: { [PROVIDER_NAME]: target },
228+
} as OcxConfig;
229+
230+
try {
231+
const response = await handleChatCompletions(
232+
new Request("http://localhost/v1/chat/completions", {
233+
method: "POST",
234+
headers: { "content-type": "application/json" },
235+
body: JSON.stringify({
236+
model: `${PROVIDER_NAME}/${MODEL_ID}`,
237+
messages: [{ role: "user", content: "ping" }],
238+
service_tier: "flex",
239+
}),
240+
}),
241+
config,
242+
{ model: "", provider: "" },
243+
);
244+
245+
expect(response.status).toBe(200);
246+
expect(captured.map(entry => entry.authorization)).toEqual(["Bearer key-one", "Bearer key-two"]);
247+
expect(captured).toHaveLength(2);
248+
for (const entry of captured) expect(entry.body).not.toHaveProperty("service_tier");
249+
} finally {
250+
if (previousHome === undefined) delete process.env.OPENCODEX_HOME;
251+
else process.env.OPENCODEX_HOME = previousHome;
252+
rmSync(home, { recursive: true, force: true });
253+
}
254+
});
160255
});
161256

162257
describe("main and native Chat tier authorization parity", () => {
@@ -205,6 +300,20 @@ describe("main and native Chat tier authorization parity", () => {
205300
mainTier: "flex",
206301
nativeTier: "flex",
207302
},
303+
{
304+
name: "classified foreign-tier drop with CallerTierForward",
305+
config: {
306+
supportsServiceTier: true,
307+
chatServiceTier: true,
308+
fastWire: {
309+
kind: "service-tier",
310+
canonicalToWire: { priority: "priority" },
311+
foreignCallerTiers: "drop",
312+
},
313+
},
314+
callerTier: "flex",
315+
forwarded: false,
316+
},
208317
] as const)("$name makes the same forward/drop decision", row => {
209318
const target = provider(row.config);
210319
const main = mainPathBody(target, row.callerTier);
@@ -218,4 +327,15 @@ describe("main and native Chat tier authorization parity", () => {
218327
expect(native.service_tier).toBe(row.nativeTier);
219328
}
220329
});
330+
331+
test("forced Fast and forced default make the same decision on both Chat paths", () => {
332+
const target = provider({ supportsServiceTier: true, chatServiceTier: true });
333+
334+
for (const fastMode of [true, false] as const) {
335+
const main = mainPathBody(target, "flex", MODEL_ID, fastMode);
336+
const native = nativeBody(target, "flex", MODEL_ID, fastMode);
337+
expect(forwardsTier(native)).toBe(forwardsTier(main));
338+
expect(native.service_tier).toBe(main.service_tier);
339+
}
340+
});
221341
});

0 commit comments

Comments
 (0)