Skip to content

Commit dec332c

Browse files
lidge-junlidge-jun
authored
Merge pull request #1892 from olddonkey/codex/fastwire-a0-characterization
test(fastwire): A0 characterization — lock current service-tier behavior (#1886)
2 parents 552a62c + 6b17d62 commit dec332c

2 files changed

Lines changed: 371 additions & 0 deletions

File tree

‎tests/fastwire-characterization-routing.test.ts‎

Lines changed: 132 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,132 @@
1+
import { describe, expect, test } from "bun:test";
2+
import { applyCatalogModelMetadata } from "../src/codex/catalog/effort";
3+
import type { CatalogModel, RawEntry } from "../src/codex/catalog/parsing";
4+
import { candidateCapabilityEvidence } from "../src/routing/capability";
5+
import { resolveProductionBehaviorValues } from "../src/routing/compatibility/behavior";
6+
import { evaluatePolicyProfile } from "../src/routing/evaluator";
7+
import type { OcxConfig, OcxProviderConfig } from "../src/types";
8+
9+
describe("FastWire characterization: routing profile service-tier evidence", () => {
10+
test("require.serviceTier sees supportsServiceTier=true plus chatServiceTier=false as unsupported", () => {
11+
const provider: OcxProviderConfig = {
12+
adapter: "openai-chat",
13+
baseUrl: "https://chat-no-tier.example.test/v1",
14+
authMode: "key",
15+
apiKey: "sk-test",
16+
supportsServiceTier: true,
17+
chatServiceTier: false,
18+
};
19+
const config = {
20+
port: 0,
21+
defaultProvider: "chat-no-tier",
22+
providers: { "chat-no-tier": provider },
23+
routingProfiles: {
24+
fast: {
25+
candidates: [{ provider: "chat-no-tier", model: "model" }],
26+
require: { serviceTier: "supported" },
27+
},
28+
},
29+
} as OcxConfig;
30+
const capability = candidateCapabilityEvidence(config, "chat-no-tier", "model");
31+
32+
expect(capability.serviceTier).toBe("unsupported");
33+
const result = evaluatePolicyProfile(config, "fast", {}, [{
34+
provider: "chat-no-tier",
35+
model: "model",
36+
capability,
37+
}]);
38+
expect(result.candidates[0]).toMatchObject({
39+
eligible: false,
40+
requirements: [{
41+
id: "service-tier",
42+
expected: "supported",
43+
actual: "unsupported",
44+
outcome: "unsatisfied",
45+
}],
46+
});
47+
expect(result.selectedIndex).toBeNull();
48+
});
49+
});
50+
51+
describe("FastWire characterization: compatibility fingerprint projection", () => {
52+
const cases: Array<{ label: string; expected: boolean; provider: OcxProviderConfig }> = [
53+
{
54+
label: "supported",
55+
expected: true,
56+
provider: {
57+
adapter: "openai-responses",
58+
baseUrl: "https://supported.example.test/v1",
59+
supportsServiceTier: true,
60+
},
61+
},
62+
{
63+
label: "unsupported",
64+
expected: false,
65+
provider: {
66+
adapter: "openai-responses",
67+
baseUrl: "https://unsupported.example.test/v1",
68+
supportsServiceTier: false,
69+
},
70+
},
71+
];
72+
73+
test.each(cases)("projects $label service-tier behavior", ({ label, expected, provider }) => {
74+
const config = {
75+
port: 0,
76+
defaultProvider: label,
77+
fastMode: true,
78+
providers: { [label]: provider },
79+
} as OcxConfig;
80+
const values = resolveProductionBehaviorValues(
81+
config,
82+
label,
83+
"model",
84+
provider,
85+
"fastwire-characterization-salt",
86+
);
87+
88+
expect(values?.["responses.serviceTier"]).toEqual({
89+
source: "provider_config",
90+
value: expected,
91+
});
92+
expect(values?.["runtime.fastMode"]).toEqual({
93+
source: "global_config",
94+
value: true,
95+
});
96+
});
97+
});
98+
99+
describe("FastWire characterization: catalog service-tier bytes", () => {
100+
function apply(supportsServiceTier?: boolean): RawEntry {
101+
const entry: RawEntry = {};
102+
const model: CatalogModel = {
103+
id: "model",
104+
provider: "fixture",
105+
...(supportsServiceTier === undefined ? {} : { supportsServiceTier }),
106+
};
107+
applyCatalogModelMetadata(entry, model);
108+
return entry;
109+
}
110+
111+
test("supportsServiceTier=true emits the current narrow byte golden", () => {
112+
const entry = apply(true);
113+
const projection = {
114+
default_service_tier: entry.default_service_tier,
115+
service_tiers: entry.service_tiers,
116+
additional_speed_tiers: entry.additional_speed_tiers,
117+
};
118+
expect(JSON.stringify(projection)).toBe(
119+
'{"default_service_tier":null,"service_tiers":[{"id":"priority","name":"Fast","description":"1.5x speed, increased usage"}],"additional_speed_tiers":["fast"]}',
120+
);
121+
});
122+
123+
test.each([
124+
{ label: "false", supportsServiceTier: false },
125+
{ label: "unset", supportsServiceTier: undefined },
126+
])("supportsServiceTier=$label omits all catalog tier fields", ({ supportsServiceTier }) => {
127+
const entry = apply(supportsServiceTier);
128+
expect(entry).not.toHaveProperty("default_service_tier");
129+
expect(entry).not.toHaveProperty("service_tiers");
130+
expect(entry).not.toHaveProperty("additional_speed_tiers");
131+
});
132+
});

‎tests/fastwire-characterization-wire.test.ts‎

Lines changed: 239 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,239 @@
1+
import { afterEach, describe, expect, spyOn, test } from "bun:test";
2+
import { buildOpenAIChatPassthroughRequest } from "../src/adapters/openai-chat";
3+
import { chatCompletionsToResponsesBody } from "../src/chat/inbound";
4+
import * as adapterResolveModule from "../src/server/adapter-resolve";
5+
import type { RequestLogContext } from "../src/server/request-log";
6+
import { handleResponses } from "../src/server/responses/core";
7+
import type { OcxConfig, OcxProviderConfig } from "../src/types";
8+
9+
const originalFetch = globalThis.fetch;
10+
11+
afterEach(() => {
12+
globalThis.fetch = originalFetch;
13+
});
14+
15+
async function driveResponses(args: {
16+
provider: OcxProviderConfig;
17+
model?: string;
18+
callerTier?: string;
19+
fastMode?: boolean;
20+
}): Promise<{ outboundBody: Record<string, unknown>; logCtx: RequestLogContext }> {
21+
const providerName = "fastwire-fixture";
22+
const model = args.model ?? "model";
23+
const bodies: Record<string, unknown>[] = [];
24+
globalThis.fetch = (async (_input: RequestInfo | URL, init?: RequestInit) => {
25+
bodies.push(JSON.parse(String(init?.body ?? "{}")) as Record<string, unknown>);
26+
return new Response("data: [DONE]\n\n", {
27+
status: 200,
28+
headers: { "content-type": "text/event-stream" },
29+
});
30+
}) as typeof fetch;
31+
32+
const config = {
33+
port: 0,
34+
defaultProvider: providerName,
35+
providers: { [providerName]: args.provider },
36+
...(args.fastMode === undefined ? {} : { fastMode: args.fastMode }),
37+
} as OcxConfig;
38+
const logCtx: RequestLogContext = { model: "", provider: "" };
39+
const requestBody = {
40+
model: `${providerName}/${model}`,
41+
input: "ping",
42+
stream: true,
43+
...(args.callerTier === undefined ? {} : { service_tier: args.callerTier }),
44+
};
45+
46+
await handleResponses(
47+
new Request("http://localhost/v1/responses", {
48+
method: "POST",
49+
headers: { "content-type": "application/json" },
50+
body: JSON.stringify(requestBody),
51+
}),
52+
config,
53+
logCtx,
54+
{},
55+
);
56+
57+
expect(bodies).toHaveLength(1);
58+
return { outboundBody: bodies[0]!, logCtx };
59+
}
60+
61+
const supportedResponsesProvider = (): OcxProviderConfig => ({
62+
adapter: "openai-responses",
63+
baseUrl: "https://supported.example.test/v1",
64+
authMode: "key",
65+
apiKey: "sk-test",
66+
supportsServiceTier: true,
67+
});
68+
69+
const unclassifiedResponsesProvider = (): OcxProviderConfig => ({
70+
adapter: "openai-responses",
71+
baseUrl: "https://unclassified.example.test/v1",
72+
authMode: "key",
73+
apiKey: "sk-test",
74+
});
75+
76+
describe("FastWire characterization: supported-route fastMode tri-state", () => {
77+
test("fastMode=true overrides caller flex with priority", async () => {
78+
const { outboundBody } = await driveResponses({
79+
provider: supportedResponsesProvider(),
80+
callerTier: "flex",
81+
fastMode: true,
82+
});
83+
expect(outboundBody.service_tier).toBe("priority");
84+
});
85+
86+
test("fastMode=false removes the caller tier", async () => {
87+
const { outboundBody } = await driveResponses({
88+
provider: supportedResponsesProvider(),
89+
callerTier: "turbo-x",
90+
fastMode: false,
91+
});
92+
expect(outboundBody).not.toHaveProperty("service_tier");
93+
});
94+
95+
test("fastMode=undefined preserves caller flex", async () => {
96+
const { outboundBody } = await driveResponses({
97+
provider: supportedResponsesProvider(),
98+
callerTier: "flex",
99+
});
100+
expect(outboundBody.service_tier).toBe("flex");
101+
});
102+
});
103+
104+
describe("FastWire characterization: unclassified support matrix", () => {
105+
const cells = ([true, false, undefined] as const).flatMap(fastMode =>
106+
(["priority", "fast", "flex"] as const).map(callerTier => ({ fastMode, callerTier }))
107+
);
108+
109+
test.each(cells)(
110+
"support=undefined preserves caller $callerTier with fastMode=$fastMode",
111+
async ({ fastMode, callerTier }) => {
112+
const { outboundBody } = await driveResponses({
113+
provider: unclassifiedResponsesProvider(),
114+
callerTier,
115+
fastMode,
116+
});
117+
expect(outboundBody.service_tier).toBe(callerTier);
118+
},
119+
);
120+
});
121+
122+
describe("FastWire characterization: exact-model Chat tier forwarding", () => {
123+
test.each(["flex", "turbo-x"])(
124+
"exact model true forwards foreign caller tier %s without chatServiceTier",
125+
async callerTier => {
126+
const { outboundBody } = await driveResponses({
127+
provider: {
128+
adapter: "openai-chat",
129+
baseUrl: "https://chat.example.test/v1",
130+
authMode: "key",
131+
apiKey: "sk-test",
132+
modelSupportsServiceTier: { model: true },
133+
},
134+
callerTier,
135+
});
136+
expect(outboundBody.service_tier).toBe(callerTier);
137+
},
138+
);
139+
});
140+
141+
describe("FastWire characterization: requestedServiceTier timing", () => {
142+
test("records the raw caller tier when fastMode overrides the wire tier", async () => {
143+
const { outboundBody, logCtx } = await driveResponses({
144+
provider: supportedResponsesProvider(),
145+
callerTier: "flex",
146+
fastMode: true,
147+
});
148+
expect(outboundBody.service_tier).toBe("priority");
149+
expect(logCtx.requestedServiceTier).toBe("flex");
150+
});
151+
152+
test("clears the caller tier after an unsupported route strips it", async () => {
153+
const { outboundBody, logCtx } = await driveResponses({
154+
provider: {
155+
...supportedResponsesProvider(),
156+
supportsServiceTier: false,
157+
},
158+
callerTier: "priority",
159+
});
160+
expect(outboundBody).not.toHaveProperty("service_tier");
161+
expect(logCtx.requestedServiceTier).toBeUndefined();
162+
});
163+
});
164+
165+
describe("FastWire characterization: rawBody observation point", () => {
166+
test("fastMode injection is visible in parsed._rawBody when the adapter is invoked", async () => {
167+
let adapterRawBody: Record<string, unknown> | undefined;
168+
const adapterSpy = spyOn(adapterResolveModule, "resolveAdapter").mockReturnValue({
169+
name: "openai-responses",
170+
passthrough: true,
171+
async buildRequest(parsed) {
172+
adapterRawBody = JSON.parse(JSON.stringify(parsed._rawBody)) as Record<string, unknown>;
173+
throw new Error("fastwire rawBody observation complete");
174+
},
175+
} as ReturnType<typeof adapterResolveModule.resolveAdapter>);
176+
177+
try {
178+
const providerName = "fastwire-raw-body";
179+
const config = {
180+
port: 0,
181+
defaultProvider: providerName,
182+
fastMode: true,
183+
providers: { [providerName]: supportedResponsesProvider() },
184+
} as OcxConfig;
185+
const request = new Request("http://localhost/v1/responses", {
186+
method: "POST",
187+
headers: { "content-type": "application/json" },
188+
body: JSON.stringify({
189+
model: `${providerName}/model`,
190+
input: "ping",
191+
stream: true,
192+
service_tier: "flex",
193+
}),
194+
});
195+
196+
await expect(handleResponses(request, config, { model: "", provider: "" }, {}))
197+
.rejects.toThrow("fastwire rawBody observation complete");
198+
// A1 intentionally moves fast-mode injection out of `_rawBody`; update this
199+
// characterization when that observation point changes.
200+
expect(adapterRawBody?.service_tier).toBe("priority");
201+
} finally {
202+
adapterSpy.mockRestore();
203+
}
204+
});
205+
});
206+
207+
describe("FastWire characterization: known bugs", () => {
208+
test("characterization (known bug): native chat passthrough ignores exact-model false", () => {
209+
const request = buildOpenAIChatPassthroughRequest(
210+
{
211+
adapter: "openai-chat",
212+
baseUrl: "https://native-chat.example.test/v1",
213+
authMode: "key",
214+
apiKey: "sk-test",
215+
supportsServiceTier: true,
216+
chatServiceTier: true,
217+
modelSupportsServiceTier: { model: false },
218+
},
219+
{
220+
model: "model",
221+
messages: [{ role: "user", content: "ping" }],
222+
service_tier: "flex",
223+
},
224+
"model",
225+
false,
226+
);
227+
const body = JSON.parse(request.body) as Record<string, unknown>;
228+
expect(body.service_tier).toBe("flex");
229+
});
230+
231+
test("characterization (known bug): chat-to-responses conversion drops service_tier", () => {
232+
const body = chatCompletionsToResponsesBody({
233+
model: "model",
234+
messages: [{ role: "user", content: "ping" }],
235+
service_tier: "priority",
236+
});
237+
expect(body).not.toHaveProperty("service_tier");
238+
});
239+
});

0 commit comments

Comments
 (0)