Skip to content

Commit f507447

Browse files
vincentkocvincentkoc
committed
fix(openai): quarantine unreadable projected tools
1 parent 5d6216a commit f507447

8 files changed

Lines changed: 443 additions & 32 deletions

‎src/agents/openai-tool-schema.test.ts‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,11 @@
11
import { beforeEach, describe, expect, it } from "vitest";
22
import {
33
clearOpenAIToolSchemaCacheForTest,
4+
findOpenAIStrictToolSchemaDiagnostics,
45
isStrictOpenAIJsonSchemaCompatible,
56
normalizeStrictOpenAIJsonSchema,
67
resolveOpenAIStrictToolFlagForInventory,
8+
snapshotOpenAIToolProjectionInputs,
79
} from "./openai-tool-schema.js";
810

911
describe("OpenAI strict tool schema normalization", () => {
@@ -91,4 +93,39 @@ describe("OpenAI strict tool schema normalization", () => {
9193
}),
9294
).toBe(third);
9395
});
96+
97+
it("records unreadable tool projection fields as diagnostics instead of throwing", () => {
98+
const unreadableTool = {
99+
name: "bad_schema",
100+
get parameters(): unknown {
101+
throw new Error("boom");
102+
},
103+
};
104+
const healthyTool = {
105+
name: "healthy_lookup",
106+
description: "Lookup",
107+
parameters: {},
108+
};
109+
110+
const snapshot = snapshotOpenAIToolProjectionInputs([unreadableTool, healthyTool]);
111+
112+
expect(snapshot.tools.map((tool) => tool.name)).toEqual(["healthy_lookup"]);
113+
expect(snapshot.diagnostics).toEqual([
114+
{
115+
toolIndex: 0,
116+
toolName: "bad_schema",
117+
violations: ["bad_schema.unreadable: boom"],
118+
},
119+
]);
120+
expect(findOpenAIStrictToolSchemaDiagnostics([unreadableTool, healthyTool])).toEqual([
121+
{
122+
toolIndex: 0,
123+
toolName: "bad_schema",
124+
violations: ["bad_schema.unreadable: boom"],
125+
},
126+
]);
127+
expect(resolveOpenAIStrictToolFlagForInventory([unreadableTool, healthyTool], true)).toBe(
128+
false,
129+
);
130+
});
94131
});

‎src/agents/openai-tool-schema.ts‎

Lines changed: 75 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -9,9 +9,19 @@ type ToolSchemaCompatInput = {
99

1010
type ToolWithParameters = {
1111
name?: unknown;
12+
description?: unknown;
1213
parameters: unknown;
1314
};
1415

16+
type OpenAIToolProjectionSnapshot = {
17+
toolIndex: number;
18+
name: string;
19+
description?: string;
20+
parameters: unknown;
21+
};
22+
23+
type OpenAIToolProjectionSnapshotDiagnostic = OpenAIStrictToolSchemaDiagnostic;
24+
1525
const MAX_STRICT_SCHEMA_CACHE_ENTRIES_PER_SCHEMA = 8;
1626
let strictOpenAISchemaCache = new WeakMap<object, Array<{ key: string; value: unknown }>>();
1727

@@ -163,25 +173,69 @@ type OpenAIStrictToolSchemaDiagnostic = {
163173
violations: string[];
164174
};
165175

176+
export function snapshotOpenAIToolProjectionInputs(tools: readonly ToolWithParameters[]): {
177+
tools: OpenAIToolProjectionSnapshot[];
178+
diagnostics: OpenAIToolProjectionSnapshotDiagnostic[];
179+
} {
180+
const snapshots: OpenAIToolProjectionSnapshot[] = [];
181+
const diagnostics: OpenAIToolProjectionSnapshotDiagnostic[] = [];
182+
tools.forEach((tool, toolIndex) => {
183+
const fallbackPath = `tool[${toolIndex}]`;
184+
let toolName: string | undefined;
185+
try {
186+
if (!tool || typeof tool !== "object") {
187+
diagnostics.push({ toolIndex, violations: [`${fallbackPath}.tool`] });
188+
return;
189+
}
190+
const name = Reflect.get(tool, "name");
191+
if (typeof name !== "string" || name.length === 0) {
192+
diagnostics.push({ toolIndex, violations: [`${fallbackPath}.name`] });
193+
return;
194+
}
195+
toolName = name;
196+
const description = Reflect.get(tool, "description");
197+
const parameters = Reflect.get(tool, "parameters");
198+
snapshots.push({
199+
toolIndex,
200+
name,
201+
...(typeof description === "string" ? { description } : {}),
202+
parameters,
203+
});
204+
} catch (error) {
205+
const message = error instanceof Error ? error.message : String(error);
206+
diagnostics.push({
207+
toolIndex,
208+
...(toolName ? { toolName } : {}),
209+
violations: [`${toolName ?? fallbackPath}.unreadable: ${message}`],
210+
});
211+
}
212+
});
213+
return { tools: snapshots, diagnostics };
214+
}
215+
166216
export function findOpenAIStrictToolSchemaDiagnostics(
167217
tools: readonly ToolWithParameters[],
168218
): OpenAIStrictToolSchemaDiagnostic[] {
169-
return tools.flatMap((tool, toolIndex) => {
170-
const violations = findStrictOpenAIJsonSchemaViolations(
171-
normalizeStrictOpenAIJsonSchema(tool.parameters),
172-
`${typeof tool.name === "string" && tool.name ? tool.name : `tool[${toolIndex}]`}.parameters`,
173-
);
174-
if (violations.length === 0) {
175-
return [];
176-
}
177-
return [
178-
{
179-
toolIndex,
180-
...(typeof tool.name === "string" && tool.name ? { toolName: tool.name } : {}),
181-
violations,
182-
},
183-
];
184-
});
219+
const snapshot = snapshotOpenAIToolProjectionInputs(tools);
220+
return [
221+
...snapshot.diagnostics,
222+
...snapshot.tools.flatMap((tool) => {
223+
const violations = findStrictOpenAIJsonSchemaViolations(
224+
normalizeStrictOpenAIJsonSchema(tool.parameters),
225+
`${tool.name}.parameters`,
226+
);
227+
if (violations.length === 0) {
228+
return [];
229+
}
230+
return [
231+
{
232+
toolIndex: tool.toolIndex,
233+
toolName: tool.name,
234+
violations,
235+
},
236+
];
237+
}),
238+
];
185239
}
186240

187241
function isStrictOpenAIJsonSchemaCompatibleRecursive(schema: unknown): boolean {
@@ -304,5 +358,9 @@ export function resolveOpenAIStrictToolFlagForInventory(
304358
if (strict !== true) {
305359
return strict === false ? false : undefined;
306360
}
307-
return tools.every((tool) => isStrictOpenAIJsonSchemaCompatible(tool.parameters));
361+
const snapshot = snapshotOpenAIToolProjectionInputs(tools);
362+
return (
363+
snapshot.diagnostics.length === 0 &&
364+
snapshot.tools.every((tool) => isStrictOpenAIJsonSchemaCompatible(tool.parameters))
365+
);
308366
}

‎src/agents/openai-transport-stream.test.ts‎

Lines changed: 201 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4573,6 +4573,207 @@ describe("openai transport stream", () => {
45734573
expect(params.tools?.[0]).not.toHaveProperty("strict");
45744574
});
45754575

4576+
it("skips unreadable tools before building OpenAI Responses params", () => {
4577+
const unreadableNameTool = {
4578+
get name(): string {
4579+
throw new Error("boom name");
4580+
},
4581+
description: "Bad",
4582+
parameters: {},
4583+
};
4584+
const unreadableParametersTool = {
4585+
name: "bad_parameters",
4586+
description: "Bad",
4587+
get parameters(): unknown {
4588+
throw new Error("boom parameters");
4589+
},
4590+
};
4591+
const params = buildOpenAIResponsesParams(
4592+
{
4593+
id: "gpt-5.5",
4594+
name: "GPT-5.5",
4595+
api: "openai-responses",
4596+
provider: "openai",
4597+
baseUrl: "https://api.openai.com/v1",
4598+
reasoning: true,
4599+
input: ["text"],
4600+
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
4601+
contextWindow: 200000,
4602+
maxTokens: 8192,
4603+
} satisfies Model<"openai-responses">,
4604+
{
4605+
systemPrompt: "system",
4606+
messages: [],
4607+
tools: [
4608+
unreadableNameTool,
4609+
{
4610+
name: "healthy_lookup",
4611+
description: "Lookup",
4612+
parameters: {},
4613+
},
4614+
unreadableParametersTool,
4615+
],
4616+
} as never,
4617+
undefined,
4618+
) as { tools?: Array<{ name?: string }> };
4619+
4620+
expect(params.tools?.map((tool) => tool.name)).toEqual(["healthy_lookup"]);
4621+
});
4622+
4623+
it("drops pinned OpenAI Responses tool choice when that tool is unreadable", () => {
4624+
const unreadablePinnedTool = {
4625+
name: "bad_parameters",
4626+
description: "Bad",
4627+
get parameters(): unknown {
4628+
throw new Error("boom parameters");
4629+
},
4630+
};
4631+
const params = buildOpenAIResponsesParams(
4632+
{
4633+
id: "gpt-5.5",
4634+
name: "GPT-5.5",
4635+
api: "openai-responses",
4636+
provider: "openai",
4637+
baseUrl: "https://api.openai.com/v1",
4638+
reasoning: true,
4639+
input: ["text"],
4640+
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
4641+
contextWindow: 200000,
4642+
maxTokens: 8192,
4643+
} satisfies Model<"openai-responses">,
4644+
{
4645+
systemPrompt: "system",
4646+
messages: [],
4647+
tools: [
4648+
unreadablePinnedTool,
4649+
{
4650+
name: "healthy_lookup",
4651+
description: "Lookup",
4652+
parameters: {},
4653+
},
4654+
],
4655+
} as never,
4656+
{
4657+
toolChoice: { type: "function", name: "bad_parameters" },
4658+
},
4659+
) as { tools?: Array<{ name?: string }>; tool_choice?: unknown };
4660+
4661+
expect(params.tools?.map((tool) => tool.name)).toEqual(["healthy_lookup"]);
4662+
expect(params.tool_choice).toBeUndefined();
4663+
});
4664+
4665+
it("drops forced OpenAI Responses tool choice when every tool is unreadable", () => {
4666+
const unreadableTool = {
4667+
name: "bad_parameters",
4668+
description: "Bad",
4669+
get parameters(): unknown {
4670+
throw new Error("boom parameters");
4671+
},
4672+
};
4673+
const params = buildOpenAIResponsesParams(
4674+
{
4675+
id: "gpt-5.5",
4676+
name: "GPT-5.5",
4677+
api: "openai-responses",
4678+
provider: "openai",
4679+
baseUrl: "https://api.openai.com/v1",
4680+
reasoning: true,
4681+
input: ["text"],
4682+
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
4683+
contextWindow: 200000,
4684+
maxTokens: 8192,
4685+
} satisfies Model<"openai-responses">,
4686+
{
4687+
systemPrompt: "system",
4688+
messages: [],
4689+
tools: [unreadableTool],
4690+
} as never,
4691+
{
4692+
toolChoice: "required",
4693+
},
4694+
) as { tools?: unknown[]; tool_choice?: unknown };
4695+
4696+
expect(params.tools).toBeUndefined();
4697+
expect(params.tool_choice).toBeUndefined();
4698+
});
4699+
4700+
it("drops forced OpenAI completions tool choice when every tool is unreadable", () => {
4701+
const unreadableTool = {
4702+
name: "bad_parameters",
4703+
description: "Bad",
4704+
get parameters(): unknown {
4705+
throw new Error("boom parameters");
4706+
},
4707+
};
4708+
const params = buildOpenAICompletionsParams(
4709+
{
4710+
id: "gpt-5.5",
4711+
name: "GPT-5.5",
4712+
api: "openai-completions",
4713+
provider: "openai",
4714+
baseUrl: "https://api.openai.com/v1",
4715+
reasoning: false,
4716+
input: ["text"],
4717+
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
4718+
contextWindow: 200000,
4719+
maxTokens: 8192,
4720+
} satisfies Model<"openai-completions">,
4721+
{
4722+
systemPrompt: "system",
4723+
messages: [],
4724+
tools: [unreadableTool],
4725+
} as never,
4726+
{
4727+
toolChoice: "required",
4728+
},
4729+
) as { tools?: unknown[]; tool_choice?: unknown };
4730+
4731+
expect(params.tools).toBeUndefined();
4732+
expect(params.tool_choice).toBeUndefined();
4733+
});
4734+
4735+
it("drops pinned OpenAI completions tool choice when that tool is unreadable", () => {
4736+
const unreadablePinnedTool = {
4737+
name: "bad_parameters",
4738+
description: "Bad",
4739+
get parameters(): unknown {
4740+
throw new Error("boom parameters");
4741+
},
4742+
};
4743+
const params = buildOpenAICompletionsParams(
4744+
{
4745+
id: "gpt-5.5",
4746+
name: "GPT-5.5",
4747+
api: "openai-completions",
4748+
provider: "openai",
4749+
baseUrl: "https://api.openai.com/v1",
4750+
reasoning: false,
4751+
input: ["text"],
4752+
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
4753+
contextWindow: 200000,
4754+
maxTokens: 8192,
4755+
} satisfies Model<"openai-completions">,
4756+
{
4757+
systemPrompt: "system",
4758+
messages: [],
4759+
tools: [
4760+
unreadablePinnedTool,
4761+
{
4762+
name: "healthy_lookup",
4763+
description: "Lookup",
4764+
parameters: {},
4765+
},
4766+
],
4767+
} as never,
4768+
{
4769+
toolChoice: { type: "function", function: { name: "bad_parameters" } },
4770+
},
4771+
) as { tools?: Array<{ function?: { name?: string } }>; tool_choice?: unknown };
4772+
4773+
expect(params.tools?.map((tool) => tool.function?.name)).toEqual(["healthy_lookup"]);
4774+
expect(params.tool_choice).toBeUndefined();
4775+
});
4776+
45764777
it("still normalizes responses tool parameters when strict is omitted", () => {
45774778
const params = buildOpenAIResponsesParams(
45784779
{

0 commit comments

Comments
 (0)