Skip to content

Commit 2cef742

Browse files
committed
fix(provider): preserve output constraints and force corrective tool choice
1 parent 2e438e2 commit 2cef742

4 files changed

Lines changed: 41 additions & 6 deletions

File tree

‎packages/cli/src/session/llm.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ import {
99
type StreamTextResult,
1010
type Tool,
1111
type ToolSet,
12+
type ToolChoice,
1213
tool,
1314
jsonSchema,
1415
} from "ai"
@@ -40,7 +41,7 @@ export namespace LLM {
4041
small?: boolean
4142
tools: Record<string, Tool>
4243
retries?: number
43-
toolChoice?: "auto" | "required" | "none"
44+
toolChoice?: ToolChoice<ToolSet>
4445
structured?: { reject: (input: string) => Promise<string>; checkpoint: () => () => void; repairs: Set<string> }
4546
}
4647

‎packages/cli/src/session/prompt.ts‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -821,7 +821,7 @@ export namespace SessionPrompt {
821821
],
822822
tools,
823823
model,
824-
toolChoice: contract ? "required" : undefined,
824+
toolChoice: contract ? (reminder ? { type: "tool", toolName: "StructuredOutput" } : "required") : undefined,
825825
structured: contract
826826
? {
827827
reject,
@@ -1125,8 +1125,14 @@ export namespace SessionPrompt {
11251125
}): AITool & { id: "StructuredOutput" } {
11261126
const validate = input.validate ?? OutputSchema.compile(input.schema)
11271127
const schema = Object.fromEntries(Object.entries(input.schema).filter(([key]) => key !== "$schema"))
1128+
const canonical = JSON.stringify(input.schema)
1129+
const bytes = Buffer.from(canonical)
1130+
const description =
1131+
bytes.length <= 8 * 1024
1132+
? `Canonical JSON Schema: ${canonical}`
1133+
: `Canonical JSON Schema (truncated; the validator enforces the full schema): ${new TextDecoder().decode(bytes.subarray(0, 8 * 1024), { stream: true })}`
11281134
const result = tool({
1129-
description: STRUCTURED_OUTPUT_DESCRIPTION,
1135+
description: `${STRUCTURED_OUTPUT_DESCRIPTION}\n\n${description}`,
11301136
inputSchema: jsonSchema(input.model ? ProviderTransform.schema(input.model, schema) : schema, {
11311137
validate(value) {
11321138
return validate(value)

‎packages/cli/test/cli/run-output-schema.test.ts‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -337,7 +337,10 @@ for (const json of [true, false]) {
337337
})
338338
expect(result.exit, result.stderr).toBe(0)
339339
expect(result.requests).toHaveLength(2)
340-
expect(result.requests.map((request) => request.tool_choice)).toEqual(["required", "required"])
340+
expect(result.requests.map((request) => request.tool_choice)).toEqual([
341+
"required",
342+
{ type: "function", function: { name: "StructuredOutput" } },
343+
])
341344
expect(JSON.stringify(result.requests[1].messages)).toContain("call StructuredOutput with the final result")
342345
expect(result.result).toBe(JSON.stringify({ result: "accepted" }, null, 2) + "\n")
343346
if (!json) return
@@ -477,7 +480,10 @@ test("schema attach repairs a prose finish through the HTTP session boundary", a
477480
retryCount: 2,
478481
})
479482
expect(result.requests).toHaveLength(2)
480-
expect(result.requests.map((request) => request.tool_choice)).toEqual(["required", "required"])
483+
expect(result.requests.map((request) => request.tool_choice)).toEqual([
484+
"required",
485+
{ type: "function", function: { name: "StructuredOutput" } },
486+
])
481487
expect(result.result).toBe(JSON.stringify({ result: "accepted" }, null, 2) + "\n")
482488
expect(result.events.filter((event) => event.type === "structured_output_rejected")).toMatchObject([
483489
{ attempt: 1, errors: [{ path: "", keyword: "missing", message: "call StructuredOutput with the final result" }] },

‎packages/cli/test/session/output-schema.test.ts‎

Lines changed: 23 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -491,7 +491,8 @@ test("Gemini tool schema is transformed but canonical numeric enum remains autho
491491
test("prose-only finish is a counted corrective attempt before a valid result", async () => {
492492
const result = await run(["prose", '{"result":"repaired"}'], 1)
493493
expect(result.requests).toHaveLength(2)
494-
result.requests.forEach((request) => expect(request.tool_choice).toBe("required"))
494+
expect(result.requests[0].tool_choice).toBe("required")
495+
expect(result.requests[1].tool_choice).toEqual({ type: "function", function: { name: "StructuredOutput" } })
495496
expect(result.info.structured).toEqual({ result: "repaired" })
496497
expect(result.info.error).toBeUndefined()
497498
expect(result.rejected).toEqual([
@@ -678,3 +679,24 @@ test("every parallel rejection is counted after the corrective budget is spent",
678679
])
679680
expect(result.outcomes).toEqual([{ status: "failed", reason: "exhausted", attempts: 3 }])
680681
})
682+
683+
test("provider tool description includes canonical pattern constraints", async () => {
684+
const canonical = {
685+
type: "object",
686+
properties: { result: { type: "string", pattern: "^OK-[0-9]{4}$" } },
687+
required: ["result"],
688+
additionalProperties: false,
689+
}
690+
const result = await run(['{"result":"OK-1234"}'], 0, { schema: canonical, gemini: true })
691+
const tools = result.requests[0].tools as { function: { name: string; description: string } }[]
692+
expect(tools.find((tool) => tool.function.name === "StructuredOutput")?.function.description).toContain(
693+
JSON.stringify(canonical),
694+
)
695+
})
696+
697+
test("tool description caps the canonical schema at 8 KiB and explains truncation", () => {
698+
const canonical = { type: "object", description: "x".repeat(9 * 1024) }
699+
const tool = SessionPrompt.createStructuredOutputTool({ schema: canonical, onSuccess() {} })
700+
expect(tool.description).toContain("truncated; the validator enforces the full schema")
701+
expect(tool.description?.split(": ").at(-1)).toBe(JSON.stringify(canonical).slice(0, 8 * 1024))
702+
})

0 commit comments

Comments
 (0)