Skip to content

Commit fd0bc41

Browse files
anandgupta42claude
andcommitted
fix(analyst): bound basis by length, state the precision range, add per-column row precision
Live sessions produced two avoidable `answer_commit` validation errors: the "at most two sentences" check split a basis on "U.S.", and a `precision` of 14 was rejected without the bound being visible to the model. `basis` is now bounded by `MAX_BASIS_CHARACTERS` (400) instead of a punctuation-based sentence count, and the description and validation message state the precision range (0 to 10). `rows` gains an optional `precision` map keyed by column name so numeric cells, including integers, render with fixed decimals; strings, nulls and unlisted columns are unchanged. Live sessions showed a numeric `5` rendered where the equivalent `"5.00"` was expected. Tests: two-sentence basis with "U.S." and "e.g." accepted; 400 characters accepted, 401 rejected naming the bound; precision range stated in the description and in the error for 14; row integers with precision 2 render `5.00`, without precision `5`, numeric strings keep trailing zeros, precision applies independently per column; the Anthropic draft-2020-12 schema test still passes. Co-Authored-By: Claude Fable 5.1 <[email protected]>
1 parent 0b52f4f commit fd0bc41

2 files changed

Lines changed: 146 additions & 66 deletions

File tree

‎packages/opencode/src/altimate/tools/answer-commit.ts‎

Lines changed: 24 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import { Tool } from "../../tool/tool"
55

66
/** Maximum rendered answer length, including Markdown and basis, before the model echoes it. */
77
export const MAX_RENDERED_ANSWER_CHARACTERS = 60_000
8+
export const MAX_BASIS_CHARACTERS = 400
89

910
const renderedLengthGuidance = `Rendered answer text must not exceed ${MAX_RENDERED_ANSWER_CHARACTERS.toLocaleString("en-US")} characters; aggregate, narrow the result, or write the full table to a file and commit the summary.`
1011

@@ -32,15 +33,18 @@ const number = z
3233
"Numeric values must have at most 15 significant digits and integers must be safe; pass the exact value as a string",
3334
)
3435

36+
const precisionRange = "precision must be an integer from 0 to 10 (inclusive)"
37+
const precision = z.number({ error: precisionRange }).int(precisionRange).min(0, precisionRange).max(10, precisionRange)
38+
3539
const field = z
3640
.object({
3741
name: z.string().describe("Field label; use an empty name for an unnamed value"),
3842
value: z
3943
.union([z.string(), number], { error: "Each field value must be one non-empty string or number" })
4044
.refine((value) => typeof value !== "string" || value.trim().length > 0, "Field value must be non-empty"),
4145
units: optional(z.string()),
42-
precision: optional(z.number().int().min(0).max(10)).describe(
43-
"Decimal places for numeric values; integers stay integers",
46+
precision: optional(precision).describe(
47+
"Decimal places for numeric values: integer from 0 to 10 (inclusive); integers stay integers",
4448
),
4549
})
4650
.strict()
@@ -54,6 +58,9 @@ const rows = z
5458
.describe(
5559
"Concrete result rows; when the request names N, the row count must equal N, even with a tie at the top-N boundary",
5660
),
61+
precision: optional(z.record(z.string(), precision)).describe(
62+
"Display decimal places by column name: integer from 0 to 10 (inclusive). Numeric cells use fixed decimals, including integers; strings and nulls are unchanged. Omitted columns keep their default formatting.",
63+
),
5764
note: optional(z.string()).describe(
5865
"Brief result note, including any tie at the top-N boundary; omitted when answer_only is true",
5966
),
@@ -71,8 +78,7 @@ const rows = z
7178
})
7279
})
7380

74-
const basisRequired =
75-
"basis is always required: supply one or two sentences naming population, grain, source object/column, and caveats"
81+
const basisRequired = `basis is always required: supply one or two sentences, at most ${MAX_BASIS_CHARACTERS} characters, naming population, grain, source object/column, and caveats`
7682

7783
const parameters = z
7884
.object({
@@ -100,17 +106,10 @@ const parameters = z
100106
),
101107
basis: z
102108
.string({ error: basisRequired })
109+
.max(MAX_BASIS_CHARACTERS, `basis must contain at most ${MAX_BASIS_CHARACTERS} characters`)
103110
.refine((value) => value.trim().length > 0, basisRequired)
104-
.refine(
105-
(value) =>
106-
value
107-
.replace(/\b(?:e\.g|i\.e|etc|Dr|vs|approx)\./gi, (word) => word.replaceAll(".", ""))
108-
.split(/[.?!](?:\s+(?=\S)|$)/u)
109-
.filter((part) => part.trim()).length <= 2,
110-
"basis must contain at most 2 sentences",
111-
)
112111
.describe(
113-
"basis is always required: one or two sentences naming population, grain, the source object/column that supplies the requested entity or value, and caveats; never rendered with answer_only",
112+
`basis is always required: one or two sentences, at most ${MAX_BASIS_CHARACTERS} characters, naming population, grain, the source object/column that supplies the requested entity or value, and caveats; never rendered with answer_only`,
114113
),
115114
answer_only: optional(z.boolean()),
116115
readings: optional(
@@ -195,11 +194,19 @@ export function renderAnswer(payload: unknown): string {
195194
return limitRenderedAnswer(args.answer_only ? values : `${basis}**Answer:** ${values}`)
196195
}
197196
if (args.rows) {
197+
const { columns, values, precision } = args.rows
198198
const row = (cells: (string | number | null)[]) => `| ${cells.map(tableCell).join(" | ")} |`
199199
const table = [
200-
row(args.rows.columns),
201-
`|${args.rows.columns.map(() => "---").join("|")}|`,
202-
...args.rows.values.map(row),
200+
row(columns),
201+
`|${columns.map(() => "---").join("|")}|`,
202+
...values.map((cells) =>
203+
row(
204+
cells.map((value, index) => {
205+
const places = precision && Object.hasOwn(precision, columns[index]!) ? precision[columns[index]!] : undefined
206+
return typeof value === "number" && places !== undefined ? value.toFixed(places) : value
207+
}),
208+
),
209+
),
203210
].join("\n")
204211
return limitRenderedAnswer(
205212
args.answer_only ? table : `${basis}**Answer:**\n${table}${args.rows.note ? `\n${args.rows.note}` : ""}`,
@@ -221,7 +228,7 @@ export const AnswerCommitTool = Tool.define(
221228
"answer_commit",
222229
Effect.succeed({
223230
description:
224-
`Deterministically finalize a data-question answer. Call this as the LAST tool call, once the answer is decided and all queries are finished. After it returns, reply with exactly the returned text and nothing else. Supply exactly one of fields, rows, or unresolved. Numbers are accepted up to 15 significant digits; any decimal that must be reproduced exactly (more digits, trailing zeros, or currency strings) must be passed as a string. Always supply \`basis\`; it is never shown when the request asks for only the answer. Supply one or two sentences naming population, grain, the source object/column that supplies the requested entity or value, and caveats. Resolve policy eligibility separately from metric and entity attribution: a rule about a related object does not change which object supplies the requested entity. Say in \`basis\` which source object and column supply it. When the request names N, rows.values must contain exactly N concrete rows, even with a tie at the top-N boundary; note the tie briefly in rows.note. Use answer_only when the request asks for only the answer or a specific format: the reply is exactly the value(s) or table, with no basis, field names, note, or answer label. An unresolved answer retains its unable-to-determine status. Omit unused optional properties. ${renderedLengthGuidance} This tool is not available for JSON-schema requests; use StructuredOutput instead.`,
231+
`Deterministically finalize a data-question answer. Call this as the LAST tool call, once the answer is decided and all queries are finished. After it returns, reply with exactly the returned text and nothing else. Supply exactly one of fields, rows, or unresolved. Numbers are accepted up to 15 significant digits; any decimal that must be reproduced exactly (more digits, trailing zeros, or currency strings) must be passed as a string. For fields.precision and each rows.precision value, use an integer from 0 to 10 (inclusive). Set rows.precision by column name to display numeric cells with fixed decimal places, including integers; strings are never changed. Always supply \`basis\`; it is never shown when the request asks for only the answer. Supply one or two sentences, at most ${MAX_BASIS_CHARACTERS} characters, naming population, grain, the source object/column that supplies the requested entity or value, and caveats. Resolve policy eligibility separately from metric and entity attribution: a rule about a related object does not change which object supplies the requested entity. Say in \`basis\` which source object and column supply it. When the request names N, rows.values must contain exactly N concrete rows, even with a tie at the top-N boundary; note the tie briefly in rows.note. Use answer_only when the request asks for only the answer or a specific format: the reply is exactly the value(s) or table, with no basis, field names, note, or answer label. An unresolved answer retains its unable-to-determine status. Omit unused optional properties. ${renderedLengthGuidance} This tool is not available for JSON-schema requests; use StructuredOutput instead.`,
225232
parameters: Parameters,
226233
formatValidationError: (error: unknown) => (Schema.isSchemaError(error) ? error.message : String(error)),
227234
// Preserve the existing wire schema; Zod and the SDK use different recursive JSON Schema types.

‎packages/opencode/test/altimate/tools/answer-commit.test.ts‎

Lines changed: 122 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -94,15 +94,15 @@ describe("answer_commit required basis", () => {
9494
"Always supply `basis`; it is never shown when the request asks for only the answer",
9595
)
9696
expect(tool.description).toContain(
97-
"one or two sentences naming population, grain, the source object/column that supplies the requested entity or value, and caveats",
97+
"one or two sentences, at most 400 characters, naming population, grain, the source object/column that supplies the requested entity or value, and caveats",
9898
)
9999
expect(tool.description).toContain(
100100
"Resolve policy eligibility separately from metric and entity attribution: a rule about a related object does not change which object supplies the requested entity. Say in `basis` which source object and column supply it.",
101101
)
102102
expect(structuredClone(tool.jsonSchema?.properties?.basis)).toMatchObject({
103103
type: "string",
104104
description: expect.stringMatching(
105-
/always required.*population.*grain.*source object\/column.*caveats.*never rendered with answer_only/i,
105+
/always required.*one or two sentences, at most 400 characters.*population.*grain.*source object\/column.*caveats.*never rendered with answer_only/i,
106106
),
107107
})
108108
expect(tool.jsonSchema?.required).toContain("basis")
@@ -243,6 +243,93 @@ describe("answer_commit numeric safety", () => {
243243
)
244244
})
245245

246+
describe("answer_commit display precision", () => {
247+
test("description and field and row schemas state the allowed precision range", async () => {
248+
const tool = await initTool(AnswerCommit.AnswerCommitTool)
249+
const range = /integer from 0 to 10 \(inclusive\)/i
250+
expect(tool.description).toMatch(range)
251+
expect(structuredClone(tool.jsonSchema?.properties?.fields)).toMatchObject({
252+
anyOf: expect.arrayContaining([
253+
expect.objectContaining({
254+
items: expect.objectContaining({
255+
properties: expect.objectContaining({
256+
precision: expect.objectContaining({ description: expect.stringMatching(range) }),
257+
}),
258+
}),
259+
}),
260+
]),
261+
})
262+
expect(structuredClone(tool.jsonSchema?.properties?.rows)).toMatchObject({
263+
anyOf: expect.arrayContaining([
264+
expect.objectContaining({
265+
properties: expect.objectContaining({
266+
precision: expect.objectContaining({ description: expect.stringMatching(range) }),
267+
}),
268+
}),
269+
]),
270+
})
271+
})
272+
273+
test.each([-1, 11, 14, 1.5, "2", true])("invalid precision %j names the allowed range", async (precision) => {
274+
const tool = await initTool(AnswerCommit.AnswerCommitTool)
275+
for (const shape of [
276+
{ fields: [{ name: "", value: 1.5, precision }] },
277+
{ rows: { columns: ["Value"], values: [[5]], precision: { Value: precision } } },
278+
]) {
279+
const payload = { basis: recordBasis, ...shape }
280+
const error = /precision.*integer from 0 to 10 \(inclusive\)/i
281+
expect(() => AnswerCommit.renderAnswer(payload)).toThrow(error)
282+
await rejects(tool.execute(payload, context), error)
283+
}
284+
})
285+
286+
test.each([
287+
["numeric integers use fixed decimals", 5, 2, "5.00"],
288+
["without precision uses the shortest numeric form", 5, undefined, "5"],
289+
["numeric strings retain trailing zeros", "5.00", 0, "5.00"],
290+
["numeric strings retain leading zeros", "001.20", 10, "001.20"],
291+
["text is unchanged", "unchanged", 2, "unchanged"],
292+
["null is unchanged", null, 2, "null"],
293+
["zero precision rounds numeric cells", 1.6, 0, "2"],
294+
["maximum precision includes ten decimals", 5, 10, "5.0000000000"],
295+
] as [string, string | number | null, number | undefined, string][])(
296+
"rows %s",
297+
async (_name, value, precision, rendered) => {
298+
const payload = {
299+
basis: recordBasis,
300+
rows: {
301+
columns: ["Value"],
302+
values: [[value]],
303+
...(precision === undefined ? {} : { precision: { Value: precision } }),
304+
},
305+
}
306+
const expected = `${recordBasis}\n\n**Answer:**\n| Value |\n|---|\n| ${rendered} |`
307+
expect(AnswerCommit.renderAnswer(payload)).toBe(expected)
308+
const tool = await initTool(AnswerCommit.AnswerCommitTool)
309+
expect((await tool.execute(payload, context)).output).toBe(expected)
310+
},
311+
)
312+
313+
test("row precision applies independently by column name", async () => {
314+
const payload = {
315+
basis: recordBasis,
316+
answer_only: true,
317+
rows: {
318+
columns: ["Quantity", "Rate", "Unspecified"],
319+
values: [
320+
[5, 1.234, 5],
321+
["5.00", null, "001.20"],
322+
],
323+
precision: { Rate: 1, Quantity: 2 },
324+
},
325+
}
326+
const expected = "| Quantity | Rate | Unspecified |\n|---|---|---|\n| 5.00 | 1.2 | 5 |\n| 5.00 | null | 001.20 |"
327+
expect(AnswerCommit.renderAnswer(payload)).toBe(expected)
328+
const tool = await initTool(AnswerCommit.AnswerCommitTool)
329+
expect((await tool.execute(payload, context)).output).toBe(expected)
330+
})
331+
})
332+
246333
describe("answer_commit rendered length limit", () => {
247334
const cap = 60_000
248335
const error = /60,000 characters.*aggregate.*narrow.*write the full table to a file.*commit the summary/i
@@ -579,52 +666,44 @@ describe("renderAnswer strict-schema null fillers", () => {
579666
})
580667
})
581668

582-
describe("renderAnswer basis sentence boundaries", () => {
583-
test.each(["e.g.", "i.e.", "etc.", "Dr.", "vs.", "approx."])("does not count %s as a sentence end", (word) => {
584-
const basis = `Used ${word} Completed orders. Count is 2.`
585-
expect(AnswerCommit.renderAnswer({ basis, fields: [{ name: "", value: 2 }] })).toBe(`${basis}\n\n**Answer:** 2`)
586-
})
587-
588-
test("does not count decimals as sentence ends", () => {
589-
const basis = "The mean is 3.5 and the median is 2.5. Count is 2."
590-
expect(AnswerCommit.renderAnswer({ basis, fields: [{ name: "", value: 2 }] })).toBe(`${basis}\n\n**Answer:** 2`)
591-
})
592-
593-
test("punctuation followed by lowercase text counts as a sentence boundary", () => {
594-
const basis = "Count? maybe! still checking. Count is 2."
595-
expect(() => AnswerCommit.renderAnswer({ basis, fields: [{ name: "", value: 2 }] })).toThrow(/basis.*2 sentences/i)
596-
})
597-
598-
test("caps lowercase sentences separated by newlines", () => {
599-
expect(() => AnswerCommit.renderAnswer({ basis: "one.\ntwo.\nthree.", fields: [{ name: "", value: 2 }] })).toThrow(
600-
/basis.*2 sentences/i,
601-
)
602-
})
603-
604-
test.each(["2 orders", '"two orders"'])("counts sentence boundaries before %s", (next) => {
605-
const basis = `First. ${next}. Third.`
606-
expect(() => AnswerCommit.renderAnswer({ basis, fields: [{ name: "", value: 2 }] })).toThrow(/basis.*2 sentences/i)
669+
describe("renderAnswer basis length bound", () => {
670+
test("accepts a two-sentence basis containing U.S. and e.g.", async () => {
671+
const basis = "Counted U.S. orders at order grain from orders.id. Excluded cancelled orders, e.g. pending refunds."
672+
const payload = { basis, fields: [{ name: "", value: 2 }] }
673+
const expected = `${basis}\n\n**Answer:** 2`
674+
expect(AnswerCommit.renderAnswer(payload)).toBe(expected)
675+
const tool = await initTool(AnswerCommit.AnswerCommitTool)
676+
expect((await tool.execute(payload, context)).output).toBe(expected)
607677
})
608678

609-
test.each([".", "?", "!"])("caps sentences separated by %s and uppercase text", (punctuation) => {
610-
const basis = `First${punctuation} Second${punctuation} Third${punctuation}`
611-
expect(() => AnswerCommit.renderAnswer({ basis, fields: [{ name: "", value: 2 }] })).toThrow(/basis.*2 sentences/i)
612-
})
679+
test.each(["First. Second? Third!", "one.\ntwo.\nthree.", "One.\n\nTwo.\n\nThree."])(
680+
"accepts punctuation within the length bound: %s",
681+
(basis) => {
682+
expect(AnswerCommit.renderAnswer({ basis, fields: [{ name: "", value: 2 }] })).toBe(`${basis}\n\n**Answer:** 2`)
683+
},
684+
)
613685

614-
test("abbreviations and decimals do not hide a third sentence", () => {
615-
expect(() =>
616-
AnswerCommit.renderAnswer({
617-
basis: "Used e.g. Orders. Mean is 3.5. Count is 2.",
618-
fields: [{ name: "", value: 2 }],
619-
}),
620-
).toThrow(/basis.*2 sentences/i)
686+
test("accepts exactly 400 basis characters", async () => {
687+
expect(answerCommitModule.MAX_BASIS_CHARACTERS).toBe(400)
688+
const basis = "x".repeat(400)
689+
const payload = { basis, fields: [{ name: "", value: 2 }] }
690+
const expected = `${basis}\n\n**Answer:** 2`
691+
expect(AnswerCommit.renderAnswer(payload)).toBe(expected)
692+
const tool = await initTool(AnswerCommit.AnswerCommitTool)
693+
expect((await tool.execute(payload, context)).output).toBe(expected)
621694
})
622695

623-
test("paragraph breaks count as sentence boundaries", () => {
624-
expect(() =>
625-
AnswerCommit.renderAnswer({ basis: "One.\n\nTwo.\n\nThree.", fields: [{ name: "", value: 2 }] }),
626-
).toThrow(/basis.*2 sentences/i)
627-
})
696+
test.each(["x".repeat(401), ` ${"x".repeat(399)} `])(
697+
"rejects 401 raw basis characters with the named bound",
698+
async (basis) => {
699+
const tool = await initTool(AnswerCommit.AnswerCommitTool)
700+
for (const answer_only of [false, true]) {
701+
const payload = { basis, answer_only, fields: [{ name: "", value: 2 }] }
702+
expect(() => AnswerCommit.renderAnswer(payload)).toThrow(/basis.*400 characters/i)
703+
await rejects(tool.execute(payload, context), /basis.*400 characters/i)
704+
}
705+
},
706+
)
628707
})
629708

630709
describe("renderAnswer structural validation", () => {
@@ -671,12 +750,6 @@ describe("renderAnswer structural validation", () => {
671750
["ragged rows", { rows: { columns: ["A", "B"], values: [[1, 2], [3]] } }, /rows.*1.*2.*cells/i],
672751
["no columns", { rows: { columns: [], values: [[1]] } }, /rows.*columns/i],
673752
["no rows", { rows: { columns: ["A"], values: [] } }, /rows.*values/i],
674-
["three sentences", { basis: "First. Second? Third!", fields: [{ name: "", value: 1 }] }, /basis.*2 sentences/i],
675-
[
676-
"unpunctuated third sentence",
677-
{ basis: "First. Second! Third", fields: [{ name: "", value: 1 }] },
678-
/basis.*2 sentences/i,
679-
],
680753
["negative precision", { fields: [{ name: "", value: 1.5, precision: -1 }] }, /precision/i],
681754
["excessive precision", { fields: [{ name: "", value: 1.5, precision: 11 }] }, /precision/i],
682755
["fractional precision", { fields: [{ name: "", value: 1.5, precision: 1.5 }] }, /precision/i],

0 commit comments

Comments
 (0)