|
| 1 | +import { describe, expect, test } from "bun:test" |
| 2 | +import { RunBudget } from "../../../src/altimate/run-budget" |
| 3 | +import { begin, downgradeEffort } from "../../../src/altimate/session/run-budget-effort" |
| 4 | +import { AnswerCommitTool, renderAnswer } from "../../../src/altimate/tools/answer-commit" |
| 5 | +import { CONJUNCTION_REJECTION_MARKER, requestedItems } from "../../../src/altimate/tools/answer-commit-conjunction" |
| 6 | +import type { Tool } from "../../../src/tool/tool" |
| 7 | +import { initTool } from "../tool-fixture" |
| 8 | + |
| 9 | +const singles = [ |
| 10 | + "Which repository has the most stars?", |
| 11 | + "How many films were released between 2010 and 2015?", |
| 12 | + "Which artist sold the most Rock and Roll albums?", |
| 13 | + "Which artist sold the most rock and roll albums?", |
| 14 | + "Which customer from Trinidad and Tobago spent the most?", |
| 15 | + "List the top product, and only consider active stores.", |
| 16 | +] |
| 17 | +const compound = "Which repository and its most copied file?" |
| 18 | +const basis = "Selected repositories at repository grain from repositories.name." |
| 19 | +const payload = { basis, fields: [{ name: "repository", value: "repo-a" }] } |
| 20 | +const complete = { basis, fields: [...payload.fields, { name: "most copied file", value: "main.swift" }] } |
| 21 | +function user(text: string) { |
| 22 | + return { info: { role: "user" }, parts: [{ type: "text", text }] } |
| 23 | +} |
| 24 | +function rejectedCommit(error: string) { |
| 25 | + return { |
| 26 | + info: { role: "assistant" }, |
| 27 | + parts: [{ type: "tool", tool: "answer_commit", state: { status: "error", error } }], |
| 28 | + } |
| 29 | +} |
| 30 | +// The rejection is recorded per session, so each call gets a fresh session unless one is shared. |
| 31 | +function context(messages: unknown[] = [], sessionID: string = crypto.randomUUID()) { |
| 32 | + return { |
| 33 | + sessionID, |
| 34 | + messageID: "test-message", |
| 35 | + agent: "analyst", |
| 36 | + abort: new AbortController().signal, |
| 37 | + messages: messages as Tool.Context["messages"], |
| 38 | + ask: () => {}, |
| 39 | + } |
| 40 | +} |
| 41 | + |
| 42 | +describe("requestedItems", () => { |
| 43 | + test.each(singles)("counts one item: %s", (request) => { |
| 44 | + expect(requestedItems(request)).toEqual({ count: 1, joins: [] }) |
| 45 | + }) |
| 46 | + test("counts and quotes the connective that introduces each extra item", () => { |
| 47 | + expect(requestedItems(compound)).toEqual({ count: 2, joins: ["and its most copied file"] }) |
| 48 | + expect(requestedItems("Identify the repository and the most copied file.").joins).toEqual([ |
| 49 | + "and the most copied file", |
| 50 | + ]) |
| 51 | + expect(requestedItems("Which repository contains Swift code and its most copied file?").count).toBe(2) |
| 52 | + }) |
| 53 | + test("takes the sentence with the most items and ignores later paragraphs", () => { |
| 54 | + const request = |
| 55 | + "Find the store with the most sales, along with its city, and the manager's name. Round to 2 decimals.\n\nThe schema has a stores table and a sales table." |
| 56 | + expect(requestedItems(request)).toEqual({ count: 3, joins: ["along with its city", "and the manager's name"] }) |
| 57 | + }) |
| 58 | +}) |
| 59 | + |
| 60 | +describe("answer_commit conjunction check", () => { |
| 61 | + test("rejects an under-asked fields commit once, naming the missing item; an unchanged recommit is accepted", async () => { |
| 62 | + const tool = await initTool(AnswerCommitTool) |
| 63 | + const session = crypto.randomUUID() |
| 64 | + const first = await tool.execute(payload, context([user(compound)], session)) |
| 65 | + expect(first.status).toBe("error") |
| 66 | + expect(first.title).toBe("Answer not committed") |
| 67 | + expect(first.output.startsWith(CONJUNCTION_REJECTION_MARKER)).toBe(true) |
| 68 | + expect(first.output).toContain('"and its most copied file"') |
| 69 | + expect(first.output).toContain('["repository"]') |
| 70 | + expect(first.metadata).not.toHaveProperty("committed_text") |
| 71 | + expect(first.metadata).toHaveProperty("failure_kind", "conjunction_check") |
| 72 | + const second = await tool.execute(payload, context([user(compound), rejectedCommit(first.output)], session)) |
| 73 | + expect(second.status).not.toBe("error") |
| 74 | + expect(second.output).toBe(renderAnswer(payload)) |
| 75 | + expect(second.metadata).toHaveProperty("committed_text", renderAnswer(payload)) |
| 76 | + }) |
| 77 | + test("the rejection in history alone allows the recommit, as after a restart", async () => { |
| 78 | + const tool = await initTool(AnswerCommitTool) |
| 79 | + const first = await tool.execute(payload, context([user(compound)])) |
| 80 | + const resumed = await tool.execute(payload, context([user(compound), rejectedCommit(first.output)])) |
| 81 | + expect(resumed.output).toBe(renderAnswer(payload)) |
| 82 | + // A rejection before the latest request does not cover the new request. |
| 83 | + const next = "Which author and their most cited paper?" |
| 84 | + const later = await tool.execute(payload, context([user(compound), rejectedCommit(first.output), user(next)])) |
| 85 | + expect(later.status).toBe("error") |
| 86 | + }) |
| 87 | + test("rejects once per session and request without history of the rejection", async () => { |
| 88 | + const tool = await initTool(AnswerCommitTool) |
| 89 | + const session = crypto.randomUUID() |
| 90 | + // Parallel commits in one step share the same history snapshot. |
| 91 | + const [first, second] = await Promise.all([ |
| 92 | + tool.execute(payload, context([user(compound)], session)), |
| 93 | + tool.execute(payload, context([user(compound)], session)), |
| 94 | + ]) |
| 95 | + expect([first.status, second.status].filter((status) => status === "error")).toHaveLength(1) |
| 96 | + // A compaction replay repeats the request as a new user message without the earlier rejection. |
| 97 | + expect((await tool.execute(payload, context([user(compound)], session))).output).toBe(renderAnswer(payload)) |
| 98 | + expect((await tool.execute(payload, context([user(compound)]))).status).toBe("error") |
| 99 | + }) |
| 100 | + test("answer_only commits are checked too", async () => { |
| 101 | + const tool = await initTool(AnswerCommitTool) |
| 102 | + const result = await tool.execute( |
| 103 | + { ...payload, answer_only: true }, |
| 104 | + context([user("Identify the repository and the most copied file.")]), |
| 105 | + ) |
| 106 | + expect(result.status).toBe("error") |
| 107 | + }) |
| 108 | + test("two fields for a three-item request are rejected; three are accepted", async () => { |
| 109 | + const tool = await initTool(AnswerCommitTool) |
| 110 | + const request = "Find the store with the most sales, along with its city, and the manager's name." |
| 111 | + expect((await tool.execute(complete, context([user(request)]))).status).toBe("error") |
| 112 | + const three = { basis, fields: [...complete.fields, { name: "manager", value: "Ana" }] } |
| 113 | + expect((await tool.execute(three, context([user(request)]))).output).toBe(renderAnswer(three)) |
| 114 | + }) |
| 115 | + test.each(singles)("accepts a single-item request: %s", async (request) => { |
| 116 | + const tool = await initTool(AnswerCommitTool) |
| 117 | + expect((await tool.execute(payload, context([user(request)]))).output).toBe(renderAnswer(payload)) |
| 118 | + }) |
| 119 | + test("accepts complete fields, rows, unresolved and commits without a user message", async () => { |
| 120 | + const tool = await initTool(AnswerCommitTool) |
| 121 | + expect((await tool.execute(complete, context([user(compound)]))).output).toBe(renderAnswer(complete)) |
| 122 | + const rows = { basis, rows: { columns: ["repository"], values: [["repo-a"]] } } |
| 123 | + expect((await tool.execute(rows, context([user(compound)]))).output).toBe(renderAnswer(rows)) |
| 124 | + const unresolved = { basis, unresolved: { reason: "No matching records." } } |
| 125 | + expect((await tool.execute(unresolved, context([user(compound)]))).output).toBe(renderAnswer(unresolved)) |
| 126 | + expect((await tool.execute(payload, context())).output).toBe(renderAnswer(payload)) |
| 127 | + }) |
| 128 | +}) |
| 129 | + |
| 130 | +describe("near the time budget a commit is never rejected", () => { |
| 131 | + test("after the query deadline", async () => { |
| 132 | + const tool = await initTool(AnswerCommitTool) |
| 133 | + const session = crypto.randomUUID() |
| 134 | + const release = RunBudget.registerQueryDeadline(session, Date.now() - 1) |
| 135 | + try { |
| 136 | + expect((await tool.execute(payload, context([user(compound)], session))).output).toBe(renderAnswer(payload)) |
| 137 | + } finally { |
| 138 | + release() |
| 139 | + } |
| 140 | + // The same session is checked again once no deadline applies. |
| 141 | + expect((await tool.execute(payload, context([user(compound)], session))).status).toBe("error") |
| 142 | + }) |
| 143 | + test("once effort is downgraded at the soft deadline", async () => { |
| 144 | + const tool = await initTool(AnswerCommitTool) |
| 145 | + const session = crypto.randomUUID() |
| 146 | + const release = begin(session) |
| 147 | + downgradeEffort(session) |
| 148 | + try { |
| 149 | + expect((await tool.execute(payload, context([user(compound)], session))).output).toBe(renderAnswer(payload)) |
| 150 | + } finally { |
| 151 | + release() |
| 152 | + } |
| 153 | + }) |
| 154 | + test("after the synthetic time-budget reminder", async () => { |
| 155 | + const tool = await initTool(AnswerCommitTool) |
| 156 | + // The reminder joins two items, so it would be rejected if synthetic text were read as the request. |
| 157 | + const text = "Answer now, and the basis in one sentence." |
| 158 | + expect(requestedItems(text).count).toBe(2) |
| 159 | + const reminder = { info: { role: "user" }, parts: [{ type: "text", text, synthetic: true }] } |
| 160 | + expect((await tool.execute(payload, context([user(compound), reminder]))).output).toBe(renderAnswer(payload)) |
| 161 | + }) |
| 162 | +}) |
| 163 | + |
| 164 | +// Native replies echo the committed text: the basis paragraph sits above the `**Answer:**` line. |
| 165 | +describe("committed answer layout", () => { |
| 166 | + const answerLine = (text: string) => text.slice(text.indexOf("**Answer:**")) |
| 167 | + test("a complete fields commit keeps the basis off the answer line", () => { |
| 168 | + const text = renderAnswer(complete) |
| 169 | + expect(text).toBe(`${basis}\n\n**Answer:** repository: repo-a; most copied file: main.swift`) |
| 170 | + expect(answerLine(text)).not.toContain(basis) |
| 171 | + }) |
| 172 | + test("a rows commit keeps the basis off the answer", () => { |
| 173 | + const text = renderAnswer({ basis, rows: { columns: ["repository"], values: [["repo-a"]] } }) |
| 174 | + expect(text.startsWith(`${basis}\n\n**Answer:**\n`)).toBe(true) |
| 175 | + expect(answerLine(text)).not.toContain(basis) |
| 176 | + }) |
| 177 | + test("an accepted under-asked commit keeps the basis in the reply", async () => { |
| 178 | + const tool = await initTool(AnswerCommitTool) |
| 179 | + const withItem = { ...payload, basis: `${basis} Its most copied file is main.swift.` } |
| 180 | + const first = await tool.execute(withItem, context([user(compound)])) |
| 181 | + const accepted = await tool.execute(withItem, context([user(compound), rejectedCommit(first.output)])) |
| 182 | + expect(accepted.metadata.committed_text).toBe(`${withItem.basis}\n\n**Answer:** repository: repo-a`) |
| 183 | + }) |
| 184 | +}) |
0 commit comments