Skip to content

Commit 215b068

Browse files
anandgupta42claude
andcommitted
feat: reject an under-asked answer_commit once, naming the missing item
- A `fields` commit with fewer fields than the items the session's latest user request joins ("the X and its Y") is rejected once per request, quoting the clause that names each extra item - An unchanged recommit is accepted; never rejected near the time budget - Lexical only: no queries, no model calls, no schema change Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent 49b5615 commit 215b068

4 files changed

Lines changed: 403 additions & 0 deletions

File tree

Lines changed: 133 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,133 @@
1+
import type { Tool } from "../../tool/tool"
2+
import { RunBudget } from "../run-budget"
3+
import { effortOverride, toolsDisabled } from "../session/run-budget-effort"
4+
5+
/** Identifies this check's rejections in the current turn's history. */
6+
export const CONJUNCTION_REJECTION_MARKER = "answer_commit conjunction check:"
7+
8+
// Narrow connectives strongly suggest another requested item.
9+
const NARROW = [
10+
/\band (?:its|their|his|her|the corresponding|the associated)\b/gi,
11+
/\balong with\b/gi,
12+
/\bas well as\b/gi,
13+
/\btogether with\b/gi,
14+
/\band also\b/gi,
15+
/\bin addition to\b/gi,
16+
/\band (?:report|list|show|include|provide|return|give|compute|calculate|identify|find)\b/gi,
17+
]
18+
// Articles and comma-separated conjunctions catch less explicit item lists.
19+
const BROAD_EXTRA = [/,\s+and\s+(?!then\b)/gi, /\band (?:the|a|an|which|what|how|whether)\b/gi]
20+
// Ranges and paired expressions do not introduce requested items.
21+
const EXCL = /(?:between|both|either|from)\s+[^.,;]{0,60}?\band\b/gi
22+
// Constraints qualify an item rather than request another one.
23+
const CONSTRAINT =
24+
/\band (?:only|also consider|consider|excluding|exclude|limit|using|ensure|ensuring|where|for|in|on|at|from|with|by|to|if)\b/gi
25+
/** Words of the request quoted after each connective, to name the item it introduces. */
26+
const JOIN_WORDS = 8
27+
28+
/**
29+
* Lexical estimate of the items a request asks for: 1 plus the most item connectives in any sentence
30+
* of its first paragraph, ignoring ranges, constraint clauses and rounding instructions. `joins` quotes
31+
* each counted connective with the words that follow it, naming the extra items.
32+
*/
33+
export function requestedItems(request: string): { count: number; joins: string[] } {
34+
const para = request
35+
.trim()
36+
.split(/\n\s*\n|\n#/)[0]!
37+
.replace(EXCL, " ")
38+
let joins: string[] = []
39+
for (const sentence of para.split(/(?<=[.?!])\s+/)) {
40+
if (/\b(decimal|precision|round(?:ed|ing)?)\b/i.test(sentence) && !/\band\b/i.test(sentence)) continue
41+
const text = sentence.replace(CONSTRAINT, " ")
42+
const spans: number[] = []
43+
for (const pattern of [...NARROW, ...BROAD_EXTRA]) {
44+
for (const match of text.matchAll(pattern)) {
45+
if (!spans.some((start) => Math.abs(match.index - start) < 4)) spans.push(match.index)
46+
}
47+
}
48+
if (spans.length > joins.length)
49+
joins = spans
50+
.sort((a, b) => a - b)
51+
.map((start) =>
52+
text
53+
.slice(start)
54+
.replace(/^,\s*/, "")
55+
.split(/[.,;?!]/)[0]!
56+
.split(/\s+/)
57+
.filter(Boolean)
58+
.slice(0, JOIN_WORDS)
59+
.join(" "),
60+
)
61+
}
62+
return { count: 1 + joins.length, joins }
63+
}
64+
65+
type Part = {
66+
type?: string
67+
tool?: string
68+
text?: string
69+
synthetic?: boolean
70+
state?: { status?: string; error?: unknown }
71+
}
72+
73+
/** Text of the latest user message, without synthetic parts such as time-budget reminders. */
74+
function latestRequest(messages: Tool.Context["messages"]): string {
75+
const latest = messages.findLast((message) => message.info.role === "user")
76+
return ((latest?.parts ?? []) as Part[])
77+
.filter((part) => part.type === "text" && !part.synthetic)
78+
.map((part) => part.text ?? "")
79+
.join("\n\n")
80+
}
81+
82+
/** Bounds the in-process record of advised requests; the oldest entries are dropped first. */
83+
const MAX_ADVISED_REQUESTS = 1_000
84+
/**
85+
* Session and request text already advised in this process. Reserved synchronously before the rejection
86+
* returns, so parallel commits in one step and a compaction replay of the same request are not rejected again.
87+
*/
88+
const advised = new Set<string>()
89+
90+
/** Near the run's time budget a rejected commit may not get another turn, so it is never rejected. */
91+
function budgetEnding(sessionID: string): boolean {
92+
const deadline = RunBudget.queryDeadline(sessionID)
93+
return toolsDisabled(sessionID) || effortOverride(sessionID) || (deadline !== undefined && Date.now() >= deadline)
94+
}
95+
96+
/**
97+
* Rejects a fields commit with fewer fields than the items the latest user request joins. Fires at most
98+
* once per session and request, so the next commit, changed or not, is accepted, and never once the run
99+
* nears its time budget. Purely lexical: no queries or model calls.
100+
*/
101+
export function conjunctionRejection(
102+
args: { fields?: { name: string }[] },
103+
messages: Tool.Context["messages"],
104+
sessionID: string,
105+
): string | undefined {
106+
if (!args.fields || budgetEnding(sessionID)) return undefined
107+
const text = latestRequest(messages)
108+
const { count, joins } = requestedItems(text)
109+
if (count <= args.fields.length) return undefined
110+
const key = `${sessionID}\u0000${text}`
111+
if (advised.has(key)) return undefined
112+
const latest = messages.findLastIndex((message) => message.info.role === "user")
113+
// History covers a resumed session whose in-process record is gone.
114+
if (
115+
messages
116+
.slice(latest)
117+
.some((message) =>
118+
(message.parts as Part[]).some(
119+
(part) =>
120+
part.type === "tool" &&
121+
part.tool === "answer_commit" &&
122+
part.state?.status === "error" &&
123+
String(part.state.error).includes(CONJUNCTION_REJECTION_MARKER),
124+
),
125+
)
126+
)
127+
return undefined
128+
advised.add(key)
129+
if (advised.size > MAX_ADVISED_REQUESTS) advised.delete(advised.values().next().value!)
130+
const names = args.fields.map((field) => field.name)
131+
const fields = `${names.length} field${names.length === 1 ? "" : "s"}`
132+
return `${CONJUNCTION_REJECTION_MARKER} the request appears to ask for ${count} items, but this commit has ${fields} (${JSON.stringify(names)}). The request also asks for ${joins.map((join) => `"${join}"`).join(", ")}. Call answer_commit again with each requested item as its own field, named after it. If the request asks for fewer items, recommit unchanged; the next commit is accepted.`
133+
}

‎packages/opencode/src/altimate/tools/answer-commit.ts‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,9 @@ import { Decimal } from "decimal.js"
33
import { Effect, Option, Schema, SchemaIssue } from "effect"
44
import type { JSONSchema7 } from "@ai-sdk/provider"
55
import { Tool } from "../../tool/tool"
6+
// altimate_change start — one-time conjunction review
7+
import { conjunctionRejection } from "./answer-commit-conjunction"
8+
// altimate_change end
69

710
/** Maximum rendered answer length, including Markdown and basis, before the model echoes it. */
811
export const MAX_RENDERED_ANSWER_CHARACTERS = 60_000
@@ -304,6 +307,19 @@ export const AnswerCommitTool = Tool.define(
304307
return yield* Effect.die(new Error("answer_commit is only available to the analyst agent"))
305308
yield* ctx.ask({ permission: "answer_commit", patterns: ["*"], always: ["*"], metadata: {} })
306309
const output = renderAnswer(args)
310+
// altimate_change start — one-time conjunction review of under-asked commits
311+
const rejection = conjunctionRejection(args, ctx.messages ?? [], ctx.sessionID)
312+
if (rejection) {
313+
const rejected: Tool.ExecuteResult = {
314+
title: "Answer not committed",
315+
// failure_kind separates this advisory from real commit failures in telemetry.
316+
metadata: { readings: args.readings ?? [], truncated: false, failure_kind: "conjunction_check" },
317+
output: rejection,
318+
status: "error",
319+
}
320+
return rejected
321+
}
322+
// altimate_change end
307323
return {
308324
title: "Answer committed",
309325
// The processor caps model-facing output but preserves metadata for the final echo.
Lines changed: 184 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,184 @@
1+
import { describe, expect, test } from "bun:test"
2+
import { RunBudget } from "../../../src/altimate/run-budget"
3+
import { begin, downgradeEffort } from "../../../src/altimate/session/run-budget-effort"
4+
import { AnswerCommitTool, renderAnswer } from "../../../src/altimate/tools/answer-commit"
5+
import { CONJUNCTION_REJECTION_MARKER, requestedItems } from "../../../src/altimate/tools/answer-commit-conjunction"
6+
import type { Tool } from "../../../src/tool/tool"
7+
import { initTool } from "../tool-fixture"
8+
9+
const singles = [
10+
"Which repository has the most stars?",
11+
"How many films were released between 2010 and 2015?",
12+
"Which artist sold the most Rock and Roll albums?",
13+
"Which artist sold the most rock and roll albums?",
14+
"Which customer from Trinidad and Tobago spent the most?",
15+
"List the top product, and only consider active stores.",
16+
]
17+
const compound = "Which repository and its most copied file?"
18+
const basis = "Selected repositories at repository grain from repositories.name."
19+
const payload = { basis, fields: [{ name: "repository", value: "repo-a" }] }
20+
const complete = { basis, fields: [...payload.fields, { name: "most copied file", value: "main.swift" }] }
21+
function user(text: string) {
22+
return { info: { role: "user" }, parts: [{ type: "text", text }] }
23+
}
24+
function rejectedCommit(error: string) {
25+
return {
26+
info: { role: "assistant" },
27+
parts: [{ type: "tool", tool: "answer_commit", state: { status: "error", error } }],
28+
}
29+
}
30+
// The rejection is recorded per session, so each call gets a fresh session unless one is shared.
31+
function context(messages: unknown[] = [], sessionID: string = crypto.randomUUID()) {
32+
return {
33+
sessionID,
34+
messageID: "test-message",
35+
agent: "analyst",
36+
abort: new AbortController().signal,
37+
messages: messages as Tool.Context["messages"],
38+
ask: () => {},
39+
}
40+
}
41+
42+
describe("requestedItems", () => {
43+
test.each(singles)("counts one item: %s", (request) => {
44+
expect(requestedItems(request)).toEqual({ count: 1, joins: [] })
45+
})
46+
test("counts and quotes the connective that introduces each extra item", () => {
47+
expect(requestedItems(compound)).toEqual({ count: 2, joins: ["and its most copied file"] })
48+
expect(requestedItems("Identify the repository and the most copied file.").joins).toEqual([
49+
"and the most copied file",
50+
])
51+
expect(requestedItems("Which repository contains Swift code and its most copied file?").count).toBe(2)
52+
})
53+
test("takes the sentence with the most items and ignores later paragraphs", () => {
54+
const request =
55+
"Find the store with the most sales, along with its city, and the manager's name. Round to 2 decimals.\n\nThe schema has a stores table and a sales table."
56+
expect(requestedItems(request)).toEqual({ count: 3, joins: ["along with its city", "and the manager's name"] })
57+
})
58+
})
59+
60+
describe("answer_commit conjunction check", () => {
61+
test("rejects an under-asked fields commit once, naming the missing item; an unchanged recommit is accepted", async () => {
62+
const tool = await initTool(AnswerCommitTool)
63+
const session = crypto.randomUUID()
64+
const first = await tool.execute(payload, context([user(compound)], session))
65+
expect(first.status).toBe("error")
66+
expect(first.title).toBe("Answer not committed")
67+
expect(first.output.startsWith(CONJUNCTION_REJECTION_MARKER)).toBe(true)
68+
expect(first.output).toContain('"and its most copied file"')
69+
expect(first.output).toContain('["repository"]')
70+
expect(first.metadata).not.toHaveProperty("committed_text")
71+
expect(first.metadata).toHaveProperty("failure_kind", "conjunction_check")
72+
const second = await tool.execute(payload, context([user(compound), rejectedCommit(first.output)], session))
73+
expect(second.status).not.toBe("error")
74+
expect(second.output).toBe(renderAnswer(payload))
75+
expect(second.metadata).toHaveProperty("committed_text", renderAnswer(payload))
76+
})
77+
test("the rejection in history alone allows the recommit, as after a restart", async () => {
78+
const tool = await initTool(AnswerCommitTool)
79+
const first = await tool.execute(payload, context([user(compound)]))
80+
const resumed = await tool.execute(payload, context([user(compound), rejectedCommit(first.output)]))
81+
expect(resumed.output).toBe(renderAnswer(payload))
82+
// A rejection before the latest request does not cover the new request.
83+
const next = "Which author and their most cited paper?"
84+
const later = await tool.execute(payload, context([user(compound), rejectedCommit(first.output), user(next)]))
85+
expect(later.status).toBe("error")
86+
})
87+
test("rejects once per session and request without history of the rejection", async () => {
88+
const tool = await initTool(AnswerCommitTool)
89+
const session = crypto.randomUUID()
90+
// Parallel commits in one step share the same history snapshot.
91+
const [first, second] = await Promise.all([
92+
tool.execute(payload, context([user(compound)], session)),
93+
tool.execute(payload, context([user(compound)], session)),
94+
])
95+
expect([first.status, second.status].filter((status) => status === "error")).toHaveLength(1)
96+
// A compaction replay repeats the request as a new user message without the earlier rejection.
97+
expect((await tool.execute(payload, context([user(compound)], session))).output).toBe(renderAnswer(payload))
98+
expect((await tool.execute(payload, context([user(compound)]))).status).toBe("error")
99+
})
100+
test("answer_only commits are checked too", async () => {
101+
const tool = await initTool(AnswerCommitTool)
102+
const result = await tool.execute(
103+
{ ...payload, answer_only: true },
104+
context([user("Identify the repository and the most copied file.")]),
105+
)
106+
expect(result.status).toBe("error")
107+
})
108+
test("two fields for a three-item request are rejected; three are accepted", async () => {
109+
const tool = await initTool(AnswerCommitTool)
110+
const request = "Find the store with the most sales, along with its city, and the manager's name."
111+
expect((await tool.execute(complete, context([user(request)]))).status).toBe("error")
112+
const three = { basis, fields: [...complete.fields, { name: "manager", value: "Ana" }] }
113+
expect((await tool.execute(three, context([user(request)]))).output).toBe(renderAnswer(three))
114+
})
115+
test.each(singles)("accepts a single-item request: %s", async (request) => {
116+
const tool = await initTool(AnswerCommitTool)
117+
expect((await tool.execute(payload, context([user(request)]))).output).toBe(renderAnswer(payload))
118+
})
119+
test("accepts complete fields, rows, unresolved and commits without a user message", async () => {
120+
const tool = await initTool(AnswerCommitTool)
121+
expect((await tool.execute(complete, context([user(compound)]))).output).toBe(renderAnswer(complete))
122+
const rows = { basis, rows: { columns: ["repository"], values: [["repo-a"]] } }
123+
expect((await tool.execute(rows, context([user(compound)]))).output).toBe(renderAnswer(rows))
124+
const unresolved = { basis, unresolved: { reason: "No matching records." } }
125+
expect((await tool.execute(unresolved, context([user(compound)]))).output).toBe(renderAnswer(unresolved))
126+
expect((await tool.execute(payload, context())).output).toBe(renderAnswer(payload))
127+
})
128+
})
129+
130+
describe("near the time budget a commit is never rejected", () => {
131+
test("after the query deadline", async () => {
132+
const tool = await initTool(AnswerCommitTool)
133+
const session = crypto.randomUUID()
134+
const release = RunBudget.registerQueryDeadline(session, Date.now() - 1)
135+
try {
136+
expect((await tool.execute(payload, context([user(compound)], session))).output).toBe(renderAnswer(payload))
137+
} finally {
138+
release()
139+
}
140+
// The same session is checked again once no deadline applies.
141+
expect((await tool.execute(payload, context([user(compound)], session))).status).toBe("error")
142+
})
143+
test("once effort is downgraded at the soft deadline", async () => {
144+
const tool = await initTool(AnswerCommitTool)
145+
const session = crypto.randomUUID()
146+
const release = begin(session)
147+
downgradeEffort(session)
148+
try {
149+
expect((await tool.execute(payload, context([user(compound)], session))).output).toBe(renderAnswer(payload))
150+
} finally {
151+
release()
152+
}
153+
})
154+
test("after the synthetic time-budget reminder", async () => {
155+
const tool = await initTool(AnswerCommitTool)
156+
// The reminder joins two items, so it would be rejected if synthetic text were read as the request.
157+
const text = "Answer now, and the basis in one sentence."
158+
expect(requestedItems(text).count).toBe(2)
159+
const reminder = { info: { role: "user" }, parts: [{ type: "text", text, synthetic: true }] }
160+
expect((await tool.execute(payload, context([user(compound), reminder]))).output).toBe(renderAnswer(payload))
161+
})
162+
})
163+
164+
// Native replies echo the committed text: the basis paragraph sits above the `**Answer:**` line.
165+
describe("committed answer layout", () => {
166+
const answerLine = (text: string) => text.slice(text.indexOf("**Answer:**"))
167+
test("a complete fields commit keeps the basis off the answer line", () => {
168+
const text = renderAnswer(complete)
169+
expect(text).toBe(`${basis}\n\n**Answer:** repository: repo-a; most copied file: main.swift`)
170+
expect(answerLine(text)).not.toContain(basis)
171+
})
172+
test("a rows commit keeps the basis off the answer", () => {
173+
const text = renderAnswer({ basis, rows: { columns: ["repository"], values: [["repo-a"]] } })
174+
expect(text.startsWith(`${basis}\n\n**Answer:**\n`)).toBe(true)
175+
expect(answerLine(text)).not.toContain(basis)
176+
})
177+
test("an accepted under-asked commit keeps the basis in the reply", async () => {
178+
const tool = await initTool(AnswerCommitTool)
179+
const withItem = { ...payload, basis: `${basis} Its most copied file is main.swift.` }
180+
const first = await tool.execute(withItem, context([user(compound)]))
181+
const accepted = await tool.execute(withItem, context([user(compound), rejectedCommit(first.output)]))
182+
expect(accepted.metadata.committed_text).toBe(`${withItem.basis}\n\n**Answer:** repository: repo-a`)
183+
})
184+
})

0 commit comments

Comments
 (0)