Skip to content

Commit a077066

Browse files
faitnoAlexedelauna
authored
fix(prompts): prevent models from hallucinating line numbers in apply_diff (#641)
* fix(prompts): prevent models from hallucinating line numbers in apply_diff * fix(prompts): correct line-number examples and add regression tests --------- Co-authored-by: Alex <[email protected]> Co-authored-by: Elliott de Launay <[email protected]>
1 parent 299bddf commit a077066

2 files changed

Lines changed: 84 additions & 3 deletions

File tree

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
import type OpenAI from "openai"
2+
import { apply_diff } from "../apply_diff"
3+
4+
// Helper type to access function tools
5+
type FunctionTool = OpenAI.Chat.ChatCompletionTool & { type: "function" }
6+
7+
// Helper to get function definition from tool
8+
const getFunctionDef = (tool: OpenAI.Chat.ChatCompletionTool) => (tool as FunctionTool).function
9+
10+
// OpenAI types the parameter schema as FunctionParameters (Record<string, unknown>).
11+
// Narrow it with runtime checks instead of assertions so the spec tracks schema changes.
12+
function asRecord(value: unknown): Record<string, unknown> {
13+
if (typeof value !== "object" || value === null) {
14+
throw new Error("expected an object value in the apply_diff parameter schema")
15+
}
16+
return value as Record<string, unknown>
17+
}
18+
19+
const getDiffDescription = (): string => {
20+
const parameters = asRecord(getFunctionDef(apply_diff).parameters)
21+
const properties = asRecord(parameters["properties"])
22+
const diff = asRecord(properties["diff"])
23+
const description = diff["description"]
24+
if (typeof description !== "string") {
25+
throw new Error("diff parameter must declare a string description")
26+
}
27+
return description
28+
}
29+
30+
const getRequiredParameters = (): unknown[] => {
31+
const parameters = asRecord(getFunctionDef(apply_diff).parameters)
32+
const required = parameters["required"]
33+
if (!Array.isArray(required)) {
34+
throw new Error("apply_diff parameters must declare a required array")
35+
}
36+
return required
37+
}
38+
39+
describe("apply_diff tool description", () => {
40+
const description = getFunctionDef(apply_diff).description
41+
42+
describe("read_file line-number format", () => {
43+
it("should document line numbers using the actual read_file output format (number, spaces, pipe)", () => {
44+
// The real format comes from addLineNumbers() in extract-text.ts: "N | content"
45+
expect(description).toContain("'1 | '")
46+
expect(description).toContain("'2 | '")
47+
})
48+
49+
it("should not describe read_file line numbers with a colon suffix", () => {
50+
// Regression guard: read_file never outputs "N: " prefixes (extract-text.ts addLineNumbers),
51+
// and the server-side auto-strip in multi-search-replace.ts only keys on the "N | " pattern.
52+
expect(description).not.toContain("'1: '")
53+
expect(description).not.toContain("'2: '")
54+
})
55+
56+
it("should instruct the model to strip line numbers from the SEARCH block", () => {
57+
expect(description).toContain("MUST STRIP all line numbers from the 'SEARCH' block")
58+
})
59+
})
60+
61+
describe("diff parameter description", () => {
62+
const diffDescription = getDiffDescription()
63+
64+
it("should use the actual read_file output format in the line-number prohibition bullet", () => {
65+
expect(diffDescription).toContain("'137 | '")
66+
expect(diffDescription).not.toContain("'137: '")
67+
})
68+
69+
it("should keep the template placeholder free of line numbers", () => {
70+
expect(diffDescription).toContain("[exact content to find WITHOUT line numbers]")
71+
})
72+
})
73+
74+
describe("schema", () => {
75+
it("should require path and diff parameters", () => {
76+
expect(getRequiredParameters()).toEqual(["path", "diff"])
77+
})
78+
})
79+
})

‎src/core/prompts/tools/native-tools/apply_diff.ts‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,20 +1,22 @@
11
import type OpenAI from "openai"
22

3-
const APPLY_DIFF_DESCRIPTION = `Apply precise, targeted modifications to an existing file using one or more search/replace blocks. This tool is for surgical edits only; the 'SEARCH' block must exactly match the existing content, including whitespace and indentation. To make multiple targeted changes, provide multiple SEARCH/REPLACE blocks in the 'diff' parameter. Use the 'read_file' tool first if you are not confident in the exact content to search for.`
3+
const APPLY_DIFF_DESCRIPTION = `Apply precise, targeted modifications to an existing file using one or more search/replace blocks. This tool is for surgical edits only; the 'SEARCH' block must exactly match the existing content, including whitespace and indentation. To make multiple targeted changes, provide multiple SEARCH/REPLACE blocks in the 'diff' parameter. Use the 'read_file' tool first if you are not confident in the exact content to search for.
4+
CRITICAL WARNING: The 'read_file' tool outputs line numbers (e.g. '1 | ', '2 | ') for your convenience, but these line numbers DO NOT exist in the actual file! You MUST STRIP all line numbers from the 'SEARCH' block. Your 'SEARCH' block must match the RAW file content exactly.`
45

56
const DIFF_PARAMETER_DESCRIPTION = `A string containing one or more search/replace blocks defining the changes. The ':start_line:' is strongly recommended and indicates the starting line number of the original content. You must not add a start line for the replacement content. Each block must follow this format:
67
<<<<<<< SEARCH
78
:start_line:[line_number]
89
-------
9-
[exact content to find]
10+
[exact content to find WITHOUT line numbers]
1011
=======
1112
[new content to replace with]
1213
>>>>>>> REPLACE
1314
1415
CRITICAL:
1516
- The ':start_line:[line_number]' header is strongly recommended for accurate matching. When provided, it must follow the exact syntax ':start_line:[integer]' (for example: ':start_line:220'). Do not write headers with shorthand forms like ':220' or variations like ':start_line=220'.
1617
- Copy the exact lines from the source file for a 100% string match including all whitespace, indentation, and newlines.
17-
- Ensure the separator '-------' is on its own line immediately following ':start_line:[line_number]' with a newline.`
18+
- Ensure the separator '-------' is on its own line immediately following ':start_line:[line_number]' with a newline.
19+
- DO NOT include line numbers (like '137 | ') in the [exact content to find] section! It must match the actual file content, which does not have line numbers.`
1820

1921
export const apply_diff = {
2022
type: "function",

0 commit comments

Comments
 (0)