Skip to content

Commit 6c993fd

Browse files
committed
fix: don't run tool calls from turns a safety filter stopped
Part of Zoo-Code-Org#1820. When Bedrock's filter stops an Opus 5.5 turn with content_filtered, it cuts the output mid-stream and then closes the open JSON of a streaming tool call, e.g. an execute_command input ending in `…c.specialisation.\"intel-nvid"}`. The truncated call still parses, so Zoo ran it. The quote check then rejected it as "Malformed command: unterminated single quote", and that was all the model learned: it took its own command to be broken and resent it three times, each filtered again. Anthropic's guidance is to treat partial output of a refused turn as incomplete and discard it. So when the turn's stop reason is content_filtered, refusal or guardrail_intervened: - no tool call from that turn runs, native MCP calls included; - each gets a tool_result stating the real stop reason, and that resending the same content is likely to be stopped again; - the chat shows an error row with the stop reason. https://platform.claude.com/docs/en/build-with-claude/refusals-and-fallback#what-a-refusal-looks-like
1 parent 1aae58d commit 6c993fd

23 files changed

Lines changed: 223 additions & 0 deletions
Lines changed: 121 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,121 @@
1+
// npx vitest run core/assistant-message/__tests__/presentAssistantMessage-content-stop.spec.ts
2+
3+
import { presentAssistantMessage } from "../presentAssistantMessage"
4+
5+
const mockExecuteCommandHandle = vi.hoisted(() => vi.fn())
6+
const mockUseMcpToolHandle = vi.hoisted(() => vi.fn())
7+
8+
vi.mock("../../task/Task")
9+
vi.mock("../../tools/validateToolUse", () => ({
10+
validateToolUse: vi.fn(),
11+
isValidToolName: vi.fn(() => true),
12+
}))
13+
vi.mock("../../tools/ExecuteCommandTool", () => ({
14+
executeCommandTool: { handle: mockExecuteCommandHandle },
15+
}))
16+
vi.mock("../../tools/UseMcpToolTool", () => ({
17+
useMcpToolTool: { handle: mockUseMcpToolHandle },
18+
}))
19+
vi.mock("@roo-code/telemetry", () => ({
20+
TelemetryService: {
21+
instance: {
22+
captureToolUsage: vi.fn(),
23+
captureConsecutiveMistakeError: vi.fn(),
24+
},
25+
},
26+
}))
27+
28+
// The truncated call from the #1820 report: Bedrock closed the open JSON string after the filter cut it off.
29+
const truncatedCommand =
30+
"cd /etc/nixos && nix-instantiate --eval --strict -E '\nlet s = import <nixpkgs/nixos> {};\n off = c.specialisation.\"intel-nvid"
31+
32+
type TestToolResult = { type: "tool_result"; tool_use_id: string; content: string; is_error?: boolean }
33+
34+
function makeTask(contentStopReason: string | undefined, block: Record<string, unknown>) {
35+
const userMessageContent: TestToolResult[] = []
36+
return {
37+
taskId: "test-task-id",
38+
instanceId: "test-instance",
39+
abort: false,
40+
presentAssistantMessageLocked: false,
41+
presentAssistantMessageHasPendingUpdates: false,
42+
currentStreamingContentIndex: 0,
43+
assistantMessageContent: [block],
44+
userMessageContent,
45+
didCompleteReadingStream: true,
46+
didRejectTool: false,
47+
didAlreadyUseTool: false,
48+
contentStopReason,
49+
consecutiveMistakeCount: 0,
50+
clineMessages: [],
51+
getTaskMode: vi.fn().mockResolvedValue("code"),
52+
api: { getModel: () => ({ id: "test-model", info: {} }) },
53+
recordToolUsage: vi.fn(),
54+
recordToolError: vi.fn(),
55+
toolRepetitionDetector: { check: vi.fn().mockReturnValue({ allowExecution: true }) },
56+
providerRef: { deref: () => ({ getState: vi.fn().mockResolvedValue({ mode: "code", customModes: [] }) }) },
57+
say: vi.fn().mockResolvedValue(undefined),
58+
ask: vi.fn().mockResolvedValue({ response: "yesButtonClicked" }),
59+
pushToolResultToUserContent: vi.fn((result: TestToolResult) => {
60+
userMessageContent.push(result)
61+
return true
62+
}),
63+
}
64+
}
65+
66+
const executeCommandBlock = {
67+
type: "tool_use",
68+
id: "tooluse_filtered",
69+
name: "execute_command",
70+
params: { command: truncatedCommand },
71+
nativeArgs: { command: truncatedCommand },
72+
partial: false,
73+
}
74+
75+
describe("presentAssistantMessage - turns ended by a content stop", () => {
76+
beforeEach(() => {
77+
mockExecuteCommandHandle.mockReset()
78+
mockUseMcpToolHandle.mockReset()
79+
})
80+
81+
it("does not run a tool call from a content_filtered turn and tells the model why", async () => {
82+
const task = makeTask("content_filtered", executeCommandBlock)
83+
84+
// `as never`: the partial mock only carries the fields presentAssistantMessage reads.
85+
await presentAssistantMessage(task as never)
86+
87+
expect(mockExecuteCommandHandle).not.toHaveBeenCalled()
88+
expect(task.userMessageContent).toHaveLength(1)
89+
const [result] = task.userMessageContent
90+
expect(result).toMatchObject({ type: "tool_result", tool_use_id: "tooluse_filtered", is_error: true })
91+
expect(result.content).toContain("stop reason: content_filtered")
92+
expect(result.content).toContain("was not run")
93+
expect(result.content).not.toContain("Malformed command")
94+
})
95+
96+
it("does not run a native MCP tool call from a refusal turn", async () => {
97+
const task = makeTask("refusal (reasoning_extraction)", {
98+
type: "mcp_tool_use",
99+
id: "tooluse_mcp",
100+
name: "mcp_server_tool",
101+
serverName: "server",
102+
toolName: "tool",
103+
arguments: {},
104+
partial: false,
105+
})
106+
107+
await presentAssistantMessage(task as never)
108+
109+
expect(mockUseMcpToolHandle).not.toHaveBeenCalled()
110+
expect(task.userMessageContent[0]).toMatchObject({ tool_use_id: "tooluse_mcp", is_error: true })
111+
expect(task.userMessageContent[0].content).toContain("refusal (reasoning_extraction)")
112+
})
113+
114+
it("runs the tool call normally when the turn ended without a content stop", async () => {
115+
const task = makeTask(undefined, executeCommandBlock)
116+
117+
await presentAssistantMessage(task as never)
118+
119+
expect(mockExecuteCommandHandle).toHaveBeenCalledTimes(1)
120+
})
121+
})

‎src/core/assistant-message/presentAssistantMessage.ts‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,17 @@ import { codebaseSearchTool } from "../tools/CodebaseSearchTool"
4444
import { formatResponse } from "../prompts/responses"
4545
import { sanitizeToolUseId } from "../../utils/tool-id"
4646

47+
/**
48+
* Tool result for a call from a turn a safety classifier or guardrail ended.
49+
* States the real stop reason, so the model changes course instead of retrying a "malformed" call.
50+
*/
51+
export function contentStopToolResult(toolName: string, stopReason: string): string {
52+
return (
53+
`The provider stopped your response (stop reason: ${stopReason}) while you were writing this ${toolName} call. ` +
54+
`The call was cut off and was not run. Resending the same content is likely to be stopped again.`
55+
)
56+
}
57+
4758
/**
4859
* Maps a raw, potentially model-controlled tool name to a safe analytics key.
4960
* Never returns the raw name unless it is a known static tool, so an
@@ -156,6 +167,20 @@ async function presentAssistantMessageBlock(cline: Task): Promise<void> {
156167
// their original name in API history
157168
const mcpBlock = block as McpToolUse
158169

170+
if (cline.contentStopReason && !mcpBlock.partial) {
171+
if (mcpBlock.id) {
172+
cline.pushToolResultToUserContent({
173+
type: "tool_result",
174+
tool_use_id: sanitizeToolUseId(mcpBlock.id),
175+
content: formatResponse.toolError(
176+
contentStopToolResult(mcpBlock.name, cline.contentStopReason),
177+
),
178+
is_error: true,
179+
})
180+
}
181+
break
182+
}
183+
159184
if (cline.didRejectTool) {
160185
// For native protocol, we must send a tool_result for every tool_use to avoid API errors
161186
const toolCallId = mcpBlock.id
@@ -549,6 +574,16 @@ async function presentAssistantMessageBlock(cline: Task): Promise<void> {
549574
//
550575
// This avoids executing an invalid tool_use block and prevents duplicate/fragmented
551576
// error reporting.
577+
if (cline.contentStopReason && !block.partial) {
578+
cline.pushToolResultToUserContent({
579+
type: "tool_result",
580+
tool_use_id: sanitizeToolUseId(toolCallId),
581+
content: formatResponse.toolError(contentStopToolResult(block.name, cline.contentStopReason)),
582+
is_error: true,
583+
})
584+
break
585+
}
586+
552587
if (!block.partial) {
553588
const customTool = stateExperiments?.customTools ? customToolRegistry.get(block.name) : undefined
554589
const isKnownTool = isValidToolName(String(block.name), stateExperiments)

‎src/core/task/Task.ts‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -102,6 +102,7 @@ import { getWorkspacePath } from "../../utils/path"
102102
import { sanitizeToolUseId } from "../../utils/tool-id"
103103
import { getTaskDirectoryPath } from "../../utils/storage"
104104
import { createRawApiDump } from "../../utils/rawApiDump"
105+
import { findContentStopReason } from "./contentStop"
105106
import { logger } from "../../utils/logging"
106107

107108
// prompts
@@ -614,6 +615,8 @@ export class Task extends EventEmitter<TaskEvents> implements TaskLike {
614615
didRejectTool = false
615616
didAlreadyUseTool = false
616617
didToolFailInCurrentTurn = false
618+
/** Stop reason of a turn ended by a safety classifier or guardrail; its tool calls must not run. */
619+
contentStopReason: string | undefined = undefined
617620
didCompleteReadingStream = false
618621
private _started = false
619622
private _runPromise: Promise<void> | undefined
@@ -3982,6 +3985,7 @@ export class Task extends EventEmitter<TaskEvents> implements TaskLike {
39823985
this.userMessageContentReady = false
39833986
this.didRejectTool = false
39843987
this.didAlreadyUseTool = false
3988+
this.contentStopReason = undefined
39853989
this.assistantMessageSavedToHistory = false
39863990
this.didFinishAbortingStream = false
39873991
this.resetAssistantMessagePersistence()
@@ -4560,6 +4564,15 @@ export class Task extends EventEmitter<TaskEvents> implements TaskLike {
45604564
// to be completed or the user to reject a block in order to proceed
45614565
// and eventually set userMessageContentReady to true.)
45624566

4567+
// Set before finalizing, so the tool calls presented below see it and are not run.
4568+
this.contentStopReason = findContentStopReason(this.streamDiagnostics.stopReasons)
4569+
if (this.contentStopReason) {
4570+
await this.say(
4571+
"error",
4572+
t("common:errors.content_stop_tool_not_run", { reason: this.contentStopReason }),
4573+
)
4574+
}
4575+
45634576
// Finalize any remaining streaming tool calls that weren't explicitly ended
45644577
// This is critical for MCP tools which need tool_call_end events to be properly
45654578
// converted from ToolUse to McpToolUse via finalizeStreamingToolCall()
Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
import { findContentStopReason } from "../contentStop"
2+
3+
describe("findContentStopReason", () => {
4+
it.each([
5+
[["content_filtered"], "content_filtered"],
6+
[["refusal"], "refusal"],
7+
[["guardrail_intervened"], "guardrail_intervened"],
8+
[["content_filtered (reasoning_extraction: Declined.)"], "content_filtered (reasoning_extraction: Declined.)"],
9+
[["tool_use", "refusal (cyber)"], "refusal (cyber)"],
10+
])("reports %j as a content stop", (stopReasons, expected) => {
11+
expect(findContentStopReason(stopReasons)).toBe(expected)
12+
})
13+
14+
it.each([[[]], [["tool_use"]], [["end_turn"]], [["max_tokens"]], [["stop_sequence"]], [["not_content_filtered"]]])(
15+
"does not treat %j as a content stop",
16+
(stopReasons) => {
17+
expect(findContentStopReason(stopReasons)).toBeUndefined()
18+
},
19+
)
20+
})

‎src/core/task/contentStop.ts‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
/**
2+
* The stop reason when a safety classifier or guardrail ended the turn, otherwise undefined.
3+
*
4+
* Such a stop cuts generation off mid-output:
5+
* Bedrock then closes the open JSON of a streaming tool call (e.g. `…"intel-nvid"}`),
6+
* so a truncated call still parses and would run.
7+
* Anthropic's guidance is to treat partial output of a refused turn as incomplete and discard it.
8+
* See #1820.
9+
*
10+
* - https://platform.claude.com/docs/en/build-with-claude/refusals-and-fallback#what-a-refusal-looks-like
11+
* - https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_ConverseStreamMetadataEvent.html
12+
*/
13+
export function findContentStopReason(stopReasons: readonly string[]): string | undefined {
14+
// Matches the bare reason and the "reason (details)" form Bedrock builds from stop_details.
15+
return stopReasons.find((reason) => /^(content_filtered|refusal|guardrail_intervened)\b/.test(reason))
16+
}

‎src/i18n/locales/ca/common.json‎

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎src/i18n/locales/de/common.json‎

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎src/i18n/locales/en/common.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,7 @@
5858
"path_outside_workspace": "Path is outside the workspace",
5959
"settings_import_failed": "Settings import failed: {{error}}.",
6060
"mistake_limit_guidance": "This may indicate a failure in the model's thought process or inability to use a tool properly, which can be mitigated with some user guidance (e.g. \"Try breaking down the task into smaller steps\").",
61+
"content_stop_tool_not_run": "The provider stopped the model's response ({{reason}}). Any tool call in it was cut off and was not run.",
6162
"violated_organization_allowlist": "Failed to run task: the current profile isn't compatible with your organization settings",
6263
"condense_failed": "Failed to condense context",
6364
"condense_not_enough_messages": "Not enough messages to condense context",

‎src/i18n/locales/es/common.json‎

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎src/i18n/locales/fr/common.json‎

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)