Skip to content

Commit 3c52ed6

Browse files
committed
fix: don't run tool calls from turns a safety filter stopped
Part of Zoo-Code-Org#1820. When Bedrock's filter stops an Opus 5.5 turn with content_filtered, it cuts the output mid-stream and then closes the open JSON of a streaming tool call, e.g. an execute_command input ending in `…c.specialisation.\"intel-nvid"}`. The truncated call still parses, so Zoo ran it. The quote check then rejected it as "Malformed command: unterminated single quote", and that was all the model learned: it took its own command to be broken and resent it three times, each filtered again. Anthropic's guidance is to treat partial output of a refused turn as incomplete and discard it. So when the turn's stop reason is content_filtered, refusal or guardrail_intervened: - no tool call from that turn runs, native MCP calls included; - each gets a tool_result stating the real stop reason, and that resending the same content is likely to be stopped again; - the chat shows an error row with the stop reason. https://platform.claude.com/docs/en/build-with-claude/refusals-and-fallback#what-a-refusal-looks-like
1 parent c688a19 commit 3c52ed6

23 files changed

Lines changed: 223 additions & 0 deletions
Lines changed: 121 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,121 @@
1+
// npx vitest run core/assistant-message/__tests__/presentAssistantMessage-content-stop.spec.ts
2+
3+
import { presentAssistantMessage } from "../presentAssistantMessage"
4+
5+
const mockExecuteCommandHandle = vi.hoisted(() => vi.fn())
6+
const mockUseMcpToolHandle = vi.hoisted(() => vi.fn())
7+
8+
vi.mock("../../task/Task")
9+
vi.mock("../../tools/validateToolUse", () => ({
10+
validateToolUse: vi.fn(),
11+
isValidToolName: vi.fn(() => true),
12+
}))
13+
vi.mock("../../tools/ExecuteCommandTool", () => ({
14+
executeCommandTool: { handle: mockExecuteCommandHandle },
15+
}))
16+
vi.mock("../../tools/UseMcpToolTool", () => ({
17+
useMcpToolTool: { handle: mockUseMcpToolHandle },
18+
}))
19+
vi.mock("@roo-code/telemetry", () => ({
20+
TelemetryService: {
21+
instance: {
22+
captureToolUsage: vi.fn(),
23+
captureConsecutiveMistakeError: vi.fn(),
24+
},
25+
},
26+
}))
27+
28+
// The truncated call from the #1820 report: Bedrock closed the open JSON string after the filter cut it off.
29+
const truncatedCommand =
30+
"cd /etc/nixos && nix-instantiate --eval --strict -E '\nlet s = import <nixpkgs/nixos> {};\n off = c.specialisation.\"intel-nvid"
31+
32+
type TestToolResult = { type: "tool_result"; tool_use_id: string; content: string; is_error?: boolean }
33+
34+
function makeTask(contentStopReason: string | undefined, block: Record<string, unknown>) {
35+
const userMessageContent: TestToolResult[] = []
36+
return {
37+
taskId: "test-task-id",
38+
instanceId: "test-instance",
39+
abort: false,
40+
presentAssistantMessageLocked: false,
41+
presentAssistantMessageHasPendingUpdates: false,
42+
currentStreamingContentIndex: 0,
43+
assistantMessageContent: [block],
44+
userMessageContent,
45+
didCompleteReadingStream: true,
46+
didRejectTool: false,
47+
didAlreadyUseTool: false,
48+
contentStopReason,
49+
consecutiveMistakeCount: 0,
50+
clineMessages: [],
51+
getTaskMode: vi.fn().mockResolvedValue("code"),
52+
api: { getModel: () => ({ id: "test-model", info: {} }) },
53+
recordToolUsage: vi.fn(),
54+
recordToolError: vi.fn(),
55+
toolRepetitionDetector: { check: vi.fn().mockReturnValue({ allowExecution: true }) },
56+
providerRef: { deref: () => ({ getState: vi.fn().mockResolvedValue({ mode: "code", customModes: [] }) }) },
57+
say: vi.fn().mockResolvedValue(undefined),
58+
ask: vi.fn().mockResolvedValue({ response: "yesButtonClicked" }),
59+
pushToolResultToUserContent: vi.fn((result: TestToolResult) => {
60+
userMessageContent.push(result)
61+
return true
62+
}),
63+
}
64+
}
65+
66+
const executeCommandBlock = {
67+
type: "tool_use",
68+
id: "tooluse_filtered",
69+
name: "execute_command",
70+
params: { command: truncatedCommand },
71+
nativeArgs: { command: truncatedCommand },
72+
partial: false,
73+
}
74+
75+
describe("presentAssistantMessage - turns ended by a content stop", () => {
76+
beforeEach(() => {
77+
mockExecuteCommandHandle.mockReset()
78+
mockUseMcpToolHandle.mockReset()
79+
})
80+
81+
it("does not run a tool call from a content_filtered turn and tells the model why", async () => {
82+
const task = makeTask("content_filtered", executeCommandBlock)
83+
84+
// `as never`: the partial mock only carries the fields presentAssistantMessage reads.
85+
await presentAssistantMessage(task as never)
86+
87+
expect(mockExecuteCommandHandle).not.toHaveBeenCalled()
88+
expect(task.userMessageContent).toHaveLength(1)
89+
const [result] = task.userMessageContent
90+
expect(result).toMatchObject({ type: "tool_result", tool_use_id: "tooluse_filtered", is_error: true })
91+
expect(result.content).toContain("stop reason: content_filtered")
92+
expect(result.content).toContain("was not run")
93+
expect(result.content).not.toContain("Malformed command")
94+
})
95+
96+
it("does not run a native MCP tool call from a refusal turn", async () => {
97+
const task = makeTask("refusal (reasoning_extraction)", {
98+
type: "mcp_tool_use",
99+
id: "tooluse_mcp",
100+
name: "mcp_server_tool",
101+
serverName: "server",
102+
toolName: "tool",
103+
arguments: {},
104+
partial: false,
105+
})
106+
107+
await presentAssistantMessage(task as never)
108+
109+
expect(mockUseMcpToolHandle).not.toHaveBeenCalled()
110+
expect(task.userMessageContent[0]).toMatchObject({ tool_use_id: "tooluse_mcp", is_error: true })
111+
expect(task.userMessageContent[0].content).toContain("refusal (reasoning_extraction)")
112+
})
113+
114+
it("runs the tool call normally when the turn ended without a content stop", async () => {
115+
const task = makeTask(undefined, executeCommandBlock)
116+
117+
await presentAssistantMessage(task as never)
118+
119+
expect(mockExecuteCommandHandle).toHaveBeenCalledTimes(1)
120+
})
121+
})

‎src/core/assistant-message/presentAssistantMessage.ts‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,17 @@ import { codebaseSearchTool } from "../tools/CodebaseSearchTool"
4242
import { formatResponse } from "../prompts/responses"
4343
import { sanitizeToolUseId } from "../../utils/tool-id"
4444

45+
/**
46+
* Tool result for a call from a turn a safety classifier or guardrail ended.
47+
* States the real stop reason, so the model changes course instead of retrying a "malformed" call.
48+
*/
49+
export function contentStopToolResult(toolName: string, stopReason: string): string {
50+
return (
51+
`The provider stopped your response (stop reason: ${stopReason}) while you were writing this ${toolName} call. ` +
52+
`The call was cut off and was not run. Resending the same content is likely to be stopped again.`
53+
)
54+
}
55+
4556
/**
4657
* Maps a raw, potentially model-controlled tool name to a safe analytics key.
4758
* Never returns the raw name unless it is a known static tool, so an
@@ -134,6 +145,20 @@ export async function presentAssistantMessage(cline: Task) {
134145
// their original name in API history
135146
const mcpBlock = block as McpToolUse
136147

148+
if (cline.contentStopReason && !mcpBlock.partial) {
149+
if (mcpBlock.id) {
150+
cline.pushToolResultToUserContent({
151+
type: "tool_result",
152+
tool_use_id: sanitizeToolUseId(mcpBlock.id),
153+
content: formatResponse.toolError(
154+
contentStopToolResult(mcpBlock.name, cline.contentStopReason),
155+
),
156+
is_error: true,
157+
})
158+
}
159+
break
160+
}
161+
137162
if (cline.didRejectTool) {
138163
// For native protocol, we must send a tool_result for every tool_use to avoid API errors
139164
const toolCallId = mcpBlock.id
@@ -442,6 +467,16 @@ export async function presentAssistantMessage(cline: Task) {
442467
//
443468
// This avoids executing an invalid tool_use block and prevents duplicate/fragmented
444469
// error reporting.
470+
if (cline.contentStopReason && !block.partial) {
471+
cline.pushToolResultToUserContent({
472+
type: "tool_result",
473+
tool_use_id: sanitizeToolUseId(toolCallId),
474+
content: formatResponse.toolError(contentStopToolResult(block.name, cline.contentStopReason)),
475+
is_error: true,
476+
})
477+
break
478+
}
479+
445480
if (!block.partial) {
446481
const customTool = stateExperiments?.customTools ? customToolRegistry.get(block.name) : undefined
447482
const isKnownTool = isValidToolName(String(block.name), stateExperiments)

‎src/core/task/Task.ts‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -95,6 +95,7 @@ import { getWorkspacePath } from "../../utils/path"
9595
import { sanitizeToolUseId } from "../../utils/tool-id"
9696
import { getTaskDirectoryPath } from "../../utils/storage"
9797
import { createRawApiDump } from "../../utils/rawApiDump"
98+
import { findContentStopReason } from "./contentStop"
9899
import { logger } from "../../utils/logging"
99100

100101
// prompts
@@ -497,6 +498,8 @@ export class Task extends EventEmitter<TaskEvents> implements TaskLike {
497498
didRejectTool = false
498499
didAlreadyUseTool = false
499500
didToolFailInCurrentTurn = false
501+
/** Stop reason of a turn ended by a safety classifier or guardrail; its tool calls must not run. */
502+
contentStopReason: string | undefined = undefined
500503
didCompleteReadingStream = false
501504
private _started = false
502505
private _runPromise: Promise<void> | undefined
@@ -3306,6 +3309,7 @@ export class Task extends EventEmitter<TaskEvents> implements TaskLike {
33063309
this.userMessageContentReady = false
33073310
this.didRejectTool = false
33083311
this.didAlreadyUseTool = false
3312+
this.contentStopReason = undefined
33093313
this.assistantMessageSavedToHistory = false
33103314
this.resetAssistantMessagePersistence()
33113315
// Reset tool failure flag for each new assistant turn - this ensures that tool failures
@@ -3880,6 +3884,15 @@ export class Task extends EventEmitter<TaskEvents> implements TaskLike {
38803884
// to be completed or the user to reject a block in order to proceed
38813885
// and eventually set userMessageContentReady to true.)
38823886

3887+
// Set before finalizing, so the tool calls presented below see it and are not run.
3888+
this.contentStopReason = findContentStopReason(this.streamDiagnostics.stopReasons)
3889+
if (this.contentStopReason) {
3890+
await this.say(
3891+
"error",
3892+
t("common:errors.content_stop_tool_not_run", { reason: this.contentStopReason }),
3893+
)
3894+
}
3895+
38833896
// Finalize any remaining streaming tool calls that weren't explicitly ended
38843897
// This is critical for MCP tools which need tool_call_end events to be properly
38853898
// converted from ToolUse to McpToolUse via finalizeStreamingToolCall()
Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
import { findContentStopReason } from "../contentStop"
2+
3+
describe("findContentStopReason", () => {
4+
it.each([
5+
[["content_filtered"], "content_filtered"],
6+
[["refusal"], "refusal"],
7+
[["guardrail_intervened"], "guardrail_intervened"],
8+
[["content_filtered (reasoning_extraction: Declined.)"], "content_filtered (reasoning_extraction: Declined.)"],
9+
[["tool_use", "refusal (cyber)"], "refusal (cyber)"],
10+
])("reports %j as a content stop", (stopReasons, expected) => {
11+
expect(findContentStopReason(stopReasons)).toBe(expected)
12+
})
13+
14+
it.each([[[]], [["tool_use"]], [["end_turn"]], [["max_tokens"]], [["stop_sequence"]], [["not_content_filtered"]]])(
15+
"does not treat %j as a content stop",
16+
(stopReasons) => {
17+
expect(findContentStopReason(stopReasons)).toBeUndefined()
18+
},
19+
)
20+
})

‎src/core/task/contentStop.ts‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
/**
2+
* The stop reason when a safety classifier or guardrail ended the turn, otherwise undefined.
3+
*
4+
* Such a stop cuts generation off mid-output:
5+
* Bedrock then closes the open JSON of a streaming tool call (e.g. `…"intel-nvid"}`),
6+
* so a truncated call still parses and would run.
7+
* Anthropic's guidance is to treat partial output of a refused turn as incomplete and discard it.
8+
* See #1820.
9+
*
10+
* - https://platform.claude.com/docs/en/build-with-claude/refusals-and-fallback#what-a-refusal-looks-like
11+
* - https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_ConverseStreamMetadataEvent.html
12+
*/
13+
export function findContentStopReason(stopReasons: readonly string[]): string | undefined {
14+
// Matches the bare reason and the "reason (details)" form Bedrock builds from stop_details.
15+
return stopReasons.find((reason) => /^(content_filtered|refusal|guardrail_intervened)\b/.test(reason))
16+
}

‎src/i18n/locales/ca/common.json‎

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎src/i18n/locales/de/common.json‎

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎src/i18n/locales/en/common.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,7 @@
5757
"cannot_access_path": "Cannot access path {{path}}: {{error}}",
5858
"settings_import_failed": "Settings import failed: {{error}}.",
5959
"mistake_limit_guidance": "This may indicate a failure in the model's thought process or inability to use a tool properly, which can be mitigated with some user guidance (e.g. \"Try breaking down the task into smaller steps\").",
60+
"content_stop_tool_not_run": "The provider stopped the model's response ({{reason}}). Any tool call in it was cut off and was not run.",
6061
"violated_organization_allowlist": "Failed to run task: the current profile isn't compatible with your organization settings",
6162
"condense_failed": "Failed to condense context",
6263
"condense_not_enough_messages": "Not enough messages to condense context",

‎src/i18n/locales/es/common.json‎

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎src/i18n/locales/fr/common.json‎

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)