Skip to content

Commit 2b873cf

Browse files
tlongwell-blockWren
andauthored
Recover from max-token response truncation (block#5223)
## Summary - treat provider `max_tokens` as an interrupted assistant response and continue the same turn with actionable feedback - discard tool calls from truncated responses, including malformed partial arguments, so they are neither executed nor replayed with invalid tool-result pairing - bound recovery to two retries while preserving normal finite `max_rounds` accounting ## Verification - `cargo fmt --all -- --check` - `cargo test -p buzz-agent` (422 unit tests plus all package integration/doc suites passed) - `cargo clippy -p buzz-agent --all-targets -- -D warnings` ## Notes The pre-push repository-wide hook also ran. Its Rust tests passed (2,270 passed, 14 ignored), but its `buzz-db` unit-test build was blocked because local rustc 1.89 is below sqlx 0.9's rustc 1.94 requirement. The affected package suite above is green on the exact pushed commit. Originating Buzz channel: `c3252dd2-0142-4e01-88c7-a2183c3960a5` Signed-off-by: Wren <5217c5c2f7bfb4333e46d17c98a9255a52dadee18dcd43a43536b95e6776dfa0@buzz.block.builderlab.xyz> Co-authored-by: Wren <5217c5c2f7bfb4333e46d17c98a9255a52dadee18dcd43a43536b95e6776dfa0@buzz.block.builderlab.xyz>
1 parent 3855687 commit 2b873cf

3 files changed

Lines changed: 261 additions & 33 deletions

File tree

‎crates/buzz-agent/src/agent.rs‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,17 @@ const ERROR_REFLECTION_SUFFIX: &str =
2424

2525
const UNSUPPORTED_IMAGE_TOOL_MESSAGE: &str = "The current model does not support image input. The image was removed from conversation history so this turn can continue. Use a text-based inspection tool or ask the user for a textual description instead.";
2626

27+
/// Model-visible feedback after the provider truncates an assistant response at
28+
/// its output-token limit. This is a user message rather than a synthetic tool
29+
/// result because truncation can happen without a tool call (and an unpaired
30+
/// tool result is invalid on every provider wire format).
31+
const MAX_TOKENS_RECOVERY_MESSAGE: &str = "Your previous response exceeded the model's output token limit and was truncated. Any incomplete tool call was not run. Continue the task, breaking the work or tool call into smaller steps and keeping the response concise.";
32+
33+
/// A provider can repeatedly spend its entire output allowance without making
34+
/// progress, while `max_rounds` is unbounded by default. Keep the in-turn rescue
35+
/// finite so a persistently truncating model eventually surfaces `max_tokens`.
36+
const MAX_TOKENS_RECOVERIES_PER_RUN: u32 = 2;
37+
2738
/// Remove image blocks that the provider has explicitly rejected while keeping
2839
/// their surrounding tool result (and therefore the tool-call/result pairing)
2940
/// intact. Returns the number of images removed; zero means the provider error
@@ -262,6 +273,10 @@ impl RunCtx<'_> {
262273
// per-session: a fresh prompt deserves a fresh chance to recover, and
263274
// `max_rounds` defaults to 0 (unbounded) so it cannot bound this.
264275
let mut context_recoveries = 0u32;
276+
// Per-run output-truncation recovery budget. Unlike context recovery,
277+
// these successful provider requests consume a real round and are not
278+
// refunded; this counter only bounds the default-unlimited case.
279+
let mut max_tokens_recoveries = 0u32;
265280
loop {
266281
if self.cfg.max_rounds > 0 && round >= self.cfg.max_rounds {
267282
return Ok(StopReason::MaxTurnRequests);
@@ -515,6 +530,37 @@ impl RunCtx<'_> {
515530
.await;
516531
}
517532

533+
// `max_tokens` describes a truncated assistant response, not turn
534+
// completion. Never execute tool calls from it: although one may
535+
// parse as valid, a later call (or surrounding instructions) may
536+
// have been cut off. Replay only the text, with no tool calls, so
537+
// the history remains valid without fabricated tool results; then
538+
// add actionable user-role feedback and ask the model to continue.
539+
if response.stop == ProviderStop::MaxTokens {
540+
self.history.push(HistoryItem::Assistant {
541+
text: response.text,
542+
tool_calls: Vec::new(),
543+
reasoning_details: response.reasoning_details,
544+
});
545+
if max_tokens_recoveries >= MAX_TOKENS_RECOVERIES_PER_RUN {
546+
tracing::warn!(
547+
recoveries = max_tokens_recoveries,
548+
"provider repeatedly hit output token limit; recovery budget exhausted"
549+
);
550+
return Ok(StopReason::MaxTokens);
551+
}
552+
max_tokens_recoveries = max_tokens_recoveries.saturating_add(1);
553+
tracing::warn!(
554+
recovery = max_tokens_recoveries,
555+
max_recoveries = MAX_TOKENS_RECOVERIES_PER_RUN,
556+
discarded_tool_calls = response.tool_calls.len(),
557+
"provider hit output token limit; asking model to continue in smaller steps"
558+
);
559+
self.history
560+
.push(HistoryItem::User(MAX_TOKENS_RECOVERY_MESSAGE.to_string()));
561+
continue;
562+
}
563+
518564
if response.tool_calls.is_empty() {
519565
if response.stop == ProviderStop::ToolUse {
520566
return Err(AgentError::Llm(

‎crates/buzz-agent/src/llm.rs‎

Lines changed: 91 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -1229,6 +1229,11 @@ fn databricks_v2_path(route: DatabricksV2Route) -> &'static str {
12291229
}
12301230

12311231
fn parse_responses(v: Value) -> Result<LlmResponse, AgentError> {
1232+
let max_tokens = v.get("status").and_then(Value::as_str) == Some("incomplete")
1233+
&& v.get("incomplete_details")
1234+
.and_then(|d| d.get("reason"))
1235+
.and_then(Value::as_str)
1236+
== Some("max_output_tokens");
12321237
let mut text = String::new();
12331238
let mut reasoning = String::new();
12341239
let mut tool_calls = Vec::new();
@@ -1259,7 +1264,7 @@ fn parse_responses(v: Value) -> Result<LlmResponse, AgentError> {
12591264
}
12601265
}
12611266
}
1262-
Some("function_call") => {
1267+
Some("function_call") if !max_tokens => {
12631268
saw_function_call = true;
12641269
let raw = item
12651270
.get("arguments")
@@ -1279,6 +1284,9 @@ fn parse_responses(v: Value) -> Result<LlmResponse, AgentError> {
12791284
Default::default(),
12801285
)?);
12811286
}
1287+
// Incomplete Responses output can carry a partial function call.
1288+
// It is intentionally discarded by the in-turn recovery path.
1289+
Some("function_call") => {}
12821290
Some("reasoning") => {
12831291
// Reasoning summary items from the Responses API. Each item has a
12841292
// `summary` array of `{"type": "summary_text", "text": "..."}` objects.
@@ -1551,13 +1559,19 @@ fn parse_anthropic(v: Value) -> Result<LlmResponse, AgentError> {
15511559
reasoning.push_str(t);
15521560
}
15531561
}
1554-
// Anthropic's replay shape is fully modelled, so nothing to keep.
1555-
Some("tool_use") => tool_calls.push(make_tool_call(
1556-
str_field(b, "id"),
1557-
str_field(b, "name"),
1558-
b.get("input").cloned().unwrap_or(Value::Null),
1559-
Default::default(),
1560-
)?),
1562+
// A max-token response may end in the middle of a tool input.
1563+
// The agent discards all calls from truncated responses, so do
1564+
// not reject the whole response trying to parse an input that
1565+
// can never be executed.
1566+
Some("tool_use") if stop != ProviderStop::MaxTokens => {
1567+
tool_calls.push(make_tool_call(
1568+
str_field(b, "id"),
1569+
str_field(b, "name"),
1570+
b.get("input").cloned().unwrap_or(Value::Null),
1571+
Default::default(),
1572+
)?)
1573+
}
1574+
Some("tool_use") => {}
15611575
_ => {}
15621576
}
15631577
}
@@ -1645,30 +1659,33 @@ fn parse_openai(v: Value) -> Result<LlmResponse, AgentError> {
16451659
}
16461660
};
16471661
let mut tool_calls = Vec::new();
1648-
if let Some(arr) = msg.get("tool_calls").and_then(Value::as_array) {
1649-
for tc in arr {
1650-
let f = tc
1651-
.get("function")
1652-
.ok_or_else(|| AgentError::Llm("tool_call missing function".into()))?;
1653-
let raw = f.get("arguments").and_then(Value::as_str).unwrap_or("{}");
1654-
let args: Value = serde_json::from_str(raw)
1655-
.map_err(|e| AgentError::Llm(format!("tool_call.arguments not valid JSON: {e}")))?;
1656-
// Everything on the wire object we do not model, kept for replay.
1657-
let extra = tc
1658-
.as_object()
1659-
.map(|o| {
1660-
o.iter()
1661-
.filter(|(k, _)| !matches!(k.as_str(), "id" | "type" | "function"))
1662-
.map(|(k, v)| (k.clone(), v.clone()))
1663-
.collect()
1664-
})
1665-
.unwrap_or_default();
1666-
tool_calls.push(make_tool_call(
1667-
str_field(tc, "id"),
1668-
str_field(f, "name"),
1669-
args,
1670-
extra,
1671-
)?);
1662+
if stop != ProviderStop::MaxTokens {
1663+
if let Some(arr) = msg.get("tool_calls").and_then(Value::as_array) {
1664+
for tc in arr {
1665+
let f = tc
1666+
.get("function")
1667+
.ok_or_else(|| AgentError::Llm("tool_call missing function".into()))?;
1668+
let raw = f.get("arguments").and_then(Value::as_str).unwrap_or("{}");
1669+
let args: Value = serde_json::from_str(raw).map_err(|e| {
1670+
AgentError::Llm(format!("tool_call.arguments not valid JSON: {e}"))
1671+
})?;
1672+
// Everything on the wire object we do not model, kept for replay.
1673+
let extra = tc
1674+
.as_object()
1675+
.map(|o| {
1676+
o.iter()
1677+
.filter(|(k, _)| !matches!(k.as_str(), "id" | "type" | "function"))
1678+
.map(|(k, v)| (k.clone(), v.clone()))
1679+
.collect()
1680+
})
1681+
.unwrap_or_default();
1682+
tool_calls.push(make_tool_call(
1683+
str_field(tc, "id"),
1684+
str_field(f, "name"),
1685+
args,
1686+
extra,
1687+
)?);
1688+
}
16721689
}
16731690
}
16741691
dedupe_provider_ids(&mut tool_calls);
@@ -3527,10 +3544,51 @@ mod tests {
35273544
let v = serde_json::json!({
35283545
"status": "incomplete",
35293546
"incomplete_details": {"reason": "max_output_tokens"},
3530-
"output": [],
3547+
"output": [{
3548+
"type": "function_call",
3549+
"call_id": "partial",
3550+
"name": "dev__shell",
3551+
"arguments": "{\"command\":\"unterminated",
3552+
}],
35313553
});
35323554
let r = parse_responses(v).unwrap();
35333555
assert_eq!(r.stop, ProviderStop::MaxTokens);
3556+
assert!(r.tool_calls.is_empty());
3557+
}
3558+
3559+
#[test]
3560+
fn truncated_openai_tool_arguments_are_discarded_not_rejected() {
3561+
let v = serde_json::json!({
3562+
"choices": [{
3563+
"finish_reason": "length",
3564+
"message": {
3565+
"content": "partial text",
3566+
"tool_calls": [{
3567+
"id": "partial",
3568+
"type": "function",
3569+
"function": {
3570+
"name": "dev__shell",
3571+
"arguments": "{\"command\":\"unterminated",
3572+
},
3573+
}],
3574+
},
3575+
}],
3576+
});
3577+
let r = parse_openai(v).unwrap();
3578+
assert_eq!(r.stop, ProviderStop::MaxTokens);
3579+
assert_eq!(r.text, "partial text");
3580+
assert!(r.tool_calls.is_empty());
3581+
}
3582+
3583+
#[test]
3584+
fn truncated_anthropic_tool_use_is_discarded_not_rejected() {
3585+
let v = serde_json::json!({
3586+
"stop_reason": "max_tokens",
3587+
"content": [{"type": "tool_use", "id": "", "name": "", "input": null}],
3588+
});
3589+
let r = parse_anthropic(v).unwrap();
3590+
assert_eq!(r.stop, ProviderStop::MaxTokens);
3591+
assert!(r.tool_calls.is_empty());
35343592
}
35353593

35363594
#[test]

‎crates/buzz-agent/tests/regressions.rs‎

Lines changed: 124 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -233,6 +233,26 @@ fn openai_text_with_usage(content: &str, prompt_tokens: u64) -> Value {
233233
v
234234
}
235235

236+
fn openai_max_tokens(content: &str, tool_calls: Value) -> Value {
237+
json!({
238+
"id": "cc-max", "object": "chat.completion", "model": "fake-model",
239+
"choices": [{
240+
"index": 0,
241+
"message": {
242+
"role": "assistant",
243+
"content": content,
244+
"tool_calls": tool_calls,
245+
},
246+
"finish_reason": "length",
247+
}],
248+
"usage": {
249+
"prompt_tokens": 10,
250+
"completion_tokens": 100,
251+
"total_tokens": 110,
252+
},
253+
})
254+
}
255+
236256
fn openai_tool_call(id: &str, name: &str, args: Value) -> Value {
237257
json!({
238258
"id": "cc-2", "object": "chat.completion", "model": "fake-model",
@@ -2437,6 +2457,110 @@ async fn context_window_400_recovers_instead_of_sticking() {
24372457
h.shutdown().await;
24382458
}
24392459

2460+
/// A provider output-token stop is an interrupted round, not completion. The
2461+
/// agent must preserve any text, discard a possibly partial tool call, provide
2462+
/// actionable feedback, and let the same prompt finish normally.
2463+
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2464+
async fn max_tokens_recovers_in_turn_without_running_partial_tool_call() {
2465+
const PARTIAL: &str = "partial-before-limit";
2466+
let partial_call = json!([{
2467+
"id": "partial-call", "type": "function",
2468+
"function": { "name": "dev__shell", "arguments": "{\"command\":\"echo" },
2469+
}]);
2470+
let llm = spawn_capturing_llm(vec![
2471+
openai_max_tokens(PARTIAL, partial_call),
2472+
openai_text("done after truncation"),
2473+
])
2474+
.await;
2475+
let mut h = Harness::spawn(&llm.url).await;
2476+
let sid = init_session(&mut h, json!([])).await;
2477+
2478+
let prompt_id = h
2479+
.send(
2480+
"session/prompt",
2481+
json!({"sessionId": sid, "prompt": [{"type":"text","text":"do the task"}]}),
2482+
)
2483+
.await;
2484+
let reply = h.recv_until(|v| v["id"] == json!(prompt_id)).await;
2485+
assert_eq!(reply["result"]["stopReason"], "end_turn", "{reply}");
2486+
2487+
let requests = llm.captured.lock().await;
2488+
assert_eq!(requests.len(), 2, "truncation should trigger one retry");
2489+
let retry = &requests[1]["messages"];
2490+
let serialized = retry.to_string();
2491+
assert!(
2492+
serialized.contains(PARTIAL),
2493+
"partial text was lost: {retry}"
2494+
);
2495+
assert!(
2496+
serialized.contains("output token limit")
2497+
&& serialized.contains("smaller steps")
2498+
&& serialized.contains("tool call"),
2499+
"retry lacks actionable truncation feedback: {retry}"
2500+
);
2501+
assert!(
2502+
!serialized.contains("partial-call") && !serialized.contains("tool_call_id"),
2503+
"partial tool call must not be replayed or executed: {retry}"
2504+
);
2505+
drop(requests);
2506+
h.shutdown().await;
2507+
}
2508+
2509+
/// `max_rounds` counts max-token responses because they are successful, billed
2510+
/// provider requests. Recovery must not grant them the refund reserved for a
2511+
/// rejected context-overflow request.
2512+
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2513+
async fn max_tokens_recovery_respects_finite_round_cap() {
2514+
let llm = spawn_capturing_llm(vec![
2515+
openai_max_tokens("cut off", json!([])),
2516+
openai_text("must not be requested"),
2517+
])
2518+
.await;
2519+
let mut h = Harness::spawn_with_env(&llm.url, &[("BUZZ_AGENT_MAX_ROUNDS", "1")]).await;
2520+
let sid = init_session(&mut h, json!([])).await;
2521+
let prompt_id = h
2522+
.send(
2523+
"session/prompt",
2524+
json!({"sessionId": sid, "prompt": [{"type":"text","text":"go"}]}),
2525+
)
2526+
.await;
2527+
let reply = h.recv_until(|v| v["id"] == json!(prompt_id)).await;
2528+
assert_eq!(
2529+
reply["result"]["stopReason"], "max_turn_requests",
2530+
"{reply}"
2531+
);
2532+
assert_eq!(llm.captured.lock().await.len(), 1);
2533+
h.shutdown().await;
2534+
}
2535+
2536+
/// With the production-unbounded round setting, a model that always fills its
2537+
/// output allowance still has to return. Two recovery prompts are allowed; the
2538+
/// third truncation surfaces the original stop reason.
2539+
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2540+
async fn repeated_max_tokens_is_bounded() {
2541+
let responses = (0..4)
2542+
.map(|_| openai_max_tokens("still truncated", json!([])))
2543+
.collect();
2544+
let llm = spawn_capturing_llm(responses).await;
2545+
let mut h = Harness::spawn_with_env(&llm.url, &[("BUZZ_AGENT_MAX_ROUNDS", "0")]).await;
2546+
let sid = init_session(&mut h, json!([])).await;
2547+
let prompt_id = h
2548+
.send(
2549+
"session/prompt",
2550+
json!({"sessionId": sid, "prompt": [{"type":"text","text":"go"}]}),
2551+
)
2552+
.await;
2553+
let reply = tokio::time::timeout(
2554+
Duration::from_secs(10),
2555+
h.recv_until(|v| v["id"] == json!(prompt_id)),
2556+
)
2557+
.await
2558+
.expect("max-token recovery must be bounded");
2559+
assert_eq!(reply["result"]["stopReason"], "max_tokens", "{reply}");
2560+
assert_eq!(llm.captured.lock().await.len(), 3);
2561+
h.shutdown().await;
2562+
}
2563+
24402564
/// A successful recovery must actually send the recovered completion, even when
24412565
/// `max_rounds` is finite. `round` is incremented BEFORE the completion that
24422566
/// gets rejected, so a naive `continue` after recovery re-enters the loop with

0 commit comments

Comments
 (0)