Skip to content

Commit e7a5d7b

Browse files
author
Eva
committed
fix(agent): retry LLM completion on malformed 2xx JSON body
A provider that returns HTTP 200 with a truncated JSON document (cleanly closed, framing intact, content cut off mid-value) previously died on the first attempt with a terminal AgentError::Llm("json: EOF while parsing a value"), killing the agent turn before it produced anything. Observed live: deepseek via OpenRouter returned a truncated body in a tb21 bench trial and the agent exited with 0 turns (code -32000 at the ACP surface). Treat an unparseable success body like a 5xx in both HTTP loops (shared post() and openrouter_post()): retry under the existing MAX_RETRIES bound with the same backoff_with_jitter, and surface terminal_llm_error with the json parse detail when retries are exhausted. Tool-safety: the retry lives entirely inside the HTTP POST helpers, before any parsed response value exists. Tool calls are only ever extracted from a successfully parsed response; a malformed body produced none, so nothing downstream of it could have dispatched. The retry re-sends the identical request bytes captured at function entry — an LLM completion request, which executes no tools. No tool call can be re-run by this change. Co-authored-by: Eva <011987e296fd5006292d2f930b574be47c7801048d1983c46c425d3c95f0cffd@buzz.block.builderlab.xyz> Signed-off-by: Eva <011987e296fd5006292d2f930b574be47c7801048d1983c46c425d3c95f0cffd@buzz.block.builderlab.xyz>
1 parent f029dea commit e7a5d7b

1 file changed

Lines changed: 283 additions & 3 deletions

File tree

‎crates/buzz-agent/src/llm.rs‎

Lines changed: 283 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2203,8 +2203,33 @@ where
22032203
}
22042204
}
22052205
}
2206-
return serde_json::from_slice(&buf)
2207-
.map_err(|e| PostError::Agent(AgentError::Llm(format!("json: {e}"))));
2206+
// A 2xx body that fails to parse (e.g. a provider flushing a
2207+
// truncated JSON document and closing the stream cleanly) is a
2208+
// transient upstream fault, not a request problem: retry it like a
2209+
// 5xx. Safe to re-send — a malformed body produced no parsed
2210+
// response, so no tool call was ever extracted from it, and the
2211+
// retried request is the identical completion POST captured in
2212+
// `body_bytes` at function entry.
2213+
match serde_json::from_slice(&buf) {
2214+
Ok(value) => return Ok(value),
2215+
Err(e) => {
2216+
if attempt + 1 < MAX_RETRIES {
2217+
tracing::warn!(
2218+
attempt = attempt + 1,
2219+
max_attempts = MAX_RETRIES,
2220+
error = %e,
2221+
"llm: malformed response body, retrying"
2222+
);
2223+
backoff_with_jitter(attempt).await;
2224+
continue;
2225+
}
2226+
return Err(PostError::Agent(terminal_llm_error(
2227+
call_start.elapsed(),
2228+
attempt + 1,
2229+
&format!("json: {e}"),
2230+
)));
2231+
}
2232+
}
22082233
}
22092234
// Unreachable in practice: every iteration either returns or continues.
22102235
// A fallthrough here would mean MAX_RETRIES was 0, which is rejected at
@@ -2610,7 +2635,31 @@ async fn openrouter_post(
26102635
}
26112636
}
26122637
}
2613-
return serde_json::from_slice(&buf).map_err(|e| AgentError::Llm(format!("json: {e}")));
2638+
// Same malformed-body retry as the shared `post()` terminal: a
2639+
// truncated 2xx JSON body is transient upstream trouble, and
2640+
// re-sending is provably tool-safe — nothing was parsed, so no tool
2641+
// call could have been extracted, and the retry re-uses the
2642+
// identical `body_bytes` completion request.
2643+
match serde_json::from_slice(&buf) {
2644+
Ok(value) => return Ok(value),
2645+
Err(e) => {
2646+
if attempt + 1 < MAX_RETRIES {
2647+
tracing::warn!(
2648+
attempt = attempt + 1,
2649+
max_attempts = MAX_RETRIES,
2650+
error = %e,
2651+
"llm: openrouter malformed response body, retrying"
2652+
);
2653+
backoff_with_jitter(attempt).await;
2654+
continue;
2655+
}
2656+
return Err(terminal_llm_error(
2657+
call_start.elapsed(),
2658+
attempt + 1,
2659+
&format!("json: {e}"),
2660+
));
2661+
}
2662+
}
26142663
}
26152664
Err(terminal_llm_error(
26162665
call_start.elapsed(),
@@ -4791,6 +4840,237 @@ mod tests {
47914840
);
47924841
}
47934842

4843+
/// Regression (write-compressor, tb21-twins-1): a provider returning
4844+
/// HTTP 200 with a *truncated* JSON body (cleanly closed, correct
4845+
/// framing, unparseable content) previously surfaced as a terminal
4846+
/// `AgentError::Llm("json: EOF while parsing a value ...")` on the very
4847+
/// first attempt, killing the agent turn. A malformed body is transient
4848+
/// upstream trouble and must be retried like a 5xx. Re-sending is
4849+
/// tool-safe: nothing was parsed, so no tool call was extracted from the
4850+
/// bad body, and the retry replays the identical completion request.
4851+
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
4852+
async fn post_retries_malformed_json_body_and_succeeds() {
4853+
use std::sync::atomic::{AtomicU32, Ordering};
4854+
use std::sync::Arc;
4855+
use tokio::io::{AsyncReadExt, AsyncWriteExt};
4856+
use tokio::net::TcpListener;
4857+
4858+
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
4859+
let url = format!("http://{}/v1/x", listener.local_addr().unwrap());
4860+
let accepts = Arc::new(AtomicU32::new(0));
4861+
let accepts_srv = accepts.clone();
4862+
4863+
tokio::spawn(async move {
4864+
loop {
4865+
let (mut sock, _) = match listener.accept().await {
4866+
Ok(p) => p,
4867+
Err(_) => return,
4868+
};
4869+
let n = accepts_srv.fetch_add(1, Ordering::SeqCst);
4870+
let mut buf = Vec::new();
4871+
let mut tmp = [0u8; 4096];
4872+
while !buf.windows(4).any(|w| w == b"\r\n\r\n") {
4873+
match sock.read(&mut tmp).await {
4874+
Ok(0) | Err(_) => return,
4875+
Ok(k) => buf.extend_from_slice(&tmp[..k]),
4876+
}
4877+
}
4878+
if n == 0 {
4879+
// First attempt: 200 OK with a truncated JSON document.
4880+
// Content-Length matches the bytes actually sent, so the
4881+
// body read completes cleanly — the fault is purely that
4882+
// the JSON is cut off mid-value.
4883+
let body = "{\"choices\":[{\"mess";
4884+
let resp = format!(
4885+
"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\n\
4886+
Content-Length: {}\r\nConnection: close\r\n\r\n{}",
4887+
body.len(),
4888+
body,
4889+
);
4890+
let _ = sock.write_all(resp.as_bytes()).await;
4891+
let _ = sock.shutdown().await;
4892+
continue;
4893+
}
4894+
// Subsequent attempts: complete valid JSON.
4895+
let body = "{\"ok\":true}";
4896+
let resp = format!(
4897+
"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\n\
4898+
Content-Length: {}\r\nConnection: close\r\n\r\n{}",
4899+
body.len(),
4900+
body,
4901+
);
4902+
let _ = sock.write_all(resp.as_bytes()).await;
4903+
let _ = sock.shutdown().await;
4904+
}
4905+
});
4906+
4907+
let client = Client::builder()
4908+
.timeout(Duration::from_secs(5))
4909+
.build()
4910+
.unwrap();
4911+
let out = post(
4912+
&client,
4913+
&url,
4914+
&serde_json::json!({}),
4915+
false,
4916+
Duration::from_secs(5),
4917+
|b| b,
4918+
)
4919+
.await
4920+
.expect("post should succeed after retrying the malformed body");
4921+
assert_eq!(out, serde_json::json!({ "ok": true }));
4922+
assert_eq!(
4923+
accepts.load(Ordering::SeqCst),
4924+
2,
4925+
"server must see exactly 2 attempts (malformed body retried once)"
4926+
);
4927+
}
4928+
4929+
/// A persistently malformed 200 body exhausts MAX_RETRIES and surfaces
4930+
/// the terminal error with the `json:` detail plus cumulative
4931+
/// duration/attempt count — never an early first-attempt death.
4932+
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
4933+
async fn post_exhausts_retries_on_persistent_malformed_json() {
4934+
use std::sync::atomic::{AtomicU32, Ordering};
4935+
use std::sync::Arc;
4936+
use tokio::io::{AsyncReadExt, AsyncWriteExt};
4937+
use tokio::net::TcpListener;
4938+
4939+
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
4940+
let url = format!("http://{}/v1/x", listener.local_addr().unwrap());
4941+
let accepts = Arc::new(AtomicU32::new(0));
4942+
let accepts_srv = accepts.clone();
4943+
4944+
tokio::spawn(async move {
4945+
loop {
4946+
let (mut sock, _) = match listener.accept().await {
4947+
Ok(p) => p,
4948+
Err(_) => return,
4949+
};
4950+
accepts_srv.fetch_add(1, Ordering::SeqCst);
4951+
let mut buf = Vec::new();
4952+
let mut tmp = [0u8; 4096];
4953+
while !buf.windows(4).any(|w| w == b"\r\n\r\n") {
4954+
match sock.read(&mut tmp).await {
4955+
Ok(0) | Err(_) => return,
4956+
Ok(k) => buf.extend_from_slice(&tmp[..k]),
4957+
}
4958+
}
4959+
let body = "{\"choices\":[{\"mess";
4960+
let resp = format!(
4961+
"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\n\
4962+
Content-Length: {}\r\nConnection: close\r\n\r\n{}",
4963+
body.len(),
4964+
body,
4965+
);
4966+
let _ = sock.write_all(resp.as_bytes()).await;
4967+
let _ = sock.shutdown().await;
4968+
}
4969+
});
4970+
4971+
let client = Client::builder()
4972+
.timeout(Duration::from_secs(5))
4973+
.build()
4974+
.unwrap();
4975+
let err = post(
4976+
&client,
4977+
&url,
4978+
&serde_json::json!({}),
4979+
false,
4980+
Duration::from_secs(5),
4981+
|b| b,
4982+
)
4983+
.await
4984+
.unwrap_err();
4985+
match &err {
4986+
PostError::Agent(AgentError::Llm(msg)) => {
4987+
assert!(
4988+
msg.contains("json:"),
4989+
"expected the json parse detail, got: {msg}"
4990+
);
4991+
assert!(
4992+
msg.contains("cumulative") && msg.contains("3 attempts"),
4993+
"expected cumulative duration + exact attempt count, got: {msg}"
4994+
);
4995+
}
4996+
other => panic!("expected PostError::Agent(AgentError::Llm), got: {other:?}"),
4997+
}
4998+
assert_eq!(
4999+
accepts.load(Ordering::SeqCst),
5000+
MAX_RETRIES,
5001+
"server must see exactly MAX_RETRIES attempts — malformed bodies must be retried"
5002+
);
5003+
}
5004+
5005+
/// Same regression coverage for `openrouter_post`, which carries its own
5006+
/// retry loop: a truncated 200 body on the first attempt is retried and
5007+
/// the call succeeds on the second.
5008+
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5009+
async fn openrouter_post_retries_malformed_json_body_and_succeeds() {
5010+
use std::sync::atomic::{AtomicU32, Ordering};
5011+
use std::sync::Arc;
5012+
use tokio::io::{AsyncReadExt, AsyncWriteExt};
5013+
use tokio::net::TcpListener;
5014+
5015+
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
5016+
let url = format!("http://{}/v1/x", listener.local_addr().unwrap());
5017+
let accepts = Arc::new(AtomicU32::new(0));
5018+
let accepts_srv = accepts.clone();
5019+
5020+
tokio::spawn(async move {
5021+
loop {
5022+
let (mut sock, _) = match listener.accept().await {
5023+
Ok(p) => p,
5024+
Err(_) => return,
5025+
};
5026+
let n = accepts_srv.fetch_add(1, Ordering::SeqCst);
5027+
let mut buf = Vec::new();
5028+
let mut tmp = [0u8; 4096];
5029+
while !buf.windows(4).any(|w| w == b"\r\n\r\n") {
5030+
match sock.read(&mut tmp).await {
5031+
Ok(0) | Err(_) => return,
5032+
Ok(k) => buf.extend_from_slice(&tmp[..k]),
5033+
}
5034+
}
5035+
if n == 0 {
5036+
let body = "{\"choices\":[{\"mess";
5037+
let resp = format!(
5038+
"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\n\
5039+
Content-Length: {}\r\nConnection: close\r\n\r\n{}",
5040+
body.len(),
5041+
body,
5042+
);
5043+
let _ = sock.write_all(resp.as_bytes()).await;
5044+
let _ = sock.shutdown().await;
5045+
continue;
5046+
}
5047+
let body = r#"{"choices":[{"message":{"role":"assistant","content":"ok"},"finish_reason":"stop"}]}"#;
5048+
let resp = format!(
5049+
"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\n\
5050+
Content-Length: {}\r\nConnection: close\r\n\r\n{}",
5051+
body.len(),
5052+
body,
5053+
);
5054+
let _ = sock.write_all(resp.as_bytes()).await;
5055+
let _ = sock.shutdown().await;
5056+
}
5057+
});
5058+
5059+
let client = Client::builder()
5060+
.timeout(Duration::from_secs(5))
5061+
.build()
5062+
.unwrap();
5063+
let out = openrouter_post(&client, &url, &json!({}), "key", Duration::from_secs(5))
5064+
.await
5065+
.expect("openrouter_post should succeed after retrying the malformed body");
5066+
assert!(out.is_object(), "expected a JSON object: {out:?}");
5067+
assert_eq!(
5068+
accepts.load(Ordering::SeqCst),
5069+
2,
5070+
"server must see exactly 2 attempts (malformed body retried once)"
5071+
);
5072+
}
5073+
47945074
/// A body-read timeout on the first attempt triggers a retry under an
47955075
/// escalated budget, and the call succeeds on the second attempt.
47965076
///

0 commit comments

Comments
 (0)