Skip to content

Commit b043860

Browse files
author
Eva
committed
fix(buzz-agent): recover from 400-shaped image rejections; unbound benchmark agent rounds
Two failure modes from the tb21-glm52-crusoe-1 benchmark run wedged or killed 13 of 89 trials without the model being at fault. **Conversation poisoning on text-only endpoints.** Crusoe's serverless `crusoeai/GLM-5.2-NVFP4` rejects any request whose history contains an image with `400: ... is not a multimodal model`. The recovery path for exactly this situation already exists (`AgentError::UnsupportedImageInput` → `replace_unsupported_images` strips the image blocks and continues the turn with a text placeholder), but classification only matched OpenRouter's 404 body ("no endpoints found that support image input") and was only consulted on the 404 arms. The Crusoe 400 fell through to terminal `AgentError::Llm`; the image stayed in history, every subsequent call failed identically, buzz-acp rode its 10-retry ladder (~40 min), and the trial idled to budget death. Measured: 8 trials, 12.7h aggregate idle. Fix: extend `is_unsupported_image_input_error()` to also match the verbatim "is not a multimodal model" body, and consult it on the 400 arms of both the shared `post()` ladder and `openrouter_post()` (a BYOK/ passthrough upstream can surface the provider's own 400). The matcher stays deliberately tight, same doctrine as `is_context_length_error`: misclassifying a generic 400 as recoverable would mutate history for an error that removing images cannot fix. **Bounded agent rounds in benchmark trials.** The harness default `DEFAULT_MAX_AGENT_ROUNDS = 32` ended solo trials mid-work when a turn rotated (max_tokens rotation on thinking-heavy models burned rounds fast). Benchmark trials already have a wall-clock budget as the real limit, so the round cap only converts recoverable rotation into trial death. Default is now 0 (unbounded — `BUZZ_AGENT_MAX_ROUNDS=0` is the agent's own documented unbounded value); per-agent `budget.max_calls` in manifests still overrides. Tests: 400-body classification asserted through `complete()` on the shared-post path and at the `openrouter_post` terminal, both proving single-attempt (no retry of a deterministic capability rejection); python tests updated for 0-is-legal with a negative arm at -1. Signed-off-by: Eva <011987e296fd5006292d2f930b574be47c7801048d1983c46c425d3c95f0cffd@buzz.block.builderlab.xyz> Co-authored-by: Eva <011987e296fd5006292d2f930b574be47c7801048d1983c46c425d3c95f0cffd@buzz.block.builderlab.xyz>
1 parent 02f640b commit b043860

3 files changed

Lines changed: 107 additions & 9 deletions

File tree

‎benchmarks/harbor-buzz-orchestra/src/harbor_buzz_orchestra/container_runtime.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@
2424
from .provisioning import AgentCredential, TrialHandle
2525
from .runtime import RuntimeResult
2626

27-
DEFAULT_MAX_AGENT_ROUNDS = 32
27+
DEFAULT_MAX_AGENT_ROUNDS = 0 # 0 = unbounded (BUZZ_AGENT_MAX_ROUNDS=0); the trial budget is the clock
2828
# Container-side layout for the uploaded Buzz stack.
2929
REMOTE_ROOT = "/opt/buzz"
3030
REMOTE_BIN = f"{REMOTE_ROOT}/bin"
@@ -80,8 +80,8 @@ def __init__(
8080
readiness_timeout_seconds: float = 60.0,
8181
poll_seconds: float = 1.0,
8282
) -> None:
83-
if max_agent_rounds <= 0:
84-
raise ValueError("max_agent_rounds must be positive")
83+
if max_agent_rounds < 0:
84+
raise ValueError("max_agent_rounds must be >= 0 (0 = unbounded)")
8585
if readiness_timeout_seconds <= 0:
8686
raise ValueError("readiness_timeout_seconds must be positive")
8787
self.logs_dir = Path(logs_dir)

‎benchmarks/harbor-buzz-orchestra/tests/test_container_runtime.py‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -247,7 +247,7 @@ async def test_forwarder_bridges_the_canonical_relay_address(tmp_path):
247247
rt._ws_authority("http://relay")
248248

249249

250-
@pytest.mark.parametrize(("configured", "expected"), [(None, "32"), (7, "7")])
250+
@pytest.mark.parametrize(("configured", "expected"), [(None, "0"), (7, "7")])
251251
async def test_launch_wires_the_desktop_environment(tmp_path, configured, expected):
252252
manifest = write_manifest(tmp_path)
253253
agent_class = manifest.roster[0]
@@ -290,9 +290,12 @@ async def test_launch_wires_the_desktop_environment(tmp_path, configured, expect
290290
)
291291

292292

293-
def test_runtime_rejects_unbounded_agent_rounds(tmp_path):
294-
with pytest.raises(ValueError, match="positive"):
295-
runtime(tmp_path, max_agent_rounds=0)
293+
def test_runtime_validates_construction_bounds(tmp_path):
294+
# 0 is legal and means unbounded (BUZZ_AGENT_MAX_ROUNDS=0); the trial
295+
# budget is the clock. Only negatives are rejected.
296+
runtime(tmp_path, max_agent_rounds=0)
297+
with pytest.raises(ValueError, match="unbounded"):
298+
runtime(tmp_path, max_agent_rounds=-1)
296299
with pytest.raises(ValueError, match="positive"):
297300
runtime(tmp_path, readiness_timeout_seconds=0)
298301

‎crates/buzz-agent/src/llm.rs‎

Lines changed: 97 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1932,9 +1932,24 @@ fn classify_body_read_error(
19321932
}
19331933
}
19341934

1935+
/// Provider bodies that mean "this model cannot accept image input", the
1936+
/// signal the agent loop uses to strip rejected images from history and
1937+
/// continue the turn (see `replace_unsupported_images`).
1938+
///
1939+
/// Deliberately tight, same doctrine as [`is_context_length_error`]: each
1940+
/// phrase is a verbatim capability rejection observed live. Misclassifying a
1941+
/// generic 400 as recoverable would mutate history for an error that removing
1942+
/// images cannot fix.
19351943
fn is_unsupported_image_input_error(body: &str) -> bool {
1936-
body.to_ascii_lowercase()
1937-
.contains("no endpoints found that support image input")
1944+
let b = body.to_ascii_lowercase();
1945+
// OpenRouter 404: no provider endpoint accepts images for this model.
1946+
b.contains("no endpoints found that support image input")
1947+
// OpenAI-compatible 400 from text-only single-model deployments,
1948+
// e.g. Crusoe serverless GLM: `"crusoeai/GLM-5.2-NVFP4 is not a
1949+
// multimodal model"`. Without this arm the 400 is terminal, the image
1950+
// stays in history, and every subsequent request in the session fails
1951+
// identically — the turn wedges until the harness/user gives up.
1952+
|| b.contains("is not a multimodal model")
19381953
}
19391954

19401955
/// Build the terminal `AgentError::Llm` for a `post()` exit that has given up
@@ -2129,6 +2144,13 @@ where
21292144
"{status}: {body}"
21302145
))));
21312146
}
2147+
// Image-capability rejection is equally recoverable and equally
2148+
// deterministic: a text-only deployment 400s the same request
2149+
// forever. Typed here (not just on the 404 arm) because
2150+
// OpenAI-compatible providers report it as a 400.
2151+
if status == 400 && is_unsupported_image_input_error(&body) {
2152+
return Err(PostError::Agent(AgentError::UnsupportedImageInput(body)));
2153+
}
21322154
return Err(PostError::Agent(AgentError::Llm(format!(
21332155
"{status}: {body}"
21342156
))));
@@ -2535,6 +2557,12 @@ async fn openrouter_post(
25352557
if status == 400 && is_context_length_error(&body) {
25362558
return Err(AgentError::LlmContextExceeded(format!("{status}: {body}")));
25372559
}
2560+
// Same 400-shaped image rejection as the shared `post()` terminal:
2561+
// OpenRouter normally reports this as a 404 (handled above), but a
2562+
// BYOK/passthrough upstream can surface the provider's own 400.
2563+
if status == 400 && is_unsupported_image_input_error(&body) {
2564+
return Err(AgentError::UnsupportedImageInput(body));
2565+
}
25382566
return Err(AgentError::Llm(format!("{status}: {body}")));
25392567
}
25402568
if let Some(len) = resp.content_length() {
@@ -7544,6 +7572,73 @@ mod tests {
75447572
);
75457573
}
75467574

7575+
/// OpenAI-compatible text-only deployments report the image rejection as a
7576+
/// 400, not OpenRouter's 404 — Crusoe serverless GLM answers
7577+
/// `"crusoeai/GLM-5.2-NVFP4 is not a multimodal model"` to every request
7578+
/// whose history contains an image. Before the 400 arm existed, this fell
7579+
/// through to terminal `AgentError::Llm`: the image stayed in history and
7580+
/// every later call in the session failed identically (measured live:
7581+
/// 8 wedged benchmark trials, 40 min of doomed retries each). Asserted
7582+
/// through `complete()` so the arm's return path into the convergence
7583+
/// mapper is covered, same doctrine as the context-400 tests above.
7584+
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7585+
async fn openai_400_unsupported_image_is_typed_through_complete() {
7586+
let (base_url, captured) = spawn_sequence_stub(vec![StubHttpResponse {
7587+
status: 400,
7588+
body: json!({"error":{"message":"crusoeai/GLM-5.2-NVFP4 is not a multimodal model","type":"invalid_request_error"}}),
7589+
}])
7590+
.await;
7591+
let mut c = cfg(Provider::OpenAi);
7592+
c.base_url = base_url;
7593+
let llm = Llm::new(&c).unwrap();
7594+
let err = complete_model(&llm, &c, "gpt-probe-model")
7595+
.await
7596+
.unwrap_err();
7597+
assert!(
7598+
matches!(&err, AgentError::UnsupportedImageInput(s) if s.contains("not a multimodal model")),
7599+
"a text-only deployment's 400 must reach the history-recovery path: got {err:?}"
7600+
);
7601+
assert_eq!(
7602+
captured.lock().await.len(),
7603+
1,
7604+
"a deterministic capability rejection must not be retried"
7605+
);
7606+
}
7607+
7608+
/// Same 400-shaped rejection at the OpenRouter terminal, which has its own
7609+
/// status ladder: a BYOK/passthrough upstream can surface the provider's
7610+
/// own 400 body instead of OpenRouter's 404 routing error.
7611+
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7612+
async fn openrouter_post_400_unsupported_image_is_typed_and_not_retried() {
7613+
let (url, _captured, attempts) = spawn_openrouter_stub(vec![CannedResponse::new(
7614+
400,
7615+
r#"{"error":{"message":"crusoeai/GLM-5.2-NVFP4 is not a multimodal model"}}"#,
7616+
)])
7617+
.await;
7618+
let http = Client::builder()
7619+
.timeout(Duration::from_secs(5))
7620+
.build()
7621+
.unwrap();
7622+
let err = openrouter_post(
7623+
&http,
7624+
&format!("{url}/x"),
7625+
&json!({}),
7626+
"key",
7627+
Duration::from_secs(5),
7628+
)
7629+
.await
7630+
.unwrap_err();
7631+
assert!(
7632+
matches!(&err, AgentError::UnsupportedImageInput(s) if s.contains("not a multimodal model")),
7633+
"image rejection must reach the history-recovery path: got {err:?}"
7634+
);
7635+
assert_eq!(
7636+
attempts.load(std::sync::atomic::Ordering::SeqCst),
7637+
1,
7638+
"a deterministic capability rejection must not be retried"
7639+
);
7640+
}
7641+
75477642
/// Every other 404 still maps to `LlmModelNotFound`, including one that
75487643
/// shares the `No endpoints found` prefix but is about the model rather than
75497644
/// the parameters — the discriminator is narrow enough that a genuinely

0 commit comments

Comments
 (0)