Skip to content

Commit fc8a174

Browse files
authored
fix(agents): harden structured error-code classification edge cases (#1663)
1 parent cecd031 commit fc8a174

6 files changed

Lines changed: 195 additions & 49 deletions

File tree

‎crates/buzz-acp/src/acp.rs‎

Lines changed: 44 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -105,6 +105,19 @@ pub enum AcpError {
105105
AgentError { code: i64, message: String },
106106
}
107107

108+
/// Build an [`AcpError::AgentError`] from a JSON-RPC error object,
109+
/// preserving the numeric code. When the `message` field is missing or
110+
/// non-string, fall back to the full JSON object so provider-specific
111+
/// detail (e.g. a `data` field) is not lost.
112+
fn agent_error_from_json(error: &serde_json::Value) -> AcpError {
113+
let code = error.get("code").and_then(|c| c.as_i64()).unwrap_or(-32000);
114+
let message = match error.get("message").and_then(|m| m.as_str()) {
115+
Some(m) => m.to_string(),
116+
None => error.to_string(),
117+
};
118+
AcpError::AgentError { code, message }
119+
}
120+
108121
/// ACP client that owns an agent subprocess and communicates over its stdio.
109122
///
110123
/// One `AcpClient` per agent process. Multiple sessions can be created on the
@@ -863,13 +876,7 @@ impl AcpClient {
863876
if let Some(id) = msg.get("id") {
864877
if *id == serde_json::json!(expected_id) && msg.get("method").is_none() {
865878
if let Some(error) = msg.get("error") {
866-
let code = error.get("code").and_then(|c| c.as_i64()).unwrap_or(-32000);
867-
let message = error
868-
.get("message")
869-
.and_then(|m| m.as_str())
870-
.unwrap_or("unknown error")
871-
.to_string();
872-
return Err(AcpError::AgentError { code, message });
879+
return Err(agent_error_from_json(error));
873880
}
874881
return Ok(msg["result"].clone());
875882
}
@@ -1191,16 +1198,7 @@ impl AcpClient {
11911198
let _ = ack_tx
11921199
.send(crate::pool::SteerAck::PromptCompletedNeutral);
11931200
}
1194-
let code = error
1195-
.get("code")
1196-
.and_then(|c| c.as_i64())
1197-
.unwrap_or(-32000);
1198-
let message = error
1199-
.get("message")
1200-
.and_then(|m| m.as_str())
1201-
.unwrap_or("unknown error")
1202-
.to_string();
1203-
return Err(AcpError::AgentError { code, message });
1201+
return Err(agent_error_from_json(error));
12041202
}
12051203
if let Some((_, ack_tx)) = pending_steer.take() {
12061204
let _ =
@@ -3022,4 +3020,33 @@ mod tests {
30223020
client.handle_goose_usage_update(&bad2);
30233021
assert!(client.take_turn_usage().is_none());
30243022
}
3023+
3024+
#[test]
3025+
fn agent_error_from_json_falls_back_to_full_json_when_message_missing() {
3026+
// Errors without a string `message` field (e.g. only a `data` field) must
3027+
// not be silently truncated to "unknown error" — the full JSON is preserved.
3028+
let error = serde_json::json!({"code": -32000, "data": "quota exceeded"});
3029+
match super::agent_error_from_json(&error) {
3030+
AcpError::AgentError { code, message } => {
3031+
assert_eq!(code, -32000);
3032+
assert!(
3033+
message.contains("quota exceeded"),
3034+
"expected full JSON in message, got: {message}"
3035+
);
3036+
}
3037+
other => panic!("expected AgentError, got {other:?}"),
3038+
}
3039+
}
3040+
3041+
#[test]
3042+
fn agent_error_from_json_uses_message_field_when_present() {
3043+
let error = serde_json::json!({"code": -32001, "message": "auth denied"});
3044+
match super::agent_error_from_json(&error) {
3045+
AcpError::AgentError { code, message } => {
3046+
assert_eq!(code, -32001);
3047+
assert_eq!(message, "auth denied");
3048+
}
3049+
other => panic!("expected AgentError, got {other:?}"),
3050+
}
3051+
}
30253052
}

‎crates/buzz-acp/src/lib.rs‎

Lines changed: 4 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -2875,6 +2875,10 @@ fn handle_prompt_result(
28752875
| acp::AcpError::Timeout(_)
28762876
| acp::AcpError::Protocol(_)
28772877
);
2878+
let error_code = match &e {
2879+
acp::AcpError::AgentError { code, .. } => Some(*code),
2880+
_ => None,
2881+
};
28782882
if is_transport_error {
28792883
tracing::warn!(
28802884
agent = agent_index,
@@ -2884,10 +2888,6 @@ fn handle_prompt_result(
28842888
error = %e,
28852889
"transport/protocol error — respawning agent"
28862890
);
2887-
let error_code = match &e {
2888-
acp::AcpError::AgentError { code, .. } => Some(*code),
2889-
_ => None,
2890-
};
28912891
emit_turn_error(&e.to_string(), error_code);
28922892

28932893
let index = result.agent.index;
@@ -2914,10 +2914,6 @@ fn handle_prompt_result(
29142914
error = %e,
29152915
"agent_returned (application error — pipe intact)"
29162916
);
2917-
let error_code = match &e {
2918-
acp::AcpError::AgentError { code, .. } => Some(*code),
2919-
_ => None,
2920-
};
29212917
emit_turn_error(&e.to_string(), error_code);
29222918
pool.return_agent(result.agent);
29232919
}

‎desktop/src-tauri/src/managed_agents/storage.rs‎

Lines changed: 42 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -532,8 +532,13 @@ fn bytecount_newlines(buf: &[u8]) -> usize {
532532
buf.iter().filter(|&&b| b == b'\n').count()
533533
}
534534

535+
/// A meaningful error recovered from an exited agent's log tail.
535536
pub struct AgentLogError {
537+
/// The full log line, wrapped as `Agent reported error…` for display.
536538
pub message: String,
539+
/// JSON-RPC error code parsed from the line's `(code N)` marker, or a
540+
/// synthetic code for known bare prefixes. `None` for legacy-format
541+
/// lines that carry no code (or when the code fails to parse as i64).
537542
pub code: Option<i64>,
538543
}
539544

@@ -557,6 +562,21 @@ pub fn meaningful_agent_error_from_log(path: &Path) -> Option<AgentLogError> {
557562
code: None,
558563
});
559564
}
565+
// Bare prefixes emitted by older agent binaries whose Display still leaks
566+
// unwrapped errors. Promote these so they surface instead of the generic
567+
// "harness exited with status N" fallback.
568+
if line.starts_with("llm auth:") {
569+
return Some(AgentLogError {
570+
message: format!("Agent reported error: {line}"),
571+
code: Some(-32001),
572+
});
573+
}
574+
if line.starts_with("llm model not found:") {
575+
return Some(AgentLogError {
576+
message: format!("Agent reported error: {line}"),
577+
code: Some(-32002),
578+
});
579+
}
560580
None
561581
})
562582
}
@@ -884,7 +904,28 @@ mod tests {
884904
#[test]
885905
fn meaningful_agent_error_from_log_promotes_unwrapped_llm_auth() {
886906
let file = write_log("noise\nllm auth: denied\n");
887-
assert!(super::meaningful_agent_error_from_log(file.path()).is_none());
907+
let result = super::meaningful_agent_error_from_log(file.path()).unwrap();
908+
assert_eq!(result.message, "Agent reported error: llm auth: denied");
909+
assert_eq!(result.code, Some(-32001));
910+
}
911+
912+
#[test]
913+
fn meaningful_agent_error_from_log_promotes_bare_model_not_found() {
914+
let file = write_log("noise\nllm model not found: (some-model) 404\n");
915+
let result = super::meaningful_agent_error_from_log(file.path()).unwrap();
916+
assert_eq!(
917+
result.message,
918+
"Agent reported error: llm model not found: (some-model) 404"
919+
);
920+
assert_eq!(result.code, Some(-32002));
921+
}
922+
923+
#[test]
924+
fn meaningful_agent_error_from_log_promotes_legacy_format() {
925+
let file = write_log("noise\nAgent reported error: llm: 500 internal\n");
926+
let result = super::meaningful_agent_error_from_log(file.path()).unwrap();
927+
assert_eq!(result.message, "Agent reported error: llm: 500 internal");
928+
assert_eq!(result.code, None);
888929
}
889930

890931
#[test]

‎desktop/src/features/agents/lib/friendlyAgentLastError.test.mjs‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -148,3 +148,68 @@ test("friendlyTurnErrorCopy: missing code falls back to raw text", () => {
148148
test("friendlyTurnErrorCopy: unknown code passes raw text through", () => {
149149
assert.equal(friendlyTurnErrorCopy("raw error", 12345), "raw error");
150150
});
151+
152+
// --- structured-code hardening ---
153+
154+
test("unknown code prevents string-pattern cross-classification", () => {
155+
// code -32003 is structured and unrecognized — must NOT fall through to
156+
// the legacy string path that would wrongly promote this to denied.
157+
const result = friendlyAgentLastError(
158+
"llm auth: rate limiter denial",
159+
-32003,
160+
);
161+
assert.deepEqual(result, {
162+
severity: "generic",
163+
copy: "llm auth: rate limiter denial",
164+
});
165+
});
166+
167+
test("NaN code param treated as absent — string path applies", () => {
168+
// NaN is not finite; falls back to string matching.
169+
const result = friendlyAgentLastError("llm auth: denied", NaN);
170+
assert.deepEqual(result, {
171+
severity: "denied",
172+
copy: RELAY_MESH_DENIED_COPY,
173+
});
174+
});
175+
176+
test("embedded code -32001 recovered from message when code param is null", () => {
177+
const result = friendlyAgentLastError(
178+
"Agent reported error (code -32001): llm auth: 401",
179+
null,
180+
);
181+
assert.deepEqual(result, {
182+
severity: "denied",
183+
copy: RELAY_MESH_DENIED_COPY,
184+
});
185+
});
186+
187+
test("embedded code -32002 recovered from message when code param is undefined", () => {
188+
const result = friendlyAgentLastError(
189+
"Agent reported error (code -32002): llm model not found: x",
190+
undefined,
191+
);
192+
assert.deepEqual(result, {
193+
severity: "denied",
194+
copy: MODEL_NOT_FOUND_COPY,
195+
});
196+
});
197+
198+
test("embedded unknown code is authoritative — no cross-classification", () => {
199+
const result = friendlyAgentLastError(
200+
"Agent reported error (code -32099): llm auth: weird",
201+
null,
202+
);
203+
assert.deepEqual(result, {
204+
severity: "generic",
205+
copy: "Agent reported error (code -32099): llm auth: weird",
206+
});
207+
});
208+
209+
test("friendlyTurnErrorCopy: garbage string code coerces to NaN → string path", () => {
210+
// "garbage" → NaN → not finite → null → string prefix matches "llm auth:".
211+
assert.equal(
212+
friendlyTurnErrorCopy("llm auth: denied", "garbage"),
213+
RELAY_MESH_DENIED_COPY,
214+
);
215+
});
Lines changed: 39 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -1,27 +1,29 @@
11
/**
22
* Promote certain machine-readable `lastError` strings to user-facing copy.
33
*
4-
* The mesh-llm seam (Max's commit `5196203…`) flows like this:
5-
* buzz-agent — gets HTTP 401/403 from the OpenAI-compatible mesh endpoint
6-
* → raises `AgentError::LlmAuth("…")` (json_rpc_code `-32001`,
7-
* Display prefix `"llm auth: …"`)
8-
* buzz-acp — wraps it as `AcpError::AgentError("Agent reported error: llm auth: …")`
9-
* desktop managed-agent supervisor — on nonzero exit, scans `read_log_tail`
10-
* for `"Agent reported error:"` / `"llm auth:"` and persists
11-
* that line into `ManagedAgent.lastError` instead of the
12-
* generic `"harness exited with status …"`.
4+
* The error classification seam flows like this:
5+
* buzz-agent — classifies LLM failures into `AgentError` variants with
6+
* JSON-RPC codes (`-32001` auth, `-32002` model-not-found,
7+
* `-32000` generic), defined in `crates/buzz-agent/src/types.rs`.
8+
* buzz-acp — preserves the code structurally in
9+
* `AcpError::AgentError { code, message }`, whose Display is
10+
* `"Agent reported error (code N): message"`, and includes
11+
* `code` in `turn_error` observer events.
12+
* desktop supervisor — on nonzero exit, recovers `{ message, code }` from
13+
* the log tail (`managed_agents/storage.rs`) into
14+
* `ManagedAgent.lastError` / `lastErrorCode`.
1315
*
14-
* v1 caveat (named, not hidden): the typed `-32001` code never reaches
15-
* desktop structurally — desktop only supervises the child process and ACP's
16-
* `ObserverHandle` is in-process inside that child. So `lastError` is the
17-
* recovered string, not the original code. The follow-up to make this 9/10
18-
* structural is an ACP status file or desktop-owned observer sink; that lives
19-
* elsewhere. For now we match the string this function exists to render.
16+
* This function dispatches on the numeric code first (works for any harness),
17+
* then recovers a code embedded in the message string (handles records where
18+
* the `lastErrorCode` field was lost, e.g. downgrade or pre-code records with
19+
* new-format strings), then falls back to legacy string prefixes for records
20+
* written before structured codes existed.
2021
*
2122
* Returns:
2223
* - null when there's nothing to show (null/empty lastError).
2324
* - A `{ severity: "denied"; copy: string }` object for the auth-failure
24-
* case, so the UI can render with the right visual weight (destructive).
25+
* and model-not-found cases, so the UI can render with the right visual
26+
* weight (destructive).
2527
* - A `{ severity: "generic"; copy: string }` pass-through for any other
2628
* lastError, so generic harness exits still surface their text instead of
2729
* being swallowed.
@@ -40,6 +42,13 @@ export const RELAY_MESH_DENIED_COPY =
4042
export const MODEL_NOT_FOUND_COPY =
4143
"The configured model is not available — open agent settings and select a different one from the dropdown.";
4244

45+
const EMBEDDED_CODE_RE = /^Agent reported error \(code (-?\d+)\): /;
46+
47+
function recoverEmbeddedCode(trimmed: string): number | null {
48+
const match = EMBEDDED_CODE_RE.exec(trimmed);
49+
return match ? Number(match[1]) : null;
50+
}
51+
4352
export function friendlyAgentLastError(
4453
raw: string | null,
4554
code?: number | null,
@@ -48,18 +57,25 @@ export function friendlyAgentLastError(
4857
const trimmed = raw.trim();
4958
if (trimmed.length === 0) return null;
5059

51-
// Structured code path — no string matching, works across all providers.
52-
if (code != null) {
53-
switch (code) {
60+
// Structured code first; a code embedded in the message string is the
61+
// same signal recovered from a record that lost the field.
62+
const effectiveCode = Number.isFinite(code)
63+
? (code as number)
64+
: recoverEmbeddedCode(trimmed);
65+
if (effectiveCode != null) {
66+
switch (effectiveCode) {
5467
case -32001:
5568
return { severity: "denied", copy: RELAY_MESH_DENIED_COPY };
5669
case -32002:
5770
return { severity: "denied", copy: MODEL_NOT_FOUND_COPY };
5871
}
72+
// A structured code we don't recognize is authoritative — don't let
73+
// string patterns cross-classify it.
74+
return { severity: "generic", copy: trimmed };
5975
}
6076

61-
// Match either the unwrapped buzz-agent prefix or the buzz-acp wrap.
62-
// The desktop supervisor recovers whichever appears first in the log tail.
77+
// Legacy string fallback for records written before codes existed.
78+
// Match either the unwrapped buzz-agent prefix or the buzz-acp v0 wrap.
6379
if (
6480
trimmed.startsWith("Agent reported error: llm auth:") ||
6581
trimmed.startsWith("llm auth:")
@@ -77,5 +93,6 @@ export function friendlyAgentLastError(
7793
*/
7894
export function friendlyTurnErrorCopy(raw: string, code: unknown): string {
7995
const numeric = code == null ? null : Number(code);
80-
return friendlyAgentLastError(raw, numeric)?.copy ?? raw;
96+
const safe = Number.isFinite(numeric) ? (numeric as number) : null;
97+
return friendlyAgentLastError(raw, safe)?.copy ?? raw;
8198
}

‎desktop/tests/e2e/agent-error-state-screenshots.spec.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ const MODEL_NOT_FOUND_AGENT = {
2626
name: "Databricks Agent",
2727
status: "stopped" as const,
2828
lastError:
29-
"Agent reported error: llm: (goose-databricks-llama-3-3-70b) 404 Not Found: model not found",
29+
"Agent reported error (code -32002): llm model not found: (goose-databricks-llama-3-3-70b) 404 Not Found: model not found",
3030
lastErrorCode: -32002,
3131
};
3232

0 commit comments

Comments
 (0)