Skip to content

Commit 051c3a2

Browse files
authored
fix(buzz-acp): report missing models without retrying (block#7538)
## Summary When an agent reports model-not-found, Buzz retries the unavailable model and delays the failure reply until retries are exhausted. Stop retrying this error and immediately post a threaded recovery notice. The notice tells users to select a different model in agent settings, save, restart the agent to apply the configuration, and re-send their request. This adds one error-handling branch and regression coverage in `buzz-acp`. It matches `-32002` errors containing `model not found`. Other resource-not-found errors, such as stale sessions, retain the existing retry behavior. Detailed error events remain available for diagnosis. The existing restart policy is unchanged. ### Related issue None found in existing issue/PR searches for model-not-found recovery. ### Testing Playwright captured and visually checked the thread UI with seeded conversation data and the exact recovery text. The check opens the request's thread, confirms no reply before the failure, injects the notice, and verifies the full text is visible. [Before/after screenshots](block#7538 (comment)) show the corrected save-and-restart instructions. These are local test captures, not a deployed provider recovery flow. Generated with Codex --------- Signed-off-by: Diem Nguyen <[email protected]>
1 parent 12023a3 commit 051c3a2

1 file changed

Lines changed: 200 additions & 7 deletions

File tree

‎crates/buzz-acp/src/lib.rs‎

Lines changed: 200 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -4783,6 +4783,23 @@ fn handle_prompt_result(
47834783
} else {
47844784
hard_timeout_fate_suffix = Some(" — requeued for retry (recently active)");
47854785
}
4786+
} else if matches!(
4787+
&result.outcome,
4788+
PromptOutcome::Error(acp::AcpError::AgentError { code: -32002, message })
4789+
if message.contains("model not found")
4790+
) {
4791+
// Retrying the same missing model cannot repair its configuration.
4792+
tracing::warn!(
4793+
channel_id = %batch.channel_id,
4794+
events = batch.events.len(),
4795+
"dead-lettering batch immediately — model not found"
4796+
);
4797+
let content = "⚠️ I couldn't process the last request: the configured model \
4798+
wasn't found at the provider's endpoint. Open agent settings, select a \
4799+
different model from the dropdown, and save your changes. Restart the agent \
4800+
to apply the new configuration, then re-send your request."
4801+
.to_string();
4802+
spawn_failure_notice(rest_client, &batch, content);
47864803
} else if matches!(&result.outcome, PromptOutcome::Error(e) if is_auth_error(e)) {
47874804
// Auth errors are non-retryable: the token won't self-repair
47884805
// between retries, so requeueing only wastes attempt slots and
@@ -11012,10 +11029,192 @@ mod error_outcome_emission_tests {
1101211029
);
1101311030
}
1101411031

11032+
#[tokio::test]
11033+
async fn model_not_found_posts_recovery_notice_without_retrying() {
11034+
use tokio::io::{AsyncBufReadExt, AsyncReadExt, AsyncWriteExt, BufReader};
11035+
11036+
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
11037+
let rest = relay::RestClient {
11038+
http: reqwest::Client::new(),
11039+
base_url: format!("http://{}", listener.local_addr().unwrap()),
11040+
keys: Keys::generate(),
11041+
auth_tag_json: None,
11042+
};
11043+
let keys = Keys::generate();
11044+
let root = nostr::EventId::from_byte_array([0xaa; 32]);
11045+
let parent = nostr::EventId::from_byte_array([0xbb; 32]);
11046+
let event = EventBuilder::new(Kind::Custom(9), "test")
11047+
.tags([
11048+
nostr::Tag::parse(["e", &root.to_hex(), "", "root"]).unwrap(),
11049+
nostr::Tag::parse(["e", &parent.to_hex(), "", "reply"]).unwrap(),
11050+
])
11051+
.sign_with_keys(&keys)
11052+
.unwrap();
11053+
let channel_id = uuid::Uuid::new_v4();
11054+
let batch = FlushBatch {
11055+
channel_id,
11056+
scope: scope::SessionScope::Conversation { channel_id },
11057+
events: vec![BatchEvent {
11058+
event,
11059+
prompt_tag: "test".into(),
11060+
received_at: std::time::Instant::now(),
11061+
}],
11062+
cancelled_events: vec![],
11063+
cancel_reason: None,
11064+
};
11065+
11066+
let raw_error = r#"llm model not found: (gpt-6-astra) 404 Not Found: {"error_code":"NOT_FOUND","message":"'gpt-6-astra' does not exist."}"#;
11067+
let model_error = AcpError::AgentError {
11068+
code: -32002,
11069+
message: raw_error.to_string(),
11070+
};
11071+
let expected_error = model_error.to_string();
11072+
let observer = ObserverHandle::in_process();
11073+
11074+
let agent = dummy_agent(0).await;
11075+
let mut pool = AgentPool::from_slots(vec![None]);
11076+
let task_id = pool.join_set.spawn(async {}).id();
11077+
pool.task_map_mut().insert(
11078+
task_id,
11079+
crate::pool::TaskMeta {
11080+
agent_index: 0,
11081+
channel_id: None,
11082+
scope: None,
11083+
turn_id: "test-turn-id".to_string(),
11084+
recoverable_batch: None,
11085+
control_tx: None,
11086+
steer_tx: None,
11087+
successful_steer_deliveries: HashSet::new(),
11088+
},
11089+
);
11090+
let mut queue = EventQueue::new(config::DedupMode::Queue);
11091+
let config = test_config();
11092+
let mut heartbeat_in_flight = false;
11093+
let removed_channels = std::collections::HashSet::new();
11094+
let mut crash_history = vec![SlotCircuit {
11095+
crash_times: Vec::new(),
11096+
open_until: None,
11097+
respawn_in_flight: false,
11098+
}];
11099+
let (respawn_tx, _respawn_rx) = mpsc::channel(8);
11100+
let mut respawn_tasks = tokio::task::JoinSet::new();
11101+
let result = PromptResult {
11102+
agent,
11103+
source: PromptSource::Channel(scope::SessionScope::Conversation { channel_id }),
11104+
turn_id: "test-turn-id".to_string(),
11105+
outcome: PromptOutcome::Error(model_error),
11106+
batch: Some(batch),
11107+
};
11108+
handle_prompt_result(
11109+
&mut pool,
11110+
&mut queue,
11111+
&config,
11112+
result,
11113+
&mut heartbeat_in_flight,
11114+
&removed_channels,
11115+
&mut crash_history,
11116+
&respawn_tx,
11117+
&mut respawn_tasks,
11118+
Some(observer.clone()),
11119+
Some(&rest),
11120+
);
11121+
11122+
// The batch must not be requeued: pending_channels returns 0.
11123+
assert_eq!(
11124+
queue.pending_channels(),
11125+
0,
11126+
"model-not-found must stop immediately — batch must not be requeued"
11127+
);
11128+
assert_eq!(
11129+
queue.queued_event_count(channel_id),
11130+
0,
11131+
"model-not-found must stop immediately — no events should be pending"
11132+
);
11133+
11134+
assert!(
11135+
pool.agents_mut()[0].is_some(),
11136+
"healthy process remains reusable"
11137+
);
11138+
assert!(respawn_tasks.is_empty());
11139+
let errors: Vec<_> = observer
11140+
.snapshot()
11141+
.into_iter()
11142+
.filter(|event| event.kind == "turn_error")
11143+
.collect();
11144+
assert_eq!(errors.len(), 1);
11145+
assert_eq!(errors[0].payload["code"], -32002);
11146+
assert_eq!(errors[0].payload["error"], expected_error);
11147+
11148+
// Capture the real signed notice sent by handle_prompt_result, without a live relay.
11149+
let notice: nostr::Event = tokio::time::timeout(Duration::from_secs(3), async {
11150+
let (socket, _) = listener.accept().await.unwrap();
11151+
let mut reader = BufReader::new(socket);
11152+
let mut line = String::new();
11153+
reader.read_line(&mut line).await.unwrap();
11154+
assert_eq!(line, "POST /events HTTP/1.1\r\n");
11155+
let mut content_length = None;
11156+
for _ in 0..64 {
11157+
line.clear();
11158+
assert_ne!(reader.read_line(&mut line).await.unwrap(), 0);
11159+
if line == "\r\n" {
11160+
break;
11161+
}
11162+
if let Some(value) = line.to_ascii_lowercase().strip_prefix("content-length:") {
11163+
content_length = Some(value.trim().parse::<usize>().unwrap());
11164+
}
11165+
}
11166+
let size = content_length.expect("request Content-Length");
11167+
assert!(size < 65536);
11168+
let mut body = vec![0; size];
11169+
reader.read_exact(&mut body).await.unwrap();
11170+
reader
11171+
.get_mut()
11172+
.write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: close\r\n\r\n{}")
11173+
.await
11174+
.unwrap();
11175+
serde_json::from_slice(&body).unwrap()
11176+
})
11177+
.await
11178+
.expect("failure notice must be posted on the first failure");
11179+
notice.verify().unwrap();
11180+
assert_eq!(notice.pubkey, rest.keys.public_key());
11181+
assert_eq!(notice.kind, Kind::Custom(9));
11182+
assert_eq!(
11183+
notice.content,
11184+
"⚠️ I couldn't process the last request: the configured model wasn't found at the provider's endpoint. Open agent settings, select a different model from the dropdown, and save your changes. Restart the agent to apply the new configuration, then re-send your request."
11185+
);
11186+
let tags = serde_json::to_value(&notice.tags).unwrap();
11187+
assert!(tags
11188+
.as_array()
11189+
.unwrap()
11190+
.iter()
11191+
.any(|tag| tag[0] == "h" && tag[1] == channel_id.to_string()));
11192+
let threading = queue::parse_thread_tags(&notice);
11193+
assert_eq!(threading.root_event_id, Some(root.to_hex()));
11194+
assert_eq!(threading.parent_event_id, Some(parent.to_hex()));
11195+
}
11196+
1101511197
/// A non-auth application error (e.g. usage credits) must still follow the
1101611198
/// standard requeue path so today's behavior is unchanged.
1101711199
#[tokio::test]
1101811200
async fn non_auth_application_error_is_requeued() {
11201+
assert_application_error_is_requeued(acp::AcpError::AgentError {
11202+
code: -32000,
11203+
message: "Usage credits required for 1M context".to_string(),
11204+
})
11205+
.await;
11206+
}
11207+
11208+
#[tokio::test]
11209+
async fn non_model_resource_not_found_is_requeued() {
11210+
assert_application_error_is_requeued(acp::AcpError::AgentError {
11211+
code: -32002,
11212+
message: "Resource not found: session no longer exists".to_string(),
11213+
})
11214+
.await;
11215+
}
11216+
11217+
async fn assert_application_error_is_requeued(error: acp::AcpError) {
1101911218
let keys = nostr::Keys::generate();
1102011219
let event = nostr::EventBuilder::new(nostr::Kind::Custom(9), "test")
1102111220
.sign_with_keys(&keys)
@@ -11033,12 +11232,6 @@ mod error_outcome_emission_tests {
1103311232
cancel_reason: None,
1103411233
};
1103511234

11036-
// Usage-credits error — AgentError but NOT an auth error.
11037-
let usage_error = acp::AcpError::AgentError {
11038-
code: -32000,
11039-
message: "Usage credits required for 1M context".to_string(),
11040-
};
11041-
1104211235
let agent = dummy_agent(0).await;
1104311236
let mut pool = AgentPool::from_slots(vec![None]);
1104411237
let task_id = pool.join_set.spawn(async {}).id();
@@ -11070,7 +11263,7 @@ mod error_outcome_emission_tests {
1107011263
agent,
1107111264
source: PromptSource::Channel(scope::SessionScope::Conversation { channel_id }),
1107211265
turn_id: "test-turn-id".to_string(),
11073-
outcome: PromptOutcome::Error(usage_error),
11266+
outcome: PromptOutcome::Error(error),
1107411267
batch: Some(batch),
1107511268
};
1107611269
handle_prompt_result(

0 commit comments

Comments
 (0)