Skip to content

Commit bcb93f5

Browse files
author
Tryn McCaffery
committed
fix(acp): survive transient discovery overload
Signed-off-by: Tryn McCaffery <[email protected]>
1 parent b49115e commit bcb93f5

2 files changed

Lines changed: 163 additions & 23 deletions

File tree

‎crates/buzz-acp/src/lib.rs‎

Lines changed: 29 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1351,7 +1351,12 @@ async fn tokio_main() -> Result<()> {
13511351
tracing::warn!("failed to set startup watermark: {e}");
13521352
}
13531353

1354-
tracing::info!("connected to relay at {}", config.relay_url);
1354+
tracing::info!(
1355+
target: "buzz_acp::runtime_lifecycle",
1356+
runtime_state = "transport_authenticated",
1357+
relay_url = %config.relay_url,
1358+
"relay transport authenticated; runtime is not ready until discovery and subscriptions finish"
1359+
);
13551360

13561361
relay
13571362
.subscribe_membership_notifications()
@@ -1426,10 +1431,21 @@ async fn tokio_main() -> Result<()> {
14261431
}
14271432
}
14281433

1429-
let channel_info_map = relay
1430-
.discover_channels()
1431-
.await
1432-
.map_err(|e| anyhow::anyhow!("channel discovery error: {e}"))?;
1434+
tracing::info!(
1435+
target: "buzz_acp::runtime_lifecycle",
1436+
runtime_state = "discovering_channels",
1437+
"starting canonical channel discovery"
1438+
);
1439+
let channel_info_map = relay.discover_channels().await.map_err(|e| {
1440+
tracing::error!(
1441+
target: "buzz_acp::runtime_lifecycle",
1442+
runtime_state = "startup_failed",
1443+
startup_stage = "channel_discovery",
1444+
error = %e,
1445+
"runtime did not become ready because channel discovery failed"
1446+
);
1447+
anyhow::anyhow!("channel discovery error: {e}")
1448+
})?;
14331449

14341450
tracing::info!("discovered {} channel(s)", channel_info_map.len());
14351451
let channel_ids: Vec<Uuid> = channel_info_map.keys().copied().collect();
@@ -1485,6 +1501,14 @@ async fn tokio_main() -> Result<()> {
14851501
}
14861502
}
14871503

1504+
tracing::info!(
1505+
target: "buzz_acp::runtime_lifecycle",
1506+
runtime_state = "subscriptions_enqueued",
1507+
discovered_channels = channel_info_map.len(),
1508+
subscribed_channels = subscribed_channel_ids.len(),
1509+
"canonical subscriptions are enqueued; runtime may now publish presence"
1510+
);
1511+
14881512
if let Some((observer, publisher, keys, agent_pubkey, owner_pubkey, owner)) =
14891513
relay_observer_publisher.take()
14901514
{

‎crates/buzz-acp/src/relay.rs‎

Lines changed: 134 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -244,12 +244,21 @@ fn is_retriable_status(status: reqwest::StatusCode) -> bool {
244244
status == 408 || status == 429 || (500..600).contains(&status)
245245
}
246246

247-
/// Base retry delays for transient HTTP failures: 500ms, 1s, 2s.
247+
/// Base retry delays for transient HTTP failures.
248+
///
249+
/// Managed desktops commonly launch several agents at once. Each agent performs
250+
/// two discovery queries, so a relay recovering from startup can remain
251+
/// overloaded for longer than the old 3.5-second window. Seven total attempts
252+
/// spread that burst over roughly 31.5 seconds (plus jitter) without retrying
253+
/// deterministic auth or request errors.
248254
/// Jitter (±20%) is applied at call time via `jittered_duration`.
249-
const REST_RETRY_BASE_DELAYS: [Duration; 3] = [
255+
const REST_RETRY_BASE_DELAYS: [Duration; 6] = [
250256
Duration::from_millis(500),
251257
Duration::from_millis(1000),
252258
Duration::from_millis(2000),
259+
Duration::from_millis(4000),
260+
Duration::from_millis(8000),
261+
Duration::from_millis(16000),
253262
];
254263

255264
fn unix_now_secs() -> u64 {
@@ -308,7 +317,8 @@ impl RestClient {
308317
Ok(format!("Nostr {}", self.sign_nip98(method, url, body)?))
309318
}
310319

311-
/// Retry helper: executes `build_request` up to 4 times (1 attempt + 3 retries)
320+
/// Retry helper: executes `build_request` up to seven times
321+
/// (one attempt plus six retries)
312322
/// on transient failures (408, 429, 5xx, timeout, connect errors).
313323
///
314324
/// NIP-98 auth events are re-signed on each attempt (they have a ±60s window).
@@ -324,24 +334,53 @@ impl RestClient {
324334
{
325335
let mut last_err = None;
326336

337+
let total_attempts = REST_RETRY_BASE_DELAYS.len() + 1;
327338
for (attempt, delay) in std::iter::once(None)
328339
.chain(REST_RETRY_BASE_DELAYS.iter().map(|d| Some(*d)))
329340
.enumerate()
330341
{
331342
if let Some(base) = delay {
332343
let jittered = jittered_duration(base);
333-
tracing::debug!(
334-
"retrying {method} {path} (attempt {attempt}) in {:.1}s",
335-
jittered.as_secs_f64()
344+
tracing::warn!(
345+
target: "buzz_acp::relay_lifecycle",
346+
relay_state = "http_retry_wait",
347+
method,
348+
path,
349+
next_attempt = attempt + 1,
350+
total_attempts,
351+
delay_seconds = jittered.as_secs_f64(),
352+
"transient relay request failure; waiting before retry"
336353
);
337354
tokio::time::sleep(jittered).await;
338355
}
339356

340357
match build_request().await {
341-
Ok(resp) if resp.status().is_success() => return Ok(resp),
358+
Ok(resp) if resp.status().is_success() => {
359+
if attempt > 0 {
360+
tracing::info!(
361+
target: "buzz_acp::relay_lifecycle",
362+
relay_state = "http_recovered",
363+
method,
364+
path,
365+
attempt = attempt + 1,
366+
total_attempts,
367+
"relay request recovered after transient failures"
368+
);
369+
}
370+
return Ok(resp);
371+
}
342372
Ok(resp) if is_retriable_status(resp.status()) => {
343373
let status = resp.status();
344-
tracing::warn!("{method} {path} returned retriable HTTP {status}");
374+
tracing::warn!(
375+
target: "buzz_acp::relay_lifecycle",
376+
relay_state = "http_transient_failure",
377+
method,
378+
path,
379+
attempt = attempt + 1,
380+
total_attempts,
381+
http_status = status.as_u16(),
382+
"relay request returned a transient HTTP status"
383+
);
345384
last_err = Some(RelayError::Http(format!(
346385
"{method} {path} returned HTTP {status}"
347386
)));
@@ -354,15 +393,34 @@ impl RestClient {
354393
)));
355394
}
356395
Err(e) if e.is_timeout() || e.is_connect() => {
357-
tracing::warn!("{method} {path} network error: {e}");
396+
tracing::warn!(
397+
target: "buzz_acp::relay_lifecycle",
398+
relay_state = "http_transient_failure",
399+
method,
400+
path,
401+
attempt = attempt + 1,
402+
total_attempts,
403+
error = %e,
404+
"relay request hit a transient network error"
405+
);
358406
last_err = Some(RelayError::Http(e.to_string()));
359407
}
360408
Err(e) => return Err(RelayError::Http(e.to_string())),
361409
}
362410
}
363411

364-
Err(last_err
365-
.unwrap_or_else(|| RelayError::Http(format!("{method} {path} failed after retries"))))
412+
let error = last_err
413+
.unwrap_or_else(|| RelayError::Http(format!("{method} {path} failed after retries")));
414+
tracing::error!(
415+
target: "buzz_acp::relay_lifecycle",
416+
relay_state = "http_retry_exhausted",
417+
method,
418+
path,
419+
total_attempts,
420+
error = %error,
421+
"relay request exhausted its transient retry budget"
422+
);
423+
Err(error)
366424
}
367425

368426
/// POST with NIP-98 auth and retry. Re-signs on each attempt.
@@ -2918,14 +2976,25 @@ async fn try_autonomous_reconnect(
29182976
let mut attempt = 0usize;
29192977
while attempt < backoffs.len() {
29202978
info!(
2921-
"autonomous reconnect attempt {}/{} to {relay_url}…",
2922-
attempt + 1,
2923-
backoffs.len()
2979+
target: "buzz_acp::relay_lifecycle",
2980+
relay_state = "reconnecting",
2981+
reconnect_mode = "autonomous",
2982+
attempt = attempt + 1,
2983+
total_attempts = backoffs.len(),
2984+
relay_url,
2985+
"attempting relay reconnect"
29242986
);
29252987
match do_connect(relay_url, keys, auth_tag).await {
29262988
Ok((new_ws, handshake_buffer)) => {
29272989
*ws = new_ws;
2928-
info!("autonomous reconnect succeeded (attempt {})", attempt + 1);
2990+
info!(
2991+
target: "buzz_acp::relay_lifecycle",
2992+
relay_state = "transport_reconnected",
2993+
reconnect_mode = "autonomous",
2994+
attempt = attempt + 1,
2995+
relay_url,
2996+
"relay transport reconnected; restoring subscriptions"
2997+
);
29292998
let handshake_ok = process_handshake_buffer(
29302999
ws,
29313000
handshake_buffer,
@@ -2950,7 +3019,16 @@ async fn try_autonomous_reconnect(
29503019
match resubscribe_after_reconnect(ws, cmd_rx, state, agent_pubkey_hex, true)
29513020
.await
29523021
{
2953-
ResubscribeResult::Ok => return ReconnectOutcome::Ok,
3022+
ResubscribeResult::Ok => {
3023+
info!(
3024+
target: "buzz_acp::relay_lifecycle",
3025+
relay_state = "online",
3026+
reconnect_mode = "autonomous",
3027+
active_subscriptions = state.active_subscriptions.len(),
3028+
"relay subscriptions restored; event replay is active"
3029+
);
3030+
return ReconnectOutcome::Ok;
3031+
}
29543032
ResubscribeResult::Shutdown => return ReconnectOutcome::Shutdown,
29553033
ResubscribeResult::RetryConnection => {
29563034
warn!("resubscribe failed after autonomous reconnect — treating as failed attempt");
@@ -3008,6 +3086,13 @@ async fn try_autonomous_reconnect(
30083086
attempt += 1;
30093087
}
30103088

3089+
tracing::error!(
3090+
target: "buzz_acp::relay_lifecycle",
3091+
relay_state = "reconnect_budget_exhausted",
3092+
reconnect_mode = "autonomous",
3093+
relay_url,
3094+
"bounded autonomous reconnect exhausted; entering persistent reconnect loop"
3095+
);
30113096
ReconnectOutcome::Failed
30123097
}
30133098

@@ -3059,11 +3144,25 @@ async fn wait_for_reconnect(
30593144
];
30603145
let mut attempt = state.backoff_step;
30613146
loop {
3062-
info!("attempting relay reconnect to {relay_url}…");
3147+
info!(
3148+
target: "buzz_acp::relay_lifecycle",
3149+
relay_state = "reconnecting",
3150+
reconnect_mode = "persistent",
3151+
attempt = attempt + 1,
3152+
relay_url,
3153+
"attempting relay reconnect"
3154+
);
30633155
match do_connect(relay_url, keys, auth_tag).await {
30643156
Ok((new_ws, handshake_buffer)) => {
30653157
*ws = new_ws;
3066-
info!("relay reconnected to {relay_url}");
3158+
info!(
3159+
target: "buzz_acp::relay_lifecycle",
3160+
relay_state = "transport_reconnected",
3161+
reconnect_mode = "persistent",
3162+
attempt = attempt + 1,
3163+
relay_url,
3164+
"relay transport reconnected; restoring subscriptions"
3165+
);
30673166
let handshake_ok = process_handshake_buffer(
30683167
ws,
30693168
handshake_buffer,
@@ -3086,6 +3185,13 @@ async fn wait_for_reconnect(
30863185
.await
30873186
{
30883187
ResubscribeResult::Ok => {
3188+
info!(
3189+
target: "buzz_acp::relay_lifecycle",
3190+
relay_state = "online",
3191+
reconnect_mode = "persistent",
3192+
active_subscriptions = state.active_subscriptions.len(),
3193+
"relay subscriptions restored; event replay is active"
3194+
);
30893195
// Drain any commands that arrived during do_connect() +
30903196
// resubscribe (which don't poll cmd_rx).
30913197
return drain_post_reconnect(ws, cmd_rx, state, agent_pubkey_hex).await;
@@ -4013,6 +4119,16 @@ mod tests {
40134119
}
40144120
}
40154121

4122+
#[test]
4123+
fn rest_retry_budget_outlasts_short_relay_startup_overload() {
4124+
assert_eq!(REST_RETRY_BASE_DELAYS.len() + 1, 7);
4125+
assert_eq!(
4126+
REST_RETRY_BASE_DELAYS.iter().sum::<Duration>(),
4127+
Duration::from_millis(31_500),
4128+
"managed-agent discovery must not exit after the old 3.5-second window"
4129+
);
4130+
}
4131+
40164132
#[test]
40174133
fn relay_ws_to_http_plain() {
40184134
assert_eq!(

0 commit comments

Comments
 (0)