Skip to content

Commit e4b3cb9

Browse files
committed
fix(acp): wake quiet hosts for respawns and queued retries
Signed-off-by: Logan Johnson <[email protected]>
1 parent 3c7f288 commit e4b3cb9

3 files changed

Lines changed: 276 additions & 48 deletions

File tree

‎crates/buzz-acp/src/lib.rs‎

Lines changed: 64 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ mod pool_lifecycle;
1111
mod prompt_framing;
1212
mod prompt_project;
1313
mod queue;
14+
mod recovery_wake;
1415
mod relay;
1516
mod scope;
1617
mod setup_mode;
@@ -3005,6 +3006,7 @@ async fn tokio_main() -> Result<()> {
30053006
Panic(tokio::task::JoinError),
30063007
SteerAck(SteerAckEvent),
30073008
Wake(u32, Result<AgentPool, String>),
3009+
Recovery(recovery_wake::RecoveryWake),
30083010
}
30093011

30103012
loop {
@@ -3090,37 +3092,8 @@ async fn tokio_main() -> Result<()> {
30903092
}
30913093
}
30923094

3093-
let mut respawn_collected = false;
3094-
while let Ok(rr) = respawn_rx.try_recv() {
3095-
crash_history[rr.index].respawn_in_flight = false;
3096-
match rr.result {
3097-
Ok((acp, protocol_version, agent_name)) => {
3098-
let agent = OwnedAgent {
3099-
index: rr.index,
3100-
acp,
3101-
state: SessionState::default(),
3102-
model_capabilities: None,
3103-
desired_model: config.model.clone(),
3104-
model_overridden: false,
3105-
desired_model_request_id: None,
3106-
desired_model_pending_ack: false,
3107-
startup_effort: config.effort_level.clone(),
3108-
agent_name,
3109-
goose_system_prompt_supported: None,
3110-
protocol_version,
3111-
};
3112-
pool.return_agent(agent);
3113-
tracing::info!(agent = rr.index, "respawn complete");
3114-
respawn_collected = true;
3115-
}
3116-
Err(e) => {
3117-
crash_history[rr.index].mark_spawn_failed();
3118-
tracing::warn!(agent = rr.index, "respawn failed: {e} — circuit re-opened");
3119-
}
3120-
}
3121-
}
31223095
// Reap completed respawn handles from the JoinSet. Payloads are
3123-
// delivered out-of-band through `respawn_rx` (drained above), so the
3096+
// delivered out-of-band through `respawn_rx` (selected below), so the
31243097
// JoinSet is never joined by the normal flow — Tokio retains finished
31253098
// tasks until `join_next`, so without this the set grows on every
31263099
// refill/crash recovery and `!respawn_tasks.is_empty()` would stay true
@@ -3130,26 +3103,27 @@ async fn tokio_main() -> Result<()> {
31303103
// slot's `respawn_in_flight` is cleared when its payload is received),
31313104
// not JoinSet occupancy.
31323105
while respawn_tasks.join_next().now_or_never().flatten().is_some() {}
3133-
// Flush requeued events that were waiting for a live agent. Without
3134-
// this, batches requeued during crash recovery sit idle until the
3135-
// next relay event arrives — which can be minutes on quiet channels.
3136-
if respawn_collected {
3137-
for (scope, thread_tags) in dispatch_pending(
3138-
&mut pool,
3139-
&mut queue,
3140-
&ctx,
3141-
&mut last_activity,
3142-
observer.as_ref(),
3143-
) {
3144-
typing_channels.insert(scope, thread_tags);
3145-
}
3146-
}
3106+
// Retry deadlines are actionable only with idle capacity. A busy pool
3107+
// wakes on its result/respawn instead of spinning on an expired retry.
3108+
let retry_at = if pool_ready && pool.any_idle() {
3109+
queue.next_retry_deadline()
3110+
} else {
3111+
None
3112+
};
3113+
let maintenance_at = pool_ready.then_some(last_maintenance + maintenance_interval);
31473114

31483115
// Borrow result_rx and join_set simultaneously via split-borrow helper.
31493116
let pool_event: Option<PoolEvent> = {
31503117
let (result_rx, join_set) = pool.rx_and_join_set();
31513118
tokio::select! {
31523119
biased;
3120+
_ = shutdown_rx.changed() => {
3121+
tracing::info!("shutting down");
3122+
break;
3123+
}
3124+
wake = recovery_wake::wait(&mut respawn_rx, retry_at, maintenance_at) => {
3125+
Some(PoolEvent::Recovery(wake))
3126+
}
31533127
// recv() returning None means all senders dropped (pool was torn down).
31543128
// Break cleanly instead of panicking.
31553129
r = result_rx.recv(), if pool_ready => match r {
@@ -3733,14 +3707,56 @@ async fn tokio_main() -> Result<()> {
37333707
}
37343708
None
37353709
}
3736-
_ = shutdown_rx.changed() => {
3737-
tracing::info!("shutting down");
3738-
break;
3739-
}
37403710
}
37413711
};
37423712

37433713
match pool_event {
3714+
Some(PoolEvent::Recovery(wake)) => {
3715+
match wake {
3716+
recovery_wake::RecoveryWake::Respawn(rr) => {
3717+
crash_history[rr.index].respawn_in_flight = false;
3718+
match rr.result {
3719+
Ok((acp, protocol_version, agent_name)) => {
3720+
let agent = OwnedAgent {
3721+
index: rr.index,
3722+
acp,
3723+
state: SessionState::default(),
3724+
model_capabilities: None,
3725+
desired_model: config.model.clone(),
3726+
model_overridden: false,
3727+
desired_model_request_id: None,
3728+
desired_model_pending_ack: false,
3729+
startup_effort: config.effort_level.clone(),
3730+
agent_name,
3731+
goose_system_prompt_supported: None,
3732+
protocol_version,
3733+
};
3734+
pool.return_agent(agent);
3735+
tracing::info!(agent = rr.index, "respawn complete");
3736+
}
3737+
Err(e) => {
3738+
crash_history[rr.index].mark_spawn_failed();
3739+
tracing::warn!(
3740+
agent = rr.index,
3741+
"respawn failed: {e} — circuit re-opened"
3742+
);
3743+
}
3744+
}
3745+
}
3746+
// Maintenance runs at the top of the next iteration.
3747+
recovery_wake::RecoveryWake::Maintenance => continue,
3748+
recovery_wake::RecoveryWake::Retry => {}
3749+
}
3750+
for (scope, thread_tags) in dispatch_pending(
3751+
&mut pool,
3752+
&mut queue,
3753+
&ctx,
3754+
&mut last_activity,
3755+
observer.as_ref(),
3756+
) {
3757+
typing_channels.insert(scope, thread_tags);
3758+
}
3759+
}
37443760
Some(PoolEvent::Result(result)) => {
37453761
// Stop the typing indicator for the completed turn's exact scope,
37463762
// not the whole channel — a sibling thread still running in the

‎crates/buzz-acp/src/queue.rs‎

Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -494,6 +494,24 @@ impl EventQueue {
494494
})
495495
}
496496

497+
/// Earliest retry throttle for queued work not already in flight.
498+
///
499+
/// Includes expired deadlines: an event-loop iteration can cross eligibility
500+
/// before it arms its timer. The caller must gate on idle pool capacity and
501+
/// dispatch when woken. Dispatch either makes the scope in-flight or releases
502+
/// it with `mark_complete`, which clears the expired throttle even when a
503+
/// busy session owner holds the batch. Empty/removed scopes never arm a timer.
504+
pub fn next_retry_deadline(&self) -> Option<Instant> {
505+
self.retry_after
506+
.iter()
507+
.filter(|(scope, _)| {
508+
!self.in_flight_scopes.contains(*scope)
509+
&& self.queues.get(*scope).is_some_and(|q| !q.is_empty())
510+
})
511+
.map(|(_, &deadline)| deadline)
512+
.min()
513+
}
514+
497515
/// Mark the prompt for `channel_id` as complete.
498516
///
499517
/// Removes the channel from `in_flight_channels` and `in_flight_deadlines`.
@@ -2200,6 +2218,95 @@ mod tests {
22002218
use nostr::{EventBuilder, Keys, Kind, Timestamp};
22012219
use std::time::Duration;
22022220

2221+
#[tokio::test]
2222+
async fn retry_wake_redelivers_before_and_after_replacement_eligibility() {
2223+
use crate::recovery_wake::{wait, RecoveryWake};
2224+
for replacement_first in [true, false] {
2225+
let mut queue = EventQueue::new(DedupMode::Queue);
2226+
let channel = Uuid::new_v4();
2227+
let event = make_queued(channel, "original batch");
2228+
let id = event.event.id;
2229+
let received_at = event.received_at;
2230+
queue.push(event);
2231+
let batch = queue.flush_next().unwrap();
2232+
assert!(queue.requeue(batch).is_none());
2233+
queue.mark_complete(channel);
2234+
// Production requeue installs a future throttle, not an immediate
2235+
// dispatch. Accelerate only the test's clock boundary.
2236+
assert!(queue.next_retry_deadline().unwrap() > Instant::now());
2237+
let deadline = if replacement_first {
2238+
Instant::now() + Duration::from_millis(30)
2239+
} else {
2240+
Instant::now() - Duration::from_millis(30)
2241+
};
2242+
queue.retry_after.insert(conv(channel), deadline);
2243+
let (tx, mut rx) = tokio::sync::mpsc::channel(1);
2244+
tx.send(crate::RespawnResult {
2245+
index: 0,
2246+
result: Err(anyhow::anyhow!("wake token")),
2247+
})
2248+
.await
2249+
.unwrap();
2250+
// With no idle agent the event loop disables the retry timer,
2251+
// even if its deadline is already past. Respawn is the wake owner.
2252+
assert!(matches!(
2253+
wait(&mut rx, None, None).await,
2254+
RecoveryWake::Respawn(_)
2255+
));
2256+
if replacement_first {
2257+
assert!(
2258+
queue.flush_next().is_none(),
2259+
"respawn must not bypass backoff"
2260+
);
2261+
}
2262+
assert!(matches!(
2263+
wait(&mut rx, queue.next_retry_deadline(), None).await,
2264+
RecoveryWake::Retry
2265+
));
2266+
let retried = queue.flush_next().unwrap();
2267+
assert_eq!(retried.events.len(), 1);
2268+
assert_eq!(retried.events[0].event.id, id);
2269+
assert_eq!(retried.events[0].received_at, received_at);
2270+
assert_eq!(
2271+
queue.next_retry_deadline(),
2272+
None,
2273+
"in-flight work cannot hot-loop"
2274+
);
2275+
queue.mark_complete(channel);
2276+
assert!(
2277+
queue.flush_next().is_none(),
2278+
"completed work must not duplicate"
2279+
);
2280+
assert_eq!(queue.next_retry_deadline(), None);
2281+
assert!(tokio::time::timeout(
2282+
Duration::from_millis(20),
2283+
wait(&mut rx, queue.next_retry_deadline(), None)
2284+
)
2285+
.await
2286+
.is_err());
2287+
}
2288+
}
2289+
2290+
#[test]
2291+
fn retry_deadline_excludes_empty_removed_and_held_scopes() {
2292+
let mut queue = EventQueue::new(DedupMode::Queue);
2293+
let channel = Uuid::new_v4();
2294+
let past = Instant::now() - Duration::from_secs(1);
2295+
queue.retry_after.insert(conv(channel), past);
2296+
assert_eq!(queue.next_retry_deadline(), None);
2297+
queue.push(make_queued(channel, "held"));
2298+
assert_eq!(queue.next_retry_deadline(), Some(past));
2299+
let held = queue.flush_next().unwrap();
2300+
assert_eq!(queue.next_retry_deadline(), None);
2301+
// This is dispatch_pending's busy-owner / no-slot release path.
2302+
queue.requeue_preserve_timestamps(held);
2303+
queue.mark_complete(channel);
2304+
assert_eq!(queue.next_retry_deadline(), None);
2305+
queue.retry_after.insert(conv(channel), past);
2306+
queue.drain_channel(channel);
2307+
assert_eq!(queue.next_retry_deadline(), None);
2308+
}
2309+
22032310
/// Build a test event with the given content and kind.
22042311
fn make_event(content: &str) -> Event {
22052312
let keys = Keys::generate();
Lines changed: 105 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,105 @@
1+
//! Quiet-host recovery wake sources, independent of presence/typing/heartbeat.
2+
3+
use std::time::Instant;
4+
use tokio::sync::mpsc;
5+
6+
use crate::RespawnResult;
7+
8+
pub(super) enum RecoveryWake {
9+
Respawn(Box<RespawnResult>),
10+
Retry,
11+
Maintenance,
12+
}
13+
14+
async fn sleep_until(deadline: Option<Instant>) {
15+
match deadline {
16+
Some(deadline) => tokio::time::sleep_until(deadline.into()).await,
17+
None => std::future::pending().await,
18+
}
19+
}
20+
21+
/// Cancellation-safe: selecting another host event never consumes a respawn.
22+
/// A closed receiver disables only that source, not retry/refill timers.
23+
pub(super) async fn wait(
24+
respawns: &mut mpsc::Receiver<RespawnResult>,
25+
retry_at: Option<Instant>,
26+
maintenance_at: Option<Instant>,
27+
) -> RecoveryWake {
28+
tokio::select! {
29+
Some(result) = respawns.recv() => RecoveryWake::Respawn(Box::new(result)),
30+
_ = sleep_until(retry_at) => RecoveryWake::Retry,
31+
_ = sleep_until(maintenance_at) => RecoveryWake::Maintenance,
32+
}
33+
}
34+
35+
#[cfg(test)]
36+
mod tests {
37+
use super::*;
38+
use std::time::Duration;
39+
use tokio::time::timeout;
40+
41+
#[tokio::test]
42+
async fn respawn_wakes_without_optional_timers() {
43+
let (tx, mut rx) = mpsc::channel(1);
44+
tx.send(RespawnResult {
45+
index: 0,
46+
result: Err(anyhow::anyhow!("fixture spawn failure")),
47+
})
48+
.await
49+
.unwrap();
50+
assert!(matches!(
51+
timeout(Duration::from_secs(1), wait(&mut rx, None, None))
52+
.await
53+
.unwrap(),
54+
RecoveryWake::Respawn(result) if result.index == 0
55+
));
56+
// Consumed exactly once: neither an empty queue nor a completed spawn
57+
// produces an immediately ready arm on the next loop iteration.
58+
assert!(
59+
timeout(Duration::from_millis(20), wait(&mut rx, None, None))
60+
.await
61+
.is_err()
62+
);
63+
}
64+
65+
#[tokio::test]
66+
async fn closed_respawn_channel_does_not_spin_or_disable_refill() {
67+
let (tx, mut rx) = mpsc::channel(1);
68+
drop(tx);
69+
assert!(
70+
timeout(Duration::from_millis(20), wait(&mut rx, None, None))
71+
.await
72+
.is_err()
73+
);
74+
assert!(matches!(
75+
wait(
76+
&mut rx,
77+
None,
78+
Some(Instant::now() + Duration::from_millis(20))
79+
)
80+
.await,
81+
RecoveryWake::Maintenance
82+
));
83+
}
84+
85+
#[tokio::test]
86+
async fn host_shutdown_can_cancel_wait_without_losing_later_respawn() {
87+
let (tx, mut rx) = mpsc::channel(1);
88+
let (shutdown, mut shutdown_rx) = tokio::sync::watch::channel(());
89+
shutdown.send(()).unwrap();
90+
tokio::select! {
91+
_ = shutdown_rx.changed() => {}
92+
_ = wait(&mut rx, None, None) => panic!("unexpected wake"),
93+
}
94+
tx.send(RespawnResult {
95+
index: 0,
96+
result: Err(anyhow::anyhow!("fixture")),
97+
})
98+
.await
99+
.unwrap();
100+
assert!(matches!(
101+
wait(&mut rx, None, None).await,
102+
RecoveryWake::Respawn(_)
103+
));
104+
}
105+
}

0 commit comments

Comments
 (0)