Skip to content

Commit 87e45c6

Browse files
wpfleger96npub1mn7jgtj4w2pd0g0zeuhxsa6jy6p0rewxz4kujt98my82ahfmp72sxjexk7
andauthored
feat(agents): show per-turn duration and prune dead turns within ~25s of host crash (#1017)
Signed-off-by: Will Pfleger <[email protected]> Co-authored-by: Will Pfleger <[email protected]> Co-authored-by: npub1mn7jgtj4w2pd0g0zeuhxsa6jy6p0rewxz4kujt98my82ahfmp72sxjexk7 <dcfd242e557282d7a1e2cf2e6877522682f1e5c6156dc92ca7d90eaedd3b0f95@sprout-oss.stage.blox.sqprod.co>
1 parent 2fef8d6 commit 87e45c6

8 files changed

Lines changed: 608 additions & 65 deletions

File tree

‎crates/buzz-acp/src/config.rs‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -250,6 +250,11 @@ pub struct CliArgs {
250250
#[arg(long, env = "BUZZ_ACP_HEARTBEAT_INTERVAL", default_value_t = 0)]
251251
pub heartbeat_interval: u64,
252252

253+
/// Seconds between per-turn liveness pings (the crash backstop signal —
254+
/// distinct from heartbeat self-prompting). 0 = disabled.
255+
#[arg(long, env = "BUZZ_ACP_TURN_LIVENESS_SECS", default_value_t = 10)]
256+
pub turn_liveness_secs: u64,
257+
253258
/// Heartbeat prompt text. Conflicts with --heartbeat-prompt-file.
254259
#[arg(
255260
long,
@@ -435,6 +440,10 @@ pub struct Config {
435440
pub max_turn_duration_secs: u64,
436441
pub agents: u32,
437442
pub heartbeat_interval_secs: u64,
443+
/// Seconds between per-turn liveness pings. 0 = disabled. Distinct from
444+
/// `heartbeat_interval_secs` (agent self-prompting) — this is the desktop
445+
/// crash-backstop signal.
446+
pub turn_liveness_secs: u64,
438447
pub heartbeat_prompt: Option<String>,
439448
pub system_prompt: Option<String>,
440449
pub initial_message: Option<String>,
@@ -599,6 +608,12 @@ impl Config {
599608
));
600609
}
601610

611+
if args.turn_liveness_secs > 0 && args.turn_liveness_secs < 5 {
612+
return Err(ConfigError::ConfigFile(
613+
"turn liveness interval must be 0 (disabled) or ≥5 seconds".into(),
614+
));
615+
}
616+
602617
let heartbeat_prompt = if let Some(text) = args.heartbeat_prompt {
603618
Some(text)
604619
} else if let Some(ref path) = args.heartbeat_prompt_file {
@@ -669,6 +684,17 @@ impl Config {
669684
args.heartbeat_interval
670685
};
671686

687+
// Cap turn-liveness interval at 86400s (24h) — same bound as heartbeat.
688+
let turn_liveness_secs = if args.turn_liveness_secs > 86400 {
689+
tracing::warn!(
690+
interval = args.turn_liveness_secs,
691+
"turn liveness interval exceeds 24h — capping at 86400s"
692+
);
693+
86400u64
694+
} else {
695+
args.turn_liveness_secs
696+
};
697+
672698
// Resolve idle_timeout_secs with deprecation handling.
673699
// Precedence: explicit --idle-timeout > --turn-timeout (deprecated) > `DEFAULT_IDLE_TIMEOUT_SECS`.
674700
let idle_timeout_secs = {
@@ -808,6 +834,7 @@ impl Config {
808834
max_turn_duration_secs,
809835
agents: args.agents,
810836
heartbeat_interval_secs: heartbeat_interval,
837+
turn_liveness_secs,
811838
heartbeat_prompt,
812839
system_prompt,
813840
initial_message: args.initial_message,
@@ -1173,6 +1200,7 @@ mod tests {
11731200
max_turn_duration_secs: 3600,
11741201
agents: 1,
11751202
heartbeat_interval_secs: 0,
1203+
turn_liveness_secs: 10,
11761204
heartbeat_prompt: None,
11771205
system_prompt: None,
11781206
initial_message: None,
@@ -1742,6 +1770,44 @@ channels = "ALL"
17421770
assert!(err.to_string().contains("heartbeat interval must be 0"));
17431771
}
17441772

1773+
// ── turn-liveness validation ─────────────────────────────────────────────
1774+
1775+
fn validate_turn_liveness(secs: u64) -> Result<(), ConfigError> {
1776+
if secs > 0 && secs < 5 {
1777+
return Err(ConfigError::ConfigFile(
1778+
"turn liveness interval must be 0 (disabled) or ≥5 seconds".into(),
1779+
));
1780+
}
1781+
Ok(())
1782+
}
1783+
1784+
#[test]
1785+
fn test_turn_liveness_zero_ok() {
1786+
assert!(validate_turn_liveness(0).is_ok());
1787+
}
1788+
1789+
#[test]
1790+
fn test_turn_liveness_five_ok() {
1791+
assert!(validate_turn_liveness(5).is_ok());
1792+
}
1793+
1794+
#[test]
1795+
fn test_turn_liveness_ten_ok() {
1796+
assert!(validate_turn_liveness(10).is_ok());
1797+
}
1798+
1799+
#[test]
1800+
fn test_turn_liveness_four_rejected() {
1801+
let err = validate_turn_liveness(4).unwrap_err();
1802+
assert!(err.to_string().contains("turn liveness interval must be 0"));
1803+
}
1804+
1805+
#[test]
1806+
fn test_turn_liveness_one_rejected() {
1807+
let err = validate_turn_liveness(1).unwrap_err();
1808+
assert!(err.to_string().contains("turn liveness interval must be 0"));
1809+
}
1810+
17451811
// ── summary includes agents and heartbeat ────────────────────────────────
17461812

17471813
#[test]

‎crates/buzz-acp/src/lib.rs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1078,6 +1078,7 @@ async fn tokio_main() -> Result<()> {
10781078
initial_message: config.initial_message.clone(),
10791079
idle_timeout: Duration::from_secs(config.idle_timeout_secs),
10801080
max_turn_duration: Duration::from_secs(config.max_turn_duration_secs),
1081+
turn_liveness_interval: Duration::from_secs(config.turn_liveness_secs),
10811082
dedup_mode: config.dedup_mode,
10821083
system_prompt: config.system_prompt.clone(),
10831084
base_prompt: if config.no_base_prompt {
@@ -3000,6 +3001,7 @@ mod build_mcp_servers_tests {
30003001
max_turn_duration_secs: 3600,
30013002
agents: 1,
30023003
heartbeat_interval_secs: 0,
3004+
turn_liveness_secs: 10,
30033005
heartbeat_prompt: None,
30043006
system_prompt: None,
30053007
initial_message: None,

‎crates/buzz-acp/src/pool.rs‎

Lines changed: 152 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -216,6 +216,10 @@ pub struct PromptContext {
216216
pub initial_message: Option<String>,
217217
pub idle_timeout: Duration,
218218
pub max_turn_duration: Duration,
219+
/// Interval between per-turn `turn_liveness` observer pings. `Duration::ZERO`
220+
/// disables emission. This is the desktop crash-backstop signal — distinct
221+
/// from `heartbeat_prompt` (agent self-prompting).
222+
pub turn_liveness_interval: Duration,
219223
pub dedup_mode: DedupMode,
220224
pub system_prompt: Option<String>,
221225
pub heartbeat_prompt: Option<String>,
@@ -1088,18 +1092,36 @@ pub async fn run_prompt_task(
10881092
// When control_rx is Some (channel tasks), wrap the prompt in select! so
10891093
// the main loop can cancel, interrupt, or rotate it. Heartbeats
10901094
// (control_rx=None) take the simple await path — they are not controllable.
1095+
//
1096+
// The liveness future emits `turn_liveness` pings on an interval and never
1097+
// resolves; it rides every prompt-await path as a non-winning select arm so
1098+
// a turn stays alive on the desktop while it runs. Built from a captured
1099+
// observer handle (not `&agent.acp`) because the prompt holds `&mut agent.acp`.
1100+
let liveness = run_turn_liveness(
1101+
agent.acp.observer_handle(),
1102+
agent.acp.observer_agent_index(),
1103+
observer::context_for(
1104+
observer_channel_id,
1105+
Some(session_id.clone()),
1106+
Some(turn_id.clone()),
1107+
),
1108+
ctx.turn_liveness_interval,
1109+
);
1110+
tokio::pin!(liveness);
1111+
10911112
let prompt_result = match control_rx {
10921113
None => {
10931114
// Heartbeat / non-cancellable path.
1094-
agent
1095-
.acp
1096-
.session_prompt_blocks_with_idle_timeout(
1115+
tokio::select! {
1116+
biased;
1117+
result = agent.acp.session_prompt_blocks_with_idle_timeout(
10971118
&session_id,
10981119
&prompt_blocks,
10991120
ctx.idle_timeout,
11001121
ctx.max_turn_duration,
1101-
)
1102-
.await
1122+
) => result,
1123+
_ = &mut liveness => unreachable!("liveness future never resolves"),
1124+
}
11031125
}
11041126
Some(rx) => {
11051127
tokio::select! {
@@ -1110,6 +1132,7 @@ pub async fn run_prompt_task(
11101132
ctx.idle_timeout,
11111133
ctx.max_turn_duration,
11121134
) => result,
1135+
_ = &mut liveness => unreachable!("liveness future never resolves"),
11131136
mode = rx => {
11141137
let control_signal = mode.unwrap_or(ControlSignal::Cancel);
11151138
// Control signal received. Guard against Race 1: the turn may
@@ -2017,6 +2040,49 @@ impl Drop for ReactionGuard {
20172040
}
20182041
}
20192042

2043+
// ── Turn liveness emission ───────────────────────────────────────────────────
2044+
// Periodically emits a `turn_liveness` observer event while a turn is in-flight,
2045+
// so the desktop can prune turns whose host died without unwinding (kill -9 /
2046+
// crash) far sooner than the no-activity backstop. Runs as a non-resolving
2047+
// `select!` arm in `run_prompt_task`: it lives and dies with the prompt future,
2048+
// so emission stops on every exit path (complete / cancel / error / panic) with
2049+
// no separate teardown to forget.
2050+
//
2051+
// Takes a captured `ObserverHandle` rather than `&agent.acp` because the prompt
2052+
// future holds `&mut agent.acp` for its whole duration — a second borrow would
2053+
// not compile.
2054+
//
2055+
// This future never resolves; callers must race it against the prompt and rely
2056+
// on drop for teardown. When `interval` is zero, liveness is disabled and the
2057+
// future parks forever without emitting.
2058+
async fn run_turn_liveness(
2059+
observer: Option<observer::ObserverHandle>,
2060+
agent_index: Option<usize>,
2061+
context: observer::ObserverContext,
2062+
interval: Duration,
2063+
) {
2064+
let Some(observer) = observer else {
2065+
return std::future::pending::<()>().await;
2066+
};
2067+
if interval.is_zero() {
2068+
return std::future::pending::<()>().await;
2069+
}
2070+
let mut ticker = tokio::time::interval(interval);
2071+
// The first tick completes immediately; skip it so the first liveness ping
2072+
// fires one interval after the turn starts, not at t=0 (turn_started already
2073+
// marks t=0).
2074+
ticker.tick().await;
2075+
loop {
2076+
ticker.tick().await;
2077+
observer.emit(
2078+
"turn_liveness",
2079+
agent_index,
2080+
&context,
2081+
serde_json::json!({}),
2082+
);
2083+
}
2084+
}
2085+
20202086
// ── Turn completion scope guard ──────────────────────────────────────────────
20212087
// Emits a `turn_completed` observer event on drop, covering ALL exit paths
20222088
// (success, error, timeout, cancel, panic) from `run_prompt_task`. Captures
@@ -2790,4 +2856,85 @@ mod tests {
27902856
assert_eq!(*s.turn_counts.get(&ch_b).unwrap(), 3);
27912857
assert_eq!(s.core_sections.get(&ch_b).unwrap(), "core-b");
27922858
}
2859+
2860+
// ── turn liveness emission ───────────────────────────────────────────────
2861+
// `run_turn_liveness` is raced against a "prompt" future the same way
2862+
// `run_prompt_task` does it: the prompt wins the select and the liveness
2863+
// future is dropped. We assert what the observer saw.
2864+
2865+
fn liveness_count(handle: &observer::ObserverHandle) -> usize {
2866+
handle
2867+
.snapshot()
2868+
.iter()
2869+
.filter(|e| e.kind == "turn_liveness")
2870+
.count()
2871+
}
2872+
2873+
#[tokio::test(start_paused = true)]
2874+
async fn test_liveness_fires_while_prompt_pends_then_stops() {
2875+
let observer = observer::ObserverHandle::in_process();
2876+
let context = observer::context_for(None, None, Some("t-1".into()));
2877+
let liveness = run_turn_liveness(
2878+
Some(observer.clone()),
2879+
Some(0),
2880+
context,
2881+
Duration::from_secs(10),
2882+
);
2883+
tokio::pin!(liveness);
2884+
2885+
// Prompt pends for 25s, then completes — first liveness tick at 10s,
2886+
// second at 20s, so the observer must see exactly two pings.
2887+
tokio::select! {
2888+
biased;
2889+
() = tokio::time::sleep(Duration::from_secs(25)) => {}
2890+
_ = &mut liveness => unreachable!("liveness future never resolves"),
2891+
}
2892+
2893+
assert_eq!(liveness_count(&observer), 2);
2894+
2895+
// The turn carried the live turn_id on each ping.
2896+
let pings: Vec<_> = observer
2897+
.snapshot()
2898+
.into_iter()
2899+
.filter(|e| e.kind == "turn_liveness")
2900+
.collect();
2901+
assert!(pings.iter().all(|e| e.turn_id.as_deref() == Some("t-1")));
2902+
2903+
// After the prompt wins the select, the liveness future is dropped —
2904+
// advancing the clock further produces no new pings.
2905+
tokio::time::sleep(Duration::from_secs(60)).await;
2906+
assert_eq!(liveness_count(&observer), 2);
2907+
}
2908+
2909+
#[tokio::test(start_paused = true)]
2910+
async fn test_liveness_disabled_when_interval_zero_emits_nothing() {
2911+
let observer = observer::ObserverHandle::in_process();
2912+
let context = observer::context_for(None, None, Some("t-1".into()));
2913+
let liveness = run_turn_liveness(Some(observer.clone()), Some(0), context, Duration::ZERO);
2914+
tokio::pin!(liveness);
2915+
2916+
tokio::select! {
2917+
biased;
2918+
() = tokio::time::sleep(Duration::from_secs(120)) => {}
2919+
_ = &mut liveness => unreachable!("disabled liveness future never resolves"),
2920+
}
2921+
2922+
assert_eq!(liveness_count(&observer), 0);
2923+
}
2924+
2925+
#[tokio::test(start_paused = true)]
2926+
async fn test_liveness_without_observer_emits_nothing() {
2927+
// A turn that never started has no observer handle — the future must
2928+
// park without emitting or panicking.
2929+
let context = observer::context_for(None, None, Some("t-1".into()));
2930+
let liveness = run_turn_liveness(None, None, context, Duration::from_secs(10));
2931+
tokio::pin!(liveness);
2932+
2933+
tokio::select! {
2934+
biased;
2935+
() = tokio::time::sleep(Duration::from_secs(120)) => {}
2936+
_ = &mut liveness => unreachable!("handle-less liveness future never resolves"),
2937+
}
2938+
// No observer to assert against — reaching here without panic is the test.
2939+
}
27932940
}

0 commit comments

Comments
 (0)