Skip to content

Commit d9c4e4a

Browse files
wesbillmanBrain
andauthored
fix: agent reliability — no restart on channel-add, visible dead-letter notice (#1468)
Signed-off-by: Wes <[email protected]> Co-authored-by: Brain <21994759fc7a6fa6b965551d35cfd7897d262f2495467f2d78694ddcfa6a5c7e@sprout-oss.stage.blox.sqprod.co>
1 parent c48006f commit d9c4e4a

10 files changed

Lines changed: 124 additions & 41 deletions

File tree

‎crates/buzz-acp/src/lib.rs‎

Lines changed: 32 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2111,6 +2111,7 @@ async fn tokio_main() -> Result<()> {
21112111
&respawn_tx,
21122112
&mut respawn_tasks,
21132113
observer.clone(),
2114+
Some(&ctx.rest_client),
21142115
) == LoopAction::Exit
21152116
{
21162117
break;
@@ -2662,6 +2663,7 @@ fn handle_prompt_result(
26622663
respawn_tx: &mpsc::Sender<RespawnResult>,
26632664
respawn_tasks: &mut tokio::task::JoinSet<()>,
26642665
observer: Option<observer::ObserverHandle>,
2666+
rest_client: Option<&relay::RestClient>,
26652667
) -> LoopAction {
26662668
let before = pool.task_map().len();
26672669
let agent_index = result.agent.index;
@@ -2674,7 +2676,7 @@ fn handle_prompt_result(
26742676
// retry_counts. If mark_complete runs first, retry_counts is cleared and
26752677
// every retry starts at attempt 1 — defeating exponential backoff and
26762678
// dead-letter protection.
2677-
if let Some(batch) = result.batch {
2679+
if let Some(batch) = result.batch.take() {
26782680
// Don't requeue batches for channels the agent was removed from —
26792681
// those events are stale and should be silently dropped.
26802682
if !removed_channels.contains(&batch.channel_id) {
@@ -2689,8 +2691,31 @@ fn handle_prompt_result(
26892691
// system default — rather than telling the agent to supersede.
26902692
let reason = batch.cancel_reason.unwrap_or(CancelReason::Steer);
26912693
queue.requeue_as_cancelled(batch, reason);
2692-
} else {
2693-
queue.requeue(batch);
2694+
} else if let Some(dead) = queue.requeue(batch) {
2695+
// Dead-lettered: retries exhausted and the events are gone.
2696+
// Post a visible notice so the channel isn't left waiting on
2697+
// a turn that will never happen.
2698+
if let Some(rest) = rest_client {
2699+
let thread_tags = dead
2700+
.events
2701+
.last()
2702+
.map(|be| queue::parse_thread_tags(&be.event))
2703+
.unwrap_or_default();
2704+
let reason = match &result.outcome {
2705+
PromptOutcome::Timeout => "the turn timed out".to_string(),
2706+
PromptOutcome::AgentExited => "the agent process exited".to_string(),
2707+
PromptOutcome::Error(e) => format!("{e}"),
2708+
_ => "repeated failures".to_string(),
2709+
};
2710+
let content = format!(
2711+
"⚠️ I couldn't process the last request after multiple retries ({reason}). Please re-send if it's still needed."
2712+
);
2713+
let rest = rest.clone();
2714+
let channel_id = dead.channel_id;
2715+
tokio::spawn(async move {
2716+
pool::post_failure_notice(&rest, channel_id, &thread_tags, &content).await;
2717+
});
2718+
}
26942719
}
26952720
} else {
26962721
tracing::debug!(
@@ -2875,7 +2900,9 @@ fn recover_panicked_agent(
28752900
if let Some(batch) = meta.recoverable_batch {
28762901
if let Some(ch) = meta.channel_id {
28772902
if !removed_channels.contains(&ch) {
2878-
queue.requeue(batch);
2903+
// Dead-letter on exhaustion is logged inside requeue(); a
2904+
// panic path has no outcome to report, so no notice here.
2905+
let _ = queue.requeue(batch);
28792906
tracing::warn!("requeued batch for panicked agent {i}");
28802907
} else {
28812908
tracing::debug!(
@@ -4020,6 +4047,7 @@ mod error_outcome_emission_tests {
40204047
&respawn_tx,
40214048
&mut respawn_tasks,
40224049
Some(observer.clone()),
4050+
None,
40234051
);
40244052

40254053
observer

‎crates/buzz-acp/src/pool.rs‎

Lines changed: 45 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ use crate::config::{DedupMode, PermissionMode};
3636
use crate::observer;
3737
use crate::queue::{
3838
CancelReason, ContextMessage, ConversationContext, FlushBatch, PromptChannelInfo,
39-
PromptProfile, PromptProfileLookup,
39+
PromptProfile, PromptProfileLookup, ThreadTags,
4040
};
4141
use crate::relay::{ChannelInfo, RestClient};
4242

@@ -2615,6 +2615,50 @@ pub(crate) async fn reaction_add(rest: &crate::relay::RestClient, event_id: &str
26152615
}
26162616
}
26172617

2618+
/// Best-effort: post a visible failure notice (kind:9) to a channel after a
2619+
/// batch is dead-lettered. Replies into the thread of `thread_tags` when the
2620+
/// triggering event was threaded. Errors are logged and swallowed — the
2621+
/// notice must never take down the main loop.
2622+
pub(crate) async fn post_failure_notice(
2623+
rest: &crate::relay::RestClient,
2624+
channel_id: Uuid,
2625+
thread_tags: &ThreadTags,
2626+
content: &str,
2627+
) {
2628+
let thread_ref = thread_tags.root_event_id.as_deref().and_then(|root| {
2629+
let root_id = nostr::EventId::from_hex(root).ok()?;
2630+
let parent_id = thread_tags
2631+
.parent_event_id
2632+
.as_deref()
2633+
.and_then(|p| nostr::EventId::from_hex(p).ok())
2634+
.unwrap_or(root_id);
2635+
Some(buzz_sdk::ThreadRef {
2636+
root_event_id: root_id,
2637+
parent_event_id: parent_id,
2638+
})
2639+
});
2640+
let builder =
2641+
match buzz_sdk::build_message(channel_id, content, thread_ref.as_ref(), &[], false, &[]) {
2642+
Ok(b) => b,
2643+
Err(e) => {
2644+
tracing::warn!(channel = %channel_id, "failure notice: build failed: {e}");
2645+
return;
2646+
}
2647+
};
2648+
let event = match builder.sign_with_keys(&rest.keys) {
2649+
Ok(e) => e,
2650+
Err(e) => {
2651+
tracing::warn!(channel = %channel_id, "failure notice: sign failed: {e}");
2652+
return;
2653+
}
2654+
};
2655+
match tokio::time::timeout(Duration::from_secs(5), rest.submit_event(&event)).await {
2656+
Ok(Ok(_)) => {}
2657+
Ok(Err(e)) => tracing::warn!(channel = %channel_id, "failure notice failed: {e}"),
2658+
Err(_) => tracing::warn!(channel = %channel_id, "failure notice timed out"),
2659+
}
2660+
}
2661+
26182662
/// Best-effort: remove a reaction via a signed kind:5 (NIP-09) deletion event.
26192663
///
26202664
/// Queries kind:7 reactions by our pubkey targeting the event, finds the matching

‎crates/buzz-acp/src/queue.rs‎

Lines changed: 37 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -128,7 +128,7 @@ pub struct FlushBatch {
128128
///
129129
/// requeue(batch):
130130
/// increment retry_counts[channel]
131-
/// if retry_counts[channel] > MAX_RETRIES: dead-letter (log ERROR, discard)
131+
/// if retry_counts[channel] > MAX_RETRIES: dead-letter (log ERROR, return batch to caller)
132132
/// else: push_front with original received_at, set exponential backoff retry_after with jitter
133133
/// ```
134134
pub struct EventQueue {
@@ -376,12 +376,13 @@ impl EventQueue {
376376
/// not from resetting received_at.
377377
///
378378
/// After [`MAX_RETRIES`] attempts the batch is dead-lettered: logged at
379-
/// ERROR and discarded rather than requeued. This prevents poison batches
380-
/// from looping forever.
379+
/// ERROR and returned to the caller (rather than requeued) so a visible
380+
/// failure notice can be posted to the channel. Returns `None` when the
381+
/// batch was requeued for another attempt.
381382
///
382383
/// Note: does NOT remove from `in_flight_channels` — caller must call
383384
/// `mark_complete` separately.
384-
pub fn requeue(&mut self, batch: FlushBatch) {
385+
pub fn requeue(&mut self, batch: FlushBatch) -> Option<FlushBatch> {
385386
let channel_id = batch.channel_id;
386387
let attempt = {
387388
let count = self.retry_counts.entry(channel_id).or_insert(0);
@@ -402,7 +403,7 @@ impl EventQueue {
402403
// Also clear retry_after so fresh traffic on this channel isn't
403404
// throttled by stale backoff from the discarded poison batch.
404405
self.retry_after.remove(&channel_id);
405-
return;
406+
return Some(batch);
406407
}
407408

408409
// Exponential backoff: BASE * 2^(attempt-1), capped at MAX, with ±20% jitter.
@@ -449,6 +450,7 @@ impl EventQueue {
449450
);
450451
}
451452
self.retry_after.insert(channel_id, Instant::now() + delay);
453+
None
452454
}
453455

454456
/// Re-queue a batch preserving original `received_at` timestamps.
@@ -2690,6 +2692,36 @@ mod tests {
26902692
);
26912693
}
26922694

2695+
#[test]
2696+
fn test_requeue_dead_letters_after_max_retries() {
2697+
let mut q = EventQueue::new(DedupMode::Queue);
2698+
let ch = Uuid::new_v4();
2699+
2700+
q.push(make_queued(ch, "poison"));
2701+
for attempt in 1..=MAX_RETRIES {
2702+
q.retry_after
2703+
.insert(ch, Instant::now() - Duration::from_secs(1));
2704+
let batch = q.flush_next().expect("flush");
2705+
assert!(
2706+
q.requeue(batch).is_none(),
2707+
"attempt {attempt} should requeue, not dead-letter"
2708+
);
2709+
q.mark_complete(ch);
2710+
}
2711+
2712+
// The MAX_RETRIES+1'th failure dead-letters: batch is returned.
2713+
q.retry_after
2714+
.insert(ch, Instant::now() - Duration::from_secs(1));
2715+
let batch = q.flush_next().expect("flush");
2716+
let dead = q.requeue(batch).expect("should dead-letter");
2717+
assert_eq!(dead.channel_id, ch);
2718+
assert_eq!(dead.events.len(), 1);
2719+
q.mark_complete(ch);
2720+
// Retry state is cleared so fresh traffic isn't throttled.
2721+
assert!(!q.retry_counts.contains_key(&ch));
2722+
assert!(!q.retry_after.contains_key(&ch));
2723+
}
2724+
26932725
#[test]
26942726
fn test_retry_throttle_blocks_requeue_channel() {
26952727
let mut q = EventQueue::new(DedupMode::Queue);

‎crates/buzz-agent/README.md‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -147,7 +147,7 @@ Everything is environment variables. No flags, no config files. (We are a subpro
147147
| `BUZZ_AGENT_MAX_OUTPUT_TOKENS` | `32768` | Per LLM call. Headroom for large tool-call inputs (e.g. file writes via heredoc); Sonnet 4 / Opus 4 cap at 64K. |
148148
| `BUZZ_AGENT_MAX_CONTEXT_TOKENS` | `200000` | Provider context window used by the handoff gate. |
149149
| `BUZZ_AGENT_MAX_HANDOFFS` | `10` | Max context handoffs per session before falling back to truncation. |
150-
| `BUZZ_AGENT_LLM_TIMEOUT_SECS` | `120` | |
150+
| `BUZZ_AGENT_LLM_TIMEOUT_SECS` | `240` | |
151151
| `BUZZ_AGENT_TOOL_TIMEOUT_SECS` | `660` | Per-tool call timeout in seconds |
152152
| `BUZZ_AGENT_MAX_PARALLEL_TOOLS` | `8` | Max concurrent tool calls per turn (1 = sequential) |
153153
| `BUZZ_AGENT_MAX_SESSIONS` | unlimited | Max concurrent ACP sessions. Sessions are cheap; default has no cap. |
@@ -244,7 +244,7 @@ The trust boundary is **the operator who launched the agent**. The harness, MCP
244244
| Tool schema bytes | 4 KiB | `MAX_SCHEMA_BYTES` (oversize → replaced with `{}`) |
245245
| Tool calls per turn | 64 | `MAX_TOOL_CALLS_PER_TURN` |
246246
| Loop rounds | 0 (unlimited) | `BUZZ_AGENT_MAX_ROUNDS` |
247-
| LLM call timeout | 120 s | `BUZZ_AGENT_LLM_TIMEOUT_SECS` |
247+
| LLM call timeout | 240 s | `BUZZ_AGENT_LLM_TIMEOUT_SECS` |
248248
| Tool call timeout | 660 s | `BUZZ_AGENT_TOOL_TIMEOUT_SECS` |
249249

250250
## What This Is NOT

‎crates/buzz-agent/src/config.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -164,7 +164,7 @@ impl Config {
164164
openai_api,
165165
max_rounds: parse_env("BUZZ_AGENT_MAX_ROUNDS", 0)?,
166166
max_output_tokens: parse_env("BUZZ_AGENT_MAX_OUTPUT_TOKENS", 32_768)?,
167-
llm_timeout: Duration::from_secs(parse_env("BUZZ_AGENT_LLM_TIMEOUT_SECS", 120)?),
167+
llm_timeout: Duration::from_secs(parse_env("BUZZ_AGENT_LLM_TIMEOUT_SECS", 240)?),
168168
tool_timeout: Duration::from_secs(parse_env("BUZZ_AGENT_TOOL_TIMEOUT_SECS", 660)?),
169169
mcp_init_timeout: Duration::from_secs(parse_env(
170170
"BUZZ_AGENT_MCP_INIT_TIMEOUT_SECS",

‎desktop/src/features/agents/channelAgents.ts‎

Lines changed: 4 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,6 @@ import {
1212
getChannelMembers,
1313
listManagedAgents,
1414
startManagedAgent,
15-
stopManagedAgent,
1615
updateManagedAgent,
1716
uploadMediaBytes,
1817
} from "@/shared/api/tauri";
@@ -38,7 +37,6 @@ export type AttachManagedAgentToChannelInput = {
3837
export type AttachManagedAgentToChannelResult = {
3938
agent: ManagedAgent;
4039
membershipAdded: boolean;
41-
restarted: boolean;
4240
started: boolean;
4341
};
4442

@@ -122,25 +120,16 @@ export async function attachManagedAgentToChannel(
122120

123121
let agent = input.agent;
124122
let started = false;
125-
let restarted = false;
126123

127124
if (ensureRunning) {
128-
// Provider-backed agents use startManagedAgent as the deploy operation.
129-
// Already-deployed providers auto-discover new channel membership via the
130-
// harness; not-yet-deployed providers need the deploy call before the
131-
// first mention can reach them.
125+
// Running agents (local or provider) auto-discover new channel membership
126+
// via the harness's membership notifications — no restart needed. Only
127+
// not-yet-running agents need a start/deploy call before the first
128+
// mention can reach them.
132129
const isRemote = input.agent.backend.type === "provider";
133130
if (isRemote && input.agent.status !== "deployed") {
134131
agent = await startManagedAgent(input.agent.pubkey);
135132
started = true;
136-
} else if (
137-
!isRemote &&
138-
membershipAdded &&
139-
(input.agent.status === "running" || input.agent.status === "deployed")
140-
) {
141-
await stopManagedAgent(input.agent.pubkey);
142-
agent = await startManagedAgent(input.agent.pubkey);
143-
restarted = true;
144133
} else if (
145134
!isRemote &&
146135
input.agent.status !== "running" &&
@@ -154,7 +143,6 @@ export async function attachManagedAgentToChannel(
154143
return {
155144
agent,
156145
membershipAdded,
157-
restarted,
158146
started,
159147
} satisfies AttachManagedAgentToChannelResult;
160148
}

‎desktop/src/features/agents/hooks.ts‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -534,7 +534,6 @@ export function useEnsureGooseInChannelMutation(channelId: string | null) {
534534
return {
535535
agent: attached.agent,
536536
membershipAdded: attached.membershipAdded,
537-
restarted: attached.restarted,
538537
started: attached.started,
539538
created: attached.created,
540539
};

‎desktop/src/features/agents/ui/AddAgentToChannelDialog.tsx‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -116,9 +116,8 @@ export function AddAgentToChannelDialog({
116116
<DialogTitle>Add agent to channel</DialogTitle>
117117
<DialogDescription>
118118
Add {agent?.name ?? "this agent"} to a channel so desktop chat can
119-
`@mention` it. Running local agents are restarted automatically
120-
when they join a new channel. Remote agents pick up new channels
121-
automatically via membership notifications.
119+
`@mention` it. Running agents pick up new channels automatically
120+
via membership notifications.
122121
</DialogDescription>
123122
</DialogHeader>
124123

‎desktop/src/features/agents/ui/useManagedAgentActions.ts‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -364,9 +364,6 @@ export function useManagedAgentActions() {
364364
) {
365365
setActionErrorMessage(null);
366366
setActionNoticeMessage(() => {
367-
if (result.restarted) {
368-
return `Added ${result.agent.name} to ${channel.name} and restarted it so the new channel subscription is live.`;
369-
}
370367
if (result.started) {
371368
return `Added ${result.agent.name} to ${channel.name} and spawned it.`;
372369
}

‎desktop/src/features/profile/ui/UserProfilePanel.tsx‎

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -662,11 +662,7 @@ export function UserProfilePanel({
662662

663663
const handleAddedToChannel = React.useCallback(
664664
(channel: Channel, result: AttachManagedAgentToChannelResult) => {
665-
if (result.restarted) {
666-
toast.success(
667-
`Added ${result.agent.name} to ${channel.name} and restarted it.`,
668-
);
669-
} else if (result.started) {
665+
if (result.started) {
670666
toast.success(`Added ${result.agent.name} to ${channel.name}.`);
671667
} else if (result.membershipAdded) {
672668
toast.success(`Added ${result.agent.name} to ${channel.name}.`);

0 commit comments

Comments
 (0)