Skip to content

Commit c4c4bd9

Browse files
committed
fix(agents): replay the failed request after workspace recovery
Signed-off-by: Tee8z <[email protected]>
1 parent b4d6557 commit c4c4bd9

9 files changed

Lines changed: 186 additions & 26 deletions

File tree

‎apps/buzz-gpui/crates/app/src/agent_plugins/remote_workspace/mod.rs‎

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -143,7 +143,7 @@ impl AgentType for RemoteWorkspace {
143143
return Ok(());
144144
};
145145
cx.update(|cx| recovery::started(&sent, &name, cx));
146-
sent.1.map_err(|error| error.message)
146+
sent.result.map_err(|error| error.message)
147147
})
148148
}
149149

@@ -174,11 +174,21 @@ pub(crate) async fn deliver(delivery: Delivery, relay: String) -> Result<Option<
174174
Ok(()) => WorkOutcome::ok(),
175175
Err(error) => WorkOutcome::failed(error.message.clone()),
176176
});
177-
Ok(Some((session, result)))
177+
Ok(Some(Sent {
178+
session,
179+
result,
180+
replay_floor,
181+
}))
178182
}
179183

180184
/// A session request the host answered.
181-
pub(crate) type Sent = (SandboxSession, Result<(), SessionError>);
185+
#[derive(Debug)]
186+
pub(crate) struct Sent {
187+
pub session: SandboxSession,
188+
pub result: Result<(), SessionError>,
189+
/// The signed mention that requested this start, retained for recovery.
190+
pub replay_floor: u64,
191+
}
182192

183193
/// The mention's thread session and the time the harness replays from,
184194
/// for a mention by the owner or an agent it attested.

‎apps/buzz-gpui/crates/app/src/agent_plugins/remote_workspace/recovery.rs‎

Lines changed: 69 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -115,10 +115,29 @@ impl View {
115115
struct Entry {
116116
/// The start that failed, sent again by its actions.
117117
session: SandboxSession,
118+
replay_floor: u64,
118119
view: View,
119120
busy: bool,
120121
}
121122

123+
impl Entry {
124+
fn request(&self, action: Action) -> Option<SandboxRequest> {
125+
if self.busy {
126+
return None;
127+
}
128+
let session = self.session.clone();
129+
match action {
130+
Action::Recover(mode) => Some(SandboxRequest::Recover {
131+
session,
132+
mode,
133+
replay_floor: Some(self.replay_floor),
134+
}),
135+
Action::End | Action::Check => Some(SandboxRequest::End { session }),
136+
Action::Dismiss => None,
137+
}
138+
}
139+
}
140+
122141
/// Failed starts awaiting a choice, and which agents' providers can end
123142
/// and recover sessions.
124143
#[derive(Default)]
@@ -151,7 +170,12 @@ impl Prompts {
151170
/// Records a start that failed with a lifecycle code, and clears the
152171
/// thread's prompt once one works. Returns whether a prompt was
153172
/// recorded.
154-
pub fn started(&mut self, (session, result): &Sent, name: &str) -> bool {
173+
pub fn started(&mut self, sent: &Sent, name: &str) -> bool {
174+
let Sent {
175+
session,
176+
result,
177+
replay_floor,
178+
} = sent;
155179
let Some((key, scope)) = key_of(session) else {
156180
return false;
157181
};
@@ -175,6 +199,7 @@ impl Prompts {
175199
});
176200
let entry = Entry {
177201
session: session.clone(),
202+
replay_floor: *replay_floor,
178203
view,
179204
busy: false,
180205
};
@@ -244,10 +269,10 @@ pub fn started(sent: &Sent, name: &str, cx: &mut App) {
244269
return;
245270
}
246271
let recorded = cx.update_global(|prompts: &mut Prompts, _| prompts.started(sent, name));
247-
let pubkey = sent.0.pubkey.clone();
272+
let pubkey = sent.session.pubkey.clone();
248273
let prompts = cx.global::<Prompts>();
249274
if recorded && !prompts.lifecycle.contains_key(&pubkey) && !prompts.checking.contains(&pubkey) {
250-
check_lifecycle(&sent.0, cx);
275+
check_lifecycle(&sent.session, cx);
251276
}
252277
thread_banners::refresh(cx);
253278
}
@@ -317,17 +342,15 @@ fn act(key: &RecoveryPromptKey, action: Action, cx: &mut App) {
317342
let Some(entry) = cx.global::<Prompts>().entries.get(key).cloned() else {
318343
return;
319344
};
320-
let (pubkey, session) = (entry.session.pubkey.clone(), entry.session);
321-
let request = match action {
322-
_ if entry.busy => return,
323-
Action::Dismiss => {
324-
cx.global_mut::<Prompts>().clear(key);
325-
thread_banners::refresh(cx);
326-
return;
327-
}
328-
Action::Recover(mode) => SandboxRequest::Recover { session, mode },
329-
Action::End | Action::Check => SandboxRequest::End { session },
345+
if action == Action::Dismiss && !entry.busy {
346+
cx.global_mut::<Prompts>().clear(key);
347+
thread_banners::refresh(cx);
348+
return;
349+
}
350+
let Some(request) = entry.request(action) else {
351+
return;
330352
};
353+
let pubkey = entry.session.pubkey.clone();
331354
let Some(link) = link_of(&pubkey, cx) else {
332355
return;
333356
};
@@ -417,10 +440,39 @@ mod tests {
417440
);
418441
}
419442

443+
fn sent(root: char, result: Result<(), SessionError>) -> Sent {
444+
Sent {
445+
session: session(root),
446+
result,
447+
replay_floor: 1_700_000_123,
448+
}
449+
}
450+
451+
#[test]
452+
fn recovery_buttons_replay_the_failed_mention_for_both_modes() {
453+
let mut prompts = Prompts::default();
454+
assert!(prompts.started(&sent('c', Err(coded("session_ended"))), "Ada"));
455+
let shown = prompts.for_thread(&thread('c'));
456+
let entry = &shown[0].1;
457+
for mode in [RecoveryMode::Checkpoint, RecoveryMode::Fresh] {
458+
let Some(SandboxRequest::Recover {
459+
session,
460+
mode: requested,
461+
replay_floor,
462+
}) = entry.request(Action::Recover(mode))
463+
else {
464+
panic!("expected recovery request")
465+
};
466+
assert_eq!(requested, mode);
467+
assert_eq!(session.scope.thread_root, "c".repeat(64));
468+
assert_eq!(replay_floor, Some(1_700_000_123));
469+
}
470+
}
471+
420472
#[test]
421473
fn a_coded_failure_records_a_prompt_and_a_success_clears_it() {
422474
let mut prompts = Prompts::default();
423-
assert!(prompts.started(&(session('c'), Err(coded("session_ended"))), "Ada"));
475+
assert!(prompts.started(&sent('c', Err(coded("session_ended"))), "Ada"));
424476
let shown = prompts.for_thread(&thread('c'));
425477
assert_eq!(shown.len(), 1);
426478
assert_eq!(shown[0].0.agent_name, "Ada");
@@ -432,17 +484,17 @@ mod tests {
432484

433485
// A failure without a lifecycle code asks for nothing.
434486
let transient = SessionError::from("The cluster is unreachable");
435-
assert!(!prompts.started(&(session('d'), Err(transient)), "Ada"));
487+
assert!(!prompts.started(&sent('d', Err(transient)), "Ada"));
436488
assert!(prompts.for_thread(&thread('d')).is_empty());
437489

438-
assert!(!prompts.started(&(session('c'), Ok(())), "Ada"));
490+
assert!(!prompts.started(&sent('c', Ok(())), "Ada"));
439491
assert!(prompts.for_thread(&thread('c')).is_empty());
440492
}
441493

442494
#[test]
443495
fn an_end_reply_picks_the_next_banner() {
444496
let mut prompts = Prompts::default();
445-
prompts.started(&(session('c'), Err(coded("session_active"))), "Ada");
497+
prompts.started(&sent('c', Err(coded("session_active"))), "Ada");
446498
let key = key_of(&session('c')).unwrap().0;
447499
let view = |prompts: &Prompts| prompts.entries[&key].view.clone();
448500
assert_eq!(view(&prompts).kind, RecoveryKind::EndFirst);

‎apps/buzz-gpui/crates/app/src/agent_plugins/remote_workspace/tests.rs‎

Lines changed: 22 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -152,7 +152,17 @@ async fn a_mention_starts_its_threads_session() {
152152
)
153153
.await
154154
.unwrap();
155-
assert!(matches!(started, Some((_, Ok(())))), "{started:?}");
155+
assert!(
156+
matches!(
157+
started,
158+
Some(Sent {
159+
result: Ok(()),
160+
replay_floor: 1_700_000_123,
161+
..
162+
})
163+
),
164+
"{started:?}"
165+
);
156166
let sent: [SandboxRequest; 1] = sent.lock().unwrap().clone().try_into().unwrap();
157167
let [
158168
SandboxRequest::Start {
@@ -350,7 +360,17 @@ async fn a_relay_mention_starts_its_session_through_the_host() {
350360
"provider_config": {"relay_url": RELAY, "manager_pubkey": MANAGER},
351361
});
352362
let started = deliver(delivery, RELAY.into()).await.unwrap();
353-
assert!(matches!(started, Some((_, Ok(())))), "{started:?}");
363+
assert!(
364+
matches!(
365+
started,
366+
Some(Sent {
367+
result: Ok(()),
368+
replay_floor: 1_700_000_123,
369+
..
370+
})
371+
),
372+
"{started:?}"
373+
);
354374
let [SandboxRequest::Start { session, .. }] = &sent.lock().unwrap()[..] else {
355375
panic!("expected one Start");
356376
};

‎apps/buzz-gpui/crates/host/src/api/sandbox.rs‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,9 @@ pub enum SandboxRequest {
3232
Recover {
3333
session: SandboxSession,
3434
mode: RecoveryMode,
35+
/// Failed mention to replay after recovery; absent for an unprompted recovery.
36+
#[serde(default, skip_serializing_if = "Option::is_none")]
37+
replay_floor: Option<u64>,
3538
},
3639
/// Whether the agent's provider can end and recover sessions. Answers
3740
/// [`SandboxResponse::Lifecycle`].
@@ -243,6 +246,7 @@ mod tests {
243246
let recover = SandboxRequest::Recover {
244247
session: session(),
245248
mode: RecoveryMode::Checkpoint,
249+
replay_floor: None,
246250
};
247251
assert!(matches!(
248252
rmp_serde::from_slice(&wire(&recover)).unwrap(),

‎apps/buzz-gpui/crates/sandbox-host/src/lib.rs‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -157,14 +157,24 @@ impl Context {
157157
},
158158
})
159159
}
160-
SandboxRequest::Recover { session, mode } => {
160+
SandboxRequest::Recover {
161+
session,
162+
mode,
163+
replay_floor,
164+
} => {
161165
let (thread, sessions) = self.prepare(session).await?;
162166
let mode = match mode {
163167
RecoveryMode::Checkpoint => remote::RecoveryMode::Checkpoint,
164168
RecoveryMode::Fresh => remote::RecoveryMode::Fresh,
165169
};
166170
sessions
167-
.recover(&thread.pubkey, thread.tenant(), &thread.scope, mode)
171+
.recover_with_replay_floor(
172+
&thread.pubkey,
173+
thread.tenant(),
174+
&thread.scope,
175+
mode,
176+
replay_floor,
177+
)
168178
.await?;
169179
Ok(SandboxResponse::Started)
170180
}

‎apps/buzz-gpui/crates/sandbox-host/tests/sandboxes.rs‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -251,7 +251,15 @@ async fn a_relay_session_runs_the_relay_provider_by_its_id() {
251251
.await;
252252
assert!(matches!(ended, SandboxResponse::Ended { .. }), "{ended:?}");
253253
let mode = RecoveryMode::Fresh;
254-
let recovered = sandbox(&host, SandboxRequest::Recover { session, mode }).await;
254+
let recovered = sandbox(
255+
&host,
256+
SandboxRequest::Recover {
257+
session,
258+
mode,
259+
replay_floor: Some(1_700_000_123),
260+
},
261+
)
262+
.await;
255263
assert!(
256264
matches!(recovered, SandboxResponse::Started),
257265
"{recovered:?}"
@@ -262,6 +270,10 @@ async fn a_relay_session_runs_the_relay_provider_by_its_id() {
262270
assert_eq!(deploy["op"], "deploy");
263271
assert_eq!(stop["op"], "stop");
264272
assert_eq!(recover["op"], "deploy");
273+
assert_eq!(
274+
recover["agent"]["launch"]["policy_env"]["BUZZ_ACP_REPLAY_FLOOR"],
275+
"1700000123"
276+
);
265277
assert_eq!(
266278
recover["provider_config"]["sandbox"]["recovery"]["mode"],
267279
"fresh"

‎crates/buzz-remote-sessions/src/client.rs‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -202,14 +202,28 @@ impl<S: InvocationSource> RemoteSessions<S> {
202202
tenant: Tenant<'_>,
203203
session: &SessionScope,
204204
mode: RecoveryMode,
205+
) -> Result<(), SessionError> {
206+
self.recover_with_replay_floor(agent_pubkey, tenant, session, mode, None)
207+
.await
208+
}
209+
210+
/// Recover and replay the signed mention whose start failed.
211+
/// The floor is an operation input, never persisted as agent configuration.
212+
pub async fn recover_with_replay_floor(
213+
&self,
214+
agent_pubkey: &str,
215+
tenant: Tenant<'_>,
216+
session: &SessionScope,
217+
mode: RecoveryMode,
218+
replay_floor_unix: Option<u64>,
205219
) -> Result<(), SessionError> {
206220
let tenant = tenant.require_both()?;
207221
self.run_deploy(
208222
agent_pubkey,
209223
tenant,
210224
Some(session),
211225
Action::Recover(mode),
212-
None,
226+
replay_floor_unix,
213227
)
214228
.await
215229
}

‎crates/buzz-remote-sessions/src/client_tests.rs‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -503,3 +503,37 @@ async fn operations_on_one_agent_wait_for_its_lock() {
503503
.unwrap();
504504
assert_eq!(*source.rebuilds.lock().unwrap(), ["idle", "busy"]);
505505
}
506+
507+
#[tokio::test]
508+
async fn recovery_replays_the_failed_mention_without_persisting_its_floor() {
509+
let source = FakeSource::default().with("thread-agent", "kubernetes", json!({"sandbox": true}));
510+
let provider = Provider::new();
511+
provider.install(
512+
"kubernetes",
513+
json!(LIFECYCLE),
514+
json!({"ok": true, "agent_id": "remote"}),
515+
);
516+
let sessions = provider.sessions(&source);
517+
for mode in [RecoveryMode::Checkpoint, RecoveryMode::Fresh] {
518+
sessions
519+
.recover_with_replay_floor(
520+
"thread-agent",
521+
tenant(RELAY, OWNER),
522+
&thread(),
523+
mode,
524+
Some(1_700_000_123),
525+
)
526+
.await
527+
.unwrap();
528+
let request = provider.take_request().unwrap();
529+
assert_eq!(
530+
request["agent"]["launch"]["policy_env"][crate::REPLAY_FLOOR_ENV_VAR],
531+
"1700000123"
532+
);
533+
assert!(
534+
source.records.lock().unwrap()["thread-agent"].payload["launch"]
535+
.get("policy_env")
536+
.is_none()
537+
);
538+
}
539+
}

‎voltage/remote-agents.md‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -334,6 +334,10 @@ Entering a Codex access token replaces a stored OpenAI API key, and entering a C
334334
3. Reply in the same thread to continue the session.
335335
4. Select **End session** in the thread banner to save a checkpoint and stop the Sandbox. Select **Recover from checkpoint** to continue from that checkpoint later.
336336

337+
When a mention fails because its session ended, recovery replays that mention after restoring or starting the workspace.
338+
This requires an updated desktop and host; existing clients retain their earlier recovery behavior.
339+
The replay timestamp belongs to that recovery request and does not change the saved agent configuration.
340+
337341
Each thread gets its own Sandbox, so threads run in parallel. A developer can have five sessions at the same time. A session stops after 12 hours.
338342

339343
## Reference

0 commit comments

Comments
 (0)