Skip to content

Commit 527c0e0

Browse files
committed
feat(agents): recover ended Sandbox sessions explicitly
An ended session is resumed only when the desktop sends provider_config.sandbox.recovery for one click: {"mode":"checkpoint"} restores the tombstone's verified checkpoint, {"mode":"fresh"} starts with an empty workspace. Recovery is refused on anything that is not an Ended, Suspended tombstone, so a live workspace is never discarded, and checkpoint mode is refused when the tombstone has no checkpoint key. Before it deletes the tombstone, the provider preserves it in an immutable <sandbox>-recovery journal Secret without an owner reference. Journal validation checks object identity and ownership. A retry resumes from the journal after an interrupted delete/create; an ordinary deploy refuses an unfinished recovery. Deletion uses uid and resourceVersion preconditions, and recovery waits for the old owned Pod to disappear (including its 240-second drain grace) before it creates the replacement under a new generation with recovered-from. The journal is removed once the replacement is bound. Signed-off-by: Tee8z <[email protected]>
1 parent 575515e commit 527c0e0

9 files changed

Lines changed: 916 additions & 33 deletions

File tree

‎crates/buzz-backend-kubernetes/src/sandbox.rs‎

Lines changed: 224 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -3,21 +3,21 @@
33
//!
44
//! Lifecycle (`docs/remote-agents.md` §Sandbox lifecycle): a session that ended
55
//! — stopped, completed, replaced, lost, or abandoned while binding — is
6-
//! tombstoned by `buzz-sandbox-reaper` and is never redeployed by an ordinary
7-
//! deploy.
6+
//! tombstoned by `buzz-sandbox-reaper` and is only ever resumed by an explicit
7+
//! `recovery` deploy, never by an ordinary one.
88
99
use std::collections::BTreeMap;
1010
use std::time::Duration;
1111

1212
use buzz_backend_kubernetes::lifecycle::{
13-
annotation, owned_pod, sandbox_resource, EndedReason, Lifecycle, OperatingMode, CHANNEL,
14-
CHECKPOINTING, CHECKPOINTING_DISABLED, CHECKPOINTING_ENABLED, ENDED_REASON, GENERATION,
15-
INITIAL_POD, LIFECYCLE, LIFECYCLE_ENDING, OWNER, RESTORE_CHECKPOINT, SANDBOX_BINDING_VERSION,
16-
SESSION, THREAD,
13+
annotation, is_checkpoint_key, owned_pod, sandbox_resource, EndedReason, Lifecycle,
14+
OperatingMode, CHANNEL, CHECKPOINT, CHECKPOINTING, CHECKPOINTING_DISABLED,
15+
CHECKPOINTING_ENABLED, ENDED_REASON, GENERATION, INITIAL_POD, LIFECYCLE, LIFECYCLE_ENDING,
16+
OWNER, RECOVERED_FROM, RESTORE_CHECKPOINT, SANDBOX_BINDING_VERSION, SESSION, THREAD,
1717
};
1818
use k8s_openapi::api::core::v1::{EnvVar, EnvVarSource, ObjectFieldSelector, Pod, Secret};
1919
use k8s_openapi::apimachinery::pkg::apis::meta::v1::OwnerReference;
20-
use kube::api::{Api, ListParams, Patch, PatchParams, PostParams};
20+
use kube::api::{Api, DeleteParams, ListParams, Patch, PatchParams, PostParams, Preconditions};
2121
use kube::core::DynamicObject;
2222
use kube::{Client, ResourceExt};
2323
use serde::Deserialize;
@@ -26,16 +26,39 @@ use sha2::{Digest, Sha256};
2626

2727
use crate::{config::ProviderConfig, intent::Fingerprint, naming, pod};
2828

29+
#[path = "sandbox_recovery.rs"]
30+
mod recovery_journal;
31+
2932
/// Secret env key carrying the checkpoint a recovered session restores from.
3033
/// Always present (empty unless recovering) so its *key* is part of every
3134
/// fingerprint and a later ordinary deploy matches the recovered Sandbox.
3235
pub const RESTORE_KEY_ENV: &str = "BUZZ_CHECKPOINT_RESTORE_KEY";
3336

37+
const SANDBOX_TERMINATION_GRACE_SECONDS: u32 = 300;
38+
39+
/// How an ended session is resumed. Sent only on explicit user action.
40+
#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize)]
41+
#[serde(rename_all = "lowercase")]
42+
pub enum RecoveryMode {
43+
/// Restore the tombstone's verified checkpoint into a new session.
44+
Checkpoint,
45+
/// Start a new session with an empty workspace.
46+
Fresh,
47+
}
48+
49+
#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize)]
50+
#[serde(deny_unknown_fields)]
51+
pub struct Recovery {
52+
pub mode: RecoveryMode,
53+
}
54+
3455
#[derive(Debug, Deserialize)]
3556
#[serde(deny_unknown_fields)]
3657
pub struct Options {
3758
pub channel_id: String,
3859
pub thread_root: String,
60+
#[serde(default)]
61+
pub recovery: Option<Recovery>,
3962
}
4063

4164
impl Options {
@@ -51,6 +74,12 @@ impl Options {
5174
if value == &serde_json::Value::Bool(false) {
5275
return Ok(None);
5376
}
77+
if let Some(recovery) = value.get("recovery") {
78+
serde_json::from_value::<Recovery>(recovery.clone()).map_err(|_| {
79+
"sandbox recovery must be {\"mode\":\"checkpoint\"} or {\"mode\":\"fresh\"}"
80+
.to_string()
81+
})?;
82+
}
5483
let options: Self = serde_json::from_value(value.clone())
5584
.map_err(|_| "sandbox requires channel_id and canonical thread_root".to_string())?;
5685
if uuid::Uuid::parse_str(&options.channel_id)
@@ -85,6 +114,8 @@ fn reason_is(error: &kube::Error, reason: &str) -> bool {
85114
struct NewSession<'a> {
86115
generation: &'a str,
87116
checkpointing: bool,
117+
restore: Option<&'a str>,
118+
recovered_from: Option<&'a str>,
88119
}
89120

90121
fn build(
@@ -121,10 +152,16 @@ fn build(
121152
.into(),
122153
),
123154
]);
155+
if let Some(key) = session.restore {
156+
annotations.insert(RESTORE_CHECKPOINT.into(), key.into());
157+
}
158+
if let Some(previous) = session.recovered_from {
159+
annotations.insert(RECOVERED_FROM.into(), previous.into());
160+
}
124161
let spec = child.spec.as_mut().ok_or("agent Pod has no spec")?;
125162
// SIGTERM must leave time to drain sessions and run the harness's bounded
126163
// (120s) workspace checkpoint before the kubelet kills the container.
127-
spec.termination_grace_period_seconds = Some(240);
164+
spec.termination_grace_period_seconds = Some(i64::from(SANDBOX_TERMINATION_GRACE_SECONDS));
128165
// The harness writes its checkpoint receipt here; the reaper reads it
129166
// back from the container status (and cross-checks it against S3).
130167
spec.containers[0].termination_message_policy = Some("File".into());
@@ -328,8 +365,8 @@ impl<'a> Session<'a> {
328365
}
329366

330367
/// Reuses a running, identically scoped Sandbox. Ended, replaced, or lost
331-
/// sessions are never redeployed implicitly. Namespace creation remains
332-
/// Terraform's concern.
368+
/// sessions are never redeployed implicitly: only `scope.recovery` replaces a
369+
/// tombstone. Namespace creation remains Terraform's concern.
333370
pub async fn deploy(
334371
client: Client,
335372
identity: &naming::AgentIdentity,
@@ -347,18 +384,48 @@ pub async fn deploy(
347384
let checkpointing = apply_session_env(scope, &mut env);
348385
let fingerprints = Fingerprints::new(cfg, &env);
349386
let session = Session::new(client, &cfg.namespace, identity, scope, owner, fingerprints);
350-
let existing = session
387+
let mut existing = session
351388
.sandboxes
352389
.get_opt(&session.name)
353390
.await
354391
.map_err(|e| format!("read Sandbox: {e}"))?;
355-
let sandbox = match existing {
356-
Some(existing) => existing,
357-
None => create_new(&session, cfg, checkpointing).await?,
392+
if existing.is_none() {
393+
if let Some(tombstone) = recovery_journal::load(&session).await? {
394+
if scope.recovery.is_none() {
395+
return Err(
396+
"Sandbox recovery was interrupted; explicit recovery is required".into(),
397+
);
398+
}
399+
existing = Some(tombstone);
400+
}
401+
}
402+
let sandbox = match (existing, scope.recovery) {
403+
(Some(tombstone), Some(recovery)) => {
404+
recover(&session, cfg, tombstone, recovery, checkpointing).await?
405+
}
406+
(Some(existing), None) => existing,
407+
(
408+
None,
409+
Some(Recovery {
410+
mode: RecoveryMode::Checkpoint,
411+
}),
412+
) => {
413+
return Err(
414+
"no ended session exists for this thread to recover; start fresh instead".into(),
415+
)
416+
}
417+
(None, _) => create_new(&session, cfg, checkpointing).await?,
358418
};
359419
verify_identity(&sandbox, identity, scope, owner)?;
360420
verify_live(&sandbox, &session.fingerprints)?;
361-
bind(&session, cfg, sandbox, env).await
421+
let previous = annotation(&sandbox, RECOVERED_FROM).map(str::to_string);
422+
let result = bind(&session, cfg, sandbox, env).await;
423+
if result.is_ok() {
424+
if let Some(previous) = previous {
425+
recovery_journal::clear(&session, &previous).await;
426+
}
427+
}
428+
result
362429
}
363430

364431
/// Thread-session env shared by every deploy. Returns whether
@@ -427,6 +494,8 @@ async fn create_new(
427494
let new = NewSession {
428495
generation: &generation,
429496
checkpointing,
497+
restore: None,
498+
recovered_from: None,
430499
};
431500
let desired = build(
432501
identity,
@@ -453,6 +522,143 @@ async fn create_new(
453522
}
454523
}
455524

525+
/// Replace an Ended tombstone with a new session (new generation). Refused on
526+
/// anything that is not a tombstone, so a live workspace is never discarded.
527+
async fn recover(
528+
session: &Session<'_>,
529+
cfg: &ProviderConfig,
530+
tombstone: DynamicObject,
531+
recovery: Recovery,
532+
checkpointing: bool,
533+
) -> Result<DynamicObject, String> {
534+
verify_identity(&tombstone, session.identity, session.scope, session.owner)?;
535+
if Lifecycle::of(&tombstone) != Lifecycle::Ended
536+
|| OperatingMode::of(&tombstone) != OperatingMode::Suspended
537+
|| tombstone.metadata.deletion_timestamp.is_some()
538+
{
539+
return Err(
540+
"recovery requires an ended session; this Sandbox is still active, so end it first"
541+
.into(),
542+
);
543+
}
544+
let previous = annotation(&tombstone, GENERATION)
545+
.ok_or("ended Sandbox has no generation")?
546+
.to_string();
547+
let restore = match recovery.mode {
548+
RecoveryMode::Fresh => None,
549+
RecoveryMode::Checkpoint => Some(
550+
annotation(&tombstone, CHECKPOINT)
551+
.filter(|key| is_checkpoint_key(key))
552+
.ok_or("this session has no verified checkpoint; start fresh instead")?
553+
.to_string(),
554+
),
555+
};
556+
// Persist the recovery pointer before deleting its only durable source.
557+
// A retry can resume even if deletion succeeds and creation never runs.
558+
recovery_journal::preserve(session, &tombstone).await?;
559+
let uid = tombstone.metadata.uid.clone();
560+
let preconditions = Preconditions {
561+
uid: uid.clone(),
562+
resource_version: tombstone.metadata.resource_version.clone(),
563+
};
564+
match session
565+
.sandboxes
566+
.delete(
567+
&session.name,
568+
&DeleteParams {
569+
preconditions: Some(preconditions),
570+
..Default::default()
571+
},
572+
)
573+
.await
574+
{
575+
Ok(_) => {}
576+
Err(e) if reason_is(&e, "NotFound") => {}
577+
Err(e) if reason_is(&e, "Conflict") => {
578+
return Err("Sandbox changed during recovery; retry".into())
579+
}
580+
Err(e) => return Err(format!("delete ended Sandbox: {e}")),
581+
}
582+
// Allow the old Pod its complete checkpoint grace plus controller/GC time.
583+
let deadline = tokio::time::Instant::now()
584+
+ Duration::from_secs(u64::from(SANDBOX_TERMINATION_GRACE_SECONDS) + 60);
585+
loop {
586+
match session
587+
.sandboxes
588+
.get_opt(&session.name)
589+
.await
590+
.map_err(|e| format!("observe ended Sandbox deletion: {e}"))?
591+
{
592+
Some(current) if current.metadata.uid == uid => {
593+
if tokio::time::Instant::now() >= deadline {
594+
return Err("ended Sandbox is still being deleted; retry recovery".into());
595+
}
596+
tokio::time::sleep(Duration::from_secs(1)).await;
597+
}
598+
// A concurrent recovery of the same tombstone already won.
599+
Some(current) => return recovered_by_peer(current, &previous),
600+
None => {
601+
// Background garbage collection can leave the old Pod after
602+
// its Sandbox disappears. Do not race it with a new owner.
603+
let child = session
604+
.pods
605+
.get_opt(&session.name)
606+
.await
607+
.map_err(|e| format!("observe ended Sandbox Pod deletion: {e}"))?;
608+
match child {
609+
None => break,
610+
Some(child) if owned_pod(&child, &tombstone) => {
611+
if tokio::time::Instant::now() >= deadline {
612+
return Err("ended Sandbox Pod is still being deleted; retry recovery".into());
613+
}
614+
tokio::time::sleep(Duration::from_secs(1)).await;
615+
}
616+
Some(_) => return Err("a foreign Pod occupies the recovered Sandbox name; retain the recovery journal for operator cleanup".into()),
617+
}
618+
}
619+
}
620+
}
621+
let generation = uuid::Uuid::new_v4().simple().to_string();
622+
let new = NewSession {
623+
generation: &generation,
624+
checkpointing,
625+
restore: restore.as_deref(),
626+
recovered_from: Some(&previous),
627+
};
628+
let desired = build(
629+
session.identity,
630+
cfg,
631+
session.scope,
632+
session.owner,
633+
&new,
634+
&session.fingerprints.current,
635+
)?;
636+
match session
637+
.sandboxes
638+
.create(&PostParams::default(), &desired)
639+
.await
640+
{
641+
Ok(created) => Ok(created),
642+
Err(e) if reason_is(&e, "AlreadyExists") => {
643+
let current = session
644+
.sandboxes
645+
.get(&session.name)
646+
.await
647+
.map_err(|e| format!("read concurrent Sandbox: {e}"))?;
648+
recovered_by_peer(current, &previous)
649+
}
650+
Err(e) => Err(format!("create recovered Sandbox: {e}")),
651+
}
652+
}
653+
654+
fn recovered_by_peer(current: DynamicObject, previous: &str) -> Result<DynamicObject, String> {
655+
if annotation(&current, RECOVERED_FROM) == Some(previous) {
656+
Ok(current)
657+
} else {
658+
Err("another session replaced the ended Sandbox during recovery".into())
659+
}
660+
}
661+
456662
fn container_started(pod: &Pod) -> bool {
457663
pod.status
458664
.as_ref()
@@ -745,6 +951,9 @@ pub async fn stop(
745951
scope: &Options,
746952
owner: &str,
747953
) -> Result<(String, StopState), String> {
954+
if scope.recovery.is_some() {
955+
return Err("stop does not accept a recovery mode".into());
956+
}
748957
let sandboxes = api(client, namespace);
749958
let name = scope.name(identity);
750959
for _ in 0..5 {

0 commit comments

Comments
 (0)