33//!
44//! Lifecycle (`docs/remote-agents.md` §Sandbox lifecycle): a session that ended
55//! — stopped, completed, replaced, lost, or abandoned while binding — is
6- //! tombstoned by `buzz-sandbox-reaper` and is never redeployed by an ordinary
7- //! deploy.
6+ //! tombstoned by `buzz-sandbox-reaper` and is only ever resumed by an explicit
7+ //! `recovery` deploy, never by an ordinary one .
88
99use std:: collections:: BTreeMap ;
1010use std:: time:: Duration ;
1111
1212use buzz_backend_kubernetes:: lifecycle:: {
13- annotation, owned_pod, sandbox_resource, EndedReason , Lifecycle , OperatingMode , CHANNEL ,
14- CHECKPOINTING , CHECKPOINTING_DISABLED , CHECKPOINTING_ENABLED , ENDED_REASON , GENERATION ,
15- INITIAL_POD , LIFECYCLE , LIFECYCLE_ENDING , OWNER , RESTORE_CHECKPOINT , SANDBOX_BINDING_VERSION ,
16- SESSION , THREAD ,
13+ annotation, is_checkpoint_key , owned_pod, sandbox_resource, EndedReason , Lifecycle ,
14+ OperatingMode , CHANNEL , CHECKPOINT , CHECKPOINTING , CHECKPOINTING_DISABLED ,
15+ CHECKPOINTING_ENABLED , ENDED_REASON , GENERATION , INITIAL_POD , LIFECYCLE , LIFECYCLE_ENDING ,
16+ OWNER , RECOVERED_FROM , RESTORE_CHECKPOINT , SANDBOX_BINDING_VERSION , SESSION , THREAD ,
1717} ;
1818use k8s_openapi:: api:: core:: v1:: { EnvVar , EnvVarSource , ObjectFieldSelector , Pod , Secret } ;
1919use k8s_openapi:: apimachinery:: pkg:: apis:: meta:: v1:: OwnerReference ;
20- use kube:: api:: { Api , ListParams , Patch , PatchParams , PostParams } ;
20+ use kube:: api:: { Api , DeleteParams , ListParams , Patch , PatchParams , PostParams , Preconditions } ;
2121use kube:: core:: DynamicObject ;
2222use kube:: { Client , ResourceExt } ;
2323use serde:: Deserialize ;
@@ -26,16 +26,39 @@ use sha2::{Digest, Sha256};
2626
2727use crate :: { config:: ProviderConfig , intent:: Fingerprint , naming, pod} ;
2828
29+ #[ path = "sandbox_recovery.rs" ]
30+ mod recovery_journal;
31+
2932/// Secret env key carrying the checkpoint a recovered session restores from.
3033/// Always present (empty unless recovering) so its *key* is part of every
3134/// fingerprint and a later ordinary deploy matches the recovered Sandbox.
3235pub const RESTORE_KEY_ENV : & str = "BUZZ_CHECKPOINT_RESTORE_KEY" ;
3336
37+ const SANDBOX_TERMINATION_GRACE_SECONDS : u32 = 300 ;
38+
39+ /// How an ended session is resumed. Sent only on explicit user action.
40+ #[ derive( Debug , Clone , Copy , PartialEq , Eq , Deserialize ) ]
41+ #[ serde( rename_all = "lowercase" ) ]
42+ pub enum RecoveryMode {
43+ /// Restore the tombstone's verified checkpoint into a new session.
44+ Checkpoint ,
45+ /// Start a new session with an empty workspace.
46+ Fresh ,
47+ }
48+
49+ #[ derive( Debug , Clone , Copy , PartialEq , Eq , Deserialize ) ]
50+ #[ serde( deny_unknown_fields) ]
51+ pub struct Recovery {
52+ pub mode : RecoveryMode ,
53+ }
54+
3455#[ derive( Debug , Deserialize ) ]
3556#[ serde( deny_unknown_fields) ]
3657pub struct Options {
3758 pub channel_id : String ,
3859 pub thread_root : String ,
60+ #[ serde( default ) ]
61+ pub recovery : Option < Recovery > ,
3962}
4063
4164impl Options {
@@ -51,6 +74,12 @@ impl Options {
5174 if value == & serde_json:: Value :: Bool ( false ) {
5275 return Ok ( None ) ;
5376 }
77+ if let Some ( recovery) = value. get ( "recovery" ) {
78+ serde_json:: from_value :: < Recovery > ( recovery. clone ( ) ) . map_err ( |_| {
79+ "sandbox recovery must be {\" mode\" :\" checkpoint\" } or {\" mode\" :\" fresh\" }"
80+ . to_string ( )
81+ } ) ?;
82+ }
5483 let options: Self = serde_json:: from_value ( value. clone ( ) )
5584 . map_err ( |_| "sandbox requires channel_id and canonical thread_root" . to_string ( ) ) ?;
5685 if uuid:: Uuid :: parse_str ( & options. channel_id )
@@ -85,6 +114,8 @@ fn reason_is(error: &kube::Error, reason: &str) -> bool {
85114struct NewSession < ' a > {
86115 generation : & ' a str ,
87116 checkpointing : bool ,
117+ restore : Option < & ' a str > ,
118+ recovered_from : Option < & ' a str > ,
88119}
89120
90121fn build (
@@ -121,10 +152,16 @@ fn build(
121152 . into ( ) ,
122153 ) ,
123154 ] ) ;
155+ if let Some ( key) = session. restore {
156+ annotations. insert ( RESTORE_CHECKPOINT . into ( ) , key. into ( ) ) ;
157+ }
158+ if let Some ( previous) = session. recovered_from {
159+ annotations. insert ( RECOVERED_FROM . into ( ) , previous. into ( ) ) ;
160+ }
124161 let spec = child. spec . as_mut ( ) . ok_or ( "agent Pod has no spec" ) ?;
125162 // SIGTERM must leave time to drain sessions and run the harness's bounded
126163 // (120s) workspace checkpoint before the kubelet kills the container.
127- spec. termination_grace_period_seconds = Some ( 240 ) ;
164+ spec. termination_grace_period_seconds = Some ( i64 :: from ( SANDBOX_TERMINATION_GRACE_SECONDS ) ) ;
128165 // The harness writes its checkpoint receipt here; the reaper reads it
129166 // back from the container status (and cross-checks it against S3).
130167 spec. containers [ 0 ] . termination_message_policy = Some ( "File" . into ( ) ) ;
@@ -328,8 +365,8 @@ impl<'a> Session<'a> {
328365}
329366
330367/// Reuses a running, identically scoped Sandbox. Ended, replaced, or lost
331- /// sessions are never redeployed implicitly. Namespace creation remains
332- /// Terraform's concern.
368+ /// sessions are never redeployed implicitly: only `scope.recovery` replaces a
369+ /// tombstone. Namespace creation remains Terraform's concern.
333370pub async fn deploy (
334371 client : Client ,
335372 identity : & naming:: AgentIdentity ,
@@ -347,18 +384,48 @@ pub async fn deploy(
347384 let checkpointing = apply_session_env ( scope, & mut env) ;
348385 let fingerprints = Fingerprints :: new ( cfg, & env) ;
349386 let session = Session :: new ( client, & cfg. namespace , identity, scope, owner, fingerprints) ;
350- let existing = session
387+ let mut existing = session
351388 . sandboxes
352389 . get_opt ( & session. name )
353390 . await
354391 . map_err ( |e| format ! ( "read Sandbox: {e}" ) ) ?;
355- let sandbox = match existing {
356- Some ( existing) => existing,
357- None => create_new ( & session, cfg, checkpointing) . await ?,
392+ if existing. is_none ( ) {
393+ if let Some ( tombstone) = recovery_journal:: load ( & session) . await ? {
394+ if scope. recovery . is_none ( ) {
395+ return Err (
396+ "Sandbox recovery was interrupted; explicit recovery is required" . into ( ) ,
397+ ) ;
398+ }
399+ existing = Some ( tombstone) ;
400+ }
401+ }
402+ let sandbox = match ( existing, scope. recovery ) {
403+ ( Some ( tombstone) , Some ( recovery) ) => {
404+ recover ( & session, cfg, tombstone, recovery, checkpointing) . await ?
405+ }
406+ ( Some ( existing) , None ) => existing,
407+ (
408+ None ,
409+ Some ( Recovery {
410+ mode : RecoveryMode :: Checkpoint ,
411+ } ) ,
412+ ) => {
413+ return Err (
414+ "no ended session exists for this thread to recover; start fresh instead" . into ( ) ,
415+ )
416+ }
417+ ( None , _) => create_new ( & session, cfg, checkpointing) . await ?,
358418 } ;
359419 verify_identity ( & sandbox, identity, scope, owner) ?;
360420 verify_live ( & sandbox, & session. fingerprints ) ?;
361- bind ( & session, cfg, sandbox, env) . await
421+ let previous = annotation ( & sandbox, RECOVERED_FROM ) . map ( str:: to_string) ;
422+ let result = bind ( & session, cfg, sandbox, env) . await ;
423+ if result. is_ok ( ) {
424+ if let Some ( previous) = previous {
425+ recovery_journal:: clear ( & session, & previous) . await ;
426+ }
427+ }
428+ result
362429}
363430
364431/// Thread-session env shared by every deploy. Returns whether
@@ -427,6 +494,8 @@ async fn create_new(
427494 let new = NewSession {
428495 generation : & generation,
429496 checkpointing,
497+ restore : None ,
498+ recovered_from : None ,
430499 } ;
431500 let desired = build (
432501 identity,
@@ -453,6 +522,143 @@ async fn create_new(
453522 }
454523}
455524
525+ /// Replace an Ended tombstone with a new session (new generation). Refused on
526+ /// anything that is not a tombstone, so a live workspace is never discarded.
527+ async fn recover (
528+ session : & Session < ' _ > ,
529+ cfg : & ProviderConfig ,
530+ tombstone : DynamicObject ,
531+ recovery : Recovery ,
532+ checkpointing : bool ,
533+ ) -> Result < DynamicObject , String > {
534+ verify_identity ( & tombstone, session. identity , session. scope , session. owner ) ?;
535+ if Lifecycle :: of ( & tombstone) != Lifecycle :: Ended
536+ || OperatingMode :: of ( & tombstone) != OperatingMode :: Suspended
537+ || tombstone. metadata . deletion_timestamp . is_some ( )
538+ {
539+ return Err (
540+ "recovery requires an ended session; this Sandbox is still active, so end it first"
541+ . into ( ) ,
542+ ) ;
543+ }
544+ let previous = annotation ( & tombstone, GENERATION )
545+ . ok_or ( "ended Sandbox has no generation" ) ?
546+ . to_string ( ) ;
547+ let restore = match recovery. mode {
548+ RecoveryMode :: Fresh => None ,
549+ RecoveryMode :: Checkpoint => Some (
550+ annotation ( & tombstone, CHECKPOINT )
551+ . filter ( |key| is_checkpoint_key ( key) )
552+ . ok_or ( "this session has no verified checkpoint; start fresh instead" ) ?
553+ . to_string ( ) ,
554+ ) ,
555+ } ;
556+ // Persist the recovery pointer before deleting its only durable source.
557+ // A retry can resume even if deletion succeeds and creation never runs.
558+ recovery_journal:: preserve ( session, & tombstone) . await ?;
559+ let uid = tombstone. metadata . uid . clone ( ) ;
560+ let preconditions = Preconditions {
561+ uid : uid. clone ( ) ,
562+ resource_version : tombstone. metadata . resource_version . clone ( ) ,
563+ } ;
564+ match session
565+ . sandboxes
566+ . delete (
567+ & session. name ,
568+ & DeleteParams {
569+ preconditions : Some ( preconditions) ,
570+ ..Default :: default ( )
571+ } ,
572+ )
573+ . await
574+ {
575+ Ok ( _) => { }
576+ Err ( e) if reason_is ( & e, "NotFound" ) => { }
577+ Err ( e) if reason_is ( & e, "Conflict" ) => {
578+ return Err ( "Sandbox changed during recovery; retry" . into ( ) )
579+ }
580+ Err ( e) => return Err ( format ! ( "delete ended Sandbox: {e}" ) ) ,
581+ }
582+ // Allow the old Pod its complete checkpoint grace plus controller/GC time.
583+ let deadline = tokio:: time:: Instant :: now ( )
584+ + Duration :: from_secs ( u64:: from ( SANDBOX_TERMINATION_GRACE_SECONDS ) + 60 ) ;
585+ loop {
586+ match session
587+ . sandboxes
588+ . get_opt ( & session. name )
589+ . await
590+ . map_err ( |e| format ! ( "observe ended Sandbox deletion: {e}" ) ) ?
591+ {
592+ Some ( current) if current. metadata . uid == uid => {
593+ if tokio:: time:: Instant :: now ( ) >= deadline {
594+ return Err ( "ended Sandbox is still being deleted; retry recovery" . into ( ) ) ;
595+ }
596+ tokio:: time:: sleep ( Duration :: from_secs ( 1 ) ) . await ;
597+ }
598+ // A concurrent recovery of the same tombstone already won.
599+ Some ( current) => return recovered_by_peer ( current, & previous) ,
600+ None => {
601+ // Background garbage collection can leave the old Pod after
602+ // its Sandbox disappears. Do not race it with a new owner.
603+ let child = session
604+ . pods
605+ . get_opt ( & session. name )
606+ . await
607+ . map_err ( |e| format ! ( "observe ended Sandbox Pod deletion: {e}" ) ) ?;
608+ match child {
609+ None => break ,
610+ Some ( child) if owned_pod ( & child, & tombstone) => {
611+ if tokio:: time:: Instant :: now ( ) >= deadline {
612+ return Err ( "ended Sandbox Pod is still being deleted; retry recovery" . into ( ) ) ;
613+ }
614+ tokio:: time:: sleep ( Duration :: from_secs ( 1 ) ) . await ;
615+ }
616+ Some ( _) => return Err ( "a foreign Pod occupies the recovered Sandbox name; retain the recovery journal for operator cleanup" . into ( ) ) ,
617+ }
618+ }
619+ }
620+ }
621+ let generation = uuid:: Uuid :: new_v4 ( ) . simple ( ) . to_string ( ) ;
622+ let new = NewSession {
623+ generation : & generation,
624+ checkpointing,
625+ restore : restore. as_deref ( ) ,
626+ recovered_from : Some ( & previous) ,
627+ } ;
628+ let desired = build (
629+ session. identity ,
630+ cfg,
631+ session. scope ,
632+ session. owner ,
633+ & new,
634+ & session. fingerprints . current ,
635+ ) ?;
636+ match session
637+ . sandboxes
638+ . create ( & PostParams :: default ( ) , & desired)
639+ . await
640+ {
641+ Ok ( created) => Ok ( created) ,
642+ Err ( e) if reason_is ( & e, "AlreadyExists" ) => {
643+ let current = session
644+ . sandboxes
645+ . get ( & session. name )
646+ . await
647+ . map_err ( |e| format ! ( "read concurrent Sandbox: {e}" ) ) ?;
648+ recovered_by_peer ( current, & previous)
649+ }
650+ Err ( e) => Err ( format ! ( "create recovered Sandbox: {e}" ) ) ,
651+ }
652+ }
653+
654+ fn recovered_by_peer ( current : DynamicObject , previous : & str ) -> Result < DynamicObject , String > {
655+ if annotation ( & current, RECOVERED_FROM ) == Some ( previous) {
656+ Ok ( current)
657+ } else {
658+ Err ( "another session replaced the ended Sandbox during recovery" . into ( ) )
659+ }
660+ }
661+
456662fn container_started ( pod : & Pod ) -> bool {
457663 pod. status
458664 . as_ref ( )
@@ -745,6 +951,9 @@ pub async fn stop(
745951 scope : & Options ,
746952 owner : & str ,
747953) -> Result < ( String , StopState ) , String > {
954+ if scope. recovery . is_some ( ) {
955+ return Err ( "stop does not accept a recovery mode" . into ( ) ) ;
956+ }
748957 let sandboxes = api ( client, namespace) ;
749958 let name = scope. name ( identity) ;
750959 for _ in 0 ..5 {
0 commit comments