@@ -10,10 +10,12 @@ use std::collections::BTreeMap;
1010use std:: time:: Duration ;
1111
1212use buzz_backend_kubernetes:: lifecycle:: {
13- annotation, owned_pod, sandbox_resource, EndedReason , Lifecycle , OperatingMode , CHANNEL ,
14- CHECKPOINTING , CHECKPOINTING_DISABLED , CHECKPOINTING_ENABLED , ENDED_REASON , GENERATION ,
15- INITIAL_POD , OWNER , SANDBOX_BINDING_VERSION , SESSION , THREAD ,
13+ annotation, owned_pod, sandbox_resource, EndedReason , Lifecycle , OperatingMode ,
14+ BIND_TIMEOUT_SECS , CHANNEL , CHECKPOINTING , CHECKPOINTING_DISABLED , CHECKPOINTING_ENABLED ,
15+ ENDED_REASON , GENERATION , INITIAL_POD , OWNER , RESTORE_CHECKPOINT , SANDBOX_BINDING_VERSION ,
16+ SESSION , THREAD ,
1617} ;
18+ use chrono:: { DateTime , Utc } ;
1719use k8s_openapi:: api:: core:: v1:: { EnvVar , EnvVarSource , ObjectFieldSelector , Pod , Secret } ;
1820use k8s_openapi:: apimachinery:: pkg:: apis:: meta:: v1:: OwnerReference ;
1921use kube:: api:: { Api , ListParams , PostParams } ;
@@ -26,6 +28,16 @@ use sha2::{Digest, Sha256};
2628use crate :: wire:: { ErrorCode , ProviderError } ;
2729use crate :: { config:: ProviderConfig , intent:: Fingerprint , naming, pod} ;
2830
31+ /// Secret env key carrying the checkpoint a recovered session restores from.
32+ /// Always present (empty unless recovering) so its *key* is part of every
33+ /// fingerprint and a later ordinary deploy matches the recovered Sandbox.
34+ pub const RESTORE_KEY_ENV : & str = "BUZZ_CHECKPOINT_RESTORE_KEY" ;
35+
36+ /// How long one deploy waits for its binding, and then for startup.
37+ const DEPLOY_DEADLINE_SECS : u64 = 120 ;
38+ /// Allowance for skew between this host's clock and the apiserver's.
39+ const CLOCK_SKEW_SECS : i64 = 60 ;
40+
2941#[ derive( Debug , Deserialize ) ]
3042#[ serde( deny_unknown_fields) ]
3143pub struct Options {
@@ -182,9 +194,35 @@ fn verify_identity(
182194 Ok ( ( ) )
183195}
184196
197+ /// Launch fingerprints a live Sandbox may carry and still be reused.
198+ struct Fingerprints {
199+ /// Every key this version writes, including [`RESTORE_KEY_ENV`].
200+ current : Fingerprint ,
201+ /// The same launch from a provider that predates [`RESTORE_KEY_ENV`].
202+ /// Accepted only on reuse, so Sandboxes created before this version keep
203+ /// working while desktop versions are mixed; never written.
204+ legacy : Fingerprint ,
205+ }
206+
207+ impl Fingerprints {
208+ fn new ( cfg : & ProviderConfig , env : & BTreeMap < String , String > ) -> Self {
209+ let keys = || env. keys ( ) . cloned ( ) ;
210+ Self {
211+ current : pod:: intent_template ( cfg, keys ( ) ) . fingerprint ( ) ,
212+ legacy : pod:: intent_template ( cfg, keys ( ) . filter ( |key| key != RESTORE_KEY_ENV ) )
213+ . fingerprint ( ) ,
214+ }
215+ }
216+
217+ fn accepts ( & self , sandbox : & DynamicObject ) -> bool {
218+ let recorded = annotation ( sandbox, naming:: ANNOTATION_CREATE_INTENT ) ;
219+ recorded == Some ( self . current . as_str ( ) ) || recorded == Some ( self . legacy . as_str ( ) )
220+ }
221+ }
222+
185223/// The Sandbox is a live session this deploy may reuse. Every refusal names the
186224/// lifecycle state so the desktop can offer the right explicit action.
187- fn verify_live ( sandbox : & DynamicObject , fingerprint : & Fingerprint ) -> Result < ( ) , ProviderError > {
225+ fn verify_live ( sandbox : & DynamicObject , fingerprints : & Fingerprints ) -> Result < ( ) , ProviderError > {
188226 let refused = |code, message : & str | Err ( ProviderError :: refused ( code, message) ) ;
189227 if sandbox. metadata . deletion_timestamp . is_some ( ) {
190228 return refused (
@@ -225,7 +263,7 @@ fn verify_live(sandbox: &DynamicObject, fingerprint: &Fingerprint) -> Result<(),
225263 "Sandbox is not an active managed session; explicit recovery is required" ,
226264 ) ;
227265 }
228- if annotation ( sandbox , naming :: ANNOTATION_CREATE_INTENT ) != Some ( fingerprint . as_str ( ) ) {
266+ if !fingerprints . accepts ( sandbox ) {
229267 return refused (
230268 ErrorCode :: ConfigDiffers ,
231269 "Sandbox launch configuration differs; end the session, then explicit recovery is required" ,
@@ -294,7 +332,7 @@ struct Session<'a> {
294332 identity : & ' a naming:: AgentIdentity ,
295333 scope : & ' a Options ,
296334 owner : & ' a str ,
297- fingerprint : Fingerprint ,
335+ fingerprints : Fingerprints ,
298336}
299337
300338impl < ' a > Session < ' a > {
@@ -304,7 +342,7 @@ impl<'a> Session<'a> {
304342 identity : & ' a naming:: AgentIdentity ,
305343 scope : & ' a Options ,
306344 owner : & ' a str ,
307- fingerprint : Fingerprint ,
345+ fingerprints : Fingerprints ,
308346 ) -> Self {
309347 Self {
310348 sandboxes : api ( client. clone ( ) , namespace) ,
@@ -314,7 +352,7 @@ impl<'a> Session<'a> {
314352 identity,
315353 scope,
316354 owner,
317- fingerprint ,
355+ fingerprints ,
318356 }
319357 }
320358}
@@ -337,8 +375,8 @@ pub async fn deploy(
337375 return Err ( "Sandbox requires pod_options.active_deadline_seconds" . into ( ) ) ;
338376 }
339377 let checkpointing = apply_session_env ( scope, & mut env) ?;
340- let fingerprint = pod :: intent_template ( cfg, env. keys ( ) . cloned ( ) ) . fingerprint ( ) ;
341- let session = Session :: new ( client, & cfg. namespace , identity, scope, owner, fingerprint ) ;
378+ let fingerprints = Fingerprints :: new ( cfg, & env) ;
379+ let session = Session :: new ( client, & cfg. namespace , identity, scope, owner, fingerprints ) ;
342380 let existing = session
343381 . sandboxes
344382 . get_opt ( & session. name )
@@ -349,7 +387,7 @@ pub async fn deploy(
349387 None => create_new ( & session, cfg, checkpointing) . await ?,
350388 } ;
351389 verify_identity ( & sandbox, identity, scope, owner) ?;
352- verify_live ( & sandbox, & session. fingerprint ) ?;
390+ verify_live ( & sandbox, & session. fingerprints ) ?;
353391 bind ( & session, cfg, sandbox, env) . await
354392}
355393
@@ -396,8 +434,10 @@ fn apply_session_env(scope: &Options, env: &mut BTreeMap<String, String>) -> Res
396434 if checkpointing {
397435 env. insert ( "BUZZ_ACP_CHECKPOINT_ON_STOP" . into ( ) , "true" . into ( ) ) ;
398436 }
399- // Include the UID binding key in the fingerprint without hashing its value.
437+ // Include the binding keys in the fingerprint without hashing their
438+ // values; both are filled in from the Sandbox when the Secret is written.
400439 env. insert ( "BUZZ_SANDBOX_INITIAL_POD_UID" . into ( ) , String :: new ( ) ) ;
440+ env. insert ( RESTORE_KEY_ENV . into ( ) , String :: new ( ) ) ;
401441 Ok ( checkpointing)
402442}
403443
@@ -449,7 +489,7 @@ async fn create_new(
449489 session. scope ,
450490 session. owner ,
451491 & new,
452- & session. fingerprint ,
492+ & session. fingerprints . current ,
453493 ) ?;
454494 match session
455495 . sandboxes
@@ -469,7 +509,39 @@ async fn create_new(
469509 }
470510}
471511
512+ /// The reaper ends a never-started binding once its Pod is
513+ /// [`BIND_TIMEOUT_SECS`] old, by a CAS on the Sandbox that a Secret write does
514+ /// not bump. A deploy stops resuming early enough to write the Secret and see
515+ /// the container start before then.
516+ fn resume_window_closed ( pod : & Pod , now : DateTime < Utc > ) -> bool {
517+ let window = BIND_TIMEOUT_SECS - DEPLOY_DEADLINE_SECS as i64 - CLOCK_SKEW_SECS ;
518+ pod. metadata
519+ . creation_timestamp
520+ . as_ref ( )
521+ . is_some_and ( |created| now - created. 0 >= chrono:: Duration :: seconds ( window) )
522+ }
523+
524+ fn container_started ( pod : & Pod ) -> bool {
525+ pod. status
526+ . as_ref ( )
527+ . and_then ( |s| s. container_statuses . as_ref ( ) )
528+ . and_then ( |containers| containers. iter ( ) . find ( |c| c. name == "agent" ) )
529+ . is_some_and ( |c| {
530+ c. restart_count > 0
531+ || c. last_state
532+ . as_ref ( )
533+ . is_some_and ( |s| s. terminated . is_some ( ) || s. running . is_some ( ) )
534+ || c. state
535+ . as_ref ( )
536+ . is_some_and ( |s| s. terminated . is_some ( ) || s. running . is_some ( ) )
537+ } )
538+ }
539+
472540/// Bind credentials to the Sandbox's first Pod and wait for it to start.
541+ ///
542+ /// Resumable: a session bound to its first Pod whose Secret is missing and
543+ /// whose container never started is completed by whichever deploy observes
544+ /// it, with the Sandbox's own generation and an ownerRef to that Pod.
473545async fn bind (
474546 session : & Session < ' _ > ,
475547 cfg : & ProviderConfig ,
@@ -481,8 +553,9 @@ async fn bind(
481553 let generation = annotation ( & sandbox, GENERATION )
482554 . ok_or ( "Sandbox generation is missing" ) ?
483555 . to_string ( ) ;
484- let deadline = tokio:: time:: Instant :: now ( ) + Duration :: from_secs ( 120 ) ;
556+ let deadline = tokio:: time:: Instant :: now ( ) + Duration :: from_secs ( DEPLOY_DEADLINE_SECS ) ;
485557 loop {
558+ let expired = tokio:: time:: Instant :: now ( ) >= deadline;
486559 let Some ( child) = session
487560 . pods
488561 . get_opt ( name)
@@ -495,7 +568,7 @@ async fn bind(
495568 "Sandbox lost its original Pod; explicit recovery is required" ,
496569 ) ) ;
497570 }
498- if tokio :: time :: Instant :: now ( ) >= deadline {
571+ if expired {
499572 return Err (
500573 "Sandbox controller has not created its Pod; inspect the Sandbox and retry"
501574 . into ( ) ,
@@ -529,62 +602,78 @@ async fn bind(
529602 sandbox
530603 . annotations_mut ( )
531604 . insert ( INITIAL_POD . into ( ) , uid. into ( ) ) ;
532- sandbox = match session
605+ match session
533606 . sandboxes
534607 . replace ( name, & PostParams :: default ( ) , & sandbox)
535608 . await
536609 {
537- Ok ( bound) => bound,
610+ Ok ( bound) => {
611+ sandbox = bound;
612+ }
538613 Err ( e) if reason_is ( & e, "Conflict" ) => {
539614 sandbox = refresh ( session, & generation) . await ?;
540- if tokio :: time :: Instant :: now ( ) >= deadline {
615+ if expired {
541616 return Err (
542617 "Sandbox kept changing during credential binding; retry deploy" . into ( ) ,
543618 ) ;
544619 }
545- continue ;
546620 }
547621 Err ( e) => return Err ( format ! ( "bind Sandbox to its first Pod: {e}" ) . into ( ) ) ,
548- } ;
549- let secret = bound_secret ( identity, cfg, & generation, name, uid, env) ;
550- session
551- . secrets
552- . create ( & PostParams :: default ( ) , & secret)
553- . await
554- . map_err ( |e| {
555- ProviderError :: refused (
556- ErrorCode :: BindingIncomplete ,
557- format ! (
558- "could not create bound Sandbox credentials ({}); explicit recovery is required" ,
559- api_failure( & e)
560- ) ,
561- )
562- } ) ?;
563- return wait_for_start ( & session. pods , & sandbox, name, uid) . await ;
622+ }
623+ continue ;
564624 } ;
565625 if uid != initial_uid {
566626 return Err ( ProviderError :: refused (
567627 ErrorCode :: PodReplaced ,
568628 "Sandbox Pod was replaced; credentials remain bound to the original session, explicit recovery is required" ,
569629 ) ) ;
570630 }
571- let secret = session
631+ let secret_name = identity. secret_name ( & generation) ;
632+ let secret =
633+ session. secrets . get_opt ( & secret_name) . await . map_err ( |e| {
634+ format ! ( "could not check Sandbox launch Secret: {}" , api_failure( & e) )
635+ } ) ?;
636+ if let Some ( secret) = secret {
637+ check_secret ( & secret, identity, uid, & env) ?;
638+ return wait_for_start ( & session. pods , & sandbox, name, uid) . await ;
639+ }
640+ if container_started ( & child) {
641+ return Err ( ProviderError :: refused (
642+ ErrorCode :: CredentialsMissing ,
643+ "Sandbox credentials are missing from a started session; explicit recovery is required" ,
644+ ) ) ;
645+ }
646+ if expired {
647+ return Err ( ProviderError :: refused (
648+ ErrorCode :: BindingIncomplete ,
649+ "Sandbox credential binding is incomplete; retry deploy" ,
650+ ) ) ;
651+ }
652+ if resume_window_closed ( & child, Utc :: now ( ) ) {
653+ return Err ( ProviderError :: refused (
654+ ErrorCode :: BindingExpired ,
655+ "Sandbox credential binding is too old to resume; explicit recovery is required once the session has ended" ,
656+ ) ) ;
657+ }
658+ let secret = bound_secret ( identity, cfg, & sandbox, & generation, name, uid, env. clone ( ) ) ;
659+ match session
572660 . secrets
573- . get_opt ( & identity . secret_name ( & generation ) )
661+ . create ( & PostParams :: default ( ) , & secret )
574662 . await
575- . map_err ( |e| format ! ( "could not check Sandbox launch Secret: {}" , api_failure( & e) ) ) ?;
576- let Some ( secret) = secret else {
577- if tokio:: time:: Instant :: now ( ) >= deadline {
578- return Err ( ProviderError :: refused (
579- ErrorCode :: BindingIncomplete ,
580- "Sandbox credential binding is incomplete; explicit recovery is required" ,
581- ) ) ;
663+ {
664+ // Either way, the next pass re-reads and verifies what was stored.
665+ Ok ( _) => { }
666+ Err ( e) if reason_is ( & e, "AlreadyExists" ) => { }
667+ Err ( e) => {
668+ // The write may have succeeded despite a lost response.
669+ // Keep the Pod binding so retries verify the same session.
670+ return Err ( format ! (
671+ "could not create bound Sandbox credentials ({}); retry deploy" ,
672+ api_failure( & e)
673+ )
674+ . into ( ) ) ;
582675 }
583- tokio:: time:: sleep ( Duration :: from_millis ( 250 ) ) . await ;
584- continue ;
585- } ;
586- check_secret ( & secret, identity, uid, & env) ?;
587- return wait_for_start ( & session. pods , & sandbox, name, uid) . await ;
676+ }
588677 }
589678}
590679
@@ -596,7 +685,7 @@ async fn refresh(session: &Session<'_>, generation: &str) -> Result<DynamicObjec
596685 . await
597686 . map_err ( |e| format ! ( "refresh Sandbox binding: {e}" ) ) ?;
598687 verify_identity ( & sandbox, session. identity , session. scope , session. owner ) ?;
599- verify_live ( & sandbox, & session. fingerprint ) ?;
688+ verify_live ( & sandbox, & session. fingerprints ) ?;
600689 if annotation ( & sandbox, GENERATION ) != Some ( generation) {
601690 return Err ( ProviderError :: refused (
602691 ErrorCode :: SessionReplaced ,
@@ -609,13 +698,22 @@ async fn refresh(session: &Session<'_>, generation: &str) -> Result<DynamicObjec
609698fn bound_secret (
610699 identity : & naming:: AgentIdentity ,
611700 cfg : & ProviderConfig ,
701+ sandbox : & DynamicObject ,
612702 generation : & str ,
613703 name : & str ,
614704 uid : & str ,
615705 mut env : BTreeMap < String , String > ,
616706) -> Secret {
617707 env. insert ( "BUZZ_MANAGED_AGENT_START_NONCE" . into ( ) , generation. into ( ) ) ;
618708 env. insert ( "BUZZ_SANDBOX_INITIAL_POD_UID" . into ( ) , uid. into ( ) ) ;
709+ // The restore key comes from the Sandbox, never from the request, so a
710+ // resumed binding keeps the recovery its Sandbox was created with.
711+ env. insert (
712+ RESTORE_KEY_ENV . into ( ) ,
713+ annotation ( sandbox, RESTORE_CHECKPOINT )
714+ . unwrap_or_default ( )
715+ . into ( ) ,
716+ ) ;
619717 let mut secret = pod:: build_secret ( identity, & cfg. namespace , generation, env) ;
620718 secret. labels_mut ( ) . insert (
621719 naming:: LABEL_BINDING_VERSION . into ( ) ,
@@ -674,7 +772,7 @@ async fn wait_for_start(
674772 name : & str ,
675773 uid : & str ,
676774) -> Result < String , ProviderError > {
677- let deadline = tokio:: time:: Instant :: now ( ) + Duration :: from_secs ( 120 ) ;
775+ let deadline = tokio:: time:: Instant :: now ( ) + Duration :: from_secs ( DEPLOY_DEADLINE_SECS ) ;
678776 loop {
679777 let child = pods
680778 . get_opt ( name)
0 commit comments