Skip to content

Commit 0397d34

Browse files
committed
feat(agents): resume interrupted Sandbox credential binding
A deploy that bound the Sandbox to its first Pod but never wrote the launch Secret used to strand the session. The next deploy now finishes the binding while the container has not started, with the Sandbox's own generation and an owner reference to that Pod. When the Secret write fails, the deploy keeps the first-Pod binding, because the write may have succeeded despite the error; the retry re-reads and verifies the Secret. A started session without credentials still requires explicit recovery (credentials_missing), and a deploy stops resuming before the reaper's bind timeout can end the session under it (binding_expired). Every launch Secret now carries BUZZ_CHECKPOINT_RESTORE_KEY (empty unless a recovered session restores a checkpoint). The value comes from the Sandbox, never the request, so a resumed binding keeps the restore its Sandbox was created with. The key is part of every launch fingerprint; Sandboxes created before it existed are still reused. That acceptance is one-way: an older provider refuses a Sandbox created by this version as a configuration change. Signed-off-by: Tee8z <[email protected]>
1 parent ffc3a25 commit 0397d34

5 files changed

Lines changed: 507 additions & 67 deletions

File tree

‎crates/buzz-backend-kubernetes/src/sandbox.rs‎

Lines changed: 149 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -10,10 +10,12 @@ use std::collections::BTreeMap;
1010
use std::time::Duration;
1111

1212
use buzz_backend_kubernetes::lifecycle::{
13-
annotation, owned_pod, sandbox_resource, EndedReason, Lifecycle, OperatingMode, CHANNEL,
14-
CHECKPOINTING, CHECKPOINTING_DISABLED, CHECKPOINTING_ENABLED, ENDED_REASON, GENERATION,
15-
INITIAL_POD, OWNER, SANDBOX_BINDING_VERSION, SESSION, THREAD,
13+
annotation, owned_pod, sandbox_resource, EndedReason, Lifecycle, OperatingMode,
14+
BIND_TIMEOUT_SECS, CHANNEL, CHECKPOINTING, CHECKPOINTING_DISABLED, CHECKPOINTING_ENABLED,
15+
ENDED_REASON, GENERATION, INITIAL_POD, OWNER, RESTORE_CHECKPOINT, SANDBOX_BINDING_VERSION,
16+
SESSION, THREAD,
1617
};
18+
use chrono::{DateTime, Utc};
1719
use k8s_openapi::api::core::v1::{EnvVar, EnvVarSource, ObjectFieldSelector, Pod, Secret};
1820
use k8s_openapi::apimachinery::pkg::apis::meta::v1::OwnerReference;
1921
use kube::api::{Api, ListParams, PostParams};
@@ -26,6 +28,16 @@ use sha2::{Digest, Sha256};
2628
use crate::wire::{ErrorCode, ProviderError};
2729
use crate::{config::ProviderConfig, intent::Fingerprint, naming, pod};
2830

31+
/// Secret env key carrying the checkpoint a recovered session restores from.
32+
/// Always present (empty unless recovering) so its *key* is part of every
33+
/// fingerprint and a later ordinary deploy matches the recovered Sandbox.
34+
pub const RESTORE_KEY_ENV: &str = "BUZZ_CHECKPOINT_RESTORE_KEY";
35+
36+
/// How long one deploy waits for its binding, and then for startup.
37+
const DEPLOY_DEADLINE_SECS: u64 = 120;
38+
/// Allowance for skew between this host's clock and the apiserver's.
39+
const CLOCK_SKEW_SECS: i64 = 60;
40+
2941
#[derive(Debug, Deserialize)]
3042
#[serde(deny_unknown_fields)]
3143
pub struct Options {
@@ -182,9 +194,35 @@ fn verify_identity(
182194
Ok(())
183195
}
184196

197+
/// Launch fingerprints a live Sandbox may carry and still be reused.
198+
struct Fingerprints {
199+
/// Every key this version writes, including [`RESTORE_KEY_ENV`].
200+
current: Fingerprint,
201+
/// The same launch from a provider that predates [`RESTORE_KEY_ENV`].
202+
/// Accepted only on reuse, so Sandboxes created before this version keep
203+
/// working while desktop versions are mixed; never written.
204+
legacy: Fingerprint,
205+
}
206+
207+
impl Fingerprints {
208+
fn new(cfg: &ProviderConfig, env: &BTreeMap<String, String>) -> Self {
209+
let keys = || env.keys().cloned();
210+
Self {
211+
current: pod::intent_template(cfg, keys()).fingerprint(),
212+
legacy: pod::intent_template(cfg, keys().filter(|key| key != RESTORE_KEY_ENV))
213+
.fingerprint(),
214+
}
215+
}
216+
217+
fn accepts(&self, sandbox: &DynamicObject) -> bool {
218+
let recorded = annotation(sandbox, naming::ANNOTATION_CREATE_INTENT);
219+
recorded == Some(self.current.as_str()) || recorded == Some(self.legacy.as_str())
220+
}
221+
}
222+
185223
/// The Sandbox is a live session this deploy may reuse. Every refusal names the
186224
/// lifecycle state so the desktop can offer the right explicit action.
187-
fn verify_live(sandbox: &DynamicObject, fingerprint: &Fingerprint) -> Result<(), ProviderError> {
225+
fn verify_live(sandbox: &DynamicObject, fingerprints: &Fingerprints) -> Result<(), ProviderError> {
188226
let refused = |code, message: &str| Err(ProviderError::refused(code, message));
189227
if sandbox.metadata.deletion_timestamp.is_some() {
190228
return refused(
@@ -225,7 +263,7 @@ fn verify_live(sandbox: &DynamicObject, fingerprint: &Fingerprint) -> Result<(),
225263
"Sandbox is not an active managed session; explicit recovery is required",
226264
);
227265
}
228-
if annotation(sandbox, naming::ANNOTATION_CREATE_INTENT) != Some(fingerprint.as_str()) {
266+
if !fingerprints.accepts(sandbox) {
229267
return refused(
230268
ErrorCode::ConfigDiffers,
231269
"Sandbox launch configuration differs; end the session, then explicit recovery is required",
@@ -294,7 +332,7 @@ struct Session<'a> {
294332
identity: &'a naming::AgentIdentity,
295333
scope: &'a Options,
296334
owner: &'a str,
297-
fingerprint: Fingerprint,
335+
fingerprints: Fingerprints,
298336
}
299337

300338
impl<'a> Session<'a> {
@@ -304,7 +342,7 @@ impl<'a> Session<'a> {
304342
identity: &'a naming::AgentIdentity,
305343
scope: &'a Options,
306344
owner: &'a str,
307-
fingerprint: Fingerprint,
345+
fingerprints: Fingerprints,
308346
) -> Self {
309347
Self {
310348
sandboxes: api(client.clone(), namespace),
@@ -314,7 +352,7 @@ impl<'a> Session<'a> {
314352
identity,
315353
scope,
316354
owner,
317-
fingerprint,
355+
fingerprints,
318356
}
319357
}
320358
}
@@ -337,8 +375,8 @@ pub async fn deploy(
337375
return Err("Sandbox requires pod_options.active_deadline_seconds".into());
338376
}
339377
let checkpointing = apply_session_env(scope, &mut env)?;
340-
let fingerprint = pod::intent_template(cfg, env.keys().cloned()).fingerprint();
341-
let session = Session::new(client, &cfg.namespace, identity, scope, owner, fingerprint);
378+
let fingerprints = Fingerprints::new(cfg, &env);
379+
let session = Session::new(client, &cfg.namespace, identity, scope, owner, fingerprints);
342380
let existing = session
343381
.sandboxes
344382
.get_opt(&session.name)
@@ -349,7 +387,7 @@ pub async fn deploy(
349387
None => create_new(&session, cfg, checkpointing).await?,
350388
};
351389
verify_identity(&sandbox, identity, scope, owner)?;
352-
verify_live(&sandbox, &session.fingerprint)?;
390+
verify_live(&sandbox, &session.fingerprints)?;
353391
bind(&session, cfg, sandbox, env).await
354392
}
355393

@@ -396,8 +434,10 @@ fn apply_session_env(scope: &Options, env: &mut BTreeMap<String, String>) -> Res
396434
if checkpointing {
397435
env.insert("BUZZ_ACP_CHECKPOINT_ON_STOP".into(), "true".into());
398436
}
399-
// Include the UID binding key in the fingerprint without hashing its value.
437+
// Include the binding keys in the fingerprint without hashing their
438+
// values; both are filled in from the Sandbox when the Secret is written.
400439
env.insert("BUZZ_SANDBOX_INITIAL_POD_UID".into(), String::new());
440+
env.insert(RESTORE_KEY_ENV.into(), String::new());
401441
Ok(checkpointing)
402442
}
403443

@@ -449,7 +489,7 @@ async fn create_new(
449489
session.scope,
450490
session.owner,
451491
&new,
452-
&session.fingerprint,
492+
&session.fingerprints.current,
453493
)?;
454494
match session
455495
.sandboxes
@@ -469,7 +509,39 @@ async fn create_new(
469509
}
470510
}
471511

512+
/// The reaper ends a never-started binding once its Pod is
513+
/// [`BIND_TIMEOUT_SECS`] old, by a CAS on the Sandbox that a Secret write does
514+
/// not bump. A deploy stops resuming early enough to write the Secret and see
515+
/// the container start before then.
516+
fn resume_window_closed(pod: &Pod, now: DateTime<Utc>) -> bool {
517+
let window = BIND_TIMEOUT_SECS - DEPLOY_DEADLINE_SECS as i64 - CLOCK_SKEW_SECS;
518+
pod.metadata
519+
.creation_timestamp
520+
.as_ref()
521+
.is_some_and(|created| now - created.0 >= chrono::Duration::seconds(window))
522+
}
523+
524+
fn container_started(pod: &Pod) -> bool {
525+
pod.status
526+
.as_ref()
527+
.and_then(|s| s.container_statuses.as_ref())
528+
.and_then(|containers| containers.iter().find(|c| c.name == "agent"))
529+
.is_some_and(|c| {
530+
c.restart_count > 0
531+
|| c.last_state
532+
.as_ref()
533+
.is_some_and(|s| s.terminated.is_some() || s.running.is_some())
534+
|| c.state
535+
.as_ref()
536+
.is_some_and(|s| s.terminated.is_some() || s.running.is_some())
537+
})
538+
}
539+
472540
/// Bind credentials to the Sandbox's first Pod and wait for it to start.
541+
///
542+
/// Resumable: a session bound to its first Pod whose Secret is missing and
543+
/// whose container never started is completed by whichever deploy observes
544+
/// it, with the Sandbox's own generation and an ownerRef to that Pod.
473545
async fn bind(
474546
session: &Session<'_>,
475547
cfg: &ProviderConfig,
@@ -481,8 +553,9 @@ async fn bind(
481553
let generation = annotation(&sandbox, GENERATION)
482554
.ok_or("Sandbox generation is missing")?
483555
.to_string();
484-
let deadline = tokio::time::Instant::now() + Duration::from_secs(120);
556+
let deadline = tokio::time::Instant::now() + Duration::from_secs(DEPLOY_DEADLINE_SECS);
485557
loop {
558+
let expired = tokio::time::Instant::now() >= deadline;
486559
let Some(child) = session
487560
.pods
488561
.get_opt(name)
@@ -495,7 +568,7 @@ async fn bind(
495568
"Sandbox lost its original Pod; explicit recovery is required",
496569
));
497570
}
498-
if tokio::time::Instant::now() >= deadline {
571+
if expired {
499572
return Err(
500573
"Sandbox controller has not created its Pod; inspect the Sandbox and retry"
501574
.into(),
@@ -529,62 +602,78 @@ async fn bind(
529602
sandbox
530603
.annotations_mut()
531604
.insert(INITIAL_POD.into(), uid.into());
532-
sandbox = match session
605+
match session
533606
.sandboxes
534607
.replace(name, &PostParams::default(), &sandbox)
535608
.await
536609
{
537-
Ok(bound) => bound,
610+
Ok(bound) => {
611+
sandbox = bound;
612+
}
538613
Err(e) if reason_is(&e, "Conflict") => {
539614
sandbox = refresh(session, &generation).await?;
540-
if tokio::time::Instant::now() >= deadline {
615+
if expired {
541616
return Err(
542617
"Sandbox kept changing during credential binding; retry deploy".into(),
543618
);
544619
}
545-
continue;
546620
}
547621
Err(e) => return Err(format!("bind Sandbox to its first Pod: {e}").into()),
548-
};
549-
let secret = bound_secret(identity, cfg, &generation, name, uid, env);
550-
session
551-
.secrets
552-
.create(&PostParams::default(), &secret)
553-
.await
554-
.map_err(|e| {
555-
ProviderError::refused(
556-
ErrorCode::BindingIncomplete,
557-
format!(
558-
"could not create bound Sandbox credentials ({}); explicit recovery is required",
559-
api_failure(&e)
560-
),
561-
)
562-
})?;
563-
return wait_for_start(&session.pods, &sandbox, name, uid).await;
622+
}
623+
continue;
564624
};
565625
if uid != initial_uid {
566626
return Err(ProviderError::refused(
567627
ErrorCode::PodReplaced,
568628
"Sandbox Pod was replaced; credentials remain bound to the original session, explicit recovery is required",
569629
));
570630
}
571-
let secret = session
631+
let secret_name = identity.secret_name(&generation);
632+
let secret =
633+
session.secrets.get_opt(&secret_name).await.map_err(|e| {
634+
format!("could not check Sandbox launch Secret: {}", api_failure(&e))
635+
})?;
636+
if let Some(secret) = secret {
637+
check_secret(&secret, identity, uid, &env)?;
638+
return wait_for_start(&session.pods, &sandbox, name, uid).await;
639+
}
640+
if container_started(&child) {
641+
return Err(ProviderError::refused(
642+
ErrorCode::CredentialsMissing,
643+
"Sandbox credentials are missing from a started session; explicit recovery is required",
644+
));
645+
}
646+
if expired {
647+
return Err(ProviderError::refused(
648+
ErrorCode::BindingIncomplete,
649+
"Sandbox credential binding is incomplete; retry deploy",
650+
));
651+
}
652+
if resume_window_closed(&child, Utc::now()) {
653+
return Err(ProviderError::refused(
654+
ErrorCode::BindingExpired,
655+
"Sandbox credential binding is too old to resume; explicit recovery is required once the session has ended",
656+
));
657+
}
658+
let secret = bound_secret(identity, cfg, &sandbox, &generation, name, uid, env.clone());
659+
match session
572660
.secrets
573-
.get_opt(&identity.secret_name(&generation))
661+
.create(&PostParams::default(), &secret)
574662
.await
575-
.map_err(|e| format!("could not check Sandbox launch Secret: {}", api_failure(&e)))?;
576-
let Some(secret) = secret else {
577-
if tokio::time::Instant::now() >= deadline {
578-
return Err(ProviderError::refused(
579-
ErrorCode::BindingIncomplete,
580-
"Sandbox credential binding is incomplete; explicit recovery is required",
581-
));
663+
{
664+
// Either way, the next pass re-reads and verifies what was stored.
665+
Ok(_) => {}
666+
Err(e) if reason_is(&e, "AlreadyExists") => {}
667+
Err(e) => {
668+
// The write may have succeeded despite a lost response.
669+
// Keep the Pod binding so retries verify the same session.
670+
return Err(format!(
671+
"could not create bound Sandbox credentials ({}); retry deploy",
672+
api_failure(&e)
673+
)
674+
.into());
582675
}
583-
tokio::time::sleep(Duration::from_millis(250)).await;
584-
continue;
585-
};
586-
check_secret(&secret, identity, uid, &env)?;
587-
return wait_for_start(&session.pods, &sandbox, name, uid).await;
676+
}
588677
}
589678
}
590679

@@ -596,7 +685,7 @@ async fn refresh(session: &Session<'_>, generation: &str) -> Result<DynamicObjec
596685
.await
597686
.map_err(|e| format!("refresh Sandbox binding: {e}"))?;
598687
verify_identity(&sandbox, session.identity, session.scope, session.owner)?;
599-
verify_live(&sandbox, &session.fingerprint)?;
688+
verify_live(&sandbox, &session.fingerprints)?;
600689
if annotation(&sandbox, GENERATION) != Some(generation) {
601690
return Err(ProviderError::refused(
602691
ErrorCode::SessionReplaced,
@@ -609,13 +698,22 @@ async fn refresh(session: &Session<'_>, generation: &str) -> Result<DynamicObjec
609698
fn bound_secret(
610699
identity: &naming::AgentIdentity,
611700
cfg: &ProviderConfig,
701+
sandbox: &DynamicObject,
612702
generation: &str,
613703
name: &str,
614704
uid: &str,
615705
mut env: BTreeMap<String, String>,
616706
) -> Secret {
617707
env.insert("BUZZ_MANAGED_AGENT_START_NONCE".into(), generation.into());
618708
env.insert("BUZZ_SANDBOX_INITIAL_POD_UID".into(), uid.into());
709+
// The restore key comes from the Sandbox, never from the request, so a
710+
// resumed binding keeps the recovery its Sandbox was created with.
711+
env.insert(
712+
RESTORE_KEY_ENV.into(),
713+
annotation(sandbox, RESTORE_CHECKPOINT)
714+
.unwrap_or_default()
715+
.into(),
716+
);
619717
let mut secret = pod::build_secret(identity, &cfg.namespace, generation, env);
620718
secret.labels_mut().insert(
621719
naming::LABEL_BINDING_VERSION.into(),
@@ -674,7 +772,7 @@ async fn wait_for_start(
674772
name: &str,
675773
uid: &str,
676774
) -> Result<String, ProviderError> {
677-
let deadline = tokio::time::Instant::now() + Duration::from_secs(120);
775+
let deadline = tokio::time::Instant::now() + Duration::from_secs(DEPLOY_DEADLINE_SECS);
678776
loop {
679777
let child = pods
680778
.get_opt(name)

0 commit comments

Comments
 (0)