Skip to content

Commit 8a78548

Browse files
committed
feat(agents): add the shared Sandbox lifecycle contract
Add a small library target to buzz-backend-kubernetes that holds the Agent Sandbox session lifecycle: the annotation keys and values, a pure classifier that maps an observed Sandbox, Pod, and apiserver time to one lifecycle state, and the checkpoint receipt that binds a tombstone to the session's own checkpoint prefix. The provider binary and the in-cluster reaper share it, so naming.rs now re-exports the management marker from the library instead of spelling it twice. The harness writes the saved checkpoint key to the kubelet-mounted termination log as {"version":1,"checkpoint":"<key>"}. It never creates the file, so local runs are unaffected. Signed-off-by: Tee8z <[email protected]>
1 parent 4bf9e54 commit 8a78548

7 files changed

Lines changed: 1167 additions & 31 deletions

File tree

‎crates/buzz-acp/src/activity.rs‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ impl ActivityRecorder {
3232
pub(crate) fn record(&mut self, event: &ObserverEvent) -> Option<Value> {
3333
let mut details = json!({});
3434
let kind = match event.kind.as_str() {
35-
"checkpoint_started" => event.kind.as_str(),
35+
"checkpoint_started" | "checkpoint_receipt_failed" => event.kind.as_str(),
3636
"checkpoint_failed" => {
3737
details["reason"] = event.payload["reason"].as_str().into();
3838
details["exit_code"] = event.payload["exit_code"].as_i64().into();
@@ -310,6 +310,15 @@ mod tests {
310310
assert!(recorder.record(&e).is_none());
311311
}
312312

313+
#[test]
314+
fn checkpoint_receipt_failures_are_platform_records() {
315+
let mut recorder = ActivityRecorder::new();
316+
let mut e = event(json!({"error_kind": "permission denied"}));
317+
e.kind = "checkpoint_receipt_failed".into();
318+
let record = recorder.record(&e).unwrap();
319+
assert_eq!(record["event"], "checkpoint_receipt_failed");
320+
}
321+
313322
#[test]
314323
fn active_tool_tracking_is_bounded() {
315324
let mut recorder = ActivityRecorder::new();

‎crates/buzz-acp/src/checkpoint.rs‎

Lines changed: 163 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
//! Checkpoint the workspace after all adapter processes have stopped.
2+
use std::path::Path;
23
use std::time::Duration;
34

45
use serde::Deserialize;
@@ -82,21 +83,62 @@ fn receipt(bytes: &[u8]) -> Result<serde_json::Value, CheckpointError> {
8283
"repositories":saved.repositories,"bytes":saved.bytes}))
8384
}
8485

86+
/// Kubernetes container termination-message path. The kubelet mounts it into
87+
/// every container, and the in-cluster reaper reads the receipt back from the
88+
/// agent container's terminated status (`lifecycle::CheckpointReceipt` in
89+
/// `buzz-backend-kubernetes`).
90+
const TERMINATION_LOG: &str = "/dev/termination-log";
91+
92+
/// Termination-message wire format: `{"version":1,"checkpoint":"<key>"}`.
93+
/// Field order is the declaration order, so the bytes are stable.
94+
#[derive(serde::Serialize)]
95+
struct TerminationReceipt<'a> {
96+
version: u32,
97+
checkpoint: &'a str,
98+
}
99+
85100
pub(crate) async fn save(observer: Option<&ObserverHandle>) -> anyhow::Result<()> {
86101
let mut command = Command::new("buzz-agent-checkpoint");
87102
command.arg("save");
88-
save_with_command(observer, command, Duration::from_secs(120))
89-
.await
90-
.map_err(|error| {
91-
anyhow::Error::new(error)
92-
.context("checkpoint failed; retain the Sandbox workspace for explicit recovery")
93-
})
103+
save_with_command(
104+
observer,
105+
command,
106+
Duration::from_secs(120),
107+
Path::new(TERMINATION_LOG),
108+
)
109+
.await
110+
.map_err(|error| {
111+
anyhow::Error::new(error)
112+
.context("checkpoint failed; retain the Sandbox workspace for explicit recovery")
113+
})
114+
}
115+
116+
/// Record the saved checkpoint key as the container's termination message.
117+
/// The file is never created: it exists only where the kubelet mounted it, so
118+
/// `NotFound` is a local run and succeeds without writing. Any other open or
119+
/// write failure is returned.
120+
fn write_termination_receipt(path: &Path, key: &str) -> std::io::Result<()> {
121+
let receipt = serde_json::to_string(&TerminationReceipt {
122+
version: 1,
123+
checkpoint: key,
124+
})?;
125+
let mut file = match std::fs::OpenOptions::new()
126+
.write(true)
127+
.truncate(true)
128+
.open(path)
129+
{
130+
Ok(file) => file,
131+
Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(()),
132+
Err(error) => return Err(error),
133+
};
134+
std::io::Write::write_all(&mut file, receipt.as_bytes())
94135
}
95136

96137
async fn save_with_command(
97138
observer: Option<&ObserverHandle>,
98139
mut command: Command,
99140
timeout: Duration,
141+
termination_log: &Path,
100142
) -> Result<(), CheckpointError> {
101143
let emit = |kind, details| {
102144
if let Some(observer) = observer {
@@ -112,6 +154,17 @@ async fn save_with_command(
112154
command.process_group(0);
113155
match run_checkpoint(command, timeout).await {
114156
Ok(details) => {
157+
if let Some(key) = details["key"].as_str() {
158+
// The checkpoint itself is saved and the stop proceeds; the
159+
// reaper finds no receipt and treats the checkpoint as missing.
160+
if let Err(error) = write_termination_receipt(termination_log, key) {
161+
tracing::warn!(%error, "could not write the checkpoint termination receipt");
162+
emit(
163+
"checkpoint_receipt_failed",
164+
serde_json::json!({"error_kind": error.kind().to_string()}),
165+
);
166+
}
167+
}
115168
emit("checkpoint_saved", details);
116169
Ok(())
117170
}
@@ -170,6 +223,9 @@ async fn run_checkpoint(
170223
mod tests {
171224
use super::*;
172225

226+
#[cfg(unix)]
227+
const NO_LOG_PATH: &str = "/nonexistent/termination-log";
228+
173229
fn valid_receipt() -> serde_json::Value {
174230
serde_json::json!({"version":1,"key":"developer/session/id.tar.gz",
175231
"sha256":"a".repeat(64),"repositories":2,"bytes":100})
@@ -181,9 +237,14 @@ mod tests {
181237
let mut command = Command::new("sh");
182238
command.args(["-c", "head -c 2049 /dev/zero; sleep 60"]);
183239
let started = std::time::Instant::now();
184-
let error = save_with_command(None, command, Duration::from_secs(10))
185-
.await
186-
.unwrap_err();
240+
let error = save_with_command(
241+
None,
242+
command,
243+
Duration::from_secs(10),
244+
Path::new(NO_LOG_PATH),
245+
)
246+
.await
247+
.unwrap_err();
187248
assert!(matches!(error, CheckpointError::ReceiptOversized));
188249
assert!(started.elapsed() < Duration::from_secs(2));
189250
}
@@ -237,11 +298,14 @@ mod tests {
237298
let observer = ObserverHandle::in_process();
238299
let mut command = Command::new("sh");
239300
command.args(["-c", script]);
240-
assert!(
241-
save_with_command(Some(&observer), command, Duration::from_millis(100))
242-
.await
243-
.is_err()
244-
);
301+
assert!(save_with_command(
302+
Some(&observer),
303+
command,
304+
Duration::from_millis(100),
305+
Path::new(NO_LOG_PATH)
306+
)
307+
.await
308+
.is_err());
245309
let events = observer.snapshot();
246310
assert_eq!(
247311
events
@@ -255,7 +319,13 @@ mod tests {
255319
let observer = ObserverHandle::in_process();
256320
let command = Command::new("/nonexistent/checkpoint-helper");
257321
assert!(matches!(
258-
save_with_command(Some(&observer), command, Duration::from_secs(1)).await,
322+
save_with_command(
323+
Some(&observer),
324+
command,
325+
Duration::from_secs(1),
326+
Path::new(NO_LOG_PATH)
327+
)
328+
.await,
259329
Err(CheckpointError::Spawn(_))
260330
));
261331
let failed = observer.snapshot().last().unwrap().clone();
@@ -278,7 +348,13 @@ mod tests {
278348
pid_file.to_str().unwrap(),
279349
]);
280350
assert!(matches!(
281-
save_with_command(None, command, Duration::from_secs(5)).await,
351+
save_with_command(
352+
None,
353+
command,
354+
Duration::from_secs(5),
355+
Path::new(NO_LOG_PATH)
356+
)
357+
.await,
282358
Err(CheckpointError::ExitStatus(_))
283359
));
284360
let pid: i32 = std::fs::read_to_string(&pid_file)
@@ -302,16 +378,85 @@ mod tests {
302378
#[tokio::test]
303379
async fn helper_success_emits_verified_receipt_after_exit() {
304380
let observer = ObserverHandle::in_process();
305-
let valid = valid_receipt().to_string();
381+
let valid = serde_json::json!({"version":1,"key":"developer/session/id.tar.gz",
382+
"sha256":"a".repeat(64),"repositories":2,"bytes":100})
383+
.to_string();
306384
let mut command = Command::new("sh");
307385
command.args(["-c", "printf '%s' \"$1\"", "checkpoint-test", &valid]);
308-
save_with_command(Some(&observer), command, Duration::from_secs(1))
386+
let dir = tempfile::tempdir().unwrap();
387+
let log = dir.path().join("termination-log");
388+
std::fs::write(&log, "stale").unwrap();
389+
save_with_command(Some(&observer), command, Duration::from_secs(1), &log)
309390
.await
310391
.unwrap();
311392
let events = observer.snapshot();
312393
assert_eq!(events.len(), 2);
313394
assert_eq!(events[0].kind, "checkpoint_started");
314395
assert_eq!(events[1].kind, "checkpoint_saved");
315396
assert_eq!(events[1].payload["key"], "developer/session/id.tar.gz");
397+
// Exact bytes: the reaper's `CheckpointReceipt` tests parse this literal.
398+
assert_eq!(
399+
std::fs::read_to_string(&log).unwrap(),
400+
r#"{"version":1,"checkpoint":"developer/session/id.tar.gz"}"#
401+
);
402+
}
403+
404+
/// A mounted log that cannot be written is reported, not swallowed; the
405+
/// checkpoint itself still counts as saved.
406+
#[cfg(unix)]
407+
#[tokio::test]
408+
async fn unwritable_receipt_is_reported_without_failing_the_checkpoint() {
409+
let observer = ObserverHandle::in_process();
410+
let valid = serde_json::json!({"version":1,"key":"developer/session/id.tar.gz",
411+
"sha256":"a".repeat(64),"repositories":2,"bytes":100})
412+
.to_string();
413+
let mut command = Command::new("sh");
414+
command.args(["-c", "printf '%s' \"$1\"", "checkpoint-test", &valid]);
415+
// A directory exists but cannot be opened for writing.
416+
let dir = tempfile::tempdir().unwrap();
417+
save_with_command(Some(&observer), command, Duration::from_secs(1), dir.path())
418+
.await
419+
.unwrap();
420+
let kinds: Vec<_> = observer
421+
.snapshot()
422+
.into_iter()
423+
.map(|event| event.kind)
424+
.collect();
425+
assert_eq!(
426+
kinds,
427+
[
428+
"checkpoint_started",
429+
"checkpoint_receipt_failed",
430+
"checkpoint_saved"
431+
]
432+
);
433+
}
434+
435+
/// Local runs have no kubelet-mounted file; the receipt must never create
436+
/// one, and a failed checkpoint must never write one.
437+
#[cfg(unix)]
438+
#[tokio::test]
439+
async fn receipt_is_never_created_and_never_written_on_failure() {
440+
let dir = tempfile::tempdir().unwrap();
441+
let absent = dir.path().join("termination-log");
442+
let valid = serde_json::json!({"version":1,"key":"developer/session/id.tar.gz",
443+
"sha256":"a".repeat(64),"repositories":2,"bytes":100})
444+
.to_string();
445+
let mut command = Command::new("sh");
446+
command.args(["-c", "printf '%s' \"$1\"", "checkpoint-test", &valid]);
447+
save_with_command(None, command, Duration::from_secs(1), &absent)
448+
.await
449+
.unwrap();
450+
assert!(!absent.exists());
451+
let present = dir.path().join("present");
452+
std::fs::write(&present, "").unwrap();
453+
let mut failing = Command::new("sh");
454+
failing.args(["-c", "exit 1"]);
455+
assert!(
456+
save_with_command(None, failing, Duration::from_secs(1), &present)
457+
.await
458+
.is_err()
459+
);
460+
assert_eq!(std::fs::read_to_string(&present).unwrap(), "");
316461
}
317462
}

‎crates/buzz-backend-kubernetes/Cargo.toml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,10 @@ rust-version.workspace = true
66
license.workspace = true
77
description = "Kubernetes backend provider for Buzz remote agents (docs/remote-agents.md)"
88

9+
[lib]
10+
name = "buzz_backend_kubernetes"
11+
path = "src/lib.rs"
12+
913
[[bin]]
1014
name = "buzz-backend-kubernetes"
1115
path = "src/main.rs"
Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
//! Library surface of the Kubernetes provider: the Agent Sandbox session
2+
//! lifecycle shared with the in-cluster `buzz-sandbox-reaper`.
3+
//!
4+
//! Deliberately small. The provider binary ships inside the desktop app, so
5+
//! anything a consumer of this library needs (S3, HTTP servers, metrics) must
6+
//! live in that consumer, never here.
7+
8+
pub mod lifecycle;

0 commit comments

Comments
 (0)