11//! Checkpoint the workspace after all adapter processes have stopped.
2+ use std:: path:: Path ;
23use std:: time:: Duration ;
34
45use serde:: Deserialize ;
@@ -82,21 +83,62 @@ fn receipt(bytes: &[u8]) -> Result<serde_json::Value, CheckpointError> {
8283 "repositories" : saved. repositories, "bytes" : saved. bytes} ) )
8384}
8485
86+ /// Kubernetes container termination-message path. The kubelet mounts it into
87+ /// every container, and the in-cluster reaper reads the receipt back from the
88+ /// agent container's terminated status (`lifecycle::CheckpointReceipt` in
89+ /// `buzz-backend-kubernetes`).
90+ const TERMINATION_LOG : & str = "/dev/termination-log" ;
91+
92+ /// Termination-message wire format: `{"version":1,"checkpoint":"<key>"}`.
93+ /// Field order is the declaration order, so the bytes are stable.
94+ #[ derive( serde:: Serialize ) ]
95+ struct TerminationReceipt < ' a > {
96+ version : u32 ,
97+ checkpoint : & ' a str ,
98+ }
99+
85100pub ( crate ) async fn save ( observer : Option < & ObserverHandle > ) -> anyhow:: Result < ( ) > {
86101 let mut command = Command :: new ( "buzz-agent-checkpoint" ) ;
87102 command. arg ( "save" ) ;
88- save_with_command ( observer, command, Duration :: from_secs ( 120 ) )
89- . await
90- . map_err ( |error| {
91- anyhow:: Error :: new ( error)
92- . context ( "checkpoint failed; retain the Sandbox workspace for explicit recovery" )
93- } )
103+ save_with_command (
104+ observer,
105+ command,
106+ Duration :: from_secs ( 120 ) ,
107+ Path :: new ( TERMINATION_LOG ) ,
108+ )
109+ . await
110+ . map_err ( |error| {
111+ anyhow:: Error :: new ( error)
112+ . context ( "checkpoint failed; retain the Sandbox workspace for explicit recovery" )
113+ } )
114+ }
115+
116+ /// Record the saved checkpoint key as the container's termination message.
117+ /// The file is never created: it exists only where the kubelet mounted it, so
118+ /// `NotFound` is a local run and succeeds without writing. Any other open or
119+ /// write failure is returned.
120+ fn write_termination_receipt ( path : & Path , key : & str ) -> std:: io:: Result < ( ) > {
121+ let receipt = serde_json:: to_string ( & TerminationReceipt {
122+ version : 1 ,
123+ checkpoint : key,
124+ } ) ?;
125+ let mut file = match std:: fs:: OpenOptions :: new ( )
126+ . write ( true )
127+ . truncate ( true )
128+ . open ( path)
129+ {
130+ Ok ( file) => file,
131+ Err ( error) if error. kind ( ) == std:: io:: ErrorKind :: NotFound => return Ok ( ( ) ) ,
132+ Err ( error) => return Err ( error) ,
133+ } ;
134+ std:: io:: Write :: write_all ( & mut file, receipt. as_bytes ( ) )
94135}
95136
96137async fn save_with_command (
97138 observer : Option < & ObserverHandle > ,
98139 mut command : Command ,
99140 timeout : Duration ,
141+ termination_log : & Path ,
100142) -> Result < ( ) , CheckpointError > {
101143 let emit = |kind, details| {
102144 if let Some ( observer) = observer {
@@ -112,6 +154,17 @@ async fn save_with_command(
112154 command. process_group ( 0 ) ;
113155 match run_checkpoint ( command, timeout) . await {
114156 Ok ( details) => {
157+ if let Some ( key) = details[ "key" ] . as_str ( ) {
158+ // The checkpoint itself is saved and the stop proceeds; the
159+ // reaper finds no receipt and treats the checkpoint as missing.
160+ if let Err ( error) = write_termination_receipt ( termination_log, key) {
161+ tracing:: warn!( %error, "could not write the checkpoint termination receipt" ) ;
162+ emit (
163+ "checkpoint_receipt_failed" ,
164+ serde_json:: json!( { "error_kind" : error. kind( ) . to_string( ) } ) ,
165+ ) ;
166+ }
167+ }
115168 emit ( "checkpoint_saved" , details) ;
116169 Ok ( ( ) )
117170 }
@@ -170,6 +223,9 @@ async fn run_checkpoint(
170223mod tests {
171224 use super :: * ;
172225
226+ #[ cfg( unix) ]
227+ const NO_LOG_PATH : & str = "/nonexistent/termination-log" ;
228+
173229 fn valid_receipt ( ) -> serde_json:: Value {
174230 serde_json:: json!( { "version" : 1 , "key" : "developer/session/id.tar.gz" ,
175231 "sha256" : "a" . repeat( 64 ) , "repositories" : 2 , "bytes" : 100 } )
@@ -181,9 +237,14 @@ mod tests {
181237 let mut command = Command :: new ( "sh" ) ;
182238 command. args ( [ "-c" , "head -c 2049 /dev/zero; sleep 60" ] ) ;
183239 let started = std:: time:: Instant :: now ( ) ;
184- let error = save_with_command ( None , command, Duration :: from_secs ( 10 ) )
185- . await
186- . unwrap_err ( ) ;
240+ let error = save_with_command (
241+ None ,
242+ command,
243+ Duration :: from_secs ( 10 ) ,
244+ Path :: new ( NO_LOG_PATH ) ,
245+ )
246+ . await
247+ . unwrap_err ( ) ;
187248 assert ! ( matches!( error, CheckpointError :: ReceiptOversized ) ) ;
188249 assert ! ( started. elapsed( ) < Duration :: from_secs( 2 ) ) ;
189250 }
@@ -237,11 +298,14 @@ mod tests {
237298 let observer = ObserverHandle :: in_process ( ) ;
238299 let mut command = Command :: new ( "sh" ) ;
239300 command. args ( [ "-c" , script] ) ;
240- assert ! (
241- save_with_command( Some ( & observer) , command, Duration :: from_millis( 100 ) )
242- . await
243- . is_err( )
244- ) ;
301+ assert ! ( save_with_command(
302+ Some ( & observer) ,
303+ command,
304+ Duration :: from_millis( 100 ) ,
305+ Path :: new( NO_LOG_PATH )
306+ )
307+ . await
308+ . is_err( ) ) ;
245309 let events = observer. snapshot ( ) ;
246310 assert_eq ! (
247311 events
@@ -255,7 +319,13 @@ mod tests {
255319 let observer = ObserverHandle :: in_process ( ) ;
256320 let command = Command :: new ( "/nonexistent/checkpoint-helper" ) ;
257321 assert ! ( matches!(
258- save_with_command( Some ( & observer) , command, Duration :: from_secs( 1 ) ) . await ,
322+ save_with_command(
323+ Some ( & observer) ,
324+ command,
325+ Duration :: from_secs( 1 ) ,
326+ Path :: new( NO_LOG_PATH )
327+ )
328+ . await ,
259329 Err ( CheckpointError :: Spawn ( _) )
260330 ) ) ;
261331 let failed = observer. snapshot ( ) . last ( ) . unwrap ( ) . clone ( ) ;
@@ -278,7 +348,13 @@ mod tests {
278348 pid_file. to_str ( ) . unwrap ( ) ,
279349 ] ) ;
280350 assert ! ( matches!(
281- save_with_command( None , command, Duration :: from_secs( 5 ) ) . await ,
351+ save_with_command(
352+ None ,
353+ command,
354+ Duration :: from_secs( 5 ) ,
355+ Path :: new( NO_LOG_PATH )
356+ )
357+ . await ,
282358 Err ( CheckpointError :: ExitStatus ( _) )
283359 ) ) ;
284360 let pid: i32 = std:: fs:: read_to_string ( & pid_file)
@@ -302,16 +378,85 @@ mod tests {
302378 #[ tokio:: test]
303379 async fn helper_success_emits_verified_receipt_after_exit ( ) {
304380 let observer = ObserverHandle :: in_process ( ) ;
305- let valid = valid_receipt ( ) . to_string ( ) ;
381+ let valid = serde_json:: json!( { "version" : 1 , "key" : "developer/session/id.tar.gz" ,
382+ "sha256" : "a" . repeat( 64 ) , "repositories" : 2 , "bytes" : 100 } )
383+ . to_string ( ) ;
306384 let mut command = Command :: new ( "sh" ) ;
307385 command. args ( [ "-c" , "printf '%s' \" $1\" " , "checkpoint-test" , & valid] ) ;
308- save_with_command ( Some ( & observer) , command, Duration :: from_secs ( 1 ) )
386+ let dir = tempfile:: tempdir ( ) . unwrap ( ) ;
387+ let log = dir. path ( ) . join ( "termination-log" ) ;
388+ std:: fs:: write ( & log, "stale" ) . unwrap ( ) ;
389+ save_with_command ( Some ( & observer) , command, Duration :: from_secs ( 1 ) , & log)
309390 . await
310391 . unwrap ( ) ;
311392 let events = observer. snapshot ( ) ;
312393 assert_eq ! ( events. len( ) , 2 ) ;
313394 assert_eq ! ( events[ 0 ] . kind, "checkpoint_started" ) ;
314395 assert_eq ! ( events[ 1 ] . kind, "checkpoint_saved" ) ;
315396 assert_eq ! ( events[ 1 ] . payload[ "key" ] , "developer/session/id.tar.gz" ) ;
397+ // Exact bytes: the reaper's `CheckpointReceipt` tests parse this literal.
398+ assert_eq ! (
399+ std:: fs:: read_to_string( & log) . unwrap( ) ,
400+ r#"{"version":1,"checkpoint":"developer/session/id.tar.gz"}"#
401+ ) ;
402+ }
403+
404+ /// A mounted log that cannot be written is reported, not swallowed; the
405+ /// checkpoint itself still counts as saved.
406+ #[ cfg( unix) ]
407+ #[ tokio:: test]
408+ async fn unwritable_receipt_is_reported_without_failing_the_checkpoint ( ) {
409+ let observer = ObserverHandle :: in_process ( ) ;
410+ let valid = serde_json:: json!( { "version" : 1 , "key" : "developer/session/id.tar.gz" ,
411+ "sha256" : "a" . repeat( 64 ) , "repositories" : 2 , "bytes" : 100 } )
412+ . to_string ( ) ;
413+ let mut command = Command :: new ( "sh" ) ;
414+ command. args ( [ "-c" , "printf '%s' \" $1\" " , "checkpoint-test" , & valid] ) ;
415+ // A directory exists but cannot be opened for writing.
416+ let dir = tempfile:: tempdir ( ) . unwrap ( ) ;
417+ save_with_command ( Some ( & observer) , command, Duration :: from_secs ( 1 ) , dir. path ( ) )
418+ . await
419+ . unwrap ( ) ;
420+ let kinds: Vec < _ > = observer
421+ . snapshot ( )
422+ . into_iter ( )
423+ . map ( |event| event. kind )
424+ . collect ( ) ;
425+ assert_eq ! (
426+ kinds,
427+ [
428+ "checkpoint_started" ,
429+ "checkpoint_receipt_failed" ,
430+ "checkpoint_saved"
431+ ]
432+ ) ;
433+ }
434+
435+ /// Local runs have no kubelet-mounted file; the receipt must never create
436+ /// one, and a failed checkpoint must never write one.
437+ #[ cfg( unix) ]
438+ #[ tokio:: test]
439+ async fn receipt_is_never_created_and_never_written_on_failure ( ) {
440+ let dir = tempfile:: tempdir ( ) . unwrap ( ) ;
441+ let absent = dir. path ( ) . join ( "termination-log" ) ;
442+ let valid = serde_json:: json!( { "version" : 1 , "key" : "developer/session/id.tar.gz" ,
443+ "sha256" : "a" . repeat( 64 ) , "repositories" : 2 , "bytes" : 100 } )
444+ . to_string ( ) ;
445+ let mut command = Command :: new ( "sh" ) ;
446+ command. args ( [ "-c" , "printf '%s' \" $1\" " , "checkpoint-test" , & valid] ) ;
447+ save_with_command ( None , command, Duration :: from_secs ( 1 ) , & absent)
448+ . await
449+ . unwrap ( ) ;
450+ assert ! ( !absent. exists( ) ) ;
451+ let present = dir. path ( ) . join ( "present" ) ;
452+ std:: fs:: write ( & present, "" ) . unwrap ( ) ;
453+ let mut failing = Command :: new ( "sh" ) ;
454+ failing. args ( [ "-c" , "exit 1" ] ) ;
455+ assert ! (
456+ save_with_command( None , failing, Duration :: from_secs( 1 ) , & present)
457+ . await
458+ . is_err( )
459+ ) ;
460+ assert_eq ! ( std:: fs:: read_to_string( & present) . unwrap( ) , "" ) ;
316461 }
317462}
0 commit comments