@@ -3249,6 +3249,13 @@ impl ComputeRuntime {
32493249 . await
32503250 . map_err ( |e| e. to_string ( ) ) ?;
32513251 if let Some ( sandbox) = sandbox. as_ref ( ) {
3252+ // The watcher told us this sandbox's compute resource is gone, so
3253+ // no request-side DeleteSandbox call is coming — release
3254+ // driver-owned resources in the background. Watch events are
3255+ // processed sequentially, so this must not block on the driver
3256+ // call itself, only on the (instant, non-blocking) decision to
3257+ // make it.
3258+ self . spawn_driver_sandbox_cleanup ( sandbox. object_id ( ) , sandbox. object_name ( ) ) ;
32523259 self . cleanup_sandbox_owned_records ( sandbox) . await ?;
32533260 }
32543261
@@ -3300,6 +3307,97 @@ impl ComputeRuntime {
33003307 Ok ( ( ) )
33013308 }
33023309
3310+ /// Best-effort driver-side cleanup for a sandbox discovered gone
3311+ /// out-of-band during the periodic prune sweep, rather than through an
3312+ /// explicit `DeleteSandbox` request.
3313+ ///
3314+ /// Skips the driver call entirely if a request-side lifecycle operation
3315+ /// (e.g. an in-flight explicit delete) already holds this sandbox's
3316+ /// lifecycle gate: that operation already owns driver-side cleanup for
3317+ /// it, and calling `DeleteSandbox` again here would race its own
3318+ /// in-flight call. Awaited inline: the prune sweep already makes a
3319+ /// blocking `GetSandbox` call per sandbox as part of its normal
3320+ /// operation, unlike the watch loop (see `spawn_driver_sandbox_cleanup`).
3321+ async fn cleanup_driver_sandbox_resources ( & self , sandbox_id : & str , sandbox_name : & str ) {
3322+ let gate = self . lifecycle_gates . gate_for ( sandbox_id) ;
3323+ let Ok ( _guard) = gate. try_lock_owned ( ) else {
3324+ debug ! (
3325+ sandbox_id,
3326+ sandbox_name,
3327+ "Skipping driver cleanup while a lifecycle operation is already in flight for this sandbox"
3328+ ) ;
3329+ return ;
3330+ } ;
3331+
3332+ self . call_driver_delete_sandbox ( sandbox_id, sandbox_name)
3333+ . await ;
3334+ }
3335+
3336+ /// Same lifecycle-gate-guarded driver cleanup as
3337+ /// `cleanup_driver_sandbox_resources`, but for the watch path: the
3338+ /// actual `DeleteSandbox` call is deferred to a background task rather
3339+ /// than awaited inline. Watch events are processed sequentially, so a
3340+ /// slow or stuck driver call here must never delay notifying
3341+ /// subscribers of other sandboxes (see `LifecycleGateRegistry`).
3342+ ///
3343+ /// The gate itself is still checked — and, on success, held for the
3344+ /// background call's duration — synchronously before returning, so this
3345+ /// cannot race a concurrent request-side operation for the same
3346+ /// sandbox; only the potentially-slow RPC is backgrounded.
3347+ fn spawn_driver_sandbox_cleanup ( & self , sandbox_id : & str , sandbox_name : & str ) {
3348+ let gate = self . lifecycle_gates . gate_for ( sandbox_id) ;
3349+ let Ok ( guard) = gate. try_lock_owned ( ) else {
3350+ debug ! (
3351+ sandbox_id,
3352+ sandbox_name,
3353+ "Skipping driver cleanup while a lifecycle operation is already in flight for this sandbox"
3354+ ) ;
3355+ return ;
3356+ } ;
3357+
3358+ let runtime = self . clone ( ) ;
3359+ let sandbox_id = sandbox_id. to_string ( ) ;
3360+ let sandbox_name = sandbox_name. to_string ( ) ;
3361+ tokio:: spawn ( async move {
3362+ let _guard = guard;
3363+ runtime
3364+ . call_driver_delete_sandbox ( & sandbox_id, & sandbox_name)
3365+ . await ;
3366+ } ) ;
3367+ }
3368+
3369+ /// `DeleteSandbox` is idempotent: drivers must reclaim owned
3370+ /// secrets/volumes/etc. even when the underlying compute resource is
3371+ /// already gone. Failures here are logged, not propagated — callers
3372+ /// already consider this sandbox gone, so a driver hiccup must not
3373+ /// block store cleanup.
3374+ async fn call_driver_delete_sandbox ( & self , sandbox_id : & str , sandbox_name : & str ) {
3375+ let result = self
3376+ . driver
3377+ . call ( "driver.delete_sandbox" , Some ( sandbox_id) , |driver| {
3378+ let sandbox_id = sandbox_id. to_string ( ) ;
3379+ let sandbox_name = sandbox_name. to_string ( ) ;
3380+ async move {
3381+ driver
3382+ . delete_sandbox ( Request :: new ( DeleteSandboxRequest {
3383+ sandbox_id,
3384+ sandbox_name,
3385+ } ) )
3386+ . await
3387+ }
3388+ } )
3389+ . await ;
3390+
3391+ if let Err ( status) = result {
3392+ warn ! (
3393+ sandbox_id,
3394+ sandbox_name,
3395+ error = %status,
3396+ "Failed to release driver-owned resources while cleaning up a sandbox discovered gone out-of-band"
3397+ ) ;
3398+ }
3399+ }
3400+
33033401 async fn cleanup_sandbox_ssh_sessions (
33043402 & self ,
33053403 sandbox_id : & str ,
@@ -3553,6 +3651,11 @@ impl ComputeRuntime {
35533651 age_secs = age_ms / 1000 ,
35543652 "Removing sandbox from store after it disappeared from the compute driver snapshot"
35553653 ) ;
3654+ // The driver's own snapshot never reported this sandbox, so no
3655+ // request-side DeleteSandbox call is coming for it either — release
3656+ // driver-owned resources here, before the store record disappears.
3657+ self . cleanup_driver_sandbox_resources ( & sandbox_id, & sandbox_name)
3658+ . await ;
35563659 self . apply_deleted_if_version_locked ( & sandbox, expected_resource_version)
35573660 . await
35583661 }
@@ -4984,6 +5087,7 @@ mod tests {
49845087 delete_release : Semaphore ,
49855088 delete_blocked : AtomicBool ,
49865089 delete_calls : AtomicUsize ,
5090+ delete_requests : TestMutex < Vec < ( String , String ) > > ,
49875091 delete_outcome : TestMutex < ControlledDeleteOutcome > ,
49885092 stop_started : Notify ,
49895093 stop_finished : Notify ,
@@ -5016,6 +5120,7 @@ mod tests {
50165120 delete_release : Semaphore :: new ( 0 ) ,
50175121 delete_blocked : AtomicBool :: new ( false ) ,
50185122 delete_calls : AtomicUsize :: new ( 0 ) ,
5123+ delete_requests : TestMutex :: new ( Vec :: new ( ) ) ,
50195124 delete_outcome : TestMutex :: new ( ControlledDeleteOutcome :: Ok ( true ) ) ,
50205125 stop_started : Notify :: new ( ) ,
50215126 stop_finished : Notify :: new ( ) ,
@@ -5099,6 +5204,13 @@ mod tests {
50995204 self . delete_calls . load ( Ordering :: SeqCst )
51005205 }
51015206
5207+ fn delete_requests ( & self ) -> Vec < ( String , String ) > {
5208+ self . delete_requests
5209+ . lock ( )
5210+ . expect ( "delete requests lock poisoned" )
5211+ . clone ( )
5212+ }
5213+
51025214 fn stop_calls ( & self ) -> usize {
51035215 self . stop_calls . load ( Ordering :: SeqCst )
51045216 }
@@ -5286,8 +5398,13 @@ mod tests {
52865398
52875399 async fn delete_sandbox (
52885400 & self ,
5289- _request : Request < DeleteSandboxRequest > ,
5401+ request : Request < DeleteSandboxRequest > ,
52905402 ) -> Result < tonic:: Response < DeleteSandboxResponse > , Status > {
5403+ let request = request. into_inner ( ) ;
5404+ self . delete_requests
5405+ . lock ( )
5406+ . expect ( "delete requests lock poisoned" )
5407+ . push ( ( request. sandbox_id , request. sandbox_name ) ) ;
52915408 self . delete_calls . fetch_add ( 1 , Ordering :: SeqCst ) ;
52925409 self . delete_started . notify_one ( ) ;
52935410 if self . delete_blocked . load ( Ordering :: SeqCst ) {
@@ -7974,6 +8091,107 @@ mod tests {
79748091 ) ) ;
79758092 }
79768093
8094+ #[ tokio:: test]
8095+ async fn apply_deleted_releases_driver_resources_for_out_of_band_removal ( ) {
8096+ // Regression test for #2352: a container removed without going
8097+ // through OpenShell's own DeleteSandbox (e.g. `podman rm -f`) must
8098+ // still release driver-owned secrets/volumes, not just the store
8099+ // record. This never calls `runtime.delete_sandbox`, matching the
8100+ // out-of-band removal the issue reports.
8101+ let driver = ControlledDriver :: new ( ) ;
8102+ let runtime = test_runtime ( driver. clone ( ) ) . await ;
8103+ let sandbox = sandbox_record ( "sb-1" , "sandbox-a" , SandboxPhase :: Ready ) ;
8104+ runtime. store . put_message ( & sandbox) . await . unwrap ( ) ;
8105+ let session = seed_sandbox_owned_records ( & runtime, & sandbox) . await ;
8106+
8107+ assert_eq ! ( driver. delete_calls( ) , 0 ) ;
8108+
8109+ runtime. apply_deleted ( "sb-1" ) . await . unwrap ( ) ;
8110+
8111+ // The driver call is backgrounded (see `spawn_driver_sandbox_cleanup`)
8112+ // so it can't block the watch loop; wait for it to actually land
8113+ // before asserting on it.
8114+ tokio:: time:: timeout ( Duration :: from_secs ( 1 ) , driver. delete_started . notified ( ) )
8115+ . await
8116+ . expect ( "background driver cleanup did not run" ) ;
8117+ assert_eq ! (
8118+ driver. delete_requests( ) ,
8119+ vec![ ( "sb-1" . to_string( ) , "sandbox-a" . to_string( ) ) ]
8120+ ) ;
8121+ assert_sandbox_owned_records ( & runtime, & sandbox, & session, false ) . await ;
8122+ assert ! (
8123+ runtime
8124+ . store
8125+ . get_message:: <Sandbox >( "sb-1" )
8126+ . await
8127+ . unwrap( )
8128+ . is_none( )
8129+ ) ;
8130+ }
8131+
8132+ #[ tokio:: test]
8133+ async fn apply_deleted_removes_store_record_even_when_driver_cleanup_fails ( ) {
8134+ // The driver has already told us (via the watch/prune path that led
8135+ // here) that this sandbox is gone. A failure releasing its
8136+ // secrets/volumes must be logged, not block store cleanup — retrying
8137+ // forever on a driver hiccup would leave the store permanently out
8138+ // of sync with reality.
8139+ let driver = ControlledDriver :: new ( ) ;
8140+ driver. set_delete_outcome ( ControlledDeleteOutcome :: Error ( "podman unreachable" ) ) ;
8141+ let runtime = test_runtime ( driver. clone ( ) ) . await ;
8142+ let sandbox = sandbox_record ( "sb-1" , "sandbox-a" , SandboxPhase :: Ready ) ;
8143+ runtime. store . put_message ( & sandbox) . await . unwrap ( ) ;
8144+
8145+ runtime. apply_deleted ( "sb-1" ) . await . unwrap ( ) ;
8146+
8147+ // Store cleanup happens synchronously in `apply_deleted_locked`, so
8148+ // this is already true even though the driver call is backgrounded.
8149+ assert ! (
8150+ runtime
8151+ . store
8152+ . get_message:: <Sandbox >( "sb-1" )
8153+ . await
8154+ . unwrap( )
8155+ . is_none( )
8156+ ) ;
8157+
8158+ tokio:: time:: timeout ( Duration :: from_secs ( 1 ) , driver. delete_started . notified ( ) )
8159+ . await
8160+ . expect ( "background driver cleanup did not run" ) ;
8161+ assert_eq ! ( driver. delete_calls( ) , 1 ) ;
8162+ }
8163+
8164+ #[ tokio:: test]
8165+ async fn prune_missing_sandbox_releases_driver_resources ( ) {
8166+ // Regression test for #2352's second reproduction: a sandbox whose
8167+ // container never survived to exist at all (e.g. the gateway was
8168+ // killed mid-create) is discovered missing by the periodic sweep,
8169+ // not the watcher. That path must also release driver resources.
8170+ let driver = ControlledDriver :: new ( ) ;
8171+ driver. set_get_outcome ( ControlledGetOutcome :: Missing ) ;
8172+ let runtime = test_runtime ( driver. clone ( ) ) . await ;
8173+ let sandbox = sandbox_record ( "sb-1" , "sandbox-a" , SandboxPhase :: Provisioning ) ;
8174+ runtime. store . put_message ( & sandbox) . await . unwrap ( ) ;
8175+
8176+ runtime
8177+ . reconcile_store_with_backend ( Duration :: ZERO )
8178+ . await
8179+ . unwrap ( ) ;
8180+
8181+ assert_eq ! (
8182+ driver. delete_requests( ) ,
8183+ vec![ ( "sb-1" . to_string( ) , "sandbox-a" . to_string( ) ) ]
8184+ ) ;
8185+ assert ! (
8186+ runtime
8187+ . store
8188+ . get_message:: <Sandbox >( "sb-1" )
8189+ . await
8190+ . unwrap( )
8191+ . is_none( )
8192+ ) ;
8193+ }
8194+
79778195 #[ tokio:: test]
79788196 async fn accepted_delete_cleans_local_state_after_another_replica_removes_row ( ) {
79798197 let driver = ControlledDriver :: new ( ) ;
0 commit comments