@@ -3308,42 +3308,24 @@ impl ComputeRuntime {
33083308 }
33093309
33103310 /// Best-effort driver-side cleanup for a sandbox discovered gone
3311- /// out-of-band during the periodic prune sweep, rather than through an
3311+ /// out-of-band — a watch deletion event, or the periodic prune sweep
3312+ /// finding no matching driver resource — rather than through an
33123313 /// explicit `DeleteSandbox` request.
33133314 ///
33143315 /// Skips the driver call entirely if a request-side lifecycle operation
33153316 /// (e.g. an in-flight explicit delete) already holds this sandbox's
33163317 /// lifecycle gate: that operation already owns driver-side cleanup for
33173318 /// it, and calling `DeleteSandbox` again here would race its own
3318- /// in-flight call. Awaited inline: the prune sweep already makes a
3319- /// blocking `GetSandbox` call per sandbox as part of its normal
3320- /// operation, unlike the watch loop (see `spawn_driver_sandbox_cleanup`).
3321- async fn cleanup_driver_sandbox_resources ( & self , sandbox_id : & str , sandbox_name : & str ) {
3322- let gate = self . lifecycle_gates . gate_for ( sandbox_id) ;
3323- let Ok ( _guard) = gate. try_lock_owned ( ) else {
3324- debug ! (
3325- sandbox_id,
3326- sandbox_name,
3327- "Skipping driver cleanup while a lifecycle operation is already in flight for this sandbox"
3328- ) ;
3329- return ;
3330- } ;
3331-
3332- self . call_driver_delete_sandbox ( sandbox_id, sandbox_name)
3333- . await ;
3334- }
3335-
3336- /// Same lifecycle-gate-guarded driver cleanup as
3337- /// `cleanup_driver_sandbox_resources`, but for the watch path: the
3338- /// actual `DeleteSandbox` call is deferred to a background task rather
3339- /// than awaited inline. Watch events are processed sequentially, so a
3340- /// slow or stuck driver call here must never delay notifying
3341- /// subscribers of other sandboxes (see `LifecycleGateRegistry`).
3319+ /// in-flight call.
33423320 ///
3343- /// The gate itself is still checked — and, on success, held for the
3344- /// background call's duration — synchronously before returning, so this
3345- /// cannot race a concurrent request-side operation for the same
3346- /// sandbox; only the potentially-slow RPC is backgrounded.
3321+ /// The gate check is synchronous, but the actual `DeleteSandbox` RPC is
3322+ /// always deferred to a background task, never awaited inline: both
3323+ /// call sites run while holding a broader lock (the watch loop's
3324+ /// sequential event processing; the prune sweep's gateway-wide
3325+ /// `sync_lock`), and a slow or stuck driver call must never block that
3326+ /// wider scope. The gate itself is held for the background call's
3327+ /// duration, so this still can't race a concurrent request-side
3328+ /// operation — only the potentially-slow RPC is backgrounded.
33473329 fn spawn_driver_sandbox_cleanup ( & self , sandbox_id : & str , sandbox_name : & str ) {
33483330 let gate = self . lifecycle_gates . gate_for ( sandbox_id) ;
33493331 let Ok ( guard) = gate. try_lock_owned ( ) else {
@@ -3653,9 +3635,12 @@ impl ComputeRuntime {
36533635 ) ;
36543636 // The driver's own snapshot never reported this sandbox, so no
36553637 // request-side DeleteSandbox call is coming for it either — release
3656- // driver-owned resources here, before the store record disappears.
3657- self . cleanup_driver_sandbox_resources ( & sandbox_id, & sandbox_name)
3658- . await ;
3638+ // driver-owned resources in the background. This function holds
3639+ // `sync_lock` (the gateway-wide state guard) through the rest of its
3640+ // body, so the driver call must not be awaited here: doing so would
3641+ // block every other sandbox operation gateway-wide on a single,
3642+ // potentially slow or stuck driver RPC.
3643+ self . spawn_driver_sandbox_cleanup ( & sandbox_id, & sandbox_name) ;
36593644 self . apply_deleted_if_version_locked ( & sandbox, expected_resource_version)
36603645 . await
36613646 }
@@ -8178,10 +8163,48 @@ mod tests {
81788163 . await
81798164 . unwrap ( ) ;
81808165
8166+ assert ! (
8167+ runtime
8168+ . store
8169+ . get_message:: <Sandbox >( "sb-1" )
8170+ . await
8171+ . unwrap( )
8172+ . is_none( )
8173+ ) ;
8174+
8175+ // The driver call is backgrounded (see `spawn_driver_sandbox_cleanup`)
8176+ // so the prune sweep never awaits it while holding the gateway-wide
8177+ // sync_lock; wait for it to actually land before asserting on it.
8178+ tokio:: time:: timeout ( Duration :: from_secs ( 1 ) , driver. delete_started . notified ( ) )
8179+ . await
8180+ . expect ( "background driver cleanup did not run" ) ;
81818181 assert_eq ! (
81828182 driver. delete_requests( ) ,
81838183 vec![ ( "sb-1" . to_string( ) , "sandbox-a" . to_string( ) ) ]
81848184 ) ;
8185+ }
8186+
8187+ #[ tokio:: test]
8188+ async fn prune_sweep_does_not_block_on_a_stuck_driver_delete_call ( ) {
8189+ // Regression test: the prune sweep's driver cleanup must not be
8190+ // awaited while holding `sync_lock` (the gateway-wide state guard).
8191+ // Block the driver's delete call indefinitely and confirm the sweep
8192+ // itself still completes promptly and removes the store record.
8193+ let driver = ControlledDriver :: new ( ) ;
8194+ driver. block_delete ( ) ;
8195+ driver. set_get_outcome ( ControlledGetOutcome :: Missing ) ;
8196+ let runtime = test_runtime ( driver. clone ( ) ) . await ;
8197+ let sandbox = sandbox_record ( "sb-1" , "sandbox-a" , SandboxPhase :: Provisioning ) ;
8198+ runtime. store . put_message ( & sandbox) . await . unwrap ( ) ;
8199+
8200+ tokio:: time:: timeout (
8201+ Duration :: from_secs ( 1 ) ,
8202+ runtime. reconcile_store_with_backend ( Duration :: ZERO ) ,
8203+ )
8204+ . await
8205+ . expect ( "prune sweep blocked on the stuck driver delete call" )
8206+ . unwrap ( ) ;
8207+
81858208 assert ! (
81868209 runtime
81878210 . store
@@ -8190,6 +8213,9 @@ mod tests {
81908213 . unwrap( )
81918214 . is_none( )
81928215 ) ;
8216+ tokio:: time:: timeout ( Duration :: from_secs ( 1 ) , driver. delete_started . notified ( ) )
8217+ . await
8218+ . expect ( "background driver cleanup did not run" ) ;
81938219 }
81948220
81958221 #[ tokio:: test]
0 commit comments