Skip to content

Commit 6d9d5e0

Browse files
committed
fix(server): do not hold gateway shutdown for the endpoint-status retry
When a replacement supervisor session failed its endpoint-status reset, the spawned cleanup task kept the session lifetime guard through retry_endpoint_status_after_supervisor_disconnect. That retry only ends once the reset is durable, so a gateway shutdown during a database or lock outage waited the full 10 s and reported ownership cleanup as incomplete. The guard now covers only the lifecycle demotion, as in finish_supervisor_session, and a SQLite test checks that shutdown completes while the retry is still blocked. Part of NVIDIA#3528 Signed-off-by: Emilien Macchi <[email protected]>
1 parent ae62a03 commit 6d9d5e0

2 files changed

Lines changed: 64 additions & 1 deletion

File tree

‎crates/openshell-server/src/grpc/policy/endpoint_status_tests.rs‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1559,6 +1559,67 @@ async fn replacement_reset_timeout_invalidates_superseded_evidence() {
15591559
.expect("the superseded session's evidence is reset once the guard is free");
15601560
}
15611561

1562+
#[tokio::test]
1563+
async fn replacement_reset_retry_does_not_hold_gateway_shutdown() {
1564+
let sandbox_id = "endpoint-replacement-reset-retry";
1565+
let (state, _report) = sandbox_with_accepted_endpoint_result(sandbox_id, true).await;
1566+
// X(workspace) times out the reset and every retry of it, but not the
1567+
// demotion, which takes only S(global) X(sandbox).
1568+
let hold = crate::persistence::lock_order::branch(
1569+
state
1570+
.compute
1571+
.mutation_guard(MutationScope::Workspace("default")),
1572+
)
1573+
.await
1574+
.expect("hold the sandbox's workspace guard");
1575+
state
1576+
.compute
1577+
.set_mutation_lock_timeout_for_tests(std::time::Duration::from_millis(50));
1578+
1579+
let hello = SupervisorMessage {
1580+
payload: Some(supervisor_message::Payload::Hello(SupervisorHello {
1581+
sandbox_id: sandbox_id.to_string(),
1582+
instance_id: "instance-b".to_string(),
1583+
connection_epoch: 1,
1584+
supports_provider_readiness: false,
1585+
redirected: false,
1586+
supports_session_redirect: false,
1587+
})),
1588+
};
1589+
let error = gateway_client(Arc::clone(&state))
1590+
.await
1591+
.connect_supervisor(tokio_stream::iter([hello]))
1592+
.await
1593+
.expect_err("the replacement's endpoint reset times out on the held guard");
1594+
assert_eq!(error.code(), Code::Unavailable);
1595+
tokio::time::timeout(std::time::Duration::from_secs(5), async {
1596+
loop {
1597+
let status = stored_sandbox(&state, sandbox_id)
1598+
.await
1599+
.status
1600+
.expect("sandbox status");
1601+
if status.phase == SandboxPhase::Provisioning as i32 {
1602+
return;
1603+
}
1604+
tokio::time::sleep(std::time::Duration::from_millis(20)).await;
1605+
}
1606+
})
1607+
.await
1608+
.expect("the failed replacement demotes the sandbox");
1609+
1610+
// The retry is still timing out on the held guard. Shutdown waits for
1611+
// the demotion only.
1612+
let shutdown = state
1613+
.supervisor_sessions
1614+
.shutdown(std::time::Duration::from_millis(500))
1615+
.await;
1616+
assert!(
1617+
shutdown.is_ok(),
1618+
"the endpoint-status retry must not hold gateway shutdown: {shutdown:?}"
1619+
);
1620+
drop(hold);
1621+
}
1622+
15621623
#[tokio::test]
15631624
async fn report_endpoint_status_is_session_bound_and_retry_idempotent() {
15641625
let state = test_server_state().await;

‎crates/openshell-server/src/supervisor_session.rs‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2243,7 +2243,6 @@ async fn establish_supervisor_session(
22432243
let state = Arc::clone(&state);
22442244
let sandbox_id = sandbox_id.clone();
22452245
tokio::spawn(async move {
2246-
let _session_lifetime = session_lifetime;
22472246
// Lifecycle demotion takes an unbounded local lock. Keep it
22482247
// off this failure path so a timed-out reset returns promptly.
22492248
if let Err(err) = state
@@ -2253,6 +2252,9 @@ async fn establish_supervisor_session(
22532252
{
22542253
warn!(sandbox_id, error = %err, "supervisor session: failed to mark sandbox disconnected after endpoint status initialization failure");
22552254
}
2255+
// Shutdown waits for the demotion, like other session cleanup,
2256+
// but not for the retry, which runs until the reset is durable.
2257+
drop(session_lifetime);
22562258
crate::grpc::policy::retry_endpoint_status_after_supervisor_disconnect(
22572259
state, sandbox_id,
22582260
)

0 commit comments

Comments
 (0)