Skip to content

Commit 7a45dec

Browse files
committed
fix(ha): wait for handoff only when a redirect went out
Address review feedback on the shutdown handoff: - Wait for a replacement session only when the stopping replica queued a SessionRedirect. Single-replica gateways, supervisors without redirect support, and full outbound channels no longer pay the grace period. - Close the session stream in both directions before cleanup so a supervisor without a redirect sees EOF and reconnects at once. - Bound the replacement wait with timeout_at and the shared session-wait backoff, so the grace is a hard ceiling. - Name the supervisor session shutdown timeout and assert at compile time that it leaves room after the grace period. - Demote the sandbox when session setup fails after the owner publish, so a failed takeover cannot leave it Ready with no session. - Move the supervisor reconnect delays into ReconnectBackoff so both loop branches reset the same way, and test reconnect sequences. Signed-off-by: Piotr Mlocek <[email protected]>
1 parent 6853205 commit 7a45dec

4 files changed

Lines changed: 384 additions & 200 deletions

File tree

‎crates/openshell-server/src/compute/mod.rs‎

Lines changed: 1 addition & 114 deletions
Original file line numberDiff line numberDiff line change
@@ -340,9 +340,6 @@ async fn wait_for_startup(
340340
/// Interval between store-vs-backend reconciliation sweeps.
341341
const RECONCILE_INTERVAL: Duration = Duration::from_mins(1);
342342

343-
/// Interval between replacement-session checks during a shutdown handoff.
344-
const SUPERVISOR_HANDOFF_POLL_INTERVAL: Duration = Duration::from_millis(100);
345-
346343
/// Restart intents live in sandbox rows. A lightweight holder-only scan makes
347344
/// cross-replica writes and lost wakeups recover without changing the general
348345
/// backend reconciliation cadence.
@@ -4763,42 +4760,7 @@ impl ComputeRuntime {
47634760
Ok(())
47644761
}
47654762

4766-
/// Wait up to `timeout` for a replacement supervisor session after this
4767-
/// replica handed the sandbox off during shutdown.
4768-
///
4769-
/// Call before [`Self::supervisor_session_disconnected`], which keeps the
4770-
/// sandbox Ready when a replacement exists. Without the wait, the stopping
4771-
/// replica demotes the sandbox before the redirected supervisor publishes
4772-
/// its new ownership, and requests fail with `sandbox is not ready` for the
4773-
/// length of the handoff. Returns whether a replacement became ready.
4774-
pub async fn wait_for_supervisor_session_handoff(
4775-
&self,
4776-
sandbox_id: &str,
4777-
timeout: Duration,
4778-
) -> bool {
4779-
let deadline = tokio::time::Instant::now() + timeout;
4780-
loop {
4781-
match self.supervisor_session_ready(sandbox_id).await {
4782-
Ok(true) => return true,
4783-
Ok(false) => {}
4784-
Err(error) => {
4785-
warn!(
4786-
sandbox_id,
4787-
error = %error,
4788-
"Failed to check for a replacement supervisor session during handoff"
4789-
);
4790-
return false;
4791-
}
4792-
}
4793-
let now = tokio::time::Instant::now();
4794-
if now >= deadline {
4795-
return false;
4796-
}
4797-
tokio::time::sleep(SUPERVISOR_HANDOFF_POLL_INTERVAL.min(deadline - now)).await;
4798-
}
4799-
}
4800-
4801-
async fn supervisor_session_ready(&self, sandbox_id: &str) -> Result<bool, String> {
4763+
pub(crate) async fn supervisor_session_ready(&self, sandbox_id: &str) -> Result<bool, String> {
48024764
if self.supervisor_sessions.has_session(sandbox_id) {
48034765
return Ok(true);
48044766
}
@@ -14328,81 +14290,6 @@ mod tests {
1432814290
);
1432914291
}
1433014292

14331-
#[tokio::test]
14332-
async fn shutdown_handoff_keeps_ready_when_replacement_publishes_during_wait() {
14333-
let runtime = Arc::new(test_runtime(Arc::new(TestDriver::default())).await);
14334-
let mut sandbox = sandbox_record("sb-1", "sandbox-a", SandboxPhase::Ready);
14335-
sandbox.set_phase(SandboxPhase::Ready as i32);
14336-
runtime.store.put_message(&sandbox).await.unwrap();
14337-
14338-
let store = runtime.store.clone();
14339-
let publish = tokio::spawn(async move {
14340-
tokio::time::sleep(Duration::from_millis(250)).await;
14341-
SupervisorOwnerIndex::new(store, OWNER_TTL)
14342-
.publish(
14343-
"sb-1",
14344-
"replacement-session",
14345-
"supervisor-instance",
14346-
2,
14347-
"gateway-b",
14348-
"http://gateway-b:8080",
14349-
)
14350-
.await
14351-
.unwrap();
14352-
});
14353-
14354-
assert!(
14355-
runtime
14356-
.wait_for_supervisor_session_handoff("sb-1", Duration::from_secs(5))
14357-
.await
14358-
);
14359-
publish.await.unwrap();
14360-
runtime
14361-
.supervisor_session_disconnected("sb-1", false)
14362-
.await
14363-
.unwrap();
14364-
14365-
let stored = runtime
14366-
.store
14367-
.get_message::<Sandbox>("sb-1")
14368-
.await
14369-
.unwrap()
14370-
.unwrap();
14371-
assert_eq!(
14372-
SandboxPhase::try_from(stored.phase()).unwrap(),
14373-
SandboxPhase::Ready
14374-
);
14375-
}
14376-
14377-
#[tokio::test]
14378-
async fn shutdown_handoff_demotes_when_no_replacement_arrives() {
14379-
let runtime = test_runtime(Arc::new(TestDriver::default())).await;
14380-
let mut sandbox = sandbox_record("sb-1", "sandbox-a", SandboxPhase::Ready);
14381-
sandbox.set_phase(SandboxPhase::Ready as i32);
14382-
runtime.store.put_message(&sandbox).await.unwrap();
14383-
14384-
assert!(
14385-
!runtime
14386-
.wait_for_supervisor_session_handoff("sb-1", Duration::from_millis(200))
14387-
.await
14388-
);
14389-
runtime
14390-
.supervisor_session_disconnected("sb-1", false)
14391-
.await
14392-
.unwrap();
14393-
14394-
let stored = runtime
14395-
.store
14396-
.get_message::<Sandbox>("sb-1")
14397-
.await
14398-
.unwrap()
14399-
.unwrap();
14400-
assert_eq!(
14401-
SandboxPhase::try_from(stored.phase()).unwrap(),
14402-
SandboxPhase::Provisioning
14403-
);
14404-
}
14405-
1440614293
// --- Composition rule tests ---
1440714294

1440814295
fn make_ready_driver_status() -> DriverSandboxStatus {

‎crates/openshell-server/src/lib.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -991,7 +991,7 @@ pub(crate) async fn run_server(
991991
// record. Drain it even when compute cleanup failed before exiting Tokio.
992992
let session_cleanup = state
993993
.supervisor_sessions
994-
.shutdown(Duration::from_secs(10))
994+
.shutdown(supervisor_session::SUPERVISOR_SESSION_SHUTDOWN_TIMEOUT)
995995
.await;
996996
if let Err(err) = &session_cleanup {
997997
warn!(error = %err, "Gateway supervisor session cleanup incomplete");

0 commit comments

Comments
 (0)