Skip to content

Commit 3f916fb

Browse files
committed
fix(ha): hold ownership through the shutdown handoff
The stopping replica released its owner record before waiting for the replacement session. A reconciliation sweep on another replica in that window saw no owner, and with the Kubernetes driver leaving readiness to the gateway it demoted the sandbox to Provisioning anyway. Keep the record until a different session supersedes it or the grace expires, then release it only if it is still ours. Also from review: - Keep polling the owner record through store errors until the deadline. - Skip the handoff wait for sandboxes that are not Ready. - Re-read membership in the membership worker's shutdown branch and await the worker before session shutdown, so the worker stays the only writer of the ring and peer map. - Share the remove, release, and demote steps of failed session setup. - Test redirect queuing directly, including unsupported supervisors and a full outbound channel. Signed-off-by: Piotr Mlocek <[email protected]>
1 parent 0824268 commit 3f916fb

3 files changed

Lines changed: 308 additions & 185 deletions

File tree

‎crates/openshell-server/src/gateway_members.rs‎

Lines changed: 79 additions & 73 deletions
Original file line numberDiff line numberDiff line change
@@ -28,8 +28,12 @@ pub const MEMBER_REFRESH_INTERVAL: Duration = Duration::from_secs(10);
2828
/// Upper bound on replicas read back when building the ring.
2929
const MEMBER_LIST_LIMIT: u32 = 1024;
3030

31-
/// How long shutdown waits to re-read membership before redirecting sessions.
32-
pub const SHUTDOWN_RING_REFRESH_TIMEOUT: Duration = Duration::from_secs(2);
31+
/// How long the worker waits on each store call while leaving at shutdown.
32+
const SHUTDOWN_STORE_TIMEOUT: Duration = Duration::from_secs(2);
33+
34+
/// How long gateway shutdown waits for the membership worker to finish
35+
/// leaving: one membership read and one deregister.
36+
pub const SHUTDOWN_MEMBERSHIP_TIMEOUT: Duration = Duration::from_secs(5);
3337

3438
fn member_object_id(replica_id: &str) -> String {
3539
format!("gateway-member:{replica_id}")
@@ -148,85 +152,35 @@ impl GatewayMemberIndex {
148152
let ring = GatewayRing::new(members.iter().map(|member| member.replica_id.as_str()));
149153
Ok((ring, members))
150154
}
151-
152-
/// Build the ring and peer endpoints a stopping replica redirects to: the
153-
/// live replicas other than `replica_id`.
154-
pub async fn ring_without(
155-
&self,
156-
replica_id: &str,
157-
) -> PersistenceResult<(GatewayRing, HashMap<String, String>)> {
158-
let members = self
159-
.live_members()
160-
.await?
161-
.into_iter()
162-
.filter(|member| member.replica_id != replica_id)
163-
.collect::<Vec<_>>();
164-
let ring = GatewayRing::new(members.iter().map(|member| member.replica_id.as_str()));
165-
let peers = members
166-
.into_iter()
167-
.map(|member| (member.replica_id, member.peer_endpoint))
168-
.collect();
169-
Ok((ring, peers))
170-
}
171-
}
172-
173-
/// Re-read membership just before shutdown redirects supervisor sessions.
174-
///
175-
/// The worker refreshes the ring only every [`MEMBER_REFRESH_INTERVAL`]. During
176-
/// a rolling update the cached ring can miss a replacement that just started
177-
/// and still name a replica that already left, so redirects built from it send
178-
/// supervisors nowhere or to a dead pod. The read is bounded by `timeout`; on
179-
/// failure the cached ring stays in place.
180-
pub async fn refresh_ring_for_shutdown(state: &crate::ServerState, timeout: Duration) {
181-
if state.store.is_single_replica() || state.peer_endpoint.is_none() {
182-
return;
183-
}
184-
let index = GatewayMemberIndex::new(state.store.clone(), MEMBER_TTL);
185-
match tokio::time::timeout(timeout, index.ring_without(&state.replica_id)).await {
186-
Ok(Ok((ring, peers))) => {
187-
if let Ok(mut slot) = state.gateway_ring.write() {
188-
*slot = ring;
189-
}
190-
if let Ok(mut slot) = state.gateway_peers.write() {
191-
*slot = peers;
192-
}
193-
}
194-
Ok(Err(error)) => {
195-
tracing::warn!(%error, "gateway membership: shutdown ring refresh failed");
196-
}
197-
Err(_) => {
198-
tracing::warn!(
199-
timeout_ms = timeout.as_millis(),
200-
"gateway membership: shutdown ring refresh timed out"
201-
);
202-
}
203-
}
204155
}
205156

206157
/// Keep this replica's membership record fresh and refresh the local ring.
207158
///
208159
/// One task does both so the ring is never newer than our own liveness claim.
209160
/// The ring starts empty, which means "serve locally" — so a store outage
210161
/// degrades to today's behaviour rather than breaking placement.
162+
///
163+
/// Returns the worker task, which finishes after this replica leaves the ring
164+
/// at shutdown. Shutdown awaits it before redirecting supervisor sessions.
211165
pub fn spawn_membership_worker(
212166
state: Arc<crate::ServerState>,
213167
interval: Duration,
214168
mut shutdown: tokio::sync::watch::Receiver<bool>,
215-
) {
169+
) -> Option<tokio::task::JoinHandle<()>> {
216170
if state.store.is_single_replica() {
217-
return;
171+
return None;
218172
}
219173
let Some(peer_endpoint) = state.peer_endpoint.clone() else {
220174
// Without a peer endpoint other replicas cannot dial us, so we must
221175
// not advertise ourselves as a placement target.
222176
tracing::debug!("gateway membership: no peer endpoint, not joining the ring");
223-
return;
177+
return None;
224178
};
225179

226180
let index = GatewayMemberIndex::new(state.store.clone(), MEMBER_TTL);
227181
let replica_id = state.replica_id.clone();
228182

229-
tokio::spawn(async move {
183+
Some(tokio::spawn(async move {
230184
let mut ticker = tokio::time::interval(interval);
231185
ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
232186
let mut last_refreshed = tokio::time::Instant::now();
@@ -236,14 +190,7 @@ pub fn spawn_membership_worker(
236190
_ = ticker.tick() => {}
237191
_ = shutdown.changed() => {
238192
if *shutdown.borrow() {
239-
// Sessions closing during shutdown redirect to the
240-
// replica that owns them once this one is gone.
241-
if let Ok(mut slot) = state.gateway_ring.write() {
242-
*slot = slot.without(&replica_id);
243-
}
244-
if let Err(error) = index.deregister(&replica_id).await {
245-
tracing::warn!(%error, "gateway membership: deregister failed");
246-
}
193+
leave_ring(&state, &index, &replica_id).await;
247194
return;
248195
}
249196
continue;
@@ -276,7 +223,50 @@ pub fn spawn_membership_worker(
276223
}
277224
}
278225
}
279-
});
226+
}))
227+
}
228+
229+
/// Point the ring at the replicas that remain once this one is gone, then
230+
/// remove this replica's record.
231+
///
232+
/// Sessions closing during shutdown redirect to the replica that owns them
233+
/// next. The periodic refresh can be up to one interval old, and during a
234+
/// rolling update that ring can miss a replacement that just started and still
235+
/// name a replica that already left. Redirects built from it send supervisors
236+
/// nowhere or to a dead pod, so re-read membership first. If the read fails,
237+
/// keep the cached ring without this replica.
238+
async fn leave_ring(state: &crate::ServerState, index: &GatewayMemberIndex, replica_id: &str) {
239+
match tokio::time::timeout(SHUTDOWN_STORE_TIMEOUT, index.ring()).await {
240+
Ok(Ok((ring, members))) => {
241+
let peers: HashMap<String, String> = members
242+
.into_iter()
243+
.map(|member| (member.replica_id, member.peer_endpoint))
244+
.collect();
245+
if let Ok(mut slot) = state.gateway_ring.write() {
246+
*slot = ring.without(replica_id);
247+
}
248+
if let Ok(mut slot) = state.gateway_peers.write() {
249+
*slot = peers;
250+
}
251+
}
252+
Ok(Err(error)) => {
253+
tracing::warn!(%error, "gateway membership: shutdown ring refresh failed");
254+
if let Ok(mut slot) = state.gateway_ring.write() {
255+
*slot = slot.without(replica_id);
256+
}
257+
}
258+
Err(_) => {
259+
tracing::warn!("gateway membership: shutdown ring refresh timed out");
260+
if let Ok(mut slot) = state.gateway_ring.write() {
261+
*slot = slot.without(replica_id);
262+
}
263+
}
264+
}
265+
match tokio::time::timeout(SHUTDOWN_STORE_TIMEOUT, index.deregister(replica_id)).await {
266+
Ok(Ok(())) => {}
267+
Ok(Err(error)) => tracing::warn!(%error, "gateway membership: deregister failed"),
268+
Err(_) => tracing::warn!("gateway membership: deregister timed out"),
269+
}
280270
}
281271

282272
fn clear_ring_if_stale(state: &crate::ServerState, last_refreshed: tokio::time::Instant) {
@@ -316,25 +306,41 @@ mod tests {
316306
}
317307

318308
#[tokio::test]
319-
async fn shutdown_ring_includes_new_replicas_and_excludes_departed_ones() {
320-
let index = test_index(MEMBER_TTL).await;
309+
async fn leaving_points_ring_at_new_replicas_and_not_departed_ones() {
310+
let state = crate::grpc::test_support::test_server_state().await;
311+
let index = GatewayMemberIndex::new(state.store.clone(), MEMBER_TTL);
321312
index.register("gw-old", "https://gw-old").await.unwrap();
322313
index
323314
.register("gw-stopping", "https://gw-stopping")
324315
.await
325316
.unwrap();
317+
*state.gateway_ring.write().unwrap() = GatewayRing::new(["gw-old", "gw-stopping"]);
326318
// A rolling update replaces gw-old after the stopping replica last
327319
// refreshed its cached ring.
328320
index.deregister("gw-old").await.unwrap();
329321
index.register("gw-new", "https://gw-new").await.unwrap();
330322

331-
let (ring, peers) = index.ring_without("gw-stopping").await.unwrap();
323+
leave_ring(&state, &index, "gw-stopping").await;
332324

325+
let ring = state.gateway_ring.read().unwrap().clone();
333326
assert_eq!(ring.owner_for("sb-1"), Some("gw-new"));
334327
assert_eq!(
335-
peers,
336-
HashMap::from([("gw-new".to_string(), "https://gw-new".to_string())])
328+
state
329+
.gateway_peers
330+
.read()
331+
.unwrap()
332+
.get("gw-new")
333+
.map(String::as_str),
334+
Some("https://gw-new")
337335
);
336+
let remaining: Vec<String> = index
337+
.live_members()
338+
.await
339+
.unwrap()
340+
.into_iter()
341+
.map(|member| member.replica_id)
342+
.collect();
343+
assert_eq!(remaining, vec!["gw-new"]);
338344
}
339345

340346
#[tokio::test]

‎crates/openshell-server/src/lib.rs‎

Lines changed: 10 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -967,7 +967,7 @@ pub(crate) async fn run_server(
967967
shutdown_rx.clone(),
968968
);
969969
}
970-
gateway_members::spawn_membership_worker(
970+
let membership_worker = gateway_members::spawn_membership_worker(
971971
state.clone(),
972972
gateway_members::MEMBER_REFRESH_INTERVAL,
973973
shutdown_rx.clone(),
@@ -987,13 +987,15 @@ pub(crate) async fn run_server(
987987
}
988988

989989
let compute_cleanup = state.compute.cleanup_on_shutdown().await;
990-
// Closing sessions redirect their supervisors through the ring, so pick up
991-
// replicas that joined since the last periodic refresh.
992-
gateway_members::refresh_ring_for_shutdown(
993-
&state,
994-
gateway_members::SHUTDOWN_RING_REFRESH_TIMEOUT,
995-
)
996-
.await;
990+
// Closing sessions redirect their supervisors through the ring. Let the
991+
// membership worker finish pointing it at the remaining replicas first.
992+
if let Some(worker) = membership_worker
993+
&& tokio::time::timeout(gateway_members::SHUTDOWN_MEMBERSHIP_TIMEOUT, worker)
994+
.await
995+
.is_err()
996+
{
997+
warn!("Gateway membership worker did not finish leaving the ring before session shutdown");
998+
}
997999
// A stopped supervisor may still have a detached task deleting its owner
9981000
// record. Drain it even when compute cleanup failed before exiting Tokio.
9991001
let session_cleanup = state

0 commit comments

Comments
 (0)