@@ -28,8 +28,12 @@ pub const MEMBER_REFRESH_INTERVAL: Duration = Duration::from_secs(10);
2828/// Upper bound on replicas read back when building the ring.
2929const MEMBER_LIST_LIMIT : u32 = 1024 ;
3030
31- /// How long shutdown waits to re-read membership before redirecting sessions.
32- pub const SHUTDOWN_RING_REFRESH_TIMEOUT : Duration = Duration :: from_secs ( 2 ) ;
31+ /// How long the worker waits on each store call while leaving at shutdown.
32+ const SHUTDOWN_STORE_TIMEOUT : Duration = Duration :: from_secs ( 2 ) ;
33+
34+ /// How long gateway shutdown waits for the membership worker to finish
35+ /// leaving: one membership read and one deregister.
36+ pub const SHUTDOWN_MEMBERSHIP_TIMEOUT : Duration = Duration :: from_secs ( 5 ) ;
3337
3438fn member_object_id ( replica_id : & str ) -> String {
3539 format ! ( "gateway-member:{replica_id}" )
@@ -148,85 +152,35 @@ impl GatewayMemberIndex {
148152 let ring = GatewayRing :: new ( members. iter ( ) . map ( |member| member. replica_id . as_str ( ) ) ) ;
149153 Ok ( ( ring, members) )
150154 }
151-
152- /// Build the ring and peer endpoints a stopping replica redirects to: the
153- /// live replicas other than `replica_id`.
154- pub async fn ring_without (
155- & self ,
156- replica_id : & str ,
157- ) -> PersistenceResult < ( GatewayRing , HashMap < String , String > ) > {
158- let members = self
159- . live_members ( )
160- . await ?
161- . into_iter ( )
162- . filter ( |member| member. replica_id != replica_id)
163- . collect :: < Vec < _ > > ( ) ;
164- let ring = GatewayRing :: new ( members. iter ( ) . map ( |member| member. replica_id . as_str ( ) ) ) ;
165- let peers = members
166- . into_iter ( )
167- . map ( |member| ( member. replica_id , member. peer_endpoint ) )
168- . collect ( ) ;
169- Ok ( ( ring, peers) )
170- }
171- }
172-
173- /// Re-read membership just before shutdown redirects supervisor sessions.
174- ///
175- /// The worker refreshes the ring only every [`MEMBER_REFRESH_INTERVAL`]. During
176- /// a rolling update the cached ring can miss a replacement that just started
177- /// and still name a replica that already left, so redirects built from it send
178- /// supervisors nowhere or to a dead pod. The read is bounded by `timeout`; on
179- /// failure the cached ring stays in place.
180- pub async fn refresh_ring_for_shutdown ( state : & crate :: ServerState , timeout : Duration ) {
181- if state. store . is_single_replica ( ) || state. peer_endpoint . is_none ( ) {
182- return ;
183- }
184- let index = GatewayMemberIndex :: new ( state. store . clone ( ) , MEMBER_TTL ) ;
185- match tokio:: time:: timeout ( timeout, index. ring_without ( & state. replica_id ) ) . await {
186- Ok ( Ok ( ( ring, peers) ) ) => {
187- if let Ok ( mut slot) = state. gateway_ring . write ( ) {
188- * slot = ring;
189- }
190- if let Ok ( mut slot) = state. gateway_peers . write ( ) {
191- * slot = peers;
192- }
193- }
194- Ok ( Err ( error) ) => {
195- tracing:: warn!( %error, "gateway membership: shutdown ring refresh failed" ) ;
196- }
197- Err ( _) => {
198- tracing:: warn!(
199- timeout_ms = timeout. as_millis( ) ,
200- "gateway membership: shutdown ring refresh timed out"
201- ) ;
202- }
203- }
204155}
205156
206157/// Keep this replica's membership record fresh and refresh the local ring.
207158///
208159/// One task does both so the ring is never newer than our own liveness claim.
209160/// The ring starts empty, which means "serve locally" — so a store outage
210161/// degrades to today's behaviour rather than breaking placement.
162+ ///
163+ /// Returns the worker task, which finishes after this replica leaves the ring
164+ /// at shutdown. Shutdown awaits it before redirecting supervisor sessions.
211165pub fn spawn_membership_worker (
212166 state : Arc < crate :: ServerState > ,
213167 interval : Duration ,
214168 mut shutdown : tokio:: sync:: watch:: Receiver < bool > ,
215- ) {
169+ ) -> Option < tokio :: task :: JoinHandle < ( ) > > {
216170 if state. store . is_single_replica ( ) {
217- return ;
171+ return None ;
218172 }
219173 let Some ( peer_endpoint) = state. peer_endpoint . clone ( ) else {
220174 // Without a peer endpoint other replicas cannot dial us, so we must
221175 // not advertise ourselves as a placement target.
222176 tracing:: debug!( "gateway membership: no peer endpoint, not joining the ring" ) ;
223- return ;
177+ return None ;
224178 } ;
225179
226180 let index = GatewayMemberIndex :: new ( state. store . clone ( ) , MEMBER_TTL ) ;
227181 let replica_id = state. replica_id . clone ( ) ;
228182
229- tokio:: spawn ( async move {
183+ Some ( tokio:: spawn ( async move {
230184 let mut ticker = tokio:: time:: interval ( interval) ;
231185 ticker. set_missed_tick_behavior ( tokio:: time:: MissedTickBehavior :: Delay ) ;
232186 let mut last_refreshed = tokio:: time:: Instant :: now ( ) ;
@@ -236,14 +190,7 @@ pub fn spawn_membership_worker(
236190 _ = ticker. tick( ) => { }
237191 _ = shutdown. changed( ) => {
238192 if * shutdown. borrow( ) {
239- // Sessions closing during shutdown redirect to the
240- // replica that owns them once this one is gone.
241- if let Ok ( mut slot) = state. gateway_ring. write( ) {
242- * slot = slot. without( & replica_id) ;
243- }
244- if let Err ( error) = index. deregister( & replica_id) . await {
245- tracing:: warn!( %error, "gateway membership: deregister failed" ) ;
246- }
193+ leave_ring( & state, & index, & replica_id) . await ;
247194 return ;
248195 }
249196 continue ;
@@ -276,7 +223,50 @@ pub fn spawn_membership_worker(
276223 }
277224 }
278225 }
279- } ) ;
226+ } ) )
227+ }
228+
229+ /// Point the ring at the replicas that remain once this one is gone, then
230+ /// remove this replica's record.
231+ ///
232+ /// Sessions closing during shutdown redirect to the replica that owns them
233+ /// next. The periodic refresh can be up to one interval old, and during a
234+ /// rolling update that ring can miss a replacement that just started and still
235+ /// name a replica that already left. Redirects built from it send supervisors
236+ /// nowhere or to a dead pod, so re-read membership first. If the read fails,
237+ /// keep the cached ring without this replica.
238+ async fn leave_ring ( state : & crate :: ServerState , index : & GatewayMemberIndex , replica_id : & str ) {
239+ match tokio:: time:: timeout ( SHUTDOWN_STORE_TIMEOUT , index. ring ( ) ) . await {
240+ Ok ( Ok ( ( ring, members) ) ) => {
241+ let peers: HashMap < String , String > = members
242+ . into_iter ( )
243+ . map ( |member| ( member. replica_id , member. peer_endpoint ) )
244+ . collect ( ) ;
245+ if let Ok ( mut slot) = state. gateway_ring . write ( ) {
246+ * slot = ring. without ( replica_id) ;
247+ }
248+ if let Ok ( mut slot) = state. gateway_peers . write ( ) {
249+ * slot = peers;
250+ }
251+ }
252+ Ok ( Err ( error) ) => {
253+ tracing:: warn!( %error, "gateway membership: shutdown ring refresh failed" ) ;
254+ if let Ok ( mut slot) = state. gateway_ring . write ( ) {
255+ * slot = slot. without ( replica_id) ;
256+ }
257+ }
258+ Err ( _) => {
259+ tracing:: warn!( "gateway membership: shutdown ring refresh timed out" ) ;
260+ if let Ok ( mut slot) = state. gateway_ring . write ( ) {
261+ * slot = slot. without ( replica_id) ;
262+ }
263+ }
264+ }
265+ match tokio:: time:: timeout ( SHUTDOWN_STORE_TIMEOUT , index. deregister ( replica_id) ) . await {
266+ Ok ( Ok ( ( ) ) ) => { }
267+ Ok ( Err ( error) ) => tracing:: warn!( %error, "gateway membership: deregister failed" ) ,
268+ Err ( _) => tracing:: warn!( "gateway membership: deregister timed out" ) ,
269+ }
280270}
281271
282272fn clear_ring_if_stale ( state : & crate :: ServerState , last_refreshed : tokio:: time:: Instant ) {
@@ -316,25 +306,41 @@ mod tests {
316306 }
317307
318308 #[ tokio:: test]
319- async fn shutdown_ring_includes_new_replicas_and_excludes_departed_ones ( ) {
320- let index = test_index ( MEMBER_TTL ) . await ;
309+ async fn leaving_points_ring_at_new_replicas_and_not_departed_ones ( ) {
310+ let state = crate :: grpc:: test_support:: test_server_state ( ) . await ;
311+ let index = GatewayMemberIndex :: new ( state. store . clone ( ) , MEMBER_TTL ) ;
321312 index. register ( "gw-old" , "https://gw-old" ) . await . unwrap ( ) ;
322313 index
323314 . register ( "gw-stopping" , "https://gw-stopping" )
324315 . await
325316 . unwrap ( ) ;
317+ * state. gateway_ring . write ( ) . unwrap ( ) = GatewayRing :: new ( [ "gw-old" , "gw-stopping" ] ) ;
326318 // A rolling update replaces gw-old after the stopping replica last
327319 // refreshed its cached ring.
328320 index. deregister ( "gw-old" ) . await . unwrap ( ) ;
329321 index. register ( "gw-new" , "https://gw-new" ) . await . unwrap ( ) ;
330322
331- let ( ring , peers ) = index. ring_without ( "gw-stopping" ) . await . unwrap ( ) ;
323+ leave_ring ( & state , & index, "gw-stopping" ) . await ;
332324
325+ let ring = state. gateway_ring . read ( ) . unwrap ( ) . clone ( ) ;
333326 assert_eq ! ( ring. owner_for( "sb-1" ) , Some ( "gw-new" ) ) ;
334327 assert_eq ! (
335- peers,
336- HashMap :: from( [ ( "gw-new" . to_string( ) , "https://gw-new" . to_string( ) ) ] )
328+ state
329+ . gateway_peers
330+ . read( )
331+ . unwrap( )
332+ . get( "gw-new" )
333+ . map( String :: as_str) ,
334+ Some ( "https://gw-new" )
337335 ) ;
336+ let remaining: Vec < String > = index
337+ . live_members ( )
338+ . await
339+ . unwrap ( )
340+ . into_iter ( )
341+ . map ( |member| member. replica_id )
342+ . collect ( ) ;
343+ assert_eq ! ( remaining, vec![ "gw-new" ] ) ;
338344 }
339345
340346 #[ tokio:: test]
0 commit comments