88//! kept in the store rather than read from Kubernetes so it works for any
99//! multi-replica deployment and adds no API server load.
1010
11+ use crate :: gateway_metrics;
1112use crate :: gateway_ring:: GatewayRing ;
1213use crate :: persistence:: { PersistenceError , PersistenceResult , Store , WriteCondition } ;
1314use openshell_core:: time:: now_ms;
@@ -186,6 +187,10 @@ pub fn spawn_membership_worker(
186187
187188 let index = GatewayMemberIndex :: new ( state. store . clone ( ) , MEMBER_TTL ) ;
188189 let replica_id = state. replica_id . clone ( ) ;
190+ // Export the ring's size now (empty until the first refresh), so only
191+ // replicas that join the ring carry the gauge, and they carry it from the
192+ // start.
193+ publish_ring ( & state, GatewayRing :: clone) ;
189194
190195 Some ( tokio:: spawn ( async move {
191196 let mut ticker = tokio:: time:: interval ( interval) ;
@@ -216,9 +221,7 @@ pub fn spawn_membership_worker(
216221 . into_iter ( )
217222 . map ( |member| ( member. replica_id , member. peer_endpoint ) )
218223 . collect ( ) ;
219- if let Ok ( mut slot) = state. gateway_ring . write ( ) {
220- * slot = ring;
221- }
224+ publish_ring ( & state, |_| ring) ;
222225 if let Ok ( mut slot) = state. gateway_peers . write ( ) {
223226 * slot = peers;
224227 }
@@ -249,24 +252,18 @@ async fn leave_ring(state: &crate::ServerState, index: &GatewayMemberIndex, repl
249252 . into_iter ( )
250253 . map ( |member| ( member. replica_id , member. peer_endpoint ) )
251254 . collect ( ) ;
252- if let Ok ( mut slot) = state. gateway_ring . write ( ) {
253- * slot = ring. without ( replica_id) ;
254- }
255+ publish_ring ( state, |_| ring. without ( replica_id) ) ;
255256 if let Ok ( mut slot) = state. gateway_peers . write ( ) {
256257 * slot = peers;
257258 }
258259 }
259260 Ok ( Err ( error) ) => {
260261 tracing:: warn!( %error, "gateway membership: shutdown ring refresh failed" ) ;
261- if let Ok ( mut slot) = state. gateway_ring . write ( ) {
262- * slot = slot. without ( replica_id) ;
263- }
262+ publish_ring ( state, |ring| ring. without ( replica_id) ) ;
264263 }
265264 Err ( _) => {
266265 tracing:: warn!( "gateway membership: shutdown ring refresh timed out" ) ;
267- if let Ok ( mut slot) = state. gateway_ring . write ( ) {
268- * slot = slot. without ( replica_id) ;
269- }
266+ publish_ring ( state, |ring| ring. without ( replica_id) ) ;
270267 }
271268 }
272269 match tokio:: time:: timeout ( SHUTDOWN_STORE_TIMEOUT , index. deregister ( replica_id) ) . await {
@@ -280,14 +277,26 @@ fn clear_ring_if_stale(state: &crate::ServerState, last_refreshed: tokio::time::
280277 if last_refreshed. elapsed ( ) <= MEMBER_TTL {
281278 return ;
282279 }
283- if let Ok ( mut slot) = state. gateway_ring . write ( ) {
284- * slot = GatewayRing :: default ( ) ;
285- }
280+ publish_ring ( state, |_| GatewayRing :: default ( ) ) ;
286281 if let Ok ( mut slot) = state. gateway_peers . write ( ) {
287282 slot. clear ( ) ;
288283 }
289284}
290285
286+ /// Replace this replica's placement ring with `next(current ring)` and record
287+ /// its size as `openshell_server_gateway_ring_members` under the same write
288+ /// lock.
289+ ///
290+ /// This is the only sanctioned non-test writer of `ServerState::gateway_ring`:
291+ /// writing the slot anywhere else lets the gauge drift from the ring. A
292+ /// poisoned lock leaves both unchanged.
293+ fn publish_ring ( state : & crate :: ServerState , next : impl FnOnce ( & GatewayRing ) -> GatewayRing ) {
294+ if let Ok ( mut slot) = state. gateway_ring . write ( ) {
295+ * slot = next ( & slot) ;
296+ gateway_metrics:: record_gateway_ring_members ( slot. replica_count ( ) ) ;
297+ }
298+ }
299+
291300#[ cfg( test) ]
292301mod tests {
293302 use super :: * ;
@@ -314,6 +323,7 @@ mod tests {
314323
315324 #[ tokio:: test]
316325 async fn leaving_points_ring_at_new_replicas_and_not_departed_ones ( ) {
326+ let metrics = gateway_metrics:: MetricsCapture :: install ( ) ;
317327 let state = crate :: grpc:: test_support:: test_server_state ( ) . await ;
318328 let index = GatewayMemberIndex :: new ( state. store . clone ( ) , MEMBER_TTL ) ;
319329 index. register ( "gw-old" , "https://gw-old" ) . await . unwrap ( ) ;
@@ -331,6 +341,10 @@ mod tests {
331341
332342 let ring = state. gateway_ring . read ( ) . unwrap ( ) . clone ( ) ;
333343 assert_eq ! ( ring. owner_for( "sb-1" ) , Some ( "gw-new" ) ) ;
344+ assert_eq ! (
345+ metrics. value( gateway_metrics:: GATEWAY_RING_MEMBERS ) ,
346+ Some ( 1 )
347+ ) ;
334348 assert_eq ! (
335349 state
336350 . gateway_peers
@@ -350,6 +364,222 @@ mod tests {
350364 assert_eq ! ( remaining, vec![ "gw-new" ] ) ;
351365 }
352366
367+ #[ tokio:: test]
368+ async fn leaving_without_a_membership_read_keeps_the_cached_ring_without_this_replica ( ) {
369+ let metrics = gateway_metrics:: MetricsCapture :: install ( ) ;
370+ let state = crate :: grpc:: test_support:: test_server_state ( ) . await ;
371+ publish_ring ( & state, |_| GatewayRing :: new ( [ "gw-a" , "gw-stopping" ] ) ) ;
372+ // Only the index's store is closed, so the shutdown membership read
373+ // fails while the state's store stays open.
374+ let store = Arc :: new ( crate :: persistence:: test_store ( ) . await ) ;
375+ store. close ( ) . await ;
376+ let index = GatewayMemberIndex :: new ( store, MEMBER_TTL ) ;
377+
378+ leave_ring ( & state, & index, "gw-stopping" ) . await ;
379+
380+ let ring = state. gateway_ring . read ( ) . unwrap ( ) . clone ( ) ;
381+ assert_eq ! ( ring. owner_for( "sb-1" ) , Some ( "gw-a" ) ) ;
382+ assert_eq ! (
383+ metrics. value( gateway_metrics:: GATEWAY_RING_MEMBERS ) ,
384+ Some ( 1 )
385+ ) ;
386+ }
387+
388+ #[ tokio:: test]
389+ async fn leaving_after_a_timed_out_membership_read_keeps_the_cached_ring_without_this_replica ( )
390+ {
391+ let metrics = gateway_metrics:: MetricsCapture :: install ( ) ;
392+ let state = crate :: grpc:: test_support:: test_server_state ( ) . await ;
393+ publish_ring ( & state, |_| GatewayRing :: new ( [ "gw-a" , "gw-stopping" ] ) ) ;
394+ // The listener accepts TCP but never answers the PostgreSQL handshake, so
395+ // the shutdown read hangs past SHUTDOWN_STORE_TIMEOUT instead of failing.
396+ let silent = tokio:: net:: TcpListener :: bind ( "127.0.0.1:0" ) . await . unwrap ( ) ;
397+ let url = format ! (
398+ "postgres://[email protected] :{}/openshell" , 399+ silent. local_addr( ) . unwrap( ) . port( )
400+ ) ;
401+ let index = GatewayMemberIndex :: new (
402+ Arc :: new ( Store :: Postgres (
403+ crate :: persistence:: PostgresStore :: connect_lazy_for_tests ( & url, 1 ) ,
404+ ) ) ,
405+ MEMBER_TTL ,
406+ ) ;
407+
408+ // The paused clock jumps to each timeout while the socket stays silent.
409+ tokio:: time:: pause ( ) ;
410+ let started = tokio:: time:: Instant :: now ( ) ;
411+ leave_ring ( & state, & index, "gw-stopping" ) . await ;
412+ // Both the read and the deregister must hit their timeouts; a read that
413+ // failed fast would take the Ok(Err) branch and finish sooner.
414+ assert ! (
415+ started. elapsed( ) >= SHUTDOWN_STORE_TIMEOUT * 2 ,
416+ "the membership read must time out, not fail fast"
417+ ) ;
418+
419+ let ring = state. gateway_ring . read ( ) . unwrap ( ) . clone ( ) ;
420+ assert_eq ! ( ring. replica_count( ) , 1 ) ;
421+ assert_eq ! ( ring. owner_for( "sb-1" ) , Some ( "gw-a" ) ) ;
422+ assert_eq ! (
423+ metrics. value( gateway_metrics:: GATEWAY_RING_MEMBERS ) ,
424+ Some ( 1 )
425+ ) ;
426+ drop ( silent) ;
427+ }
428+
429+ #[ tokio:: test]
430+ async fn stale_ring_is_cleared_only_after_the_member_ttl ( ) {
431+ let metrics = gateway_metrics:: MetricsCapture :: install ( ) ;
432+ let state = crate :: grpc:: test_support:: test_server_state ( ) . await ;
433+ publish_ring ( & state, |_| GatewayRing :: new ( [ "gw-a" , "gw-b" ] ) ) ;
434+ * state. gateway_peers . write ( ) . unwrap ( ) = HashMap :: from ( [
435+ ( "gw-a" . to_string ( ) , "https://gw-a" . to_string ( ) ) ,
436+ ( "gw-b" . to_string ( ) , "https://gw-b" . to_string ( ) ) ,
437+ ] ) ;
438+ assert_eq ! (
439+ metrics. value( gateway_metrics:: GATEWAY_RING_MEMBERS ) ,
440+ Some ( 2 )
441+ ) ;
442+
443+ clear_ring_if_stale ( & state, tokio:: time:: Instant :: now ( ) ) ;
444+ assert_eq ! ( state. gateway_ring. read( ) . unwrap( ) . replica_count( ) , 2 ) ;
445+ assert_eq ! ( state. gateway_peers. read( ) . unwrap( ) . len( ) , 2 ) ;
446+ assert_eq ! (
447+ metrics. value( gateway_metrics:: GATEWAY_RING_MEMBERS ) ,
448+ Some ( 2 )
449+ ) ;
450+
451+ let last_refreshed = tokio:: time:: Instant :: now ( )
452+ . checked_sub ( MEMBER_TTL + Duration :: from_millis ( 1 ) )
453+ . expect ( "the monotonic clock is past one member TTL" ) ;
454+ clear_ring_if_stale ( & state, last_refreshed) ;
455+ assert ! ( state. gateway_ring. read( ) . unwrap( ) . is_empty( ) ) ;
456+ assert ! ( state. gateway_peers. read( ) . unwrap( ) . is_empty( ) ) ;
457+ assert_eq ! (
458+ metrics. value( gateway_metrics:: GATEWAY_RING_MEMBERS ) ,
459+ Some ( 0 )
460+ ) ;
461+ }
462+
463+ #[ tokio:: test]
464+ async fn poisoned_ring_lock_publishes_no_ring_size ( ) {
465+ let metrics = gateway_metrics:: MetricsCapture :: install ( ) ;
466+ let state = crate :: grpc:: test_support:: test_server_state ( ) . await ;
467+ let ring = Arc :: clone ( & state. gateway_ring ) ;
468+ let poisoned = std:: thread:: spawn ( move || {
469+ std:: panic:: catch_unwind ( std:: panic:: AssertUnwindSafe ( || {
470+ let _guard = ring. write ( ) . unwrap ( ) ;
471+ // `resume_unwind` drops the guard and poisons the lock without
472+ // running the panic hook, keeping the expected panic out of
473+ // the test output.
474+ std:: panic:: resume_unwind ( Box :: new ( "poison the placement ring lock" ) ) ;
475+ } ) )
476+ } )
477+ . join ( )
478+ . expect ( "the poisoning thread should catch its own unwind" ) ;
479+ assert ! ( poisoned. is_err( ) ) ;
480+ assert ! ( state. gateway_ring. is_poisoned( ) ) ;
481+
482+ publish_ring ( & state, |_| GatewayRing :: new ( [ "gw-a" , "gw-b" ] ) ) ;
483+
484+ assert_eq ! ( metrics. value( gateway_metrics:: GATEWAY_RING_MEMBERS ) , None ) ;
485+ }
486+
487+ #[ tokio:: test]
488+ async fn single_replica_store_neither_joins_the_ring_nor_exports_its_size ( ) {
489+ let metrics = gateway_metrics:: MetricsCapture :: install ( ) ;
490+ let mut state = crate :: grpc:: test_support:: test_server_state ( ) . await ;
491+ // SQLite keeps the replica out even with a peer endpoint.
492+ Arc :: get_mut ( & mut state)
493+ . expect ( "test state is not shared yet" )
494+ . peer_endpoint = Some ( "http://gw-self:8081" . to_string ( ) ) ;
495+ let ( _shutdown_tx, shutdown_rx) = tokio:: sync:: watch:: channel ( false ) ;
496+
497+ assert ! ( spawn_membership_worker( state, MEMBER_REFRESH_INTERVAL , shutdown_rx) . is_none( ) ) ;
498+ assert_eq ! ( metrics. value( gateway_metrics:: GATEWAY_RING_MEMBERS ) , None ) ;
499+ }
500+
501+ #[ tokio:: test]
502+ #[ ignore = "requires a disposable PostgreSQL database in OPENSHELL_TEST_POSTGRES_URL; run mise run test:rust:postgres" ]
503+ async fn postgres_replica_without_peer_endpoint_neither_joins_the_ring_nor_exports_its_size ( ) {
504+ let metrics = gateway_metrics:: MetricsCapture :: install ( ) ;
505+ let schema =
506+ crate :: persistence:: test_postgres:: TestSchema :: create ( "membership_no_peer" ) . await ;
507+ let mut state =
508+ crate :: grpc:: test_support:: test_server_state_for_store ( schema. connect_store ( ) . await )
509+ . await ;
510+ // peer_endpoint is env-derived; pin it so the test does not depend on the environment.
511+ Arc :: get_mut ( & mut state)
512+ . expect ( "test state is not shared yet" )
513+ . peer_endpoint = None ;
514+ let ( _shutdown_tx, shutdown_rx) = tokio:: sync:: watch:: channel ( false ) ;
515+
516+ assert ! (
517+ spawn_membership_worker( state. clone( ) , MEMBER_REFRESH_INTERVAL , shutdown_rx) . is_none( )
518+ ) ;
519+ assert_eq ! ( metrics. value( gateway_metrics:: GATEWAY_RING_MEMBERS ) , None ) ;
520+ assert ! (
521+ !metrics
522+ . render( )
523+ . contains( gateway_metrics:: GATEWAY_RING_MEMBERS )
524+ ) ;
525+
526+ state. store . close ( ) . await ;
527+ schema. drop_schema ( ) . await ;
528+ }
529+
530+ #[ tokio:: test]
531+ #[ ignore = "requires a disposable PostgreSQL database in OPENSHELL_TEST_POSTGRES_URL; run mise run test:rust:postgres" ]
532+ async fn postgres_membership_worker_exports_ring_size ( ) {
533+ let metrics = gateway_metrics:: MetricsCapture :: install ( ) ;
534+ let schema =
535+ crate :: persistence:: test_postgres:: TestSchema :: create ( "membership_worker" ) . await ;
536+ let mut state =
537+ crate :: grpc:: test_support:: test_server_state_for_store ( schema. connect_store ( ) . await )
538+ . await ;
539+ {
540+ let state = Arc :: get_mut ( & mut state) . expect ( "test state is not shared yet" ) ;
541+ state. replica_id = "gw-self" . to_string ( ) ;
542+ state. peer_endpoint = Some ( "http://gw-self:8081" . to_string ( ) ) ;
543+ }
544+ GatewayMemberIndex :: new ( state. store . clone ( ) , MEMBER_TTL )
545+ . register ( "gw-other" , "http://gw-other:8081" )
546+ . await
547+ . unwrap ( ) ;
548+ let ( shutdown_tx, shutdown_rx) = tokio:: sync:: watch:: channel ( false ) ;
549+
550+ let worker = spawn_membership_worker ( state. clone ( ) , Duration :: from_millis ( 50 ) , shutdown_rx)
551+ . expect ( "a PostgreSQL replica with a peer endpoint joins the ring" ) ;
552+ assert_eq ! (
553+ metrics. value( gateway_metrics:: GATEWAY_RING_MEMBERS ) ,
554+ Some ( 0 )
555+ ) ;
556+
557+ tokio:: time:: timeout ( Duration :: from_secs ( 5 ) , async {
558+ while metrics. value ( gateway_metrics:: GATEWAY_RING_MEMBERS ) != Some ( 2 ) {
559+ tokio:: time:: sleep ( Duration :: from_millis ( 10 ) ) . await ;
560+ }
561+ } )
562+ . await
563+ . expect ( "the first refresh publishes both replicas" ) ;
564+ assert_eq ! ( state. gateway_ring. read( ) . unwrap( ) . replica_count( ) , 2 ) ;
565+
566+ // Leaving re-reads membership and keeps only the other replica.
567+ shutdown_tx. send ( true ) . unwrap ( ) ;
568+ tokio:: time:: timeout ( SHUTDOWN_MEMBERSHIP_TIMEOUT , worker)
569+ . await
570+ . expect ( "the worker leaves the ring" )
571+ . expect ( "membership worker task" ) ;
572+ let ring = state. gateway_ring . read ( ) . unwrap ( ) . clone ( ) ;
573+ assert_eq ! ( ring. owner_for( "sb-1" ) , Some ( "gw-other" ) ) ;
574+ assert_eq ! (
575+ metrics. value( gateway_metrics:: GATEWAY_RING_MEMBERS ) ,
576+ Some ( 1 )
577+ ) ;
578+
579+ state. store . close ( ) . await ;
580+ schema. drop_schema ( ) . await ;
581+ }
582+
353583 #[ tokio:: test]
354584 async fn register_is_idempotent_and_updates_the_endpoint ( ) {
355585 let index = test_index ( MEMBER_TTL ) . await ;
0 commit comments