Skip to content

Commit 5e82f09

Browse files
committed
feat(server): export gateway ring membership size
Each replica builds its placement ring from live membership, but nothing showed how many replicas a given replica had in its ring, so a replica that dropped out of its peers' rings, or a ring cleared after failed refreshes, went unnoticed. openshell_server_gateway_ring_members now reports the replicas in this replica's ring. Every ring write goes through one helper that records the size under the same lock: the periodic refresh, leaving the ring at shutdown, and clearing a stale ring. Only replicas that join the ring export the gauge, so SQLite gateways and replicas without a peer endpoint show no series rather than a misleading 0. The HA guide's Monitor Capacity table gains a Session placement row scoped to gateways that place supervisor sessions by ring: under the multi-tenant topology the shared supervisor's host moves sessions, and that section says the placement metrics do not reflect those moves. Part of #4223 Closes #4688 Signed-off-by: Emilien Macchi <[email protected]>
1 parent 0af14d4 commit 5e82f09

6 files changed

Lines changed: 285 additions & 33 deletions

File tree

‎crates/openshell-server/src/gateway_members.rs‎

Lines changed: 245 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
//! kept in the store rather than read from Kubernetes so it works for any
99
//! multi-replica deployment and adds no API server load.
1010
11+
use crate::gateway_metrics;
1112
use crate::gateway_ring::GatewayRing;
1213
use crate::persistence::{PersistenceError, PersistenceResult, Store, WriteCondition};
1314
use openshell_core::time::now_ms;
@@ -186,6 +187,10 @@ pub fn spawn_membership_worker(
186187

187188
let index = GatewayMemberIndex::new(state.store.clone(), MEMBER_TTL);
188189
let replica_id = state.replica_id.clone();
190+
// Export the ring's size now (empty until the first refresh), so only
191+
// replicas that join the ring carry the gauge, and they carry it from the
192+
// start.
193+
publish_ring(&state, GatewayRing::clone);
189194

190195
Some(tokio::spawn(async move {
191196
let mut ticker = tokio::time::interval(interval);
@@ -216,9 +221,7 @@ pub fn spawn_membership_worker(
216221
.into_iter()
217222
.map(|member| (member.replica_id, member.peer_endpoint))
218223
.collect();
219-
if let Ok(mut slot) = state.gateway_ring.write() {
220-
*slot = ring;
221-
}
224+
publish_ring(&state, |_| ring);
222225
if let Ok(mut slot) = state.gateway_peers.write() {
223226
*slot = peers;
224227
}
@@ -249,24 +252,18 @@ async fn leave_ring(state: &crate::ServerState, index: &GatewayMemberIndex, repl
249252
.into_iter()
250253
.map(|member| (member.replica_id, member.peer_endpoint))
251254
.collect();
252-
if let Ok(mut slot) = state.gateway_ring.write() {
253-
*slot = ring.without(replica_id);
254-
}
255+
publish_ring(state, |_| ring.without(replica_id));
255256
if let Ok(mut slot) = state.gateway_peers.write() {
256257
*slot = peers;
257258
}
258259
}
259260
Ok(Err(error)) => {
260261
tracing::warn!(%error, "gateway membership: shutdown ring refresh failed");
261-
if let Ok(mut slot) = state.gateway_ring.write() {
262-
*slot = slot.without(replica_id);
263-
}
262+
publish_ring(state, |ring| ring.without(replica_id));
264263
}
265264
Err(_) => {
266265
tracing::warn!("gateway membership: shutdown ring refresh timed out");
267-
if let Ok(mut slot) = state.gateway_ring.write() {
268-
*slot = slot.without(replica_id);
269-
}
266+
publish_ring(state, |ring| ring.without(replica_id));
270267
}
271268
}
272269
match tokio::time::timeout(SHUTDOWN_STORE_TIMEOUT, index.deregister(replica_id)).await {
@@ -280,14 +277,26 @@ fn clear_ring_if_stale(state: &crate::ServerState, last_refreshed: tokio::time::
280277
if last_refreshed.elapsed() <= MEMBER_TTL {
281278
return;
282279
}
283-
if let Ok(mut slot) = state.gateway_ring.write() {
284-
*slot = GatewayRing::default();
285-
}
280+
publish_ring(state, |_| GatewayRing::default());
286281
if let Ok(mut slot) = state.gateway_peers.write() {
287282
slot.clear();
288283
}
289284
}
290285

286+
/// Replace this replica's placement ring with `next(current ring)` and record
287+
/// its size as `openshell_server_gateway_ring_members` under the same write
288+
/// lock.
289+
///
290+
/// This is the only sanctioned non-test writer of `ServerState::gateway_ring`:
291+
/// writing the slot anywhere else lets the gauge drift from the ring. A
292+
/// poisoned lock leaves both unchanged.
293+
fn publish_ring(state: &crate::ServerState, next: impl FnOnce(&GatewayRing) -> GatewayRing) {
294+
if let Ok(mut slot) = state.gateway_ring.write() {
295+
*slot = next(&slot);
296+
gateway_metrics::record_gateway_ring_members(slot.replica_count());
297+
}
298+
}
299+
291300
#[cfg(test)]
292301
mod tests {
293302
use super::*;
@@ -314,6 +323,7 @@ mod tests {
314323

315324
#[tokio::test]
316325
async fn leaving_points_ring_at_new_replicas_and_not_departed_ones() {
326+
let metrics = gateway_metrics::MetricsCapture::install();
317327
let state = crate::grpc::test_support::test_server_state().await;
318328
let index = GatewayMemberIndex::new(state.store.clone(), MEMBER_TTL);
319329
index.register("gw-old", "https://gw-old").await.unwrap();
@@ -331,6 +341,10 @@ mod tests {
331341

332342
let ring = state.gateway_ring.read().unwrap().clone();
333343
assert_eq!(ring.owner_for("sb-1"), Some("gw-new"));
344+
assert_eq!(
345+
metrics.value(gateway_metrics::GATEWAY_RING_MEMBERS),
346+
Some(1)
347+
);
334348
assert_eq!(
335349
state
336350
.gateway_peers
@@ -350,6 +364,222 @@ mod tests {
350364
assert_eq!(remaining, vec!["gw-new"]);
351365
}
352366

367+
#[tokio::test]
368+
async fn leaving_without_a_membership_read_keeps_the_cached_ring_without_this_replica() {
369+
let metrics = gateway_metrics::MetricsCapture::install();
370+
let state = crate::grpc::test_support::test_server_state().await;
371+
publish_ring(&state, |_| GatewayRing::new(["gw-a", "gw-stopping"]));
372+
// Only the index's store is closed, so the shutdown membership read
373+
// fails while the state's store stays open.
374+
let store = Arc::new(crate::persistence::test_store().await);
375+
store.close().await;
376+
let index = GatewayMemberIndex::new(store, MEMBER_TTL);
377+
378+
leave_ring(&state, &index, "gw-stopping").await;
379+
380+
let ring = state.gateway_ring.read().unwrap().clone();
381+
assert_eq!(ring.owner_for("sb-1"), Some("gw-a"));
382+
assert_eq!(
383+
metrics.value(gateway_metrics::GATEWAY_RING_MEMBERS),
384+
Some(1)
385+
);
386+
}
387+
388+
#[tokio::test]
389+
async fn leaving_after_a_timed_out_membership_read_keeps_the_cached_ring_without_this_replica()
390+
{
391+
let metrics = gateway_metrics::MetricsCapture::install();
392+
let state = crate::grpc::test_support::test_server_state().await;
393+
publish_ring(&state, |_| GatewayRing::new(["gw-a", "gw-stopping"]));
394+
// The listener accepts TCP but never answers the PostgreSQL handshake, so
395+
// the shutdown read hangs past SHUTDOWN_STORE_TIMEOUT instead of failing.
396+
let silent = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
397+
let url = format!(
398+
"postgres://[email protected]:{}/openshell",
399+
silent.local_addr().unwrap().port()
400+
);
401+
let index = GatewayMemberIndex::new(
402+
Arc::new(Store::Postgres(
403+
crate::persistence::PostgresStore::connect_lazy_for_tests(&url, 1),
404+
)),
405+
MEMBER_TTL,
406+
);
407+
408+
// The paused clock jumps to each timeout while the socket stays silent.
409+
tokio::time::pause();
410+
let started = tokio::time::Instant::now();
411+
leave_ring(&state, &index, "gw-stopping").await;
412+
// Both the read and the deregister must hit their timeouts; a read that
413+
// failed fast would take the Ok(Err) branch and finish sooner.
414+
assert!(
415+
started.elapsed() >= SHUTDOWN_STORE_TIMEOUT * 2,
416+
"the membership read must time out, not fail fast"
417+
);
418+
419+
let ring = state.gateway_ring.read().unwrap().clone();
420+
assert_eq!(ring.replica_count(), 1);
421+
assert_eq!(ring.owner_for("sb-1"), Some("gw-a"));
422+
assert_eq!(
423+
metrics.value(gateway_metrics::GATEWAY_RING_MEMBERS),
424+
Some(1)
425+
);
426+
drop(silent);
427+
}
428+
429+
#[tokio::test]
430+
async fn stale_ring_is_cleared_only_after_the_member_ttl() {
431+
let metrics = gateway_metrics::MetricsCapture::install();
432+
let state = crate::grpc::test_support::test_server_state().await;
433+
publish_ring(&state, |_| GatewayRing::new(["gw-a", "gw-b"]));
434+
*state.gateway_peers.write().unwrap() = HashMap::from([
435+
("gw-a".to_string(), "https://gw-a".to_string()),
436+
("gw-b".to_string(), "https://gw-b".to_string()),
437+
]);
438+
assert_eq!(
439+
metrics.value(gateway_metrics::GATEWAY_RING_MEMBERS),
440+
Some(2)
441+
);
442+
443+
clear_ring_if_stale(&state, tokio::time::Instant::now());
444+
assert_eq!(state.gateway_ring.read().unwrap().replica_count(), 2);
445+
assert_eq!(state.gateway_peers.read().unwrap().len(), 2);
446+
assert_eq!(
447+
metrics.value(gateway_metrics::GATEWAY_RING_MEMBERS),
448+
Some(2)
449+
);
450+
451+
let last_refreshed = tokio::time::Instant::now()
452+
.checked_sub(MEMBER_TTL + Duration::from_millis(1))
453+
.expect("the monotonic clock is past one member TTL");
454+
clear_ring_if_stale(&state, last_refreshed);
455+
assert!(state.gateway_ring.read().unwrap().is_empty());
456+
assert!(state.gateway_peers.read().unwrap().is_empty());
457+
assert_eq!(
458+
metrics.value(gateway_metrics::GATEWAY_RING_MEMBERS),
459+
Some(0)
460+
);
461+
}
462+
463+
#[tokio::test]
464+
async fn poisoned_ring_lock_publishes_no_ring_size() {
465+
let metrics = gateway_metrics::MetricsCapture::install();
466+
let state = crate::grpc::test_support::test_server_state().await;
467+
let ring = Arc::clone(&state.gateway_ring);
468+
let poisoned = std::thread::spawn(move || {
469+
std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
470+
let _guard = ring.write().unwrap();
471+
// `resume_unwind` drops the guard and poisons the lock without
472+
// running the panic hook, keeping the expected panic out of
473+
// the test output.
474+
std::panic::resume_unwind(Box::new("poison the placement ring lock"));
475+
}))
476+
})
477+
.join()
478+
.expect("the poisoning thread should catch its own unwind");
479+
assert!(poisoned.is_err());
480+
assert!(state.gateway_ring.is_poisoned());
481+
482+
publish_ring(&state, |_| GatewayRing::new(["gw-a", "gw-b"]));
483+
484+
assert_eq!(metrics.value(gateway_metrics::GATEWAY_RING_MEMBERS), None);
485+
}
486+
487+
#[tokio::test]
488+
async fn single_replica_store_neither_joins_the_ring_nor_exports_its_size() {
489+
let metrics = gateway_metrics::MetricsCapture::install();
490+
let mut state = crate::grpc::test_support::test_server_state().await;
491+
// SQLite keeps the replica out even with a peer endpoint.
492+
Arc::get_mut(&mut state)
493+
.expect("test state is not shared yet")
494+
.peer_endpoint = Some("http://gw-self:8081".to_string());
495+
let (_shutdown_tx, shutdown_rx) = tokio::sync::watch::channel(false);
496+
497+
assert!(spawn_membership_worker(state, MEMBER_REFRESH_INTERVAL, shutdown_rx).is_none());
498+
assert_eq!(metrics.value(gateway_metrics::GATEWAY_RING_MEMBERS), None);
499+
}
500+
501+
#[tokio::test]
502+
#[ignore = "requires a disposable PostgreSQL database in OPENSHELL_TEST_POSTGRES_URL; run mise run test:rust:postgres"]
503+
async fn postgres_replica_without_peer_endpoint_neither_joins_the_ring_nor_exports_its_size() {
504+
let metrics = gateway_metrics::MetricsCapture::install();
505+
let schema =
506+
crate::persistence::test_postgres::TestSchema::create("membership_no_peer").await;
507+
let mut state =
508+
crate::grpc::test_support::test_server_state_for_store(schema.connect_store().await)
509+
.await;
510+
// peer_endpoint is env-derived; pin it so the test does not depend on the environment.
511+
Arc::get_mut(&mut state)
512+
.expect("test state is not shared yet")
513+
.peer_endpoint = None;
514+
let (_shutdown_tx, shutdown_rx) = tokio::sync::watch::channel(false);
515+
516+
assert!(
517+
spawn_membership_worker(state.clone(), MEMBER_REFRESH_INTERVAL, shutdown_rx).is_none()
518+
);
519+
assert_eq!(metrics.value(gateway_metrics::GATEWAY_RING_MEMBERS), None);
520+
assert!(
521+
!metrics
522+
.render()
523+
.contains(gateway_metrics::GATEWAY_RING_MEMBERS)
524+
);
525+
526+
state.store.close().await;
527+
schema.drop_schema().await;
528+
}
529+
530+
#[tokio::test]
531+
#[ignore = "requires a disposable PostgreSQL database in OPENSHELL_TEST_POSTGRES_URL; run mise run test:rust:postgres"]
532+
async fn postgres_membership_worker_exports_ring_size() {
533+
let metrics = gateway_metrics::MetricsCapture::install();
534+
let schema =
535+
crate::persistence::test_postgres::TestSchema::create("membership_worker").await;
536+
let mut state =
537+
crate::grpc::test_support::test_server_state_for_store(schema.connect_store().await)
538+
.await;
539+
{
540+
let state = Arc::get_mut(&mut state).expect("test state is not shared yet");
541+
state.replica_id = "gw-self".to_string();
542+
state.peer_endpoint = Some("http://gw-self:8081".to_string());
543+
}
544+
GatewayMemberIndex::new(state.store.clone(), MEMBER_TTL)
545+
.register("gw-other", "http://gw-other:8081")
546+
.await
547+
.unwrap();
548+
let (shutdown_tx, shutdown_rx) = tokio::sync::watch::channel(false);
549+
550+
let worker = spawn_membership_worker(state.clone(), Duration::from_millis(50), shutdown_rx)
551+
.expect("a PostgreSQL replica with a peer endpoint joins the ring");
552+
assert_eq!(
553+
metrics.value(gateway_metrics::GATEWAY_RING_MEMBERS),
554+
Some(0)
555+
);
556+
557+
tokio::time::timeout(Duration::from_secs(5), async {
558+
while metrics.value(gateway_metrics::GATEWAY_RING_MEMBERS) != Some(2) {
559+
tokio::time::sleep(Duration::from_millis(10)).await;
560+
}
561+
})
562+
.await
563+
.expect("the first refresh publishes both replicas");
564+
assert_eq!(state.gateway_ring.read().unwrap().replica_count(), 2);
565+
566+
// Leaving re-reads membership and keeps only the other replica.
567+
shutdown_tx.send(true).unwrap();
568+
tokio::time::timeout(SHUTDOWN_MEMBERSHIP_TIMEOUT, worker)
569+
.await
570+
.expect("the worker leaves the ring")
571+
.expect("membership worker task");
572+
let ring = state.gateway_ring.read().unwrap().clone();
573+
assert_eq!(ring.owner_for("sb-1"), Some("gw-other"));
574+
assert_eq!(
575+
metrics.value(gateway_metrics::GATEWAY_RING_MEMBERS),
576+
Some(1)
577+
);
578+
579+
state.store.close().await;
580+
schema.drop_schema().await;
581+
}
582+
353583
#[tokio::test]
354584
async fn register_is_idempotent_and_updates_the_endpoint() {
355585
let index = test_index(MEMBER_TTL).await;

0 commit comments

Comments
 (0)