Skip to content

Commit 6af3db2

Browse files
committed
feat(server): export gateway ring membership size
Each replica builds its placement ring from live membership, but nothing showed how many replicas a given replica had in its ring, so a replica that dropped out of its peers' rings, or a ring cleared after failed refreshes, went unnoticed. openshell_server_gateway_ring_members now reports the replicas in this replica's ring. Every ring write goes through one helper that records the size under the same lock: the periodic refresh, leaving the ring at shutdown, and clearing a stale ring. Only replicas that join the ring export the gauge, so SQLite gateways and replicas without a peer endpoint show no series rather than a misleading 0. The HA guide's Monitor Capacity table gains a Session placement row for all three placement metrics. Part of #4223 Closes #4688 Signed-off-by: Emilien Macchi <[email protected]>
1 parent f37e2c8 commit 6af3db2

6 files changed

Lines changed: 285 additions & 19 deletions

File tree

‎crates/openshell-server/src/gateway_members.rs‎

Lines changed: 238 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
//! kept in the store rather than read from Kubernetes so it works for any
99
//! multi-replica deployment and adds no API server load.
1010
11+
use crate::gateway_metrics;
1112
use crate::gateway_ring::GatewayRing;
1213
use crate::persistence::{PersistenceError, PersistenceResult, Store, WriteCondition};
1314
use openshell_core::time::now_ms;
@@ -186,6 +187,10 @@ pub fn spawn_membership_worker(
186187

187188
let index = GatewayMemberIndex::new(state.store.clone(), MEMBER_TTL);
188189
let replica_id = state.replica_id.clone();
190+
// Export the ring's size now (empty until the first refresh), so only
191+
// replicas that join the ring carry the gauge, and they carry it from the
192+
// start.
193+
publish_ring(&state, GatewayRing::clone);
189194

190195
Some(tokio::spawn(async move {
191196
let mut ticker = tokio::time::interval(interval);
@@ -216,9 +221,7 @@ pub fn spawn_membership_worker(
216221
.into_iter()
217222
.map(|member| (member.replica_id, member.peer_endpoint))
218223
.collect();
219-
if let Ok(mut slot) = state.gateway_ring.write() {
220-
*slot = ring;
221-
}
224+
publish_ring(&state, |_| ring);
222225
if let Ok(mut slot) = state.gateway_peers.write() {
223226
*slot = peers;
224227
}
@@ -249,24 +252,18 @@ async fn leave_ring(state: &crate::ServerState, index: &GatewayMemberIndex, repl
249252
.into_iter()
250253
.map(|member| (member.replica_id, member.peer_endpoint))
251254
.collect();
252-
if let Ok(mut slot) = state.gateway_ring.write() {
253-
*slot = ring.without(replica_id);
254-
}
255+
publish_ring(state, |_| ring.without(replica_id));
255256
if let Ok(mut slot) = state.gateway_peers.write() {
256257
*slot = peers;
257258
}
258259
}
259260
Ok(Err(error)) => {
260261
tracing::warn!(%error, "gateway membership: shutdown ring refresh failed");
261-
if let Ok(mut slot) = state.gateway_ring.write() {
262-
*slot = slot.without(replica_id);
263-
}
262+
publish_ring(state, |ring| ring.without(replica_id));
264263
}
265264
Err(_) => {
266265
tracing::warn!("gateway membership: shutdown ring refresh timed out");
267-
if let Ok(mut slot) = state.gateway_ring.write() {
268-
*slot = slot.without(replica_id);
269-
}
266+
publish_ring(state, |ring| ring.without(replica_id));
270267
}
271268
}
272269
match tokio::time::timeout(SHUTDOWN_STORE_TIMEOUT, index.deregister(replica_id)).await {
@@ -280,14 +277,26 @@ fn clear_ring_if_stale(state: &crate::ServerState, last_refreshed: tokio::time::
280277
if last_refreshed.elapsed() <= MEMBER_TTL {
281278
return;
282279
}
283-
if let Ok(mut slot) = state.gateway_ring.write() {
284-
*slot = GatewayRing::default();
285-
}
280+
publish_ring(state, |_| GatewayRing::default());
286281
if let Ok(mut slot) = state.gateway_peers.write() {
287282
slot.clear();
288283
}
289284
}
290285

286+
/// Replace this replica's placement ring with `next(current ring)` and record
287+
/// its size as `openshell_server_gateway_ring_members` under the same write
288+
/// lock.
289+
///
290+
/// This is the only sanctioned non-test writer of `ServerState::gateway_ring`:
291+
/// writing the slot anywhere else lets the gauge drift from the ring. A
292+
/// poisoned lock leaves both unchanged.
293+
fn publish_ring(state: &crate::ServerState, next: impl FnOnce(&GatewayRing) -> GatewayRing) {
294+
if let Ok(mut slot) = state.gateway_ring.write() {
295+
*slot = next(&slot);
296+
gateway_metrics::record_gateway_ring_members(slot.replica_count());
297+
}
298+
}
299+
291300
#[cfg(test)]
292301
mod tests {
293302
use super::*;
@@ -314,6 +323,7 @@ mod tests {
314323

315324
#[tokio::test]
316325
async fn leaving_points_ring_at_new_replicas_and_not_departed_ones() {
326+
let metrics = gateway_metrics::MetricsCapture::install();
317327
let state = crate::grpc::test_support::test_server_state().await;
318328
let index = GatewayMemberIndex::new(state.store.clone(), MEMBER_TTL);
319329
index.register("gw-old", "https://gw-old").await.unwrap();
@@ -331,6 +341,10 @@ mod tests {
331341

332342
let ring = state.gateway_ring.read().unwrap().clone();
333343
assert_eq!(ring.owner_for("sb-1"), Some("gw-new"));
344+
assert_eq!(
345+
metrics.value(gateway_metrics::GATEWAY_RING_MEMBERS),
346+
Some(1)
347+
);
334348
assert_eq!(
335349
state
336350
.gateway_peers
@@ -350,6 +364,215 @@ mod tests {
350364
assert_eq!(remaining, vec!["gw-new"]);
351365
}
352366

367+
#[tokio::test]
368+
async fn leaving_without_a_membership_read_keeps_the_cached_ring_without_this_replica() {
369+
let metrics = gateway_metrics::MetricsCapture::install();
370+
let state = crate::grpc::test_support::test_server_state().await;
371+
publish_ring(&state, |_| GatewayRing::new(["gw-a", "gw-stopping"]));
372+
// Only the index's store is closed, so the shutdown membership read
373+
// fails while the state's store stays open.
374+
let store = Arc::new(crate::persistence::test_store().await);
375+
store.close().await;
376+
let index = GatewayMemberIndex::new(store, MEMBER_TTL);
377+
378+
leave_ring(&state, &index, "gw-stopping").await;
379+
380+
let ring = state.gateway_ring.read().unwrap().clone();
381+
assert_eq!(ring.owner_for("sb-1"), Some("gw-a"));
382+
assert_eq!(
383+
metrics.value(gateway_metrics::GATEWAY_RING_MEMBERS),
384+
Some(1)
385+
);
386+
}
387+
388+
#[tokio::test]
389+
async fn leaving_after_a_timed_out_membership_read_keeps_the_cached_ring_without_this_replica()
390+
{
391+
let metrics = gateway_metrics::MetricsCapture::install();
392+
let state = crate::grpc::test_support::test_server_state().await;
393+
publish_ring(&state, |_| GatewayRing::new(["gw-a", "gw-stopping"]));
394+
// The listener accepts TCP but never answers the PostgreSQL handshake, so
395+
// the shutdown read hangs past SHUTDOWN_STORE_TIMEOUT instead of failing.
396+
let silent = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
397+
let url = format!(
398+
"postgres://[email protected]:{}/openshell",
399+
silent.local_addr().unwrap().port()
400+
);
401+
let index = GatewayMemberIndex::new(
402+
Arc::new(Store::Postgres(
403+
crate::persistence::PostgresStore::connect_lazy_for_tests(&url, 1),
404+
)),
405+
MEMBER_TTL,
406+
);
407+
408+
// The paused clock jumps to each timeout while the socket stays silent.
409+
tokio::time::pause();
410+
let started = tokio::time::Instant::now();
411+
leave_ring(&state, &index, "gw-stopping").await;
412+
// Both the read and the deregister must hit their timeouts; a read that
413+
// failed fast would take the Ok(Err) branch and finish sooner.
414+
assert!(
415+
started.elapsed() >= SHUTDOWN_STORE_TIMEOUT * 2,
416+
"the membership read must time out, not fail fast"
417+
);
418+
419+
let ring = state.gateway_ring.read().unwrap().clone();
420+
assert_eq!(ring.replica_count(), 1);
421+
assert_eq!(ring.owner_for("sb-1"), Some("gw-a"));
422+
assert_eq!(
423+
metrics.value(gateway_metrics::GATEWAY_RING_MEMBERS),
424+
Some(1)
425+
);
426+
drop(silent);
427+
}
428+
429+
#[tokio::test]
430+
async fn stale_ring_is_cleared_only_after_the_member_ttl() {
431+
let metrics = gateway_metrics::MetricsCapture::install();
432+
let state = crate::grpc::test_support::test_server_state().await;
433+
publish_ring(&state, |_| GatewayRing::new(["gw-a", "gw-b"]));
434+
*state.gateway_peers.write().unwrap() = HashMap::from([
435+
("gw-a".to_string(), "https://gw-a".to_string()),
436+
("gw-b".to_string(), "https://gw-b".to_string()),
437+
]);
438+
assert_eq!(
439+
metrics.value(gateway_metrics::GATEWAY_RING_MEMBERS),
440+
Some(2)
441+
);
442+
443+
clear_ring_if_stale(&state, tokio::time::Instant::now());
444+
assert_eq!(state.gateway_ring.read().unwrap().replica_count(), 2);
445+
assert_eq!(state.gateway_peers.read().unwrap().len(), 2);
446+
assert_eq!(
447+
metrics.value(gateway_metrics::GATEWAY_RING_MEMBERS),
448+
Some(2)
449+
);
450+
451+
let last_refreshed = tokio::time::Instant::now()
452+
.checked_sub(MEMBER_TTL + Duration::from_millis(1))
453+
.expect("the monotonic clock is past one member TTL");
454+
clear_ring_if_stale(&state, last_refreshed);
455+
assert!(state.gateway_ring.read().unwrap().is_empty());
456+
assert!(state.gateway_peers.read().unwrap().is_empty());
457+
assert_eq!(
458+
metrics.value(gateway_metrics::GATEWAY_RING_MEMBERS),
459+
Some(0)
460+
);
461+
}
462+
463+
#[tokio::test]
464+
async fn poisoned_ring_lock_publishes_no_ring_size() {
465+
let metrics = gateway_metrics::MetricsCapture::install();
466+
let state = crate::grpc::test_support::test_server_state().await;
467+
let ring = Arc::clone(&state.gateway_ring);
468+
let _ = std::thread::spawn(move || {
469+
let _guard = ring.write().unwrap();
470+
panic!("poison the placement ring lock");
471+
})
472+
.join();
473+
assert!(state.gateway_ring.is_poisoned());
474+
475+
publish_ring(&state, |_| GatewayRing::new(["gw-a", "gw-b"]));
476+
477+
assert_eq!(metrics.value(gateway_metrics::GATEWAY_RING_MEMBERS), None);
478+
}
479+
480+
#[tokio::test]
481+
async fn single_replica_store_neither_joins_the_ring_nor_exports_its_size() {
482+
let metrics = gateway_metrics::MetricsCapture::install();
483+
let mut state = crate::grpc::test_support::test_server_state().await;
484+
// SQLite keeps the replica out even with a peer endpoint.
485+
Arc::get_mut(&mut state)
486+
.expect("test state is not shared yet")
487+
.peer_endpoint = Some("http://gw-self:8081".to_string());
488+
let (_shutdown_tx, shutdown_rx) = tokio::sync::watch::channel(false);
489+
490+
assert!(spawn_membership_worker(state, MEMBER_REFRESH_INTERVAL, shutdown_rx).is_none());
491+
assert_eq!(metrics.value(gateway_metrics::GATEWAY_RING_MEMBERS), None);
492+
}
493+
494+
#[tokio::test]
495+
#[ignore = "requires a disposable PostgreSQL database in OPENSHELL_TEST_POSTGRES_URL; run mise run test:rust:postgres"]
496+
async fn postgres_replica_without_peer_endpoint_neither_joins_the_ring_nor_exports_its_size() {
497+
let metrics = gateway_metrics::MetricsCapture::install();
498+
let schema =
499+
crate::persistence::test_postgres::TestSchema::create("membership_no_peer").await;
500+
let mut state =
501+
crate::grpc::test_support::test_server_state_for_store(schema.connect_store().await)
502+
.await;
503+
// peer_endpoint is env-derived; pin it so the test does not depend on the environment.
504+
Arc::get_mut(&mut state)
505+
.expect("test state is not shared yet")
506+
.peer_endpoint = None;
507+
let (_shutdown_tx, shutdown_rx) = tokio::sync::watch::channel(false);
508+
509+
assert!(
510+
spawn_membership_worker(state.clone(), MEMBER_REFRESH_INTERVAL, shutdown_rx).is_none()
511+
);
512+
assert_eq!(metrics.value(gateway_metrics::GATEWAY_RING_MEMBERS), None);
513+
assert!(
514+
!metrics
515+
.render()
516+
.contains(gateway_metrics::GATEWAY_RING_MEMBERS)
517+
);
518+
519+
state.store.close().await;
520+
schema.drop_schema().await;
521+
}
522+
523+
#[tokio::test]
524+
#[ignore = "requires a disposable PostgreSQL database in OPENSHELL_TEST_POSTGRES_URL; run mise run test:rust:postgres"]
525+
async fn postgres_membership_worker_exports_ring_size() {
526+
let metrics = gateway_metrics::MetricsCapture::install();
527+
let schema =
528+
crate::persistence::test_postgres::TestSchema::create("membership_worker").await;
529+
let mut state =
530+
crate::grpc::test_support::test_server_state_for_store(schema.connect_store().await)
531+
.await;
532+
{
533+
let state = Arc::get_mut(&mut state).expect("test state is not shared yet");
534+
state.replica_id = "gw-self".to_string();
535+
state.peer_endpoint = Some("http://gw-self:8081".to_string());
536+
}
537+
GatewayMemberIndex::new(state.store.clone(), MEMBER_TTL)
538+
.register("gw-other", "http://gw-other:8081")
539+
.await
540+
.unwrap();
541+
let (shutdown_tx, shutdown_rx) = tokio::sync::watch::channel(false);
542+
543+
let worker = spawn_membership_worker(state.clone(), Duration::from_millis(50), shutdown_rx)
544+
.expect("a PostgreSQL replica with a peer endpoint joins the ring");
545+
assert_eq!(
546+
metrics.value(gateway_metrics::GATEWAY_RING_MEMBERS),
547+
Some(0)
548+
);
549+
550+
tokio::time::timeout(Duration::from_secs(5), async {
551+
while metrics.value(gateway_metrics::GATEWAY_RING_MEMBERS) != Some(2) {
552+
tokio::time::sleep(Duration::from_millis(10)).await;
553+
}
554+
})
555+
.await
556+
.expect("the first refresh publishes both replicas");
557+
assert_eq!(state.gateway_ring.read().unwrap().replica_count(), 2);
558+
559+
// Leaving re-reads membership and keeps only the other replica.
560+
shutdown_tx.send(true).unwrap();
561+
tokio::time::timeout(SHUTDOWN_MEMBERSHIP_TIMEOUT, worker)
562+
.await
563+
.expect("the worker leaves the ring")
564+
.expect("membership worker task");
565+
let ring = state.gateway_ring.read().unwrap().clone();
566+
assert_eq!(ring.owner_for("sb-1"), Some("gw-other"));
567+
assert_eq!(
568+
metrics.value(gateway_metrics::GATEWAY_RING_MEMBERS),
569+
Some(1)
570+
);
571+
572+
state.store.close().await;
573+
schema.drop_schema().await;
574+
}
575+
353576
#[tokio::test]
354577
async fn register_is_idempotent_and_updates_the_endpoint() {
355578
let index = test_index(MEMBER_TTL).await;

‎crates/openshell-server/src/gateway_metrics.rs‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ pub const MUTATION_LOCK_CONNECTIONS_IN_USE: &str =
2929
"openshell_server_mutation_lock_connections_in_use";
3030
pub const MUTATION_LOCK_CONNECTIONS_CAPACITY: &str =
3131
"openshell_server_mutation_lock_connections_capacity";
32+
pub const GATEWAY_RING_MEMBERS: &str = "openshell_server_gateway_ring_members";
3233
// Counters
3334
pub const RELAY_REJECTED_TOTAL: &str = "openshell_server_relay_rejected_total";
3435
pub const RELAY_EXPIRED_TOTAL: &str = "openshell_server_relay_expired_total";
@@ -297,6 +298,13 @@ pub fn describe_and_initialize(relay: RelayCapacity, mutation_lock_connections:
297298
Unit::Count,
298299
"Maximum pending relay channels on one gateway replica."
299300
);
301+
// Not zero-initialized: only replicas that join the placement ring record it, so SQLite
302+
// gateways and replicas without a peer endpoint export nothing instead of a misleading 0.
303+
describe_gauge!(
304+
GATEWAY_RING_MEMBERS,
305+
Unit::Count,
306+
"Gateway replicas in this replica's placement ring, including itself."
307+
);
300308
describe_counter!(
301309
ROUTED_REQUEST_ATTEMPTS_TOTAL,
302310
Unit::Count,
@@ -488,6 +496,11 @@ pub fn record_off_ring_admission() {
488496
counter!(SUPERVISOR_SESSION_OFF_RING_ADMISSIONS_TOTAL).increment(1);
489497
}
490498

499+
/// Replicas in this replica's placement ring. Recorded together with each ring write.
500+
pub fn record_gateway_ring_members(count: usize) {
501+
gauge!(GATEWAY_RING_MEMBERS).set(count_as_f64(count));
502+
}
503+
491504
/// `count` pending relays were dropped unclaimed (late claim or reaper).
492505
pub fn record_relay_expired(count: usize) {
493506
if count > 0 {
@@ -750,6 +763,9 @@ mod tests {
750763
] {
751764
assert_eq!(metrics.value(series), Some(expected), "{series}");
752765
}
766+
// Described only, so it renders nothing, not even HELP, until the ring is published.
767+
assert_eq!(metrics.value(GATEWAY_RING_MEMBERS), None);
768+
assert!(!metrics.render().contains(GATEWAY_RING_MEMBERS));
753769
for kind in ["ssh", "tcp"] {
754770
for route in ["local", "peer"] {
755771
let series = format!(
@@ -793,6 +809,27 @@ mod tests {
793809
);
794810
}
795811

812+
#[test]
813+
fn placement_metrics_export_help_and_type_once_present() {
814+
let metrics = MetricsCapture::install();
815+
describe_and_initialize(RelayCapacity { per_replica: 256 }, None);
816+
record_gateway_ring_members(3);
817+
818+
let rendered = metrics.render();
819+
for (name, kind) in [
820+
(SUPERVISOR_SESSION_REDIRECTS_TOTAL, "counter"),
821+
(SUPERVISOR_SESSION_OFF_RING_ADMISSIONS_TOTAL, "counter"),
822+
(GATEWAY_RING_MEMBERS, "gauge"),
823+
] {
824+
assert!(rendered.contains(&format!("# HELP {name} ")), "{name} HELP");
825+
assert!(
826+
rendered.contains(&format!("# TYPE {name} {kind}")),
827+
"{name} TYPE"
828+
);
829+
}
830+
assert_eq!(metrics.value(GATEWAY_RING_MEMBERS), Some(3));
831+
}
832+
796833
#[test]
797834
fn routed_attempts_have_seven_bounded_success_series_and_keep_counts_on_initialize() {
798835
let metrics = MetricsCapture::install();

0 commit comments

Comments
 (0)