Skip to content

Commit a1143e2

Browse files
committed
feat(server): export mutation lock errors and lock connections in use
A mutation lock acquisition that failed without timing out, such as a lock connection PostgreSQL never opened or a failed lock statement, only logged a warning, and readiness stays healthy because it pings the data pool. Count those failures in openshell_server_mutation_lock_errors_total{scope}, kept apart from timeouts because they point at PostgreSQL capacity or connectivity rather than contention. Export openshell_server_mutation_lock_connections_in_use as a gauge slot held by each checked-out lock connection, next to openshell_server_mutation_lock_connections_capacity, which reads the configured lock pool size; both exist only with PostgreSQL. The metrics catalog, the HA guide's signals and PromQL examples, and the debug skill cover the new series. Part of NVIDIA#3528 Signed-off-by: Emilien Macchi <[email protected]>
1 parent d874f84 commit a1143e2

10 files changed

Lines changed: 320 additions & 43 deletions

File tree

‎crates/openshell-server/src/compute/mutation_guard.rs‎

Lines changed: 21 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -295,7 +295,9 @@ impl ComputeRuntime {
295295
/// [`PersistenceError::LockTimeout`]: the mutation lock was not acquired,
296296
/// so the caller's guarded writes did not run. A lock connection that
297297
/// `PostgreSQL` does not open with at least `LOCK_CONNECTION_MIN_BUDGET`
298-
/// left fails with [`PersistenceError::Database`].
298+
/// left fails with [`PersistenceError::Database`]. Timeouts count in
299+
/// `openshell_server_mutation_lock_timeouts_total` and other failures in
300+
/// `openshell_server_mutation_lock_errors_total`.
299301
pub(crate) async fn mutation_guard(
300302
&self,
301303
scope: MutationScope<'_>,
@@ -316,12 +318,15 @@ impl ComputeRuntime {
316318
"mutation lock acquisition timed out"
317319
);
318320
}
319-
Err(error) => warn!(
320-
scope = scope.lock_scope().label(),
321-
waited_ms = duration_millis(started.elapsed()),
322-
error = %error,
323-
"mutation lock acquisition failed"
324-
),
321+
Err(error) => {
322+
gateway_metrics::record_lock_error(scope.lock_scope());
323+
warn!(
324+
scope = scope.lock_scope().label(),
325+
waited_ms = duration_millis(started.elapsed()),
326+
error = %error,
327+
"mutation lock acquisition failed"
328+
);
329+
}
325330
}
326331
result
327332
}
@@ -809,6 +814,7 @@ mod tests {
809814
async fn lock_metrics_record_wait_and_timeout() {
810815
const WAITS: &str = "openshell_server_mutation_lock_wait_seconds_count{scope=\"sandbox\"}";
811816
const TIMEOUTS: &str = "openshell_server_mutation_lock_timeouts_total{scope=\"sandbox\"}";
817+
const ERRORS: &str = "openshell_server_mutation_lock_errors_total{scope=\"sandbox\"}";
812818
let metrics = MetricsCapture::install();
813819
let runtime = test_runtime().await;
814820
runtime.set_mutation_lock_timeout_for_tests(Duration::from_millis(50));
@@ -826,6 +832,11 @@ mod tests {
826832
.is_err()
827833
);
828834
assert_eq!(metrics.value(TIMEOUTS), Some(1));
835+
assert_eq!(
836+
metrics.value(ERRORS),
837+
None,
838+
"a timeout is not also an error"
839+
);
829840
assert_eq!(metrics.value(WAITS), Some(1));
830841

831842
drop(runtime.lock_sandbox_local("b").await);
@@ -834,8 +845,9 @@ mod tests {
834845
}
835846

836847
#[tokio::test]
837-
async fn lock_connection_failure_is_not_counted_as_a_timeout() {
848+
async fn lock_connection_failure_counts_as_a_lock_error_not_a_timeout() {
838849
const TIMEOUTS: &str = "openshell_server_mutation_lock_timeouts_total{scope=\"sandbox\"}";
850+
const ERRORS: &str = "openshell_server_mutation_lock_errors_total{scope=\"sandbox\"}";
839851
let metrics = MetricsCapture::install();
840852
let url = crate::persistence::PostgresStore::refusing_url_for_tests().await;
841853
let store = crate::persistence::PostgresStore::connect_lazy_for_tests(&url, 1);
@@ -861,6 +873,7 @@ mod tests {
861873
Ok(_) => panic!("nothing listens, yet the guard was acquired"),
862874
}
863875
assert_eq!(metrics.value(TIMEOUTS), None);
876+
assert_eq!(metrics.value(ERRORS), Some(1));
864877
}
865878

866879
/// Workspaces and sandboxes the random scope mix draws from.

‎crates/openshell-server/src/gateway_metrics.rs‎

Lines changed: 92 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -25,11 +25,16 @@ use tonic::{Code, Status};
2525
pub const SUPERVISOR_SESSIONS: &str = "openshell_server_supervisor_sessions";
2626
pub const RELAY_PENDING: &str = "openshell_server_relay_pending";
2727
pub const RELAY_PENDING_CAPACITY: &str = "openshell_server_relay_pending_capacity";
28+
pub const MUTATION_LOCK_CONNECTIONS_IN_USE: &str =
29+
"openshell_server_mutation_lock_connections_in_use";
30+
pub const MUTATION_LOCK_CONNECTIONS_CAPACITY: &str =
31+
"openshell_server_mutation_lock_connections_capacity";
2832
// Counters
2933
pub const RELAY_REJECTED_TOTAL: &str = "openshell_server_relay_rejected_total";
3034
pub const RELAY_EXPIRED_TOTAL: &str = "openshell_server_relay_expired_total";
3135
pub const ROUTED_REQUEST_ATTEMPTS_TOTAL: &str = "openshell_server_routed_request_attempts_total";
3236
pub const MUTATION_LOCK_TIMEOUTS_TOTAL: &str = "openshell_server_mutation_lock_timeouts_total";
37+
pub const MUTATION_LOCK_ERRORS_TOTAL: &str = "openshell_server_mutation_lock_errors_total";
3338
// Histograms (explicit buckets, see BUCKETED_HISTOGRAMS)
3439
pub const RELAY_CLAIM_DURATION_SECONDS: &str = "openshell_server_relay_claim_duration_seconds";
3540
pub const PEER_REQUEST_DURATION_SECONDS: &str = "openshell_server_peer_request_duration_seconds";
@@ -226,16 +231,21 @@ pub fn configure_exporter(builder: PrometheusBuilder) -> Result<PrometheusBuilde
226231
}
227232

228233
/// Install the process-wide recorder, then describe and zero-initialize the catalog. Call
229-
/// once, from `run_server`.
230-
pub fn install_global_recorder(relay: RelayCapacity) -> Result<PrometheusHandle, BuildError> {
234+
/// once, from `run_server`. `mutation_lock_connections` is the `PostgreSQL` lock pool size,
235+
/// `None` on a backend without a lock pool.
236+
pub fn install_global_recorder(
237+
relay: RelayCapacity,
238+
mutation_lock_connections: Option<u32>,
239+
) -> Result<PrometheusHandle, BuildError> {
231240
let handle = configure_exporter(PrometheusBuilder::new())?.install_recorder()?;
232-
describe_and_initialize(relay);
241+
describe_and_initialize(relay, mutation_lock_connections);
233242
Ok(handle)
234243
}
235244

236-
/// Emit HELP metadata, create every fixed-label series at 0, and publish the relay caps. An
237-
/// idle replica then exports 0 instead of "no data", which HPA and `rate()` need.
238-
pub fn describe_and_initialize(relay: RelayCapacity) {
245+
/// Emit HELP metadata, create every fixed-label series at 0, and publish the relay caps and,
246+
/// with a lock pool, its size. An idle replica then exports 0 instead of "no data", which HPA
247+
/// and `rate()` need.
248+
pub fn describe_and_initialize(relay: RelayCapacity, mutation_lock_connections: Option<u32>) {
239249
describe_gauge!(
240250
SUPERVISOR_SESSIONS,
241251
Unit::Count,
@@ -271,6 +281,11 @@ pub fn describe_and_initialize(relay: RelayCapacity) {
271281
Unit::Count,
272282
"Mutation lock acquisitions that timed out."
273283
);
284+
describe_counter!(
285+
MUTATION_LOCK_ERRORS_TOTAL,
286+
Unit::Count,
287+
"Mutation lock acquisitions that failed without timing out, such as a lock connection PostgreSQL did not open."
288+
);
274289
describe_histogram!(
275290
RELAY_CLAIM_DURATION_SECONDS,
276291
Unit::Seconds,
@@ -310,6 +325,21 @@ pub fn describe_and_initialize(relay: RelayCapacity) {
310325
counter!(RELAY_EXPIRED_TOTAL).increment(0);
311326
for scope in LockScope::ALL {
312327
counter!(MUTATION_LOCK_TIMEOUTS_TOTAL, LABEL_SCOPE => scope.label()).increment(0);
328+
counter!(MUTATION_LOCK_ERRORS_TOTAL, LABEL_SCOPE => scope.label()).increment(0);
329+
}
330+
if let Some(capacity) = mutation_lock_connections {
331+
describe_gauge!(
332+
MUTATION_LOCK_CONNECTIONS_IN_USE,
333+
Unit::Count,
334+
"PostgreSQL lock-pool connections checked out on this replica: held by mutation guards, by acquisitions waiting for an advisory lock, or being returned to the pool."
335+
);
336+
describe_gauge!(
337+
MUTATION_LOCK_CONNECTIONS_CAPACITY,
338+
Unit::Count,
339+
"Maximum PostgreSQL lock-pool connections on one gateway replica."
340+
);
341+
gauge!(MUTATION_LOCK_CONNECTIONS_IN_USE).increment(0.0);
342+
gauge!(MUTATION_LOCK_CONNECTIONS_CAPACITY).set(f64::from(capacity));
313343
}
314344
for rpc in PeerRpc::ALL {
315345
if rpc == PeerRpc::Relay {
@@ -350,6 +380,11 @@ impl GaugeSlot {
350380
Self::acquire(RELAY_PENDING)
351381
}
352382

383+
/// Share of `openshell_server_mutation_lock_connections_in_use`.
384+
pub fn mutation_lock_connection() -> Self {
385+
Self::acquire(MUTATION_LOCK_CONNECTIONS_IN_USE)
386+
}
387+
353388
fn acquire(name: &'static str) -> Self {
354389
let gauge = gauge!(name);
355390
gauge.increment(1.0);
@@ -386,12 +421,19 @@ pub fn record_lock_wait(scope: LockScope, waited: Duration) {
386421

387422
/// A guard acquisition that timed out: a local wait, a full lock pool, too little time left to
388423
/// open a lock connection, or Postgres `lock_timeout` (SQLSTATE 55P03). A lock connection that
389-
/// Postgres does not open with at least `LOCK_CONNECTION_MIN_BUDGET` left is not counted. RPC
390-
/// callers return the timeout as `Status::unavailable`.
424+
/// Postgres does not open with at least `LOCK_CONNECTION_MIN_BUDGET` left counts in
425+
/// [`record_lock_error`] instead. RPC callers return the timeout as `Status::unavailable`.
391426
pub fn record_lock_timeout(scope: LockScope) {
392427
counter!(MUTATION_LOCK_TIMEOUTS_TOTAL, LABEL_SCOPE => scope.label()).increment(1);
393428
}
394429

430+
/// A guard acquisition that failed without timing out: Postgres did not open a lock connection,
431+
/// or a lock statement failed. RPC callers return it as `Status::internal`. A cancelled
432+
/// acquisition is not counted.
433+
pub fn record_lock_error(scope: LockScope) {
434+
counter!(MUTATION_LOCK_ERRORS_TOTAL, LABEL_SCOPE => scope.label()).increment(1);
435+
}
436+
395437
/// Counts one local relay setup or outbound peer attempt exactly once, and times peer requests.
396438
/// A relay succeeds when the supervisor claims it, on either route. Dropping an unfinished
397439
/// timer (the caller gave up) records `local_error` / `cancelled`.
@@ -541,10 +583,12 @@ mod tests {
541583
#[test]
542584
fn describe_and_initialize_exports_capacity_and_zero_series() {
543585
let metrics = MetricsCapture::install();
544-
describe_and_initialize(RelayCapacity { per_replica: 256 });
586+
describe_and_initialize(RelayCapacity { per_replica: 256 }, Some(4));
545587

546588
for (series, expected) in [
547589
("openshell_server_relay_pending_capacity", 256),
590+
("openshell_server_mutation_lock_connections_capacity", 4),
591+
("openshell_server_mutation_lock_connections_in_use", 0),
548592
("openshell_server_supervisor_sessions", 0),
549593
("openshell_server_relay_pending", 0),
550594
(
@@ -568,6 +612,18 @@ mod tests {
568612
"openshell_server_mutation_lock_timeouts_total{scope=\"sandbox\"}",
569613
0,
570614
),
615+
(
616+
"openshell_server_mutation_lock_errors_total{scope=\"global\"}",
617+
0,
618+
),
619+
(
620+
"openshell_server_mutation_lock_errors_total{scope=\"workspace\"}",
621+
0,
622+
),
623+
(
624+
"openshell_server_mutation_lock_errors_total{scope=\"sandbox\"}",
625+
0,
626+
),
571627
] {
572628
assert_eq!(metrics.value(series), Some(expected), "{series}");
573629
}
@@ -596,6 +652,24 @@ mod tests {
596652
);
597653
}
598654

655+
#[test]
656+
fn describe_and_initialize_without_a_lock_pool_omits_lock_connection_gauges() {
657+
let metrics = MetricsCapture::install();
658+
describe_and_initialize(RelayCapacity { per_replica: 256 }, None);
659+
660+
assert_eq!(metrics.value(MUTATION_LOCK_CONNECTIONS_IN_USE), None);
661+
assert_eq!(metrics.value(MUTATION_LOCK_CONNECTIONS_CAPACITY), None);
662+
assert!(
663+
!metrics
664+
.render()
665+
.contains("openshell_server_mutation_lock_connections")
666+
);
667+
assert_eq!(
668+
metrics.value("openshell_server_mutation_lock_errors_total{scope=\"sandbox\"}"),
669+
Some(0)
670+
);
671+
}
672+
599673
#[test]
600674
fn routed_attempts_have_seven_bounded_success_series_and_keep_counts_on_initialize() {
601675
let metrics = MetricsCapture::install();
@@ -605,7 +679,7 @@ mod tests {
605679
}
606680
}
607681
RoutedRequestTimer::relay(RelayKind::Tcp, RelayRoute::Peer).finish(&Ok::<(), Status>(()));
608-
describe_and_initialize(RelayCapacity { per_replica: 256 });
682+
describe_and_initialize(RelayCapacity { per_replica: 256 }, None);
609683

610684
for kind in ["ssh", "tcp"] {
611685
for route in ["local", "peer"] {
@@ -705,6 +779,14 @@ mod tests {
705779
assert_eq!(metrics.value(SUPERVISOR_SESSIONS), Some(1));
706780
drop(session);
707781
assert_eq!(metrics.value(SUPERVISOR_SESSIONS), Some(0));
782+
783+
let first = GaugeSlot::mutation_lock_connection();
784+
let second = GaugeSlot::mutation_lock_connection();
785+
assert_eq!(metrics.value(MUTATION_LOCK_CONNECTIONS_IN_USE), Some(2));
786+
drop(first);
787+
assert_eq!(metrics.value(MUTATION_LOCK_CONNECTIONS_IN_USE), Some(1));
788+
drop(second);
789+
assert_eq!(metrics.value(MUTATION_LOCK_CONNECTIONS_IN_USE), Some(0));
708790
}
709791

710792
#[test]

‎crates/openshell-server/src/lib.rs‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -870,9 +870,11 @@ pub(crate) async fn run_server(
870870

871871
// Bind the Prometheus metrics endpoint on a dedicated port when configured.
872872
if let Some(metrics_bind_address) = config.metrics_bind_address {
873-
let prometheus_handle =
874-
gateway_metrics::install_global_recorder(supervisor_session::RELAY_CAPACITY)
875-
.map_err(|e| Error::config(format!("failed to install metrics recorder: {e}")))?;
873+
let prometheus_handle = gateway_metrics::install_global_recorder(
874+
supervisor_session::RELAY_CAPACITY,
875+
store.mutation_lock_connections(),
876+
)
877+
.map_err(|e| Error::config(format!("failed to install metrics recorder: {e}")))?;
876878
let metrics_listener = TcpListener::bind(metrics_bind_address).await.map_err(|e| {
877879
Error::transport(format!(
878880
"failed to bind metrics port {metrics_bind_address}: {e}",

‎crates/openshell-server/src/persistence/mod.rs‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -316,6 +316,15 @@ impl Store {
316316
matches!(self, Self::Sqlite(_))
317317
}
318318

319+
/// Configured size of the `PostgreSQL` mutation lock pool, or `None` on
320+
/// `SQLite`, which has no lock pool.
321+
pub fn mutation_lock_connections(&self) -> Option<u32> {
322+
match self {
323+
Self::Postgres(store) => Some(store.lock_connection_capacity()),
324+
Self::Sqlite(_) => None,
325+
}
326+
}
327+
319328
/// Serialize mutations whose invariants span multiple persisted objects.
320329
///
321330
/// `SQLite` deployments are single-replica and use only the caller's local

‎crates/openshell-server/src/persistence/mutation_lock.rs‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -96,7 +96,9 @@ pub const LOCK_CONNECTION_MIN_BUDGET: Duration = Duration::from_secs(1);
9696
/// staying well under the data pool. Each extra slot costs
9797
/// `2 × replicas + surge` server connections during a rollout, so raise it
9898
/// for replicas whose guarded operations wait for a lock connection rather
99-
/// than by default.
99+
/// than by default. `openshell_server_mutation_lock_connections_in_use`
100+
/// exports how many are checked out, against the configured size in
101+
/// `openshell_server_mutation_lock_connections_capacity`.
100102
pub(super) const MUTATION_LOCK_POOL_MAX_CONNECTIONS: u32 = 4;
101103

102104
/// Domain separator hashed into every derived key.

0 commit comments

Comments
 (0)