Skip to content

Commit fa82963

Browse files
bjw123FrostGod
authored andcommitted
feat(server): make the database pool ceiling configurable
Both stores hardcoded their pool ceiling: 10 connections for Postgres, 5 for on-disk SQLite. One pool is shared by every database-backed RPC, so that number is also the gateway's ceiling on concurrent database work. Once every connection is checked out, callers queue on acquire and sqlx logs "time to acquire exceeded slow threshold"; sandbox creates then time out and are retried, which adds load rather than shedding it. The right ceiling is deployment-specific — it depends on how many gateway replicas share the database and what max_connections the server itself allows — so it cannot be a single number baked into the binary. Expose it on the same config surface as the rest of the gateway's settings: --db-max-connections, OPENSHELL_DB_MAX_CONNECTIONS, and the TOML key database_max_connections, resolved in that precedence order and rendered by the Helm chart from server.dbMaxConnections. Omitting it keeps each backend's previous value, so existing deployments do not move. Values below 1 are rejected rather than silently replaced by the default: a zero pool would block every acquire, and quietly ignoring a typo would reproduce the ceiling the operator is trying to lift. An in-memory SQLite database stays pinned to one connection, since the database lives inside that connection. Refs: NVIDIA#2561 Signed-off-by: Bryce Wilkinson <[email protected]> (cherry picked from commit 33517b3) Signed-off-by: Emilien Macchi <[email protected]>
1 parent 4676933 commit fa82963

16 files changed

Lines changed: 337 additions & 16 deletions

File tree

‎crates/openshell-core/src/config.rs‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -243,6 +243,16 @@ pub struct Config {
243243
/// Database URL for persistence.
244244
pub database_url: String,
245245

246+
/// Connection ceiling for the persistence pool.
247+
///
248+
/// `None` leaves the backend's built-in default in place. The right
249+
/// ceiling depends on the deployment — how many gateway replicas share
250+
/// the database, and what `max_connections` a Postgres server itself
251+
/// allows — so it cannot be one number baked into the binary. An
252+
/// in-memory `SQLite` database ignores it: the database lives in its single
253+
/// connection.
254+
pub database_max_connections: Option<u32>,
255+
246256
/// Explicit compute driver configured for the gateway.
247257
/// `None` enables runtime auto-detection.
248258
pub compute_driver: Option<String>,
@@ -892,6 +902,7 @@ impl Config {
892902
mtls_auth: MtlsAuthConfig::default(),
893903
gateway_jwt: None,
894904
database_url: String::new(),
905+
database_max_connections: None,
895906
compute_driver: None,
896907
compute_driver_endpoints: BTreeMap::new(),
897908
credential_drivers: Vec::new(),
@@ -944,6 +955,15 @@ impl Config {
944955
self
945956
}
946957

958+
/// Create a new configuration with a database pool connection ceiling.
959+
///
960+
/// `None` keeps the persistence backend's default.
961+
#[must_use]
962+
pub const fn with_database_max_connections(mut self, max_connections: Option<u32>) -> Self {
963+
self.database_max_connections = max_connections;
964+
self
965+
}
966+
947967
/// Create a new configuration with an explicit compute driver.
948968
#[must_use]
949969
pub fn with_compute_driver(mut self, driver: impl ToString) -> Self {

‎crates/openshell-server/src/cli.rs‎

Lines changed: 92 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -128,6 +128,20 @@ struct RunArgs {
128128
#[arg(long, env = "OPENSHELL_DB_URL")]
129129
db_url: Option<String>,
130130

131+
/// Connection ceiling for the database pool, shared by every
132+
/// database-backed request.
133+
///
134+
/// When unset, the gateway uses its backend default: 10 with Postgres,
135+
/// 5 with an on-disk `SQLite` database. An in-memory `SQLite` database is
136+
/// always a single connection. Each replica opens its own pool, so keep
137+
/// the total below what the database server itself admits.
138+
#[arg(
139+
long,
140+
env = "OPENSHELL_DB_MAX_CONNECTIONS",
141+
value_parser = clap::value_parser!(u32).range(1..)
142+
)]
143+
db_max_connections: Option<u32>,
144+
131145
/// Compute driver configured for this gateway.
132146
///
133147
/// Accepts one registered driver name. When unset, the gateway runs
@@ -516,7 +530,9 @@ fn prepare_server_config_with_drivers(
516530
config = config.with_metrics_bind_address(addr);
517531
}
518532

519-
config = config.with_database_url(db_url);
533+
config = config
534+
.with_database_url(db_url)
535+
.with_database_max_connections(args.db_max_connections);
520536
if let Some(driver) = &args.compute_driver {
521537
config = config.with_compute_driver(driver);
522538
}
@@ -1267,6 +1283,12 @@ fn merge_file_into_args(args: &mut RunArgs, file: &GatewayFileSection, matches:
12671283
{
12681284
args.grpc_rate_limit_window_seconds = Some(window);
12691285
}
1286+
if let Some(max_connections) = file.database_max_connections
1287+
&& args.db_max_connections.is_none()
1288+
&& arg_defaulted(matches, "db_max_connections")
1289+
{
1290+
args.db_max_connections = Some(max_connections);
1291+
}
12701292
}
12711293

12721294
fn validate_grpc_rate_limit_args(requests: Option<u64>, window_seconds: Option<u64>) -> Result<()> {
@@ -3143,6 +3165,75 @@ grpc_rate_limit_window_seconds = 30
31433165
assert_eq!(args.grpc_rate_limit_window_seconds, Some(30));
31443166
}
31453167

3168+
#[test]
3169+
fn file_db_max_connections_populates_args_when_cli_omits() {
3170+
let _lock = ENV_LOCK
3171+
.lock()
3172+
.unwrap_or_else(std::sync::PoisonError::into_inner);
3173+
let _g = EnvVarGuard::remove("OPENSHELL_DB_MAX_CONNECTIONS");
3174+
3175+
let (mut args, matches) =
3176+
parse_with_args(&["openshell-gateway", "--db-url", "sqlite::memory:"]);
3177+
assert_eq!(
3178+
args.db_max_connections, None,
3179+
"default leaves the pool alone"
3180+
);
3181+
let file = config_file_from_toml(
3182+
r"
3183+
[openshell.gateway]
3184+
database_max_connections = 64
3185+
",
3186+
);
3187+
merge_file_into_args(&mut args, &file.openshell.gateway, &matches);
3188+
3189+
assert_eq!(args.db_max_connections, Some(64));
3190+
}
3191+
3192+
#[test]
3193+
fn env_db_max_connections_overrides_file_value() {
3194+
let _lock = ENV_LOCK
3195+
.lock()
3196+
.unwrap_or_else(std::sync::PoisonError::into_inner);
3197+
let _g = EnvVarGuard::set("OPENSHELL_DB_MAX_CONNECTIONS", "128");
3198+
3199+
let (mut args, matches) =
3200+
parse_with_args(&["openshell-gateway", "--db-url", "sqlite::memory:"]);
3201+
let file = config_file_from_toml(
3202+
r"
3203+
[openshell.gateway]
3204+
database_max_connections = 64
3205+
",
3206+
);
3207+
merge_file_into_args(&mut args, &file.openshell.gateway, &matches);
3208+
3209+
assert_eq!(args.db_max_connections, Some(128));
3210+
}
3211+
3212+
#[test]
3213+
fn db_max_connections_rejects_a_zero_or_unparseable_ceiling() {
3214+
let _lock = ENV_LOCK
3215+
.lock()
3216+
.unwrap_or_else(std::sync::PoisonError::into_inner);
3217+
let _g = EnvVarGuard::remove("OPENSHELL_DB_MAX_CONNECTIONS");
3218+
3219+
// Zero would deadlock every `acquire`, and a silent fallback to the
3220+
// default would reproduce the ceiling the operator is trying to lift.
3221+
for bad in ["0", "-5", "many"] {
3222+
assert!(
3223+
command()
3224+
.try_get_matches_from([
3225+
"openshell-gateway",
3226+
"--db-url",
3227+
"sqlite::memory:",
3228+
"--db-max-connections",
3229+
bad,
3230+
])
3231+
.is_err(),
3232+
"input {bad:?} must be rejected"
3233+
);
3234+
}
3235+
}
3236+
31463237
#[test]
31473238
fn aux_listener_preserves_file_ip_against_public_bind() {
31483239
use std::net::SocketAddr;

‎crates/openshell-server/src/config_file.rs‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -102,6 +102,17 @@ pub struct GatewayFileSection {
102102
#[serde(default)]
103103
pub log_level: Option<String>,
104104

105+
// ── Database ─────────────────────────────────────────────────────────
106+
/// Connection ceiling for the persistence pool. One shared pool serves
107+
/// every database-backed RPC, so this is also the gateway's ceiling on
108+
/// concurrent database work. On Postgres, total server connections are
109+
/// `database_max_connections × replica_count`, which must stay under the
110+
/// server's own `max_connections`. Omit to keep the backend default (10
111+
/// for Postgres, 5 for on-disk `SQLite`); an in-memory `SQLite` database
112+
/// is always a single connection.
113+
#[serde(default)]
114+
pub database_max_connections: Option<u32>,
115+
105116
// ── Drivers ──────────────────────────────────────────────────────────
106117
/// Explicit compute driver selection. `None` enables auto-detection.
107118
#[serde(default, skip_serializing_if = "Option::is_none")]
@@ -590,6 +601,14 @@ fn parse_and_validate(path: &Path, contents: &str) -> Result<ConfigFile, ConfigF
590601
cli: "--db-url",
591602
});
592603
}
604+
// A zero-sized pool would deadlock every `acquire` rather than degrade,
605+
// so reject it at load instead of starting a gateway that cannot serve.
606+
if file.openshell.gateway.database_max_connections == Some(0) {
607+
return Err(ConfigFileError::InvalidValue {
608+
field: "openshell.gateway.database_max_connections",
609+
message: "must be greater than zero; omit the key to use the built-in default",
610+
});
611+
}
593612
if file
594613
.openshell
595614
.gateway
@@ -1288,6 +1307,34 @@ database_url = "sqlite::memory:"
12881307
));
12891308
}
12901309

1310+
#[test]
1311+
fn parses_database_max_connections() {
1312+
let toml = r"
1313+
[openshell.gateway]
1314+
database_max_connections = 64
1315+
";
1316+
let tmp = write_tmp(toml);
1317+
let file = load(tmp.path()).expect("a positive pool ceiling parses");
1318+
assert_eq!(file.openshell.gateway.database_max_connections, Some(64));
1319+
}
1320+
1321+
#[test]
1322+
fn rejects_zero_database_max_connections() {
1323+
let toml = r"
1324+
[openshell.gateway]
1325+
database_max_connections = 0
1326+
";
1327+
let tmp = write_tmp(toml);
1328+
let err = load(tmp.path()).expect_err("a zero pool ceiling must be rejected");
1329+
assert!(matches!(
1330+
err,
1331+
ConfigFileError::InvalidValue {
1332+
field: "openshell.gateway.database_max_connections",
1333+
..
1334+
}
1335+
));
1336+
}
1337+
12911338
#[test]
12921339
fn rejects_unknown_gateway_field() {
12931340
let toml = r"

‎crates/openshell-server/src/lib.rs‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -604,7 +604,9 @@ pub(crate) async fn run_server(
604604
.map_err(|error| Error::config(format!("middleware registration failed: {error}")))?,
605605
);
606606

607-
let store = Arc::new(Store::connect(database_url).await?);
607+
let store = Arc::new(
608+
Store::connect_with_pool_size(database_url, config.database_max_connections).await?,
609+
);
608610
let credentials = credentials::CredentialRuntime::from_config_file_with_store(
609611
&config,
610612
config_file.as_ref(),

‎crates/openshell-server/src/persistence/mod.rs‎

Lines changed: 17 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -381,10 +381,24 @@ impl Store {
381381
}
382382
}
383383

384-
/// Connect to a persistence store based on the database URL.
384+
/// Connect to a persistence store based on the database URL, leaving pool
385+
/// sizing at the backend default.
385386
pub async fn connect(url: &str) -> CoreResult<Self> {
387+
Self::connect_with_pool_size(url, None).await
388+
}
389+
390+
/// Connect to a persistence store based on the database URL.
391+
///
392+
/// `max_connections` overrides the backend's pool ceiling, which stays at
393+
/// its built-in default when `None`. The two backends have different
394+
/// defaults, and an in-memory `SQLite` database ignores the override
395+
/// entirely — it must be held by exactly one connection.
396+
pub async fn connect_with_pool_size(
397+
url: &str,
398+
max_connections: Option<u32>,
399+
) -> CoreResult<Self> {
386400
if url.starts_with("postgres://") || url.starts_with("postgresql://") {
387-
let store = PostgresStore::connect(url)
401+
let store = PostgresStore::connect(url, max_connections)
388402
.await
389403
.map_err(|e| CoreError::execution(e.to_string()))?;
390404
store
@@ -393,7 +407,7 @@ impl Store {
393407
.map_err(|e| CoreError::execution(e.to_string()))?;
394408
Ok(Self::Postgres(store))
395409
} else if url.starts_with("sqlite:") {
396-
let store = SqliteStore::connect(url)
410+
let store = SqliteStore::connect(url, max_connections)
397411
.await
398412
.map_err(|e| CoreError::execution(e.to_string()))?;
399413
store

‎crates/openshell-server/src/persistence/mutation_lock_pg_tests.rs‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -290,7 +290,7 @@ impl LockFixture {
290290
/// locks.
291291
async fn store(&self, lock_pool_size: u32) -> Store {
292292
let store = Store::Postgres(
293-
PostgresStore::connect_with_lock_pool_size(self.schema.url(), lock_pool_size)
293+
PostgresStore::connect_with_lock_pool_size(self.schema.url(), None, lock_pool_size)
294294
.await
295295
.expect("connect a lock store"),
296296
);
@@ -575,7 +575,7 @@ async fn postgres_mutation_lock_stalled_release_closes_session_and_recovers_pool
575575
let fixture = LockFixture::new().await;
576576
let proxy = StallingProxy::start(fixture.schema.url()).await;
577577
let store = Store::Postgres(
578-
PostgresStore::connect_with_lock_pool_size(&proxy.url, 1)
578+
PostgresStore::connect_with_lock_pool_size(&proxy.url, None, 1)
579579
.await
580580
.unwrap(),
581581
);
@@ -922,7 +922,7 @@ async fn postgres_mutation_lock_released_connection_is_reused_without_residual_l
922922
#[ignore = "requires a disposable PostgreSQL database in OPENSHELL_TEST_POSTGRES_URL; run mise run test:rust:postgres"]
923923
async fn postgres_mutation_lock_pool_never_exceeds_configured_size() {
924924
let fixture = LockFixture::new().await;
925-
let postgres = PostgresStore::connect(fixture.schema.url())
925+
let postgres = PostgresStore::connect(fixture.schema.url(), None)
926926
.await
927927
.expect("connect a store with the production lock pool");
928928
let store = Store::Postgres(postgres.clone());
@@ -1000,7 +1000,7 @@ async fn postgres_mutation_lock_connection_failure_is_not_a_lock_timeout() {
10001000
// but PostgreSQL never answers one: a database failure, not contention.
10011001
let proxy = StallingProxy::start(fixture.schema.url()).await;
10021002
let unanswered = Store::Postgres(
1003-
PostgresStore::connect_with_lock_pool_size(&proxy.url, 1)
1003+
PostgresStore::connect_with_lock_pool_size(&proxy.url, None, 1)
10041004
.await
10051005
.expect("connect a store through the proxy"),
10061006
);

‎crates/openshell-server/src/persistence/postgres.rs‎

Lines changed: 16 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -255,17 +255,30 @@ impl Drop for PendingLockConnection {
255255
}
256256
}
257257

258+
/// Pool ceiling used when the operator has not configured one.
259+
///
260+
/// A single shared pool serves every database-backed RPC, so this value is
261+
/// also the gateway's ceiling on concurrent database work. It is high enough
262+
/// for a single-replica gateway against a default-sized server; fleets that
263+
/// outgrow it raise it through `[openshell.gateway] database_max_connections`.
264+
pub const DEFAULT_MAX_CONNECTIONS: u32 = 10;
265+
258266
impl PostgresStore {
259-
pub async fn connect(url: &str) -> PersistenceResult<Self> {
260-
Self::connect_with_lock_pool_size(url, MUTATION_LOCK_POOL_MAX_CONNECTIONS).await
267+
/// Connect and size the pool, falling back to [`DEFAULT_MAX_CONNECTIONS`].
268+
pub async fn connect(url: &str, max_connections: Option<u32>) -> PersistenceResult<Self> {
269+
Self::connect_with_lock_pool_size(url, max_connections, MUTATION_LOCK_POOL_MAX_CONNECTIONS)
270+
.await
261271
}
262272

263273
pub(super) async fn connect_with_lock_pool_size(
264274
url: &str,
275+
max_connections: Option<u32>,
265276
lock_pool_size: u32,
266277
) -> PersistenceResult<Self> {
278+
let max_connections = max_connections.unwrap_or(DEFAULT_MAX_CONNECTIONS);
279+
tracing::info!(max_connections, "sizing Postgres connection pool");
267280
let pool = PgPoolOptions::new()
268-
.max_connections(10)
281+
.max_connections(max_connections)
269282
.connect(url)
270283
.await
271284
.map_err(|e| map_db_error(&e))?;

0 commit comments

Comments
 (0)