|
4 | 4 | //! `PostgreSQL` tests of the mutation advisory locks: exclusion between two |
5 | 5 | //! stores (as between two gateway replicas), exclusion against the legacy |
6 | 6 | //! global key, cleanup after timed-out and cancelled acquisitions, a stalled |
7 | | -//! holder, the lock-pool bound and its in-use gauge, and lock connections that |
8 | | -//! `PostgreSQL` does not open. |
| 7 | +//! holder, the lock-pool bound and its in-use gauge, lock connections that |
| 8 | +//! `PostgreSQL` does not open, and the bound on data-pool lock holders at the |
| 9 | +//! smallest data pool. |
9 | 10 | //! |
10 | 11 | //! Advisory locks are database-wide, not per schema, so every test uses |
11 | 12 | //! random workspace and sandbox ids, and `mise run test:rust:postgres` runs |
12 | | -//! the tests one at a time. Apart from the pool-size test, which connects as |
13 | | -//! the gateway does, the stores here run no migrations: the schema only |
14 | | -//! scopes their connections. |
| 13 | +//! the tests one at a time. Apart from the pool-size and SSH identity tests, |
| 14 | +//! which connect as the gateway does, the stores here run no migrations: the |
| 15 | +//! schema only scopes their connections. |
15 | 16 |
|
16 | 17 | use super::mutation_lock::{ |
17 | 18 | GLOBAL_MUTATION_LOCK_KEY, LOCK_CONNECTION_MIN_BUDGET, MUTATION_LOCK_POOL_MAX_CONNECTIONS, |
18 | | - MUTATION_LOCK_TIMEOUT, MutationLockKey, MutationLockSet, |
| 19 | + MUTATION_LOCK_TIMEOUT, MutationLockKey, MutationLockSet, SSH_IDENTITY_LOCK_KEY, |
19 | 20 | }; |
20 | | -use super::postgres::LOCK_CONNECTION_RELEASE_TIMEOUT; |
| 21 | +use super::postgres::{LOCK_CONNECTION_RELEASE_TIMEOUT, MIN_MAX_CONNECTIONS}; |
21 | 22 | use super::test_postgres::TestSchema; |
22 | 23 | use super::{ |
23 | 24 | DistributedMutationGuard, PersistenceError, PersistenceResult, PostgresStore, Store, |
@@ -181,6 +182,117 @@ async fn postgres_concurrent_sandbox_creates_with_ssh_identity_do_not_exhaust_lo |
181 | 182 | schema.drop_schema().await; |
182 | 183 | } |
183 | 184 |
|
| 185 | +/// Launch authentication for a supervisor whose SSH host key is not yet |
| 186 | +/// resolved. |
| 187 | +fn ssh_launch_authentication() -> openshell_core::jwt::SandboxLaunchAuthentication { |
| 188 | + use openshell_core::jwt::{ |
| 189 | + CredentialEpoch, SandboxLaunchAuthentication, SecretJwt, SessionRotation, |
| 190 | + SupervisorAuthBundle, |
| 191 | + }; |
| 192 | + use openshell_core::{SandboxSessionId, sandbox_generation::SandboxGenerationId}; |
| 193 | + |
| 194 | + SandboxLaunchAuthentication { |
| 195 | + supervisor: SupervisorAuthBundle { |
| 196 | + session_id: SandboxSessionId::new(), |
| 197 | + runtime_generation: SandboxGenerationId::parse("generation").unwrap(), |
| 198 | + session_rotation: SessionRotation::new(1).unwrap(), |
| 199 | + auth_epoch: CredentialEpoch::new(1).unwrap(), |
| 200 | + gateway_token: SecretJwt::parse("gateway-token").unwrap(), |
| 201 | + gateway_expires_at: 0, |
| 202 | + sandbox_token: SecretJwt::parse("sandbox-token").unwrap(), |
| 203 | + sandbox_expires_at: 0, |
| 204 | + ssh_host_private_key: None, |
| 205 | + ocsf_otlp_headers: None, |
| 206 | + }, |
| 207 | + gateway_id: "gateway".to_string(), |
| 208 | + verification_keys: Vec::new(), |
| 209 | + } |
| 210 | +} |
| 211 | + |
| 212 | +/// At the smallest data pool `PostgreSQL` accepts, a provider-refresh holder |
| 213 | +/// and an SSH identity holder on one replica leave a data connection free for |
| 214 | +/// queries: the SSH identity holder waits for the one data-pool lock slot |
| 215 | +/// instead of taking the last connection. |
| 216 | +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] |
| 217 | +#[ignore = "requires a disposable PostgreSQL database in OPENSHELL_TEST_POSTGRES_URL; run mise run test:rust:postgres"] |
| 218 | +async fn postgres_data_pool_lock_holders_leave_a_connection_free_at_floor() { |
| 219 | + use crate::credentials::CredentialRuntime; |
| 220 | + use crate::ssh_identity::SshIdentityStore; |
| 221 | + use openshell_core::proto::Sandbox; |
| 222 | + use openshell_core::proto::datamodel::v1::ObjectMeta; |
| 223 | + use std::sync::Arc; |
| 224 | + |
| 225 | + let fixture = LockFixture::new().await; |
| 226 | + let store = |
| 227 | + Store::connect_with_pool_sizes(fixture.schema.url(), Some(MIN_MAX_CONNECTIONS), None) |
| 228 | + .await |
| 229 | + .expect("connect a store with the smallest data pool"); |
| 230 | + let shared = Arc::new(store.clone()); |
| 231 | + let config = crate::Config::new(None) |
| 232 | + .with_credential_drivers(["test-static"]) |
| 233 | + .with_default_credential_driver(Some("test-static")); |
| 234 | + let identities = SshIdentityStore::new( |
| 235 | + shared.clone(), |
| 236 | + CredentialRuntime::from_config_with_store(&config, shared).unwrap(), |
| 237 | + ); |
| 238 | + let id = random_id("sb"); |
| 239 | + let sandbox = Sandbox { |
| 240 | + metadata: Some(ObjectMeta { |
| 241 | + id: id.clone(), |
| 242 | + name: id.clone(), |
| 243 | + workspace: "default".to_string(), |
| 244 | + ..Default::default() |
| 245 | + }), |
| 246 | + ..Default::default() |
| 247 | + }; |
| 248 | + store |
| 249 | + .put_message(&sandbox) |
| 250 | + .await |
| 251 | + .expect("store the sandbox"); |
| 252 | + |
| 253 | + let refresh = store |
| 254 | + .acquire_refresh_guard(&random_id("provider"), "ACCESS_TOKEN") |
| 255 | + .await |
| 256 | + .expect("hold a provider refresh lock"); |
| 257 | + let preparing = { |
| 258 | + let identities = identities.clone(); |
| 259 | + let mut sandbox = sandbox.clone(); |
| 260 | + tokio::spawn(async move { |
| 261 | + identities |
| 262 | + .prepare(&mut sandbox, &mut ssh_launch_authentication()) |
| 263 | + .await |
| 264 | + .map(|()| sandbox) |
| 265 | + }) |
| 266 | + }; |
| 267 | + // Long enough for the SSH identity holder to take a data connection if |
| 268 | + // nothing stops it. |
| 269 | + tokio::time::sleep(Duration::from_secs(1)).await; |
| 270 | + |
| 271 | + tokio::time::timeout(PROCEEDS_WITHIN, store.get_message::<Sandbox>(&id)) |
| 272 | + .await |
| 273 | + .expect("a query must find a free data connection while both lock holders run") |
| 274 | + .expect("load the sandbox") |
| 275 | + .expect("the sandbox exists"); |
| 276 | + assert_eq!( |
| 277 | + fixture.lock_count(SSH_IDENTITY_LOCK_KEY).await, |
| 278 | + 0, |
| 279 | + "the SSH identity holder waits for a data-pool lock slot, not in PostgreSQL" |
| 280 | + ); |
| 281 | + |
| 282 | + drop(refresh); |
| 283 | + let prepared = tokio::time::timeout(PROCEEDS_WITHIN, preparing) |
| 284 | + .await |
| 285 | + .expect("the SSH identity holder proceeds once the refresh lock is released") |
| 286 | + .expect("prepare task") |
| 287 | + .expect("prepare the SSH identity"); |
| 288 | + assert!( |
| 289 | + !prepared.host_key_fingerprint.is_empty(), |
| 290 | + "prepare publishes the SSH host key fingerprint" |
| 291 | + ); |
| 292 | + |
| 293 | + fixture.finish(vec![store]).await; |
| 294 | +} |
| 295 | + |
184 | 296 | /// Drop client traffic while keeping sockets open, like a failed network path. |
185 | 297 | /// Client EOF still closes the upstream socket so `PostgreSQL` can release locks. |
186 | 298 | struct StallingProxy { |
|
0 commit comments