Skip to content

Commit 547b3f0

Browse files
committed
fix(providers): make refresh configuration atomic
Signed-off-by: Mrunal Patel <[email protected]>
1 parent 16005c6 commit 547b3f0

9 files changed

Lines changed: 339 additions & 8 deletions

File tree

‎.agents/skills/openshell-cli/SKILL.md‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -184,9 +184,12 @@ The gateway stores secret refresh material through its active credential driver.
184184
With Vault selected, refresh tokens, client secrets, and private keys live in
185185
Vault alongside injectable provider credentials; refresh state contains only
186186
opaque handles. A credential-backend read or write failure makes refresh fail
187-
closed rather than falling back to inline storage. During an upgrade that
188-
migrates legacy inline refresh material, stop all older gateway replicas before
189-
starting the new version; do not use a mixed-version rolling update.
187+
closed rather than falling back to inline storage. Before OpenShell 0.1.0, the
188+
gateway does not migrate legacy inline refresh material or move secrets between
189+
credential backends. Reconfigure affected grants after upgrading, and remove or
190+
reconfigure credentials while the original backend remains available before
191+
changing backends. Do not run mixed gateway versions against the same refresh
192+
records.
190193

191194
Gateway-managed refresh credentials use an identity-stable workload handle.
192195
Routine automatic refresh and `provider refresh rotate` update the access token

‎crates/openshell-server/src/grpc/provider.rs‎

Lines changed: 122 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3831,7 +3831,7 @@ pub(super) async fn handle_configure_provider_refresh(
38313831
Err(err) => Err(err),
38323832
}
38333833
} else {
3834-
crate::provider_refresh::put_refresh_state(state.store.as_ref(), &state_record).await
3834+
crate::provider_refresh::create_refresh_state(state.store.as_ref(), &state_record).await
38353835
};
38363836
if let Err(err) = persist_result {
38373837
if let Err(cleanup_err) = state
@@ -6041,6 +6041,127 @@ mod tests {
60416041
);
60426042
}
60436043

6044+
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
6045+
async fn concurrent_first_refresh_configure_keeps_only_winner_material() {
6046+
let first_state = test_server_state().await;
6047+
import_test_graph_refresh_profile(&first_state).await;
6048+
create_provider_record(
6049+
first_state.store.as_ref(),
6050+
"default",
6051+
Provider {
6052+
metadata: Some(openshell_core::proto::datamodel::v1::ObjectMeta {
6053+
name: "configure-create-race".to_string(),
6054+
workspace: "default".to_string(),
6055+
..Default::default()
6056+
}),
6057+
r#type: TEST_GRAPH_PROVIDER_TYPE.to_string(),
6058+
profile_workspace: "default".to_string(),
6059+
..Default::default()
6060+
},
6061+
)
6062+
.await
6063+
.unwrap();
6064+
6065+
// Model a second gateway replica: it shares the durable database and
6066+
// credential backend, but owns an independent sandbox synchronization
6067+
// guard, so process-local serialization cannot hide this race.
6068+
let second_state = Arc::new(ServerState::new_with_credentials(
6069+
first_state.config.clone(),
6070+
Arc::clone(&first_state.store),
6071+
crate::compute::new_test_runtime(Arc::clone(&first_state.store)).await,
6072+
crate::sandbox_index::SandboxIndex::new(),
6073+
crate::sandbox_watch::SandboxWatchBus::new(),
6074+
crate::tracing_bus::TracingLogBus::new(),
6075+
Arc::new(crate::supervisor_session::SupervisorSessionRegistry::new()),
6076+
None,
6077+
first_state.credentials.clone(),
6078+
));
6079+
let request = |client_secret: &str| ConfigureProviderRefreshRequest {
6080+
provider: "configure-create-race".to_string(),
6081+
credential_key: "MS_GRAPH_ACCESS_TOKEN".to_string(),
6082+
strategy: ProviderCredentialRefreshStrategy::Oauth2ClientCredentials as i32,
6083+
material: HashMap::from([
6084+
("tenant_id".to_string(), "tenant".to_string()),
6085+
("client_id".to_string(), "client-id".to_string()),
6086+
("client_secret".to_string(), client_secret.to_string()),
6087+
]),
6088+
secret_material_keys: vec!["client_secret".to_string()],
6089+
expires_at_ms: None,
6090+
workspace: "default".to_string(),
6091+
};
6092+
let (first_store_hit, release_first_store) = first_state.credentials.gate_next_store();
6093+
6094+
let first = handle_configure_provider_refresh(
6095+
&first_state,
6096+
authed_request(request("loser-secret")),
6097+
);
6098+
let second = async {
6099+
first_store_hit.await.unwrap();
6100+
let result = handle_configure_provider_refresh(
6101+
&second_state,
6102+
authed_request(request("winner-secret")),
6103+
)
6104+
.await;
6105+
release_first_store.send(()).unwrap();
6106+
result
6107+
};
6108+
let (first_result, second_result) = tokio::join!(first, second);
6109+
6110+
assert_eq!(first_result.unwrap_err().code(), Code::Aborted);
6111+
second_result.unwrap();
6112+
assert_eq!(first_state.credentials.stored_credential_count(), Some(1));
6113+
6114+
let provider = first_state
6115+
.store
6116+
.get_message_by_name::<Provider>("default", "configure-create-race")
6117+
.await
6118+
.unwrap()
6119+
.unwrap();
6120+
let stored = crate::provider_refresh::get_refresh_state(
6121+
first_state.store.as_ref(),
6122+
"default",
6123+
provider.object_id(),
6124+
"MS_GRAPH_ACCESS_TOKEN",
6125+
)
6126+
.await
6127+
.unwrap()
6128+
.unwrap();
6129+
assert_eq!(
6130+
first_state
6131+
.credentials
6132+
.resolve_refresh_material(
6133+
crate::provider_refresh::refresh_material_scope(&stored),
6134+
&stored.secret_material_handles,
6135+
)
6136+
.await
6137+
.unwrap()
6138+
.get("client_secret"),
6139+
Some(&"winner-secret".to_string())
6140+
);
6141+
6142+
let physical = first_state
6143+
.store
6144+
.get_by_name(
6145+
StoredProviderCredentialRefreshState::object_type(),
6146+
"default",
6147+
stored.object_name(),
6148+
)
6149+
.await
6150+
.unwrap()
6151+
.unwrap();
6152+
assert_eq!(physical.id, stored.object_id());
6153+
assert_eq!(
6154+
crate::provider_refresh::list_refresh_states_for_provider(
6155+
first_state.store.as_ref(),
6156+
provider.object_id(),
6157+
)
6158+
.await
6159+
.unwrap()
6160+
.len(),
6161+
1
6162+
);
6163+
}
6164+
60446165
async fn state_with_authoritative_profiles_over_default_grants() -> Arc<ServerState> {
60456166
let state = test_server_state().await;
60466167
let store = state.store.as_ref();

‎crates/openshell-server/src/persistence/mod.rs‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -339,6 +339,38 @@ impl Store {
339339
))
340340
}
341341

342+
/// Atomically insert a generic named object with an application-owned scope.
343+
///
344+
/// Unlike [`Self::put_scoped`], this never updates an existing object. A
345+
/// duplicate id or `(object_type, workspace, name)` returns
346+
/// [`PersistenceError::UniqueViolation`].
347+
#[allow(clippy::too_many_arguments)]
348+
#[tracing::instrument(
349+
name = "store",
350+
skip_all,
351+
fields(otel.name = "store.create_scoped", otel.status_code = tracing::field::Empty, object_type = %object_type, object.id = %id, object.name = %name, workspace = %workspace, scope = %scope)
352+
)]
353+
pub async fn create_scoped(
354+
&self,
355+
object_type: &str,
356+
id: &str,
357+
name: &str,
358+
workspace: &str,
359+
scope: &str,
360+
payload: &[u8],
361+
labels: Option<&str>,
362+
) -> PersistenceResult<WriteResult> {
363+
store_dispatch_traced!(self.create_scoped(
364+
object_type,
365+
id,
366+
name,
367+
workspace,
368+
scope,
369+
payload,
370+
labels
371+
))
372+
}
373+
342374
/// Fetch an object by id.
343375
#[tracing::instrument(
344376
name = "store",

‎crates/openshell-server/src/persistence/postgres.rs‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -293,6 +293,50 @@ ON CONFLICT (object_type, workspace, name) WHERE name IS NOT NULL DO UPDATE SET
293293
Ok(())
294294
}
295295

296+
#[allow(clippy::too_many_arguments)]
297+
pub async fn create_scoped(
298+
&self,
299+
object_type: &str,
300+
id: &str,
301+
name: &str,
302+
workspace: &str,
303+
scope: &str,
304+
payload: &[u8],
305+
labels: Option<&str>,
306+
) -> PersistenceResult<WriteResult> {
307+
let now_ms = current_time_ms();
308+
let labels_jsonb: Option<serde_json::Value> = labels
309+
.map(serde_json::from_str)
310+
.transpose()
311+
.map_err(|e| PersistenceError::Encode(format!("invalid labels JSON: {e}")))?;
312+
313+
let row = sqlx::query(
314+
r"
315+
INSERT INTO objects (object_type, id, name, workspace, scope, payload, created_at_ms, updated_at_ms, labels, resource_version)
316+
VALUES ($1, $2, $3, $4, $5, $6, $7, $7, COALESCE($8, '{}'::jsonb), 1)
317+
RETURNING resource_version, created_at_ms, updated_at_ms
318+
",
319+
)
320+
.bind(object_type)
321+
.bind(id)
322+
.bind(name)
323+
.bind(workspace)
324+
.bind(scope)
325+
.bind(payload)
326+
.bind(now_ms)
327+
.bind(labels_jsonb)
328+
.fetch_one(&self.pool)
329+
.await
330+
.map_err(|e| map_db_error(&e))?;
331+
332+
let resource_version_i64: i64 = row.try_get("resource_version").unwrap_or(1);
333+
Ok(WriteResult {
334+
resource_version: resource_version_i64.max(1).cast_unsigned(),
335+
created_at_ms: row.get("created_at_ms"),
336+
updated_at_ms: row.get("updated_at_ms"),
337+
})
338+
}
339+
296340
pub async fn get(
297341
&self,
298342
object_type: &str,

‎crates/openshell-server/src/persistence/sqlite.rs‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -319,6 +319,44 @@ ON CONFLICT ("object_type", "workspace", "name") WHERE "name" IS NOT NULL DO UPD
319319
Ok(())
320320
}
321321

322+
#[allow(clippy::too_many_arguments)]
323+
pub async fn create_scoped(
324+
&self,
325+
object_type: &str,
326+
id: &str,
327+
name: &str,
328+
workspace: &str,
329+
scope: &str,
330+
payload: &[u8],
331+
labels: Option<&str>,
332+
) -> PersistenceResult<WriteResult> {
333+
let now_ms = current_time_ms();
334+
335+
sqlx::query(
336+
r#"
337+
INSERT INTO "objects" ("object_type", "id", "name", "workspace", "scope", "payload", "created_at_ms", "updated_at_ms", "labels", "resource_version")
338+
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?7, ?8, 1)
339+
"#,
340+
)
341+
.bind(object_type)
342+
.bind(id)
343+
.bind(name)
344+
.bind(workspace)
345+
.bind(scope)
346+
.bind(payload)
347+
.bind(now_ms)
348+
.bind(labels.unwrap_or("{}"))
349+
.execute(&self.pool)
350+
.await
351+
.map_err(|e| map_db_error(&e))?;
352+
353+
Ok(WriteResult {
354+
resource_version: 1,
355+
created_at_ms: now_ms,
356+
updated_at_ms: now_ms,
357+
})
358+
}
359+
322360
pub async fn get(
323361
&self,
324362
object_type: &str,

‎crates/openshell-server/src/persistence/tests.rs‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1302,6 +1302,65 @@ fn parse_label_selector_handles_whitespace() {
13021302
assert_eq!(result.get("tier"), Some(&"frontend".to_string()));
13031303
}
13041304

1305+
#[tokio::test]
1306+
async fn create_scoped_is_insert_only_and_preserves_scope() {
1307+
use super::PersistenceError;
1308+
1309+
let store = test_store().await;
1310+
let created = store
1311+
.create_scoped(
1312+
"refresh",
1313+
"winner-id",
1314+
"provider-token",
1315+
"default",
1316+
"winner-provider",
1317+
b"winner",
1318+
None,
1319+
)
1320+
.await
1321+
.unwrap();
1322+
assert_eq!(created.resource_version, 1);
1323+
1324+
let duplicate = store
1325+
.create_scoped(
1326+
"refresh",
1327+
"loser-id",
1328+
"provider-token",
1329+
"default",
1330+
"loser-provider",
1331+
b"loser",
1332+
None,
1333+
)
1334+
.await;
1335+
assert!(matches!(
1336+
duplicate,
1337+
Err(PersistenceError::UniqueViolation { .. })
1338+
));
1339+
1340+
let winner = store
1341+
.get_by_name("refresh", "default", "provider-token")
1342+
.await
1343+
.unwrap()
1344+
.unwrap();
1345+
assert_eq!(winner.id, "winner-id");
1346+
assert_eq!(winner.payload, b"winner");
1347+
assert_eq!(
1348+
store
1349+
.list_by_scope("refresh", "winner-provider", 10, 0)
1350+
.await
1351+
.unwrap()
1352+
.len(),
1353+
1
1354+
);
1355+
assert!(
1356+
store
1357+
.list_by_scope("refresh", "loser-provider", 10, 0)
1358+
.await
1359+
.unwrap()
1360+
.is_empty()
1361+
);
1362+
}
1363+
13051364
// ---------------------------------------------------------------------------
13061365
// CAS (compare-and-swap) tests
13071366
// ---------------------------------------------------------------------------

‎crates/openshell-server/src/provider_refresh.rs‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,7 @@ pub fn effective_authorization_epoch(
7373
})
7474
}
7575

76+
#[cfg(test)]
7677
pub async fn put_refresh_state(
7778
store: &Store,
7879
state: &StoredProviderCredentialRefreshState,
@@ -83,6 +84,37 @@ pub async fn put_refresh_state(
8384
.map_err(|e| Status::internal(format!("persist provider refresh state failed: {e}")))
8485
}
8586

87+
/// Atomically claim a new provider-and-credential refresh identity.
88+
///
89+
/// The refresh name is unique within a workspace. A concurrent creator must
90+
/// lose instead of overwriting the winner so its caller can delete any secret
91+
/// material staged before this write.
92+
pub async fn create_refresh_state(
93+
store: &Store,
94+
state: &StoredProviderCredentialRefreshState,
95+
) -> Result<(), Status> {
96+
match store
97+
.create_scoped(
98+
StoredProviderCredentialRefreshState::object_type(),
99+
state.object_id(),
100+
state.object_name(),
101+
state.object_workspace(),
102+
&state.provider_id,
103+
&state.encode_to_vec(),
104+
None,
105+
)
106+
.await
107+
{
108+
Ok(_) => Ok(()),
109+
Err(PersistenceError::UniqueViolation { .. }) => Err(Status::aborted(
110+
"provider refresh was concurrently configured",
111+
)),
112+
Err(err) => Err(Status::internal(format!(
113+
"create provider refresh state failed: {err}"
114+
))),
115+
}
116+
}
117+
86118
/// Persist an updated refresh state only if the row still exists with the
87119
/// generation read at the start of the rotation.
88120
///

0 commit comments

Comments
 (0)