Skip to content

Commit a5a89f0

Browse files
committed
fix(providers): gate provider credential write on refresh generation ownership
After minting, the rotation checked only that a refresh with the same name still existed, wrote credentials to the provider, and only then did the version-matched refresh-state write. A refresh deleted-and-recreated during mint, or a losing concurrent rotation, could write credentials minted from a stale generation before returning Aborted (CWE-362). Claim the refresh generation first: perform the version-matched refresh-state write before touching the provider. It succeeds only if the refresh still holds the generation the rotation started from; on a lost claim (deleted, recreated, or superseded) it returns None and the rotation aborts leaving both the refresh state and provider unchanged. No transaction spans the two objects — the store exposes only per-object CAS — so this optimistic claim-then-write is the ownership guarantee: only the generation owner writes credentials. Uses the typed conditional-write Conflict result instead of matching error strings. Adds a deterministic superseded-mid-flight regression test (a concurrent write bumps the generation while STS is paused) alongside the existing deleted-mid-flight test. Signed-off-by: Russell Bryant <[email protected]>
1 parent 31c339e commit a5a89f0

1 file changed

Lines changed: 166 additions & 53 deletions

File tree

‎crates/openshell-server/src/provider_refresh.rs‎

Lines changed: 166 additions & 53 deletions
Original file line numberDiff line numberDiff line change
@@ -52,14 +52,14 @@ pub async fn put_refresh_state(
5252
///
5353
/// Uses a version-matched UPDATE, which never inserts — so a refresh deleted
5454
/// while an STS (or OAuth) request was in flight is not resurrected, and its
55-
/// stored source-credential material is not recreated (CWE-362). Returns `true`
56-
/// when persisted, `false` when the refresh was deleted or superseded by a
57-
/// concurrent write (in which case nothing was written).
55+
/// stored source-credential material is not recreated (CWE-362). Returns the new
56+
/// resource version when persisted, or `None` when the refresh was deleted or
57+
/// superseded by a concurrent write (in which case nothing was written).
5858
async fn persist_refresh_state_if_current(
5959
store: &Store,
6060
state: &StoredProviderCredentialRefreshState,
6161
expected_version: u64,
62-
) -> Result<bool, Status> {
62+
) -> Result<Option<u64>, Status> {
6363
match store
6464
.put_if(
6565
StoredProviderCredentialRefreshState::object_type(),
@@ -71,11 +71,11 @@ async fn persist_refresh_state_if_current(
7171
)
7272
.await
7373
{
74-
Ok(_) => Ok(true),
75-
// Version mismatch (superseded) or the row is gone (deleted): the
76-
// version-matched UPDATE matched no row, so nothing was written.
77-
Err(PersistenceError::Conflict { .. }) => Ok(false),
78-
Err(PersistenceError::Database(message)) if message.contains("not found") => Ok(false),
74+
Ok(result) => Ok(Some(result.resource_version)),
75+
// The version-matched UPDATE matched no row: the refresh was deleted
76+
// (current version `None`) or superseded by a concurrent write (a
77+
// different current version). Either way nothing was written.
78+
Err(PersistenceError::Conflict { .. }) => Ok(None),
7979
Err(e) => Err(Status::internal(format!(
8080
"persist provider refresh state failed: {e}"
8181
))),
@@ -383,47 +383,9 @@ pub async fn refresh_provider_credential(
383383
match mint_credential(&state).await {
384384
Ok(minted) => {
385385
let now_ms = current_time_ms();
386-
// A delete-refresh can complete while the mint request is in flight.
387-
// Verify the refresh still exists before writing minted credentials
388-
// into the provider, so a removed refresh is not silently re-minted
389-
// (CWE-362). The version-matched persist below is the authoritative
390-
// guard against resurrecting deleted state; this check just avoids
391-
// writing credentials we are about to discard.
392-
if get_refresh_state(store, provider.object_id(), credential_key)
393-
.await?
394-
.is_none()
395-
{
396-
warn!(
397-
provider = %state.provider_name,
398-
credential_key = %state.credential_key,
399-
strategy = %refresh_strategy_name(state.strategy),
400-
"provider credential refresh deleted during rotation; discarding minted credentials"
401-
);
402-
return Err(Status::aborted(
403-
"provider refresh was deleted during rotation",
404-
));
405-
}
406-
if let Err(err) =
407-
apply_minted_credential(store, &provider, credential_key, &minted).await
408-
{
409-
state.status = "error".to_string();
410-
state.last_error = err.message().to_string();
411-
state.next_refresh_at_ms =
412-
now_ms.saturating_add(REFRESH_ERROR_RETRY_SECONDS.saturating_mul(1000));
413-
persist_refresh_state_if_current(store, &state, expected_version).await?;
414-
warn!(
415-
provider = %state.provider_name,
416-
credential_key = %state.credential_key,
417-
strategy = %refresh_strategy_name(state.strategy),
418-
status = %state.status,
419-
next_refresh_at_ms = state.next_refresh_at_ms,
420-
seconds_until_refresh = seconds_until_ms(now_ms, state.next_refresh_at_ms),
421-
error = %err,
422-
"provider credential refresh errored"
423-
);
424-
return Err(err);
425-
}
426-
if let Some(refresh_token) = minted.refresh_token {
386+
// Fold the minted result into the refresh state before claiming the
387+
// generation.
388+
if let Some(refresh_token) = minted.refresh_token.clone() {
427389
state
428390
.material
429391
.insert("refresh_token".to_string(), refresh_token);
@@ -445,16 +407,51 @@ pub async fn refresh_provider_credential(
445407
state.last_refresh_at_ms = now_ms;
446408
state.status = "refreshed".to_string();
447409
state.last_error.clear();
448-
if !persist_refresh_state_if_current(store, &state, expected_version).await? {
410+
411+
// Claim the refresh generation with a version-matched write BEFORE
412+
// touching the provider. It succeeds only if the refresh still holds
413+
// the generation we started from; if it was deleted, recreated, or a
414+
// concurrent rotation won, it returns `None` and we leave both the
415+
// refresh state and the provider unchanged — no credentials minted
416+
// from a stale generation are written, and a deleted refresh is not
417+
// resurrected (CWE-362). This makes generation ownership the gate on
418+
// the provider credential write.
419+
let Some(new_version) =
420+
persist_refresh_state_if_current(store, &state, expected_version).await?
421+
else {
449422
warn!(
450423
provider = %state.provider_name,
451424
credential_key = %state.credential_key,
452425
strategy = %refresh_strategy_name(state.strategy),
453-
"provider credential refresh deleted or superseded during rotation; not resurrecting refresh state"
426+
"provider credential refresh deleted or superseded during rotation; discarding minted credentials"
454427
);
455428
return Err(Status::aborted(
456-
"provider refresh was deleted during rotation",
429+
"provider refresh was deleted or superseded during rotation",
457430
));
431+
};
432+
433+
// Generation is ours; write the minted credentials into the provider.
434+
if let Err(err) =
435+
apply_minted_credential(store, &provider, credential_key, &minted).await
436+
{
437+
state.status = "error".to_string();
438+
state.last_error = err.message().to_string();
439+
state.next_refresh_at_ms =
440+
now_ms.saturating_add(REFRESH_ERROR_RETRY_SECONDS.saturating_mul(1000));
441+
// Reflect the failure on the state we just wrote; skip silently
442+
// if it was deleted concurrently (it is not recreated).
443+
persist_refresh_state_if_current(store, &state, new_version).await?;
444+
warn!(
445+
provider = %state.provider_name,
446+
credential_key = %state.credential_key,
447+
strategy = %refresh_strategy_name(state.strategy),
448+
status = %state.status,
449+
next_refresh_at_ms = state.next_refresh_at_ms,
450+
seconds_until_refresh = seconds_until_ms(now_ms, state.next_refresh_at_ms),
451+
error = %err,
452+
"provider credential refresh errored"
453+
);
454+
return Err(err);
458455
}
459456
info!(
460457
provider = %state.provider_name,
@@ -1954,6 +1951,122 @@ mod tests {
19541951
assert!(!stored.credentials.contains_key("AWS_SESSION_TOKEN"));
19551952
}
19561953

1954+
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
1955+
async fn rotation_superseded_mid_flight_discards_credentials() {
1956+
let mock_server = MockServer::start().await;
1957+
let (hit_tx, hit_rx) = tokio::sync::oneshot::channel::<()>();
1958+
let (release_tx, release_rx) = std::sync::mpsc::channel::<()>();
1959+
Mock::given(method("POST"))
1960+
.and(body_string_contains("Action=AssumeRole"))
1961+
.respond_with(GatedStsResponder {
1962+
hit: std::sync::Mutex::new(Some(hit_tx)),
1963+
release: std::sync::Mutex::new(release_rx),
1964+
body: r#"<AssumeRoleResponse xmlns="https://sts.amazonaws.com/doc/2011-06-15/">
1965+
<AssumeRoleResult>
1966+
<Credentials>
1967+
<AccessKeyId>ASIAMOCKKEY</AccessKeyId>
1968+
<SecretAccessKey>MockSecretAccessKey123</SecretAccessKey>
1969+
<SessionToken>MockSessionTokenXYZ</SessionToken>
1970+
<Expiration>2099-01-01T00:00:00Z</Expiration>
1971+
</Credentials>
1972+
</AssumeRoleResult>
1973+
</AssumeRoleResponse>"#
1974+
.to_string(),
1975+
})
1976+
.mount(&mock_server)
1977+
.await;
1978+
1979+
let store = test_store().await;
1980+
crate::grpc::policy::set_global_bool_setting_for_test(
1981+
&store,
1982+
openshell_core::settings::PROVIDERS_V2_ENABLED_KEY,
1983+
true,
1984+
)
1985+
.await
1986+
.unwrap();
1987+
let prov = provider("aws-superseded", "aws");
1988+
store.put_message(&prov).await.unwrap();
1989+
let provider_id = prov.object_id().to_string();
1990+
1991+
let state = new_refresh_state(
1992+
&prov,
1993+
"AWS_ACCESS_KEY_ID",
1994+
NewRefreshStateConfig {
1995+
additional_output_keys: HashMap::from([
1996+
(
1997+
"secret_access_key".to_string(),
1998+
"AWS_SECRET_ACCESS_KEY".to_string(),
1999+
),
2000+
("session_token".to_string(), "AWS_SESSION_TOKEN".to_string()),
2001+
]),
2002+
strategy: ProviderCredentialRefreshStrategy::AwsStsAssumeRole,
2003+
material: HashMap::from([
2004+
(
2005+
"role_arn".to_string(),
2006+
"arn:aws:iam::123456789012:role/TestRole".to_string(),
2007+
),
2008+
("aws_access_key_id".to_string(), "AKIATESTKEY".to_string()),
2009+
(
2010+
"aws_secret_access_key".to_string(),
2011+
"TestSecretKey".to_string(),
2012+
),
2013+
("sts_endpoint_url".to_string(), mock_server.uri()),
2014+
]),
2015+
secret_material_keys: vec!["aws_secret_access_key".to_string()],
2016+
expires_at_ms: 0,
2017+
token_url: String::new(),
2018+
scopes: Vec::new(),
2019+
refresh_before_seconds: 300,
2020+
max_lifetime_seconds: 3600,
2021+
},
2022+
)
2023+
.unwrap();
2024+
put_refresh_state(&store, &state).await.unwrap();
2025+
2026+
let rotate = refresh_provider_credential(&store, "aws-superseded", "AWS_ACCESS_KEY_ID");
2027+
let interfere = async {
2028+
if tokio::time::timeout(std::time::Duration::from_secs(15), hit_rx)
2029+
.await
2030+
.is_err()
2031+
{
2032+
return;
2033+
}
2034+
// Simulate a concurrent rotation or reconfigure winning the
2035+
// generation: any write to the refresh state bumps its version, so
2036+
// the in-flight rotation's version-matched persist will lose.
2037+
let mut winner = get_refresh_state(&store, &provider_id, "AWS_ACCESS_KEY_ID")
2038+
.await
2039+
.unwrap()
2040+
.unwrap();
2041+
winner.last_error = "won-by-concurrent-writer".to_string();
2042+
put_refresh_state(&store, &winner).await.unwrap();
2043+
let _ = release_tx.send(());
2044+
};
2045+
let (rotate_result, ()) = tokio::join!(rotate, interfere);
2046+
2047+
// The superseded (losing) rotation must abort rather than complete.
2048+
assert!(
2049+
rotate_result.is_err(),
2050+
"a rotation whose generation was superseded must abort"
2051+
);
2052+
// It must not write its stale-generation credentials into the provider.
2053+
let stored = store
2054+
.get_message_by_name::<Provider>("aws-superseded")
2055+
.await
2056+
.unwrap()
2057+
.unwrap();
2058+
assert!(!stored.credentials.contains_key("AWS_ACCESS_KEY_ID"));
2059+
assert!(!stored.credentials.contains_key("AWS_SECRET_ACCESS_KEY"));
2060+
assert!(!stored.credentials.contains_key("AWS_SESSION_TOKEN"));
2061+
// The concurrent writer's refresh state must survive untouched.
2062+
let refresh = get_refresh_state(&store, &provider_id, "AWS_ACCESS_KEY_ID")
2063+
.await
2064+
.unwrap()
2065+
.expect("refresh state should still exist");
2066+
assert_eq!(refresh.last_error, "won-by-concurrent-writer");
2067+
assert_ne!(refresh.status, "refreshed");
2068+
}
2069+
19572070
fn provider(name: &str, provider_type: &str) -> Provider {
19582071
Provider {
19592072
metadata: Some(ObjectMeta {

0 commit comments

Comments
 (0)