Skip to content

Commit f462775

Browse files
committed
fix(providers): don't let a rotation resurrect a deleted refresh
refresh_provider_credential read the refresh state, awaited the STS (or OAuth) request, then wrote credentials and persisted state with an unconditional upsert. A delete-refresh completing while the request was in flight was silently undone: the rotation re-minted credentials and recreated the deleted refresh state, including stored source-credential material (CWE-362). Capture the refresh generation at the start and make every terminal persist a version-matched UPDATE (never an insert), so a deleted or superseded refresh is detected and never recreated. Re-check that the refresh still exists after minting, before writing credentials into the provider, so a removed refresh is not re-minted. Add a deterministic regression test that pauses mock STS, deletes the refresh, releases STS, and verifies the refresh state and provider credentials are unchanged. Signed-off-by: Russell Bryant <[email protected]>
1 parent 623512e commit f462775

1 file changed

Lines changed: 212 additions & 7 deletions

File tree

‎crates/openshell-server/src/provider_refresh.rs‎

Lines changed: 212 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
66
#![allow(clippy::result_large_err)]
77

8-
use crate::persistence::{ObjectType, Store, current_time_ms};
8+
use crate::persistence::{ObjectType, PersistenceError, Store, WriteCondition, current_time_ms};
99
use openshell_core::proto::{
1010
Provider, ProviderCredentialRefreshStatus, ProviderCredentialRefreshStrategy,
1111
StoredProviderCredentialRefreshState,
@@ -47,6 +47,41 @@ pub async fn put_refresh_state(
4747
.map_err(|e| Status::internal(format!("persist provider refresh state failed: {e}")))
4848
}
4949

50+
/// Persist an updated refresh state only if the row still exists with the
51+
/// generation read at the start of the rotation.
52+
///
53+
/// Uses a version-matched UPDATE, which never inserts — so a refresh deleted
54+
/// while an STS (or OAuth) request was in flight is not resurrected, and its
55+
/// stored source-credential material is not recreated (CWE-362). Returns `true`
56+
/// when persisted, `false` when the refresh was deleted or superseded by a
57+
/// concurrent write (in which case nothing was written).
58+
async fn persist_refresh_state_if_current(
59+
store: &Store,
60+
state: &StoredProviderCredentialRefreshState,
61+
expected_version: u64,
62+
) -> Result<bool, Status> {
63+
match store
64+
.put_if(
65+
StoredProviderCredentialRefreshState::object_type(),
66+
state.object_id(),
67+
state.object_name(),
68+
&state.encode_to_vec(),
69+
None,
70+
WriteCondition::MatchResourceVersion(expected_version),
71+
)
72+
.await
73+
{
74+
Ok(_) => Ok(true),
75+
// Version mismatch (superseded) or the row is gone (deleted): the
76+
// version-matched UPDATE matched no row, so nothing was written.
77+
Err(PersistenceError::Conflict { .. }) => Ok(false),
78+
Err(PersistenceError::Database(message)) if message.contains("not found") => Ok(false),
79+
Err(e) => Err(Status::internal(format!(
80+
"persist provider refresh state failed: {e}"
81+
))),
82+
}
83+
}
84+
5085
pub async fn list_refresh_states_for_provider(
5186
store: &Store,
5287
provider_id: &str,
@@ -306,6 +341,13 @@ pub async fn refresh_provider_credential(
306341
else {
307342
return Err(Status::not_found("provider refresh state not found"));
308343
};
344+
// Generation of the refresh at the start of the rotation. Terminal persists
345+
// match on it so a concurrent delete or rotation is detected rather than
346+
// clobbered, and a deleted refresh is never recreated (CWE-362).
347+
let expected_version = state
348+
.metadata
349+
.as_ref()
350+
.map_or(0, |meta| meta.resource_version);
309351

310352
info!(
311353
provider = %state.provider_name,
@@ -326,7 +368,7 @@ pub async fn refresh_provider_credential(
326368
state.last_error = err.message().to_string();
327369
state.next_refresh_at_ms =
328370
now_ms.saturating_add(REFRESH_ERROR_RETRY_SECONDS.saturating_mul(1000));
329-
put_refresh_state(store, &state).await?;
371+
persist_refresh_state_if_current(store, &state, expected_version).await?;
330372
warn!(
331373
provider = %state.provider_name,
332374
credential_key = %state.credential_key,
@@ -341,14 +383,34 @@ pub async fn refresh_provider_credential(
341383
match mint_credential(&state).await {
342384
Ok(minted) => {
343385
let now_ms = current_time_ms();
386+
// A delete-refresh can complete while the mint request is in flight.
387+
// Verify the refresh still exists before writing minted credentials
388+
// into the provider, so a removed refresh is not silently re-minted
389+
// (CWE-362). The version-matched persist below is the authoritative
390+
// guard against resurrecting deleted state; this check just avoids
391+
// writing credentials we are about to discard.
392+
if get_refresh_state(store, provider.object_id(), credential_key)
393+
.await?
394+
.is_none()
395+
{
396+
warn!(
397+
provider = %state.provider_name,
398+
credential_key = %state.credential_key,
399+
strategy = %refresh_strategy_name(state.strategy),
400+
"provider credential refresh deleted during rotation; discarding minted credentials"
401+
);
402+
return Err(Status::aborted(
403+
"provider refresh was deleted during rotation",
404+
));
405+
}
344406
if let Err(err) =
345407
apply_minted_credential(store, &provider, credential_key, &minted).await
346408
{
347409
state.status = "error".to_string();
348410
state.last_error = err.message().to_string();
349411
state.next_refresh_at_ms =
350412
now_ms.saturating_add(REFRESH_ERROR_RETRY_SECONDS.saturating_mul(1000));
351-
put_refresh_state(store, &state).await?;
413+
persist_refresh_state_if_current(store, &state, expected_version).await?;
352414
warn!(
353415
provider = %state.provider_name,
354416
credential_key = %state.credential_key,
@@ -383,7 +445,17 @@ pub async fn refresh_provider_credential(
383445
state.last_refresh_at_ms = now_ms;
384446
state.status = "refreshed".to_string();
385447
state.last_error.clear();
386-
put_refresh_state(store, &state).await?;
448+
if !persist_refresh_state_if_current(store, &state, expected_version).await? {
449+
warn!(
450+
provider = %state.provider_name,
451+
credential_key = %state.credential_key,
452+
strategy = %refresh_strategy_name(state.strategy),
453+
"provider credential refresh deleted or superseded during rotation; not resurrecting refresh state"
454+
);
455+
return Err(Status::aborted(
456+
"provider refresh was deleted during rotation",
457+
));
458+
}
387459
info!(
388460
provider = %state.provider_name,
389461
credential_key = %state.credential_key,
@@ -402,7 +474,7 @@ pub async fn refresh_provider_credential(
402474
state.last_error = err.message().to_string();
403475
state.next_refresh_at_ms =
404476
now_ms.saturating_add(REFRESH_ERROR_RETRY_SECONDS.saturating_mul(1000));
405-
put_refresh_state(store, &state).await?;
477+
persist_refresh_state_if_current(store, &state, expected_version).await?;
406478
warn!(
407479
provider = %state.provider_name,
408480
credential_key = %state.credential_key,
@@ -982,8 +1054,8 @@ async fn run_refresh_worker_tick(store: &Store) -> Result<(), Status> {
9821054
#[cfg(test)]
9831055
mod tests {
9841056
use super::{
985-
NewRefreshStateConfig, get_refresh_state, new_refresh_state, put_refresh_state,
986-
refresh_provider_credential, refresh_state_name, refresh_strategy_name,
1057+
NewRefreshStateConfig, delete_refresh_state, get_refresh_state, new_refresh_state,
1058+
put_refresh_state, refresh_provider_credential, refresh_state_name, refresh_strategy_name,
9871059
run_refresh_worker_tick, seconds_until_ms,
9881060
};
9891061
use crate::persistence::test_store;
@@ -1749,6 +1821,139 @@ mod tests {
17491821
assert!(err.message().contains("AWS_SECRET_ACCESS_KEY"));
17501822
}
17511823

1824+
// A wiremock responder that blocks the STS response until the test releases
1825+
// it, so a delete-refresh can be interleaved deterministically while the
1826+
// rotation is parked awaiting STS.
1827+
struct GatedStsResponder {
1828+
hit: std::sync::Mutex<Option<tokio::sync::oneshot::Sender<()>>>,
1829+
release: std::sync::Mutex<std::sync::mpsc::Receiver<()>>,
1830+
body: String,
1831+
}
1832+
1833+
impl wiremock::Respond for GatedStsResponder {
1834+
fn respond(&self, _request: &wiremock::Request) -> ResponseTemplate {
1835+
let hit = self.hit.lock().unwrap().take();
1836+
if let Some(hit) = hit {
1837+
let _ = hit.send(());
1838+
}
1839+
let _ = self.release.lock().unwrap().recv();
1840+
ResponseTemplate::new(200).set_body_string(self.body.clone())
1841+
}
1842+
}
1843+
1844+
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
1845+
async fn rotation_does_not_resurrect_refresh_deleted_mid_flight() {
1846+
let mock_server = MockServer::start().await;
1847+
let (hit_tx, hit_rx) = tokio::sync::oneshot::channel::<()>();
1848+
let (release_tx, release_rx) = std::sync::mpsc::channel::<()>();
1849+
Mock::given(method("POST"))
1850+
.and(body_string_contains("Action=AssumeRole"))
1851+
.respond_with(GatedStsResponder {
1852+
hit: std::sync::Mutex::new(Some(hit_tx)),
1853+
release: std::sync::Mutex::new(release_rx),
1854+
body: r#"<AssumeRoleResponse xmlns="https://sts.amazonaws.com/doc/2011-06-15/">
1855+
<AssumeRoleResult>
1856+
<Credentials>
1857+
<AccessKeyId>ASIAMOCKKEY</AccessKeyId>
1858+
<SecretAccessKey>MockSecretAccessKey123</SecretAccessKey>
1859+
<SessionToken>MockSessionTokenXYZ</SessionToken>
1860+
<Expiration>2099-01-01T00:00:00Z</Expiration>
1861+
</Credentials>
1862+
</AssumeRoleResult>
1863+
</AssumeRoleResponse>"#
1864+
.to_string(),
1865+
})
1866+
.mount(&mock_server)
1867+
.await;
1868+
1869+
let store = test_store().await;
1870+
crate::grpc::policy::set_global_bool_setting_for_test(
1871+
&store,
1872+
openshell_core::settings::PROVIDERS_V2_ENABLED_KEY,
1873+
true,
1874+
)
1875+
.await
1876+
.unwrap();
1877+
let prov = provider("aws-race", "aws");
1878+
store.put_message(&prov).await.unwrap();
1879+
let provider_id = prov.object_id().to_string();
1880+
1881+
let state = new_refresh_state(
1882+
&prov,
1883+
"AWS_ACCESS_KEY_ID",
1884+
NewRefreshStateConfig {
1885+
additional_output_keys: HashMap::from([
1886+
(
1887+
"secret_access_key".to_string(),
1888+
"AWS_SECRET_ACCESS_KEY".to_string(),
1889+
),
1890+
("session_token".to_string(), "AWS_SESSION_TOKEN".to_string()),
1891+
]),
1892+
strategy: ProviderCredentialRefreshStrategy::AwsStsAssumeRole,
1893+
material: HashMap::from([
1894+
(
1895+
"role_arn".to_string(),
1896+
"arn:aws:iam::123456789012:role/TestRole".to_string(),
1897+
),
1898+
("aws_access_key_id".to_string(), "AKIATESTKEY".to_string()),
1899+
(
1900+
"aws_secret_access_key".to_string(),
1901+
"TestSecretKey".to_string(),
1902+
),
1903+
("sts_endpoint_url".to_string(), mock_server.uri()),
1904+
]),
1905+
secret_material_keys: vec!["aws_secret_access_key".to_string()],
1906+
expires_at_ms: 0,
1907+
token_url: String::new(),
1908+
scopes: Vec::new(),
1909+
refresh_before_seconds: 300,
1910+
max_lifetime_seconds: 3600,
1911+
},
1912+
)
1913+
.unwrap();
1914+
put_refresh_state(&store, &state).await.unwrap();
1915+
1916+
let rotate = refresh_provider_credential(&store, "aws-race", "AWS_ACCESS_KEY_ID");
1917+
let interfere = async {
1918+
// Wait until the rotation is inside the STS call (its state read has
1919+
// already happened), then delete the refresh and release STS.
1920+
if tokio::time::timeout(std::time::Duration::from_secs(15), hit_rx)
1921+
.await
1922+
.is_err()
1923+
{
1924+
return;
1925+
}
1926+
delete_refresh_state(&store, &provider_id, "AWS_ACCESS_KEY_ID")
1927+
.await
1928+
.unwrap();
1929+
let _ = release_tx.send(());
1930+
};
1931+
let (rotate_result, ()) = tokio::join!(rotate, interfere);
1932+
1933+
// The rotation must fail rather than complete against a deleted refresh.
1934+
assert!(
1935+
rotate_result.is_err(),
1936+
"rotation should abort when its refresh is deleted mid-flight"
1937+
);
1938+
// The deleted refresh state must not be resurrected.
1939+
assert!(
1940+
get_refresh_state(&store, &provider_id, "AWS_ACCESS_KEY_ID")
1941+
.await
1942+
.unwrap()
1943+
.is_none(),
1944+
"deleted refresh state must not be recreated"
1945+
);
1946+
// No credentials were minted into the provider.
1947+
let stored = store
1948+
.get_message_by_name::<Provider>("aws-race")
1949+
.await
1950+
.unwrap()
1951+
.unwrap();
1952+
assert!(!stored.credentials.contains_key("AWS_ACCESS_KEY_ID"));
1953+
assert!(!stored.credentials.contains_key("AWS_SECRET_ACCESS_KEY"));
1954+
assert!(!stored.credentials.contains_key("AWS_SESSION_TOKEN"));
1955+
}
1956+
17521957
fn provider(name: &str, provider_type: &str) -> Provider {
17531958
Provider {
17541959
metadata: Some(ObjectMeta {

0 commit comments

Comments
 (0)