55
66#![ allow( clippy:: result_large_err) ]
77
8- use crate :: persistence:: { ObjectType , Store , current_time_ms} ;
8+ use crate :: persistence:: { ObjectType , PersistenceError , Store , WriteCondition , current_time_ms} ;
99use openshell_core:: proto:: {
1010 Provider , ProviderCredentialRefreshStatus , ProviderCredentialRefreshStrategy ,
1111 StoredProviderCredentialRefreshState ,
@@ -47,6 +47,41 @@ pub async fn put_refresh_state(
4747 . map_err ( |e| Status :: internal ( format ! ( "persist provider refresh state failed: {e}" ) ) )
4848}
4949
50+ /// Persist an updated refresh state only if the row still exists with the
51+ /// generation read at the start of the rotation.
52+ ///
53+ /// Uses a version-matched UPDATE, which never inserts — so a refresh deleted
54+ /// while an STS (or OAuth) request was in flight is not resurrected, and its
55+ /// stored source-credential material is not recreated (CWE-362). Returns `true`
56+ /// when persisted, `false` when the refresh was deleted or superseded by a
57+ /// concurrent write (in which case nothing was written).
58+ async fn persist_refresh_state_if_current (
59+ store : & Store ,
60+ state : & StoredProviderCredentialRefreshState ,
61+ expected_version : u64 ,
62+ ) -> Result < bool , Status > {
63+ match store
64+ . put_if (
65+ StoredProviderCredentialRefreshState :: object_type ( ) ,
66+ state. object_id ( ) ,
67+ state. object_name ( ) ,
68+ & state. encode_to_vec ( ) ,
69+ None ,
70+ WriteCondition :: MatchResourceVersion ( expected_version) ,
71+ )
72+ . await
73+ {
74+ Ok ( _) => Ok ( true ) ,
75+ // Version mismatch (superseded) or the row is gone (deleted): the
76+ // version-matched UPDATE matched no row, so nothing was written.
77+ Err ( PersistenceError :: Conflict { .. } ) => Ok ( false ) ,
78+ Err ( PersistenceError :: Database ( message) ) if message. contains ( "not found" ) => Ok ( false ) ,
79+ Err ( e) => Err ( Status :: internal ( format ! (
80+ "persist provider refresh state failed: {e}"
81+ ) ) ) ,
82+ }
83+ }
84+
5085pub async fn list_refresh_states_for_provider (
5186 store : & Store ,
5287 provider_id : & str ,
@@ -306,6 +341,13 @@ pub async fn refresh_provider_credential(
306341 else {
307342 return Err ( Status :: not_found ( "provider refresh state not found" ) ) ;
308343 } ;
344+ // Generation of the refresh at the start of the rotation. Terminal persists
345+ // match on it so a concurrent delete or rotation is detected rather than
346+ // clobbered, and a deleted refresh is never recreated (CWE-362).
347+ let expected_version = state
348+ . metadata
349+ . as_ref ( )
350+ . map_or ( 0 , |meta| meta. resource_version ) ;
309351
310352 info ! (
311353 provider = %state. provider_name,
@@ -326,7 +368,7 @@ pub async fn refresh_provider_credential(
326368 state. last_error = err. message ( ) . to_string ( ) ;
327369 state. next_refresh_at_ms =
328370 now_ms. saturating_add ( REFRESH_ERROR_RETRY_SECONDS . saturating_mul ( 1000 ) ) ;
329- put_refresh_state ( store, & state) . await ?;
371+ persist_refresh_state_if_current ( store, & state, expected_version ) . await ?;
330372 warn ! (
331373 provider = %state. provider_name,
332374 credential_key = %state. credential_key,
@@ -341,14 +383,34 @@ pub async fn refresh_provider_credential(
341383 match mint_credential ( & state) . await {
342384 Ok ( minted) => {
343385 let now_ms = current_time_ms ( ) ;
386+ // A delete-refresh can complete while the mint request is in flight.
387+ // Verify the refresh still exists before writing minted credentials
388+ // into the provider, so a removed refresh is not silently re-minted
389+ // (CWE-362). The version-matched persist below is the authoritative
390+ // guard against resurrecting deleted state; this check just avoids
391+ // writing credentials we are about to discard.
392+ if get_refresh_state ( store, provider. object_id ( ) , credential_key)
393+ . await ?
394+ . is_none ( )
395+ {
396+ warn ! (
397+ provider = %state. provider_name,
398+ credential_key = %state. credential_key,
399+ strategy = %refresh_strategy_name( state. strategy) ,
400+ "provider credential refresh deleted during rotation; discarding minted credentials"
401+ ) ;
402+ return Err ( Status :: aborted (
403+ "provider refresh was deleted during rotation" ,
404+ ) ) ;
405+ }
344406 if let Err ( err) =
345407 apply_minted_credential ( store, & provider, credential_key, & minted) . await
346408 {
347409 state. status = "error" . to_string ( ) ;
348410 state. last_error = err. message ( ) . to_string ( ) ;
349411 state. next_refresh_at_ms =
350412 now_ms. saturating_add ( REFRESH_ERROR_RETRY_SECONDS . saturating_mul ( 1000 ) ) ;
351- put_refresh_state ( store, & state) . await ?;
413+ persist_refresh_state_if_current ( store, & state, expected_version ) . await ?;
352414 warn ! (
353415 provider = %state. provider_name,
354416 credential_key = %state. credential_key,
@@ -383,7 +445,17 @@ pub async fn refresh_provider_credential(
383445 state. last_refresh_at_ms = now_ms;
384446 state. status = "refreshed" . to_string ( ) ;
385447 state. last_error . clear ( ) ;
386- put_refresh_state ( store, & state) . await ?;
448+ if !persist_refresh_state_if_current ( store, & state, expected_version) . await ? {
449+ warn ! (
450+ provider = %state. provider_name,
451+ credential_key = %state. credential_key,
452+ strategy = %refresh_strategy_name( state. strategy) ,
453+ "provider credential refresh deleted or superseded during rotation; not resurrecting refresh state"
454+ ) ;
455+ return Err ( Status :: aborted (
456+ "provider refresh was deleted during rotation" ,
457+ ) ) ;
458+ }
387459 info ! (
388460 provider = %state. provider_name,
389461 credential_key = %state. credential_key,
@@ -402,7 +474,7 @@ pub async fn refresh_provider_credential(
402474 state. last_error = err. message ( ) . to_string ( ) ;
403475 state. next_refresh_at_ms =
404476 now_ms. saturating_add ( REFRESH_ERROR_RETRY_SECONDS . saturating_mul ( 1000 ) ) ;
405- put_refresh_state ( store, & state) . await ?;
477+ persist_refresh_state_if_current ( store, & state, expected_version ) . await ?;
406478 warn ! (
407479 provider = %state. provider_name,
408480 credential_key = %state. credential_key,
@@ -982,8 +1054,8 @@ async fn run_refresh_worker_tick(store: &Store) -> Result<(), Status> {
9821054#[ cfg( test) ]
9831055mod tests {
9841056 use super :: {
985- NewRefreshStateConfig , get_refresh_state , new_refresh_state , put_refresh_state ,
986- refresh_provider_credential, refresh_state_name, refresh_strategy_name,
1057+ NewRefreshStateConfig , delete_refresh_state , get_refresh_state , new_refresh_state ,
1058+ put_refresh_state , refresh_provider_credential, refresh_state_name, refresh_strategy_name,
9871059 run_refresh_worker_tick, seconds_until_ms,
9881060 } ;
9891061 use crate :: persistence:: test_store;
@@ -1749,6 +1821,139 @@ mod tests {
17491821 assert ! ( err. message( ) . contains( "AWS_SECRET_ACCESS_KEY" ) ) ;
17501822 }
17511823
1824+ // A wiremock responder that blocks the STS response until the test releases
1825+ // it, so a delete-refresh can be interleaved deterministically while the
1826+ // rotation is parked awaiting STS.
1827+ struct GatedStsResponder {
1828+ hit : std:: sync:: Mutex < Option < tokio:: sync:: oneshot:: Sender < ( ) > > > ,
1829+ release : std:: sync:: Mutex < std:: sync:: mpsc:: Receiver < ( ) > > ,
1830+ body : String ,
1831+ }
1832+
1833+ impl wiremock:: Respond for GatedStsResponder {
1834+ fn respond ( & self , _request : & wiremock:: Request ) -> ResponseTemplate {
1835+ let hit = self . hit . lock ( ) . unwrap ( ) . take ( ) ;
1836+ if let Some ( hit) = hit {
1837+ let _ = hit. send ( ( ) ) ;
1838+ }
1839+ let _ = self . release . lock ( ) . unwrap ( ) . recv ( ) ;
1840+ ResponseTemplate :: new ( 200 ) . set_body_string ( self . body . clone ( ) )
1841+ }
1842+ }
1843+
1844+ #[ tokio:: test( flavor = "multi_thread" , worker_threads = 4 ) ]
1845+ async fn rotation_does_not_resurrect_refresh_deleted_mid_flight ( ) {
1846+ let mock_server = MockServer :: start ( ) . await ;
1847+ let ( hit_tx, hit_rx) = tokio:: sync:: oneshot:: channel :: < ( ) > ( ) ;
1848+ let ( release_tx, release_rx) = std:: sync:: mpsc:: channel :: < ( ) > ( ) ;
1849+ Mock :: given ( method ( "POST" ) )
1850+ . and ( body_string_contains ( "Action=AssumeRole" ) )
1851+ . respond_with ( GatedStsResponder {
1852+ hit : std:: sync:: Mutex :: new ( Some ( hit_tx) ) ,
1853+ release : std:: sync:: Mutex :: new ( release_rx) ,
1854+ body : r#"<AssumeRoleResponse xmlns="https://sts.amazonaws.com/doc/2011-06-15/">
1855+ <AssumeRoleResult>
1856+ <Credentials>
1857+ <AccessKeyId>ASIAMOCKKEY</AccessKeyId>
1858+ <SecretAccessKey>MockSecretAccessKey123</SecretAccessKey>
1859+ <SessionToken>MockSessionTokenXYZ</SessionToken>
1860+ <Expiration>2099-01-01T00:00:00Z</Expiration>
1861+ </Credentials>
1862+ </AssumeRoleResult>
1863+ </AssumeRoleResponse>"#
1864+ . to_string ( ) ,
1865+ } )
1866+ . mount ( & mock_server)
1867+ . await ;
1868+
1869+ let store = test_store ( ) . await ;
1870+ crate :: grpc:: policy:: set_global_bool_setting_for_test (
1871+ & store,
1872+ openshell_core:: settings:: PROVIDERS_V2_ENABLED_KEY ,
1873+ true ,
1874+ )
1875+ . await
1876+ . unwrap ( ) ;
1877+ let prov = provider ( "aws-race" , "aws" ) ;
1878+ store. put_message ( & prov) . await . unwrap ( ) ;
1879+ let provider_id = prov. object_id ( ) . to_string ( ) ;
1880+
1881+ let state = new_refresh_state (
1882+ & prov,
1883+ "AWS_ACCESS_KEY_ID" ,
1884+ NewRefreshStateConfig {
1885+ additional_output_keys : HashMap :: from ( [
1886+ (
1887+ "secret_access_key" . to_string ( ) ,
1888+ "AWS_SECRET_ACCESS_KEY" . to_string ( ) ,
1889+ ) ,
1890+ ( "session_token" . to_string ( ) , "AWS_SESSION_TOKEN" . to_string ( ) ) ,
1891+ ] ) ,
1892+ strategy : ProviderCredentialRefreshStrategy :: AwsStsAssumeRole ,
1893+ material : HashMap :: from ( [
1894+ (
1895+ "role_arn" . to_string ( ) ,
1896+ "arn:aws:iam::123456789012:role/TestRole" . to_string ( ) ,
1897+ ) ,
1898+ ( "aws_access_key_id" . to_string ( ) , "AKIATESTKEY" . to_string ( ) ) ,
1899+ (
1900+ "aws_secret_access_key" . to_string ( ) ,
1901+ "TestSecretKey" . to_string ( ) ,
1902+ ) ,
1903+ ( "sts_endpoint_url" . to_string ( ) , mock_server. uri ( ) ) ,
1904+ ] ) ,
1905+ secret_material_keys : vec ! [ "aws_secret_access_key" . to_string( ) ] ,
1906+ expires_at_ms : 0 ,
1907+ token_url : String :: new ( ) ,
1908+ scopes : Vec :: new ( ) ,
1909+ refresh_before_seconds : 300 ,
1910+ max_lifetime_seconds : 3600 ,
1911+ } ,
1912+ )
1913+ . unwrap ( ) ;
1914+ put_refresh_state ( & store, & state) . await . unwrap ( ) ;
1915+
1916+ let rotate = refresh_provider_credential ( & store, "aws-race" , "AWS_ACCESS_KEY_ID" ) ;
1917+ let interfere = async {
1918+ // Wait until the rotation is inside the STS call (its state read has
1919+ // already happened), then delete the refresh and release STS.
1920+ if tokio:: time:: timeout ( std:: time:: Duration :: from_secs ( 15 ) , hit_rx)
1921+ . await
1922+ . is_err ( )
1923+ {
1924+ return ;
1925+ }
1926+ delete_refresh_state ( & store, & provider_id, "AWS_ACCESS_KEY_ID" )
1927+ . await
1928+ . unwrap ( ) ;
1929+ let _ = release_tx. send ( ( ) ) ;
1930+ } ;
1931+ let ( rotate_result, ( ) ) = tokio:: join!( rotate, interfere) ;
1932+
1933+ // The rotation must fail rather than complete against a deleted refresh.
1934+ assert ! (
1935+ rotate_result. is_err( ) ,
1936+ "rotation should abort when its refresh is deleted mid-flight"
1937+ ) ;
1938+ // The deleted refresh state must not be resurrected.
1939+ assert ! (
1940+ get_refresh_state( & store, & provider_id, "AWS_ACCESS_KEY_ID" )
1941+ . await
1942+ . unwrap( )
1943+ . is_none( ) ,
1944+ "deleted refresh state must not be recreated"
1945+ ) ;
1946+ // No credentials were minted into the provider.
1947+ let stored = store
1948+ . get_message_by_name :: < Provider > ( "aws-race" )
1949+ . await
1950+ . unwrap ( )
1951+ . unwrap ( ) ;
1952+ assert ! ( !stored. credentials. contains_key( "AWS_ACCESS_KEY_ID" ) ) ;
1953+ assert ! ( !stored. credentials. contains_key( "AWS_SECRET_ACCESS_KEY" ) ) ;
1954+ assert ! ( !stored. credentials. contains_key( "AWS_SESSION_TOKEN" ) ) ;
1955+ }
1956+
17521957 fn provider ( name : & str , provider_type : & str ) -> Provider {
17531958 Provider {
17541959 metadata : Some ( ObjectMeta {
0 commit comments