@@ -52,14 +52,14 @@ pub async fn put_refresh_state(
5252///
5353/// Uses a version-matched UPDATE, which never inserts — so a refresh deleted
5454/// while an STS (or OAuth) request was in flight is not resurrected, and its
55- /// stored source-credential material is not recreated (CWE-362). Returns `true`
56- /// when persisted, `false ` when the refresh was deleted or superseded by a
57- /// concurrent write (in which case nothing was written).
55+ /// stored source-credential material is not recreated (CWE-362). Returns the new
56+ /// resource version when persisted, or `None ` when the refresh was deleted or
57+ /// superseded by a concurrent write (in which case nothing was written).
5858async fn persist_refresh_state_if_current (
5959 store : & Store ,
6060 state : & StoredProviderCredentialRefreshState ,
6161 expected_version : u64 ,
62- ) -> Result < bool , Status > {
62+ ) -> Result < Option < u64 > , Status > {
6363 match store
6464 . put_if (
6565 StoredProviderCredentialRefreshState :: object_type ( ) ,
@@ -71,11 +71,11 @@ async fn persist_refresh_state_if_current(
7171 )
7272 . await
7373 {
74- Ok ( _ ) => Ok ( true ) ,
75- // Version mismatch (superseded) or the row is gone (deleted): the
76- // version-matched UPDATE matched no row, so nothing was written.
77- Err ( PersistenceError :: Conflict { .. } ) => Ok ( false ) ,
78- Err ( PersistenceError :: Database ( message ) ) if message . contains ( "not found" ) => Ok ( false ) ,
74+ Ok ( result ) => Ok ( Some ( result . resource_version ) ) ,
75+ // The version-matched UPDATE matched no row: the refresh was deleted
76+ // (current version `None`) or superseded by a concurrent write (a
77+ // different current version). Either way nothing was written.
78+ Err ( PersistenceError :: Conflict { .. } ) => Ok ( None ) ,
7979 Err ( e) => Err ( Status :: internal ( format ! (
8080 "persist provider refresh state failed: {e}"
8181 ) ) ) ,
@@ -383,47 +383,9 @@ pub async fn refresh_provider_credential(
383383 match mint_credential ( & state) . await {
384384 Ok ( minted) => {
385385 let now_ms = current_time_ms ( ) ;
386- // A delete-refresh can complete while the mint request is in flight.
387- // Verify the refresh still exists before writing minted credentials
388- // into the provider, so a removed refresh is not silently re-minted
389- // (CWE-362). The version-matched persist below is the authoritative
390- // guard against resurrecting deleted state; this check just avoids
391- // writing credentials we are about to discard.
392- if get_refresh_state ( store, provider. object_id ( ) , credential_key)
393- . await ?
394- . is_none ( )
395- {
396- warn ! (
397- provider = %state. provider_name,
398- credential_key = %state. credential_key,
399- strategy = %refresh_strategy_name( state. strategy) ,
400- "provider credential refresh deleted during rotation; discarding minted credentials"
401- ) ;
402- return Err ( Status :: aborted (
403- "provider refresh was deleted during rotation" ,
404- ) ) ;
405- }
406- if let Err ( err) =
407- apply_minted_credential ( store, & provider, credential_key, & minted) . await
408- {
409- state. status = "error" . to_string ( ) ;
410- state. last_error = err. message ( ) . to_string ( ) ;
411- state. next_refresh_at_ms =
412- now_ms. saturating_add ( REFRESH_ERROR_RETRY_SECONDS . saturating_mul ( 1000 ) ) ;
413- persist_refresh_state_if_current ( store, & state, expected_version) . await ?;
414- warn ! (
415- provider = %state. provider_name,
416- credential_key = %state. credential_key,
417- strategy = %refresh_strategy_name( state. strategy) ,
418- status = %state. status,
419- next_refresh_at_ms = state. next_refresh_at_ms,
420- seconds_until_refresh = seconds_until_ms( now_ms, state. next_refresh_at_ms) ,
421- error = %err,
422- "provider credential refresh errored"
423- ) ;
424- return Err ( err) ;
425- }
426- if let Some ( refresh_token) = minted. refresh_token {
386+ // Fold the minted result into the refresh state before claiming the
387+ // generation.
388+ if let Some ( refresh_token) = minted. refresh_token . clone ( ) {
427389 state
428390 . material
429391 . insert ( "refresh_token" . to_string ( ) , refresh_token) ;
@@ -445,16 +407,51 @@ pub async fn refresh_provider_credential(
445407 state. last_refresh_at_ms = now_ms;
446408 state. status = "refreshed" . to_string ( ) ;
447409 state. last_error . clear ( ) ;
448- if !persist_refresh_state_if_current ( store, & state, expected_version) . await ? {
410+
411+ // Claim the refresh generation with a version-matched write BEFORE
412+ // touching the provider. It succeeds only if the refresh still holds
413+ // the generation we started from; if it was deleted, recreated, or a
414+ // concurrent rotation won, it returns `None` and we leave both the
415+ // refresh state and the provider unchanged — no credentials minted
416+ // from a stale generation are written, and a deleted refresh is not
417+ // resurrected (CWE-362). This makes generation ownership the gate on
418+ // the provider credential write.
419+ let Some ( new_version) =
420+ persist_refresh_state_if_current ( store, & state, expected_version) . await ?
421+ else {
449422 warn ! (
450423 provider = %state. provider_name,
451424 credential_key = %state. credential_key,
452425 strategy = %refresh_strategy_name( state. strategy) ,
453- "provider credential refresh deleted or superseded during rotation; not resurrecting refresh state "
426+ "provider credential refresh deleted or superseded during rotation; discarding minted credentials "
454427 ) ;
455428 return Err ( Status :: aborted (
456- "provider refresh was deleted during rotation" ,
429+ "provider refresh was deleted or superseded during rotation" ,
457430 ) ) ;
431+ } ;
432+
433+ // Generation is ours; write the minted credentials into the provider.
434+ if let Err ( err) =
435+ apply_minted_credential ( store, & provider, credential_key, & minted) . await
436+ {
437+ state. status = "error" . to_string ( ) ;
438+ state. last_error = err. message ( ) . to_string ( ) ;
439+ state. next_refresh_at_ms =
440+ now_ms. saturating_add ( REFRESH_ERROR_RETRY_SECONDS . saturating_mul ( 1000 ) ) ;
441+ // Reflect the failure on the state we just wrote; skip silently
442+ // if it was deleted concurrently (it is not recreated).
443+ persist_refresh_state_if_current ( store, & state, new_version) . await ?;
444+ warn ! (
445+ provider = %state. provider_name,
446+ credential_key = %state. credential_key,
447+ strategy = %refresh_strategy_name( state. strategy) ,
448+ status = %state. status,
449+ next_refresh_at_ms = state. next_refresh_at_ms,
450+ seconds_until_refresh = seconds_until_ms( now_ms, state. next_refresh_at_ms) ,
451+ error = %err,
452+ "provider credential refresh errored"
453+ ) ;
454+ return Err ( err) ;
458455 }
459456 info ! (
460457 provider = %state. provider_name,
@@ -1954,6 +1951,122 @@ mod tests {
19541951 assert ! ( !stored. credentials. contains_key( "AWS_SESSION_TOKEN" ) ) ;
19551952 }
19561953
1954+ #[ tokio:: test( flavor = "multi_thread" , worker_threads = 4 ) ]
1955+ async fn rotation_superseded_mid_flight_discards_credentials ( ) {
1956+ let mock_server = MockServer :: start ( ) . await ;
1957+ let ( hit_tx, hit_rx) = tokio:: sync:: oneshot:: channel :: < ( ) > ( ) ;
1958+ let ( release_tx, release_rx) = std:: sync:: mpsc:: channel :: < ( ) > ( ) ;
1959+ Mock :: given ( method ( "POST" ) )
1960+ . and ( body_string_contains ( "Action=AssumeRole" ) )
1961+ . respond_with ( GatedStsResponder {
1962+ hit : std:: sync:: Mutex :: new ( Some ( hit_tx) ) ,
1963+ release : std:: sync:: Mutex :: new ( release_rx) ,
1964+ body : r#"<AssumeRoleResponse xmlns="https://sts.amazonaws.com/doc/2011-06-15/">
1965+ <AssumeRoleResult>
1966+ <Credentials>
1967+ <AccessKeyId>ASIAMOCKKEY</AccessKeyId>
1968+ <SecretAccessKey>MockSecretAccessKey123</SecretAccessKey>
1969+ <SessionToken>MockSessionTokenXYZ</SessionToken>
1970+ <Expiration>2099-01-01T00:00:00Z</Expiration>
1971+ </Credentials>
1972+ </AssumeRoleResult>
1973+ </AssumeRoleResponse>"#
1974+ . to_string ( ) ,
1975+ } )
1976+ . mount ( & mock_server)
1977+ . await ;
1978+
1979+ let store = test_store ( ) . await ;
1980+ crate :: grpc:: policy:: set_global_bool_setting_for_test (
1981+ & store,
1982+ openshell_core:: settings:: PROVIDERS_V2_ENABLED_KEY ,
1983+ true ,
1984+ )
1985+ . await
1986+ . unwrap ( ) ;
1987+ let prov = provider ( "aws-superseded" , "aws" ) ;
1988+ store. put_message ( & prov) . await . unwrap ( ) ;
1989+ let provider_id = prov. object_id ( ) . to_string ( ) ;
1990+
1991+ let state = new_refresh_state (
1992+ & prov,
1993+ "AWS_ACCESS_KEY_ID" ,
1994+ NewRefreshStateConfig {
1995+ additional_output_keys : HashMap :: from ( [
1996+ (
1997+ "secret_access_key" . to_string ( ) ,
1998+ "AWS_SECRET_ACCESS_KEY" . to_string ( ) ,
1999+ ) ,
2000+ ( "session_token" . to_string ( ) , "AWS_SESSION_TOKEN" . to_string ( ) ) ,
2001+ ] ) ,
2002+ strategy : ProviderCredentialRefreshStrategy :: AwsStsAssumeRole ,
2003+ material : HashMap :: from ( [
2004+ (
2005+ "role_arn" . to_string ( ) ,
2006+ "arn:aws:iam::123456789012:role/TestRole" . to_string ( ) ,
2007+ ) ,
2008+ ( "aws_access_key_id" . to_string ( ) , "AKIATESTKEY" . to_string ( ) ) ,
2009+ (
2010+ "aws_secret_access_key" . to_string ( ) ,
2011+ "TestSecretKey" . to_string ( ) ,
2012+ ) ,
2013+ ( "sts_endpoint_url" . to_string ( ) , mock_server. uri ( ) ) ,
2014+ ] ) ,
2015+ secret_material_keys : vec ! [ "aws_secret_access_key" . to_string( ) ] ,
2016+ expires_at_ms : 0 ,
2017+ token_url : String :: new ( ) ,
2018+ scopes : Vec :: new ( ) ,
2019+ refresh_before_seconds : 300 ,
2020+ max_lifetime_seconds : 3600 ,
2021+ } ,
2022+ )
2023+ . unwrap ( ) ;
2024+ put_refresh_state ( & store, & state) . await . unwrap ( ) ;
2025+
2026+ let rotate = refresh_provider_credential ( & store, "aws-superseded" , "AWS_ACCESS_KEY_ID" ) ;
2027+ let interfere = async {
2028+ if tokio:: time:: timeout ( std:: time:: Duration :: from_secs ( 15 ) , hit_rx)
2029+ . await
2030+ . is_err ( )
2031+ {
2032+ return ;
2033+ }
2034+ // Simulate a concurrent rotation or reconfigure winning the
2035+ // generation: any write to the refresh state bumps its version, so
2036+ // the in-flight rotation's version-matched persist will lose.
2037+ let mut winner = get_refresh_state ( & store, & provider_id, "AWS_ACCESS_KEY_ID" )
2038+ . await
2039+ . unwrap ( )
2040+ . unwrap ( ) ;
2041+ winner. last_error = "won-by-concurrent-writer" . to_string ( ) ;
2042+ put_refresh_state ( & store, & winner) . await . unwrap ( ) ;
2043+ let _ = release_tx. send ( ( ) ) ;
2044+ } ;
2045+ let ( rotate_result, ( ) ) = tokio:: join!( rotate, interfere) ;
2046+
2047+ // The superseded (losing) rotation must abort rather than complete.
2048+ assert ! (
2049+ rotate_result. is_err( ) ,
2050+ "a rotation whose generation was superseded must abort"
2051+ ) ;
2052+ // It must not write its stale-generation credentials into the provider.
2053+ let stored = store
2054+ . get_message_by_name :: < Provider > ( "aws-superseded" )
2055+ . await
2056+ . unwrap ( )
2057+ . unwrap ( ) ;
2058+ assert ! ( !stored. credentials. contains_key( "AWS_ACCESS_KEY_ID" ) ) ;
2059+ assert ! ( !stored. credentials. contains_key( "AWS_SECRET_ACCESS_KEY" ) ) ;
2060+ assert ! ( !stored. credentials. contains_key( "AWS_SESSION_TOKEN" ) ) ;
2061+ // The concurrent writer's refresh state must survive untouched.
2062+ let refresh = get_refresh_state ( & store, & provider_id, "AWS_ACCESS_KEY_ID" )
2063+ . await
2064+ . unwrap ( )
2065+ . expect ( "refresh state should still exist" ) ;
2066+ assert_eq ! ( refresh. last_error, "won-by-concurrent-writer" ) ;
2067+ assert_ne ! ( refresh. status, "refreshed" ) ;
2068+ }
2069+
19572070 fn provider ( name : & str , provider_type : & str ) -> Provider {
19582071 Provider {
19592072 metadata : Some ( ObjectMeta {
0 commit comments