@@ -926,7 +926,35 @@ async fn cleanup_pending_secret_deletions_coordinated(
926926 Ok ( ( ) )
927927}
928928
929- /// Cancellation-safe entry point shared by explicit rotation and the worker.
929+ #[ derive( Clone , Copy , PartialEq , Eq ) ]
930+ enum RefreshTrigger {
931+ Automatic ,
932+ ExplicitRotation ,
933+ }
934+
935+ /// Automatic callers must honor the live failure schedule, not the eligibility
936+ /// observed before they queued. Configuration/investigation failures can still
937+ /// be retried by the worker once due; reauthorization stays parked.
938+ fn ensure_automatic_refresh_ready (
939+ state : & StoredProviderCredentialRefreshState ,
940+ now_ms : i64 ,
941+ ) -> Result < ( ) , Status > {
942+ if state. recovery_action == ProviderCredentialRefreshRecoveryAction :: Reauthorize as i32 {
943+ return Err ( Status :: failed_precondition (
944+ "credential refresh requires recovery" ,
945+ ) ) ;
946+ }
947+ if state. recovery_action != ProviderCredentialRefreshRecoveryAction :: Unspecified as i32
948+ && state. next_refresh_at_ms > now_ms
949+ {
950+ return Err ( Status :: unavailable (
951+ "credential refresh is in retry backoff" ,
952+ ) ) ;
953+ }
954+ Ok ( ( ) )
955+ }
956+
957+ /// Cancellation-safe entry point for a separately requested manual rotation.
930958pub async fn refresh_provider_credential (
931959 store : & Store ,
932960 workspace : & str ,
@@ -943,15 +971,40 @@ pub async fn refresh_provider_credential(
943971 let snapshot = get_refresh_state ( store, workspace, provider. object_id ( ) , credential_key)
944972 . await ?
945973 . ok_or_else ( || Status :: not_found ( "provider refresh state not found" ) ) ?;
946- refresh_from_snapshot ( store, credentials, compute, snapshot) . await
974+ refresh_from_snapshot_with_trigger (
975+ store,
976+ credentials,
977+ compute,
978+ snapshot,
979+ RefreshTrigger :: ExplicitRotation ,
980+ )
981+ . await
947982}
948983
949- /// Coalesce against the caller's observed generation, not a name-keyed replacement.
984+ /// Coalesce an automatic refresh against the caller's observed generation, not
985+ /// a name-keyed replacement. Recheck recovery/backoff after acquiring the locks.
950986pub async fn refresh_from_snapshot (
951987 store : & Store ,
952988 credentials : & crate :: credentials:: CredentialRuntime ,
953989 compute : Option < & crate :: compute:: ComputeRuntime > ,
954990 snapshot : StoredProviderCredentialRefreshState ,
991+ ) -> Result < StoredProviderCredentialRefreshState , Status > {
992+ refresh_from_snapshot_with_trigger (
993+ store,
994+ credentials,
995+ compute,
996+ snapshot,
997+ RefreshTrigger :: Automatic ,
998+ )
999+ . await
1000+ }
1001+
1002+ async fn refresh_from_snapshot_with_trigger (
1003+ store : & Store ,
1004+ credentials : & crate :: credentials:: CredentialRuntime ,
1005+ compute : Option < & crate :: compute:: ComputeRuntime > ,
1006+ snapshot : StoredProviderCredentialRefreshState ,
1007+ trigger : RefreshTrigger ,
9551008) -> Result < StoredProviderCredentialRefreshState , Status > {
9561009 let admission = REFRESH_ADMISSIONS
9571010 . try_acquire ( )
@@ -1021,6 +1074,9 @@ pub async fn refresh_from_snapshot(
10211074 "previous credential mint has an uncertain outcome; reconfigure the refresh grant" ,
10221075 ) ) ;
10231076 }
1077+ if trigger == RefreshTrigger :: Automatic {
1078+ ensure_automatic_refresh_ready ( & latest, current_time_ms ( ) ) ?;
1079+ }
10241080 refresh_provider_credential_locked (
10251081 & store,
10261082 snapshot. object_workspace ( ) ,
@@ -2318,7 +2374,15 @@ async fn refresh_states(
23182374 status = %state. status,
23192375 "refreshing provider credential"
23202376 ) ;
2321- if let Err ( err) = refresh_from_snapshot ( store, credentials, compute, state. clone ( ) ) . await {
2377+ let trigger = if rotation_requested {
2378+ RefreshTrigger :: ExplicitRotation
2379+ } else {
2380+ RefreshTrigger :: Automatic
2381+ } ;
2382+ if let Err ( err) =
2383+ refresh_from_snapshot_with_trigger ( store, credentials, compute, state. clone ( ) , trigger)
2384+ . await
2385+ {
23222386 warn ! (
23232387 provider = %state. provider_name,
23242388 credential_key = %state. credential_key,
@@ -2338,9 +2402,9 @@ mod tests {
23382402 MAX_OAUTH_ERROR_RESPONSE_BYTES , NewRefreshStateConfig , OAuthGrantKind , RefreshFailure ,
23392403 RefreshRetrySchedule , Status , classify_oauth_token_error,
23402404 delete_refresh_state_with_credentials, effective_authorization_epoch,
2341- enqueue_pending_secret_deletion, get_refresh_state , list_all_refresh_states ,
2342- list_refresh_states_for_provider , max_lifetime_seconds , new_refresh_state ,
2343- next_refresh_at_ms, put_refresh_state, read_bounded_oauth_error_body,
2405+ enqueue_pending_secret_deletion, ensure_automatic_refresh_ready , get_refresh_state ,
2406+ list_all_refresh_states , list_refresh_states_for_provider , max_lifetime_seconds ,
2407+ new_refresh_state , next_refresh_at_ms, put_refresh_state, read_bounded_oauth_error_body,
23442408 refresh_has_expiration, refresh_material_scope, refresh_provider_credential,
23452409 refresh_state_name, refresh_status_from_state, refresh_strategy_name,
23462410 run_refresh_worker_tick, seconds_until_ms, set_refresh_expiration_presence,
@@ -3849,6 +3913,48 @@ mod tests {
38493913 ) ;
38503914 }
38513915
3916+ #[ test]
3917+ fn automatic_refresh_honors_recovery_schedule_without_disabling_due_retries ( ) {
3918+ let now_ms = 10_000 ;
3919+ for recovery in [
3920+ ProviderCredentialRefreshRecoveryAction :: Retry ,
3921+ ProviderCredentialRefreshRecoveryAction :: FixConfiguration ,
3922+ ProviderCredentialRefreshRecoveryAction :: Investigate ,
3923+ ] {
3924+ let mut state = StoredProviderCredentialRefreshState {
3925+ recovery_action : recovery as i32 ,
3926+ next_refresh_at_ms : now_ms + 1 ,
3927+ ..Default :: default ( )
3928+ } ;
3929+ assert_eq ! (
3930+ ensure_automatic_refresh_ready( & state, now_ms)
3931+ . unwrap_err( )
3932+ . code( ) ,
3933+ tonic:: Code :: Unavailable ,
3934+ ) ;
3935+ state. next_refresh_at_ms = now_ms;
3936+ ensure_automatic_refresh_ready ( & state, now_ms) . unwrap ( ) ;
3937+ }
3938+ let state = StoredProviderCredentialRefreshState {
3939+ recovery_action : ProviderCredentialRefreshRecoveryAction :: Reauthorize as i32 ,
3940+ next_refresh_at_ms : now_ms,
3941+ ..Default :: default ( )
3942+ } ;
3943+ assert_eq ! (
3944+ ensure_automatic_refresh_ready( & state, now_ms)
3945+ . unwrap_err( )
3946+ . code( ) ,
3947+ tonic:: Code :: FailedPrecondition ,
3948+ ) ;
3949+ // A healthy credential's proactive schedule must not block an on-read
3950+ // mint requested to meet a longer minimum remaining lifetime.
3951+ let state = StoredProviderCredentialRefreshState {
3952+ next_refresh_at_ms : now_ms + 1 ,
3953+ ..Default :: default ( )
3954+ } ;
3955+ ensure_automatic_refresh_ready ( & state, now_ms) . unwrap ( ) ;
3956+ }
3957+
38523958 #[ tokio:: test]
38533959 async fn refresh_worker_skips_parked_reauthorization_state ( ) {
38543960 let store = test_store ( ) . await ;
0 commit comments