Skip to content

Commit 2b89ee5

Browse files
committed
fix(server): honor recovery state for queued automatic refreshes
Signed-off-by: Seth Jennings <[email protected]>
1 parent 53593c9 commit 2b89ee5

3 files changed

Lines changed: 253 additions & 15 deletions

File tree

‎crates/openshell-server/README.md‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -35,8 +35,8 @@ closed without falling back to inline values.
3535

3636
## Refresh coordination
3737

38-
Explicit rotation, background refresh, and operator refresh-on-read share
39-
`refresh_from_snapshot`:
38+
Explicit rotation, background refresh, and operator refresh-on-read share the
39+
same refresh coordination:
4040

4141
- A process-local weak keyed mutex coalesces waiters without retaining deleted
4242
providers indefinitely.
@@ -50,6 +50,9 @@ Explicit rotation, background refresh, and operator refresh-on-read share
5050
`refreshed` state, with a unique completed-mint identity in metadata
5151
annotations. Waiters coalesce against that identity, not bookkeeping changes
5252
to the resource version.
53+
- Automatic callers recheck live recovery and retry deadlines under both locks.
54+
A failed mint blocks queued automatic exchanges without disabling scheduled
55+
worker retries or a separately requested manual rotation.
5356
- Detached refresh tasks complete despite request cancellation, preserving
5457
replacement refresh tokens returned by the issuer.
5558
- Existing generation/CAS fencing remains active for deletion and

‎crates/openshell-server/src/grpc/provider_credentials_tests.rs‎

Lines changed: 135 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -101,9 +101,12 @@ async fn audits_exports_and_invalid_attempts_without_values() {
101101
.with_writer(move || AuditWriter(writer.clone()))
102102
.finish();
103103
let _guard = tracing::subscriber::set_default(subscriber);
104-
handle(&state, authenticated(request(&["ACCESS_TOKEN"])))
105-
.await
106-
.unwrap();
104+
handle(
105+
&state,
106+
authenticated(request(&["ACCESS_TOKEN", "STATIC_KEY"])),
107+
)
108+
.await
109+
.unwrap();
107110
let error = handle(
108111
&state,
109112
authenticated(request(&["API_KEY=secret-from-invalid-input"])),
@@ -115,12 +118,13 @@ async fn audits_exports_and_invalid_attempts_without_values() {
115118
assert!(output.contains("operator-test"));
116119
assert!(output.contains("verified-test-certificate"));
117120
assert!(output.contains("ACCESS_TOKEN"));
121+
assert!(output.contains("STATIC_KEY"));
118122
assert!(output.contains("delivered"));
119123
assert!(output.contains("invalid_request"));
120124
assert!(output.contains("refresh_attempts=0"));
121125
for secret in [
122-
"access-value",
123-
"static-value",
126+
"old-access-token",
127+
"static-secret",
124128
"must-not-export",
125129
"secret-from-invalid-input",
126130
] {
@@ -360,7 +364,7 @@ async fn missing_static_handle_and_invalid_refresh_driver_fail_closed() {
360364
}
361365

362366
#[tokio::test]
363-
async fn bookkeeping_changes_do_not_coalesce_explicit_rotation() {
367+
async fn bookkeeping_changes_do_not_coalesce_refresh_requests() {
364368
// Cover both legacy states without a mint identity and states that already
365369
// have one. Cleanup/metadata updates must not count as another mint.
366370
for previously_minted in [false, true] {
@@ -556,6 +560,131 @@ async fn coalesces_observed_refresh_generation_and_parks_uncertain_mint() {
556560
issuer.verify().await;
557561
}
558562

563+
async fn automatic_waiters_honor_failed_refresh(
564+
response: wiremock::ResponseTemplate,
565+
expected_code: Code,
566+
expected_status: &str,
567+
expected_recovery: ProviderCredentialRefreshRecoveryAction,
568+
) {
569+
let (state, provider) = fixture().await;
570+
let issuer = wiremock::MockServer::start().await;
571+
wiremock::Mock::given(wiremock::matchers::method("POST"))
572+
.respond_with(response.set_delay(Duration::from_millis(100)))
573+
.expect(1)
574+
.mount(&issuer)
575+
.await;
576+
let snapshot = refresh_fixture(&state, &provider, issuer.uri()).await;
577+
let (a, b) = tokio::join!(
578+
crate::provider_refresh::refresh_from_snapshot(
579+
state.store.as_ref(),
580+
&state.credentials,
581+
Some(&state.compute),
582+
snapshot.clone(),
583+
),
584+
crate::provider_refresh::refresh_from_snapshot(
585+
state.store.as_ref(),
586+
&state.credentials,
587+
Some(&state.compute),
588+
snapshot.clone(),
589+
),
590+
);
591+
assert_eq!(a.unwrap_err().code(), expected_code);
592+
assert_eq!(b.unwrap_err().code(), expected_code);
593+
issuer.verify().await;
594+
let failed = crate::provider_refresh::get_refresh_state(
595+
state.store.as_ref(),
596+
"other",
597+
provider.object_id(),
598+
"ACCESS_TOKEN",
599+
)
600+
.await
601+
.unwrap()
602+
.unwrap();
603+
assert_eq!(failed.status, expected_status);
604+
assert_eq!(failed.recovery_action, expected_recovery as i32);
605+
assert!(!failed.failure_code.is_empty());
606+
assert!(failed.next_refresh_at_ms > crate::persistence::current_time_ms());
607+
if expected_recovery == ProviderCredentialRefreshRecoveryAction::Reauthorize {
608+
assert_eq!(failed.next_refresh_at_ms, i64::MAX);
609+
}
610+
// Both a stale snapshot and a fresh automatic request must respect the
611+
// committed failure, without changing its retry deadline or error receipt.
612+
for observed in [snapshot, failed.clone()] {
613+
let error = crate::provider_refresh::refresh_from_snapshot(
614+
state.store.as_ref(),
615+
&state.credentials,
616+
Some(&state.compute),
617+
observed,
618+
)
619+
.await
620+
.unwrap_err();
621+
assert_eq!(error.code(), expected_code);
622+
}
623+
assert_eq!(
624+
crate::provider_refresh::get_refresh_state(
625+
state.store.as_ref(),
626+
"other",
627+
provider.object_id(),
628+
"ACCESS_TOKEN",
629+
)
630+
.await
631+
.unwrap()
632+
.unwrap(),
633+
failed,
634+
);
635+
issuer.verify().await;
636+
637+
// A separately requested manual rotation can retry a parked grant or
638+
// override retry backoff; it still goes through the same coordination.
639+
issuer.reset().await;
640+
wiremock::Mock::given(wiremock::matchers::method("POST"))
641+
.respond_with(wiremock::ResponseTemplate::new(200).set_body_json(
642+
serde_json::json!({"access_token":"manually-refreshed","expires_in":3600}),
643+
))
644+
.expect(1)
645+
.mount(&issuer)
646+
.await;
647+
let rotated = crate::provider_refresh::refresh_provider_credential(
648+
state.store.as_ref(),
649+
"other",
650+
&state.credentials,
651+
Some(&state.compute),
652+
"provider",
653+
"ACCESS_TOKEN",
654+
)
655+
.await
656+
.unwrap();
657+
assert_eq!(rotated.status, "refreshed");
658+
assert_eq!(
659+
rotated.recovery_action,
660+
ProviderCredentialRefreshRecoveryAction::Unspecified as i32,
661+
);
662+
issuer.verify().await;
663+
}
664+
665+
#[tokio::test]
666+
async fn automatic_waiters_do_not_repeat_unusable_refresh_token_exchange() {
667+
automatic_waiters_honor_failed_refresh(
668+
wiremock::ResponseTemplate::new(200).set_body_string("not JSON"),
669+
Code::FailedPrecondition,
670+
"reauthorization_required",
671+
ProviderCredentialRefreshRecoveryAction::Reauthorize,
672+
)
673+
.await;
674+
}
675+
676+
#[tokio::test]
677+
async fn automatic_waiters_do_not_bypass_rate_limit_backoff() {
678+
automatic_waiters_honor_failed_refresh(
679+
wiremock::ResponseTemplate::new(429)
680+
.set_body_json(serde_json::json!({"error":"temporarily_unavailable"})),
681+
Code::Unavailable,
682+
"error",
683+
ProviderCredentialRefreshRecoveryAction::Retry,
684+
)
685+
.await;
686+
}
687+
559688
#[tokio::test]
560689
async fn rejects_supervisor_dynamic_grants_even_with_stored_values() {
561690
let (state, _) = fixture().await;

‎crates/openshell-server/src/provider_refresh.rs‎

Lines changed: 113 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -926,7 +926,35 @@ async fn cleanup_pending_secret_deletions_coordinated(
926926
Ok(())
927927
}
928928

929-
/// Cancellation-safe entry point shared by explicit rotation and the worker.
929+
#[derive(Clone, Copy, PartialEq, Eq)]
930+
enum RefreshTrigger {
931+
Automatic,
932+
ExplicitRotation,
933+
}
934+
935+
/// Automatic callers must honor the live failure schedule, not the eligibility
936+
/// observed before they queued. Configuration/investigation failures can still
937+
/// be retried by the worker once due; reauthorization stays parked.
938+
fn ensure_automatic_refresh_ready(
939+
state: &StoredProviderCredentialRefreshState,
940+
now_ms: i64,
941+
) -> Result<(), Status> {
942+
if state.recovery_action == ProviderCredentialRefreshRecoveryAction::Reauthorize as i32 {
943+
return Err(Status::failed_precondition(
944+
"credential refresh requires recovery",
945+
));
946+
}
947+
if state.recovery_action != ProviderCredentialRefreshRecoveryAction::Unspecified as i32
948+
&& state.next_refresh_at_ms > now_ms
949+
{
950+
return Err(Status::unavailable(
951+
"credential refresh is in retry backoff",
952+
));
953+
}
954+
Ok(())
955+
}
956+
957+
/// Cancellation-safe entry point for a separately requested manual rotation.
930958
pub async fn refresh_provider_credential(
931959
store: &Store,
932960
workspace: &str,
@@ -943,15 +971,40 @@ pub async fn refresh_provider_credential(
943971
let snapshot = get_refresh_state(store, workspace, provider.object_id(), credential_key)
944972
.await?
945973
.ok_or_else(|| Status::not_found("provider refresh state not found"))?;
946-
refresh_from_snapshot(store, credentials, compute, snapshot).await
974+
refresh_from_snapshot_with_trigger(
975+
store,
976+
credentials,
977+
compute,
978+
snapshot,
979+
RefreshTrigger::ExplicitRotation,
980+
)
981+
.await
947982
}
948983

949-
/// Coalesce against the caller's observed generation, not a name-keyed replacement.
984+
/// Coalesce an automatic refresh against the caller's observed generation, not
985+
/// a name-keyed replacement. Recheck recovery/backoff after acquiring the locks.
950986
pub async fn refresh_from_snapshot(
951987
store: &Store,
952988
credentials: &crate::credentials::CredentialRuntime,
953989
compute: Option<&crate::compute::ComputeRuntime>,
954990
snapshot: StoredProviderCredentialRefreshState,
991+
) -> Result<StoredProviderCredentialRefreshState, Status> {
992+
refresh_from_snapshot_with_trigger(
993+
store,
994+
credentials,
995+
compute,
996+
snapshot,
997+
RefreshTrigger::Automatic,
998+
)
999+
.await
1000+
}
1001+
1002+
async fn refresh_from_snapshot_with_trigger(
1003+
store: &Store,
1004+
credentials: &crate::credentials::CredentialRuntime,
1005+
compute: Option<&crate::compute::ComputeRuntime>,
1006+
snapshot: StoredProviderCredentialRefreshState,
1007+
trigger: RefreshTrigger,
9551008
) -> Result<StoredProviderCredentialRefreshState, Status> {
9561009
let admission = REFRESH_ADMISSIONS
9571010
.try_acquire()
@@ -1021,6 +1074,9 @@ pub async fn refresh_from_snapshot(
10211074
"previous credential mint has an uncertain outcome; reconfigure the refresh grant",
10221075
));
10231076
}
1077+
if trigger == RefreshTrigger::Automatic {
1078+
ensure_automatic_refresh_ready(&latest, current_time_ms())?;
1079+
}
10241080
refresh_provider_credential_locked(
10251081
&store,
10261082
snapshot.object_workspace(),
@@ -2318,7 +2374,15 @@ async fn refresh_states(
23182374
status = %state.status,
23192375
"refreshing provider credential"
23202376
);
2321-
if let Err(err) = refresh_from_snapshot(store, credentials, compute, state.clone()).await {
2377+
let trigger = if rotation_requested {
2378+
RefreshTrigger::ExplicitRotation
2379+
} else {
2380+
RefreshTrigger::Automatic
2381+
};
2382+
if let Err(err) =
2383+
refresh_from_snapshot_with_trigger(store, credentials, compute, state.clone(), trigger)
2384+
.await
2385+
{
23222386
warn!(
23232387
provider = %state.provider_name,
23242388
credential_key = %state.credential_key,
@@ -2338,9 +2402,9 @@ mod tests {
23382402
MAX_OAUTH_ERROR_RESPONSE_BYTES, NewRefreshStateConfig, OAuthGrantKind, RefreshFailure,
23392403
RefreshRetrySchedule, Status, classify_oauth_token_error,
23402404
delete_refresh_state_with_credentials, effective_authorization_epoch,
2341-
enqueue_pending_secret_deletion, get_refresh_state, list_all_refresh_states,
2342-
list_refresh_states_for_provider, max_lifetime_seconds, new_refresh_state,
2343-
next_refresh_at_ms, put_refresh_state, read_bounded_oauth_error_body,
2405+
enqueue_pending_secret_deletion, ensure_automatic_refresh_ready, get_refresh_state,
2406+
list_all_refresh_states, list_refresh_states_for_provider, max_lifetime_seconds,
2407+
new_refresh_state, next_refresh_at_ms, put_refresh_state, read_bounded_oauth_error_body,
23442408
refresh_has_expiration, refresh_material_scope, refresh_provider_credential,
23452409
refresh_state_name, refresh_status_from_state, refresh_strategy_name,
23462410
run_refresh_worker_tick, seconds_until_ms, set_refresh_expiration_presence,
@@ -3849,6 +3913,48 @@ mod tests {
38493913
);
38503914
}
38513915

3916+
#[test]
3917+
fn automatic_refresh_honors_recovery_schedule_without_disabling_due_retries() {
3918+
let now_ms = 10_000;
3919+
for recovery in [
3920+
ProviderCredentialRefreshRecoveryAction::Retry,
3921+
ProviderCredentialRefreshRecoveryAction::FixConfiguration,
3922+
ProviderCredentialRefreshRecoveryAction::Investigate,
3923+
] {
3924+
let mut state = StoredProviderCredentialRefreshState {
3925+
recovery_action: recovery as i32,
3926+
next_refresh_at_ms: now_ms + 1,
3927+
..Default::default()
3928+
};
3929+
assert_eq!(
3930+
ensure_automatic_refresh_ready(&state, now_ms)
3931+
.unwrap_err()
3932+
.code(),
3933+
tonic::Code::Unavailable,
3934+
);
3935+
state.next_refresh_at_ms = now_ms;
3936+
ensure_automatic_refresh_ready(&state, now_ms).unwrap();
3937+
}
3938+
let state = StoredProviderCredentialRefreshState {
3939+
recovery_action: ProviderCredentialRefreshRecoveryAction::Reauthorize as i32,
3940+
next_refresh_at_ms: now_ms,
3941+
..Default::default()
3942+
};
3943+
assert_eq!(
3944+
ensure_automatic_refresh_ready(&state, now_ms)
3945+
.unwrap_err()
3946+
.code(),
3947+
tonic::Code::FailedPrecondition,
3948+
);
3949+
// A healthy credential's proactive schedule must not block an on-read
3950+
// mint requested to meet a longer minimum remaining lifetime.
3951+
let state = StoredProviderCredentialRefreshState {
3952+
next_refresh_at_ms: now_ms + 1,
3953+
..Default::default()
3954+
};
3955+
ensure_automatic_refresh_ready(&state, now_ms).unwrap();
3956+
}
3957+
38523958
#[tokio::test]
38533959
async fn refresh_worker_skips_parked_reauthorization_state() {
38543960
let store = test_store().await;

0 commit comments

Comments
 (0)