Skip to content

Commit 557221f

Browse files
committed
fix(providers): decide refresh-owned expiry cleanup inside the CAS closure
Deleting a refresh decided which expiries to clear from a provider snapshot read at the top of the handler, then applied an unconditional zero to each. A rotation or provider update landing between that read and the write was silently clobbered (CWE-362). Move the equality check and removal into a pure helper run inside the provider CAS closure, so the decision reflects the current stored provider rather than a stale snapshot. Only expiries that still match the deleted refresh are cleared; concurrently changed values are preserved. Signed-off-by: Russell Bryant <[email protected]>
1 parent e8495fd commit 557221f

1 file changed

Lines changed: 100 additions & 36 deletions

File tree

‎crates/openshell-server/src/grpc/provider.rs‎

Lines changed: 100 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -2319,6 +2319,34 @@ pub(super) async fn handle_rotate_provider_credential(
23192319
}))
23202320
}
23212321

2322+
/// Remove the expiry for each key the refresh still owns — one whose stored
2323+
/// value equals the refresh's `expires_at_ms`. Independently updated expiries (no
2324+
/// longer matching) are left in place.
2325+
///
2326+
/// This operates on `provider` in place so it can run inside a CAS closure
2327+
/// against the current stored provider. The equality check must see the latest
2328+
/// state, not a snapshot read earlier in the handler: a concurrent rotation or
2329+
/// provider update between the read and the write could otherwise be clobbered
2330+
/// by an unconditional zero (CWE-362).
2331+
fn clear_refresh_owned_expiries(
2332+
provider: &mut Provider,
2333+
refresh_expires_at_ms: i64,
2334+
owned_keys: &[String],
2335+
) {
2336+
if refresh_expires_at_ms <= 0 {
2337+
return;
2338+
}
2339+
for key in owned_keys {
2340+
if provider
2341+
.credential_expires_at_ms
2342+
.get(key)
2343+
.is_some_and(|expires_at_ms| *expires_at_ms == refresh_expires_at_ms)
2344+
{
2345+
provider.credential_expires_at_ms.remove(key);
2346+
}
2347+
}
2348+
}
2349+
23222350
pub(super) async fn handle_delete_provider_refresh(
23232351
state: &Arc<ServerState>,
23242352
request: Request<DeleteProviderRefreshRequest>,
@@ -2352,44 +2380,29 @@ pub(super) async fn handle_delete_provider_refresh(
23522380
.await?;
23532381

23542382
// A refresh co-manages the expiry of its primary credential and every pinned
2355-
// additional output. Clear each one this refresh owned (its stored expiry
2356-
// still matches the refresh's), but leave independently updated expiries in
2357-
// place. Doing it in one provider update keeps the record consistent.
2358-
let mut cleared_expiries = std::collections::HashMap::new();
2359-
if let Some(refresh_state) = existing_refresh_state.as_ref()
2383+
// additional output. Clear each expiry this refresh still owns, leaving
2384+
// independently updated ones in place. The equality check and removal run
2385+
// inside the CAS closure so they see the current stored provider — deciding
2386+
// from the snapshot read above would let a concurrent rotation or provider
2387+
// update land between the read and the write and then be clobbered (CWE-362).
2388+
if let Some(refresh_state) = existing_refresh_state
23602389
&& refresh_state.expires_at_ms > 0
23612390
{
2362-
let owned_keys = std::iter::once(credential_key).chain(
2363-
refresh_state
2364-
.additional_output_keys
2365-
.values()
2366-
.map(String::as_str),
2367-
);
2368-
for key in owned_keys {
2369-
if provider
2370-
.credential_expires_at_ms
2371-
.get(key)
2372-
.is_some_and(|expires_at_ms| *expires_at_ms == refresh_state.expires_at_ms)
2373-
{
2374-
cleared_expiries.insert(key.to_string(), 0);
2375-
}
2376-
}
2377-
}
2378-
if !cleared_expiries.is_empty() {
2379-
let updated = Provider {
2380-
metadata: Some(openshell_core::proto::datamodel::v1::ObjectMeta {
2381-
id: String::new(),
2382-
name: provider_name.to_string(),
2383-
created_at_ms: 0,
2384-
labels: std::collections::HashMap::new(),
2385-
resource_version: 0,
2386-
}),
2387-
r#type: String::new(),
2388-
credentials: std::collections::HashMap::new(),
2389-
config: std::collections::HashMap::new(),
2390-
credential_expires_at_ms: cleared_expiries,
2391-
};
2392-
update_provider_record(state.store.as_ref(), updated).await?;
2391+
let refresh_expires_at_ms = refresh_state.expires_at_ms;
2392+
let owned_keys: Vec<String> = std::iter::once(credential_key.to_string())
2393+
.chain(refresh_state.additional_output_keys.into_values())
2394+
.collect();
2395+
state
2396+
.store
2397+
.update_message_cas::<Provider, _>(provider.object_id(), 0, |current| {
2398+
clear_refresh_owned_expiries(current, refresh_expires_at_ms, &owned_keys);
2399+
})
2400+
.await
2401+
.map_err(|e| {
2402+
Status::internal(format!(
2403+
"clear refresh-owned credential expiries failed: {e}"
2404+
))
2405+
})?;
23932406
}
23942407

23952408
Ok(Response::new(DeleteProviderRefreshResponse {
@@ -4133,6 +4146,57 @@ mod tests {
41334146
);
41344147
}
41354148

4149+
#[test]
4150+
fn clear_refresh_owned_expiries_preserves_concurrently_changed_expiry() {
4151+
// Regression for the delete-refresh TOCTOU: the clearing decision runs
4152+
// inside the CAS closure against the current provider. Simulate a
4153+
// concurrent rotation that updated AWS_SESSION_TOKEN's expiry after the
4154+
// handler's initial read but before its write — by the time the closure
4155+
// runs, that value no longer matches the deleted refresh's expiry and
4156+
// must survive, while the still-owned keys are cleared.
4157+
let refresh_expires_at_ms = 1_000_000;
4158+
let concurrently_changed = refresh_expires_at_ms + 500_000;
4159+
let mut provider = Provider {
4160+
metadata: Some(openshell_core::proto::datamodel::v1::ObjectMeta {
4161+
id: "p".to_string(),
4162+
name: "p".to_string(),
4163+
created_at_ms: 0,
4164+
labels: HashMap::new(),
4165+
resource_version: 0,
4166+
}),
4167+
r#type: "aws".to_string(),
4168+
credentials: HashMap::new(),
4169+
config: HashMap::new(),
4170+
credential_expires_at_ms: HashMap::from([
4171+
("AWS_ACCESS_KEY_ID".to_string(), refresh_expires_at_ms),
4172+
("AWS_SECRET_ACCESS_KEY".to_string(), refresh_expires_at_ms),
4173+
("AWS_SESSION_TOKEN".to_string(), concurrently_changed),
4174+
]),
4175+
};
4176+
let owned_keys = vec![
4177+
"AWS_ACCESS_KEY_ID".to_string(),
4178+
"AWS_SECRET_ACCESS_KEY".to_string(),
4179+
"AWS_SESSION_TOKEN".to_string(),
4180+
];
4181+
4182+
clear_refresh_owned_expiries(&mut provider, refresh_expires_at_ms, &owned_keys);
4183+
4184+
assert!(
4185+
!provider
4186+
.credential_expires_at_ms
4187+
.contains_key("AWS_ACCESS_KEY_ID")
4188+
);
4189+
assert!(
4190+
!provider
4191+
.credential_expires_at_ms
4192+
.contains_key("AWS_SECRET_ACCESS_KEY")
4193+
);
4194+
assert_eq!(
4195+
provider.credential_expires_at_ms.get("AWS_SESSION_TOKEN"),
4196+
Some(&concurrently_changed)
4197+
);
4198+
}
4199+
41364200
#[tokio::test]
41374201
async fn configure_provider_refresh_rejects_credential_key_collision_for_attached_sandbox() {
41384202
let state = test_server_state().await;

0 commit comments

Comments
 (0)