Skip to content

Commit 16005c6

Browse files
committed
refactor(providers): defer credential migration
Signed-off-by: Mrunal Patel <[email protected]>
1 parent 0541d45 commit 16005c6

4 files changed

Lines changed: 40 additions & 234 deletions

File tree

‎architecture/gateway.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -335,6 +335,11 @@ Multi-replica deployments can use that default with a shared database and
335335
shared key-encryption key, or opt into an external backend such as Vault or
336336
Kubernetes Secrets.
337337

338+
Credential handles remain bound to the driver that created them. Before the
339+
0.1.0 compatibility boundary, gateways do not migrate inline refresh material
340+
or move handles between credential drivers; operators reconfigure affected
341+
grants when upgrading or changing backends.
342+
338343
### Optimistic Concurrency (CAS)
339344

340345
Every object row carries a `resource_version` that the database increments

‎crates/openshell-server/src/provider_refresh.rs‎

Lines changed: 28 additions & 225 deletions
Original file line numberDiff line numberDiff line change
@@ -568,85 +568,6 @@ fn validate_secret_material_references(
568568
)))
569569
}
570570

571-
async fn migrate_inline_secret_material(
572-
store: &Store,
573-
credentials: Option<&crate::credentials::CredentialRuntime>,
574-
state: &StoredProviderCredentialRefreshState,
575-
) -> Result<StoredProviderCredentialRefreshState, Status> {
576-
let strategy = ProviderCredentialRefreshStrategy::try_from(state.strategy)
577-
.unwrap_or(ProviderCredentialRefreshStrategy::Unspecified);
578-
let mut secret_keys: std::collections::HashSet<String> =
579-
state.secret_material_keys.iter().cloned().collect();
580-
secret_keys.extend(
581-
strategy_secret_material_keys(strategy)
582-
.iter()
583-
.map(|key| (*key).to_string()),
584-
);
585-
let inline_secrets: HashMap<String, String> = secret_keys
586-
.into_iter()
587-
.filter_map(|key| state.material.get(&key).cloned().map(|value| (key, value)))
588-
.collect();
589-
if inline_secrets.is_empty() {
590-
return Ok(state.clone());
591-
}
592-
let Some(credentials) = credentials else {
593-
// Production refresh entry points always supply the server credential
594-
// runtime. Keeping the no-runtime path readable preserves focused unit
595-
// tests and test-only helpers that exercise minting in isolation.
596-
return Ok(state.clone());
597-
};
598-
let values_to_stage: HashMap<_, _> = inline_secrets
599-
.iter()
600-
.filter(|(key, _)| !state.secret_material_handles.contains_key(*key))
601-
.map(|(key, value)| (key.clone(), value.clone()))
602-
.collect();
603-
let staging_id = format!(
604-
"{}-refresh-migration-{}",
605-
state.object_id(),
606-
uuid::Uuid::new_v4()
607-
);
608-
let staged = credentials
609-
.store_refresh_material_with_object_id(
610-
refresh_material_scope(state),
611-
&staging_id,
612-
&values_to_stage,
613-
&HashMap::new(),
614-
)
615-
.await?;
616-
let mut migrated = state.clone();
617-
migrated.secret_material_handles.extend(staged.clone());
618-
for key in inline_secrets.keys() {
619-
migrated.material.remove(key);
620-
if !migrated.secret_material_keys.contains(key) {
621-
migrated.secret_material_keys.push(key.clone());
622-
}
623-
}
624-
migrated.secret_material_keys.sort();
625-
migrated.secret_material_keys.dedup();
626-
let expected_version = state
627-
.metadata
628-
.as_ref()
629-
.map_or(0, |metadata| metadata.resource_version);
630-
match persist_refresh_state_if_current(store, &migrated, expected_version).await {
631-
Ok(Some(new_version)) => {
632-
if let Some(metadata) = migrated.metadata.as_mut() {
633-
metadata.resource_version = new_version;
634-
}
635-
Ok(migrated)
636-
}
637-
Ok(None) => {
638-
cleanup_staged_refresh_material_handles(credentials, state, &staged).await;
639-
Err(Status::aborted(
640-
"provider refresh was deleted or superseded during material migration",
641-
))
642-
}
643-
Err(err) => {
644-
cleanup_staged_refresh_material_handles(credentials, state, &staged).await;
645-
Err(err)
646-
}
647-
}
648-
}
649-
650571
pub async fn refresh_provider_credential(
651572
store: &Store,
652573
workspace: &str,
@@ -674,7 +595,7 @@ pub async fn refresh_provider_credential(
674595
"provider refresh is being deleted",
675596
));
676597
}
677-
let mut state = migrate_inline_secret_material(store, Some(credentials), &state).await?;
598+
let mut state = state;
678599
validate_secret_material_references(&state)?;
679600
// Generation of the refresh at the start of the rotation. Terminal persists
680601
// match on it so a concurrent delete or rotation is detected rather than
@@ -1634,18 +1555,7 @@ async fn run_refresh_worker_tick(
16341555
}
16351556
continue;
16361557
}
1637-
let mut state = match migrate_inline_secret_material(store, credentials, &state).await {
1638-
Ok(state) => state,
1639-
Err(err) => {
1640-
warn!(
1641-
provider = %state.provider_name,
1642-
credential_key = %state.credential_key,
1643-
error = %err,
1644-
"provider refresh material migration failed"
1645-
);
1646-
continue;
1647-
}
1648-
};
1558+
let mut state = state;
16491559
let expected_version = state
16501560
.metadata
16511561
.as_ref()
@@ -1735,9 +1645,9 @@ mod tests {
17351645
use super::{
17361646
NewRefreshStateConfig, delete_refresh_state_with_credentials,
17371647
effective_authorization_epoch, enqueue_pending_secret_deletion, get_refresh_state,
1738-
list_all_refresh_states, list_refresh_states_for_provider, migrate_inline_secret_material,
1739-
new_refresh_state, put_refresh_state, refresh_material_scope, refresh_provider_credential,
1740-
refresh_state_name, refresh_strategy_name, run_refresh_worker_tick, seconds_until_ms,
1648+
list_all_refresh_states, list_refresh_states_for_provider, new_refresh_state,
1649+
put_refresh_state, refresh_material_scope, refresh_provider_credential, refresh_state_name,
1650+
refresh_strategy_name, run_refresh_worker_tick, seconds_until_ms,
17411651
validate_secret_material_references,
17421652
};
17431653
use crate::credentials::CredentialRuntime;
@@ -1801,76 +1711,6 @@ mod tests {
18011711
assert_eq!(all[0].metadata.as_ref().unwrap().resource_version, 1);
18021712
}
18031713

1804-
#[tokio::test]
1805-
async fn superseded_inline_material_migration_cleans_staged_handle() {
1806-
let store = test_store().await;
1807-
let provider = provider("migration-race", "outlook");
1808-
let state = new_refresh_state(
1809-
&provider,
1810-
"default",
1811-
"MS_GRAPH_ACCESS_TOKEN",
1812-
NewRefreshStateConfig {
1813-
strategy: ProviderCredentialRefreshStrategy::Oauth2ClientCredentials,
1814-
material: HashMap::from([
1815-
("client_id".to_string(), "client-id".to_string()),
1816-
("client_secret".to_string(), "client-secret".to_string()),
1817-
]),
1818-
secret_material_keys: vec!["client_secret".to_string()],
1819-
expires_at_ms: 0,
1820-
token_url: "https://issuer.example/token".to_string(),
1821-
scopes: Vec::new(),
1822-
refresh_before_seconds: 30,
1823-
max_lifetime_seconds: 60,
1824-
additional_output_keys: HashMap::new(),
1825-
},
1826-
)
1827-
.unwrap();
1828-
put_refresh_state(&store, &state).await.unwrap();
1829-
let state = get_refresh_state(
1830-
&store,
1831-
"default",
1832-
provider.object_id(),
1833-
"MS_GRAPH_ACCESS_TOKEN",
1834-
)
1835-
.await
1836-
.unwrap()
1837-
.unwrap();
1838-
let credentials = test_credentials();
1839-
let (store_hit, release_store) = credentials.gate_next_store();
1840-
1841-
let migration = migrate_inline_secret_material(&store, Some(&credentials), &state);
1842-
let supersede = async {
1843-
store_hit.await.unwrap();
1844-
let mut winner = get_refresh_state(
1845-
&store,
1846-
"default",
1847-
provider.object_id(),
1848-
"MS_GRAPH_ACCESS_TOKEN",
1849-
)
1850-
.await
1851-
.unwrap()
1852-
.unwrap();
1853-
winner.last_error = "won-by-concurrent-writer".to_string();
1854-
put_refresh_state(&store, &winner).await.unwrap();
1855-
release_store.send(()).unwrap();
1856-
};
1857-
let (result, ()) = tokio::join!(migration, supersede);
1858-
1859-
assert_eq!(result.unwrap_err().code(), tonic::Code::Aborted);
1860-
assert_eq!(credentials.stored_credential_count(), Some(0));
1861-
let winner = get_refresh_state(
1862-
&store,
1863-
"default",
1864-
provider.object_id(),
1865-
"MS_GRAPH_ACCESS_TOKEN",
1866-
)
1867-
.await
1868-
.unwrap()
1869-
.unwrap();
1870-
assert_eq!(winner.last_error, "won-by-concurrent-writer");
1871-
assert!(winner.secret_material_handles.is_empty());
1872-
}
1873-
18741714
#[test]
18751715
fn new_refresh_configuration_rotates_authorization_epoch_and_legacy_state_is_stable() {
18761716
let provider = Provider {
@@ -2086,7 +1926,7 @@ mod tests {
20861926
let store = test_store().await;
20871927
let provider = provider("my-stored-graph", "outlook");
20881928
store.put_message(&provider).await.unwrap();
2089-
let state = new_refresh_state(
1929+
let mut state = new_refresh_state(
20901930
&provider,
20911931
"default",
20921932
"MS_GRAPH_ACCESS_TOKEN",
@@ -2106,10 +1946,20 @@ mod tests {
21061946
},
21071947
)
21081948
.unwrap();
2109-
put_refresh_state(&store, &state).await.unwrap();
2110-
let authorization_epoch = state.authorization_epoch.clone();
21111949
let config = Config::new(None).with_credential_drivers(["test-static"]);
21121950
let credentials = CredentialRuntime::from_config(&config).unwrap();
1951+
state.secret_material_handles = credentials
1952+
.store_refresh_material_with_object_id(
1953+
refresh_material_scope(&state),
1954+
"configured-client-secret",
1955+
&HashMap::from([("client_secret".to_string(), "client-secret".to_string())]),
1956+
&HashMap::new(),
1957+
)
1958+
.await
1959+
.unwrap();
1960+
state.material.remove("client_secret");
1961+
put_refresh_state(&store, &state).await.unwrap();
1962+
let authorization_epoch = state.authorization_epoch.clone();
21131963

21141964
let refreshed = refresh_provider_credential(
21151965
&store,
@@ -2426,6 +2276,16 @@ mod tests {
24262276
},
24272277
)
24282278
.unwrap();
2279+
state.secret_material_handles = credentials
2280+
.store_refresh_material_with_object_id(
2281+
refresh_material_scope(&state),
2282+
"current-refresh-object",
2283+
&HashMap::from([("client_secret".to_string(), "client-secret".to_string())]),
2284+
&HashMap::new(),
2285+
)
2286+
.await
2287+
.unwrap();
2288+
state.material.remove("client_secret");
24292289
let old = credentials
24302290
.store_refresh_material_with_object_id(
24312291
refresh_material_scope(&state),
@@ -2692,63 +2552,6 @@ mod tests {
26922552
);
26932553
}
26942554

2695-
#[tokio::test]
2696-
async fn refresh_worker_migrates_legacy_secrets_before_skipping_non_mintable_state() {
2697-
let store = test_store().await;
2698-
let provider = provider("legacy-external", "outlook");
2699-
store.put_message(&provider).await.unwrap();
2700-
let state = new_refresh_state(
2701-
&provider,
2702-
"default",
2703-
"MS_GRAPH_ACCESS_TOKEN",
2704-
NewRefreshStateConfig {
2705-
strategy: ProviderCredentialRefreshStrategy::External,
2706-
material: HashMap::from([(
2707-
"legacy_secret".to_string(),
2708-
"move-me-out-of-the-database".to_string(),
2709-
)]),
2710-
secret_material_keys: vec!["legacy_secret".to_string()],
2711-
expires_at_ms: 0,
2712-
token_url: String::new(),
2713-
scopes: Vec::new(),
2714-
refresh_before_seconds: 0,
2715-
max_lifetime_seconds: 0,
2716-
additional_output_keys: HashMap::new(),
2717-
},
2718-
)
2719-
.unwrap();
2720-
put_refresh_state(&store, &state).await.unwrap();
2721-
let credentials = test_credentials();
2722-
2723-
run_refresh_worker_tick(&store, Some(&credentials), None)
2724-
.await
2725-
.unwrap();
2726-
2727-
let stored = get_refresh_state(
2728-
&store,
2729-
"default",
2730-
provider.object_id(),
2731-
"MS_GRAPH_ACCESS_TOKEN",
2732-
)
2733-
.await
2734-
.unwrap()
2735-
.unwrap();
2736-
assert!(!stored.material.contains_key("legacy_secret"));
2737-
assert!(stored.secret_material_handles.contains_key("legacy_secret"));
2738-
assert_eq!(stored.status, "configured");
2739-
assert_eq!(
2740-
credentials
2741-
.resolve_refresh_material(
2742-
refresh_material_scope(&stored),
2743-
&stored.secret_material_handles,
2744-
)
2745-
.await
2746-
.unwrap()
2747-
.get("legacy_secret"),
2748-
Some(&"move-me-out-of-the-database".to_string())
2749-
);
2750-
}
2751-
27522555
#[tokio::test]
27532556
async fn refresh_worker_finalizes_tombstoned_refresh_material() {
27542557
let store = test_store().await;

‎docs/reference/gateway-config.mdx‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -379,7 +379,7 @@ For `kubernetes-secrets`, `namespace` sets where OpenShell-managed provider Secr
379379

380380
For `vault`, `address` points at the Vault service, `mount` and `kv_version` describe the KV engine where OpenShell-managed provider secrets are stored, and `auth_method = "kubernetes"` logs in with the gateway Pod's ServiceAccount token. For local or development validation, use `auth_method = "token_file"` with `token_path = "/path/to/token"`. Do not put literal Vault tokens in TOML.
381381

382-
Provider records that already contain inline database credentials remain readable for upgrade compatibility. New provider create/update requests still submit credential values through the normal API, but the gateway stores those values through the active credential storage path and persists only handles. The refresh worker migrates legacy inline secret refresh material to the same storage path before minting. Stop all older gateway replicas before starting the version that performs this migration; older protobuf runtimes do not preserve the new internal handle fields when they rewrite refresh state.
382+
Provider records that already contain inline database credentials remain readable for upgrade compatibility. New provider create/update requests still submit credential values through the normal API, but the gateway stores those values through the active credential storage path and persists only handles. Before OpenShell 0.1.0, OpenShell does not automatically migrate inline refresh material or credential handles between drivers. Reconfigure refresh grants after an upgrade. Before changing credential drivers, remove affected credentials while the original driver is still available, then select the new driver and create them again. Do not run mixed gateway versions against the same refresh records.
383383

384384
For remote credential drivers, set `transport = "uds"` with `socket_path`. Omit `command`, `args`, and `startup_timeout_secs` when another service manager prestarts the driver socket. Keep backend tokens out of TOML; point the driver at mounted token files or native identity mechanisms instead.
385385

‎docs/sandboxes/providers-v2.mdx‎

Lines changed: 6 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -625,14 +625,12 @@ The gateway sends a complete host, port, and path binding for every emitted stat
625625

626626
Refresh configuration is stored separately from the current injectable credential value. Non-secret refresh configuration remains in the refresh-state record. Secret material is resolved from the active credential driver only while the gateway mints a new short-lived token. The gateway writes the token back through credential storage and updates credential expiry metadata. If an OAuth issuer rotates its refresh token, the gateway stages the replacement through credential storage before committing the refresh generation and removes the previous handle afterward.
627627

628-
On upgrade, the refresh worker migrates legacy inline secret material before it
629-
mints again. For a multi-replica gateway, stop every older replica before the
630-
new version begins migration. A normal mixed-version rolling update is unsafe
631-
because older protobuf runtimes can discard internal handle fields when they
632-
rewrite refresh state. Migration removes plaintext values from current records,
633-
but database backups, SQLite WAL/free pages, and Postgres dead tuples may retain
634-
historical bytes. Rotate the upstream grant and apply the database's retention
635-
or compaction procedure when historical erasure is required.
628+
Before OpenShell 0.1.0, refresh-state and credential-driver migrations are not
629+
supported. Upgrading from a build that stored refresh material inline requires
630+
reconfiguring the refresh grant. To change credential drivers, delete or
631+
reconfigure affected providers while the original driver is still available,
632+
then select the new driver and create the credentials again. Do not run mixed
633+
gateway versions against the same refresh records.
636634

637635
Each explicit `refresh configure` call also starts a new gateway-owned
638636
authorization epoch. Automatic refresh and `refresh rotate` preserve that epoch,

0 commit comments

Comments
 (0)