Skip to content

Commit d241836

Browse files
committed
refactor(server): make the platform profile lock scope explicit
Platform-scope provider profile writers (import, update, delete) pass an empty workspace on purpose and need X(global), because they validate sandboxes in every workspace. MutationScope::profiles() now names that escalation and picks Global for the platform scope, instead of relying on MutationScope::Workspace("") to mean it. An empty workspace name now trips a debug assertion, and release builds keep X(global) as a fail-safe that over-locks rather than under-locks. Part of NVIDIA#3528 Signed-off-by: Emilien Macchi <[email protected]>
1 parent 6d9d5e0 commit d241836

3 files changed

Lines changed: 95 additions & 17 deletions

File tree

‎crates/openshell-server/src/compute/mutation_guard.rs‎

Lines changed: 49 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -37,13 +37,15 @@ use tracing::warn;
3737
/// and workspace-scoped profile writers hold their workspace key exclusively;
3838
/// sandbox-scoped settings and policy writers hold only their sandbox key
3939
/// exclusively. A new cross-object writer needs a scope from the same table.
40+
/// Profile writers pick theirs with [`MutationScope::profiles`].
4041
#[derive(Clone, Copy, Debug)]
4142
pub enum MutationScope<'a> {
4243
/// Global policy/settings and platform-scope profiles. Excludes every
4344
/// other scope fleet-wide.
4445
Global,
45-
/// Provider and workspace-scoped profile mutations. `""` (platform)
46-
/// behaves as `Global`.
46+
/// Provider and workspace-scoped profile mutations in a named workspace.
47+
/// The platform scope is `Global`: an empty name trips a debug assertion
48+
/// and locks as `Global` in release builds.
4749
Workspace(&'a str),
4850
/// Any mutation of one sandbox's records, admin or supervisor.
4951
Sandbox {
@@ -60,23 +62,42 @@ impl<'a> MutationScope<'a> {
6062
}
6163
}
6264

65+
/// Scope of a provider profile write. Platform-scope profiles (`""`)
66+
/// feed every workspace's catalog and are validated against sandboxes in
67+
/// all workspaces, so they take `Global`; a named workspace takes
68+
/// `Workspace(name)`.
69+
pub(crate) const fn profiles(workspace: &'a str) -> Self {
70+
if workspace.is_empty() {
71+
Self::Global
72+
} else {
73+
Self::Workspace(workspace)
74+
}
75+
}
76+
6377
/// Keys and modes of this scope:
6478
///
65-
/// - `Global` and `Workspace("")`: X(global).
66-
/// - `Workspace(ws)`: S(global) X(workspace).
79+
/// - `Global`: X(global).
80+
/// - `Workspace(ws)`: S(global) X(workspace). An empty name is a caller
81+
/// bug that debug builds catch; release builds take X(global), which
82+
/// over-locks rather than under-locks.
6783
/// - `Sandbox`: S(global) S(workspace) X(sandbox). A legacy sandbox with
6884
/// an empty workspace locks the default workspace, where its providers
6985
/// resolve.
7086
pub(crate) fn lock_set(&self) -> MutationLockSet {
7187
let mut set = MutationLockSet::default();
7288
match *self {
7389
Self::Global => set.insert(MutationLockKey::Global, LockMode::Exclusive),
74-
Self::Workspace("") => {
75-
set.insert(MutationLockKey::Global, LockMode::Exclusive);
76-
}
7790
Self::Workspace(workspace) => {
78-
set.insert(MutationLockKey::Global, LockMode::Shared);
79-
set.insert(MutationLockKey::Workspace(workspace), LockMode::Exclusive);
91+
debug_assert!(
92+
!workspace.is_empty(),
93+
"the platform scope is MutationScope::Global; use MutationScope::profiles"
94+
);
95+
if workspace.is_empty() {
96+
set.insert(MutationLockKey::Global, LockMode::Exclusive);
97+
} else {
98+
set.insert(MutationLockKey::Global, LockMode::Shared);
99+
set.insert(MutationLockKey::Workspace(workspace), LockMode::Exclusive);
100+
}
80101
}
81102
Self::Sandbox {
82103
workspace,
@@ -460,7 +481,11 @@ mod tests {
460481

461482
let cases = [
462483
(MutationScope::Global, keys(&[(Global, Exclusive)])),
463-
(MutationScope::Workspace(""), keys(&[(Global, Exclusive)])),
484+
(MutationScope::profiles(""), keys(&[(Global, Exclusive)])),
485+
(
486+
MutationScope::profiles("team-a"),
487+
keys(&[(Global, Shared), (Workspace("team-a"), Exclusive)]),
488+
),
464489
(
465490
MutationScope::Workspace("team-a"),
466491
keys(&[(Global, Shared), (Workspace("team-a"), Exclusive)]),
@@ -504,9 +529,13 @@ mod tests {
504529
}
505530

506531
#[test]
507-
fn scope_labels_map_platform_workspace_to_global() {
532+
fn scope_labels_map_platform_profiles_to_global() {
508533
assert_eq!(MutationScope::Global.lock_scope(), LockScope::Global);
509-
assert_eq!(MutationScope::Workspace("").lock_scope(), LockScope::Global);
534+
assert_eq!(MutationScope::profiles("").lock_scope(), LockScope::Global);
535+
assert_eq!(
536+
MutationScope::profiles("team-a").lock_scope(),
537+
LockScope::Workspace
538+
);
510539
assert_eq!(
511540
MutationScope::Workspace("team-a").lock_scope(),
512541
LockScope::Workspace
@@ -570,7 +599,7 @@ mod tests {
570599

571600
let mut waiting_guards = vec![
572601
spawn_guard(&runtime, MutationScope::Global),
573-
spawn_guard(&runtime, MutationScope::Workspace("")),
602+
spawn_guard(&runtime, MutationScope::profiles("")),
574603
spawn_guard(&runtime, MutationScope::Workspace("w1")),
575604
spawn_guard(&runtime, MutationScope::sandbox("w1", "a")),
576605
];
@@ -1031,6 +1060,13 @@ mod tests {
10311060
run_random_scope_mix(&[runtime], "", 64, 50, Duration::from_secs(20)).await;
10321061
}
10331062

1063+
#[cfg(debug_assertions)]
1064+
#[test]
1065+
#[should_panic(expected = "platform scope")]
1066+
fn empty_workspace_scope_panics_in_debug_builds() {
1067+
let _ = MutationScope::Workspace("").lock_set();
1068+
}
1069+
10341070
#[cfg(debug_assertions)]
10351071
#[tokio::test]
10361072
#[should_panic(expected = "mutation lock ordering violated")]

‎crates/openshell-server/src/grpc/provider.rs‎

Lines changed: 45 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2896,7 +2896,7 @@ pub(super) async fn handle_import_provider_profiles(
28962896
add_empty_profile_set_diagnostic(&profiles, &mut diagnostics);
28972897
let _mutation_guard = state
28982898
.compute
2899-
.mutation_guard(MutationScope::Workspace(&workspace))
2899+
.mutation_guard(MutationScope::profiles(&workspace))
29002900
.await
29012901
.map_err(|err| super::persistence_error_to_status(err, "acquire provider mutation lock"))?;
29022902
let catalog = state
@@ -2992,7 +2992,7 @@ pub(super) async fn handle_update_provider_profiles(
29922992
let target_id = normalize_profile_id_request(&request.id)?;
29932993
let _mutation_guard = state
29942994
.compute
2995-
.mutation_guard(MutationScope::Workspace(&workspace))
2995+
.mutation_guard(MutationScope::profiles(&workspace))
29962996
.await
29972997
.map_err(|err| super::persistence_error_to_status(err, "acquire provider mutation lock"))?;
29982998
let catalog = state
@@ -3157,7 +3157,7 @@ pub(super) async fn handle_delete_provider_profile(
31573157
let id = normalize_profile_id_request(&id)?;
31583158
let _mutation_guard = state
31593159
.compute
3160-
.mutation_guard(MutationScope::Workspace(&workspace))
3160+
.mutation_guard(MutationScope::profiles(&workspace))
31613161
.await
31623162
.map_err(|err| super::persistence_error_to_status(err, "acquire provider mutation lock"))?;
31633163
let catalog = state
@@ -6019,6 +6019,48 @@ mod tests {
60196019
assert!(response.imported);
60206020
}
60216021

6022+
#[tokio::test]
6023+
async fn import_platform_provider_profile_waits_for_sandbox_mutation_in_any_workspace() {
6024+
let state = test_server_state().await;
6025+
let guard = state
6026+
.compute
6027+
.mutation_guard(MutationScope::sandbox("team-a", "platform-import-guard"))
6028+
.await
6029+
.unwrap();
6030+
let task_state = state.clone();
6031+
let mut task = tokio::spawn(async move {
6032+
handle_import_provider_profiles(
6033+
&task_state,
6034+
authed_request(ImportProviderProfilesRequest {
6035+
request_id: String::new(),
6036+
profiles: vec![ProviderProfileImportItem {
6037+
profile: Some(custom_profile("platform-guarded-import")),
6038+
source: "platform-guarded-import.yaml".to_string(),
6039+
}],
6040+
workspace_scope: None,
6041+
}),
6042+
)
6043+
.await
6044+
});
6045+
6046+
// An import that does not wait finishes well within this window.
6047+
assert!(
6048+
tokio::time::timeout(std::time::Duration::from_millis(200), &mut task)
6049+
.await
6050+
.is_err(),
6051+
"platform profile import should wait for a sandbox mutation in any workspace"
6052+
);
6053+
drop(guard);
6054+
6055+
let response = tokio::time::timeout(std::time::Duration::from_secs(5), task)
6056+
.await
6057+
.expect("import should finish after guard release")
6058+
.expect("join import task")
6059+
.expect("import should succeed")
6060+
.into_inner();
6061+
assert!(response.imported, "{:?}", response.diagnostics);
6062+
}
6063+
60226064
#[tokio::test]
60236065
async fn update_provider_profile_replaces_custom_profile_and_preserves_metadata() {
60246066
let state = test_server_state().await;

‎crates/openshell-server/src/persistence/mutation_lock_pg_tests.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -656,7 +656,7 @@ async fn postgres_mutation_lock_global_exclusive_blocks_every_scope() {
656656
let sandbox = random_id("sb");
657657
let scopes = [
658658
MutationScope::Global,
659-
MutationScope::Workspace(""),
659+
MutationScope::profiles(""),
660660
MutationScope::Workspace(&workspace),
661661
MutationScope::sandbox(&workspace, &sandbox),
662662
];

0 commit comments

Comments
 (0)