Skip to content

Commit 628a4b7

Browse files
feat(k8s): add operator namespace label watcher
Spawn a background kube::runtime::watcher in the K8s driver that watches namespaces matching the configured label selector and populates the OperatorNamespaceAllowlist at runtime. The driver owns the allowlist and exposes its Arc so the server can share the same set with the SA token authenticator. create_sandbox now gates pod creation on the allowlist in operator mode — workspaces whose namespace is not yet labeled are rejected at resource render time rather than silently proceeding. Workspace lifecycle itself is unaffected; only sandbox (resource) creation is gated. Signed-off-by: Derek Carr <[email protected]>
1 parent 485c65f commit 628a4b7

4 files changed

Lines changed: 200 additions & 68 deletions

File tree

‎crates/openshell-driver-kubernetes/src/driver.rs‎

Lines changed: 109 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,8 @@
66
use super::AppArmorProfile;
77
use crate::config::{
88
DEFAULT_PROXY_UID, DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME, DEFAULT_SANDBOX_UID,
9-
DEFAULT_WORKSPACE_STORAGE_SIZE, KubernetesComputeConfig, SupervisorSideloadMethod,
10-
SupervisorTopology, WorkspaceMode, managed_namespace,
9+
DEFAULT_WORKSPACE_STORAGE_SIZE, KubernetesComputeConfig, OperatorNamespaceAllowlist,
10+
SupervisorSideloadMethod, SupervisorTopology, WorkspaceMode, managed_namespace,
1111
};
1212
use futures::{Stream, StreamExt, TryStreamExt};
1313
use k8s_openapi::api::core::v1::{
@@ -438,6 +438,7 @@ pub struct KubernetesComputeDriver {
438438
watch_client: Client,
439439
sandbox_api_version: Arc<OnceCell<&'static str>>,
440440
config: KubernetesComputeConfig,
441+
operator_allowlist: Option<OperatorNamespaceAllowlist>,
441442
}
442443

443444
impl std::fmt::Debug for KubernetesComputeDriver {
@@ -485,11 +486,26 @@ impl KubernetesComputeDriver {
485486
let watch_client =
486487
Client::try_from(watch_kube_config).map_err(KubernetesDriverError::from_kube)?;
487488

489+
let operator_allowlist = if matches!(config.workspace_mode, WorkspaceMode::Operator) {
490+
config.operator_namespace_label.as_ref().map(|label| {
491+
let allowlist = OperatorNamespaceAllowlist::new();
492+
spawn_namespace_label_watcher(
493+
watch_client.clone(),
494+
label.clone(),
495+
allowlist.clone(),
496+
);
497+
allowlist
498+
})
499+
} else {
500+
None
501+
};
502+
488503
Ok(Self {
489504
client,
490505
watch_client,
491506
sandbox_api_version: Arc::new(OnceCell::new()),
492507
config,
508+
operator_allowlist,
493509
})
494510
}
495511

@@ -501,6 +517,10 @@ impl KubernetesComputeDriver {
501517
))
502518
}
503519

520+
pub fn operator_allowlist(&self) -> Option<&OperatorNamespaceAllowlist> {
521+
self.operator_allowlist.as_ref()
522+
}
523+
504524
pub fn default_image(&self) -> &str {
505525
&self.config.default_image
506526
}
@@ -1046,7 +1066,16 @@ impl KubernetesComputeDriver {
10461066
let target_namespace = match self.config.workspace_mode {
10471067
WorkspaceMode::Shared => self.config.namespace.clone(),
10481068
WorkspaceMode::Managed => self.ensure_namespace(workspace).await?,
1049-
WorkspaceMode::Operator => workspace.to_string(),
1069+
WorkspaceMode::Operator => {
1070+
if let Some(ref allowlist) = self.operator_allowlist
1071+
&& !allowlist.contains(workspace)
1072+
{
1073+
return Err(KubernetesDriverError::InvalidArgument(format!(
1074+
"workspace '{workspace}' is not in the operator namespace allowlist"
1075+
)));
1076+
}
1077+
workspace.to_string()
1078+
}
10501079
};
10511080

10521081
info!(
@@ -3479,6 +3508,83 @@ fn condition_from_value(value: &serde_json::Value) -> Option<SandboxCondition> {
34793508
})
34803509
}
34813510

3511+
fn spawn_namespace_label_watcher(
3512+
client: Client,
3513+
label_selector: String,
3514+
allowlist: OperatorNamespaceAllowlist,
3515+
) {
3516+
let ns_api: Api<Namespace> = Api::all(client);
3517+
let watcher_config = watcher::Config::default().labels(&label_selector);
3518+
3519+
tokio::spawn(async move {
3520+
loop {
3521+
let mut stream = watcher::watcher(ns_api.clone(), watcher_config.clone()).boxed();
3522+
3523+
loop {
3524+
match stream.try_next().await {
3525+
Ok(Some(Event::Applied(ns))) => {
3526+
if let Some(name) = ns.metadata.name.as_deref() {
3527+
let inner = allowlist.shared();
3528+
let mut guard = inner.write().expect("allowlist lock poisoned");
3529+
if guard.insert(name.to_string()) {
3530+
let count = guard.len();
3531+
drop(guard);
3532+
info!(
3533+
namespace = name,
3534+
total = count,
3535+
"operator namespace added to allowlist"
3536+
);
3537+
}
3538+
}
3539+
}
3540+
Ok(Some(Event::Deleted(ns))) => {
3541+
if let Some(name) = ns.metadata.name.as_deref() {
3542+
let inner = allowlist.shared();
3543+
let mut guard = inner.write().expect("allowlist lock poisoned");
3544+
if guard.remove(name) {
3545+
let count = guard.len();
3546+
drop(guard);
3547+
info!(
3548+
namespace = name,
3549+
total = count,
3550+
"operator namespace removed from allowlist"
3551+
);
3552+
}
3553+
}
3554+
}
3555+
Ok(Some(Event::Restarted(namespaces))) => {
3556+
let names: std::collections::BTreeSet<String> = namespaces
3557+
.into_iter()
3558+
.filter_map(|ns| ns.metadata.name)
3559+
.collect();
3560+
let count = names.len();
3561+
allowlist.replace(names);
3562+
info!(
3563+
total = count,
3564+
"operator namespace allowlist replaced from full relist"
3565+
);
3566+
}
3567+
Ok(None) => {
3568+
warn!("operator namespace watcher stream ended unexpectedly");
3569+
break;
3570+
}
3571+
Err(err) => {
3572+
warn!(error = %err, "operator namespace watcher stream error");
3573+
break;
3574+
}
3575+
}
3576+
}
3577+
3578+
tokio::time::sleep(Duration::from_secs(2)).await;
3579+
}
3580+
});
3581+
3582+
info!(
3583+
label_selector = %label_selector,
3584+
"operator namespace label watcher started"
3585+
);
3586+
}
3587+
34823588
#[cfg(test)]
34833589
mod tests {
34843590
use super::*;

‎crates/openshell-server/src/compute/mod.rs‎

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@ use openshell_core::{ObjectLabels, ObjectWorkspace};
4545
use openshell_driver_docker::DockerComputeDriver;
4646
use openshell_driver_kubernetes::{
4747
ComputeDriverService as KubernetesDriverService, KubernetesComputeDriver,
48+
OperatorNamespaceAllowlist,
4849
};
4950
use openshell_driver_podman::{ComputeDriverService as PodmanDriverService, PodmanComputeDriver};
5051
use prost::Message;
@@ -749,12 +750,21 @@ impl ComputeRuntime {
749750
sandbox_watch_bus: SandboxWatchBus,
750751
tracing_log_bus: TracingLogBus,
751752
supervisor_sessions: Arc<SupervisorSessionRegistry>,
752-
) -> Result<Self, ComputeError> {
753+
) -> Result<
754+
(
755+
Self,
756+
Option<Arc<std::sync::RwLock<std::collections::BTreeSet<String>>>>,
757+
),
758+
ComputeError,
759+
> {
753760
let driver = KubernetesComputeDriver::new(config)
754761
.await
755762
.map_err(|err| ComputeError::Message(err.to_string()))?;
763+
let operator_allowlist_arc = driver
764+
.operator_allowlist()
765+
.map(OperatorNamespaceAllowlist::shared);
756766
let driver: SharedComputeDriver = Arc::new(KubernetesDriverService::new(driver));
757-
Self::from_driver(
767+
let runtime = Self::from_driver(
758768
ComputeDriverKind::Kubernetes.as_str().to_string(),
759769
driver,
760770
None,
@@ -766,7 +776,8 @@ impl ComputeRuntime {
766776
tracing_log_bus,
767777
supervisor_sessions,
768778
)
769-
.await
779+
.await?;
780+
Ok((runtime, operator_allowlist_arc))
770781
}
771782

772783
pub(crate) async fn new_remote_driver(

‎crates/openshell-server/src/lib.rs‎

Lines changed: 27 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -349,7 +349,7 @@ pub(crate) async fn run_server(
349349
gateway_tls_enabled: config.tls.is_some(),
350350
endpoint_overrides: &config.compute_driver_endpoints,
351351
};
352-
let compute = build_compute_runtime(
352+
let (compute, operator_allowlist) = build_compute_runtime(
353353
&config,
354354
driver_startup,
355355
store.clone(),
@@ -468,12 +468,12 @@ pub(crate) async fn run_server(
468468
)
469469
}
470470
openshell_driver_kubernetes::WorkspaceMode::Operator => {
471-
// The operator allowlist is populated at runtime by the label
472-
// watcher and file watcher. An empty initial set is fail-closed
473-
// until the watcher populates it.
474-
auth::k8s_sa::NamespaceValidator::Allowlist(Arc::new(std::sync::RwLock::new(
475-
std::collections::BTreeSet::new(),
476-
)))
471+
// Share the driver's allowlist Arc so the SA authenticator and
472+
// the driver's namespace label watcher use the same set.
473+
let allowlist = operator_allowlist.clone().unwrap_or_else(|| {
474+
Arc::new(std::sync::RwLock::new(std::collections::BTreeSet::new()))
475+
});
476+
auth::k8s_sa::NamespaceValidator::Allowlist(allowlist)
477477
}
478478
};
479479
match kube::Client::try_default().await {
@@ -860,6 +860,8 @@ async fn terminate_signal() {
860860
// Internal wiring helper: each argument is a distinct piece of runtime state
861861
// that must be passed through, so the count is justified.
862862
#[allow(clippy::too_many_arguments)]
863+
type OperatorAllowlistArc = Option<Arc<std::sync::RwLock<std::collections::BTreeSet<String>>>>;
864+
863865
async fn build_compute_runtime(
864866
config: &Config,
865867
driver_startup: compute::driver_config::DriverStartupContext<'_>,
@@ -868,16 +870,16 @@ async fn build_compute_runtime(
868870
sandbox_watch_bus: SandboxWatchBus,
869871
tracing_log_bus: TracingLogBus,
870872
supervisor_sessions: Arc<supervisor_session::SupervisorSessionRegistry>,
871-
) -> Result<ComputeRuntime> {
873+
) -> Result<(ComputeRuntime, OperatorAllowlistArc)> {
872874
let driver = configured_compute_driver(config, driver_startup)?;
873875
info!(driver = %driver.name(), "Using compute driver");
874876

875-
let runtime = match driver {
877+
let (runtime, operator_allowlist) = match driver {
876878
ConfiguredComputeDriver::Builtin(ComputeDriverKind::Kubernetes) => {
877879
warn_if_kubernetes_sandbox_jwt_expiry_disabled(config);
878880
let k8s_config =
879881
compute::driver_config::kubernetes_config_from_context(driver_startup)?;
880-
ComputeRuntime::new_kubernetes(
882+
let (rt, allowlist) = ComputeRuntime::new_kubernetes(
881883
k8s_config,
882884
store,
883885
sandbox_index,
@@ -886,10 +888,12 @@ async fn build_compute_runtime(
886888
supervisor_sessions.clone(),
887889
)
888890
.await
891+
.map_err(|e| Error::execution(format!("failed to create compute runtime: {e}")))?;
892+
(rt, allowlist)
889893
}
890894
ConfiguredComputeDriver::Builtin(ComputeDriverKind::Docker) => {
891895
let docker_config = compute::driver_config::docker_config_from_context(driver_startup)?;
892-
ComputeRuntime::new_docker(
896+
let rt = ComputeRuntime::new_docker(
893897
config.clone(),
894898
docker_config,
895899
store,
@@ -899,10 +903,12 @@ async fn build_compute_runtime(
899903
supervisor_sessions,
900904
)
901905
.await
906+
.map_err(|e| Error::execution(format!("failed to create compute runtime: {e}")))?;
907+
(rt, None)
902908
}
903909
ConfiguredComputeDriver::Builtin(ComputeDriverKind::Podman) => {
904910
let podman_config = compute::driver_config::podman_config_from_context(driver_startup)?;
905-
ComputeRuntime::new_podman(
911+
let rt = ComputeRuntime::new_podman(
906912
podman_config,
907913
store,
908914
sandbox_index,
@@ -911,14 +917,16 @@ async fn build_compute_runtime(
911917
supervisor_sessions,
912918
)
913919
.await
920+
.map_err(|e| Error::execution(format!("failed to create compute runtime: {e}")))?;
921+
(rt, None)
914922
}
915923
ConfiguredComputeDriver::Builtin(ComputeDriverKind::Vm) => {
916924
let vm_config = compute::driver_config::vm_config_from_context(driver_startup)?;
917925
let otlp_config = driver_startup
918926
.file
919927
.and_then(|file| file.openshell.gateway.otlp.as_ref());
920928
let endpoint = compute::vm::spawn(config, &vm_config, otlp_config).await?;
921-
ComputeRuntime::new_remote_driver(
929+
let rt = ComputeRuntime::new_remote_driver(
922930
endpoint,
923931
store,
924932
sandbox_index,
@@ -927,6 +935,8 @@ async fn build_compute_runtime(
927935
supervisor_sessions,
928936
)
929937
.await
938+
.map_err(|e| Error::execution(format!("failed to create compute runtime: {e}")))?;
939+
(rt, None)
930940
}
931941
ConfiguredComputeDriver::Remote { name } => {
932942
let remote_config =
@@ -939,7 +949,7 @@ async fn build_compute_runtime(
939949
let endpoint = compute::connect_remote_compute_driver(name, &remote_config.socket_path)
940950
.await
941951
.map_err(|e| Error::execution(format!("failed to create compute runtime: {e}")))?;
942-
ComputeRuntime::new_remote_driver(
952+
let rt = ComputeRuntime::new_remote_driver(
943953
endpoint,
944954
store,
945955
sandbox_index,
@@ -948,10 +958,12 @@ async fn build_compute_runtime(
948958
supervisor_sessions,
949959
)
950960
.await
961+
.map_err(|e| Error::execution(format!("failed to create compute runtime: {e}")))?;
962+
(rt, None)
951963
}
952964
};
953965

954-
runtime.map_err(|e| Error::execution(format!("failed to create compute runtime: {e}")))
966+
Ok((runtime, operator_allowlist))
955967
}
956968

957969
#[derive(Debug, Clone)]

0 commit comments

Comments
 (0)