Skip to content

Commit 92fb8a7

Browse files
committed
fix(driver-k8s): add label selector to sandbox watch stream and list (NVIDIA#2211)
The Kubernetes driver's watch_sandboxes() crashed with a file descriptor leak when non-gateway Sandbox resources (e.g. from SandboxWarmPool) existed in the namespace. The unfiltered watch received events for all Sandbox objects, failed to extract IDs from warm pool objects, and propagated errors that triggered a 2-second reconnect loop leaking HTTP/2 connections until FD exhaustion. Two fixes: - Add openshell.ai/managed-by=openshell label selector to the watcher config and list params so only gateway-managed objects are received. - Defensively skip unrecognized objects with a debug log instead of sending errors to the channel. Signed-off-by: Roland Huß <[email protected]> Assisted-By: 🤖 Claude Code Signed-off-by: Roland Huß <[email protected]>
1 parent 8ba7247 commit 92fb8a7

1 file changed

Lines changed: 110 additions & 50 deletions

File tree

  • crates/openshell-driver-kubernetes/src

‎crates/openshell-driver-kubernetes/src/driver.rs‎

Lines changed: 110 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -485,7 +485,9 @@ impl KubernetesComputeDriver {
485485
.await?;
486486
match tokio::time::timeout(
487487
KUBE_API_TIMEOUT,
488-
agent_sandbox_api.api.list(&ListParams::default()),
488+
agent_sandbox_api
489+
.api
490+
.list(&ListParams::default().labels(&format!("{LABEL_MANAGED_BY}={LABEL_MANAGED_BY_VALUE}"))),
489491
)
490492
.await
491493
{
@@ -713,8 +715,10 @@ impl KubernetesComputeDriver {
713715
.supported_agent_sandbox_api(self.watch_client.clone())
714716
.await?;
715717
let event_api: Api<KubeEventObj> = Api::namespaced(self.watch_client.clone(), &namespace);
718+
let sandbox_watcher_config = watcher::Config::default()
719+
.labels(&format!("{LABEL_MANAGED_BY}={LABEL_MANAGED_BY_VALUE}"));
716720
let mut sandbox_stream =
717-
watcher::watcher(agent_sandbox_api.api, watcher::Config::default()).boxed();
721+
watcher::watcher(agent_sandbox_api.api, sandbox_watcher_config).boxed();
718722
let mut event_stream = watcher::watcher(event_api, watcher::Config::default()).boxed();
719723
let (tx, rx) = mpsc::channel(256);
720724

@@ -726,64 +730,52 @@ impl KubernetesComputeDriver {
726730
tokio::select! {
727731
result = sandbox_stream.try_next() => match result {
728732
Ok(Some(Event::Applied(obj))) => {
729-
match sandbox_from_object(&namespace, obj) {
730-
Ok(sandbox) => {
731-
update_indexes(&mut sandbox_name_to_id, &mut agent_pod_to_id, &sandbox);
732-
let event = WatchSandboxesEvent {
733-
payload: Some(watch_sandboxes_event::Payload::Sandbox(
734-
WatchSandboxesSandboxEvent { sandbox: Some(sandbox) }
735-
)),
736-
};
737-
if tx.send(Ok(event)).await.is_err() {
738-
break;
739-
}
740-
}
741-
Err(err) => {
742-
if tx.send(Err(KubernetesDriverError::Message(err))).await.is_err() {
743-
break;
744-
}
733+
let obj_name = obj.metadata.name.clone().unwrap_or_default();
734+
if let Ok(sandbox) = sandbox_from_object(&namespace, obj) {
735+
update_indexes(&mut sandbox_name_to_id, &mut agent_pod_to_id, &sandbox);
736+
let event = WatchSandboxesEvent {
737+
payload: Some(watch_sandboxes_event::Payload::Sandbox(
738+
WatchSandboxesSandboxEvent { sandbox: Some(sandbox) }
739+
)),
740+
};
741+
if tx.send(Ok(event)).await.is_err() {
742+
break;
745743
}
744+
} else {
745+
debug!(object_name = %obj_name, "skipping unrecognized Sandbox object in Applied event");
746746
}
747747
}
748748
Ok(Some(Event::Deleted(obj))) => {
749-
match sandbox_id_from_object(&obj) {
750-
Ok(sandbox_id) => {
751-
remove_indexes(&mut sandbox_name_to_id, &mut agent_pod_to_id, &sandbox_id);
752-
let event = WatchSandboxesEvent {
753-
payload: Some(watch_sandboxes_event::Payload::Deleted(
754-
WatchSandboxesDeletedEvent { sandbox_id }
755-
)),
756-
};
757-
if tx.send(Ok(event)).await.is_err() {
758-
break;
759-
}
760-
}
761-
Err(err) => {
762-
if tx.send(Err(KubernetesDriverError::Message(err))).await.is_err() {
763-
break;
764-
}
749+
let obj_name = obj.metadata.name.clone().unwrap_or_default();
750+
if let Ok(sandbox_id) = sandbox_id_from_object(&obj) {
751+
remove_indexes(&mut sandbox_name_to_id, &mut agent_pod_to_id, &sandbox_id);
752+
let event = WatchSandboxesEvent {
753+
payload: Some(watch_sandboxes_event::Payload::Deleted(
754+
WatchSandboxesDeletedEvent { sandbox_id }
755+
)),
756+
};
757+
if tx.send(Ok(event)).await.is_err() {
758+
break;
765759
}
760+
} else {
761+
debug!(object_name = %obj_name, "skipping unrecognized Sandbox object in Deleted event");
766762
}
767763
}
768764
Ok(Some(Event::Restarted(objs))) => {
769765
for obj in objs {
770-
match sandbox_from_object(&namespace, obj) {
771-
Ok(sandbox) => {
772-
update_indexes(&mut sandbox_name_to_id, &mut agent_pod_to_id, &sandbox);
773-
let event = WatchSandboxesEvent {
774-
payload: Some(watch_sandboxes_event::Payload::Sandbox(
775-
WatchSandboxesSandboxEvent { sandbox: Some(sandbox) }
776-
)),
777-
};
778-
if tx.send(Ok(event)).await.is_err() {
779-
return;
780-
}
781-
}
782-
Err(err) => {
783-
if tx.send(Err(KubernetesDriverError::Message(err))).await.is_err() {
784-
return;
785-
}
766+
let obj_name = obj.metadata.name.clone().unwrap_or_default();
767+
if let Ok(sandbox) = sandbox_from_object(&namespace, obj) {
768+
update_indexes(&mut sandbox_name_to_id, &mut agent_pod_to_id, &sandbox);
769+
let event = WatchSandboxesEvent {
770+
payload: Some(watch_sandboxes_event::Payload::Sandbox(
771+
WatchSandboxesSandboxEvent { sandbox: Some(sandbox) }
772+
)),
773+
};
774+
if tx.send(Ok(event)).await.is_err() {
775+
return;
786776
}
777+
} else {
778+
debug!(object_name = %obj_name, "skipping unrecognized Sandbox object in Restarted event");
787779
}
788780
}
789781
}
@@ -3842,4 +3834,72 @@ mod tests {
38423834
let storage = &vct[0]["spec"]["resources"]["requests"]["storage"];
38433835
assert_eq!(storage, DEFAULT_WORKSPACE_STORAGE_SIZE);
38443836
}
3837+
3838+
fn make_dynamic_object(
3839+
name: &str,
3840+
labels: Option<BTreeMap<String, String>>,
3841+
) -> DynamicObject {
3842+
DynamicObject {
3843+
metadata: ObjectMeta {
3844+
name: Some(name.to_string()),
3845+
namespace: Some("test-ns".to_string()),
3846+
labels,
3847+
..Default::default()
3848+
},
3849+
types: None,
3850+
data: serde_json::json!({}),
3851+
}
3852+
}
3853+
3854+
#[test]
3855+
fn test_sandbox_id_from_object_with_label() {
3856+
let mut labels = BTreeMap::new();
3857+
labels.insert(LABEL_SANDBOX_ID.to_string(), "my-sandbox-id".to_string());
3858+
let obj = make_dynamic_object("sandbox-obj", Some(labels));
3859+
let result = sandbox_id_from_object(&obj);
3860+
assert_eq!(result, Ok("my-sandbox-id".to_string()));
3861+
}
3862+
3863+
#[test]
3864+
fn test_sandbox_id_from_object_with_name_prefix() {
3865+
let obj = make_dynamic_object("sandbox-abc123", None);
3866+
let result = sandbox_id_from_object(&obj);
3867+
assert_eq!(result, Ok("abc123".to_string()));
3868+
}
3869+
3870+
#[test]
3871+
fn test_sandbox_id_from_object_unknown() {
3872+
let obj = make_dynamic_object("openshell-warm-pool-xyz", None);
3873+
let result = sandbox_id_from_object(&obj);
3874+
assert!(result.is_err());
3875+
}
3876+
3877+
#[test]
3878+
fn test_sandbox_id_from_object_managed_by_but_no_sandbox_id() {
3879+
let mut labels = BTreeMap::new();
3880+
labels.insert(LABEL_MANAGED_BY.to_string(), LABEL_MANAGED_BY_VALUE.to_string());
3881+
let obj = make_dynamic_object("openshell-warm-pool-xyz", Some(labels));
3882+
let result = sandbox_id_from_object(&obj);
3883+
assert!(result.is_err());
3884+
}
3885+
3886+
#[test]
3887+
fn test_sandbox_from_object_success() {
3888+
let mut labels = BTreeMap::new();
3889+
labels.insert(LABEL_SANDBOX_ID.to_string(), "test-id".to_string());
3890+
let obj = make_dynamic_object("sandbox-test-id", Some(labels));
3891+
let result = sandbox_from_object("test-ns", obj);
3892+
assert!(result.is_ok());
3893+
let sandbox = result.unwrap();
3894+
assert_eq!(sandbox.id, "test-id");
3895+
assert_eq!(sandbox.name, "sandbox-test-id");
3896+
assert_eq!(sandbox.namespace, "test-ns");
3897+
}
3898+
3899+
#[test]
3900+
fn test_sandbox_from_object_unknown() {
3901+
let obj = make_dynamic_object("openshell-warm-pool-abc", None);
3902+
let result = sandbox_from_object("test-ns", obj);
3903+
assert!(result.is_err());
3904+
}
38453905
}

0 commit comments

Comments
 (0)