Skip to content

Commit ba21bb3

Browse files
authored
feat(kubernetes): support agent-sandbox v1beta1 (NVIDIA#2009)
* feat(kubernetes): support agent-sandbox v1beta1 Signed-off-by: Taylor Mutch <[email protected]> * ci(kubernetes): test agent-sandbox api versions Signed-off-by: Taylor Mutch <[email protected]> * fix(kubernetes): retry agent-sandbox raw 404s Signed-off-by: Taylor Mutch <[email protected]> * fix(kubernetes): harden agent-sandbox api setup Signed-off-by: Taylor Mutch <[email protected]> * fix(kubernetes): cache agent-sandbox api version Signed-off-by: Taylor Mutch <[email protected]> * docs(kubernetes): document agent-sandbox upgrade behavior Signed-off-by: Taylor Mutch <[email protected]> * refactor(kubernetes): collapse agent-sandbox api selection Signed-off-by: Taylor Mutch <[email protected]> --------- Signed-off-by: Taylor Mutch <[email protected]>
1 parent f569a0a commit ba21bb3

11 files changed

Lines changed: 382 additions & 56 deletions

File tree

‎.github/workflows/branch-e2e.yml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -115,12 +115,22 @@ jobs:
115115
kubernetes-e2e:
116116
needs: [pr_metadata, build-gateway, build-supervisor]
117117
if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true'
118+
strategy:
119+
fail-fast: false
120+
matrix:
121+
include:
122+
- agent_sandbox_api: v1beta1
123+
agent_sandbox_version: v0.5.0
124+
- agent_sandbox_api: v1alpha1
125+
agent_sandbox_version: v0.4.6
118126
permissions:
119127
contents: read
120128
packages: read
121129
uses: ./.github/workflows/e2e-kubernetes-test.yml
122130
with:
123131
image-tag: ${{ github.sha }}
132+
job-name: Kubernetes E2E (Rust smoke, Agent Sandbox ${{ matrix.agent_sandbox_api }})
133+
agent-sandbox-version: ${{ matrix.agent_sandbox_version }}
124134

125135
kubernetes-ha-e2e:
126136
needs: [pr_metadata, build-gateway, build-supervisor]

‎.github/workflows/e2e-kubernetes-test.yml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,11 @@ on:
3232
required: false
3333
type: string
3434
default: ""
35+
agent-sandbox-version:
36+
description: "Agent Sandbox release to install before OpenShell"
37+
required: false
38+
type: string
39+
default: "v0.5.0"
3540
mise-version:
3641
description: "mise version to install on the bare Kubernetes e2e runner"
3742
required: false
@@ -114,6 +119,7 @@ jobs:
114119
115120
- name: Run Kubernetes E2E (Rust smoke)
116121
env:
122+
AGENT_SANDBOX_VERSION: ${{ inputs.agent-sandbox-version }}
117123
OPENSHELL_E2E_KUBE_CONTEXT: kind-${{ env.KIND_CLUSTER_NAME }}
118124
OPENSHELL_E2E_KUBE_EXTRA_VALUES: ${{ inputs.extra-helm-values }}
119125
OPENSHELL_E2E_KUBE_EXTERNAL_POSTGRES_SECRET: ${{ inputs.external-postgres-secret }}

‎architecture/gateway.md‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -66,10 +66,13 @@ token through `IssueSandboxToken`. The gateway validates that projected token
6666
with Kubernetes `TokenReview`, requires the configured sandbox service account,
6767
checks the returned pod binding against the live pod UID, and verifies the pod's
6868
controlling `Sandbox` ownerReference against the live Sandbox CR UID and
69-
sandbox-id label before minting the gateway JWT. Supervisors renew gateway JWTs
70-
in memory before expiry only while the sandbox record still exists. Older tokens
71-
are not server-revoked; shared deployments bound replay exposure with short
72-
`gateway_jwt.ttl_secs` lifetimes. The config default is
69+
sandbox-id label before minting the gateway JWT. The bootstrap path accepts
70+
both `agents.x-k8s.io/v1beta1` ownerReferences from newer Agent Sandbox
71+
controllers and `agents.x-k8s.io/v1alpha1` ownerReferences from existing
72+
deployments. Supervisors renew gateway JWTs in memory before expiry only while
73+
the sandbox record still exists. Older tokens are not server-revoked; shared
74+
deployments bound replay exposure with short `gateway_jwt.ttl_secs` lifetimes.
75+
The config default is
7376
`gateway_jwt.ttl_secs = 0` for local single-player Docker, Podman, and VM
7477
gateways; those tokens carry `exp = 0` and do not expire. Kubernetes and other
7578
shared deployments should set a positive TTL.

‎crates/openshell-driver-kubernetes/README.md‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -15,9 +15,13 @@ credential injection, policy polling, logs, and the gateway relay.
1515

1616
## Sandbox Resource
1717

18-
The driver works with the `agents.x-k8s.io/v1alpha1` `Sandbox` custom resource.
19-
Driver events map Kubernetes object state and platform events into the shared
20-
compute-driver protobuf surface used by the gateway.
18+
The driver works with the `agents.x-k8s.io` `Sandbox` custom resource. It
19+
detects the served Sandbox API at runtime, caches the selected API version for
20+
the gateway process, and uses `v1beta1` when available before falling back to
21+
`v1alpha1`. Restart the gateway after an in-place Agent Sandbox upgrade so the
22+
driver can detect served API versions again. Driver events map Kubernetes object
23+
state and platform events into the shared compute-driver protobuf surface used
24+
by the gateway.
2125

2226
Kubernetes API calls use explicit timeouts so gRPC handlers do not block
2327
indefinitely when the API server is slow or unavailable.

‎crates/openshell-driver-kubernetes/src/driver.rs‎

Lines changed: 147 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -34,8 +34,9 @@ use openshell_core::proto_struct::{struct_to_json_object, value_to_json};
3434
use serde::Deserialize;
3535
use std::collections::BTreeMap;
3636
use std::pin::Pin;
37+
use std::sync::Arc;
3738
use std::time::Duration;
38-
use tokio::sync::mpsc;
39+
use tokio::sync::{OnceCell, mpsc};
3940
use tokio_stream::wrappers::ReceiverStream;
4041
use tracing::{debug, info, warn};
4142

@@ -80,12 +81,19 @@ impl From<KubernetesDriverError> for openshell_core::ComputeDriverError {
8081
const KUBE_API_TIMEOUT: Duration = Duration::from_secs(30);
8182

8283
const SANDBOX_GROUP: &str = "agents.x-k8s.io";
83-
const SANDBOX_VERSION: &str = "v1alpha1";
84+
const SANDBOX_VERSION_V1BETA1: &str = "v1beta1";
85+
const SANDBOX_VERSION_V1ALPHA1: &str = "v1alpha1";
86+
const SANDBOX_VERSIONS: &[&str] = &[SANDBOX_VERSION_V1BETA1, SANDBOX_VERSION_V1ALPHA1];
8487
pub const SANDBOX_KIND: &str = "Sandbox";
8588

8689
const GPU_RESOURCE_NAME: &str = "nvidia.com/gpu";
8790
const SPIFFE_WORKLOAD_API_VOLUME_NAME: &str = "spiffe-workload-api";
8891

92+
struct AgentSandboxApi {
93+
api: Api<DynamicObject>,
94+
resource: ApiResource,
95+
}
96+
8997
// This POC treats the selected Struct as a driver-local typed schema. Once the
9098
// Kubernetes shape stabilizes, these serde structs may move to driver-local
9199
// protobuf definitions, but the typed decode should stay inside this driver.
@@ -190,6 +198,7 @@ const WORKSPACE_SENTINEL: &str = ".workspace-initialized";
190198
pub struct KubernetesComputeDriver {
191199
client: Client,
192200
watch_client: Client,
201+
sandbox_api_version: Arc<OnceCell<&'static str>>,
193202
config: KubernetesComputeConfig,
194203
}
195204

@@ -232,6 +241,7 @@ impl KubernetesComputeDriver {
232241
Ok(Self {
233242
client,
234243
watch_client,
244+
sandbox_api_version: Arc::new(OnceCell::new()),
235245
config,
236246
})
237247
}
@@ -256,16 +266,68 @@ impl KubernetesComputeDriver {
256266
&self.config.ssh_socket_path
257267
}
258268

259-
fn watch_api(&self) -> Api<DynamicObject> {
260-
let gvk = GroupVersionKind::gvk(SANDBOX_GROUP, SANDBOX_VERSION, SANDBOX_KIND);
269+
fn agent_sandbox_api(&self, client: Client, sandbox_api_version: &str) -> AgentSandboxApi {
270+
let gvk = GroupVersionKind::gvk(SANDBOX_GROUP, sandbox_api_version, SANDBOX_KIND);
261271
let resource = ApiResource::from_gvk(&gvk);
262-
Api::namespaced_with(self.watch_client.clone(), &self.config.namespace, &resource)
272+
let api = Api::namespaced_with(client, &self.config.namespace, &resource);
273+
AgentSandboxApi { api, resource }
263274
}
264275

265-
fn api(&self) -> Api<DynamicObject> {
266-
let gvk = GroupVersionKind::gvk(SANDBOX_GROUP, SANDBOX_VERSION, SANDBOX_KIND);
267-
let resource = ApiResource::from_gvk(&gvk);
268-
Api::namespaced_with(self.client.clone(), &self.config.namespace, &resource)
276+
async fn supported_agent_sandbox_api(&self, client: Client) -> Result<AgentSandboxApi, String> {
277+
let sandbox_api_version = self.supported_sandbox_api_version(client.clone()).await?;
278+
Ok(self.agent_sandbox_api(client, sandbox_api_version))
279+
}
280+
281+
async fn supported_sandbox_api_version(&self, client: Client) -> Result<&'static str, String> {
282+
self.sandbox_api_version
283+
.get_or_try_init(
284+
|| async move { self.detect_supported_sandbox_api_version(client).await },
285+
)
286+
.await
287+
.copied()
288+
}
289+
290+
async fn detect_supported_sandbox_api_version(
291+
&self,
292+
client: Client,
293+
) -> Result<&'static str, String> {
294+
for sandbox_api_version in SANDBOX_VERSIONS {
295+
let agent_sandbox_api = self.agent_sandbox_api(client.clone(), sandbox_api_version);
296+
match tokio::time::timeout(
297+
KUBE_API_TIMEOUT,
298+
agent_sandbox_api.api.list(&ListParams::default().limit(1)),
299+
)
300+
.await
301+
{
302+
Ok(Ok(_)) => {
303+
debug!(
304+
namespace = %self.config.namespace,
305+
sandbox_api_version = %sandbox_api_version,
306+
"Selected Agent Sandbox API version"
307+
);
308+
return Ok(sandbox_api_version);
309+
}
310+
Ok(Err(err)) if should_try_next_sandbox_api_version(&err) => {
311+
debug!(
312+
namespace = %self.config.namespace,
313+
sandbox_api_version = %sandbox_api_version,
314+
error = %err,
315+
"Sandbox API version is not available; trying next supported version"
316+
);
317+
}
318+
Ok(Err(err)) => return Err(err.to_string()),
319+
Err(_elapsed) => {
320+
return Err(format!(
321+
"timed out after {}s waiting for Kubernetes API",
322+
KUBE_API_TIMEOUT.as_secs()
323+
));
324+
}
325+
}
326+
}
327+
Err(format!(
328+
"no supported Agent Sandbox API version is available; tried {}",
329+
SANDBOX_VERSIONS.join(", ")
330+
))
269331
}
270332

271333
async fn has_gpu_capacity(&self) -> Result<bool, KubeError> {
@@ -306,8 +368,10 @@ impl KubernetesComputeDriver {
306368
"Fetching sandbox from Kubernetes"
307369
);
308370

309-
let api = self.api();
310-
match tokio::time::timeout(KUBE_API_TIMEOUT, api.get(name)).await {
371+
let agent_sandbox_api = self
372+
.supported_agent_sandbox_api(self.client.clone())
373+
.await?;
374+
match tokio::time::timeout(KUBE_API_TIMEOUT, agent_sandbox_api.api.get(name)).await {
311375
Ok(Ok(obj)) => sandbox_from_object(&self.config.namespace, obj).map(Some),
312376
Ok(Err(KubeError::Api(err))) if err.code == 404 => {
313377
debug!(sandbox_name = %name, "Sandbox not found in Kubernetes");
@@ -341,8 +405,15 @@ impl KubernetesComputeDriver {
341405
"Listing sandboxes from Kubernetes"
342406
);
343407

344-
let api = self.api();
345-
match tokio::time::timeout(KUBE_API_TIMEOUT, api.list(&ListParams::default())).await {
408+
let agent_sandbox_api = self
409+
.supported_agent_sandbox_api(self.client.clone())
410+
.await?;
411+
match tokio::time::timeout(
412+
KUBE_API_TIMEOUT,
413+
agent_sandbox_api.api.list(&ListParams::default()),
414+
)
415+
.await
416+
{
346417
Ok(Ok(list)) => {
347418
let mut sandboxes = list
348419
.items
@@ -396,9 +467,11 @@ impl KubernetesComputeDriver {
396467
"Creating sandbox in Kubernetes"
397468
);
398469

399-
let gvk = GroupVersionKind::gvk(SANDBOX_GROUP, SANDBOX_VERSION, SANDBOX_KIND);
400-
let resource = ApiResource::from_gvk(&gvk);
401-
let mut obj = DynamicObject::new(name, &resource);
470+
let agent_sandbox_api = self
471+
.supported_agent_sandbox_api(self.client.clone())
472+
.await
473+
.map_err(KubernetesDriverError::Message)?;
474+
let mut obj = DynamicObject::new(name, &agent_sandbox_api.resource);
402475
obj.metadata = ObjectMeta {
403476
name: Some(name.to_string()),
404477
namespace: Some(self.config.namespace.clone()),
@@ -430,9 +503,11 @@ impl KubernetesComputeDriver {
430503
.provider_spiffe_workload_api_socket_path,
431504
};
432505
obj.data = sandbox_to_k8s_spec(sandbox.spec.as_ref(), &params);
433-
let api = self.api();
434-
435-
match tokio::time::timeout(KUBE_API_TIMEOUT, api.create(&PostParams::default(), &obj)).await
506+
match tokio::time::timeout(
507+
KUBE_API_TIMEOUT,
508+
agent_sandbox_api.api.create(&PostParams::default(), &obj),
509+
)
510+
.await
436511
{
437512
Ok(Ok(_result)) => {
438513
info!(
@@ -473,9 +548,14 @@ impl KubernetesComputeDriver {
473548
"Deleting sandbox from Kubernetes"
474549
);
475550

476-
let api = self.api();
477-
match tokio::time::timeout(KUBE_API_TIMEOUT, api.delete(name, &DeleteParams::default()))
478-
.await
551+
let agent_sandbox_api = self
552+
.supported_agent_sandbox_api(self.client.clone())
553+
.await?;
554+
match tokio::time::timeout(
555+
KUBE_API_TIMEOUT,
556+
agent_sandbox_api.api.delete(name, &DeleteParams::default()),
557+
)
558+
.await
479559
{
480560
Ok(Ok(_response)) => {
481561
info!(sandbox_name = %name, "Sandbox deleted from Kubernetes");
@@ -508,8 +588,10 @@ impl KubernetesComputeDriver {
508588
}
509589

510590
pub async fn sandbox_exists(&self, name: &str) -> Result<bool, String> {
511-
let api = self.api();
512-
match tokio::time::timeout(KUBE_API_TIMEOUT, api.get(name)).await {
591+
let agent_sandbox_api = self
592+
.supported_agent_sandbox_api(self.client.clone())
593+
.await?;
594+
match tokio::time::timeout(KUBE_API_TIMEOUT, agent_sandbox_api.api.get(name)).await {
513595
Ok(Ok(_)) => Ok(true),
514596
Ok(Err(KubeError::Api(err))) if err.code == 404 => Ok(false),
515597
Ok(Err(err)) => Err(err.to_string()),
@@ -524,9 +606,12 @@ impl KubernetesComputeDriver {
524606
#[allow(clippy::unused_async)]
525607
pub async fn watch_sandboxes(&self) -> Result<WatchStream, String> {
526608
let namespace = self.config.namespace.clone();
527-
let sandbox_api = self.watch_api();
609+
let agent_sandbox_api = self
610+
.supported_agent_sandbox_api(self.watch_client.clone())
611+
.await?;
528612
let event_api: Api<KubeEventObj> = Api::namespaced(self.watch_client.clone(), &namespace);
529-
let mut sandbox_stream = watcher::watcher(sandbox_api, watcher::Config::default()).boxed();
613+
let mut sandbox_stream =
614+
watcher::watcher(agent_sandbox_api.api, watcher::Config::default()).boxed();
530615
let mut event_stream = watcher::watcher(event_api, watcher::Config::default()).boxed();
531616
let (tx, rx) = mpsc::channel(256);
532617

@@ -650,6 +735,14 @@ impl KubernetesComputeDriver {
650735
}
651736
}
652737

738+
fn should_try_next_sandbox_api_version(err: &KubeError) -> bool {
739+
// Kubernetes returns a structured 404 for some missing API resources and a
740+
// raw "404 page not found" body for others. Both mean the probed
741+
// group/version is unavailable and the next supported Sandbox API version
742+
// should be tried.
743+
matches!(err, KubeError::Api(api) if api.code == 404)
744+
}
745+
653746
fn validate_gpu_request(
654747
gpu_requirements: Option<&GpuResourceRequirements>,
655748
) -> Result<(), tonic::Status> {
@@ -2013,6 +2106,34 @@ mod tests {
20132106
}
20142107
}
20152108

2109+
fn kube_api_error(code: u16, message: &str) -> KubeError {
2110+
KubeError::Api(kube::core::ErrorResponse {
2111+
status: if code == 404 {
2112+
"404 Not Found".to_string()
2113+
} else {
2114+
"Failure".to_string()
2115+
},
2116+
message: message.to_string(),
2117+
reason: "Failed to parse error data".to_string(),
2118+
code,
2119+
})
2120+
}
2121+
2122+
#[test]
2123+
fn sandbox_api_version_probe_retries_on_structured_and_raw_404() {
2124+
let structured = kube_api_error(404, "could not find the requested resource");
2125+
assert!(should_try_next_sandbox_api_version(&structured));
2126+
2127+
let raw = kube_api_error(404, "404 page not found\n");
2128+
assert!(should_try_next_sandbox_api_version(&raw));
2129+
}
2130+
2131+
#[test]
2132+
fn sandbox_api_version_probe_keeps_non_404_errors() {
2133+
let err = kube_api_error(403, "sandboxes.agents.x-k8s.io is forbidden");
2134+
assert!(!should_try_next_sandbox_api_version(&err));
2135+
}
2136+
20162137
#[test]
20172138
fn driver_config_rejects_invalid_shape() {
20182139
let template = SandboxTemplate {

0 commit comments

Comments
 (0)