Skip to content

Commit 9057afb

Browse files
committed
fix(kubernetes): align warm pool CRDs with operator v1beta1 schema
The Milestone 1 implementation used agents.x-k8s.io/v1alpha1 for the warm pool CRDs, but the Red Hat Agent Sandbox operator v0.9.0 ships extensions.agents.x-k8s.io/v1beta1. This fixes the API group, version, status field parsing (conditions instead of phase, podIPs instead of podIP), warmPoolRef format, and image matching (resolve SandboxTemplate reference instead of reading image from the pool directly). Also adds a smoke test guide and automated script for validating the end-to-end warm pool claim + ActivateSandbox flow, along with K8s manifests for deploying the unidentified supervisor warm pool. Validated on ROSA HCP 4.22.3: 3 runs averaged 1.9s (claim 1.35s + activate 0.56s), under the 2s target. Assisted-By: 🤖 Claude Code Signed-off-by: Roland Huß <[email protected]>
1 parent 954e1d5 commit 9057afb

6 files changed

Lines changed: 1099 additions & 80 deletions

File tree

‎crates/openshell-driver-kubernetes/src/driver.rs‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -966,7 +966,9 @@ impl KubernetesComputeDriver {
966966
}
967967
};
968968

969-
let Some(pool) = crate::warm_pool::find_matching_pool(&pools, image) else {
969+
let Some(pool) =
970+
crate::warm_pool::find_matching_pool(&self.client, &self.config.namespace, &pools, image).await
971+
else {
970972
debug!(image = %image, "No warm pool found for image");
971973
return None;
972974
};

‎crates/openshell-driver-kubernetes/src/warm_pool.rs‎

Lines changed: 185 additions & 79 deletions
Original file line numberDiff line numberDiff line change
@@ -9,10 +9,11 @@ use kube::core::{DynamicObject, GroupVersionKind, ObjectMeta};
99
use serde_json::json;
1010
use tracing::{debug, info, warn};
1111

12-
const WARM_POOL_GROUP: &str = "agents.x-k8s.io";
13-
const WARM_POOL_VERSION: &str = "v1alpha1";
12+
const WARM_POOL_GROUP: &str = "extensions.agents.x-k8s.io";
13+
const WARM_POOL_VERSION: &str = "v1beta1";
1414
const WARM_POOL_KIND: &str = "SandboxWarmPool";
1515
const CLAIM_KIND: &str = "SandboxClaim";
16+
const TEMPLATE_KIND: &str = "SandboxTemplate";
1617

1718
#[derive(Debug, thiserror::Error)]
1819
pub enum WarmPoolError {
@@ -48,31 +49,80 @@ pub async fn list_warm_pools(
4849
Ok(list.items)
4950
}
5051

51-
pub fn find_matching_pool<'a>(
52+
fn template_api(client: Client, namespace: &str) -> Api<DynamicObject> {
53+
let gvk = GroupVersionKind::gvk(WARM_POOL_GROUP, WARM_POOL_VERSION, TEMPLATE_KIND);
54+
let resource = ApiResource::from_gvk(&gvk);
55+
Api::namespaced_with(client, namespace, &resource)
56+
}
57+
58+
fn extract_template_image(template: &DynamicObject) -> Option<String> {
59+
template
60+
.data
61+
.get("spec")
62+
.and_then(|s| s.get("podTemplate"))
63+
.and_then(|pt| pt.get("spec"))
64+
.and_then(|s| s.get("containers"))
65+
.and_then(|c| c.as_array())
66+
.and_then(|arr| arr.first())
67+
.and_then(|c| c.get("image"))
68+
.and_then(|i| i.as_str())
69+
.map(String::from)
70+
}
71+
72+
pub async fn find_matching_pool<'a>(
73+
client: &Client,
74+
namespace: &str,
5275
pools: &'a [DynamicObject],
5376
image: &str,
5477
) -> Option<&'a DynamicObject> {
55-
pools.iter().find(|pool| {
56-
let image_match = pool
57-
.data
58-
.get("spec")
59-
.and_then(|s| s.get("template"))
60-
.and_then(|t| t.get("containers"))
61-
.and_then(|c| c.as_array())
62-
.and_then(|arr| arr.first())
63-
.and_then(|c| c.get("image"))
64-
.and_then(|i| i.as_str())
65-
.is_some_and(|i| i == image);
78+
let tmpl_api = template_api(client.clone(), namespace);
6679

80+
for pool in pools {
6781
let ready = pool
6882
.data
6983
.get("status")
7084
.and_then(|s| s.get("readyReplicas"))
7185
.and_then(serde_json::Value::as_i64)
7286
.unwrap_or(0);
7387

74-
image_match && ready > 0
75-
})
88+
if ready == 0 {
89+
continue;
90+
}
91+
92+
let template_name = pool
93+
.data
94+
.get("spec")
95+
.and_then(|s| s.get("sandboxTemplateRef"))
96+
.and_then(|r| r.get("name"))
97+
.and_then(|n| n.as_str());
98+
99+
let Some(tmpl_name) = template_name else {
100+
warn!(pool = ?pool.metadata.name, "SandboxWarmPool missing sandboxTemplateRef");
101+
continue;
102+
};
103+
104+
match tmpl_api.get(tmpl_name).await {
105+
Ok(template) => {
106+
if let Some(tmpl_image) = extract_template_image(&template) {
107+
if tmpl_image == image {
108+
info!(
109+
pool = ?pool.metadata.name,
110+
template = %tmpl_name,
111+
image = %image,
112+
ready_replicas = %ready,
113+
"Found matching warm pool"
114+
);
115+
return Some(pool);
116+
}
117+
}
118+
}
119+
Err(e) => {
120+
warn!(template = %tmpl_name, error = %e, "Failed to fetch SandboxTemplate");
121+
}
122+
}
123+
}
124+
125+
None
76126
}
77127

78128
pub async fn create_claim(
@@ -91,8 +141,9 @@ pub async fn create_claim(
91141
};
92142
obj.data = json!({
93143
"spec": {
94-
"warmPoolRef": pool_name,
95-
"sandboxId": sandbox_id,
144+
"warmPoolRef": {
145+
"name": pool_name,
146+
},
96147
}
97148
});
98149

@@ -118,44 +169,58 @@ pub async fn wait_for_claim_ready(
118169

119170
match api.get(claim_name).await {
120171
Ok(obj) => {
121-
let phase = obj
172+
let conditions = obj
122173
.data
123174
.get("status")
124-
.and_then(|s| s.get("phase"))
125-
.and_then(|p| p.as_str())
126-
.unwrap_or("Pending");
127-
128-
match phase {
129-
"Ready" => {
130-
let pod_ip = obj
131-
.data
132-
.get("status")
133-
.and_then(|s| s.get("sandbox"))
134-
.and_then(|s| s.get("podIP"))
135-
.and_then(|ip| ip.as_str())
136-
.map(String::from);
137-
138-
match pod_ip {
139-
Some(ip) => {
140-
info!(claim = %claim_name, pod_ip = %ip, "SandboxClaim ready");
141-
return Ok(ip);
142-
}
143-
None => return Err(WarmPoolError::MissingPodIp),
175+
.and_then(|s| s.get("conditions"))
176+
.and_then(|c| c.as_array());
177+
178+
let ready_condition = conditions.and_then(|conds| {
179+
conds.iter().find(|c| {
180+
c.get("type").and_then(|t| t.as_str()) == Some("Ready")
181+
})
182+
});
183+
184+
let is_ready = ready_condition
185+
.and_then(|c| c.get("status"))
186+
.and_then(|s| s.as_str())
187+
== Some("True");
188+
189+
let is_failed = ready_condition
190+
.and_then(|c| c.get("reason"))
191+
.and_then(|r| r.as_str())
192+
.is_some_and(|r| r.contains("Failed"));
193+
194+
if is_ready {
195+
let pod_ip = obj
196+
.data
197+
.get("status")
198+
.and_then(|s| s.get("sandbox"))
199+
.and_then(|s| s.get("podIPs"))
200+
.and_then(|ips| ips.as_array())
201+
.and_then(|arr| arr.first())
202+
.and_then(|ip| ip.as_str())
203+
.map(String::from);
204+
205+
match pod_ip {
206+
Some(ip) => {
207+
info!(claim = %claim_name, pod_ip = %ip, "SandboxClaim ready");
208+
return Ok(ip);
144209
}
210+
None => return Err(WarmPoolError::MissingPodIp),
145211
}
146-
"Failed" => {
147-
return Err(WarmPoolError::ClaimFailed(
148-
obj.data
149-
.get("status")
150-
.and_then(|s| s.get("message"))
151-
.and_then(|m| m.as_str())
152-
.unwrap_or("unknown failure")
153-
.to_string(),
154-
));
155-
}
156-
_ => {
157-
debug!(claim = %claim_name, phase = %phase, "Waiting for SandboxClaim");
158-
}
212+
} else if is_failed {
213+
let message = ready_condition
214+
.and_then(|c| c.get("message"))
215+
.and_then(|m| m.as_str())
216+
.unwrap_or("unknown failure");
217+
return Err(WarmPoolError::ClaimFailed(message.to_string()));
218+
} else {
219+
let reason = ready_condition
220+
.and_then(|c| c.get("reason"))
221+
.and_then(|r| r.as_str())
222+
.unwrap_or("Pending");
223+
debug!(claim = %claim_name, reason = %reason, "Waiting for SandboxClaim");
159224
}
160225
}
161226
Err(e) => {
@@ -171,51 +236,92 @@ pub async fn wait_for_claim_ready(
171236
mod tests {
172237
use super::*;
173238

174-
fn make_pool(image: &str, ready_replicas: i64) -> DynamicObject {
175-
let gvk = GroupVersionKind::gvk(WARM_POOL_GROUP, WARM_POOL_VERSION, WARM_POOL_KIND);
239+
fn make_template(image: &str) -> DynamicObject {
240+
let gvk = GroupVersionKind::gvk(WARM_POOL_GROUP, WARM_POOL_VERSION, TEMPLATE_KIND);
176241
let resource = ApiResource::from_gvk(&gvk);
177-
let mut obj = DynamicObject::new("test-pool", &resource);
242+
let mut obj = DynamicObject::new("test-template", &resource);
178243
obj.data = json!({
179244
"spec": {
180-
"template": {
181-
"containers": [
182-
{"image": image}
183-
]
245+
"podTemplate": {
246+
"spec": {
247+
"containers": [
248+
{"image": image, "name": "agent"}
249+
]
250+
}
184251
}
185-
},
186-
"status": {
187-
"readyReplicas": ready_replicas
188252
}
189253
});
190254
obj
191255
}
192256

193257
#[test]
194-
fn find_matching_pool_exact_image() {
195-
let pools = vec![
196-
make_pool("registry/sandbox:v1", 3),
197-
make_pool("registry/sandbox:v2", 1),
198-
];
199-
let found = find_matching_pool(&pools, "registry/sandbox:v2");
200-
assert!(found.is_some());
201-
assert_eq!(found.unwrap().metadata.name.as_deref(), Some("test-pool"));
258+
fn extract_template_image_finds_first_container() {
259+
let tmpl = make_template("ghcr.io/nvidia/sandbox:latest");
260+
assert_eq!(
261+
extract_template_image(&tmpl).as_deref(),
262+
Some("ghcr.io/nvidia/sandbox:latest")
263+
);
202264
}
203265

204266
#[test]
205-
fn find_matching_pool_no_match() {
206-
let pools = vec![make_pool("registry/sandbox:v1", 3)];
207-
assert!(find_matching_pool(&pools, "registry/other:v1").is_none());
267+
fn extract_template_image_returns_none_for_empty() {
268+
let gvk = GroupVersionKind::gvk(WARM_POOL_GROUP, WARM_POOL_VERSION, TEMPLATE_KIND);
269+
let resource = ApiResource::from_gvk(&gvk);
270+
let mut obj = DynamicObject::new("empty", &resource);
271+
obj.data = json!({"spec": {}});
272+
assert!(extract_template_image(&obj).is_none());
273+
}
274+
275+
fn make_pool_with_ready(ready_replicas: i64) -> DynamicObject {
276+
let gvk = GroupVersionKind::gvk(WARM_POOL_GROUP, WARM_POOL_VERSION, WARM_POOL_KIND);
277+
let resource = ApiResource::from_gvk(&gvk);
278+
let mut obj = DynamicObject::new("test-pool", &resource);
279+
obj.data = json!({
280+
"spec": {
281+
"sandboxTemplateRef": {"name": "test-template"},
282+
"replicas": 5,
283+
},
284+
"status": {
285+
"readyReplicas": ready_replicas,
286+
"replicas": 5,
287+
}
288+
});
289+
obj
208290
}
209291

210292
#[test]
211-
fn find_matching_pool_zero_replicas() {
212-
let pools = vec![make_pool("registry/sandbox:v1", 0)];
213-
assert!(find_matching_pool(&pools, "registry/sandbox:v1").is_none());
293+
fn pool_ready_replicas_check() {
294+
let pool = make_pool_with_ready(3);
295+
let ready = pool
296+
.data
297+
.get("status")
298+
.and_then(|s| s.get("readyReplicas"))
299+
.and_then(serde_json::Value::as_i64)
300+
.unwrap_or(0);
301+
assert_eq!(ready, 3);
302+
}
303+
304+
#[test]
305+
fn pool_zero_replicas_skipped() {
306+
let pool = make_pool_with_ready(0);
307+
let ready = pool
308+
.data
309+
.get("status")
310+
.and_then(|s| s.get("readyReplicas"))
311+
.and_then(serde_json::Value::as_i64)
312+
.unwrap_or(0);
313+
assert_eq!(ready, 0);
214314
}
215315

216316
#[test]
217-
fn find_matching_pool_empty_list() {
218-
let pools: Vec<DynamicObject> = vec![];
219-
assert!(find_matching_pool(&pools, "registry/sandbox:v1").is_none());
317+
fn pool_template_ref_extraction() {
318+
let pool = make_pool_with_ready(3);
319+
let tmpl_name = pool
320+
.data
321+
.get("spec")
322+
.and_then(|s| s.get("sandboxTemplateRef"))
323+
.and_then(|r| r.get("name"))
324+
.and_then(|n| n.as_str());
325+
assert_eq!(tmpl_name, Some("test-template"));
220326
}
221327
}

0 commit comments

Comments
 (0)