@@ -34,8 +34,9 @@ use openshell_core::proto_struct::{struct_to_json_object, value_to_json};
3434use serde:: Deserialize ;
3535use std:: collections:: BTreeMap ;
3636use std:: pin:: Pin ;
37+ use std:: sync:: Arc ;
3738use std:: time:: Duration ;
38- use tokio:: sync:: mpsc;
39+ use tokio:: sync:: { OnceCell , mpsc} ;
3940use tokio_stream:: wrappers:: ReceiverStream ;
4041use tracing:: { debug, info, warn} ;
4142
@@ -80,12 +81,19 @@ impl From<KubernetesDriverError> for openshell_core::ComputeDriverError {
8081const KUBE_API_TIMEOUT : Duration = Duration :: from_secs ( 30 ) ;
8182
8283const SANDBOX_GROUP : & str = "agents.x-k8s.io" ;
83- const SANDBOX_VERSION : & str = "v1alpha1" ;
84+ const SANDBOX_VERSION_V1BETA1 : & str = "v1beta1" ;
85+ const SANDBOX_VERSION_V1ALPHA1 : & str = "v1alpha1" ;
86+ const SANDBOX_VERSIONS : & [ & str ] = & [ SANDBOX_VERSION_V1BETA1 , SANDBOX_VERSION_V1ALPHA1 ] ;
8487pub const SANDBOX_KIND : & str = "Sandbox" ;
8588
8689const GPU_RESOURCE_NAME : & str = "nvidia.com/gpu" ;
8790const SPIFFE_WORKLOAD_API_VOLUME_NAME : & str = "spiffe-workload-api" ;
8891
92+ struct AgentSandboxApi {
93+ api : Api < DynamicObject > ,
94+ resource : ApiResource ,
95+ }
96+
8997// This POC treats the selected Struct as a driver-local typed schema. Once the
9098// Kubernetes shape stabilizes, these serde structs may move to driver-local
9199// protobuf definitions, but the typed decode should stay inside this driver.
@@ -190,6 +198,7 @@ const WORKSPACE_SENTINEL: &str = ".workspace-initialized";
190198pub struct KubernetesComputeDriver {
191199 client : Client ,
192200 watch_client : Client ,
201+ sandbox_api_version : Arc < OnceCell < & ' static str > > ,
193202 config : KubernetesComputeConfig ,
194203}
195204
@@ -232,6 +241,7 @@ impl KubernetesComputeDriver {
232241 Ok ( Self {
233242 client,
234243 watch_client,
244+ sandbox_api_version : Arc :: new ( OnceCell :: new ( ) ) ,
235245 config,
236246 } )
237247 }
@@ -256,16 +266,68 @@ impl KubernetesComputeDriver {
256266 & self . config . ssh_socket_path
257267 }
258268
259- fn watch_api ( & self ) -> Api < DynamicObject > {
260- let gvk = GroupVersionKind :: gvk ( SANDBOX_GROUP , SANDBOX_VERSION , SANDBOX_KIND ) ;
269+ fn agent_sandbox_api ( & self , client : Client , sandbox_api_version : & str ) -> AgentSandboxApi {
270+ let gvk = GroupVersionKind :: gvk ( SANDBOX_GROUP , sandbox_api_version , SANDBOX_KIND ) ;
261271 let resource = ApiResource :: from_gvk ( & gvk) ;
262- Api :: namespaced_with ( self . watch_client . clone ( ) , & self . config . namespace , & resource)
272+ let api = Api :: namespaced_with ( client, & self . config . namespace , & resource) ;
273+ AgentSandboxApi { api, resource }
263274 }
264275
265- fn api ( & self ) -> Api < DynamicObject > {
266- let gvk = GroupVersionKind :: gvk ( SANDBOX_GROUP , SANDBOX_VERSION , SANDBOX_KIND ) ;
267- let resource = ApiResource :: from_gvk ( & gvk) ;
268- Api :: namespaced_with ( self . client . clone ( ) , & self . config . namespace , & resource)
276+ async fn supported_agent_sandbox_api ( & self , client : Client ) -> Result < AgentSandboxApi , String > {
277+ let sandbox_api_version = self . supported_sandbox_api_version ( client. clone ( ) ) . await ?;
278+ Ok ( self . agent_sandbox_api ( client, sandbox_api_version) )
279+ }
280+
281+ async fn supported_sandbox_api_version ( & self , client : Client ) -> Result < & ' static str , String > {
282+ self . sandbox_api_version
283+ . get_or_try_init (
284+ || async move { self . detect_supported_sandbox_api_version ( client) . await } ,
285+ )
286+ . await
287+ . copied ( )
288+ }
289+
290+ async fn detect_supported_sandbox_api_version (
291+ & self ,
292+ client : Client ,
293+ ) -> Result < & ' static str , String > {
294+ for sandbox_api_version in SANDBOX_VERSIONS {
295+ let agent_sandbox_api = self . agent_sandbox_api ( client. clone ( ) , sandbox_api_version) ;
296+ match tokio:: time:: timeout (
297+ KUBE_API_TIMEOUT ,
298+ agent_sandbox_api. api . list ( & ListParams :: default ( ) . limit ( 1 ) ) ,
299+ )
300+ . await
301+ {
302+ Ok ( Ok ( _) ) => {
303+ debug ! (
304+ namespace = %self . config. namespace,
305+ sandbox_api_version = %sandbox_api_version,
306+ "Selected Agent Sandbox API version"
307+ ) ;
308+ return Ok ( sandbox_api_version) ;
309+ }
310+ Ok ( Err ( err) ) if should_try_next_sandbox_api_version ( & err) => {
311+ debug ! (
312+ namespace = %self . config. namespace,
313+ sandbox_api_version = %sandbox_api_version,
314+ error = %err,
315+ "Sandbox API version is not available; trying next supported version"
316+ ) ;
317+ }
318+ Ok ( Err ( err) ) => return Err ( err. to_string ( ) ) ,
319+ Err ( _elapsed) => {
320+ return Err ( format ! (
321+ "timed out after {}s waiting for Kubernetes API" ,
322+ KUBE_API_TIMEOUT . as_secs( )
323+ ) ) ;
324+ }
325+ }
326+ }
327+ Err ( format ! (
328+ "no supported Agent Sandbox API version is available; tried {}" ,
329+ SANDBOX_VERSIONS . join( ", " )
330+ ) )
269331 }
270332
271333 async fn has_gpu_capacity ( & self ) -> Result < bool , KubeError > {
@@ -306,8 +368,10 @@ impl KubernetesComputeDriver {
306368 "Fetching sandbox from Kubernetes"
307369 ) ;
308370
309- let api = self . api ( ) ;
310- match tokio:: time:: timeout ( KUBE_API_TIMEOUT , api. get ( name) ) . await {
371+ let agent_sandbox_api = self
372+ . supported_agent_sandbox_api ( self . client . clone ( ) )
373+ . await ?;
374+ match tokio:: time:: timeout ( KUBE_API_TIMEOUT , agent_sandbox_api. api . get ( name) ) . await {
311375 Ok ( Ok ( obj) ) => sandbox_from_object ( & self . config . namespace , obj) . map ( Some ) ,
312376 Ok ( Err ( KubeError :: Api ( err) ) ) if err. code == 404 => {
313377 debug ! ( sandbox_name = %name, "Sandbox not found in Kubernetes" ) ;
@@ -341,8 +405,15 @@ impl KubernetesComputeDriver {
341405 "Listing sandboxes from Kubernetes"
342406 ) ;
343407
344- let api = self . api ( ) ;
345- match tokio:: time:: timeout ( KUBE_API_TIMEOUT , api. list ( & ListParams :: default ( ) ) ) . await {
408+ let agent_sandbox_api = self
409+ . supported_agent_sandbox_api ( self . client . clone ( ) )
410+ . await ?;
411+ match tokio:: time:: timeout (
412+ KUBE_API_TIMEOUT ,
413+ agent_sandbox_api. api . list ( & ListParams :: default ( ) ) ,
414+ )
415+ . await
416+ {
346417 Ok ( Ok ( list) ) => {
347418 let mut sandboxes = list
348419 . items
@@ -396,9 +467,11 @@ impl KubernetesComputeDriver {
396467 "Creating sandbox in Kubernetes"
397468 ) ;
398469
399- let gvk = GroupVersionKind :: gvk ( SANDBOX_GROUP , SANDBOX_VERSION , SANDBOX_KIND ) ;
400- let resource = ApiResource :: from_gvk ( & gvk) ;
401- let mut obj = DynamicObject :: new ( name, & resource) ;
470+ let agent_sandbox_api = self
471+ . supported_agent_sandbox_api ( self . client . clone ( ) )
472+ . await
473+ . map_err ( KubernetesDriverError :: Message ) ?;
474+ let mut obj = DynamicObject :: new ( name, & agent_sandbox_api. resource ) ;
402475 obj. metadata = ObjectMeta {
403476 name : Some ( name. to_string ( ) ) ,
404477 namespace : Some ( self . config . namespace . clone ( ) ) ,
@@ -430,9 +503,11 @@ impl KubernetesComputeDriver {
430503 . provider_spiffe_workload_api_socket_path ,
431504 } ;
432505 obj. data = sandbox_to_k8s_spec ( sandbox. spec . as_ref ( ) , & params) ;
433- let api = self . api ( ) ;
434-
435- match tokio:: time:: timeout ( KUBE_API_TIMEOUT , api. create ( & PostParams :: default ( ) , & obj) ) . await
506+ match tokio:: time:: timeout (
507+ KUBE_API_TIMEOUT ,
508+ agent_sandbox_api. api . create ( & PostParams :: default ( ) , & obj) ,
509+ )
510+ . await
436511 {
437512 Ok ( Ok ( _result) ) => {
438513 info ! (
@@ -473,9 +548,14 @@ impl KubernetesComputeDriver {
473548 "Deleting sandbox from Kubernetes"
474549 ) ;
475550
476- let api = self . api ( ) ;
477- match tokio:: time:: timeout ( KUBE_API_TIMEOUT , api. delete ( name, & DeleteParams :: default ( ) ) )
478- . await
551+ let agent_sandbox_api = self
552+ . supported_agent_sandbox_api ( self . client . clone ( ) )
553+ . await ?;
554+ match tokio:: time:: timeout (
555+ KUBE_API_TIMEOUT ,
556+ agent_sandbox_api. api . delete ( name, & DeleteParams :: default ( ) ) ,
557+ )
558+ . await
479559 {
480560 Ok ( Ok ( _response) ) => {
481561 info ! ( sandbox_name = %name, "Sandbox deleted from Kubernetes" ) ;
@@ -508,8 +588,10 @@ impl KubernetesComputeDriver {
508588 }
509589
510590 pub async fn sandbox_exists ( & self , name : & str ) -> Result < bool , String > {
511- let api = self . api ( ) ;
512- match tokio:: time:: timeout ( KUBE_API_TIMEOUT , api. get ( name) ) . await {
591+ let agent_sandbox_api = self
592+ . supported_agent_sandbox_api ( self . client . clone ( ) )
593+ . await ?;
594+ match tokio:: time:: timeout ( KUBE_API_TIMEOUT , agent_sandbox_api. api . get ( name) ) . await {
513595 Ok ( Ok ( _) ) => Ok ( true ) ,
514596 Ok ( Err ( KubeError :: Api ( err) ) ) if err. code == 404 => Ok ( false ) ,
515597 Ok ( Err ( err) ) => Err ( err. to_string ( ) ) ,
@@ -524,9 +606,12 @@ impl KubernetesComputeDriver {
524606 #[ allow( clippy:: unused_async) ]
525607 pub async fn watch_sandboxes ( & self ) -> Result < WatchStream , String > {
526608 let namespace = self . config . namespace . clone ( ) ;
527- let sandbox_api = self . watch_api ( ) ;
609+ let agent_sandbox_api = self
610+ . supported_agent_sandbox_api ( self . watch_client . clone ( ) )
611+ . await ?;
528612 let event_api: Api < KubeEventObj > = Api :: namespaced ( self . watch_client . clone ( ) , & namespace) ;
529- let mut sandbox_stream = watcher:: watcher ( sandbox_api, watcher:: Config :: default ( ) ) . boxed ( ) ;
613+ let mut sandbox_stream =
614+ watcher:: watcher ( agent_sandbox_api. api , watcher:: Config :: default ( ) ) . boxed ( ) ;
530615 let mut event_stream = watcher:: watcher ( event_api, watcher:: Config :: default ( ) ) . boxed ( ) ;
531616 let ( tx, rx) = mpsc:: channel ( 256 ) ;
532617
@@ -650,6 +735,14 @@ impl KubernetesComputeDriver {
650735 }
651736}
652737
738+ fn should_try_next_sandbox_api_version ( err : & KubeError ) -> bool {
739+ // Kubernetes returns a structured 404 for some missing API resources and a
740+ // raw "404 page not found" body for others. Both mean the probed
741+ // group/version is unavailable and the next supported Sandbox API version
742+ // should be tried.
743+ matches ! ( err, KubeError :: Api ( api) if api. code == 404 )
744+ }
745+
653746fn validate_gpu_request (
654747 gpu_requirements : Option < & GpuResourceRequirements > ,
655748) -> Result < ( ) , tonic:: Status > {
@@ -2013,6 +2106,34 @@ mod tests {
20132106 }
20142107 }
20152108
2109+ fn kube_api_error ( code : u16 , message : & str ) -> KubeError {
2110+ KubeError :: Api ( kube:: core:: ErrorResponse {
2111+ status : if code == 404 {
2112+ "404 Not Found" . to_string ( )
2113+ } else {
2114+ "Failure" . to_string ( )
2115+ } ,
2116+ message : message. to_string ( ) ,
2117+ reason : "Failed to parse error data" . to_string ( ) ,
2118+ code,
2119+ } )
2120+ }
2121+
2122+ #[ test]
2123+ fn sandbox_api_version_probe_retries_on_structured_and_raw_404 ( ) {
2124+ let structured = kube_api_error ( 404 , "could not find the requested resource" ) ;
2125+ assert ! ( should_try_next_sandbox_api_version( & structured) ) ;
2126+
2127+ let raw = kube_api_error ( 404 , "404 page not found\n " ) ;
2128+ assert ! ( should_try_next_sandbox_api_version( & raw) ) ;
2129+ }
2130+
2131+ #[ test]
2132+ fn sandbox_api_version_probe_keeps_non_404_errors ( ) {
2133+ let err = kube_api_error ( 403 , "sandboxes.agents.x-k8s.io is forbidden" ) ;
2134+ assert ! ( !should_try_next_sandbox_api_version( & err) ) ;
2135+ }
2136+
20162137 #[ test]
20172138 fn driver_config_rejects_invalid_shape ( ) {
20182139 let template = SandboxTemplate {
0 commit comments