@@ -37,7 +37,9 @@ pub use crate::commands::provider_readiness::{ProviderWaitOptions, sandbox_provi
3737
3838use crate :: color:: Colorize ;
3939use crate :: policy_update:: build_policy_update_plan;
40- use crate :: tls:: { TlsOptions , grpc_client} ;
40+ use crate :: tls:: {
41+ TlsOptions , grpc_client, owner_replica, retry_unrouted, routed_to, with_owner_hint,
42+ } ;
4143use futures:: StreamExt ;
4244use miette:: { IntoDiagnostic , Result , WrapErr , miette} ;
4345use openshell_bootstrap:: {
@@ -71,6 +73,7 @@ use std::io::{ErrorKind, IsTerminal, Read, Write};
7173use std:: path:: { Path , PathBuf } ;
7274use std:: process:: Command ;
7375use std:: time:: { Duration , Instant } ;
76+ use tonic:: metadata:: AsciiMetadataValue ;
7477use tonic:: { Code , Status } ;
7578
7679const PROVISIONAL_CONTAINER_EXIT_RECONCILIATION_TIMEOUT : Duration = Duration :: from_secs ( 5 ) ;
@@ -1981,15 +1984,17 @@ pub async fn sandbox_exec_grpc(
19811984 let mut client = grpc_client ( server, tls) . await ?;
19821985
19831986 // Resolve sandbox name to id.
1984- let sandbox = client
1985- . get_sandbox ( GetSandboxRequest {
1987+ let response = client
1988+ . get_sandbox ( with_owner_hint ( GetSandboxRequest {
19861989 name : name. to_string ( ) ,
19871990 workspace_scope : Some ( openshell_core:: proto:: workspace_selector (
19881991 workspace. to_string ( ) ,
19891992 ) ) ,
1990- } )
1993+ } ) )
19911994 . await
1992- . into_diagnostic ( ) ?
1995+ . into_diagnostic ( ) ?;
1996+ let replica = owner_replica ( & response) ;
1997+ let sandbox = response
19931998 . into_inner ( )
19941999 . sandbox
19952000 . ok_or_else ( || miette:: miette!( "sandbox not found" ) ) ?;
@@ -2077,16 +2082,26 @@ pub async fn sandbox_exec_grpc(
20772082 stdin_is_terminal,
20782083 std:: mem:: take ( & mut request. stdin ) ,
20792084 stdin_rest,
2085+ replica. as_ref ( ) ,
20802086 )
20812087 . await ;
20822088 }
20832089
20842090 // Make the streaming gRPC call.
2085- let mut stream = client
2086- . exec_sandbox ( request)
2087- . await
2088- . into_diagnostic ( ) ?
2089- . into_inner ( ) ;
2091+ let retry = replica. is_some ( ) . then ( || request. clone ( ) ) ;
2092+ let mut stream = match (
2093+ client
2094+ . exec_sandbox ( routed_to ( request, replica. as_ref ( ) ) )
2095+ . await ,
2096+ retry,
2097+ ) {
2098+ ( Err ( status) , Some ( request) ) if retry_unrouted ( replica. as_ref ( ) , & status) => {
2099+ client. exec_sandbox ( request) . await
2100+ }
2101+ ( response, _) => response,
2102+ }
2103+ . into_diagnostic ( ) ?
2104+ . into_inner ( ) ;
20902105
20912106 // Stream output to terminal in real-time.
20922107 let mut exit_code = 0i32 ;
@@ -2139,7 +2154,7 @@ pub async fn service_forward_tcp(
21392154 let ( bind_addr, bind_port) = parse_tcp_forward_spec ( local, target_port) ?;
21402155 let mut client = grpc_client ( server, tls) . await ?;
21412156
2142- fetch_ready_sandbox_for_forward ( & mut client, name, workspace) . await ?;
2157+ let mut replica = fetch_ready_sandbox_for_forward ( & mut client, name, workspace) . await ?;
21432158
21442159 let listener = tokio:: net:: TcpListener :: bind ( ( bind_addr. as_str ( ) , bind_port) )
21452160 . await
@@ -2173,7 +2188,7 @@ pub async fn service_forward_tcp(
21732188 }
21742189
21752190 _ = health_check. tick( ) => {
2176- fetch_ready_sandbox_for_forward( & mut client, name, workspace) . await ?;
2191+ replica = fetch_ready_sandbox_for_forward( & mut client, name, workspace) . await ?;
21772192 }
21782193
21792194 accepted = listener. accept( ) => {
@@ -2188,6 +2203,7 @@ pub async fn service_forward_tcp(
21882203 let service_id = format!( "service-forward:{name}:{target_host}:{target_port}" ) ;
21892204 let fatal_tx = fatal_tx. clone( ) ;
21902205 let legacy_session_tokens = legacy_session_tokens. clone( ) ;
2206+ let replica = replica. clone( ) ;
21912207 tokio:: spawn( async move {
21922208 if let Err ( err) = forward_one_tcp_connection(
21932209 & mut client,
@@ -2198,6 +2214,7 @@ pub async fn service_forward_tcp(
21982214 target_port,
21992215 service_id,
22002216 legacy_session_tokens,
2217+ replica. as_ref( ) ,
22012218 )
22022219 . await
22032220 {
@@ -2237,18 +2254,19 @@ fn forward_requires_session_token(status: &Status) -> bool {
22372254 && status. message ( ) . contains ( "authorization_token is required" )
22382255}
22392256
2257+ /// Confirm the sandbox is still ready, returning the replica that owns it.
22402258async fn fetch_ready_sandbox_for_forward (
22412259 client : & mut crate :: tls:: GrpcClient ,
22422260 name : & str ,
22432261 workspace : & str ,
2244- ) -> Result < Sandbox > {
2262+ ) -> Result < Option < AsciiMetadataValue > > {
22452263 let response = match client
2246- . get_sandbox ( GetSandboxRequest {
2264+ . get_sandbox ( with_owner_hint ( GetSandboxRequest {
22472265 name : name. to_string ( ) ,
22482266 workspace_scope : Some ( openshell_core:: proto:: workspace_selector (
22492267 workspace. to_string ( ) ,
22502268 ) ) ,
2251- } )
2269+ } ) )
22522270 . await
22532271 {
22542272 Ok ( response) => response,
@@ -2260,6 +2278,7 @@ async fn fetch_ready_sandbox_for_forward(
22602278 Err ( status) => return Err ( status) . into_diagnostic ( ) ,
22612279 } ;
22622280
2281+ let replica = owner_replica ( & response) ;
22632282 let sandbox = response
22642283 . into_inner ( )
22652284 . sandbox
@@ -2273,7 +2292,7 @@ async fn fetch_ready_sandbox_for_forward(
22732292 ) ) ;
22742293 }
22752294
2276- Ok ( sandbox )
2295+ Ok ( replica )
22772296}
22782297
22792298#[ derive( Debug ) ]
@@ -2339,6 +2358,7 @@ async fn forward_one_tcp_connection(
23392358 target_port : u16 ,
23402359 service_id : String ,
23412360 legacy_session_tokens : std:: sync:: Arc < std:: sync:: atomic:: AtomicBool > ,
2361+ replica : Option < & AsciiMetadataValue > ,
23422362) -> std:: result:: Result < ( ) , ForwardTcpConnectionError > {
23432363 let mut init = TcpForwardInit {
23442364 sandbox : sandbox_name. clone ( ) ,
@@ -2369,7 +2389,7 @@ async fn forward_one_tcp_connection(
23692389 }
23702390 }
23712391
2372- let opened = match open_forward_tcp_stream ( client, init. clone ( ) ) . await {
2392+ let opened = match open_forward_tcp_stream ( client, init. clone ( ) , replica ) . await {
23732393 Ok ( opened) => opened,
23742394 Err ( status) if session_token. is_none ( ) && forward_requires_session_token ( & status) => {
23752395 tracing:: info!(
@@ -2387,7 +2407,7 @@ async fn forward_one_tcp_connection(
23872407 } ;
23882408 init. authorization_token . clone_from ( & token) ;
23892409 session_token = Some ( token) ;
2390- match open_forward_tcp_stream ( client, init) . await {
2410+ match open_forward_tcp_stream ( client, init, replica ) . await {
23912411 Ok ( opened) => opened,
23922412 Err ( status) => {
23932413 drain_and_shutdown_local_socket ( socket) . await ;
@@ -2418,22 +2438,15 @@ type OpenForwardTcpStream = (
24182438async fn open_forward_tcp_stream (
24192439 client : & mut crate :: tls:: GrpcClient ,
24202440 init : TcpForwardInit ,
2441+ replica : Option < & AsciiMetadataValue > ,
24212442) -> std:: result:: Result < OpenForwardTcpStream , Status > {
2422- use tokio_stream:: wrappers:: ReceiverStream ;
2423-
2424- let ( tx, rx) = tokio:: sync:: mpsc:: channel :: < TcpForwardFrame > ( 16 ) ;
2425- tx. send ( TcpForwardFrame {
2443+ let init = TcpForwardFrame {
24262444 payload : Some ( openshell_core:: proto:: tcp_forward_frame:: Payload :: Init (
24272445 init,
24282446 ) ) ,
2429- } )
2430- . await
2431- . map_err ( |_| Status :: internal ( "failed to initialize forward stream" ) ) ?;
2432- let response = client
2433- . forward_tcp ( ReceiverStream :: new ( rx) )
2434- . await ?
2435- . into_inner ( ) ;
2436- Ok ( ( tx, response) )
2447+ } ;
2448+ let ( tx, response) = crate :: ssh:: open_forward_tcp ( client, init, replica) . await ;
2449+ Ok ( ( tx, response?. into_inner ( ) ) )
24372450}
24382451
24392452async fn revoke_forward_session_token ( client : & mut crate :: tls:: GrpcClient , token : Option < String > ) {
@@ -2561,6 +2574,7 @@ async fn sandbox_exec_streaming_grpc(
25612574 stdin_is_terminal : bool ,
25622575 stdin_prefix : Vec < u8 > ,
25632576 stdin_rest : Option < tokio:: sync:: mpsc:: Receiver < std:: io:: Result < Vec < u8 > > > > ,
2577+ replica : Option < & AsciiMetadataValue > ,
25642578) -> Result < i32 > {
25652579 #[ cfg( unix) ]
25662580 use openshell_core:: proto:: ExecSandboxWindowResize ;
@@ -2573,36 +2587,44 @@ async fn sandbox_exec_streaming_grpc(
25732587 ( 0 , 0 )
25742588 } ;
25752589
2576- let ( input_tx, input_rx) = tokio:: sync:: mpsc:: channel :: < ExecSandboxInput > ( 64 ) ;
2577-
2578- // Send the start message with exec metadata.
2579- input_tx
2580- . send ( ExecSandboxInput {
2581- payload : Some ( exec_sandbox_input:: Payload :: Start ( ExecSandboxRequest {
2582- request_id : String :: new ( ) ,
2583- sandbox : sandbox. object_name ( ) . to_string ( ) ,
2584- workspace_scope : Some ( openshell_core:: proto:: workspace_selector (
2585- ( sandbox. object_workspace ( ) ) . to_string ( ) ,
2586- ) ) ,
2587- command : command. to_vec ( ) ,
2588- workdir : workdir. unwrap_or_default ( ) . to_string ( ) ,
2589- environment : environment. clone ( ) ,
2590- no_login_shell,
2591- execution_timeout : proto_execution_timeout ( timeout_seconds) ?,
2592- stdin : Vec :: new ( ) ,
2593- tty,
2594- cols,
2595- rows,
2596- } ) ) ,
2597- } )
2598- . await
2599- . into_diagnostic ( ) ?;
2590+ // The start message carries the exec metadata.
2591+ let start = ExecSandboxInput {
2592+ payload : Some ( exec_sandbox_input:: Payload :: Start ( ExecSandboxRequest {
2593+ request_id : String :: new ( ) ,
2594+ sandbox : sandbox. object_name ( ) . to_string ( ) ,
2595+ workspace_scope : Some ( openshell_core:: proto:: workspace_selector (
2596+ ( sandbox. object_workspace ( ) ) . to_string ( ) ,
2597+ ) ) ,
2598+ command : command. to_vec ( ) ,
2599+ workdir : workdir. unwrap_or_default ( ) . to_string ( ) ,
2600+ environment : environment. clone ( ) ,
2601+ no_login_shell,
2602+ execution_timeout : proto_execution_timeout ( timeout_seconds) ?,
2603+ stdin : Vec :: new ( ) ,
2604+ tty,
2605+ cols,
2606+ rows,
2607+ } ) ) ,
2608+ } ;
26002609
2601- let mut stream = client
2602- . exec_sandbox_interactive ( ReceiverStream :: new ( input_rx) )
2610+ let ( mut input_tx, input_rx) = tokio:: sync:: mpsc:: channel :: < ExecSandboxInput > ( 64 ) ;
2611+ input_tx. send ( start. clone ( ) ) . await . into_diagnostic ( ) ?;
2612+ let mut stream = match client
2613+ . exec_sandbox_interactive ( routed_to ( ReceiverStream :: new ( input_rx) , replica) )
26032614 . await
2604- . into_diagnostic ( ) ?
2605- . into_inner ( ) ;
2615+ {
2616+ Err ( status) if retry_unrouted ( replica, & status) => {
2617+ let ( tx, rx) = tokio:: sync:: mpsc:: channel :: < ExecSandboxInput > ( 64 ) ;
2618+ tx. send ( start) . await . into_diagnostic ( ) ?;
2619+ input_tx = tx;
2620+ client
2621+ . exec_sandbox_interactive ( ReceiverStream :: new ( rx) )
2622+ . await
2623+ }
2624+ response => response,
2625+ }
2626+ . into_diagnostic ( ) ?
2627+ . into_inner ( ) ;
26062628
26072629 // Raw mode is only appropriate for an interactive terminal, not a pipe.
26082630 let raw_guard = if tty && stdin_is_terminal {
0 commit comments