Skip to content

Commit 834b79a

Browse files
authored
feat(server): place supervisor sessions by consistent hash (#3661)
* feat(server): place supervisor sessions by consistent hash and hand them off on shutdown Signed-off-by: divesh <[email protected]> * feat: route long-lived sandbox connections to the owning gateway replica Signed-off-by: divesh <[email protected]> * feat(helm): add opt-in per-replica GRPCRoute routing Signed-off-by: divesh <[email protected]> --------- Signed-off-by: divesh <[email protected]>
1 parent 9a6148f commit 834b79a

27 files changed

Lines changed: 2413 additions & 811 deletions

File tree

‎crates/openshell-cli/src/run.rs‎

Lines changed: 80 additions & 58 deletions
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,9 @@ pub use crate::commands::provider_readiness::{ProviderWaitOptions, sandbox_provi
3737

3838
use crate::color::Colorize;
3939
use crate::policy_update::build_policy_update_plan;
40-
use crate::tls::{TlsOptions, grpc_client};
40+
use crate::tls::{
41+
TlsOptions, grpc_client, owner_replica, retry_unrouted, routed_to, with_owner_hint,
42+
};
4143
use futures::StreamExt;
4244
use miette::{IntoDiagnostic, Result, WrapErr, miette};
4345
use openshell_bootstrap::{
@@ -71,6 +73,7 @@ use std::io::{ErrorKind, IsTerminal, Read, Write};
7173
use std::path::{Path, PathBuf};
7274
use std::process::Command;
7375
use std::time::{Duration, Instant};
76+
use tonic::metadata::AsciiMetadataValue;
7477
use tonic::{Code, Status};
7578

7679
const PROVISIONAL_CONTAINER_EXIT_RECONCILIATION_TIMEOUT: Duration = Duration::from_secs(5);
@@ -1981,15 +1984,17 @@ pub async fn sandbox_exec_grpc(
19811984
let mut client = grpc_client(server, tls).await?;
19821985

19831986
// Resolve sandbox name to id.
1984-
let sandbox = client
1985-
.get_sandbox(GetSandboxRequest {
1987+
let response = client
1988+
.get_sandbox(with_owner_hint(GetSandboxRequest {
19861989
name: name.to_string(),
19871990
workspace_scope: Some(openshell_core::proto::workspace_selector(
19881991
workspace.to_string(),
19891992
)),
1990-
})
1993+
}))
19911994
.await
1992-
.into_diagnostic()?
1995+
.into_diagnostic()?;
1996+
let replica = owner_replica(&response);
1997+
let sandbox = response
19931998
.into_inner()
19941999
.sandbox
19952000
.ok_or_else(|| miette::miette!("sandbox not found"))?;
@@ -2077,16 +2082,26 @@ pub async fn sandbox_exec_grpc(
20772082
stdin_is_terminal,
20782083
std::mem::take(&mut request.stdin),
20792084
stdin_rest,
2085+
replica.as_ref(),
20802086
)
20812087
.await;
20822088
}
20832089

20842090
// Make the streaming gRPC call.
2085-
let mut stream = client
2086-
.exec_sandbox(request)
2087-
.await
2088-
.into_diagnostic()?
2089-
.into_inner();
2091+
let retry = replica.is_some().then(|| request.clone());
2092+
let mut stream = match (
2093+
client
2094+
.exec_sandbox(routed_to(request, replica.as_ref()))
2095+
.await,
2096+
retry,
2097+
) {
2098+
(Err(status), Some(request)) if retry_unrouted(replica.as_ref(), &status) => {
2099+
client.exec_sandbox(request).await
2100+
}
2101+
(response, _) => response,
2102+
}
2103+
.into_diagnostic()?
2104+
.into_inner();
20902105

20912106
// Stream output to terminal in real-time.
20922107
let mut exit_code = 0i32;
@@ -2139,7 +2154,7 @@ pub async fn service_forward_tcp(
21392154
let (bind_addr, bind_port) = parse_tcp_forward_spec(local, target_port)?;
21402155
let mut client = grpc_client(server, tls).await?;
21412156

2142-
fetch_ready_sandbox_for_forward(&mut client, name, workspace).await?;
2157+
let mut replica = fetch_ready_sandbox_for_forward(&mut client, name, workspace).await?;
21432158

21442159
let listener = tokio::net::TcpListener::bind((bind_addr.as_str(), bind_port))
21452160
.await
@@ -2173,7 +2188,7 @@ pub async fn service_forward_tcp(
21732188
}
21742189

21752190
_ = health_check.tick() => {
2176-
fetch_ready_sandbox_for_forward(&mut client, name, workspace).await?;
2191+
replica = fetch_ready_sandbox_for_forward(&mut client, name, workspace).await?;
21772192
}
21782193

21792194
accepted = listener.accept() => {
@@ -2188,6 +2203,7 @@ pub async fn service_forward_tcp(
21882203
let service_id = format!("service-forward:{name}:{target_host}:{target_port}");
21892204
let fatal_tx = fatal_tx.clone();
21902205
let legacy_session_tokens = legacy_session_tokens.clone();
2206+
let replica = replica.clone();
21912207
tokio::spawn(async move {
21922208
if let Err(err) = forward_one_tcp_connection(
21932209
&mut client,
@@ -2198,6 +2214,7 @@ pub async fn service_forward_tcp(
21982214
target_port,
21992215
service_id,
22002216
legacy_session_tokens,
2217+
replica.as_ref(),
22012218
)
22022219
.await
22032220
{
@@ -2237,18 +2254,19 @@ fn forward_requires_session_token(status: &Status) -> bool {
22372254
&& status.message().contains("authorization_token is required")
22382255
}
22392256

2257+
/// Confirm the sandbox is still ready, returning the replica that owns it.
22402258
async fn fetch_ready_sandbox_for_forward(
22412259
client: &mut crate::tls::GrpcClient,
22422260
name: &str,
22432261
workspace: &str,
2244-
) -> Result<Sandbox> {
2262+
) -> Result<Option<AsciiMetadataValue>> {
22452263
let response = match client
2246-
.get_sandbox(GetSandboxRequest {
2264+
.get_sandbox(with_owner_hint(GetSandboxRequest {
22472265
name: name.to_string(),
22482266
workspace_scope: Some(openshell_core::proto::workspace_selector(
22492267
workspace.to_string(),
22502268
)),
2251-
})
2269+
}))
22522270
.await
22532271
{
22542272
Ok(response) => response,
@@ -2260,6 +2278,7 @@ async fn fetch_ready_sandbox_for_forward(
22602278
Err(status) => return Err(status).into_diagnostic(),
22612279
};
22622280

2281+
let replica = owner_replica(&response);
22632282
let sandbox = response
22642283
.into_inner()
22652284
.sandbox
@@ -2273,7 +2292,7 @@ async fn fetch_ready_sandbox_for_forward(
22732292
));
22742293
}
22752294

2276-
Ok(sandbox)
2295+
Ok(replica)
22772296
}
22782297

22792298
#[derive(Debug)]
@@ -2339,6 +2358,7 @@ async fn forward_one_tcp_connection(
23392358
target_port: u16,
23402359
service_id: String,
23412360
legacy_session_tokens: std::sync::Arc<std::sync::atomic::AtomicBool>,
2361+
replica: Option<&AsciiMetadataValue>,
23422362
) -> std::result::Result<(), ForwardTcpConnectionError> {
23432363
let mut init = TcpForwardInit {
23442364
sandbox: sandbox_name.clone(),
@@ -2369,7 +2389,7 @@ async fn forward_one_tcp_connection(
23692389
}
23702390
}
23712391

2372-
let opened = match open_forward_tcp_stream(client, init.clone()).await {
2392+
let opened = match open_forward_tcp_stream(client, init.clone(), replica).await {
23732393
Ok(opened) => opened,
23742394
Err(status) if session_token.is_none() && forward_requires_session_token(&status) => {
23752395
tracing::info!(
@@ -2387,7 +2407,7 @@ async fn forward_one_tcp_connection(
23872407
};
23882408
init.authorization_token.clone_from(&token);
23892409
session_token = Some(token);
2390-
match open_forward_tcp_stream(client, init).await {
2410+
match open_forward_tcp_stream(client, init, replica).await {
23912411
Ok(opened) => opened,
23922412
Err(status) => {
23932413
drain_and_shutdown_local_socket(socket).await;
@@ -2418,22 +2438,15 @@ type OpenForwardTcpStream = (
24182438
async fn open_forward_tcp_stream(
24192439
client: &mut crate::tls::GrpcClient,
24202440
init: TcpForwardInit,
2441+
replica: Option<&AsciiMetadataValue>,
24212442
) -> std::result::Result<OpenForwardTcpStream, Status> {
2422-
use tokio_stream::wrappers::ReceiverStream;
2423-
2424-
let (tx, rx) = tokio::sync::mpsc::channel::<TcpForwardFrame>(16);
2425-
tx.send(TcpForwardFrame {
2443+
let init = TcpForwardFrame {
24262444
payload: Some(openshell_core::proto::tcp_forward_frame::Payload::Init(
24272445
init,
24282446
)),
2429-
})
2430-
.await
2431-
.map_err(|_| Status::internal("failed to initialize forward stream"))?;
2432-
let response = client
2433-
.forward_tcp(ReceiverStream::new(rx))
2434-
.await?
2435-
.into_inner();
2436-
Ok((tx, response))
2447+
};
2448+
let (tx, response) = crate::ssh::open_forward_tcp(client, init, replica).await;
2449+
Ok((tx, response?.into_inner()))
24372450
}
24382451

24392452
async fn revoke_forward_session_token(client: &mut crate::tls::GrpcClient, token: Option<String>) {
@@ -2561,6 +2574,7 @@ async fn sandbox_exec_streaming_grpc(
25612574
stdin_is_terminal: bool,
25622575
stdin_prefix: Vec<u8>,
25632576
stdin_rest: Option<tokio::sync::mpsc::Receiver<std::io::Result<Vec<u8>>>>,
2577+
replica: Option<&AsciiMetadataValue>,
25642578
) -> Result<i32> {
25652579
#[cfg(unix)]
25662580
use openshell_core::proto::ExecSandboxWindowResize;
@@ -2573,36 +2587,44 @@ async fn sandbox_exec_streaming_grpc(
25732587
(0, 0)
25742588
};
25752589

2576-
let (input_tx, input_rx) = tokio::sync::mpsc::channel::<ExecSandboxInput>(64);
2577-
2578-
// Send the start message with exec metadata.
2579-
input_tx
2580-
.send(ExecSandboxInput {
2581-
payload: Some(exec_sandbox_input::Payload::Start(ExecSandboxRequest {
2582-
request_id: String::new(),
2583-
sandbox: sandbox.object_name().to_string(),
2584-
workspace_scope: Some(openshell_core::proto::workspace_selector(
2585-
(sandbox.object_workspace()).to_string(),
2586-
)),
2587-
command: command.to_vec(),
2588-
workdir: workdir.unwrap_or_default().to_string(),
2589-
environment: environment.clone(),
2590-
no_login_shell,
2591-
execution_timeout: proto_execution_timeout(timeout_seconds)?,
2592-
stdin: Vec::new(),
2593-
tty,
2594-
cols,
2595-
rows,
2596-
})),
2597-
})
2598-
.await
2599-
.into_diagnostic()?;
2590+
// The start message carries the exec metadata.
2591+
let start = ExecSandboxInput {
2592+
payload: Some(exec_sandbox_input::Payload::Start(ExecSandboxRequest {
2593+
request_id: String::new(),
2594+
sandbox: sandbox.object_name().to_string(),
2595+
workspace_scope: Some(openshell_core::proto::workspace_selector(
2596+
(sandbox.object_workspace()).to_string(),
2597+
)),
2598+
command: command.to_vec(),
2599+
workdir: workdir.unwrap_or_default().to_string(),
2600+
environment: environment.clone(),
2601+
no_login_shell,
2602+
execution_timeout: proto_execution_timeout(timeout_seconds)?,
2603+
stdin: Vec::new(),
2604+
tty,
2605+
cols,
2606+
rows,
2607+
})),
2608+
};
26002609

2601-
let mut stream = client
2602-
.exec_sandbox_interactive(ReceiverStream::new(input_rx))
2610+
let (mut input_tx, input_rx) = tokio::sync::mpsc::channel::<ExecSandboxInput>(64);
2611+
input_tx.send(start.clone()).await.into_diagnostic()?;
2612+
let mut stream = match client
2613+
.exec_sandbox_interactive(routed_to(ReceiverStream::new(input_rx), replica))
26032614
.await
2604-
.into_diagnostic()?
2605-
.into_inner();
2615+
{
2616+
Err(status) if retry_unrouted(replica, &status) => {
2617+
let (tx, rx) = tokio::sync::mpsc::channel::<ExecSandboxInput>(64);
2618+
tx.send(start).await.into_diagnostic()?;
2619+
input_tx = tx;
2620+
client
2621+
.exec_sandbox_interactive(ReceiverStream::new(rx))
2622+
.await
2623+
}
2624+
response => response,
2625+
}
2626+
.into_diagnostic()?
2627+
.into_inner();
26062628

26072629
// Raw mode is only appropriate for an interactive terminal, not a pipe.
26082630
let raw_guard = if tty && stdin_is_terminal {

0 commit comments

Comments
 (0)