Skip to content

Commit 9294765

Browse files
n1hilityjohntmyers
authored andcommitted
perf(server): drop per-connection session tokens for TCP forwards
`openshell forward service` minted an SSH session token before every forwarded TCP connection and revoked it afterwards: two store commits per connection. The token added nothing on that path. `ForwardTcp` already authenticates the caller and authorizes it against the sandbox's workspace on every stream before it looks at the token, the relay to the supervisor is opened with the sandbox id and target only, and the token is never forwarded, audited, or visible to the target service. The mechanism exists for `openshell sandbox ssh`, where the process that opens the stream is an ssh ProxyCommand holding nothing but the token. Reusing it per TCP connection put a store write on the connect path and serialized concurrent forwards on commit latency: #3494 measured the symptom, and #3543 made the commits cheaper, but each one still holds SQLite's writer lock for an fsync, so connection setup under a burst stayed linear in the number of concurrent connections. Let `target.tcp` streams omit `authorization_token`. The gateway admits them on the already-authorized principal, counts them against the same per-sandbox connection cap, and touches no store. `target.ssh` streams keep requiring the token. A token supplied with a TCP target is still validated and counted per token, so an older CLI against a new gateway is unchanged. The CLI stops minting and revoking a session per forwarded connection; against a gateway that predates this change it recognizes the `authorization_token is required` rejection once and falls back to per-connection tokens for the rest of that forward. Tests cover token-less TCP admission and slot release, SSH targets still rejected without a token, a supplied token still validated, and the per-sandbox cap for token-less forwards. CLI integration tests run `service_forward_tcp` against a mock gateway: token-less inits echo data with no CreateSshSession or RevokeSshSession call, and a gateway that rejects the empty token is detected once, after which every connection in that forward mints and revokes its own token. Architecture and security docs describe which targets carry a token, and the per-token connection limit now reads 3, matching the gateway. Signed-off-by: Jason T. Greene <[email protected]>
1 parent 4188eab commit 9294765

4 files changed

Lines changed: 536 additions & 70 deletions

File tree

‎crates/openshell-cli/src/run.rs‎

Lines changed: 132 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -2120,6 +2120,9 @@ pub async fn service_forward_tcp(
21202120
let sandbox_name = name.to_string();
21212121
let sandbox_workspace = workspace.to_string();
21222122
let (fatal_tx, mut fatal_rx) = tokio::sync::mpsc::channel::<String>(1);
2123+
// Set once this forward learns that the gateway predates principal-authorized
2124+
// TCP forwards and still requires a `CreateSshSession` token per connection.
2125+
let legacy_session_tokens = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false));
21232126
let mut health_check = tokio::time::interval(Duration::from_secs(2));
21242127
health_check.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
21252128
loop {
@@ -2143,21 +2146,8 @@ pub async fn service_forward_tcp(
21432146
let target_host = target_host.to_string();
21442147
let service_id = format!("service-forward:{name}:{target_host}:{target_port}");
21452148
let fatal_tx = fatal_tx.clone();
2149+
let legacy_session_tokens = legacy_session_tokens.clone();
21462150
tokio::spawn(async move {
2147-
let token = match create_forward_session_token(
2148-
&mut client,
2149-
&sandbox_name,
2150-
&sandbox_workspace,
2151-
).await {
2152-
Ok(token) => token,
2153-
Err(err) => {
2154-
tracing::warn!(peer = %peer, error = %err, "service forward session creation failed");
2155-
if err.fatal {
2156-
let _ = fatal_tx.send(err.message).await;
2157-
}
2158-
return;
2159-
}
2160-
};
21612151
if let Err(err) = forward_one_tcp_connection(
21622152
&mut client,
21632153
socket,
@@ -2166,7 +2156,7 @@ pub async fn service_forward_tcp(
21662156
target_host,
21672157
target_port,
21682158
service_id,
2169-
token.clone(),
2159+
legacy_session_tokens,
21702160
)
21712161
.await
21722162
{
@@ -2175,9 +2165,6 @@ pub async fn service_forward_tcp(
21752165
let _ = fatal_tx.send(err.message).await;
21762166
}
21772167
}
2178-
let _ = client
2179-
.revoke_ssh_session(RevokeSshSessionRequest { allow_missing: true, token })
2180-
.await;
21812168
});
21822169
}
21832170
}
@@ -2201,6 +2188,14 @@ async fn create_forward_session_token(
22012188
Ok(response.into_inner().token)
22022189
}
22032190

2191+
/// Older gateways reject a token-less `ForwardTcp` init with this
2192+
/// `Unauthenticated` status; newer ones authorize TCP targets on the caller's
2193+
/// principal and only demand a token for SSH targets.
2194+
fn forward_requires_session_token(status: &Status) -> bool {
2195+
status.code() == Code::Unauthenticated
2196+
&& status.message().contains("authorization_token is required")
2197+
}
2198+
22042199
async fn fetch_ready_sandbox_for_forward(
22052200
client: &mut crate::tls::GrpcClient,
22062201
name: &str,
@@ -2302,37 +2297,120 @@ async fn forward_one_tcp_connection(
23022297
target_host: String,
23032298
target_port: u16,
23042299
service_id: String,
2305-
authorization_token: String,
2300+
legacy_session_tokens: std::sync::Arc<std::sync::atomic::AtomicBool>,
23062301
) -> std::result::Result<(), ForwardTcpConnectionError> {
2307-
use tokio::io::{AsyncReadExt, AsyncWriteExt};
2302+
let mut init = TcpForwardInit {
2303+
sandbox: sandbox_name.clone(),
2304+
workspace: workspace.clone(),
2305+
service_id,
2306+
target: Some(tcp_forward_init::Target::Tcp(TcpRelayTarget {
2307+
host: target_host,
2308+
port: u32::from(target_port),
2309+
})),
2310+
// The gateway authorizes TCP forwards on this client's credentials for
2311+
// every stream, so no per-connection session token is minted unless the
2312+
// gateway turns out to predate that (`forward_requires_session_token`),
2313+
// which this forward remembers in `legacy_session_tokens`.
2314+
authorization_token: String::new(),
2315+
};
2316+
2317+
let mut session_token = None;
2318+
if legacy_session_tokens.load(std::sync::atomic::Ordering::Relaxed) {
2319+
match create_forward_session_token(client, &sandbox_name, &workspace).await {
2320+
Ok(token) => {
2321+
init.authorization_token.clone_from(&token);
2322+
session_token = Some(token);
2323+
}
2324+
Err(err) => {
2325+
drain_and_shutdown_local_socket(socket).await;
2326+
return Err(err);
2327+
}
2328+
}
2329+
}
2330+
2331+
let opened = match open_forward_tcp_stream(client, init.clone()).await {
2332+
Ok(opened) => opened,
2333+
Err(status) if session_token.is_none() && forward_requires_session_token(&status) => {
2334+
tracing::info!(
2335+
"gateway requires an SSH session token per forwarded connection; \
2336+
minting one per connection for the rest of this forward"
2337+
);
2338+
legacy_session_tokens.store(true, std::sync::atomic::Ordering::Relaxed);
2339+
let token = match create_forward_session_token(client, &sandbox_name, &workspace).await
2340+
{
2341+
Ok(token) => token,
2342+
Err(err) => {
2343+
drain_and_shutdown_local_socket(socket).await;
2344+
return Err(err);
2345+
}
2346+
};
2347+
init.authorization_token.clone_from(&token);
2348+
session_token = Some(token);
2349+
match open_forward_tcp_stream(client, init).await {
2350+
Ok(opened) => opened,
2351+
Err(status) => {
2352+
drain_and_shutdown_local_socket(socket).await;
2353+
revoke_forward_session_token(client, session_token).await;
2354+
return Err(ForwardTcpConnectionError::from_status(status));
2355+
}
2356+
}
2357+
}
2358+
Err(status) => {
2359+
drain_and_shutdown_local_socket(socket).await;
2360+
revoke_forward_session_token(client, session_token).await;
2361+
return Err(ForwardTcpConnectionError::from_status(status));
2362+
}
2363+
};
2364+
2365+
let result = bridge_local_socket_to_forward_stream(socket, opened).await;
2366+
revoke_forward_session_token(client, session_token).await;
2367+
result
2368+
}
2369+
2370+
/// An open `ForwardTcp` stream: the sender for local-to-gateway frames and the
2371+
/// gateway-to-local response stream.
2372+
type OpenForwardTcpStream = (
2373+
tokio::sync::mpsc::Sender<TcpForwardFrame>,
2374+
tonic::Streaming<TcpForwardFrame>,
2375+
);
2376+
2377+
async fn open_forward_tcp_stream(
2378+
client: &mut crate::tls::GrpcClient,
2379+
init: TcpForwardInit,
2380+
) -> std::result::Result<OpenForwardTcpStream, Status> {
23082381
use tokio_stream::wrappers::ReceiverStream;
23092382

23102383
let (tx, rx) = tokio::sync::mpsc::channel::<TcpForwardFrame>(16);
23112384
tx.send(TcpForwardFrame {
23122385
payload: Some(openshell_core::proto::tcp_forward_frame::Payload::Init(
2313-
TcpForwardInit {
2314-
sandbox: sandbox_name,
2315-
workspace: workspace.clone(),
2316-
service_id,
2317-
target: Some(tcp_forward_init::Target::Tcp(TcpRelayTarget {
2318-
host: target_host,
2319-
port: u32::from(target_port),
2320-
})),
2321-
authorization_token,
2322-
},
2386+
init,
23232387
)),
23242388
})
23252389
.await
2326-
.map_err(|_| ForwardTcpConnectionError::transient("failed to initialize forward stream"))?;
2390+
.map_err(|_| Status::internal("failed to initialize forward stream"))?;
2391+
let response = client
2392+
.forward_tcp(ReceiverStream::new(rx))
2393+
.await?
2394+
.into_inner();
2395+
Ok((tx, response))
2396+
}
23272397

2328-
let mut response = match client.forward_tcp(ReceiverStream::new(rx)).await {
2329-
Ok(response) => response.into_inner(),
2330-
Err(status) => {
2331-
let err = ForwardTcpConnectionError::from_status(status);
2332-
drain_and_shutdown_local_socket(socket).await;
2333-
return Err(err);
2334-
}
2335-
};
2398+
async fn revoke_forward_session_token(client: &mut crate::tls::GrpcClient, token: Option<String>) {
2399+
if let Some(token) = token {
2400+
let _ = client
2401+
.revoke_ssh_session(RevokeSshSessionRequest {
2402+
allow_missing: true,
2403+
token,
2404+
})
2405+
.await;
2406+
}
2407+
}
2408+
2409+
async fn bridge_local_socket_to_forward_stream(
2410+
socket: tokio::net::TcpStream,
2411+
(tx, mut response): OpenForwardTcpStream,
2412+
) -> std::result::Result<(), ForwardTcpConnectionError> {
2413+
use tokio::io::{AsyncReadExt, AsyncWriteExt};
23362414

23372415
let (mut local_read, mut local_write) = socket.into_split();
23382416

@@ -6694,8 +6772,8 @@ fn format_endpoint(endpoint: &openshell_core::proto::NetworkEndpoint) -> String
66946772
mod tests {
66956773
use super::{
66966774
PolicyGetView, ProvisioningStep, build_sandbox_resource_limits, format_endpoint,
6697-
format_log_line, git_sync_files, has_main_process_result, parse_cli_setting_value,
6698-
parse_credential_expiry_cli_value, parse_driver_config_json,
6775+
format_log_line, forward_requires_session_token, git_sync_files, has_main_process_result,
6776+
parse_cli_setting_value, parse_credential_expiry_cli_value, parse_driver_config_json,
66996777
parse_secret_material_env_pairs, policy_revision_list_json, policy_revision_to_json,
67006778
proto_execution_timeout, provisioning_timeout_message, ready_false_condition_message,
67016779
resolve_from, rootfs_tar_sources_supported_for_gateway, sandbox_should_persist,
@@ -8602,4 +8680,17 @@ mod tests {
86028680
.expect("over the limit must fail");
86038681
assert!(error.to_string().contains("sandbox upload"), "{error}");
86048682
}
8683+
8684+
#[test]
8685+
fn forward_requires_session_token_matches_only_the_legacy_gateway_error() {
8686+
assert!(forward_requires_session_token(&Status::unauthenticated(
8687+
"authorization_token is required for ForwardTcp"
8688+
)));
8689+
assert!(!forward_requires_session_token(&Status::unauthenticated(
8690+
"SSH session token not found"
8691+
)));
8692+
assert!(!forward_requires_session_token(&Status::permission_denied(
8693+
"authorization_token is required for ForwardTcp"
8694+
)));
8695+
}
86058696
}

0 commit comments

Comments
 (0)