Skip to content

Commit 6a1a792

Browse files
n1hilityjohntmyers
authored andcommitted
perf(server): drop per-connection session tokens for TCP forwards
`openshell forward service` minted an SSH session token before every forwarded TCP connection and revoked it afterwards: two store commits per connection. The token added nothing on that path. `ForwardTcp` already authenticates the caller and authorizes it against the sandbox's workspace on every stream before it looks at the token, the relay to the supervisor is opened with the sandbox id and target only, and the token is never forwarded, audited, or visible to the target service. The mechanism exists for `openshell sandbox ssh`, where the process that opens the stream is an ssh ProxyCommand holding nothing but the token. Reusing it per TCP connection put a store write on the connect path and serialized concurrent forwards on commit latency: #3494 measured the symptom, and #3543 made the commits cheaper, but each one still holds SQLite's writer lock for an fsync, so connection setup under a burst stayed linear in the number of concurrent connections. Let `target.tcp` streams omit `authorization_token`. The gateway admits them on the already-authorized principal, counts them against the same per-sandbox connection cap, and touches no store. `target.ssh` streams keep requiring the token. A token supplied with a TCP target is still validated and counted per token, so an older CLI against a new gateway is unchanged. The CLI stops minting and revoking a session per forwarded connection; against a gateway that predates this change it recognizes the `authorization_token is required` rejection once and falls back to per-connection tokens for the rest of that forward. Tests cover token-less TCP admission and slot release, SSH targets still rejected without a token, a supplied token still validated, and the per-sandbox cap for token-less forwards. CLI integration tests run `service_forward_tcp` against a mock gateway: token-less inits echo data with no CreateSshSession or RevokeSshSession call, and a gateway that rejects the empty token is detected once, after which every connection in that forward mints and revokes its own token. Architecture and security docs describe which targets carry a token, and the per-token connection limit now reads 3, matching the gateway. Signed-off-by: Jason T. Greene <[email protected]>
1 parent a2429fc commit 6a1a792

4 files changed

Lines changed: 540 additions & 74 deletions

File tree

‎crates/openshell-cli/src/run.rs‎

Lines changed: 136 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -2120,6 +2120,9 @@ pub async fn service_forward_tcp(
21202120
let sandbox_name = name.to_string();
21212121
let sandbox_workspace = workspace.to_string();
21222122
let (fatal_tx, mut fatal_rx) = tokio::sync::mpsc::channel::<String>(1);
2123+
// Set once this forward learns that the gateway predates principal-authorized
2124+
// TCP forwards and still requires a `CreateSshSession` token per connection.
2125+
let legacy_session_tokens = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false));
21232126
let mut health_check = tokio::time::interval(Duration::from_secs(2));
21242127
health_check.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
21252128
loop {
@@ -2143,21 +2146,8 @@ pub async fn service_forward_tcp(
21432146
let target_host = target_host.to_string();
21442147
let service_id = format!("service-forward:{name}:{target_host}:{target_port}");
21452148
let fatal_tx = fatal_tx.clone();
2149+
let legacy_session_tokens = legacy_session_tokens.clone();
21462150
tokio::spawn(async move {
2147-
let token = match create_forward_session_token(
2148-
&mut client,
2149-
&sandbox_name,
2150-
&sandbox_workspace,
2151-
).await {
2152-
Ok(token) => token,
2153-
Err(err) => {
2154-
tracing::warn!(peer = %peer, error = %err, "service forward session creation failed");
2155-
if err.fatal {
2156-
let _ = fatal_tx.send(err.message).await;
2157-
}
2158-
return;
2159-
}
2160-
};
21612151
if let Err(err) = forward_one_tcp_connection(
21622152
&mut client,
21632153
socket,
@@ -2166,7 +2156,7 @@ pub async fn service_forward_tcp(
21662156
target_host,
21672157
target_port,
21682158
service_id,
2169-
token.clone(),
2159+
legacy_session_tokens,
21702160
)
21712161
.await
21722162
{
@@ -2175,9 +2165,6 @@ pub async fn service_forward_tcp(
21752165
let _ = fatal_tx.send(err.message).await;
21762166
}
21772167
}
2178-
let _ = client
2179-
.revoke_ssh_session(RevokeSshSessionRequest { allow_missing: true, token })
2180-
.await;
21812168
});
21822169
}
21832170
}
@@ -2201,6 +2188,14 @@ async fn create_forward_session_token(
22012188
Ok(response.into_inner().token)
22022189
}
22032190

2191+
/// Older gateways reject a token-less `ForwardTcp` init with this
2192+
/// `Unauthenticated` status; newer ones authorize TCP targets on the caller's
2193+
/// principal and only demand a token for SSH targets.
2194+
fn forward_requires_session_token(status: &Status) -> bool {
2195+
status.code() == Code::Unauthenticated
2196+
&& status.message().contains("authorization_token is required")
2197+
}
2198+
22042199
async fn fetch_ready_sandbox_for_forward(
22052200
client: &mut crate::tls::GrpcClient,
22062201
name: &str,
@@ -2302,37 +2297,119 @@ async fn forward_one_tcp_connection(
23022297
target_host: String,
23032298
target_port: u16,
23042299
service_id: String,
2305-
authorization_token: String,
2300+
legacy_session_tokens: std::sync::Arc<std::sync::atomic::AtomicBool>,
23062301
) -> std::result::Result<(), ForwardTcpConnectionError> {
2302+
let mut init = TcpForwardInit {
2303+
sandbox: sandbox_name.clone(),
2304+
workspace: workspace.clone(),
2305+
service_id,
2306+
target: Some(tcp_forward_init::Target::Tcp(TcpRelayTarget {
2307+
host: target_host,
2308+
port: u32::from(target_port),
2309+
})),
2310+
// The gateway authorizes TCP forwards on this client's credentials for
2311+
// every stream, so no per-connection session token is minted unless the
2312+
// gateway turns out to predate that (`forward_requires_session_token`),
2313+
// which this forward remembers in `legacy_session_tokens`.
2314+
authorization_token: String::new(),
2315+
};
2316+
2317+
let mut session_token = None;
2318+
if legacy_session_tokens.load(std::sync::atomic::Ordering::Relaxed) {
2319+
match create_forward_session_token(client, &sandbox_name, &workspace).await {
2320+
Ok(token) => {
2321+
init.authorization_token.clone_from(&token);
2322+
session_token = Some(token);
2323+
}
2324+
Err(err) => {
2325+
drain_and_shutdown_local_socket(socket).await;
2326+
return Err(err);
2327+
}
2328+
}
2329+
}
2330+
2331+
let opened = match open_forward_tcp_stream(client, init.clone()).await {
2332+
Ok(opened) => opened,
2333+
Err(status) if session_token.is_none() && forward_requires_session_token(&status) => {
2334+
tracing::info!(
2335+
"gateway requires an SSH session token per forwarded connection; \
2336+
minting one per connection for the rest of this forward"
2337+
);
2338+
legacy_session_tokens.store(true, std::sync::atomic::Ordering::Relaxed);
2339+
let token = match create_forward_session_token(client, &sandbox_name, &workspace).await
2340+
{
2341+
Ok(token) => token,
2342+
Err(err) => {
2343+
drain_and_shutdown_local_socket(socket).await;
2344+
return Err(err);
2345+
}
2346+
};
2347+
init.authorization_token.clone_from(&token);
2348+
session_token = Some(token);
2349+
match open_forward_tcp_stream(client, init).await {
2350+
Ok(opened) => opened,
2351+
Err(status) => {
2352+
drain_and_shutdown_local_socket(socket).await;
2353+
revoke_forward_session_token(client, session_token).await;
2354+
return Err(ForwardTcpConnectionError::from_status(status));
2355+
}
2356+
}
2357+
}
2358+
Err(status) => {
2359+
drain_and_shutdown_local_socket(socket).await;
2360+
revoke_forward_session_token(client, session_token).await;
2361+
return Err(ForwardTcpConnectionError::from_status(status));
2362+
}
2363+
};
2364+
2365+
let result = bridge_local_socket_to_forward_stream(socket, opened).await;
2366+
revoke_forward_session_token(client, session_token).await;
2367+
result
2368+
}
2369+
2370+
/// An open `ForwardTcp` stream: the sender for local-to-gateway frames and the
2371+
/// gateway-to-local response stream.
2372+
type OpenForwardTcpStream = (
2373+
tokio::sync::mpsc::Sender<TcpForwardFrame>,
2374+
tonic::Streaming<TcpForwardFrame>,
2375+
);
2376+
2377+
async fn open_forward_tcp_stream(
2378+
client: &mut crate::tls::GrpcClient,
2379+
init: TcpForwardInit,
2380+
) -> std::result::Result<OpenForwardTcpStream, Status> {
23072381
use tokio_stream::wrappers::ReceiverStream;
23082382

23092383
let (tx, rx) = tokio::sync::mpsc::channel::<TcpForwardFrame>(16);
23102384
tx.send(TcpForwardFrame {
23112385
payload: Some(openshell_core::proto::tcp_forward_frame::Payload::Init(
2312-
TcpForwardInit {
2313-
sandbox: sandbox_name,
2314-
workspace: workspace.clone(),
2315-
service_id,
2316-
target: Some(tcp_forward_init::Target::Tcp(TcpRelayTarget {
2317-
host: target_host,
2318-
port: u32::from(target_port),
2319-
})),
2320-
authorization_token,
2321-
},
2386+
init,
23222387
)),
23232388
})
23242389
.await
2325-
.map_err(|_| ForwardTcpConnectionError::transient("failed to initialize forward stream"))?;
2390+
.map_err(|_| Status::internal("failed to initialize forward stream"))?;
2391+
let response = client
2392+
.forward_tcp(ReceiverStream::new(rx))
2393+
.await?
2394+
.into_inner();
2395+
Ok((tx, response))
2396+
}
23262397

2327-
let response = match client.forward_tcp(ReceiverStream::new(rx)).await {
2328-
Ok(response) => response.into_inner(),
2329-
Err(status) => {
2330-
let err = ForwardTcpConnectionError::from_status(status);
2331-
drain_and_shutdown_local_socket(socket).await;
2332-
return Err(err);
2333-
}
2334-
};
2398+
async fn revoke_forward_session_token(client: &mut crate::tls::GrpcClient, token: Option<String>) {
2399+
if let Some(token) = token {
2400+
let _ = client
2401+
.revoke_ssh_session(RevokeSshSessionRequest {
2402+
allow_missing: true,
2403+
token,
2404+
})
2405+
.await;
2406+
}
2407+
}
23352408

2409+
async fn bridge_local_socket_to_forward_stream(
2410+
socket: tokio::net::TcpStream,
2411+
(tx, response): OpenForwardTcpStream,
2412+
) -> std::result::Result<(), ForwardTcpConnectionError> {
23362413
let (local_read, local_write) = socket.into_split();
23372414
relay_local_socket(local_read, local_write, tx, response).await
23382415
}
@@ -6714,13 +6791,14 @@ fn format_endpoint(endpoint: &openshell_core::proto::NetworkEndpoint) -> String
67146791
mod tests {
67156792
use super::{
67166793
ForwardTcpConnectionError, PolicyGetView, ProvisioningStep, build_sandbox_resource_limits,
6717-
format_endpoint, format_log_line, git_sync_files, has_main_process_result,
6718-
parse_cli_setting_value, parse_credential_expiry_cli_value, parse_driver_config_json,
6719-
parse_secret_material_env_pairs, policy_revision_list_json, policy_revision_to_json,
6720-
proto_execution_timeout, provisioning_timeout_message, ready_false_condition_message,
6721-
relay_local_socket, resolve_from, rootfs_tar_sources_supported_for_gateway,
6722-
sandbox_should_persist, sandbox_upload_plan, service_endpoint_to_json,
6723-
service_expose_status_error, service_url_for_gateway, workspace_member_to_json,
6794+
format_endpoint, format_log_line, forward_requires_session_token, git_sync_files,
6795+
has_main_process_result, parse_cli_setting_value, parse_credential_expiry_cli_value,
6796+
parse_driver_config_json, parse_secret_material_env_pairs, policy_revision_list_json,
6797+
policy_revision_to_json, proto_execution_timeout, provisioning_timeout_message,
6798+
ready_false_condition_message, relay_local_socket, resolve_from,
6799+
rootfs_tar_sources_supported_for_gateway, sandbox_should_persist, sandbox_upload_plan,
6800+
service_endpoint_to_json, service_expose_status_error, service_url_for_gateway,
6801+
workspace_member_to_json,
67246802
};
67256803
use openshell_core::proto::TcpForwardFrame;
67266804

@@ -8704,4 +8782,17 @@ mod tests {
87048782
.expect("over the limit must fail");
87058783
assert!(error.to_string().contains("sandbox upload"), "{error}");
87068784
}
8785+
8786+
#[test]
8787+
fn forward_requires_session_token_matches_only_the_legacy_gateway_error() {
8788+
assert!(forward_requires_session_token(&Status::unauthenticated(
8789+
"authorization_token is required for ForwardTcp"
8790+
)));
8791+
assert!(!forward_requires_session_token(&Status::unauthenticated(
8792+
"SSH session token not found"
8793+
)));
8794+
assert!(!forward_requires_session_token(&Status::permission_denied(
8795+
"authorization_token is required for ForwardTcp"
8796+
)));
8797+
}
87078798
}

0 commit comments

Comments
 (0)