Skip to content

Commit fb94c0b

Browse files
ericcurtinedhemphill
authored andcommitted
fix(relay): close outbound relay stream when target closes first
Closes NVIDIA#3724 out_tx was cloned into the target-reading task, so the function's own copy kept the outbound relay stream open until the client side also ended. A target that closed a keep-alive connection never reached the client as EOF, so reused connections hung forever. Move out_tx into the target-reading task instead of cloning it, so dropping it on target EOF ends the outbound stream right away. Signed-off-by: Eric Curtin <[email protected]>
1 parent db3a843 commit fb94c0b

1 file changed

Lines changed: 134 additions & 9 deletions

File tree

‎crates/openshell-supervisor-process/src/supervisor_session.rs‎

Lines changed: 134 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -735,18 +735,51 @@ async fn handle_relay_open(
735735
}
736736
Err(e) => return Err(format!("relay_stream RPC failed: {e}").into()),
737737
};
738-
let mut inbound = response.into_inner();
738+
let inbound = response.into_inner();
739739

740740
// Connect to the local SSH daemon on its Unix socket.
741-
let (mut target_r, mut target_w) = tokio::io::split(target);
741+
let (target_r, target_w) = tokio::io::split(target);
742742

743743
debug!(
744744
channel_id = %channel_id,
745745
"relay bridge: connected to local target"
746746
);
747747

748+
bridge_relay_bytes(
749+
&channel_id,
750+
target_r,
751+
target_w,
752+
out_tx,
753+
inbound,
754+
&terminating,
755+
)
756+
.await
757+
}
758+
759+
/// Bridge bytes between a local target socket and an inbound `RelayFrame`
760+
/// stream, sending target bytes out through `out_tx`.
761+
///
762+
/// `out_tx` is moved into the target-reading task rather than cloned. A
763+
/// clone would let the sender-side task's copy be dropped on target EOF
764+
/// while this function's own copy stayed alive until `inbound` also ended,
765+
/// which keeps the outbound gRPC stream open indefinitely after the target
766+
/// closes. Moving it in means the outbound stream (and therefore the
767+
/// client's view of the connection) closes as soon as the target does,
768+
/// regardless of whether the client side has sent anything else.
769+
async fn bridge_relay_bytes<S>(
770+
channel_id: &str,
771+
mut target_r: impl AsyncRead + Unpin + Send + 'static,
772+
mut target_w: impl AsyncWrite + Unpin,
773+
out_tx: mpsc::Sender<RelayFrame>,
774+
mut inbound: S,
775+
terminating: &AtomicBool,
776+
) -> Result<(), Box<dyn std::error::Error + Send + Sync>>
777+
where
778+
S: tokio_stream::Stream<Item = Result<RelayFrame, tonic::Status>> + Unpin,
779+
{
748780
// Target → gRPC (out_tx): read local target, forward as `RelayFrame::data`.
749-
let out_tx_writer = out_tx.clone();
781+
// `out_tx` is owned by this task, so dropping it on target EOF ends the
782+
// outbound stream immediately, without waiting on the inbound side.
750783
let target_to_grpc = tokio::spawn(async move {
751784
let mut buf = vec![0u8; RELAY_CHUNK_SIZE];
752785
loop {
@@ -758,7 +791,7 @@ async fn handle_relay_open(
758791
buf[..n].to_vec(),
759792
)),
760793
};
761-
if out_tx_writer.send(chunk).await.is_err() {
794+
if out_tx.send(chunk).await.is_err() {
762795
break;
763796
}
764797
}
@@ -785,7 +818,7 @@ async fn handle_relay_open(
785818
}
786819
}
787820
Err(e) => {
788-
if expected_transport_close_during_shutdown(&e, &terminating) {
821+
if expected_transport_close_during_shutdown(&e, terminating) {
789822
debug!(
790823
channel_id = %channel_id,
791824
error = %e,
@@ -801,10 +834,6 @@ async fn handle_relay_open(
801834

802835
// Half-close the target socket's write side so the service sees EOF.
803836
let _ = target_w.shutdown().await;
804-
805-
// Dropping out_tx closes the outbound gRPC stream, letting the gateway
806-
// observe EOF on its side too.
807-
drop(out_tx);
808837
let _ = target_to_grpc.await;
809838

810839
if let Some(e) = inbound_err {
@@ -1240,4 +1269,100 @@ mod ocsf_event_tests {
12401269
assert!(err.to_string().contains("peer PID mismatch"));
12411270
accept_task.await.unwrap();
12421271
}
1272+
1273+
/// Regression test for #3724: when the target closes after sending
1274+
/// data, the outbound relay stream must close too, even though the
1275+
/// inbound (client) side is still open. Before the fix, `out_tx` was
1276+
/// cloned into the target-reading task, so the function's own copy kept
1277+
/// the outbound stream alive until `inbound` also ended.
1278+
#[tokio::test]
1279+
async fn bridge_closes_outbound_when_target_closes_first() {
1280+
let (target, mut remote) = tokio::io::duplex(4096);
1281+
let (target_r, target_w) = tokio::io::split(target);
1282+
1283+
let (out_tx, mut out_rx) = mpsc::channel::<RelayFrame>(16);
1284+
1285+
// Inbound stream the client never closes during this test.
1286+
let (inbound_tx, inbound_rx) = mpsc::channel::<Result<RelayFrame, tonic::Status>>(16);
1287+
let inbound = tokio_stream::wrappers::ReceiverStream::new(inbound_rx);
1288+
1289+
let terminating = AtomicBool::new(false);
1290+
let bridge = tokio::spawn(async move {
1291+
bridge_relay_bytes("chan-1", target_r, target_w, out_tx, inbound, &terminating).await
1292+
});
1293+
1294+
remote.write_all(b"hello").await.unwrap();
1295+
remote.shutdown().await.unwrap();
1296+
1297+
let frame = out_rx.recv().await.expect("data frame expected");
1298+
assert_eq!(
1299+
frame.payload,
1300+
Some(openshell_core::proto::relay_frame::Payload::Data(
1301+
b"hello".to_vec()
1302+
))
1303+
);
1304+
1305+
// The outbound stream must end here, without the inbound side (still
1306+
// held open by `inbound_tx`) ending first.
1307+
assert!(
1308+
out_rx.recv().await.is_none(),
1309+
"outbound stream should close once the target closes"
1310+
);
1311+
1312+
drop(inbound_tx);
1313+
bridge
1314+
.await
1315+
.unwrap()
1316+
.expect("bridge should finish cleanly when target closes first");
1317+
}
1318+
1319+
/// A well-behaved round trip: bytes flow both directions and the bridge
1320+
/// ends cleanly when the client closes its side.
1321+
#[tokio::test]
1322+
async fn bridge_round_trips_bytes_until_client_closes() {
1323+
let (target, mut remote) = tokio::io::duplex(4096);
1324+
let (target_r, target_w) = tokio::io::split(target);
1325+
1326+
let (out_tx, mut out_rx) = mpsc::channel::<RelayFrame>(16);
1327+
let (inbound_tx, inbound_rx) = mpsc::channel::<Result<RelayFrame, tonic::Status>>(16);
1328+
let inbound = tokio_stream::wrappers::ReceiverStream::new(inbound_rx);
1329+
1330+
let terminating = AtomicBool::new(false);
1331+
let bridge = tokio::spawn(async move {
1332+
bridge_relay_bytes("chan-2", target_r, target_w, out_tx, inbound, &terminating).await
1333+
});
1334+
1335+
// Client -> target.
1336+
inbound_tx
1337+
.send(Ok(RelayFrame {
1338+
payload: Some(openshell_core::proto::relay_frame::Payload::Data(
1339+
b"ping".to_vec(),
1340+
)),
1341+
}))
1342+
.await
1343+
.unwrap();
1344+
let mut buf = [0u8; 4];
1345+
remote.read_exact(&mut buf).await.unwrap();
1346+
assert_eq!(&buf, b"ping");
1347+
1348+
// Target -> client.
1349+
remote.write_all(b"pong").await.unwrap();
1350+
let frame = out_rx.recv().await.expect("data frame expected");
1351+
assert_eq!(
1352+
frame.payload,
1353+
Some(openshell_core::proto::relay_frame::Payload::Data(
1354+
b"pong".to_vec()
1355+
))
1356+
);
1357+
1358+
// Client closes its side first; the bridge should still complete
1359+
// once the target also closes.
1360+
drop(inbound_tx);
1361+
remote.shutdown().await.unwrap();
1362+
1363+
bridge
1364+
.await
1365+
.unwrap()
1366+
.expect("bridge should finish cleanly on an ordinary round trip");
1367+
}
12431368
}

0 commit comments

Comments
 (0)