Skip to content

Commit 2768c08

Browse files
committed
fix(sandbox,podman): cap each proxied CONNECT attempt within the shared budget
A proxy that accepted the first CONNECT request but never responded consumed the entire aggregate handshake timeout, so later validated addresses were never tried and the hang defeated the multi-address fallback. Each attempt is now time-boxed to its fair share of the time remaining before the shared deadline (remaining / attempts_left): a hanging attempt is cut off with enough budget left for every remaining address, while time a fast failure does not use rolls over to later attempts and the total never exceeds CONNECT_HANDSHAKE_TIMEOUT. A timed-out attempt is recorded like any other failure, and the aggregate error distinguishes all-attempted from budget-exhausted runs. Adds a first-hangs/second-succeeds regression driven through a test-visible budget parameter so it runs in about a second instead of a real 30s window. Signed-off-by: Philippe Martin <[email protected]>
1 parent 707a50f commit 2768c08

1 file changed

Lines changed: 114 additions & 33 deletions

File tree

‎crates/openshell-supervisor-network/src/upstream_proxy.rs‎

Lines changed: 114 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -654,57 +654,90 @@ pub async fn connect_via(
654654
/// dual-stack destination whose first validated address is unreachable
655655
/// through the corporate proxy still connects via a later one. All attempts
656656
/// share one aggregate handshake budget ([`CONNECT_HANDSHAKE_TIMEOUT`]),
657-
/// matching the single-attempt paths.
657+
/// matching the single-attempt paths; within it, each attempt is capped at
658+
/// its fair share of the remaining time (`remaining / attempts_left`) so a
659+
/// proxy that accepts a CONNECT but never answers cannot starve the
660+
/// remaining validated addresses. Time an attempt fails without using
661+
/// rolls over to later attempts.
658662
///
659663
/// # Errors
660664
///
661-
/// Returns an error when `addrs` is empty, when every attempt fails (the
662-
/// last attempt's error, annotated with the attempt count when there were
663-
/// several), or when the aggregate budget expires.
665+
/// Returns an error when `addrs` is empty, or when every attempt fails or
666+
/// times out (the last attempt's error, annotated with the attempt count
667+
/// when there were several).
664668
pub async fn connect_via_validated(
665669
endpoint: &ProxyEndpoint,
666670
host: &str,
667671
port: u16,
668672
addrs: &[SocketAddr],
673+
) -> std::io::Result<PrefixedStream> {
674+
connect_via_validated_with_budget(endpoint, host, port, addrs, CONNECT_HANDSHAKE_TIMEOUT).await
675+
}
676+
677+
/// [`connect_via_validated`] with an explicit aggregate budget, split out so
678+
/// tests can exercise the hang-fallback behavior without real 30s waits.
679+
async fn connect_via_validated_with_budget(
680+
endpoint: &ProxyEndpoint,
681+
host: &str,
682+
port: u16,
683+
addrs: &[SocketAddr],
684+
budget: Duration,
669685
) -> std::io::Result<PrefixedStream> {
670686
if addrs.is_empty() {
671687
return Err(IoError::new(
672688
ErrorKind::InvalidInput,
673689
format!("no validated addresses to CONNECT to for {host}"),
674690
));
675691
}
676-
tokio::time::timeout(CONNECT_HANDSHAKE_TIMEOUT, async {
677-
let mut last_err = None;
678-
for addr in addrs {
679-
match connect_via_inner(endpoint, host, port, ConnectTarget::Ip(addr.ip())).await {
680-
Ok(stream) => return Ok(stream),
681-
Err(err) => last_err = Some(err),
682-
}
692+
let deadline = tokio::time::Instant::now() + budget;
693+
let mut last_err = None;
694+
let mut attempted = 0usize;
695+
for (index, addr) in addrs.iter().enumerate() {
696+
let remaining = deadline.saturating_duration_since(tokio::time::Instant::now());
697+
if remaining.is_zero() {
698+
break;
683699
}
684-
// `addrs` is non-empty, so at least one attempt recorded an error.
685-
let last = last_err.expect("at least one CONNECT attempt");
686-
if addrs.len() > 1 {
687-
Err(IoError::new(
688-
last.kind(),
689-
format!(
690-
"CONNECT failed for all {} validated addresses of {host}; last: {last}",
691-
addrs.len()
692-
),
693-
))
694-
} else {
695-
Err(last)
700+
let attempts_left = u32::try_from(addrs.len() - index).unwrap_or(u32::MAX);
701+
let attempt_budget = remaining / attempts_left;
702+
attempted += 1;
703+
match tokio::time::timeout(
704+
attempt_budget,
705+
connect_via_inner(endpoint, host, port, ConnectTarget::Ip(addr.ip())),
706+
)
707+
.await
708+
{
709+
Ok(Ok(stream)) => return Ok(stream),
710+
Ok(Err(err)) => last_err = Some(err),
711+
Err(_) => {
712+
last_err = Some(IoError::new(
713+
ErrorKind::TimedOut,
714+
format!(
715+
"upstream proxy {} CONNECT to {} timed out",
716+
endpoint.display_addr(),
717+
addr.ip(),
718+
),
719+
));
720+
}
696721
}
697-
})
698-
.await
699-
.map_err(|_| {
700-
IoError::new(
701-
ErrorKind::TimedOut,
702-
format!(
703-
"upstream proxy {} CONNECT handshake timed out",
704-
endpoint.display_addr()
705-
),
722+
}
723+
// The first iteration always runs (`remaining` starts at the full
724+
// budget), so at least one attempt recorded an error.
725+
let last = last_err.expect("at least one CONNECT attempt");
726+
if addrs.len() == 1 {
727+
return Err(last);
728+
}
729+
let scope = if attempted == addrs.len() {
730+
format!("all {} validated addresses", addrs.len())
731+
} else {
732+
format!(
733+
"{attempted} of {} validated addresses (handshake budget exhausted)",
734+
addrs.len()
706735
)
707-
})?
736+
};
737+
Err(IoError::new(
738+
last.kind(),
739+
format!("CONNECT failed for {scope} of {host}; last: {last}"),
740+
))
708741
}
709742

710743
async fn connect_via_inner(
@@ -1371,6 +1404,54 @@ mod tests {
13711404
);
13721405
}
13731406

1407+
#[tokio::test]
1408+
async fn connect_via_validated_survives_a_hanging_first_attempt() {
1409+
// A proxy that accepts the CONNECT but never answers must not
1410+
// consume the whole aggregate budget: the per-address cap moves on
1411+
// to the next validated address.
1412+
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
1413+
let addr = listener.local_addr().unwrap();
1414+
let handle = tokio::spawn(async move {
1415+
// First attempt: read the CONNECT, then hang with the socket
1416+
// held open until the client times the attempt out.
1417+
let (mut hung, _) = listener.accept().await.unwrap();
1418+
let first = read_request(&mut hung).await;
1419+
// Second attempt: establish the tunnel.
1420+
let (mut accepted, _) = listener.accept().await.unwrap();
1421+
let second = read_request(&mut accepted).await;
1422+
accepted
1423+
.write_all(b"HTTP/1.1 200 Connection established\r\n\r\n")
1424+
.await
1425+
.unwrap();
1426+
drop(hung);
1427+
(first, second, accepted)
1428+
});
1429+
1430+
let endpoint = endpoint_for(addr, None);
1431+
let addrs = [sock("192.0.2.1", 443), sock("192.0.2.2", 443)];
1432+
// Small real-time budget: the first attempt hangs for its fair
1433+
// share (half), then the second succeeds within the remainder.
1434+
let stream = connect_via_validated_with_budget(
1435+
&endpoint,
1436+
"api.example.com",
1437+
443,
1438+
&addrs,
1439+
Duration::from_secs(2),
1440+
)
1441+
.await
1442+
.unwrap();
1443+
let (first, second, _accepted) = handle.await.unwrap();
1444+
drop(stream);
1445+
assert!(
1446+
first.starts_with("CONNECT 192.0.2.1:443 HTTP/1.1\r\n"),
1447+
"{first}"
1448+
);
1449+
assert!(
1450+
second.starts_with("CONNECT 192.0.2.2:443 HTTP/1.1\r\n"),
1451+
"{second}"
1452+
);
1453+
}
1454+
13741455
#[tokio::test]
13751456
async fn connect_via_validated_reports_failure_across_all_addresses() {
13761457
let (addr, _handle) = fake_proxy("HTTP/1.1 502 Bad Gateway\r\n\r\n").await;

0 commit comments

Comments
 (0)