Skip to content

Commit 6427db9

Browse files
fix(testkit): bound readiness and cleanup with timeouts
wait_ready checked elapsed time only after a health call returned, and cleanup started its timeout only after the delete returned, so a gateway that accepts a connection but never answers could hang either one. Wrap each whole operation in tokio::time::timeout: the readiness wait, including every connect and health call, and each sandbox's delete plus wait for deletion. A test covers a peer that accepts connections and never replies. Signed-off-by: Matthew Grossman <[email protected]>
1 parent 5cf4df2 commit 6427db9

2 files changed

Lines changed: 74 additions & 24 deletions

File tree

‎tests/testkit/testkit/src/context.rs‎

Lines changed: 17 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -153,16 +153,24 @@ impl Context {
153153
/// others and cleanup takes about one timeout however many there are.
154154
pub(crate) async fn cleanup(&self) -> Vec<String> {
155155
let names: Vec<String> = lock(&self.sandboxes).iter().cloned().collect();
156+
// The timeout wraps the whole delete, not just the wait for it, so a
157+
// gateway that never answers the delete cannot hang cleanup.
156158
let results = futures::future::join_all(names.iter().map(|name| async move {
157-
self.client
158-
.delete_sandbox(
159-
name,
160-
DeleteOptions {
161-
allow_missing: true,
162-
},
163-
)
164-
.await?;
165-
self.client.wait_deleted(name, CLEANUP_TIMEOUT, None).await
159+
let delete = async {
160+
self.client
161+
.delete_sandbox(
162+
name,
163+
DeleteOptions {
164+
allow_missing: true,
165+
},
166+
)
167+
.await?;
168+
self.client.wait_deleted(name, CLEANUP_TIMEOUT, None).await
169+
};
170+
match tokio::time::timeout(CLEANUP_TIMEOUT, delete).await {
171+
Ok(result) => result.map_err(|error| error.to_string()),
172+
Err(_) => Err(format!("timed out after {}s", CLEANUP_TIMEOUT.as_secs())),
173+
}
166174
}))
167175
.await;
168176
let mut failures = Vec::new();

‎tests/testkit/testkit/src/gateway.rs‎

Lines changed: 57 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515
//! `mtls/` directory. `OPENSHELL_WORKSPACE` selects the workspace.
1616
1717
use std::path::{Path, PathBuf};
18-
use std::time::{Duration, Instant};
18+
use std::time::Duration;
1919

2020
use openshell_sdk::{ClientConfig, ClientIdentity, Health, OpenShellClient, ServiceStatus};
2121

@@ -160,24 +160,35 @@ impl Gateway {
160160
/// Wait until the gateway accepts a connection and reports itself
161161
/// healthy, retrying until `timeout`.
162162
///
163+
/// The whole wait, including each connect and health call, is bounded by
164+
/// `timeout`, so a gateway that accepts connections but never answers
165+
/// cannot hang it.
166+
///
163167
/// This is a readiness check for whoever brings the environment up, run
164168
/// once before its tests. Tests themselves connect once and fail fast.
165169
pub async fn wait_ready(&self, timeout: Duration) -> Result<Health> {
166-
let started = Instant::now();
167-
loop {
168-
let error = match self.check_health().await {
169-
Ok(health) => return Ok(health),
170-
Err(error) => error,
171-
};
172-
if started.elapsed() >= timeout {
173-
return Err(error.context(format!(
174-
"gateway at {} not ready after {}s",
175-
self.endpoint,
176-
timeout.as_secs()
177-
)));
170+
let mut last_error = None;
171+
let outcome = tokio::time::timeout(timeout, async {
172+
loop {
173+
match self.check_health().await {
174+
Ok(health) => return health,
175+
Err(error) => last_error = Some(error),
176+
}
177+
tokio::time::sleep(READY_INTERVAL).await;
178178
}
179-
tokio::time::sleep(READY_INTERVAL).await;
180-
}
179+
})
180+
.await;
181+
outcome.map_err(|_| {
182+
let message = format!(
183+
"gateway at {} not ready after {}s",
184+
self.endpoint,
185+
timeout.as_secs()
186+
);
187+
match last_error.take() {
188+
Some(error) => error.context(message),
189+
None => Error::msg(message),
190+
}
191+
})
181192
}
182193

183194
async fn check_health(&self) -> Result<Health> {
@@ -224,3 +235,34 @@ fn read(path: &Path) -> Result<Vec<u8>> {
224235
.map_err(Error::from_display)
225236
.context(format!("read {}", path.display()))
226237
}
238+
239+
#[cfg(test)]
240+
mod tests {
241+
use std::net::TcpListener;
242+
use std::time::{Duration, Instant};
243+
244+
use super::Gateway;
245+
246+
/// A peer that accepts connections and never answers must not hang the
247+
/// readiness wait: the timeout covers the connect and health calls too.
248+
#[tokio::test]
249+
async fn wait_ready_gives_up_on_a_gateway_that_never_answers() {
250+
let listener = TcpListener::bind("127.0.0.1:0").expect("bind");
251+
let endpoint = format!("http://{}", listener.local_addr().expect("address"));
252+
std::thread::spawn(move || {
253+
// Hold every connection open without reading or replying.
254+
let held: Vec<_> = listener.incoming().flatten().collect();
255+
drop(held);
256+
});
257+
258+
let gateway = Gateway::from_endpoint(endpoint, "default".to_string());
259+
let started = Instant::now();
260+
let error = gateway
261+
.wait_ready(Duration::from_secs(1))
262+
.await
263+
.expect_err("an unresponsive gateway is not ready");
264+
265+
assert!(started.elapsed() < Duration::from_secs(5), "{error}");
266+
assert!(error.to_string().contains("not ready after 1s"), "{error}");
267+
}
268+
}

0 commit comments

Comments
 (0)