Skip to content

Commit 0fd7918

Browse files
committed
refactor(sdk): report the last observation when a wait times out
A wait_ready or wait_deleted timeout said only which sandbox it was waiting on, which does not help diagnose a sandbox that is stuck. Have the private poll_until step return Poll::Ready or Poll::Pending(observation) and append the last observation to the timeout error. wait_ready reports the phase and exit code, and wait_deleted reports the phase. Signed-off-by: Evan Lezar <[email protected]>
1 parent 947d537 commit 0fd7918

2 files changed

Lines changed: 44 additions & 19 deletions

File tree

‎crates/openshell-sdk/src/client.rs‎

Lines changed: 28 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -49,26 +49,36 @@ pub struct OpenShellClient {
4949
owners: OwnerCache,
5050
}
5151

52-
/// Run `step` until it yields `Some`, backing off from 250ms to 2s between
53-
/// attempts, or until `timeout` elapses.
52+
/// Outcome of one poll attempt in [`poll_until`].
53+
enum Poll<T> {
54+
/// The wait is satisfied.
55+
Ready(T),
56+
/// Keep polling; the string describes what the attempt observed.
57+
Pending(String),
58+
}
59+
60+
/// Run `step` until it yields [`Poll::Ready`], backing off from 250ms to 2s
61+
/// between attempts, or until `timeout` elapses.
5462
///
55-
/// `step` returns `Ok(Some(value))` when the wait is satisfied, `Ok(None)` to
56-
/// keep polling, and `Err` to stop with that error unchanged. Only the timeout
57-
/// error is created here; `waiting_for` names what was being waited on.
63+
/// `step` returns `Ok(Poll::Pending(observation))` to keep polling and `Err`
64+
/// to stop with that error unchanged. Only the timeout error is created here;
65+
/// `waiting_for` names what was being waited on and the timeout error appends
66+
/// the last observation.
5867
async fn poll_until<T, F, Fut>(timeout: Duration, waiting_for: String, mut step: F) -> Result<T>
5968
where
6069
F: FnMut() -> Fut,
61-
Fut: Future<Output = Result<Option<T>>>,
70+
Fut: Future<Output = Result<Poll<T>>>,
6271
{
6372
let deadline = Instant::now() + timeout;
6473
let mut delay = Duration::from_millis(250);
6574
loop {
66-
if let Some(value) = step().await? {
67-
return Ok(value);
68-
}
75+
let observation = match step().await? {
76+
Poll::Ready(value) => return Ok(value),
77+
Poll::Pending(observation) => observation,
78+
};
6979
if Instant::now() >= deadline {
7080
return Err(SdkError::connect(format!(
71-
"timed out waiting for {waiting_for}"
81+
"timed out waiting for {waiting_for}; last observation: {observation}"
7282
)));
7383
}
7484
tokio::time::sleep(delay).await;
@@ -975,7 +985,7 @@ impl OpenShellClient {
975985
|| async {
976986
let snapshot = self.get_sandbox_in(workspace, name).await?;
977987
match snapshot.phase {
978-
SandboxPhase::Ready => Ok(Some(snapshot)),
988+
SandboxPhase::Ready => Ok(Poll::Ready(snapshot)),
979989
SandboxPhase::Completed => Err(SdkError::connect(format!(
980990
"sandbox '{name}' completed before becoming ready"
981991
))),
@@ -994,7 +1004,10 @@ impl OpenShellClient {
9941004
"sandbox '{name}' is suspended; resume it before waiting for ready"
9951005
)))
9961006
}
997-
_ => Ok(None),
1007+
phase => Ok(Poll::Pending(format!(
1008+
"phase {phase:?}, exit code {:?}",
1009+
snapshot.exit_code
1010+
))),
9981011
}
9991012
},
10001013
)
@@ -1013,12 +1026,12 @@ impl OpenShellClient {
10131026
format!("sandbox '{name}' to be deleted"),
10141027
|| async {
10151028
match self.get_sandbox_in(workspace, name).await {
1016-
Err(SdkError::NotFound { .. }) => Ok(Some(())),
1029+
Err(SdkError::NotFound { .. }) => Ok(Poll::Ready(())),
10171030
Err(other) => Err(other),
10181031
Ok(snapshot) if expected_sandbox_id.is_some_and(|id| snapshot.id != id) => {
1019-
Ok(Some(()))
1032+
Ok(Poll::Ready(()))
10201033
}
1021-
Ok(_) => Ok(None),
1034+
Ok(snapshot) => Ok(Poll::Pending(format!("phase {:?}", snapshot.phase))),
10221035
}
10231036
},
10241037
)

‎crates/openshell-sdk/tests/client_mock.rs‎

Lines changed: 16 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1753,10 +1753,14 @@ async fn wait_ready_times_out_while_provisioning() {
17531753
for (result, _, calls) in wait_ready_both(state, Duration::from_millis(50)).await {
17541754
let err = result.unwrap_err();
17551755
assert_eq!(err.code(), "connect");
1756+
let message = err.to_string();
17561757
assert!(
1757-
err.to_string()
1758-
.contains("timed out waiting for sandbox 'my-box'"),
1759-
"{err}"
1758+
message.contains("timed out waiting for sandbox 'my-box' to become ready"),
1759+
"{message}"
1760+
);
1761+
assert!(
1762+
message.contains("last observation: phase Provisioning"),
1763+
"{message}"
17601764
);
17611765
assert!(calls >= 1);
17621766
}
@@ -1859,7 +1863,15 @@ async fn wait_deleted_does_not_complete_while_expected_identity_remains() {
18591863
}
18601864
.unwrap_err();
18611865
assert_eq!(error.code(), "connect");
1862-
assert!(error.to_string().contains("timed out waiting"));
1866+
let message = error.to_string();
1867+
assert!(
1868+
message.contains("timed out waiting for sandbox 'my-box' to be deleted"),
1869+
"{message}"
1870+
);
1871+
assert!(
1872+
message.contains("last observation: phase Ready"),
1873+
"{message}"
1874+
);
18631875
assert_eq!(state.get_calls.load(Ordering::SeqCst), 1);
18641876
}
18651877
}

0 commit comments

Comments
 (0)