Skip to content

Commit 3e2dfab

Browse files
committed
test(e2e): harden the supervisor tree check's port-forward and time limit
Signed-off-by: divesh <[email protected]>
1 parent f7593db commit 3e2dfab

2 files changed

Lines changed: 54 additions & 30 deletions

File tree

‎.config/nextest.toml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,11 @@ test-group = "kubernetes-shared-supervisor-pool"
4646
filter = "test(/^scaling_a_supervisor_pool_up_and_down_keeps_its_sandboxes_running$/)"
4747
slow-timeout = { period = "60s", terminate-after = 20 }
4848

49+
# Three bounded rollouts (up to 5 minutes each) and four settle and exec rounds.
50+
[[profile.e2e-kubernetes.overrides]]
51+
filter = "test(/^pool_sessions_follow_their_supervisors_gateway$/)"
52+
slow-timeout = { period = "60s", terminate-after = 20 }
53+
4954
# Relative to the profile store dir (`e2e/rust/target/nextest/e2e-kubernetes/`).
5055
[profile.e2e-kubernetes.junit]
5156
path = "../../../../../results/e2e-kubernetes.xml"

‎e2e/rust/tests/kubernetes_shared_supervisor_ha.rs‎

Lines changed: 49 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -477,6 +477,10 @@ async fn assert_tree(step: &str, routes: &[String], gone: &[String]) -> Vec<Bran
477477
}
478478
})
479479
.await;
480+
assert!(
481+
!last.is_empty(),
482+
"{step}: no pool records for {routes:?}; TREE may not match the gateway's `objects` schema"
483+
);
480484
assert!(
481485
settled.is_ok(),
482486
"{step}: sessions did not settle on their supervisor's gateway within {SETTLE_DEADLINE:?}: {last:#?}"
@@ -488,6 +492,7 @@ async fn assert_tree(step: &str, routes: &[String], gone: &[String]) -> Vec<Bran
488492
/// new port-forward outlives the test, so later tests keep the CLI's address.
489493
struct GatewayForward {
490494
port: u16,
495+
child: Option<tokio::process::Child>,
491496
}
492497

493498
impl GatewayForward {
@@ -505,7 +510,7 @@ impl GatewayForward {
505510
.as_u64()
506511
.and_then(|port| u16::try_from(port).ok())
507512
.expect("gateway_port");
508-
Self { port }
513+
Self { port, child: None }
509514
}
510515

511516
async fn reachable(&self) -> bool {
@@ -514,34 +519,48 @@ impl GatewayForward {
514519
.is_ok()
515520
}
516521

517-
async fn ensure(&self) {
518-
if self.reachable().await {
519-
return;
520-
}
521-
kubectl_command()
522-
.args([
523-
"-n",
524-
&release_namespace(),
525-
"port-forward",
526-
&format!("svc/{}", release()),
527-
&format!("{}:8080", self.port),
528-
])
529-
.stdout(Stdio::null())
530-
.stderr(Stdio::null())
531-
.kill_on_drop(false)
532-
.spawn()
533-
.expect("spawn kubectl port-forward");
534-
let opened = tokio::time::timeout(Duration::from_secs(30), async {
535-
while !self.reachable().await {
536-
tokio::time::sleep(Duration::from_millis(500)).await;
522+
/// Returns once the port answers through a live port-forward: the
523+
/// harness's, or one this started that has not exited, for example after
524+
/// losing the port to a dying predecessor.
525+
async fn ensure(&mut self) {
526+
let deadline = tokio::time::Instant::now() + Duration::from_mins(1);
527+
loop {
528+
let ours_alive = self
529+
.child
530+
.as_mut()
531+
.map(|child| matches!(child.try_wait(), Ok(None)));
532+
match (ours_alive, self.reachable().await) {
533+
(None | Some(true), true) => return,
534+
(Some(true), false) => {}
535+
_ => {
536+
self.child = Some(
537+
kubectl_command()
538+
.args([
539+
"-n",
540+
&release_namespace(),
541+
"port-forward",
542+
&format!("svc/{}", release()),
543+
&format!("{}:8080", self.port),
544+
])
545+
.stdout(Stdio::null())
546+
.stderr(Stdio::null())
547+
.kill_on_drop(false)
548+
.spawn()
549+
.expect("spawn kubectl port-forward"),
550+
);
551+
}
537552
}
538-
})
539-
.await;
540-
assert!(opened.is_ok(), "port-forward on {} never opened", self.port);
553+
assert!(
554+
tokio::time::Instant::now() < deadline,
555+
"no live port-forward on {}",
556+
self.port
557+
);
558+
tokio::time::sleep(Duration::from_millis(500)).await;
559+
}
541560
}
542561
}
543562

544-
async fn assert_exec(step: &str, forward: &GatewayForward, sandboxes: &[SandboxGuard]) {
563+
async fn assert_exec(step: &str, forward: &mut GatewayForward, sandboxes: &[SandboxGuard]) {
545564
for sandbox in sandboxes {
546565
let mut last = String::new();
547566
let reached = tokio::time::timeout(SETTLE_DEADLINE, async {
@@ -638,7 +657,7 @@ async fn pool_sessions_follow_their_supervisors_gateway() {
638657
);
639658
wait_until_attached(&supervisor_pods().await).await;
640659

641-
let forward = GatewayForward::new();
660+
let mut forward = GatewayForward::new();
642661
forward.ensure().await;
643662
let mut sandboxes = Vec::new();
644663
for _ in 0..SANDBOXES {
@@ -653,25 +672,25 @@ async fn pool_sessions_follow_their_supervisors_gateway() {
653672
routes.push(generation(&sandbox.name).await.route);
654673
}
655674
let branches = assert_tree("start", &routes, &[]).await;
656-
assert_exec("start", &forward, &sandboxes).await;
675+
assert_exec("start", &mut forward, &sandboxes).await;
657676

658677
let crashed = vec![branches[0].supervisor_gateway.clone()];
659678
force_delete(&crashed[0]).await;
660679
rollout("status", &gateway).await;
661680
assert_tree("gateway crash", &routes, &crashed).await;
662-
assert_exec("gateway crash", &forward, &sandboxes).await;
681+
assert_exec("gateway crash", &mut forward, &sandboxes).await;
663682

664683
let restarted = gateway_pods().await;
665684
kubectl(&["-n", &release_namespace(), "rollout", "restart", &gateway]).await;
666685
rollout("status", &gateway).await;
667686
let branches = assert_tree("gateway rolling restart", &routes, &restarted).await;
668-
assert_exec("gateway rolling restart", &forward, &sandboxes).await;
687+
assert_exec("gateway rolling restart", &mut forward, &sandboxes).await;
669688

670689
let crashed = vec![branches[0].member.clone()];
671690
force_delete(&crashed[0]).await;
672691
rollout("status", &supervisor).await;
673692
assert_tree("supervisor crash", &routes, &crashed).await;
674-
assert_exec("supervisor crash", &forward, &sandboxes).await;
693+
assert_exec("supervisor crash", &mut forward, &sandboxes).await;
675694

676695
forward.ensure().await;
677696
drop(sandboxes);

0 commit comments

Comments
 (0)