@@ -477,6 +477,10 @@ async fn assert_tree(step: &str, routes: &[String], gone: &[String]) -> Vec<Bran
477477 }
478478 } )
479479 . await ;
480+ assert ! (
481+ !last. is_empty( ) ,
482+ "{step}: no pool records for {routes:?}; TREE may not match the gateway's `objects` schema"
483+ ) ;
480484 assert ! (
481485 settled. is_ok( ) ,
482486 "{step}: sessions did not settle on their supervisor's gateway within {SETTLE_DEADLINE:?}: {last:#?}"
@@ -488,6 +492,7 @@ async fn assert_tree(step: &str, routes: &[String], gone: &[String]) -> Vec<Bran
488492/// new port-forward outlives the test, so later tests keep the CLI's address.
489493struct GatewayForward {
490494 port : u16 ,
495+ child : Option < tokio:: process:: Child > ,
491496}
492497
493498impl GatewayForward {
@@ -505,7 +510,7 @@ impl GatewayForward {
505510 . as_u64 ( )
506511 . and_then ( |port| u16:: try_from ( port) . ok ( ) )
507512 . expect ( "gateway_port" ) ;
508- Self { port }
513+ Self { port, child : None }
509514 }
510515
511516 async fn reachable ( & self ) -> bool {
@@ -514,34 +519,48 @@ impl GatewayForward {
514519 . is_ok ( )
515520 }
516521
517- async fn ensure ( & self ) {
518- if self . reachable ( ) . await {
519- return ;
520- }
521- kubectl_command ( )
522- . args ( [
523- "-n" ,
524- & release_namespace ( ) ,
525- "port-forward" ,
526- & format ! ( "svc/{}" , release( ) ) ,
527- & format ! ( "{}:8080" , self . port) ,
528- ] )
529- . stdout ( Stdio :: null ( ) )
530- . stderr ( Stdio :: null ( ) )
531- . kill_on_drop ( false )
532- . spawn ( )
533- . expect ( "spawn kubectl port-forward" ) ;
534- let opened = tokio:: time:: timeout ( Duration :: from_secs ( 30 ) , async {
535- while !self . reachable ( ) . await {
536- tokio:: time:: sleep ( Duration :: from_millis ( 500 ) ) . await ;
522+ /// Returns once the port answers through a live port-forward: the
523+ /// harness's, or one this started that has not exited, for example after
524+ /// losing the port to a dying predecessor.
525+ async fn ensure ( & mut self ) {
526+ let deadline = tokio:: time:: Instant :: now ( ) + Duration :: from_mins ( 1 ) ;
527+ loop {
528+ let ours_alive = self
529+ . child
530+ . as_mut ( )
531+ . map ( |child| matches ! ( child. try_wait( ) , Ok ( None ) ) ) ;
532+ match ( ours_alive, self . reachable ( ) . await ) {
533+ ( None | Some ( true ) , true ) => return ,
534+ ( Some ( true ) , false ) => { }
535+ _ => {
536+ self . child = Some (
537+ kubectl_command ( )
538+ . args ( [
539+ "-n" ,
540+ & release_namespace ( ) ,
541+ "port-forward" ,
542+ & format ! ( "svc/{}" , release( ) ) ,
543+ & format ! ( "{}:8080" , self . port) ,
544+ ] )
545+ . stdout ( Stdio :: null ( ) )
546+ . stderr ( Stdio :: null ( ) )
547+ . kill_on_drop ( false )
548+ . spawn ( )
549+ . expect ( "spawn kubectl port-forward" ) ,
550+ ) ;
551+ }
537552 }
538- } )
539- . await ;
540- assert ! ( opened. is_ok( ) , "port-forward on {} never opened" , self . port) ;
553+ assert ! (
554+ tokio:: time:: Instant :: now( ) < deadline,
555+ "no live port-forward on {}" ,
556+ self . port
557+ ) ;
558+ tokio:: time:: sleep ( Duration :: from_millis ( 500 ) ) . await ;
559+ }
541560 }
542561}
543562
544- async fn assert_exec ( step : & str , forward : & GatewayForward , sandboxes : & [ SandboxGuard ] ) {
563+ async fn assert_exec ( step : & str , forward : & mut GatewayForward , sandboxes : & [ SandboxGuard ] ) {
545564 for sandbox in sandboxes {
546565 let mut last = String :: new ( ) ;
547566 let reached = tokio:: time:: timeout ( SETTLE_DEADLINE , async {
@@ -638,7 +657,7 @@ async fn pool_sessions_follow_their_supervisors_gateway() {
638657 ) ;
639658 wait_until_attached ( & supervisor_pods ( ) . await ) . await ;
640659
641- let forward = GatewayForward :: new ( ) ;
660+ let mut forward = GatewayForward :: new ( ) ;
642661 forward. ensure ( ) . await ;
643662 let mut sandboxes = Vec :: new ( ) ;
644663 for _ in 0 ..SANDBOXES {
@@ -653,25 +672,25 @@ async fn pool_sessions_follow_their_supervisors_gateway() {
653672 routes. push ( generation ( & sandbox. name ) . await . route ) ;
654673 }
655674 let branches = assert_tree ( "start" , & routes, & [ ] ) . await ;
656- assert_exec ( "start" , & forward, & sandboxes) . await ;
675+ assert_exec ( "start" , & mut forward, & sandboxes) . await ;
657676
658677 let crashed = vec ! [ branches[ 0 ] . supervisor_gateway. clone( ) ] ;
659678 force_delete ( & crashed[ 0 ] ) . await ;
660679 rollout ( "status" , & gateway) . await ;
661680 assert_tree ( "gateway crash" , & routes, & crashed) . await ;
662- assert_exec ( "gateway crash" , & forward, & sandboxes) . await ;
681+ assert_exec ( "gateway crash" , & mut forward, & sandboxes) . await ;
663682
664683 let restarted = gateway_pods ( ) . await ;
665684 kubectl ( & [ "-n" , & release_namespace ( ) , "rollout" , "restart" , & gateway] ) . await ;
666685 rollout ( "status" , & gateway) . await ;
667686 let branches = assert_tree ( "gateway rolling restart" , & routes, & restarted) . await ;
668- assert_exec ( "gateway rolling restart" , & forward, & sandboxes) . await ;
687+ assert_exec ( "gateway rolling restart" , & mut forward, & sandboxes) . await ;
669688
670689 let crashed = vec ! [ branches[ 0 ] . member. clone( ) ] ;
671690 force_delete ( & crashed[ 0 ] ) . await ;
672691 rollout ( "status" , & supervisor) . await ;
673692 assert_tree ( "supervisor crash" , & routes, & crashed) . await ;
674- assert_exec ( "supervisor crash" , & forward, & sandboxes) . await ;
693+ assert_exec ( "supervisor crash" , & mut forward, & sandboxes) . await ;
675694
676695 forward. ensure ( ) . await ;
677696 drop ( sandboxes) ;
0 commit comments