3838//!
3939//! `LEAF_PUSHDOWN_FUZZ_CASES` also raises the case count of a normal run.
4040
41+ use std:: sync:: atomic:: { AtomicU64 , Ordering } ;
4142use std:: sync:: Arc ;
4243
4344use arrow:: array:: {
@@ -47,7 +48,9 @@ use arrow::array::{
4748use arrow:: datatypes:: { DataType , Field , Fields , Int32Type , Schema } ;
4849use arrow:: util:: pretty:: pretty_format_batches;
4950use datafusion:: prelude:: { ParquetReadOptions , SessionConfig , SessionContext } ;
51+ use datafusion_common:: instant:: Instant ;
5052use datafusion_common:: Result ;
53+ use datafusion_common_runtime:: SpawnedTask ;
5154use parquet:: arrow:: ArrowWriter ;
5255use rand:: rngs:: SmallRng ;
5356use rand:: { Rng , SeedableRng } ;
@@ -638,47 +641,175 @@ async fn check_case(
638641 }
639642}
640643
641- /// Runs `cases` seeded cases against the two contexts. `left` and `right` name
642- /// the two settings in the failure report. Every case runs, so the report holds
643- /// the share of the seeds that fail, not only the first one.
644+ /// Limit concurrent worker tasks to prevent thread thrashing
645+ /// since individual queries already execute with `target_partitions = 2`.
646+ const MAX_CONCURRENT_WORKERS : usize = 16 ;
647+ const DEFAULT_FALLBACK_WORKERS : usize = 1 ;
648+
649+ #[ derive( Debug , Clone , Copy , PartialEq , Eq ) ]
650+ struct Seed ( u64 ) ;
651+
652+ #[ derive( Debug , Clone , Copy , PartialEq , Eq ) ]
653+ struct Cases ( usize ) ;
654+
655+ /// Couples an engine's execution context with its diagnostic label.
656+ /// Guarantees at compile time that an engine context cannot be mispaired with the wrong label.
657+ #[ derive( Clone , Copy ) ]
658+ struct TestEngine < ' a > {
659+ ctx : & ' a SessionContext ,
660+ label : & ' a str ,
661+ }
662+
663+ impl < ' a > TestEngine < ' a > {
664+ fn new ( ctx : & ' a SessionContext , label : & ' a str ) -> Self {
665+ Self { ctx, label }
666+ }
667+ }
668+
669+ /// Aggregate execution metrics collected per worker task.
670+ #[ derive( Default ) ]
671+ struct WorkerResult {
672+ skipped : usize ,
673+ failed : usize ,
674+ /// Holds `(seed, error_report)` for the lowest numerical seed that failed
675+ /// in this worker. Preserving the earliest seed guarantees 100% deterministic
676+ /// reproduction (`LEAF_PUSHDOWN_FUZZ_SEED=<seed> LEAF_PUSHDOWN_FUZZ_CASES=1`)
677+ /// even though workers finish out of order.
678+ first_failure : Option < ( u64 , String ) > ,
679+ }
680+
681+ /// Runs `cases` seeded cases against the two contexts.
682+ ///
683+ /// # Concurrency & Dynamic Work-Stealing Architecture
684+ /// - Scales worker tasks to host vCPUs via `available_parallelism`, capped at
685+ /// [`MAX_CONCURRENT_WORKERS`] to prevent thread thrashing.
686+ /// - Seeds are dynamically claimed from a central [`AtomicU64`] counter with `fetch_add`,
687+ /// preventing straggler stalls when complex plans take longer to execute.
688+ /// - `SessionContext` is safely cloned across tasks since tables are pre-registered
689+ /// and differential queries only acquire read locks on `SessionState`.
690+ /// - Each worker tallies its local `skipped`, `failed`, and earliest `first_failure`.
691+ /// The coordinator reduces across all workers by minimal seed number so failure
692+ /// reporting matches sequential execution order.
644693async fn run (
645- ctx_left : & SessionContext ,
646- ctx_right : & SessionContext ,
647- left : & str ,
648- right : & str ,
649- cases : usize ,
650- seed0 : u64 ,
694+ left : TestEngine < ' _ > ,
695+ right : TestEngine < ' _ > ,
696+ cases : Cases ,
697+ seed0 : Seed ,
651698) {
652- let mut skipped = 0 ;
653- let mut failed = 0 ;
654- let mut first: Option < String > = None ;
655- for seed in seed0..seed0 + cases as u64 {
656- let case = gen_case ( & mut SmallRng :: seed_from_u64 ( seed) ) ;
657- let Some ( detail) =
658- check_case ( ctx_left, ctx_right, left, right, & case, & mut skipped) . await
659- else {
660- continue ;
661- } ;
662- failed += 1 ;
663- first. get_or_insert_with ( || {
664- format ! (
665- " seed: {seed}\n sql: {}\n {detail}\n reproduce with \
666- LEAF_PUSHDOWN_FUZZ_SEED={seed} LEAF_PUSHDOWN_FUZZ_CASES=1",
667- case. sql
668- )
669- } ) ;
699+ let Cases ( num_cases) = cases;
700+ let Seed ( start_seed) = seed0;
701+
702+ if num_cases == 0 {
703+ return ;
670704 }
671- let ratio = skipped as f64 / cases as f64 ;
705+
706+ let start_instant = Instant :: now ( ) ;
707+
708+ let num_workers = match std:: thread:: available_parallelism ( ) {
709+ Ok ( n) => n. get ( ) . clamp ( 1 , MAX_CONCURRENT_WORKERS ) . min ( num_cases) ,
710+ Err ( e) => {
711+ eprintln ! (
712+ "\n [WARNING] leaf pushdown fuzz: failed to detect CPU cores ({e}).\n \
713+ Falling back to {DEFAULT_FALLBACK_WORKERS} worker.\n "
714+ ) ;
715+ DEFAULT_FALLBACK_WORKERS . min ( num_cases)
716+ }
717+ } ;
718+
719+ let next_seed = Arc :: new ( AtomicU64 :: new ( start_seed) ) ;
720+ let max_seed = start_seed + num_cases as u64 ;
721+
722+ let mut handles = Vec :: with_capacity ( num_workers) ;
723+ for _ in 0 ..num_workers {
724+ let ctx_left = left. ctx . clone ( ) ;
725+ let ctx_right = right. ctx . clone ( ) ;
726+ let left_label = left. label . to_string ( ) ;
727+ let right_label = right. label . to_string ( ) ;
728+ let next_seed = Arc :: clone ( & next_seed) ;
729+
730+ handles. push ( SpawnedTask :: spawn ( async move {
731+ let mut res = WorkerResult :: default ( ) ;
732+
733+ loop {
734+ if next_seed. load ( Ordering :: Relaxed ) >= max_seed {
735+ break ;
736+ }
737+ let seed = next_seed. fetch_add ( 1 , Ordering :: Relaxed ) ;
738+ if seed >= max_seed {
739+ break ;
740+ }
741+
742+ let case = gen_case ( & mut SmallRng :: seed_from_u64 ( seed) ) ;
743+ let Some ( detail) = check_case (
744+ & ctx_left,
745+ & ctx_right,
746+ & left_label,
747+ & right_label,
748+ & case,
749+ & mut res. skipped ,
750+ )
751+ . await
752+ else {
753+ continue ;
754+ } ;
755+
756+ res. failed += 1 ;
757+ if res. first_failure . is_none ( ) {
758+ res. first_failure = Some ( (
759+ seed,
760+ format ! (
761+ " seed: {seed}\n sql: {}\n {detail}\n reproduce with \
762+ LEAF_PUSHDOWN_FUZZ_SEED={seed} LEAF_PUSHDOWN_FUZZ_CASES=1",
763+ case. sql
764+ ) ,
765+ ) ) ;
766+ }
767+ }
768+
769+ res
770+ } ) ) ;
771+ }
772+
773+ let mut total = WorkerResult :: default ( ) ;
774+
775+ for handle in handles {
776+ let worker_res = handle. await . expect ( "fuzz worker task panicked" ) ;
777+ total. skipped += worker_res. skipped ;
778+ total. failed += worker_res. failed ;
779+
780+ if let Some ( ( seed, report) ) = worker_res. first_failure {
781+ match & total. first_failure {
782+ Some ( ( best_seed, _) ) if seed < * best_seed => {
783+ total. first_failure = Some ( ( seed, report) ) ;
784+ }
785+ None => {
786+ total. first_failure = Some ( ( seed, report) ) ;
787+ }
788+ _ => { }
789+ }
790+ }
791+ }
792+
793+ let succeeded = num_cases - total. skipped - total. failed ;
794+ let first = total. first_failure . map ( |( _, report) | report) ;
795+ let ratio = total. skipped as f64 / num_cases as f64 ;
796+ let ( left_label, right_label) = ( left. label , right. label ) ;
797+ let elapsed = start_instant. elapsed ( ) ;
798+
672799 println ! (
673- "leaf pushdown fuzz: {left} vs {right}, seeds {seed0}..{}, ok {}, \
674- skipped {skipped} ({:.1}%), failed {failed}",
675- seed0 + cases as u64 ,
676- cases - skipped - failed,
677- ratio * 100.0
800+ "leaf pushdown fuzz: {left_label} vs {right_label}, seeds {start_seed}..{}, ok {succeeded}, \
801+ skipped {} ({:.1}%), failed {} in {:.2?}",
802+ start_seed + num_cases as u64 ,
803+ total. skipped,
804+ ratio * 100.0 ,
805+ total. failed,
806+ elapsed
678807 ) ;
808+
679809 assert ! (
680810 first. is_none( ) ,
681- "leaf pushdown fuzz: {failed} of {cases} seeds failed, first one:\n {}" ,
811+ "leaf pushdown fuzz: {} of {num_cases} seeds failed, first one:\n {}" ,
812+ total. failed,
682813 first. unwrap_or_default( )
683814 ) ;
684815 assert ! (
@@ -695,14 +826,20 @@ fn env_usize(name: &str, default: usize) -> usize {
695826 . unwrap_or ( default)
696827}
697828
698- fn seed ( ) -> u64 {
699- env_usize ( "LEAF_PUSHDOWN_FUZZ_SEED" , 0 ) as u64
829+ fn seed ( ) -> Seed {
830+ Seed ( env_usize ( "LEAF_PUSHDOWN_FUZZ_SEED" , 0 ) as u64 )
700831}
701832
702833async fn run_memory ( cases : usize ) {
703834 let on = memory_ctx ( true ) ;
704835 let off = memory_ctx ( false ) ;
705- run ( & on, & off, "pushdown on" , "pushdown off" , cases, seed ( ) ) . await ;
836+ run (
837+ TestEngine :: new ( & on, "pushdown on" ) ,
838+ TestEngine :: new ( & off, "pushdown off" ) ,
839+ Cases ( cases) ,
840+ seed ( ) ,
841+ )
842+ . await ;
706843}
707844
708845// ---------------------------------------------------------------------------
@@ -753,11 +890,9 @@ async fn run_parquet(cases: usize) {
753890 let on = parquet_ctx ( & dir, true ) . await ;
754891 let off = parquet_ctx ( & dir, false ) . await ;
755892 run (
756- & on,
757- & off,
758- "parquet pushdown on" ,
759- "parquet pushdown off" ,
760- cases,
893+ TestEngine :: new ( & on, "parquet pushdown on" ) ,
894+ TestEngine :: new ( & off, "parquet pushdown off" ) ,
895+ Cases ( cases) ,
761896 seed ( ) ,
762897 )
763898 . await ;
@@ -820,11 +955,9 @@ async fn leaf_pushdown_parquet_schema_evolution() {
820955 }
821956 check_evolved_filters ( & contexts[ 0 ] , & contexts[ 1 ] , & wide) . await ;
822957 run (
823- & contexts[ 0 ] ,
824- & contexts[ 1 ] ,
825- "parquet pushdown on" ,
826- "parquet pushdown off" ,
827- DEFAULT_CASES / 5 ,
958+ TestEngine :: new ( & contexts[ 0 ] , "parquet pushdown on" ) ,
959+ TestEngine :: new ( & contexts[ 1 ] , "parquet pushdown off" ) ,
960+ Cases ( DEFAULT_CASES / 5 ) ,
828961 seed ( ) ,
829962 )
830963 . await ;
0 commit comments