3939//! `LEAF_PUSHDOWN_FUZZ_CASES` also raises the case count of a normal run.
4040
4141use std:: sync:: Arc ;
42+ use std:: sync:: atomic:: { AtomicU64 , Ordering } ;
4243
4344use arrow:: array:: {
4445 Array , ArrayRef , AsArray , Int32Array , Int64Array , RecordBatch , StringArray ,
@@ -48,6 +49,8 @@ use arrow::datatypes::{DataType, Field, Fields, Int32Type, Schema};
4849use arrow:: util:: pretty:: pretty_format_batches;
4950use datafusion:: prelude:: { ParquetReadOptions , SessionConfig , SessionContext } ;
5051use datafusion_common:: Result ;
52+ use datafusion_common:: instant:: Instant ;
53+ use datafusion_common_runtime:: SpawnedTask ;
5154use parquet:: arrow:: ArrowWriter ;
5255use rand:: rngs:: SmallRng ;
5356use rand:: { Rng , SeedableRng } ;
@@ -638,47 +641,170 @@ async fn check_case(
638641 }
639642}
640643
641- /// Runs `cases` seeded cases against the two contexts. `left` and `right` name
642- /// the two settings in the failure report. Every case runs, so the report holds
643- /// the share of the seeds that fail, not only the first one.
644- async fn run (
645- ctx_left : & SessionContext ,
646- ctx_right : & SessionContext ,
647- left : & str ,
648- right : & str ,
649- cases : usize ,
650- seed0 : u64 ,
651- ) {
652- let mut skipped = 0 ;
653- let mut failed = 0 ;
654- let mut first: Option < String > = None ;
655- for seed in seed0..seed0 + cases as u64 {
656- let case = gen_case ( & mut SmallRng :: seed_from_u64 ( seed) ) ;
657- let Some ( detail) =
658- check_case ( ctx_left, ctx_right, left, right, & case, & mut skipped) . await
659- else {
660- continue ;
661- } ;
662- failed += 1 ;
663- first. get_or_insert_with ( || {
664- format ! (
665- " seed: {seed}\n sql: {}\n {detail}\n reproduce with \
666- LEAF_PUSHDOWN_FUZZ_SEED={seed} LEAF_PUSHDOWN_FUZZ_CASES=1",
667- case. sql
668- )
669- } ) ;
644+ /// Limit concurrent worker tasks to prevent thread thrashing
645+ /// since individual queries already execute with `target_partitions = 2`.
646+ const MAX_CONCURRENT_WORKERS : usize = 16 ;
647+ const DEFAULT_FALLBACK_WORKERS : usize = 1 ;
648+
649+ #[ derive( Debug , Clone , Copy , PartialEq , Eq ) ]
650+ struct Seed ( u64 ) ;
651+
652+ #[ derive( Debug , Clone , Copy , PartialEq , Eq ) ]
653+ struct Cases ( usize ) ;
654+
655+ /// Couples an engine's execution context with its diagnostic label.
656+ /// Guarantees at compile time that an engine context cannot be mispaired with the wrong label.
657+ #[ derive( Clone , Copy ) ]
658+ struct TestEngine < ' a > {
659+ ctx : & ' a SessionContext ,
660+ label : & ' a str ,
661+ }
662+
663+ impl < ' a > TestEngine < ' a > {
664+ fn new ( ctx : & ' a SessionContext , label : & ' a str ) -> Self {
665+ Self { ctx, label }
670666 }
671- let ratio = skipped as f64 / cases as f64 ;
667+ }
668+
669+ /// Aggregate execution metrics collected per worker task.
670+ #[ derive( Default ) ]
671+ struct WorkerResult {
672+ skipped : usize ,
673+ failed : usize ,
674+ /// Holds `(seed, error_report)` for the lowest numerical seed that failed
675+ /// in this worker. Preserving the earliest seed guarantees 100% deterministic
676+ /// reproduction (`LEAF_PUSHDOWN_FUZZ_SEED=<seed> LEAF_PUSHDOWN_FUZZ_CASES=1`)
677+ /// even though workers finish out of order.
678+ first_failure : Option < ( u64 , String ) > ,
679+ }
680+
681+ /// Runs `cases` seeded cases against the two contexts.
682+ ///
683+ /// # Concurrency & Dynamic Work-Stealing Architecture
684+ /// - Scales worker tasks to host vCPUs via `available_parallelism`, capped at
685+ /// [`MAX_CONCURRENT_WORKERS`] to prevent thread thrashing.
686+ /// - Seeds are dynamically claimed from a central [`AtomicU64`] counter with `fetch_add`,
687+ /// preventing straggler stalls when complex plans take longer to execute.
688+ /// - `SessionContext` is safely cloned across tasks since tables are pre-registered
689+ /// and differential queries only acquire read locks on `SessionState`.
690+ /// - Each worker tallies its local `skipped`, `failed`, and earliest `first_failure`.
691+ /// The coordinator reduces across all workers by minimal seed number so failure
692+ /// reporting matches sequential execution order.
693+ async fn run ( left : TestEngine < ' _ > , right : TestEngine < ' _ > , cases : Cases , seed0 : Seed ) {
694+ let Cases ( num_cases) = cases;
695+ let Seed ( start_seed) = seed0;
696+
697+ if num_cases == 0 {
698+ return ;
699+ }
700+
701+ let start_instant = Instant :: now ( ) ;
702+
703+ let num_workers = match std:: thread:: available_parallelism ( ) {
704+ Ok ( n) => n. get ( ) . clamp ( 1 , MAX_CONCURRENT_WORKERS ) . min ( num_cases) ,
705+ Err ( e) => {
706+ eprintln ! (
707+ "\n [WARNING] leaf pushdown fuzz: failed to detect CPU cores ({e}).\n \
708+ Falling back to {DEFAULT_FALLBACK_WORKERS} worker.\n "
709+ ) ;
710+ DEFAULT_FALLBACK_WORKERS . min ( num_cases)
711+ }
712+ } ;
713+
714+ let next_seed = Arc :: new ( AtomicU64 :: new ( start_seed) ) ;
715+ let max_seed = start_seed + num_cases as u64 ;
716+
717+ let mut handles = Vec :: with_capacity ( num_workers) ;
718+ for _ in 0 ..num_workers {
719+ let ctx_left = left. ctx . clone ( ) ;
720+ let ctx_right = right. ctx . clone ( ) ;
721+ let left_label = left. label . to_string ( ) ;
722+ let right_label = right. label . to_string ( ) ;
723+ let next_seed = Arc :: clone ( & next_seed) ;
724+
725+ handles. push ( SpawnedTask :: spawn ( async move {
726+ let mut res = WorkerResult :: default ( ) ;
727+
728+ loop {
729+ if next_seed. load ( Ordering :: Relaxed ) >= max_seed {
730+ break ;
731+ }
732+ let seed = next_seed. fetch_add ( 1 , Ordering :: Relaxed ) ;
733+ if seed >= max_seed {
734+ break ;
735+ }
736+
737+ let case = gen_case ( & mut SmallRng :: seed_from_u64 ( seed) ) ;
738+ let Some ( detail) = check_case (
739+ & ctx_left,
740+ & ctx_right,
741+ & left_label,
742+ & right_label,
743+ & case,
744+ & mut res. skipped ,
745+ )
746+ . await
747+ else {
748+ continue ;
749+ } ;
750+
751+ res. failed += 1 ;
752+ if res. first_failure . is_none ( ) {
753+ res. first_failure = Some ( (
754+ seed,
755+ format ! (
756+ " seed: {seed}\n sql: {}\n {detail}\n reproduce with \
757+ LEAF_PUSHDOWN_FUZZ_SEED={seed} LEAF_PUSHDOWN_FUZZ_CASES=1",
758+ case. sql
759+ ) ,
760+ ) ) ;
761+ }
762+ }
763+
764+ res
765+ } ) ) ;
766+ }
767+
768+ let mut total = WorkerResult :: default ( ) ;
769+
770+ for handle in handles {
771+ let worker_res = handle. await . expect ( "fuzz worker task panicked" ) ;
772+ total. skipped += worker_res. skipped ;
773+ total. failed += worker_res. failed ;
774+
775+ if let Some ( ( seed, report) ) = worker_res. first_failure {
776+ match & total. first_failure {
777+ Some ( ( best_seed, _) ) if seed < * best_seed => {
778+ total. first_failure = Some ( ( seed, report) ) ;
779+ }
780+ None => {
781+ total. first_failure = Some ( ( seed, report) ) ;
782+ }
783+ _ => { }
784+ }
785+ }
786+ }
787+
788+ let succeeded = num_cases - total. skipped - total. failed ;
789+ let first = total. first_failure . map ( |( _, report) | report) ;
790+ let ratio = total. skipped as f64 / num_cases as f64 ;
791+ let ( left_label, right_label) = ( left. label , right. label ) ;
792+ let elapsed = start_instant. elapsed ( ) ;
793+
672794 println ! (
673- "leaf pushdown fuzz: {left} vs {right}, seeds {seed0}..{}, ok {}, \
674- skipped {skipped} ({:.1}%), failed {failed}",
675- seed0 + cases as u64 ,
676- cases - skipped - failed,
677- ratio * 100.0
795+ "leaf pushdown fuzz: {left_label} vs {right_label}, seeds {start_seed}..{}, ok {succeeded}, \
796+ skipped {} ({:.1}%), failed {} in {:.2?}",
797+ start_seed + num_cases as u64 ,
798+ total. skipped,
799+ ratio * 100.0 ,
800+ total. failed,
801+ elapsed
678802 ) ;
803+
679804 assert ! (
680805 first. is_none( ) ,
681- "leaf pushdown fuzz: {failed} of {cases} seeds failed, first one:\n {}" ,
806+ "leaf pushdown fuzz: {} of {num_cases} seeds failed, first one:\n {}" ,
807+ total. failed,
682808 first. unwrap_or_default( )
683809 ) ;
684810 assert ! (
@@ -695,14 +821,20 @@ fn env_usize(name: &str, default: usize) -> usize {
695821 . unwrap_or ( default)
696822}
697823
698- fn seed ( ) -> u64 {
699- env_usize ( "LEAF_PUSHDOWN_FUZZ_SEED" , 0 ) as u64
824+ fn seed ( ) -> Seed {
825+ Seed ( env_usize ( "LEAF_PUSHDOWN_FUZZ_SEED" , 0 ) as u64 )
700826}
701827
702828async fn run_memory ( cases : usize ) {
703829 let on = memory_ctx ( true ) ;
704830 let off = memory_ctx ( false ) ;
705- run ( & on, & off, "pushdown on" , "pushdown off" , cases, seed ( ) ) . await ;
831+ run (
832+ TestEngine :: new ( & on, "pushdown on" ) ,
833+ TestEngine :: new ( & off, "pushdown off" ) ,
834+ Cases ( cases) ,
835+ seed ( ) ,
836+ )
837+ . await ;
706838}
707839
708840// ---------------------------------------------------------------------------
@@ -753,11 +885,9 @@ async fn run_parquet(cases: usize) {
753885 let on = parquet_ctx ( & dir, true ) . await ;
754886 let off = parquet_ctx ( & dir, false ) . await ;
755887 run (
756- & on,
757- & off,
758- "parquet pushdown on" ,
759- "parquet pushdown off" ,
760- cases,
888+ TestEngine :: new ( & on, "parquet pushdown on" ) ,
889+ TestEngine :: new ( & off, "parquet pushdown off" ) ,
890+ Cases ( cases) ,
761891 seed ( ) ,
762892 )
763893 . await ;
@@ -820,11 +950,9 @@ async fn leaf_pushdown_parquet_schema_evolution() {
820950 }
821951 check_evolved_filters ( & contexts[ 0 ] , & contexts[ 1 ] , & wide) . await ;
822952 run (
823- & contexts[ 0 ] ,
824- & contexts[ 1 ] ,
825- "parquet pushdown on" ,
826- "parquet pushdown off" ,
827- DEFAULT_CASES / 5 ,
953+ TestEngine :: new ( & contexts[ 0 ] , "parquet pushdown on" ) ,
954+ TestEngine :: new ( & contexts[ 1 ] , "parquet pushdown off" ) ,
955+ Cases ( DEFAULT_CASES / 5 ) ,
828956 seed ( ) ,
829957 )
830958 . await ;
0 commit comments