@@ -22,6 +22,7 @@ use std::fmt::Write as _;
2222use std:: sync:: Arc ;
2323
2424use arrow:: array:: RecordBatch ;
25+ use arrow:: compute:: concat_batches;
2526use arrow_schema:: SchemaRef ;
2627use datafusion:: datasource:: MemTable ;
2728use datafusion:: prelude:: { SessionConfig , SessionContext } ;
@@ -37,8 +38,6 @@ use rand::Rng;
3738use rand:: prelude:: IndexedRandom ;
3839use rand:: { SeedableRng , rngs:: StdRng } ;
3940
40- use crate :: fuzz_cases:: aggregation_fuzzer:: check_equality_of_batches;
41-
4241use super :: aggregation_fuzzer:: ColumnDescr ;
4342use super :: record_batch_generator:: { RecordBatchGenerator , get_supported_types_columns} ;
4443
@@ -199,7 +198,7 @@ impl SortQueryFuzzer {
199198 let init_seed = self . runner_rng . random ( ) ;
200199 for query_i in 0 ..self . queries_per_round {
201200 let query_seed = self . runner_rng . random ( ) ;
202- let mut expected_results: Option < Vec < RecordBatch > > = None ; // use first config's result as the expected result
201+ let mut expected_results: Option < RecordBatch > = None ; // use first config's result as the expected result
203202 for config_i in 0 ..self . config_variations_per_query {
204203 if self . should_stop_due_to_time_limit ( start_time, round, query_i) {
205204 return Ok ( ( ) ) ;
@@ -217,18 +216,16 @@ impl SortQueryFuzzer {
217216
218217 let results = self
219218 . test_gen
220- . fuzzer_run ( init_seed, query_seed, config_seed)
219+ . fuzzer_run ( init_seed, query_seed, config_seed, config_i == 0 )
221220 . await ?;
222221 println ! ( "\n " ) ; // Separator between tested runs
223222
224223 if expected_results. is_none ( ) {
225224 expected_results = Some ( results) ;
226225 } else if let Some ( ref expected) = expected_results {
227- // `fuzzer_run` might append `LIMIT k` to either the
228- // expected or actual query. The number of results is
229- // checked inside `fuzzer_run()`. Here we only check
230- // that the first k rows of each result are consistent.
231- check_equality_of_batches ( expected, & results) . unwrap ( ) ;
226+ // Every projected column is an ORDER BY key, so tied
227+ // rows have identical output and can be compared in order.
228+ assert_eq ! ( expected, & results) ;
232229 } else {
233230 unreachable ! ( ) ;
234231 }
@@ -480,6 +477,7 @@ impl SortFuzzerTestGenerator {
480477 & self ,
481478 rng_seed : u64 ,
482479 with_memory_limit : bool ,
480+ sort_in_place_threshold_override : Option < usize > ,
483481 ) -> Result < SessionContext > {
484482 let mut rng = StdRng :: seed_from_u64 ( rng_seed) ;
485483 let init_state = self . dataset_state . as_ref ( ) . unwrap ( ) ;
@@ -508,6 +506,8 @@ impl SortFuzzerTestGenerator {
508506 let dataset_size = self . dataset_state . as_ref ( ) . unwrap ( ) . dataset_size ;
509507 rng. random_range ( 0 ..=dataset_size * 2_usize )
510508 } ;
509+ let sort_in_place_threshold_bytes =
510+ sort_in_place_threshold_override. unwrap_or ( sort_in_place_threshold_bytes) ;
511511
512512 // Set up strings for printing
513513 let memory_limit_str = if with_memory_limit {
@@ -539,6 +539,8 @@ impl SortFuzzerTestGenerator {
539539 let config = SessionConfig :: new ( )
540540 . with_target_partitions ( num_partitions)
541541 . with_batch_size ( init_state. approx_batch_num_rows / 2 )
542+ // Keep small input batches separate when comparing the two sort paths.
543+ . with_coalesce_batches ( sort_in_place_threshold_override. is_none ( ) )
542544 . with_sort_spill_reservation_bytes ( sort_spill_reservation_bytes)
543545 . with_sort_in_place_threshold_bytes ( sort_in_place_threshold_bytes) ;
544546
@@ -569,7 +571,8 @@ impl SortFuzzerTestGenerator {
569571 dataset_seed : u64 ,
570572 query_seed : u64 ,
571573 config_seed : u64 ,
572- ) -> Result < Vec < RecordBatch > > {
574+ compare_sort_paths : bool ,
575+ ) -> Result < RecordBatch > {
573576 self . init_partitioned_staggered_batches ( dataset_seed) ;
574577 let ( query_str, limit) = self . generate_random_query ( query_seed) ;
575578 println ! ( " Query:" ) ;
@@ -582,19 +585,35 @@ impl SortFuzzerTestGenerator {
582585 // 2. Memory limiting is enabled in the test generator config
583586 let with_mem_limit = !query_str. contains ( "LIMIT" ) && self . set_memory_limit ;
584587
585- let ctx = self . generate_random_config ( config_seed, with_mem_limit) ?;
588+ let ctx = self . generate_random_config ( config_seed, with_mem_limit, None ) ?;
586589 let df = ctx. sql ( & query_str) . await . unwrap ( ) ;
587- let results = df. collect ( ) . await . unwrap ( ) ;
590+ let schema = Arc :: clone ( df. schema ( ) . inner ( ) ) ;
591+ let results = concat_batches ( & schema, & df. collect ( ) . await ?) ?;
588592
589593 // ==== Check the result size is consistent with the limit ====
590- let result_num_rows = results. iter ( ) . map ( |batch| batch . num_rows ( ) ) . sum :: < usize > ( ) ;
594+ let result_num_rows = results. num_rows ( ) ;
591595 let dataset_size = self . dataset_state . as_ref ( ) . unwrap ( ) . dataset_size ;
592596
593597 if let Some ( limit) = limit {
594598 let expected_num_rows = min ( limit, dataset_size) ;
595599 assert_eq ! ( result_num_rows, expected_num_rows) ;
596600 }
597601
602+ if compare_sort_paths && limit. is_none ( ) && !with_mem_limit {
603+ // Run the same query inline and with bounded parallel preparation.
604+ // An unbounded memory pool allows the inline path to concatenate all runs.
605+ for threshold in [ usize:: MAX , 0 ] {
606+ let ctx =
607+ self . generate_random_config ( config_seed, false , Some ( threshold) ) ?;
608+ let actual = ctx. sql ( & query_str) . await ?. collect ( ) . await ?;
609+ assert_eq ! (
610+ results,
611+ concat_batches( & schema, & actual) ?,
612+ "sort_in_place_threshold_bytes={threshold}"
613+ ) ;
614+ }
615+ }
616+
598617 Ok ( results)
599618 }
600619}
@@ -616,8 +635,8 @@ mod test {
616635 gen_seed,
617636 ) ;
618637
619- let res1 = test_generator. fuzzer_run ( 1 , 2 , 3 ) . await . unwrap ( ) ;
620- let res2 = test_generator. fuzzer_run ( 1 , 2 , 3 ) . await . unwrap ( ) ;
621- check_equality_of_batches ( & res1, & res2) . unwrap ( ) ;
638+ let res1 = test_generator. fuzzer_run ( 1 , 2 , 3 , false ) . await . unwrap ( ) ;
639+ let res2 = test_generator. fuzzer_run ( 1 , 2 , 3 , false ) . await . unwrap ( ) ;
640+ assert_eq ! ( res1, res2) ;
622641 }
623642}
0 commit comments