@@ -270,6 +270,12 @@ pub(super) struct ParquetMorselizer {
270270 pub enable_bloom_filter : bool ,
271271 /// Should row group pruning be applied
272272 pub enable_row_group_stats_pruning : bool ,
273+ /// Should a file range that partially overlaps a row group read the
274+ /// proportional slice of the row group's rows (via a `RowSelection`)
275+ /// instead of assigning the entire row group to the range containing its
276+ /// first data page. This allows partitions to decode disjoint parts of
277+ /// the same row group in parallel.
278+ pub split_row_groups_by_range : bool ,
273279 /// Coerce INT96 timestamps to specific TimeUnit
274280 pub coerce_int96 : Option < TimeUnit > ,
275281 /// Optional timezone applied to INT96-coerced timestamps. When `Some`, the
@@ -444,6 +450,7 @@ struct PreparedParquetOpen {
444450 enable_page_index : bool ,
445451 enable_bloom_filter : bool ,
446452 enable_row_group_stats_pruning : bool ,
453+ split_row_groups_by_range : bool ,
447454 limit : Option < usize > ,
448455 coerce_int96 : Option < TimeUnit > ,
449456 coerce_int96_tz : Option < Arc < str > > ,
@@ -843,6 +850,7 @@ impl ParquetMorselizer {
843850 enable_page_index : self . enable_page_index ,
844851 enable_bloom_filter : self . enable_bloom_filter ,
845852 enable_row_group_stats_pruning : self . enable_row_group_stats_pruning ,
853+ split_row_groups_by_range : self . split_row_groups_by_range ,
846854 limit : self . limit ,
847855 coerce_int96 : self . coerce_int96 ,
848856 coerce_int96_tz : self . coerce_int96_tz . clone ( ) ,
@@ -1093,7 +1101,14 @@ impl FiltersPreparedParquetOpen {
10931101
10941102 // If there is a range restricting what parts of the file to read
10951103 if let Some ( range) = prepared. file_range . as_ref ( ) {
1096- row_groups. prune_by_range ( rg_metadata, range) ;
1104+ if prepared. split_row_groups_by_range {
1105+ // Read the proportional slice of rows from row groups that
1106+ // partially overlap the range, so partitions can decode
1107+ // disjoint parts of the same row group in parallel.
1108+ row_groups. split_by_range ( rg_metadata, range) ;
1109+ } else {
1110+ row_groups. prune_by_range ( rg_metadata, range) ;
1111+ }
10971112 }
10981113
10991114 // If there is a predicate that can be evaluated against the metadata
@@ -1648,12 +1663,19 @@ fn should_load_page_index(
16481663 page_pruning_predicate : Option < & Arc < PagePruningAccessPlanFilter > > ,
16491664 row_groups : & RowGroupAccessPlanFilter ,
16501665) -> bool {
1651- page_pruning_predicate. is_some_and ( |_| {
1666+ // Load to prune pages with the predicate, unless every surviving row
1667+ // group is already known to fully match it.
1668+ let for_page_pruning = page_pruning_predicate. is_some_and ( |_| {
16521669 let fully_matched = row_groups. is_fully_matched ( ) ;
16531670 row_groups
16541671 . row_group_indexes ( )
16551672 . any ( |idx| !fully_matched[ idx] )
1656- } )
1673+ } ) ;
1674+
1675+ // Row selections (e.g. from `split_by_range`) need the offset index so
1676+ // the reader fetches and decodes only the pages covering the selected
1677+ // rows rather than entire column chunks.
1678+ for_page_pruning || row_groups. has_selection ( )
16571679}
16581680
16591681/// Returns a `ArrowReaderMetadata` with the page index loaded, loading
@@ -1749,6 +1771,7 @@ mod test {
17491771 enable_page_index : bool ,
17501772 enable_bloom_filter : bool ,
17511773 enable_row_group_stats_pruning : bool ,
1774+ split_row_groups_by_range : bool ,
17521775 coerce_int96 : Option < TimeUnit > ,
17531776 max_predicate_cache_size : Option < usize > ,
17541777 reverse_row_groups : bool ,
@@ -1857,6 +1880,7 @@ mod test {
18571880 enable_page_index : false ,
18581881 enable_bloom_filter : false ,
18591882 enable_row_group_stats_pruning : false ,
1883+ split_row_groups_by_range : false ,
18601884 coerce_int96 : None ,
18611885 max_predicate_cache_size : None ,
18621886 reverse_row_groups : false ,
@@ -1928,6 +1952,12 @@ mod test {
19281952 self
19291953 }
19301954
1955+ /// Enable splitting row groups that partially overlap a file range.
1956+ fn with_split_row_groups_by_range ( mut self , enable : bool ) -> Self {
1957+ self . split_row_groups_by_range = enable;
1958+ self
1959+ }
1960+
19311961 /// Enable page index.
19321962 fn with_enable_page_index ( mut self , enable : bool ) -> Self {
19331963 self . enable_page_index = enable;
@@ -2025,6 +2055,7 @@ mod test {
20252055 enable_page_index : self . enable_page_index ,
20262056 enable_bloom_filter : self . enable_bloom_filter ,
20272057 enable_row_group_stats_pruning : self . enable_row_group_stats_pruning ,
2058+ split_row_groups_by_range : self . split_row_groups_by_range ,
20282059 coerce_int96 : self . coerce_int96 ,
20292060 // End-to-end coercion behavior (including timezone) is
20302061 // covered by parquet.slt. No opener-level test currently
@@ -2246,6 +2277,106 @@ mod test {
22462277 ) )
22472278 }
22482279
2280+ /// Read `test.parquet` through `num_ranges` byte ranges tiling the file
2281+ /// and return the values produced by each range.
2282+ async fn read_by_ranges (
2283+ opener : & ParquetMorselizer ,
2284+ data_size : usize ,
2285+ num_ranges : i64 ,
2286+ ) -> Vec < Vec < i32 > > {
2287+ let chunk = ( data_size as i64 + num_ranges - 1 ) / num_ranges;
2288+ let mut results = vec ! [ ] ;
2289+ for i in 0 ..num_ranges {
2290+ let file = PartitionedFile :: new ( "test.parquet" . to_string ( ) , data_size as u64 )
2291+ . with_range ( i * chunk, ( ( i + 1 ) * chunk) . min ( data_size as i64 ) ) ;
2292+ let stream = open_file ( opener, file) . await . unwrap ( ) ;
2293+ results. push ( collect_int32_values ( stream) . await ) ;
2294+ }
2295+ results
2296+ }
2297+
2298+ /// A file with a single row group scanned through byte ranges: with
2299+ /// `split_row_groups_by_range` each range decodes a disjoint slice of the
2300+ /// row group (so multiple partitions share the work), and every row is
2301+ /// read exactly once.
2302+ #[ tokio:: test]
2303+ async fn test_split_row_groups_by_range ( ) {
2304+ let store = Arc :: new ( InMemory :: new ( ) ) as Arc < dyn ObjectStore > ;
2305+
2306+ let schema: SchemaRef =
2307+ Arc :: new ( Schema :: new ( vec ! [ Field :: new( "a" , DataType :: Int32 , false ) ] ) ) ;
2308+ let batch = RecordBatch :: try_new (
2309+ Arc :: clone ( & schema) ,
2310+ vec ! [ Arc :: new( arrow:: array:: Int32Array :: from_iter_values(
2311+ 0 ..1000 ,
2312+ ) ) ] ,
2313+ )
2314+ . unwrap ( ) ;
2315+ let data_size = write_parquet ( Arc :: clone ( & store) , "test.parquet" , batch) . await ;
2316+
2317+ let make_opener = |split| {
2318+ ParquetMorselizerBuilder :: new ( )
2319+ . with_store ( Arc :: clone ( & store) )
2320+ . with_schema ( Arc :: clone ( & schema) )
2321+ . with_projection_indices ( & [ 0 ] )
2322+ . with_split_row_groups_by_range ( split)
2323+ . build ( )
2324+ } ;
2325+
2326+ // With splitting: several ranges produce disjoint, contiguous slices
2327+ // that together cover the whole row group exactly once, in order.
2328+ let per_range = read_by_ranges ( & make_opener ( true ) , data_size, 4 ) . await ;
2329+ let non_empty = per_range. iter ( ) . filter ( |v| !v. is_empty ( ) ) . count ( ) ;
2330+ assert ! (
2331+ non_empty > 1 ,
2332+ "expected the row group to be split across ranges, got {non_empty} non-empty ranges"
2333+ ) ;
2334+ let all_values: Vec < i32 > = per_range. into_iter ( ) . flatten ( ) . collect ( ) ;
2335+ assert_eq ! ( all_values, ( 0 ..1000 ) . collect:: <Vec <i32 >>( ) ) ;
2336+
2337+ // Without splitting: the single row group is read entirely by the
2338+ // one range containing its first data page.
2339+ let per_range = read_by_ranges ( & make_opener ( false ) , data_size, 4 ) . await ;
2340+ let non_empty = per_range. iter ( ) . filter ( |v| !v. is_empty ( ) ) . count ( ) ;
2341+ assert_eq ! ( non_empty, 1 ) ;
2342+ let all_values: Vec < i32 > = per_range. into_iter ( ) . flatten ( ) . collect ( ) ;
2343+ assert_eq ! ( all_values, ( 0 ..1000 ) . collect:: <Vec <i32 >>( ) ) ;
2344+ }
2345+
2346+ /// Range splitting composes with predicate pushdown: rows are still read
2347+ /// exactly once and filters apply within each slice.
2348+ #[ tokio:: test]
2349+ async fn test_split_row_groups_by_range_with_predicate ( ) {
2350+ let store = Arc :: new ( InMemory :: new ( ) ) as Arc < dyn ObjectStore > ;
2351+
2352+ let schema: SchemaRef =
2353+ Arc :: new ( Schema :: new ( vec ! [ Field :: new( "a" , DataType :: Int32 , false ) ] ) ) ;
2354+ let batch = RecordBatch :: try_new (
2355+ Arc :: clone ( & schema) ,
2356+ vec ! [ Arc :: new( arrow:: array:: Int32Array :: from_iter_values(
2357+ 0 ..1000 ,
2358+ ) ) ] ,
2359+ )
2360+ . unwrap ( ) ;
2361+ let data_size = write_parquet ( Arc :: clone ( & store) , "test.parquet" , batch) . await ;
2362+
2363+ let expr = col ( "a" ) . gt_eq ( lit ( 100 ) ) . and ( col ( "a" ) . lt ( lit ( 900 ) ) ) ;
2364+ let predicate = logical2physical ( & expr, & schema) ;
2365+ let opener = ParquetMorselizerBuilder :: new ( )
2366+ . with_store ( Arc :: clone ( & store) )
2367+ . with_schema ( Arc :: clone ( & schema) )
2368+ . with_projection_indices ( & [ 0 ] )
2369+ . with_predicate ( predicate)
2370+ . with_pushdown_filters ( true )
2371+ . with_enable_page_index ( true )
2372+ . with_split_row_groups_by_range ( true )
2373+ . build ( ) ;
2374+
2375+ let per_range = read_by_ranges ( & opener, data_size, 4 ) . await ;
2376+ let all_values: Vec < i32 > = per_range. into_iter ( ) . flatten ( ) . collect ( ) ;
2377+ assert_eq ! ( all_values, ( 100 ..900 ) . collect:: <Vec <i32 >>( ) ) ;
2378+ }
2379+
22492380 #[ tokio:: test]
22502381 async fn test_prune_on_statistics ( ) {
22512382 let store = Arc :: new ( InMemory :: new ( ) ) as Arc < dyn ObjectStore > ;
0 commit comments