Skip to content

Commit 6d1a25a

Browse files
Dandandanclaude
andcommitted
perf: split row groups by file range for parallel single-row-group scans (morsel splitting)
When a parquet scan is restricted to a byte range that only partially overlaps a row group, read the proportional slice of the row group's rows (via a RowSelection) instead of assigning the whole row group to the one range containing its first data page. Since FileGroupPartitioner already tiles files into byte ranges, this lets all partitions decode disjoint slices of the same row group in parallel, parallelizing scans of files with fewer row groups than partitions (e.g. a single large row group). Row boundaries are computed with identical integer arithmetic on both sides of each range boundary, so every row is read exactly once. The offset index is now loaded whenever the access plan contains row selections so each partition fetches and decodes only the pages covering its slice. Controlled by `datafusion.execution.parquet.split_row_groups_by_range` (default: true). TPC-DS SF=1 with one row group per file: 2.2x faster overall (16.8s -> 7.5s), 82/99 queries faster, up to 5.3x. TPC-H SF=1 (multi-row-group files): 1.16x faster overall, no significant regressions. Co-Authored-By: Claude Fable 5 <[email protected]>
1 parent 3bb9314 commit 6d1a25a

15 files changed

Lines changed: 476 additions & 3 deletions

File tree

‎datafusion/common/src/config.rs‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1106,6 +1106,16 @@ config_namespace! {
11061106
/// the parquet file
11071107
pub pruning: bool, default = true
11081108

1109+
/// (reading) If true, a scan restricted to a byte range of a parquet
1110+
/// file reads the proportional slice of rows from row groups that
1111+
/// partially overlap the range, so multiple partitions can decode
1112+
/// different rows of the same row group in parallel ("morsel"
1113+
/// splitting). This parallelizes reading of files with fewer row
1114+
/// groups than partitions, e.g. files with a single large row group.
1115+
/// If false, each row group is read entirely by the single partition
1116+
/// whose range contains the offset of its first data page.
1117+
pub split_row_groups_by_range: bool, default = true
1118+
11091119
/// (reading) If true, the parquet reader skip the optional embedded metadata that may be in
11101120
/// the file Schema. This setting can help avoid schema conflicts when querying
11111121
/// multiple parquet files with schemas containing compatible types but different metadata

‎datafusion/common/src/file_options/parquet_writer.rs‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -238,6 +238,7 @@ impl ParquetOptions {
238238
pushdown_filters: _,
239239
reorder_filters: _,
240240
force_filter_selections: _, // not used for writer props
241+
split_row_groups_by_range: _, // not used for writer props
241242
allow_single_file_parallelism: _,
242243
maximum_parallel_row_group_writers: _,
243244
maximum_buffered_record_batches_per_stream: _,
@@ -494,6 +495,7 @@ mod tests {
494495
pushdown_filters: defaults.pushdown_filters,
495496
reorder_filters: defaults.reorder_filters,
496497
force_filter_selections: defaults.force_filter_selections,
498+
split_row_groups_by_range: defaults.split_row_groups_by_range,
497499
allow_single_file_parallelism: defaults.allow_single_file_parallelism,
498500
maximum_parallel_row_group_writers: defaults
499501
.maximum_parallel_row_group_writers,
@@ -613,6 +615,8 @@ mod tests {
613615
pushdown_filters: global_options_defaults.pushdown_filters,
614616
reorder_filters: global_options_defaults.reorder_filters,
615617
force_filter_selections: global_options_defaults.force_filter_selections,
618+
split_row_groups_by_range: global_options_defaults
619+
.split_row_groups_by_range,
616620
allow_single_file_parallelism: global_options_defaults
617621
.allow_single_file_parallelism,
618622
maximum_parallel_row_group_writers: global_options_defaults

‎datafusion/datasource-parquet/src/access_plan.rs‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -360,6 +360,13 @@ impl ParquetAccessPlan {
360360
self.row_groups[idx].should_scan()
361361
}
362362

363+
/// Return true if any row group is accessed via a [`RowGroupAccess::Selection`]
364+
pub fn has_selection(&self) -> bool {
365+
self.row_groups
366+
.iter()
367+
.any(|rg| matches!(rg, RowGroupAccess::Selection(_)))
368+
}
369+
363370
/// Marks the i-th row group as fully matched.
364371
///
365372
/// Fully matched row groups are still read according to their

‎datafusion/datasource-parquet/src/opener/mod.rs‎

Lines changed: 134 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -270,6 +270,12 @@ pub(super) struct ParquetMorselizer {
270270
pub enable_bloom_filter: bool,
271271
/// Should row group pruning be applied
272272
pub enable_row_group_stats_pruning: bool,
273+
/// Should a file range that partially overlaps a row group read the
274+
/// proportional slice of the row group's rows (via a `RowSelection`)
275+
/// instead of assigning the entire row group to the range containing its
276+
/// first data page. This allows partitions to decode disjoint parts of
277+
/// the same row group in parallel.
278+
pub split_row_groups_by_range: bool,
273279
/// Coerce INT96 timestamps to specific TimeUnit
274280
pub coerce_int96: Option<TimeUnit>,
275281
/// Optional timezone applied to INT96-coerced timestamps. When `Some`, the
@@ -444,6 +450,7 @@ struct PreparedParquetOpen {
444450
enable_page_index: bool,
445451
enable_bloom_filter: bool,
446452
enable_row_group_stats_pruning: bool,
453+
split_row_groups_by_range: bool,
447454
limit: Option<usize>,
448455
coerce_int96: Option<TimeUnit>,
449456
coerce_int96_tz: Option<Arc<str>>,
@@ -843,6 +850,7 @@ impl ParquetMorselizer {
843850
enable_page_index: self.enable_page_index,
844851
enable_bloom_filter: self.enable_bloom_filter,
845852
enable_row_group_stats_pruning: self.enable_row_group_stats_pruning,
853+
split_row_groups_by_range: self.split_row_groups_by_range,
846854
limit: self.limit,
847855
coerce_int96: self.coerce_int96,
848856
coerce_int96_tz: self.coerce_int96_tz.clone(),
@@ -1093,7 +1101,14 @@ impl FiltersPreparedParquetOpen {
10931101

10941102
// If there is a range restricting what parts of the file to read
10951103
if let Some(range) = prepared.file_range.as_ref() {
1096-
row_groups.prune_by_range(rg_metadata, range);
1104+
if prepared.split_row_groups_by_range {
1105+
// Read the proportional slice of rows from row groups that
1106+
// partially overlap the range, so partitions can decode
1107+
// disjoint parts of the same row group in parallel.
1108+
row_groups.split_by_range(rg_metadata, range);
1109+
} else {
1110+
row_groups.prune_by_range(rg_metadata, range);
1111+
}
10971112
}
10981113

10991114
// If there is a predicate that can be evaluated against the metadata
@@ -1648,12 +1663,19 @@ fn should_load_page_index(
16481663
page_pruning_predicate: Option<&Arc<PagePruningAccessPlanFilter>>,
16491664
row_groups: &RowGroupAccessPlanFilter,
16501665
) -> bool {
1651-
page_pruning_predicate.is_some_and(|_| {
1666+
// Load to prune pages with the predicate, unless every surviving row
1667+
// group is already known to fully match it.
1668+
let for_page_pruning = page_pruning_predicate.is_some_and(|_| {
16521669
let fully_matched = row_groups.is_fully_matched();
16531670
row_groups
16541671
.row_group_indexes()
16551672
.any(|idx| !fully_matched[idx])
1656-
})
1673+
});
1674+
1675+
// Row selections (e.g. from `split_by_range`) need the offset index so
1676+
// the reader fetches and decodes only the pages covering the selected
1677+
// rows rather than entire column chunks.
1678+
for_page_pruning || row_groups.has_selection()
16571679
}
16581680

16591681
/// Returns a `ArrowReaderMetadata` with the page index loaded, loading
@@ -1749,6 +1771,7 @@ mod test {
17491771
enable_page_index: bool,
17501772
enable_bloom_filter: bool,
17511773
enable_row_group_stats_pruning: bool,
1774+
split_row_groups_by_range: bool,
17521775
coerce_int96: Option<TimeUnit>,
17531776
max_predicate_cache_size: Option<usize>,
17541777
reverse_row_groups: bool,
@@ -1857,6 +1880,7 @@ mod test {
18571880
enable_page_index: false,
18581881
enable_bloom_filter: false,
18591882
enable_row_group_stats_pruning: false,
1883+
split_row_groups_by_range: false,
18601884
coerce_int96: None,
18611885
max_predicate_cache_size: None,
18621886
reverse_row_groups: false,
@@ -1928,6 +1952,12 @@ mod test {
19281952
self
19291953
}
19301954

1955+
/// Enable splitting row groups that partially overlap a file range.
1956+
fn with_split_row_groups_by_range(mut self, enable: bool) -> Self {
1957+
self.split_row_groups_by_range = enable;
1958+
self
1959+
}
1960+
19311961
/// Enable page index.
19321962
fn with_enable_page_index(mut self, enable: bool) -> Self {
19331963
self.enable_page_index = enable;
@@ -2025,6 +2055,7 @@ mod test {
20252055
enable_page_index: self.enable_page_index,
20262056
enable_bloom_filter: self.enable_bloom_filter,
20272057
enable_row_group_stats_pruning: self.enable_row_group_stats_pruning,
2058+
split_row_groups_by_range: self.split_row_groups_by_range,
20282059
coerce_int96: self.coerce_int96,
20292060
// End-to-end coercion behavior (including timezone) is
20302061
// covered by parquet.slt. No opener-level test currently
@@ -2246,6 +2277,106 @@ mod test {
22462277
))
22472278
}
22482279

2280+
/// Read `test.parquet` through `num_ranges` byte ranges tiling the file
2281+
/// and return the values produced by each range.
2282+
async fn read_by_ranges(
2283+
opener: &ParquetMorselizer,
2284+
data_size: usize,
2285+
num_ranges: i64,
2286+
) -> Vec<Vec<i32>> {
2287+
let chunk = (data_size as i64 + num_ranges - 1) / num_ranges;
2288+
let mut results = vec![];
2289+
for i in 0..num_ranges {
2290+
let file = PartitionedFile::new("test.parquet".to_string(), data_size as u64)
2291+
.with_range(i * chunk, ((i + 1) * chunk).min(data_size as i64));
2292+
let stream = open_file(opener, file).await.unwrap();
2293+
results.push(collect_int32_values(stream).await);
2294+
}
2295+
results
2296+
}
2297+
2298+
/// A file with a single row group scanned through byte ranges: with
2299+
/// `split_row_groups_by_range` each range decodes a disjoint slice of the
2300+
/// row group (so multiple partitions share the work), and every row is
2301+
/// read exactly once.
2302+
#[tokio::test]
2303+
async fn test_split_row_groups_by_range() {
2304+
let store = Arc::new(InMemory::new()) as Arc<dyn ObjectStore>;
2305+
2306+
let schema: SchemaRef =
2307+
Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)]));
2308+
let batch = RecordBatch::try_new(
2309+
Arc::clone(&schema),
2310+
vec![Arc::new(arrow::array::Int32Array::from_iter_values(
2311+
0..1000,
2312+
))],
2313+
)
2314+
.unwrap();
2315+
let data_size = write_parquet(Arc::clone(&store), "test.parquet", batch).await;
2316+
2317+
let make_opener = |split| {
2318+
ParquetMorselizerBuilder::new()
2319+
.with_store(Arc::clone(&store))
2320+
.with_schema(Arc::clone(&schema))
2321+
.with_projection_indices(&[0])
2322+
.with_split_row_groups_by_range(split)
2323+
.build()
2324+
};
2325+
2326+
// With splitting: several ranges produce disjoint, contiguous slices
2327+
// that together cover the whole row group exactly once, in order.
2328+
let per_range = read_by_ranges(&make_opener(true), data_size, 4).await;
2329+
let non_empty = per_range.iter().filter(|v| !v.is_empty()).count();
2330+
assert!(
2331+
non_empty > 1,
2332+
"expected the row group to be split across ranges, got {non_empty} non-empty ranges"
2333+
);
2334+
let all_values: Vec<i32> = per_range.into_iter().flatten().collect();
2335+
assert_eq!(all_values, (0..1000).collect::<Vec<i32>>());
2336+
2337+
// Without splitting: the single row group is read entirely by the
2338+
// one range containing its first data page.
2339+
let per_range = read_by_ranges(&make_opener(false), data_size, 4).await;
2340+
let non_empty = per_range.iter().filter(|v| !v.is_empty()).count();
2341+
assert_eq!(non_empty, 1);
2342+
let all_values: Vec<i32> = per_range.into_iter().flatten().collect();
2343+
assert_eq!(all_values, (0..1000).collect::<Vec<i32>>());
2344+
}
2345+
2346+
/// Range splitting composes with predicate pushdown: rows are still read
2347+
/// exactly once and filters apply within each slice.
2348+
#[tokio::test]
2349+
async fn test_split_row_groups_by_range_with_predicate() {
2350+
let store = Arc::new(InMemory::new()) as Arc<dyn ObjectStore>;
2351+
2352+
let schema: SchemaRef =
2353+
Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)]));
2354+
let batch = RecordBatch::try_new(
2355+
Arc::clone(&schema),
2356+
vec![Arc::new(arrow::array::Int32Array::from_iter_values(
2357+
0..1000,
2358+
))],
2359+
)
2360+
.unwrap();
2361+
let data_size = write_parquet(Arc::clone(&store), "test.parquet", batch).await;
2362+
2363+
let expr = col("a").gt_eq(lit(100)).and(col("a").lt(lit(900)));
2364+
let predicate = logical2physical(&expr, &schema);
2365+
let opener = ParquetMorselizerBuilder::new()
2366+
.with_store(Arc::clone(&store))
2367+
.with_schema(Arc::clone(&schema))
2368+
.with_projection_indices(&[0])
2369+
.with_predicate(predicate)
2370+
.with_pushdown_filters(true)
2371+
.with_enable_page_index(true)
2372+
.with_split_row_groups_by_range(true)
2373+
.build();
2374+
2375+
let per_range = read_by_ranges(&opener, data_size, 4).await;
2376+
let all_values: Vec<i32> = per_range.into_iter().flatten().collect();
2377+
assert_eq!(all_values, (100..900).collect::<Vec<i32>>());
2378+
}
2379+
22492380
#[tokio::test]
22502381
async fn test_prune_on_statistics() {
22512382
let store = Arc::new(InMemory::new()) as Arc<dyn ObjectStore>;

0 commit comments

Comments
 (0)