Skip to content

Commit 605d2fb

Browse files
committed
update comment
1 parent 62182a4 commit 605d2fb

3 files changed

Lines changed: 20 additions & 59 deletions

File tree

‎datafusion/core/tests/parquet/external_access_plan.rs‎

Lines changed: 3 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -221,9 +221,7 @@ async fn row_selection_extension_spanning_row_groups() {
221221

222222
#[tokio::test]
223223
async fn bitmap_row_selection_extension_spanning_row_groups() {
224-
// Keep the same cross-row-group pattern as the selector-backed test, but
225-
// provide it as a packed bitmap. The bitmap should remain packed through
226-
// ParquetAccessPlan and reach the parquet reader unchanged.
224+
// Repeat the cross-row-group case with a bitmap-backed selection.
227225
let parquet_metrics = TestFull {
228226
access_plan: None,
229227
row_selection: Some(ParquetRowSelection::new(RowSelection::from_boolean_buffer(
@@ -253,10 +251,7 @@ async fn bitmap_row_selection_extension_spanning_row_groups() {
253251

254252
#[tokio::test]
255253
async fn bitmap_row_selection_extension_with_predicate() {
256-
// The bitmap selects rows 2 (c) and 4 (null) in row group 0 and rows 5
257-
// and 6 (e, f) in row group 1. The predicate `utf8 = 'e'` prunes row
258-
// group 0 via statistics (its max value "d" sorts before "e"), so only
259-
// the bitmap-selected rows of row group 1 are returned.
254+
// Pruning removes row group 0, leaving the bitmap-selected rows in group 1.
260255
let parquet_metrics = TestFull {
261256
access_plan: None,
262257
row_selection: Some(ParquetRowSelection::new(RowSelection::from_boolean_buffer(
@@ -279,8 +274,7 @@ async fn bitmap_row_selection_extension_with_predicate() {
279274
.await
280275
.unwrap();
281276

282-
// Row group 0 was pruned by statistics even though the bitmap selected
283-
// rows in it.
277+
// Verify that statistics pruned row group 0.
284278
let row_groups_pruned_statistics = parquet_metrics
285279
.sum_by_name("row_groups_pruned_statistics")
286280
.unwrap();

‎datafusion/datasource-parquet/src/access_plan.rs‎

Lines changed: 14 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -162,8 +162,7 @@ impl RowGroupAccess {
162162
}
163163
}
164164

165-
/// Single-pass cursor that partitions a file-level [`RowSelection`] into row
166-
/// groups while preserving its backing representation.
165+
/// Splits an overall selection into row groups in one pass.
167166
enum OverallRowSelectionCursor {
168167
Mask { mask: BooleanBuffer, offset: usize },
169168
Selectors(SelectorRowSelectionCursor),
@@ -180,10 +179,7 @@ impl OverallRowSelectionCursor {
180179
}
181180
}
182181

183-
/// Take the selection for the next row group.
184-
///
185-
/// Returns `None` if the selection contains fewer than `row_group_rows`
186-
/// remaining rows.
182+
/// Takes the next row group, or `None` if too few rows remain.
187183
fn take_row_group(&mut self, row_group_rows: usize) -> Option<RowGroupAccess> {
188184
match self {
189185
Self::Mask { mask, offset } => {
@@ -210,7 +206,6 @@ impl OverallRowSelectionCursor {
210206
}
211207
}
212208

213-
/// Return the total number of rows in the original selection.
214209
fn total_rows(self) -> usize {
215210
match self {
216211
Self::Mask { mask, .. } => mask.len(),
@@ -219,7 +214,7 @@ impl OverallRowSelectionCursor {
219214
}
220215
}
221216

222-
/// Cursor over a selector-backed [`RowSelection`].
217+
/// Cursor for a selector-backed selection.
223218
///
224219
/// `take` returns the next selector fragment capped to the requested row count,
225220
/// splitting the current selector when it straddles a row group boundary.
@@ -330,17 +325,15 @@ impl RowGroupAccessBuilder {
330325
}
331326
}
332327

333-
/// Return the total number of rows described by `selection`.
328+
/// Returns the selection length.
334329
fn row_selection_len(selection: &RowSelection) -> usize {
335330
match selection.as_mask() {
336331
Some(mask) => mask.len(),
337332
None => selection.iter().map(|selector| selector.row_count).sum(),
338333
}
339334
}
340335

341-
/// Convert `selection` to a bitmap-backed [`RowSelection`].
342-
///
343-
/// No-op if the selection is already bitmap-backed.
336+
/// Converts to bitmap backing if needed.
344337
fn into_mask_backed(selection: RowSelection) -> RowSelection {
345338
match selection.as_mask() {
346339
Some(_) => selection,
@@ -408,8 +401,7 @@ impl ParquetAccessPlan {
408401
.map_while(|rg_meta| cursor.take_row_group(rg_meta.num_rows() as usize))
409402
.collect::<Vec<_>>();
410403

411-
// For selector-backed selections this computes the total while
412-
// consuming the stream above, rather than requiring a separate pass.
404+
// Count selector rows during traversal.
413405
let selection_rows = cursor.total_rows();
414406
if row_groups.len() != row_group_meta_data.len() || selection_rows != file_rows {
415407
return exec_err!(
@@ -477,11 +469,7 @@ impl ParquetAccessPlan {
477469
RowGroupAccess::Skip => RowGroupAccess::Skip,
478470
RowGroupAccess::Scan => RowGroupAccess::Selection(selection),
479471
RowGroupAccess::Selection(existing_selection) => {
480-
// `RowSelection::intersection` only stays bitmap-backed when
481-
// both sides are bitmap-backed, so promote the incoming
482-
// selection (e.g. from page index pruning) to match an
483-
// existing bitmap: the intersection is then a bitwise AND
484-
// instead of a selector merge.
472+
// Keep intersections bitmap-backed when the existing selection is.
485473
let selection = match existing_selection.as_mask() {
486474
Some(_) => into_mask_backed(selection),
487475
None => selection,
@@ -513,10 +501,7 @@ impl ParquetAccessPlan {
513501
/// is returned for *all* the rows in the row groups that are not skipped.
514502
/// Thus it includes a `Select` selection for any [`RowGroupAccess::Scan`].
515503
///
516-
/// If any [`RowGroupAccess::Selection`] is bitmap-backed
517-
/// ([`RowSelection::as_mask`] returns `Some`), the overall selection is
518-
/// bitmap-backed as well and any selector-backed selections are promoted
519-
/// to bitmaps; otherwise the overall selection is selector-backed.
504+
/// The result is bitmap-backed if any row-group selection is bitmap-backed.
520505
///
521506
/// # Errors
522507
///
@@ -605,11 +590,7 @@ impl ParquetAccessPlan {
605590
);
606591
}
607592

608-
// A bitmap-backed group selection signals the plan originated from a
609-
// bitmap (e.g. a mask-backed `ParquetRowSelection`), even if later
610-
// pruning such as the page index intersected some groups back to
611-
// selectors. Promote the selector-backed groups in that case so the
612-
// reader still receives a bitmap-backed selection.
593+
// Preserve bitmap backing across mixed row-group selections.
613594
let any_selection_mask_backed = self.row_groups.iter().any(|rg| {
614595
matches!(rg, RowGroupAccess::Selection(selection) if selection.as_mask().is_some())
615596
});
@@ -626,15 +607,12 @@ impl ParquetAccessPlan {
626607

627608
for (rg, rg_meta) in self.row_groups.into_iter().zip(row_group_meta_data) {
628609
match rg {
629-
// Skipped row groups are not passed to the parquet reader.
630610
RowGroupAccess::Skip => {}
631-
// Represent scanned row groups as all-set bitmap ranges.
632611
RowGroupAccess::Scan => {
633612
mask.append_n(rg_meta.num_rows() as usize, true)
634613
}
635614
RowGroupAccess::Selection(selection) => match selection.as_mask() {
636615
Some(buffer) => mask.append_buffer(buffer),
637-
// Promote selector-backed groups to bitmap ranges.
638616
None => {
639617
for selector in selection.iter() {
640618
mask.append_n(selector.row_count, !selector.skip);
@@ -646,8 +624,7 @@ impl ParquetAccessPlan {
646624

647625
RowSelection::from_boolean_buffer(mask.finish())
648626
} else {
649-
// Preserve the existing selector path when no row group
650-
// selection is bitmap-backed.
627+
// Keep selector backing when possible.
651628
self.row_groups
652629
.into_iter()
653630
.zip(row_group_meta_data.iter())
@@ -1046,10 +1023,7 @@ mod test {
10461023

10471024
#[test]
10481025
fn test_mixed_backings_promote_to_mask() {
1049-
// One bitmap-backed group next to a selector-backed group (e.g. one
1050-
// that page index pruning intersected): the overall selection is
1051-
// promoted to a bitmap so the caller-provided mask backing survives
1052-
// to the reader.
1026+
// Mixed backing produces a bitmap-backed overall selection.
10531027
let access_plan = ParquetAccessPlan::new(vec![
10541028
RowGroupAccess::Selection(RowSelection::from_boolean_buffer(
10551029
BooleanBuffer::from(vec![
@@ -1085,14 +1059,12 @@ mod test {
10851059
])),
10861060
)]);
10871061

1088-
// Intersect with a selector-backed selection, as produced by page
1089-
// index pruning.
1062+
// Simulate selector-backed page pruning.
10901063
access_plan.scan_selection(
10911064
0,
10921065
RowSelection::from(vec![RowSelector::select(5), RowSelector::skip(5)]),
10931066
);
1094-
// Intersect with an already mask-backed selection to exercise the
1095-
// no-op path in `into_mask_backed`.
1067+
// Exercise the already bitmap-backed path.
10961068
access_plan.scan_selection(
10971069
0,
10981070
RowSelection::from_boolean_buffer(BooleanBuffer::new_set(10)),
@@ -1258,8 +1230,7 @@ mod test {
12581230
])
12591231
);
12601232

1261-
// The fully skipped row group is omitted from the reader selection;
1262-
// scanned groups are represented as all-set bitmap ranges.
1233+
// Skipped groups are omitted; scanned groups become set bits.
12631234
let overall = access_plan
12641235
.into_overall_row_selection(&ROW_GROUP_METADATA)
12651236
.unwrap()

‎datafusion/datasource-parquet/src/sort.rs‎

Lines changed: 3 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -54,9 +54,7 @@ pub fn reverse_row_selection(
5454
) -> Result<RowSelection> {
5555
let rg_metadata = parquet_metadata.row_groups();
5656

57-
// Splitting a bitmap-backed selection slices its BooleanBuffer without
58-
// materializing selectors. Collecting the reversed slices concatenates
59-
// them back into a bitmap-backed selection.
57+
// Reverse bitmap slices without materializing selectors.
6058
if row_selection.as_mask().is_some() {
6159
let mut remaining = row_selection.clone();
6260
let mut row_group_selections = Vec::with_capacity(row_groups_to_scan.len());
@@ -66,8 +64,7 @@ pub fn reverse_row_selection(
6664
row_group_selections.push(remaining.split_off(num_rows));
6765
}
6866

69-
// `split_off` silently returns short slices when the selection runs
70-
// out early, which would misalign the reversed row group boundaries.
67+
// No rows should remain after splitting all row groups.
7168
debug_assert_eq!(
7269
remaining.row_count() + remaining.skipped_row_count(),
7370
0,
@@ -412,8 +409,7 @@ mod tests {
412409
assert_eq!(reversed_plan.row_group_indexes, vec![3, 2, 0]);
413410
let reversed_selection = reversed_plan.row_selection.unwrap();
414411

415-
// The fully scanned row group 3 becomes an all-set range at the
416-
// front of the reversed selection.
412+
// Fully scanned row group 3 becomes the leading set bits.
417413
let expected = BooleanBuffer::from(
418414
[true, true]
419415
.into_iter()

0 commit comments

Comments
 (0)