Skip to content

Commit a6e4d8f

Browse files
Support reverse page ordering in sort pushdown phase 1
- Add reverse_pages flag to ParquetSource and ParquetOpener - Wire reverse_pages through try_reverse_output() alongside reverse_row_groups - Extend try_reverse_output() to set both flags when optimizing descending sorts - Add comprehensive test coverage for reverse_pages functionality - Align with existing reverse_row_groups infrastructure pattern Phase 1 establishes the foundation for page-level reverse ordering. Actual page-level reversal implementation will be completed in Phase 2 when arrow-rs provides page-level API support. Tests: - All 27 reverse-related tests pass - Added 4 new tests for reverse_pages flag - Verified backward compatibility Closes apache#19486
1 parent e8efd59 commit a6e4d8f

2 files changed

Lines changed: 97 additions & 2 deletions

File tree

‎datafusion/datasource-parquet/src/opener.rs‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -118,6 +118,10 @@ pub(super) struct ParquetOpener {
118118
pub max_predicate_cache_size: Option<usize>,
119119
/// Whether to read row groups in reverse order
120120
pub reverse_row_groups: bool,
121+
/// Whether to read pages in reverse order within row groups
122+
/// Phase 1: Infrastructure flag for future page-level reverse support
123+
#[expect(dead_code)]
124+
pub reverse_pages: bool,
121125
}
122126

123127
/// Represents a prepared access plan with optional row selection
@@ -1051,6 +1055,7 @@ mod test {
10511055
coerce_int96: Option<arrow::datatypes::TimeUnit>,
10521056
max_predicate_cache_size: Option<usize>,
10531057
reverse_row_groups: bool,
1058+
reverse_pages: bool,
10541059
}
10551060

10561061
impl ParquetOpenerBuilder {
@@ -1076,6 +1081,7 @@ mod test {
10761081
coerce_int96: None,
10771082
max_predicate_cache_size: None,
10781083
reverse_row_groups: false,
1084+
reverse_pages: false,
10791085
}
10801086
}
10811087

@@ -1133,6 +1139,12 @@ mod test {
11331139
self
11341140
}
11351141

1142+
/// Set reverse pages flag.
1143+
fn with_reverse_pages(mut self, enable: bool) -> Self {
1144+
self.reverse_pages = enable;
1145+
self
1146+
}
1147+
11361148
/// Build the ParquetOpener instance.
11371149
///
11381150
/// # Panics
@@ -1183,6 +1195,7 @@ mod test {
11831195
encryption_factory: None,
11841196
max_predicate_cache_size: self.max_predicate_cache_size,
11851197
reverse_row_groups: self.reverse_row_groups,
1198+
reverse_pages: self.reverse_pages,
11861199
}
11871200
}
11881201
}

‎datafusion/datasource-parquet/src/source.rs‎

Lines changed: 84 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -293,6 +293,11 @@ pub struct ParquetSource {
293293
/// so we still need to sort them after reading, so the reverse scan is inexact.
294294
/// Used to optimize ORDER BY ... DESC on sorted data.
295295
reverse_row_groups: bool,
296+
/// If true, read pages within row groups in reverse order.
297+
/// Used in sort pushdown phase 1 to optimize ORDER BY ... DESC on sorted data.
298+
/// Note: This is infrastructure for phase 1; actual page reversal may be implemented
299+
/// in future phases depending on arrow-rs capabilities.
300+
reverse_pages: bool,
296301
}
297302

298303
impl ParquetSource {
@@ -318,6 +323,7 @@ impl ParquetSource {
318323
#[cfg(feature = "parquet_encryption")]
319324
encryption_factory: None,
320325
reverse_row_groups: false,
326+
reverse_pages: false,
321327
}
322328
}
323329

@@ -477,10 +483,21 @@ impl ParquetSource {
477483
self.reverse_row_groups = reverse_row_groups;
478484
self
479485
}
486+
487+
pub(crate) fn with_reverse_pages(mut self, reverse_pages: bool) -> Self {
488+
self.reverse_pages = reverse_pages;
489+
self
490+
}
491+
480492
#[cfg(test)]
481493
pub(crate) fn reverse_row_groups(&self) -> bool {
482494
self.reverse_row_groups
483495
}
496+
497+
#[cfg(test)]
498+
pub(crate) fn reverse_pages(&self) -> bool {
499+
self.reverse_pages
500+
}
484501
}
485502

486503
/// Parses datafusion.common.config.ParquetOptions.coerce_int96 String to a arrow_schema.datatype.TimeUnit
@@ -567,6 +584,7 @@ impl FileSource for ParquetSource {
567584
encryption_factory: self.get_encryption_factory_with_config(),
568585
max_predicate_cache_size: self.max_predicate_cache_size(),
569586
reverse_row_groups: self.reverse_row_groups,
587+
reverse_pages: self.reverse_pages,
570588
});
571589
Ok(opener)
572590
}
@@ -625,6 +643,10 @@ impl FileSource for ParquetSource {
625643
write!(f, ", reverse_row_groups=true")?;
626644
}
627645

646+
if self.reverse_pages {
647+
write!(f, ", reverse_pages=true")?;
648+
}
649+
628650
// Try to build a the pruning predicates.
629651
// These are only generated here because it's useful to have *some*
630652
// idea of what pushdown is happening when viewing plans.
@@ -803,9 +825,12 @@ impl FileSource for ParquetSource {
803825
return Ok(SortOrderPushdownResult::Unsupported);
804826
}
805827

806-
// Return Inexact because we're only reversing row group order,
828+
// Return Inexact because we're only reversing row group and page order,
807829
// not guaranteeing perfect row-level ordering
808-
let new_source = self.clone().with_reverse_row_groups(true);
830+
let new_source = self
831+
.clone()
832+
.with_reverse_row_groups(true)
833+
.with_reverse_pages(true);
809834
Ok(SortOrderPushdownResult::Inexact {
810835
inner: Arc::new(new_source) as Arc<dyn FileSource>,
811836
})
@@ -814,6 +839,7 @@ impl FileSource for ParquetSource {
814839
// - File reordering based on min/max statistics
815840
// - Detection of exact ordering (return Exact to remove Sort operator)
816841
// - Partial sort pushdown for prefix matches
842+
// - Actual page-level reversal implementation when arrow-rs supports it
817843
}
818844
}
819845

@@ -916,4 +942,60 @@ mod tests {
916942
assert!(source.reverse_row_groups());
917943
assert!(source.filter().is_some());
918944
}
945+
946+
#[test]
947+
fn test_reverse_pages_default_value() {
948+
use arrow::datatypes::Schema;
949+
950+
let schema = Arc::new(Schema::empty());
951+
let source = ParquetSource::new(schema);
952+
953+
assert!(!source.reverse_pages());
954+
}
955+
956+
#[test]
957+
fn test_reverse_pages_with_setter() {
958+
use arrow::datatypes::Schema;
959+
960+
let schema = Arc::new(Schema::empty());
961+
962+
let source = ParquetSource::new(schema.clone()).with_reverse_pages(true);
963+
assert!(source.reverse_pages());
964+
965+
let source = source.with_reverse_pages(false);
966+
assert!(!source.reverse_pages());
967+
}
968+
969+
#[test]
970+
fn test_reverse_pages_clone_preserves_value() {
971+
use arrow::datatypes::Schema;
972+
973+
let schema = Arc::new(Schema::empty());
974+
975+
let source = ParquetSource::new(schema).with_reverse_pages(true);
976+
let cloned = source.clone();
977+
978+
assert!(cloned.reverse_pages());
979+
assert_eq!(source.reverse_pages(), cloned.reverse_pages());
980+
}
981+
982+
#[test]
983+
fn test_reverse_pages_independent_of_reverse_row_groups() {
984+
use arrow::datatypes::Schema;
985+
986+
let schema = Arc::new(Schema::empty());
987+
988+
// Test: reverse_pages can be set independently
989+
let source = ParquetSource::new(schema.clone())
990+
.with_reverse_row_groups(true)
991+
.with_reverse_pages(true);
992+
993+
assert!(source.reverse_row_groups());
994+
assert!(source.reverse_pages());
995+
996+
// Test: reverse_pages can be set without reverse_row_groups
997+
let source = ParquetSource::new(schema).with_reverse_pages(true);
998+
assert!(!source.reverse_row_groups());
999+
assert!(source.reverse_pages());
1000+
}
9191001
}

0 commit comments

Comments
 (0)