Skip to content

Commit 3017761

Browse files
committed
perf: do not build pruning predicates if no page index
1 parent ec00112 commit 3017761

1 file changed

Lines changed: 130 additions & 65 deletions

File tree

‎datafusion/datasource-parquet/src/opener.rs‎

Lines changed: 130 additions & 65 deletions
Original file line numberDiff line numberDiff line change
@@ -926,24 +926,30 @@ impl MetadataLoadedParquetOpen {
926926
&prepared.predicate_creation_errors,
927927
);
928928

929-
// Only build page pruning predicate if page index is enabled
930-
let page_pruning_predicate = if prepared.enable_page_index {
931-
prepared.predicate.as_ref().and_then(|predicate| {
932-
let p = build_page_pruning_predicate(predicate, &physical_file_schema);
933-
(p.filter_number() > 0).then_some(p)
934-
})
935-
} else {
936-
None
937-
};
938-
939929
Ok(FiltersPreparedParquetOpen {
940930
loaded: MetadataLoadedParquetOpen {
941931
prepared,
942932
reader_metadata,
943933
options,
944934
},
945935
pruning_predicate,
946-
page_pruning_predicate,
936+
page_pruning_predicate: None,
937+
})
938+
}
939+
}
940+
941+
impl MetadataLoadedParquetOpen {
942+
fn has_page_index(&self) -> bool {
943+
self.reader_metadata.metadata().column_index().is_some()
944+
&& self.reader_metadata.metadata().offset_index().is_some()
945+
}
946+
}
947+
948+
impl PreparedParquetOpen {
949+
fn build_page_pruning_predicate(&self) -> Option<Arc<PagePruningAccessPlanFilter>> {
950+
self.predicate.as_ref().and_then(|predicate| {
951+
let p = build_page_pruning_predicate(predicate, &self.physical_file_schema);
952+
(p.filter_number() > 0).then_some(p)
947953
})
948954
}
949955
}
@@ -953,9 +959,9 @@ impl FiltersPreparedParquetOpen {
953959
async fn load_page_index(mut self) -> Result<Self> {
954960
// The page index is not stored inline in the parquet footer so the
955961
// metadata load above may not have read the page index structures yet.
956-
// If we need them for reading and they aren't yet loaded, we need to
962+
// If we need them for reading, and they aren't yet loaded, we need to
957963
// load them now.
958-
if self.page_pruning_predicate.is_some() {
964+
if self.loaded.prepared.enable_page_index {
959965
self.loaded.reader_metadata = load_page_index(
960966
self.loaded.reader_metadata,
961967
&mut self.loaded.prepared.async_file_reader,
@@ -965,6 +971,13 @@ impl FiltersPreparedParquetOpen {
965971
.with_page_index_policy(PageIndexPolicy::Optional),
966972
)
967973
.await?;
974+
975+
// Since creating a page pruning predicate can be non trivial
976+
// only do it when we are sure we have enough metadata to use it
977+
if self.loaded.has_page_index() {
978+
self.page_pruning_predicate =
979+
self.loaded.prepared.build_page_pruning_predicate();
980+
}
968981
}
969982

970983
Ok(self)
@@ -2708,70 +2721,122 @@ mod test {
27082721
);
27092722
}
27102723

2711-
/// Test that page pruning predicates are only built and applied when `enable_page_index` is true.
2712-
///
2713-
/// The file has a single row group with 10 pages (10 rows each, values 1..100).
2714-
/// With page index enabled, pages whose max value <= 90 are pruned, returning only
2715-
/// the last page (rows 91..100). With page index disabled, all 100 rows are returned
2716-
/// since neither pushdown nor row-group pruning is active.
2724+
/// Test that page pruning occurs when enabled and page index metadata is available.
27172725
#[tokio::test]
27182726
async fn test_page_pruning_predicate_respects_enable_page_index() {
2719-
use parquet::file::properties::WriterProperties;
2727+
let fixture = PagePruningFixture::new("test-with-page-index.parquet")
2728+
.with_write_page_index(true);
2729+
let rows = fixture.test(true).await;
27202730

2721-
let store = Arc::new(InMemory::new()) as Arc<dyn ObjectStore>;
2731+
assert_eq!(rows, 10, "page index should prune 9 of 10 pages");
2732+
}
27222733

2723-
// 100 rows with values 1..=100, written as a single row group with 10 rows per page
2724-
let values: Vec<i32> = (1..=100).collect();
2725-
let batch = record_batch!((
2726-
"a",
2727-
Int32,
2728-
values.iter().map(|v| Some(*v)).collect::<Vec<_>>()
2729-
))
2730-
.unwrap();
2731-
let props = WriterProperties::builder()
2732-
.set_data_page_row_count_limit(10)
2733-
.set_write_batch_size(10)
2734-
.build();
2735-
let schema = batch.schema();
2736-
let data_size = write_parquet_batches(
2737-
Arc::clone(&store),
2738-
"test.parquet",
2739-
vec![batch],
2740-
Some(props),
2741-
)
2742-
.await;
2734+
/// Test that page pruning does not occur when page index pruning is disabled.
2735+
#[tokio::test]
2736+
async fn test_page_pruning_predicate_disabled_without_option() {
2737+
let fixture = PagePruningFixture::new("test-page-index-disabled.parquet")
2738+
.with_write_page_index(true);
2739+
let rows = fixture.test(false).await;
27432740

2744-
let file = PartitionedFile::new("test.parquet".to_string(), data_size as u64);
2741+
assert_eq!(rows, 100, "without page index all rows are returned");
2742+
}
27452743

2746-
// predicate: a > 90 — should allow page index to prune first 9 pages
2747-
let predicate = logical2physical(&col("a").gt(lit(90i32)), &schema);
2744+
/// Test that page pruning does not occur when page index metadata is unavailable.
2745+
#[tokio::test]
2746+
async fn test_page_pruning_predicate_disabled_without_page_index() {
2747+
let fixture = PagePruningFixture::new("test-without-page-index.parquet")
2748+
.with_write_page_index(false);
2749+
let rows = fixture.test(true).await;
27482750

2749-
let make_opener = |enable_page_index| {
2751+
assert_eq!(
2752+
rows, 100,
2753+
"when page index metadata is unavailable all rows are returned"
2754+
);
2755+
}
2756+
2757+
struct PagePruningFixture {
2758+
store: Arc<dyn ObjectStore>,
2759+
schema: SchemaRef,
2760+
path: String,
2761+
predicate: Arc<dyn PhysicalExpr>,
2762+
write_page_index: bool,
2763+
}
2764+
2765+
impl PagePruningFixture {
2766+
// Creates a page pruning fixture with default settings.
2767+
fn new(path: &str) -> Self {
2768+
let store = Arc::new(InMemory::new()) as Arc<dyn ObjectStore>;
2769+
let values: Vec<i32> = (1..=100).collect();
2770+
let batch = record_batch!((
2771+
"a",
2772+
Int32,
2773+
values.iter().map(|v| Some(*v)).collect::<Vec<_>>()
2774+
))
2775+
.unwrap();
2776+
let schema = batch.schema();
2777+
let predicate = logical2physical(&col("a").gt(lit(90i32)), &schema);
2778+
2779+
Self {
2780+
store,
2781+
schema,
2782+
path: path.to_string(),
2783+
predicate,
2784+
write_page_index: false,
2785+
}
2786+
}
2787+
2788+
// Configures whether the generated parquet file should include page indexes.
2789+
fn with_write_page_index(mut self, write_page_index: bool) -> Self {
2790+
self.write_page_index = write_page_index;
2791+
self
2792+
}
2793+
2794+
// Writes the parquet test file and returns the matching partitioned file.
2795+
async fn create_file(&self) -> PartitionedFile {
2796+
use parquet::file::properties::WriterProperties;
2797+
2798+
let values: Vec<i32> = (1..=100).collect();
2799+
let batch = record_batch!((
2800+
"a",
2801+
Int32,
2802+
values.iter().map(|v| Some(*v)).collect::<Vec<_>>()
2803+
))
2804+
.unwrap();
2805+
let props = WriterProperties::builder()
2806+
.set_data_page_row_count_limit(10)
2807+
.set_write_batch_size(10)
2808+
.set_offset_index_disabled(!self.write_page_index)
2809+
.build();
2810+
let size = write_parquet_batches(
2811+
Arc::clone(&self.store),
2812+
&self.path,
2813+
vec![batch],
2814+
Some(props),
2815+
)
2816+
.await;
2817+
2818+
PartitionedFile::new(self.path.clone(), size as u64)
2819+
}
2820+
2821+
// Builds a parquet opener configured so page pruning is the only pruning path.
2822+
fn opener(&self, enable_page_index: bool) -> ParquetOpener {
27502823
ParquetOpenerBuilder::new()
2751-
.with_store(Arc::clone(&store))
2752-
.with_schema(Arc::clone(&schema))
2753-
.with_predicate(Arc::clone(&predicate))
2824+
.with_store(Arc::clone(&self.store))
2825+
.with_schema(Arc::clone(&self.schema))
2826+
.with_predicate(Arc::clone(&self.predicate))
27542827
.with_enable_page_index(enable_page_index)
2755-
// disable pushdown and row-group pruning so the only pruning path is page index
27562828
.with_pushdown_filters(false)
27572829
.with_row_group_stats_pruning(false)
27582830
.build()
2759-
};
2760-
let (_, rows_with_page_index) = count_batches_and_rows(
2761-
make_opener(true).open(file.clone()).unwrap().await.unwrap(),
2762-
)
2763-
.await;
2764-
let (_, rows_without_page_index) =
2765-
count_batches_and_rows(make_opener(false).open(file).unwrap().await.unwrap())
2766-
.await;
2831+
}
27672832

2768-
assert_eq!(
2769-
rows_with_page_index, 10,
2770-
"page index should prune 9 of 10 pages"
2771-
);
2772-
assert_eq!(
2773-
rows_without_page_index, 100,
2774-
"without page index all rows are returned"
2775-
);
2833+
// Writes the test file, opens it, and returns only the produced row count.
2834+
async fn test(&self, enable_page_index: bool) -> usize {
2835+
let file = self.create_file().await;
2836+
let opener = self.opener(enable_page_index);
2837+
let (_, rows) =
2838+
count_batches_and_rows(opener.open(file).unwrap().await.unwrap()).await;
2839+
rows
2840+
}
27762841
}
27772842
}

0 commit comments

Comments
 (0)