Skip to content

Commit f05869b

Browse files
authored
refactor(pruning): deprecate PruningPredicate::try_new (apache#24129)
## Which issue does this PR close? - Closes apache#24128. ## Rationale for this change `PruningPredicateBuilder` is the extensible entry point for constructing pruning predicates, but the construction logic still lived behind `PruningPredicate::try_new`. Keeping the logic in the builder provides one place to add future construction options and guides callers toward the extensible API. ## What changes are included in this PR? - Move pruning predicate construction into `PruningPredicateBuilder::try_build`. - Deprecate `PruningPredicate::try_new` and retain it as a compatibility wrapper using the builder defaults. - Migrate DataFusion call sites and examples to `PruningPredicateBuilder`. - Add a regression test confirming the deprecated constructor and builder produce equivalent predicates. ## Are these changes tested? Yes. The following checks pass: - `cargo fmt --all -- --check` - `cargo test -p datafusion-pruning` - `cargo test -p datafusion-datasource-parquet` - `cargo check -p datafusion-examples --examples` - `cargo clippy -p datafusion-pruning -p datafusion-datasource-parquet -p datafusion-examples --all-targets --all-features -- -D warnings` - `cargo test --profile=ci --test sqllogictests` - `cargo test -p datafusion` - `cargo test -p datafusion-cli` - `RUSTDOCFLAGS="-D warnings" cargo doc -p datafusion-pruning --no-deps` The workspace-wide clippy command is currently blocked on `main` by an existing `clippy::uninlined_format_args` diagnostic in `datafusion/proto-common/src/generated/pbjson.rs`. The same failure reproduces from a clean checkout of the base commit; all modified packages pass strict clippy checks. ## Are there any user-facing changes? Yes. `PruningPredicate::try_new` is deprecated as of 55.0.0. It remains available as a compatibility wrapper with unchanged behavior. New callers should construct predicates with `PruningPredicateBuilder`.
1 parent 491de04 commit f05869b

8 files changed

Lines changed: 174 additions & 103 deletions

File tree

‎datafusion-examples/examples/data_io/parquet_advanced_index.rs‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@ use datafusion::parquet::file::properties::{EnabledStatistics, WriterProperties}
4747
use datafusion::parquet::schema::types::ColumnPath;
4848
use datafusion::physical_expr::PhysicalExpr;
4949
use datafusion::physical_expr::utils::{Guarantee, LiteralGuarantee};
50-
use datafusion::physical_optimizer::pruning::PruningPredicate;
50+
use datafusion::physical_optimizer::pruning::PruningPredicateBuilder;
5151
use datafusion::physical_plan::ExecutionPlan;
5252
use datafusion::physical_plan::metrics::ExecutionPlanMetricsSet;
5353
use datafusion::prelude::*;
@@ -155,6 +155,7 @@ use url::Url;
155155
/// ```
156156
///
157157
/// [`ListingTable`]: datafusion::datasource::listing::ListingTable
158+
/// [`PruningPredicate`]: datafusion::physical_optimizer::pruning::PruningPredicate
158159
/// [Page Index](https://github.com/apache/parquet-format/blob/master/PageIndex.md)
159160
pub async fn parquet_advanced_index() -> Result<()> {
160161
// the object store is used to read the parquet files (in this case, it is
@@ -300,8 +301,9 @@ impl IndexTableProvider {
300301
// In this example, we use the PruningPredicate's literal guarantees to
301302
// analyze the predicate. In a real system, using
302303
// `PruningPredicate::prune` would likely be easier to do.
303-
let pruning_predicate =
304-
PruningPredicate::try_new(Arc::clone(predicate), self.schema())?;
304+
let pruning_predicate = PruningPredicateBuilder::new()
305+
.with_file_schema(self.schema())
306+
.try_build(Arc::clone(predicate))?;
305307

306308
// The PruningPredicate's guarantees must all be satisfied in order for
307309
// the predicate to possibly evaluate to true.

‎datafusion-examples/examples/data_io/parquet_index.rs‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@ use datafusion::parquet::arrow::{
4242
ArrowWriter, arrow_reader::ParquetRecordBatchReaderBuilder,
4343
};
4444
use datafusion::physical_expr::PhysicalExpr;
45-
use datafusion::physical_optimizer::pruning::PruningPredicate;
45+
use datafusion::physical_optimizer::pruning::PruningPredicateBuilder;
4646
use datafusion::physical_plan::ExecutionPlan;
4747
use datafusion::prelude::*;
4848
use std::collections::HashSet;
@@ -274,7 +274,8 @@ impl TableProvider for IndexTableProvider {
274274
/// Simple in memory secondary index for a set of parquet files
275275
///
276276
/// The index is represented as an arrow [`RecordBatch`] that can be passed
277-
/// directly by the DataFusion [`PruningPredicate`] API
277+
/// directly by the DataFusion
278+
/// [`datafusion::physical_optimizer::pruning::PruningPredicate`] API
278279
///
279280
/// The `RecordBatch` looks as follows.
280281
///
@@ -362,8 +363,9 @@ impl ParquetMetadataIndex {
362363
) -> Result<Vec<(&str, u64)>> {
363364
// Use the PruningPredicate API to determine which files can not
364365
// possibly have any relevant data.
365-
let pruning_predicate =
366-
PruningPredicate::try_new(predicate, self.schema().clone())?;
366+
let pruning_predicate = PruningPredicateBuilder::new()
367+
.with_file_schema(self.schema().clone())
368+
.try_build(predicate)?;
367369

368370
// Now evaluate the pruning predicate into a boolean mask, one element per
369371
// file in the index. If the mask is true, the file may have rows that

‎datafusion-examples/examples/query_planning/pruning.rs‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,9 @@ use datafusion::error::Result;
2828
use datafusion::execution::context::ExecutionProps;
2929
use datafusion::logical_expr::physical_planning_context::PhysicalPlanningContext;
3030
use datafusion::physical_expr::create_physical_expr;
31-
use datafusion::physical_optimizer::pruning::PruningPredicate;
31+
use datafusion::physical_optimizer::pruning::{
32+
PruningPredicate, PruningPredicateBuilder,
33+
};
3234
use datafusion::prelude::*;
3335

3436
/// This example shows how to use DataFusion's `PruningPredicate` to prove
@@ -202,7 +204,10 @@ fn create_pruning_predicate(expr: Expr, schema: &SchemaRef) -> PruningPredicate
202204
&PhysicalPlanningContext::default(),
203205
)
204206
.unwrap();
205-
PruningPredicate::try_new(physical_expr, Arc::clone(schema)).unwrap()
207+
PruningPredicateBuilder::new()
208+
.with_file_schema(Arc::clone(schema))
209+
.try_build(physical_expr)
210+
.unwrap()
206211
}
207212

208213
fn i32_array<'a>(values: impl Iterator<Item = &'a Option<i32>>) -> ArrayRef {

‎datafusion/datasource-parquet/src/bloom_filter.rs‎

Lines changed: 15 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -249,13 +249,23 @@ mod tests {
249249
use datafusion_expr::{Expr, col, lit};
250250
use datafusion_physical_expr::planner::logical2physical;
251251
use datafusion_physical_plan::metrics::ExecutionPlanMetricsSet;
252-
use datafusion_pruning::PruningPredicate;
252+
use datafusion_pruning::{PruningPredicate, PruningPredicateBuilder};
253253
use object_store::{ObjectStore, ObjectStoreExt};
254254
use parquet::arrow::ArrowWriter;
255255
use parquet::arrow::ParquetRecordBatchStreamBuilder;
256256
use parquet::arrow::async_reader::ParquetObjectReader;
257257
use parquet::file::properties::{EnabledStatistics, WriterProperties};
258258

259+
fn build_test_pruning_predicate(
260+
expr: Arc<dyn datafusion_physical_plan::PhysicalExpr>,
261+
schema: Schema,
262+
) -> PruningPredicate {
263+
PruningPredicateBuilder::new()
264+
.with_file_schema(Arc::new(schema))
265+
.try_build(expr)
266+
.unwrap()
267+
}
268+
259269
#[tokio::test]
260270
async fn test_row_group_bloom_filter_pruning_predicate_simple_expr() {
261271
BloomFilterTest::new_data_index_bloom_encoding_stats()
@@ -321,8 +331,7 @@ mod tests {
321331
false,
322332
);
323333
let expr = logical2physical(&expr, &schema);
324-
let pruning_predicate =
325-
PruningPredicate::try_new(expr, Arc::new(schema)).unwrap();
334+
let pruning_predicate = build_test_pruning_predicate(expr, schema);
326335

327336
let pruned_row_groups = test_row_group_bloom_filter_pruning_predicate(
328337
file_name,
@@ -439,8 +448,7 @@ mod tests {
439448
None,
440449
));
441450
let expr = logical2physical(&expr, &schema);
442-
let pruning_predicate =
443-
PruningPredicate::try_new(expr, Arc::new(schema)).unwrap();
451+
let pruning_predicate = build_test_pruning_predicate(expr, schema);
444452

445453
let pruned_row_groups = test_row_group_bloom_filter_pruning_predicate(
446454
&format!("decimal128-{precision}.parquet"),
@@ -477,8 +485,7 @@ mod tests {
477485
None,
478486
));
479487
let expr = logical2physical(&expr, &schema);
480-
let pruning_predicate =
481-
PruningPredicate::try_new(expr, Arc::new(schema)).unwrap();
488+
let pruning_predicate = build_test_pruning_predicate(expr, schema);
482489

483490
let pruned_row_groups = test_row_group_bloom_filter_pruning_predicate(
484491
&format!("negative-decimal128-{precision}.parquet"),
@@ -573,8 +580,7 @@ mod tests {
573580
let data = bytes::Bytes::from(std::fs::read(path).unwrap());
574581

575582
let expr = logical2physical(&expr, &schema);
576-
let pruning_predicate =
577-
PruningPredicate::try_new(expr, Arc::new(schema)).unwrap();
583+
let pruning_predicate = build_test_pruning_predicate(expr, schema);
578584

579585
let pruned_row_groups = test_row_group_bloom_filter_pruning_predicate(
580586
&file_name,

‎datafusion/datasource-parquet/src/page_filter.rs‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ use arrow::{
3131
use datafusion_common::ScalarValue;
3232
use datafusion_common::pruning::PruningStatistics;
3333
use datafusion_physical_expr::{PhysicalExpr, split_conjunction};
34-
use datafusion_pruning::PruningPredicate;
34+
use datafusion_pruning::{PruningPredicate, PruningPredicateBuilder};
3535

3636
use log::{debug, trace};
3737
use parquet::arrow::arrow_reader::statistics::StatisticsConverter;
@@ -144,10 +144,10 @@ impl PagePruningAccessPlanFilter {
144144
let predicates = split_conjunction(expr)
145145
.into_iter()
146146
.filter_map(|predicate| {
147-
let pp = match PruningPredicate::try_new(
148-
Arc::clone(predicate),
149-
Arc::clone(&schema),
150-
) {
147+
let pp = match PruningPredicateBuilder::new()
148+
.with_file_schema(Arc::clone(&schema))
149+
.try_build(Arc::clone(predicate))
150+
{
151151
Ok(pp) => pp,
152152
Err(e) => {
153153
debug!("Ignoring error creating page pruning predicate: {e}");

‎datafusion/datasource-parquet/src/row_group_filter.rs‎

Lines changed: 27 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ use datafusion_expr::Operator;
2929
use datafusion_physical_expr::expressions::{BinaryExpr, IsNullExpr, NotExpr};
3030
use datafusion_physical_expr::utils::collect_columns;
3131
use datafusion_physical_expr::{PhysicalExpr, PhysicalExprSimplifier};
32-
use datafusion_pruning::PruningPredicate;
32+
use datafusion_pruning::{PruningPredicate, PruningPredicateBuilder};
3333
use parquet::arrow::arrow_reader::statistics::StatisticsConverter;
3434
use parquet::file::metadata::RowGroupMetaData;
3535
use parquet::schema::types::SchemaDescriptor;
@@ -373,8 +373,9 @@ impl RowGroupAccessPlanFilter {
373373
return;
374374
};
375375

376-
let Ok(inverted_predicate) =
377-
PruningPredicate::try_new(inverted_expr, Arc::clone(predicate.schema()))
376+
let Ok(inverted_predicate) = PruningPredicateBuilder::new()
377+
.with_file_schema(Arc::clone(predicate.schema()))
378+
.try_build(inverted_expr)
378379
else {
379380
return;
380381
};
@@ -550,6 +551,16 @@ mod tests {
550551
schema::types::SchemaDescPtr,
551552
};
552553

554+
fn build_test_pruning_predicate(
555+
expr: Arc<dyn PhysicalExpr>,
556+
schema: Arc<Schema>,
557+
) -> PruningPredicate {
558+
PruningPredicateBuilder::new()
559+
.with_file_schema(schema)
560+
.try_build(expr)
561+
.unwrap()
562+
}
563+
553564
struct PrimitiveTypeField {
554565
name: &'static str,
555566
physical_ty: PhysicalType,
@@ -612,7 +623,7 @@ mod tests {
612623
Arc::new(Schema::new(vec![Field::new("c1", DataType::Int32, false)]));
613624
let expr = col("c1").gt(lit(15));
614625
let expr = logical2physical(&expr, &schema);
615-
let pruning_predicate = PruningPredicate::try_new(expr, schema.clone()).unwrap();
626+
let pruning_predicate = build_test_pruning_predicate(expr, Arc::clone(&schema));
616627

617628
let field = PrimitiveTypeField::new("c1", PhysicalType::INT32);
618629
let schema_descr = get_test_schema_descr(vec![field]);
@@ -655,7 +666,7 @@ mod tests {
655666

656667
let schema = Arc::new(Schema::new(vec![Field::new("c1", DataType::Int32, true)]));
657668
let expr = logical2physical(&col("c1").gt(lit(15)), &schema);
658-
let pruning_predicate = PruningPredicate::try_new(expr, schema.clone()).unwrap();
669+
let pruning_predicate = build_test_pruning_predicate(expr, Arc::clone(&schema));
659670

660671
let field = PrimitiveTypeField::new("c1", PhysicalType::INT32);
661672
let schema_descr = get_test_schema_descr(vec![field]);
@@ -781,7 +792,7 @@ mod tests {
781792
Arc::new(Schema::new(vec![Field::new("c1", DataType::Int32, false)]));
782793
let expr = col("c1").gt(lit(15));
783794
let expr = logical2physical(&expr, &schema);
784-
let pruning_predicate = PruningPredicate::try_new(expr, schema.clone()).unwrap();
795+
let pruning_predicate = build_test_pruning_predicate(expr, Arc::clone(&schema));
785796

786797
let field = PrimitiveTypeField::new("c1", PhysicalType::INT32);
787798
let schema_descr = get_test_schema_descr(vec![field]);
@@ -824,7 +835,7 @@ mod tests {
824835
]));
825836
let expr = col("c1").gt(lit(15)).and(col("c2").rem(lit(2)).eq(lit(0)));
826837
let expr = logical2physical(&expr, &schema);
827-
let pruning_predicate = PruningPredicate::try_new(expr, schema.clone()).unwrap();
838+
let pruning_predicate = build_test_pruning_predicate(expr, Arc::clone(&schema));
828839

829840
let schema_descr = get_test_schema_descr(vec![
830841
PrimitiveTypeField::new("c1", PhysicalType::INT32),
@@ -863,7 +874,7 @@ mod tests {
863874
// this bypasses the entire predicate expression and no row groups are filtered out
864875
let expr = col("c1").gt(lit(15)).or(col("c2").rem(lit(2)).eq(lit(0)));
865876
let expr = logical2physical(&expr, &schema);
866-
let pruning_predicate = PruningPredicate::try_new(expr, schema.clone()).unwrap();
877+
let pruning_predicate = build_test_pruning_predicate(expr, Arc::clone(&schema));
867878

868879
// if conditions in predicate are joined with OR and an unsupported expression is used
869880
// this bypasses the entire predicate expression and no row groups are filtered out
@@ -890,7 +901,7 @@ mod tests {
890901
let expr = col("c1").gt(lit(0));
891902
let expr = logical2physical(&expr, &table_schema);
892903
let pruning_predicate =
893-
PruningPredicate::try_new(expr, table_schema.clone()).unwrap();
904+
build_test_pruning_predicate(expr, Arc::clone(&table_schema));
894905

895906
// Model a file schema's column order c2 then c1, which is the opposite
896907
// of the table schema
@@ -967,7 +978,7 @@ mod tests {
967978
let schema_descr = ArrowSchemaConverter::new().convert(&schema).unwrap();
968979
let expr = col("c1").gt(lit(15)).and(col("c2").is_null());
969980
let expr = logical2physical(&expr, &schema);
970-
let pruning_predicate = PruningPredicate::try_new(expr, schema.clone()).unwrap();
981+
let pruning_predicate = build_test_pruning_predicate(expr, Arc::clone(&schema));
971982
let groups = gen_row_group_meta_data_for_pruning_predicate();
972983

973984
let metrics = parquet_file_metrics();
@@ -998,7 +1009,7 @@ mod tests {
9981009
.gt(lit(15))
9991010
.and(col("c2").eq(lit(ScalarValue::Boolean(None))));
10001011
let expr = logical2physical(&expr, &schema);
1001-
let pruning_predicate = PruningPredicate::try_new(expr, schema.clone()).unwrap();
1012+
let pruning_predicate = build_test_pruning_predicate(expr, Arc::clone(&schema));
10021013
let groups = gen_row_group_meta_data_for_pruning_predicate();
10031014

10041015
let metrics = parquet_file_metrics();
@@ -1033,7 +1044,7 @@ mod tests {
10331044
let schema_descr = get_test_schema_descr(vec![field]);
10341045
let expr = col("c1").gt(lit(ScalarValue::Decimal128(Some(500), 9, 2)));
10351046
let expr = logical2physical(&expr, &schema);
1036-
let pruning_predicate = PruningPredicate::try_new(expr, schema.clone()).unwrap();
1047+
let pruning_predicate = build_test_pruning_predicate(expr, Arc::clone(&schema));
10371048
let rgm1 = get_row_group_meta_data(
10381049
&schema_descr,
10391050
// [1.00, 6.00]
@@ -1101,7 +1112,7 @@ mod tests {
11011112
Decimal128(11, 2),
11021113
));
11031114
let expr = logical2physical(&expr, &schema);
1104-
let pruning_predicate = PruningPredicate::try_new(expr, schema.clone()).unwrap();
1115+
let pruning_predicate = build_test_pruning_predicate(expr, Arc::clone(&schema));
11051116
let rgm1 = get_row_group_meta_data(
11061117
&schema_descr,
11071118
// [100, 600]
@@ -1190,7 +1201,7 @@ mod tests {
11901201
let schema_descr = get_test_schema_descr(vec![field]);
11911202
let expr = col("c1").lt(lit(ScalarValue::Decimal128(Some(500), 18, 2)));
11921203
let expr = logical2physical(&expr, &schema);
1193-
let pruning_predicate = PruningPredicate::try_new(expr, schema.clone()).unwrap();
1204+
let pruning_predicate = build_test_pruning_predicate(expr, Arc::clone(&schema));
11941205
let rgm1 = get_row_group_meta_data(
11951206
&schema_descr,
11961207
// [6.00, 8.00]
@@ -1248,7 +1259,7 @@ mod tests {
12481259
let left = cast(col("c1"), Decimal128(28, 3));
12491260
let expr = left.eq(lit(ScalarValue::Decimal128(Some(100000), 28, 3)));
12501261
let expr = logical2physical(&expr, &schema);
1251-
let pruning_predicate = PruningPredicate::try_new(expr, schema.clone()).unwrap();
1262+
let pruning_predicate = build_test_pruning_predicate(expr, Arc::clone(&schema));
12521263
// we must use the big-endian when encode the i128 to bytes or vec[u8].
12531264
let rgm1 = get_row_group_meta_data(
12541265
&schema_descr,
@@ -1323,7 +1334,7 @@ mod tests {
13231334
let left = cast(col("c1"), Decimal128(28, 3));
13241335
let expr = left.eq(lit(ScalarValue::Decimal128(Some(100000), 28, 3)));
13251336
let expr = logical2physical(&expr, &schema);
1326-
let pruning_predicate = PruningPredicate::try_new(expr, schema.clone()).unwrap();
1337+
let pruning_predicate = build_test_pruning_predicate(expr, Arc::clone(&schema));
13271338
// we must use the big-endian when encode the i128 to bytes or vec[u8].
13281339
let rgm1 = get_row_group_meta_data(
13291340
&schema_descr,

0 commit comments

Comments
 (0)