Repository navigation
Expand file tree
/
Copy pathrow_filter.rs
More file actions
2117 lines (1910 loc) · 82.5 KB
/
Copy pathrow_filter.rs
File metadata and controls
2117 lines (1910 loc) · 82.5 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
//! Utilities to push down of DataFusion filter predicates (any DataFusion
//! `PhysicalExpr` that evaluates to a [`BooleanArray`]) to the parquet decoder
//! level in `arrow-rs`.
//!
//! DataFusion will use a `ParquetRecordBatchStream` to read data from parquet
//! into [`RecordBatch`]es.
//!
//! The `ParquetRecordBatchStream` takes an optional `RowFilter` which is itself
//! a Vec of `Box<dyn ArrowPredicate>`. During decoding, the predicates are
//! evaluated in order, to generate a mask which is used to avoid decoding rows
//! in projected columns which do not pass the filter which can significantly
//! reduce the amount of compute required for decoding and thus improve query
//! performance.
//!
//! Since the predicates are applied serially in the order defined in the
//! `RowFilter`, the optimal ordering depends on the exact filters. The best
//! filters to execute first have two properties:
//!
//! 1. They are relatively inexpensive to evaluate (e.g. they read
//! column chunks which are relatively small)
//!
//! 2. They filter many (contiguous) rows, reducing the amount of decoding
//! required for subsequent filters and projected columns
//!
//! If requested, this code will reorder the filters based on heuristics try and
//! reduce the evaluation cost.
//!
//! The basic algorithm for constructing the `RowFilter` is as follows
//!
//! 1. Break conjunctions into separate predicates. An expression
//! like `a = 1 AND (b = 2 AND c = 3)` would be
//! separated into the expressions `a = 1`, `b = 2`, and `c = 3`.
//! 2. Determine whether each predicate can be evaluated as an `ArrowPredicate`.
//! 3. Determine, for each predicate, the total compressed size of all
//! columns required to evaluate the predicate.
//! 4. Re-order predicates by total size (from step 3).
//! 5. "Compile" each predicate `Expr` to a `DatafusionArrowPredicate`.
//! 6. Build the `RowFilter` from the ordered predicates.
//!
//! List-aware predicates (for example, `array_has`, `array_has_all`, and
//! `array_has_any`) can be evaluated directly during Parquet decoding.
//! Struct field access via `get_field` is also supported when the accessed
//! leaf is a primitive type. Filters that reference entire struct columns
//! rather than individual fields cannot be pushed down and are instead
//! evaluated after the full batches are materialized.
//!
//! For example, given a struct column `s {name: Utf8, value: Int32}`:
//! - `WHERE s['value'] > 5` — pushed down (accesses a primitive leaf)
//! - `WHERE s IS NOT NULL` — not pushed down (references the whole struct)
use std::sync::Arc;
use arrow::array::BooleanArray;
use arrow::datatypes::{Schema, SchemaRef};
use arrow::error::{ArrowError, Result as ArrowResult};
use arrow::record_batch::RecordBatch;
use parquet::arrow::ProjectionMask;
use parquet::arrow::arrow_reader::{ArrowPredicate, RowFilter};
use parquet::file::metadata::ParquetMetaData;
use datafusion_common::Result;
use datafusion_common::cast::as_boolean_array;
use datafusion_common::tree_node::TreeNode;
use datafusion_physical_expr::utils::reassign_expr_columns;
use datafusion_physical_expr::{PhysicalExpr, split_conjunction};
use datafusion_physical_plan::metrics;
use super::ParquetFileMetrics;
use super::supported_predicates::supports_list_predicates;
use crate::projection_read_plan::{
ParquetReadPlan, PushdownChecker, PushdownColumns, assemble_read_plan,
build_read_plan_with_cast_clipping,
};
/// A "compiled" predicate passed to `ParquetRecordBatchStream` to perform
/// row-level filtering during parquet decoding.
///
/// See the module level documentation for more information.
///
/// Implements the `ArrowPredicate` trait used by the parquet decoder
///
/// An expression can be evaluated as a `DatafusionArrowPredicate` if it:
/// * Does not reference any projected columns
/// * References either primitive columns or list columns used by
/// supported predicates (such as `array_has_all` or NULL checks).
/// * References struct fields via `get_field` where the accessed leaf
/// is a primitive type (e.g. `get_field(struct_col, 'field') > 5`).
/// Direct references to whole struct columns are still evaluated after
/// decoding.
#[derive(Debug)]
pub(crate) struct DatafusionArrowPredicate {
/// the filter expression
physical_expr: Arc<dyn PhysicalExpr>,
/// Path to the leaf columns in the parquet schema required to evaluate the
/// expression
projection_mask: ProjectionMask,
/// how many rows were filtered out by this predicate
rows_pruned: metrics::Count,
/// how many rows passed this predicate
rows_matched: metrics::Count,
/// how long was spent evaluating this predicate
time: metrics::Time,
}
impl DatafusionArrowPredicate {
/// Create a new `DatafusionArrowPredicate` from a `FilterCandidate`.
///
/// Production code goes through [`prebuild_row_filter_candidates`] +
/// [`row_filter_from_prebuilt`]; this constructor remains as a test
/// convenience for exercising a single candidate.
#[cfg(test)]
pub fn try_new(
candidate: FilterCandidate,
rows_pruned: metrics::Count,
rows_matched: metrics::Count,
time: metrics::Time,
) -> Result<Self> {
let physical_expr =
reassign_expr_columns(candidate.expr, &candidate.read_plan.projected_schema)?;
Ok(Self {
physical_expr,
projection_mask: candidate.read_plan.projection_mask,
rows_pruned,
rows_matched,
time,
})
}
}
impl ArrowPredicate for DatafusionArrowPredicate {
fn projection(&self) -> &ProjectionMask {
&self.projection_mask
}
fn evaluate(&mut self, batch: RecordBatch) -> ArrowResult<BooleanArray> {
// scoped timer updates on drop
let mut timer = self.time.timer();
self.physical_expr
.evaluate(&batch)
.and_then(|v| v.into_array(batch.num_rows()))
.and_then(|array| {
let bool_arr = as_boolean_array(&array)?.clone();
let num_matched = bool_arr.true_count();
let num_pruned = bool_arr.len() - num_matched;
self.rows_pruned.add(num_pruned);
self.rows_matched.add(num_matched);
timer.stop();
Ok(bool_arr)
})
// `ExternalError` is the only `ArrowError` variant that keeps a
// source, and therefore the only one that leaves the original error
// recoverable (see `DataFusionError::find_root`)
.map_err(|e| {
ArrowError::ExternalError(Box::new(
e.context("Error evaluating filter predicate"),
))
})
}
}
/// A candidate expression for creating a `RowFilter`.
///
/// Each candidate contains the expression as well as data to estimate the cost
/// of evaluating the resulting expression.
///
/// See the module level documentation for more information.
pub(crate) struct FilterCandidate {
expr: Arc<dyn PhysicalExpr>,
/// Estimate for the total number of bytes that will need to be processed
/// to evaluate this filter. This is used to estimate the cost of evaluating
/// the filter and to order the filters when `reorder_predicates` is true.
/// This is generated by summing the compressed size of all columns that the filter references.
required_bytes: usize,
/// The resolved Parquet read plan (leaf indices + projected schema).
read_plan: ParquetReadPlan,
}
/// Helper to build a `FilterCandidate`.
///
/// This will do several things:
/// 1. Determine the columns required to evaluate the expression
/// 2. Calculate data required to estimate the cost of evaluating the filter
///
/// Note: This does *not* handle any adaptation of the expression to the file schema.
/// The expression must already be adapted before being passed in here, generally using
/// [`PhysicalExprAdapter`](datafusion_physical_expr_adapter::PhysicalExprAdapter).
struct FilterCandidateBuilder {
expr: Arc<dyn PhysicalExpr>,
/// The Arrow schema of this parquet file (the result of converting the
/// parquet schema to Arrow, potentially with type coercions applied).
file_schema: SchemaRef,
}
impl FilterCandidateBuilder {
pub fn new(expr: Arc<dyn PhysicalExpr>, file_schema: Arc<Schema>) -> Self {
Self { expr, file_schema }
}
/// Attempt to build a `FilterCandidate` from the expression
///
/// # Return values
///
/// * `Ok(Some(candidate))` if the expression can be used as an ArrowFilter
/// * `Ok(None)` if the expression cannot be used as an ArrowFilter
/// * `Err(e)` if an error occurs while building the candidate
pub fn build(self, metadata: &ParquetMetaData) -> Result<Option<FilterCandidate>> {
Ok(
build_parquet_read_plan(&self.expr, &self.file_schema, metadata)?.map(
|(read_plan, required_bytes)| FilterCandidate {
expr: self.expr,
required_bytes,
read_plan,
},
),
)
}
}
/// Checks if a given expression can be pushed down to the parquet decoder.
///
/// Returns `Some(PushdownColumns)` if the expression can be pushed down,
/// where the struct contains the indices into the file schema of all columns
/// required to evaluate the expression.
///
/// Returns `None` if the expression cannot be pushed down (e.g., references
/// unsupported nested types or columns not in the file).
/// Struct casts are accepted only after schema adaptation, not while planning
/// against the table schema: adaptation may insert another cast underneath an
/// explicit cast, leaving an expression the runtime checker cannot handle.
fn pushdown_columns(
expr: &Arc<dyn PhysicalExpr>,
file_schema: &Schema,
allow_struct_casts: bool,
) -> Result<Option<PushdownColumns>> {
let allow_list_columns = supports_list_predicates(expr);
let mut checker =
PushdownChecker::new(file_schema, allow_list_columns, allow_struct_casts);
expr.visit(&mut checker)?;
Ok((!checker.prevents_pushdown()).then(|| checker.into_sorted_columns()))
}
/// Resolves which Parquet leaf columns and Arrow schema fields are needed
/// to evaluate `expr` against a Parquet file
///
/// Returns `Ok(Some((plan, required_bytes)))` when the expression can be
/// evaluated using only pushdown-compatible columns. `Ok(None)` when it
/// cannot (it references whole struct columns or columns missing from disk).
///
/// The `required_bytes` is the total compressed size of all referenced columns
/// across all row groups, used to estimate filter evaluation cost.
///
/// Note: this is a shared entry point used by both row filter construction and
/// the opener's projection logic
pub(crate) fn build_parquet_read_plan(
expr: &Arc<dyn PhysicalExpr>,
file_schema: &Schema,
metadata: &ParquetMetaData,
) -> Result<Option<(ParquetReadPlan, usize)>> {
let schema_descr = metadata.file_metadata().schema_descr();
let Some(required_columns) = pushdown_columns(expr, file_schema, true)? else {
return Ok(None);
};
// A retained Struct cast names the fields its conversion touches, so the
// read is clipped to those leaves rather than decoding the whole root.
// A cast whose target covers every leaf, or that cannot be clipped safely,
// falls back to a full read of that root inside the helper.
let (read_plan, leaf_indices) = if required_columns.cast_accesses.is_empty() {
assemble_read_plan(
&required_columns.required_columns,
&required_columns.struct_field_accesses,
file_schema,
schema_descr,
)
} else {
build_read_plan_with_cast_clipping(
file_schema,
schema_descr,
&required_columns.required_columns,
&required_columns.struct_field_accesses,
&required_columns.cast_accesses,
)
};
let required_bytes = size_of_columns(&leaf_indices, metadata)?;
Ok(Some((read_plan, required_bytes)))
}
/// Checks if a predicate expression can be pushed down to the parquet decoder.
///
/// Returns `true` if all columns referenced by the expression:
/// - Exist in the provided schema
/// - Are primitive types OR list columns with supported predicates
/// (e.g., `array_has`, `array_has_all`, `array_has_any`, IS NULL, IS NOT NULL)
/// - Are struct columns accessed via `get_field` where the leaf type is primitive
/// - Direct references to whole struct columns will prevent pushdown
///
/// # Arguments
/// * `expr` - The filter expression to check
/// * `file_schema` - The Arrow schema of the parquet file (or table schema when
/// the file schema is not yet available during planning)
///
/// # Examples
///
/// Primitive column filters can be pushed down:
/// ```ignore
/// use datafusion_expr::{col, Expr};
/// use datafusion_common::ScalarValue;
/// use arrow::datatypes::{DataType, Field, Schema};
/// use std::sync::Arc;
///
/// let schema = Arc::new(Schema::new(vec![
/// Field::new("age", DataType::Int32, false),
/// ]));
///
/// // Primitive filter: can be pushed down
/// let expr = col("age").gt(Expr::Literal(ScalarValue::Int32(Some(30)), None));
/// let expr = logical2physical(&expr, &schema);
/// assert!(can_expr_be_pushed_down_with_schemas(&expr, &schema));
/// ```
///
/// Struct column filters cannot be pushed down:
/// ```ignore
/// use arrow::datatypes::Fields;
///
/// let schema = Arc::new(Schema::new(vec![
/// Field::new("person", DataType::Struct(
/// Fields::from(vec![Field::new("name", DataType::Utf8, true)])
/// ), true),
/// ]));
///
/// // Struct filter: cannot be pushed down
/// let expr = col("person").is_not_null();
/// let expr = logical2physical(&expr, &schema);
/// assert!(!can_expr_be_pushed_down_with_schemas(&expr, &schema));
/// ```
///
/// List column filters with supported predicates can be pushed down:
/// ```ignore
/// use datafusion_functions_nested::expr_fn::{array_has_all, make_array};
///
/// let schema = Arc::new(Schema::new(vec![
/// Field::new("tags", DataType::List(
/// Arc::new(Field::new("item", DataType::Utf8, true))
/// ), true),
/// ]));
///
/// // Array filter with supported predicate: can be pushed down
/// let expr = array_has_all(col("tags"), make_array(vec![
/// Expr::Literal(ScalarValue::Utf8(Some("rust".to_string())), None)
/// ]));
/// let expr = logical2physical(&expr, &schema);
/// assert!(can_expr_be_pushed_down_with_schemas(&expr, &schema));
/// ```
pub fn can_expr_be_pushed_down_with_schemas(
expr: &Arc<dyn PhysicalExpr>,
file_schema: &Schema,
) -> bool {
match pushdown_columns(expr, file_schema, false) {
Ok(Some(_)) => true,
Ok(None) | Err(_) => false,
}
}
/// Calculate the total compressed size of all leaf columns required for
/// predicate `Expr`.
///
/// This value represents the total amount of IO required to evaluate the
/// predicate.
fn size_of_columns(columns: &[usize], metadata: &ParquetMetaData) -> Result<usize> {
let mut total_size = 0;
let row_groups = metadata.row_groups();
for idx in columns {
for rg in row_groups.iter() {
total_size += rg.column(*idx).compressed_size() as usize;
}
}
Ok(total_size)
}
/// Build a [`RowFilter`] from the given predicate expression if possible.
///
/// # Arguments
/// * `expr` - The filter predicate, already adapted to reference columns in `file_schema`
/// * `file_schema` - The Arrow schema of the parquet file (the result of converting
/// the parquet schema to Arrow, potentially with type coercions applied)
/// * `metadata` - Parquet file metadata used for cost estimation
/// * `reorder_predicates` - If true, reorder predicates to minimize I/O
/// * `file_metrics` - Metrics for tracking filter performance
///
/// # Returns
/// * `Ok(Some(row_filter))` if the expression can be used as a RowFilter
/// * `Ok(None)` if the expression cannot be used as a RowFilter
/// * `Err(e)` if an error occurs while building the filter
///
/// Note: The returned `RowFilter` may not contain all conjuncts from the original
/// expression. Conjuncts that cannot be evaluated as an `ArrowPredicate` are ignored.
///
/// For example, if the expression is `a = 1 AND b = 2 AND c = 3` and `b = 2`
/// cannot be evaluated for some reason, the returned `RowFilter` will contain
/// only `a = 1` and `c = 3`.
pub fn build_row_filter(
expr: &Arc<dyn PhysicalExpr>,
file_schema: &SchemaRef,
metadata: &ParquetMetaData,
reorder_predicates: bool,
file_metrics: &ParquetFileMetrics,
) -> Result<Option<RowFilter>> {
// Implemented on top of the prebuild split so there is a single place
// that splits conjuncts, orders candidates, and wires metrics — callers
// that build once per file go through the same code as the per-row-group
// rebuild path in `RowFilterContext`.
let Some(prebuilt) = prebuild_row_filter_candidates(expr, file_schema, metadata)?
else {
return Ok(None);
};
Ok(Some(row_filter_from_prebuilt(
&prebuilt,
reorder_predicates,
file_metrics,
)))
}
/// A precomputed [`FilterCandidate`] with its expression column-reassigned to
/// the projected schema, ready to be wrapped into a [`DatafusionArrowPredicate`]
/// on demand.
///
/// Extracting this from [`build_row_filter`] lets callers pay the tree-walk +
/// column-resolution + `reassign_expr_columns` cost **once per file** instead
/// of once per row group, which is the hot path for
/// [`RowFilterContext::build_row_filter`](crate::push_decoder::RowFilterContext) rebuilds
/// on `fully_matched → not-fully-matched` boundaries.
#[derive(Clone, Debug)]
pub(crate) struct PrebuiltRowFilterCandidate {
/// The predicate expression with all `Column` indices rewritten to point
/// into the projected file schema.
physical_expr: Arc<dyn PhysicalExpr>,
/// Projection mask over the parquet leaf columns needed to evaluate this
/// predicate.
projection_mask: ProjectionMask,
/// Precomputed sum-of-compressed-bytes for the referenced columns across
/// all row groups in the file. Used to sort predicates when
/// `reorder_predicates` is enabled. Stable across row groups within a
/// file, so we cache it once.
required_bytes: usize,
}
impl PrebuiltRowFilterCandidate {
pub(crate) fn reads_leaf(&self, leaf_idx: usize) -> bool {
self.projection_mask.leaf_included(leaf_idx)
}
}
/// Precompute the list of [`PrebuiltRowFilterCandidate`]s for a predicate.
///
/// This is the expensive part of [`build_row_filter`]: split into conjuncts,
/// resolve columns for each conjunct against the file schema, reassign
/// `Column` indices, and compute the sort-order metadata. Doing it once per
/// file (and reusing across row groups) avoids repeated `TreeNode::transform`
/// walks and `Arc<PhysicalExpr>` allocations that showed up as top hot spots
/// in TPCH profiles.
///
/// Returns `Ok(None)` when the predicate has no push-downable conjuncts, in
/// which case callers should skip installing a `RowFilter` entirely.
pub(crate) fn prebuild_row_filter_candidates(
expr: &Arc<dyn PhysicalExpr>,
file_schema: &SchemaRef,
metadata: &ParquetMetaData,
) -> Result<Option<Vec<PrebuiltRowFilterCandidate>>> {
// Split into conjuncts:
// `a = 1 AND b = 2 AND c = 3` -> [`a = 1`, `b = 2`, `c = 3`]
let predicates = split_conjunction(expr);
let candidates: Vec<FilterCandidate> = predicates
.into_iter()
.map(|expr| {
FilterCandidateBuilder::new(Arc::clone(expr), Arc::clone(file_schema))
.build(metadata)
})
.collect::<Result<Vec<_>, _>>()?
.into_iter()
.flatten()
.collect();
if candidates.is_empty() {
return Ok(None);
}
let prebuilt: Vec<PrebuiltRowFilterCandidate> = candidates
.into_iter()
.map(|candidate| {
let physical_expr = reassign_expr_columns(
Arc::clone(&candidate.expr),
&candidate.read_plan.projected_schema,
)?;
Ok(PrebuiltRowFilterCandidate {
physical_expr,
projection_mask: candidate.read_plan.projection_mask.clone(),
required_bytes: candidate.required_bytes,
})
})
.collect::<Result<Vec<_>>>()?;
Ok(Some(prebuilt))
}
/// Wrap a list of prebuilt candidates into a fresh [`RowFilter`], assigning
/// per-predicate metric counters and (optionally) reordering by
/// `required_bytes`. This is the cheap per-row-group rebuild path — no tree
/// walks, no column resolution, only counter allocation.
pub(crate) fn row_filter_from_prebuilt(
prebuilt: &[PrebuiltRowFilterCandidate],
reorder_predicates: bool,
file_metrics: &ParquetFileMetrics,
) -> RowFilter {
let rows_pruned = &file_metrics.pushdown_rows_pruned;
let rows_matched = &file_metrics.pushdown_rows_matched;
let time = &file_metrics.row_pushdown_eval_time;
// Clone (cheap: Arc bumps + ProjectionMask clone) into a working list we
// can sort without disturbing the shared cache.
let mut ordered: Vec<&PrebuiltRowFilterCandidate> = prebuilt.iter().collect();
if reorder_predicates {
ordered.sort_unstable_by_key(|c| c.required_bytes);
}
let total = ordered.len();
let filters: Vec<Box<dyn ArrowPredicate>> = ordered
.into_iter()
.enumerate()
.map(|(idx, candidate)| {
let is_last = idx == total - 1;
let predicate_rows_pruned = rows_pruned.clone();
let predicate_rows_matched = if is_last {
rows_matched.clone()
} else {
metrics::Count::new()
};
Box::new(DatafusionArrowPredicate {
physical_expr: Arc::clone(&candidate.physical_expr),
projection_mask: candidate.projection_mask.clone(),
rows_pruned: predicate_rows_pruned,
rows_matched: predicate_rows_matched,
time: time.clone(),
}) as Box<dyn ArrowPredicate>
})
.collect();
RowFilter::new(filters)
}
#[cfg(test)]
mod test {
use super::*;
use arrow::datatypes::{DataType, Fields};
use datafusion_common::{DataFusionError, ScalarValue};
use arrow::array::{
Int32Array, ListBuilder, StringArray, StringBuilder, StructArray,
};
use arrow::datatypes::{Field, TimeUnit::Nanosecond};
use datafusion_expr::{Cast, Expr, col, lit};
use datafusion_functions::core::get_field;
use datafusion_functions_nested::array_has::{
array_has_all_udf, array_has_any_udf, array_has_udf,
};
use datafusion_functions_nested::expr_fn::{
array_has, array_has_all, array_has_any, make_array,
};
use datafusion_physical_expr::expressions::Column;
use datafusion_physical_expr::planner::logical2physical;
use datafusion_physical_expr_adapter::{
DefaultPhysicalExprAdapterFactory, PhysicalExprAdapterFactory,
};
use datafusion_physical_plan::metrics::{Count, ExecutionPlanMetricsSet, Time};
use parquet::arrow::ArrowWriter;
use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder;
use parquet::arrow::parquet_to_arrow_schema;
use parquet::file::reader::{FileReader, SerializedFileReader};
use tempfile::NamedTempFile;
// List predicates used by the decoder should be accepted for pushdown
#[test]
fn test_filter_candidate_builder_supports_list_types() {
let testdata = datafusion_common::test_util::parquet_test_data();
let file = std::fs::File::open(format!("{testdata}/list_columns.parquet"))
.expect("opening file");
let reader = SerializedFileReader::new(file).expect("creating reader");
let metadata = reader.metadata();
let table_schema =
parquet_to_arrow_schema(metadata.file_metadata().schema_descr(), None)
.expect("parsing schema");
let expr = col("int64_list").is_not_null();
let expr = logical2physical(&expr, &table_schema);
let table_schema = Arc::new(table_schema.clone());
let list_index = table_schema
.index_of("int64_list")
.expect("list column should exist");
let candidate = FilterCandidateBuilder::new(expr, table_schema)
.build(metadata)
.expect("building candidate")
.expect("list pushdown should be supported");
let expected_mask =
ProjectionMask::leaves(metadata.file_metadata().schema_descr(), [list_index]);
assert_eq!(candidate.read_plan.projection_mask, expected_mask);
}
#[test]
fn test_filter_type_coercion() {
let testdata = datafusion_common::test_util::parquet_test_data();
let file = std::fs::File::open(format!("{testdata}/alltypes_plain.parquet"))
.expect("opening file");
let parquet_reader_builder =
ParquetRecordBatchReaderBuilder::try_new(file).expect("creating reader");
let metadata = parquet_reader_builder.metadata().clone();
let file_schema = parquet_reader_builder.schema().clone();
// This is the schema we would like to coerce to,
// which is different from the physical schema of the file.
let table_schema = Schema::new(vec![Field::new(
"timestamp_col",
DataType::Timestamp(Nanosecond, Some(Arc::from("UTC"))),
false,
)]);
// Test all should fail
let expr = col("timestamp_col").lt(Expr::Literal(
ScalarValue::TimestampNanosecond(Some(1), Some(Arc::from("UTC"))),
None,
));
let expr = logical2physical(&expr, &table_schema);
let expr = DefaultPhysicalExprAdapterFactory {}
.create(Arc::new(table_schema.clone()), Arc::clone(&file_schema))
.expect("creating expr adapter")
.rewrite(expr)
.expect("rewriting expression");
let candidate = FilterCandidateBuilder::new(expr, file_schema.clone())
.build(&metadata)
.expect("building candidate")
.expect("candidate expected");
let mut row_filter = DatafusionArrowPredicate::try_new(
candidate,
Count::new(),
Count::new(),
Time::new(),
)
.expect("creating filter predicate");
let mut parquet_reader = parquet_reader_builder
.with_projection(row_filter.projection().clone())
.build()
.expect("building reader");
// Parquet file is small, we only need 1 record batch
let first_rb = parquet_reader
.next()
.expect("expected record batch")
.expect("expected error free record batch");
let filtered = row_filter.evaluate(first_rb.clone());
assert!(matches!(filtered, Ok(a) if a == BooleanArray::from(vec![false; 8])));
// Test all should pass
let expr = col("timestamp_col").gt(Expr::Literal(
ScalarValue::TimestampNanosecond(Some(0), Some(Arc::from("UTC"))),
None,
));
let expr = logical2physical(&expr, &table_schema);
// Rewrite the expression to add CastExpr for type coercion
let expr = DefaultPhysicalExprAdapterFactory {}
.create(Arc::new(table_schema), Arc::clone(&file_schema))
.expect("creating expr adapter")
.rewrite(expr)
.expect("rewriting expression");
let candidate = FilterCandidateBuilder::new(expr, file_schema)
.build(&metadata)
.expect("building candidate")
.expect("candidate expected");
let mut row_filter = DatafusionArrowPredicate::try_new(
candidate,
Count::new(),
Count::new(),
Time::new(),
)
.expect("creating filter predicate");
let filtered = row_filter.evaluate(first_rb);
assert!(matches!(filtered, Ok(a) if a == BooleanArray::from(vec![true; 8])));
}
/// A predicate that fails while it is being evaluated must report the
/// original error, not an opaque string, so that callers can still tell a
/// user error apart from an internal one.
#[test]
fn evaluate_reports_the_original_error() {
let schema = Arc::new(Schema::new(vec![Field::new("s", DataType::Utf8, false)]));
let batch = RecordBatch::try_new(
Arc::clone(&schema),
vec![Arc::new(StringArray::from(vec!["not_an_int"]))],
)
.expect("record batch");
let file = NamedTempFile::new().expect("temp file");
let mut writer =
ArrowWriter::try_new(file.reopen().unwrap(), Arc::clone(&schema), None)
.expect("writer");
writer.write(&batch).expect("write batch");
writer.close().expect("close writer");
let parquet_reader_builder =
ParquetRecordBatchReaderBuilder::try_new(file.reopen().expect("reopen file"))
.expect("reader builder");
let metadata = parquet_reader_builder.metadata().clone();
let file_schema = parquet_reader_builder.schema().clone();
// Casting the column in the file to `Int32` fails on this data
let expr = Expr::Cast(Cast::new(Box::new(col("s")), DataType::Int32)).eq(lit(1));
let expr = logical2physical(&expr, &file_schema);
let candidate = FilterCandidateBuilder::new(expr, Arc::clone(&file_schema))
.build(&metadata)
.expect("building candidate")
.expect("candidate expected");
let mut predicate = DatafusionArrowPredicate::try_new(
candidate,
Count::new(),
Count::new(),
Time::new(),
)
.expect("creating filter predicate");
let mut parquet_reader = parquet_reader_builder
.with_projection(predicate.projection().clone())
.build()
.expect("building reader");
let first_rb = parquet_reader
.next()
.expect("expected record batch")
.expect("expected error free record batch");
let err = predicate
.evaluate(first_rb)
.expect_err("evaluating the predicate should fail");
// The cast failure is still reachable, rather than being flattened into
// an untyped `ArrowError::ComputeError`
let err = DataFusionError::from(err);
let root = err.find_root();
assert!(
matches!(
root,
DataFusionError::ArrowError(inner, _)
if matches!(inner.as_ref(), ArrowError::CastError(_))
),
"expected the original cast error, got {root:?}"
);
// and the message still says where the failure happened
let message = err.to_string();
assert!(
message.contains("Error evaluating filter predicate"),
"{message}"
);
assert!(message.contains("Cannot cast string"), "{message}");
}
#[test]
fn struct_data_structures_prevent_pushdown() {
let table_schema = Arc::new(Schema::new(vec![Field::new(
"struct_col",
DataType::Struct(
vec![Arc::new(Field::new("a", DataType::Int32, true))].into(),
),
true,
)]));
let expr = col("struct_col").is_not_null();
let expr = logical2physical(&expr, &table_schema);
assert!(!can_expr_be_pushed_down_with_schemas(&expr, &table_schema));
}
#[test]
fn mixed_primitive_and_struct_prevents_pushdown() {
// Even when a predicate contains both primitive and unsupported nested columns,
// the entire predicate should not be pushed down because the struct column
// cannot be evaluated during Parquet decoding.
let table_schema = Arc::new(Schema::new(vec![
Field::new(
"struct_col",
DataType::Struct(
vec![Arc::new(Field::new("a", DataType::Int32, true))].into(),
),
true,
),
Field::new("int_col", DataType::Int32, false),
]));
// Expression: (struct_col IS NOT NULL) AND (int_col = 5)
// Even though int_col is primitive, the presence of struct_col in the
// conjunction should prevent pushdown of the entire expression.
let expr = col("struct_col")
.is_not_null()
.and(col("int_col").eq(Expr::Literal(ScalarValue::Int32(Some(5)), None)));
let expr = logical2physical(&expr, &table_schema);
// The entire expression should not be pushed down
assert!(!can_expr_be_pushed_down_with_schemas(&expr, &table_schema));
// However, just the int_col predicate alone should be pushable
let expr_int_only =
col("int_col").eq(Expr::Literal(ScalarValue::Int32(Some(5)), None));
let expr_int_only = logical2physical(&expr_int_only, &table_schema);
assert!(can_expr_be_pushed_down_with_schemas(
&expr_int_only,
&table_schema
));
}
#[test]
fn nested_lists_allow_pushdown_checks() {
let table_schema = Arc::new(get_lists_table_schema());
let expr = col("utf8_list").is_not_null();
let expr = logical2physical(&expr, &table_schema);
check_expression_can_evaluate_against_schema(&expr, &table_schema);
assert!(can_expr_be_pushed_down_with_schemas(&expr, &table_schema));
}
#[test]
fn array_has_all_pushdown_filters_rows() {
// Test array_has_all: checks if array contains all of ["c"]
// Rows with "c": row 1 and row 2
let expr = array_has_all(
col("letters"),
make_array(vec![Expr::Literal(
ScalarValue::Utf8(Some("c".to_string())),
None,
)]),
);
test_array_predicate_pushdown("array_has_all", expr, 1, 2, true);
}
/// Helper function to test array predicate pushdown functionality.
///
/// Creates a Parquet file with a list column, applies the given predicate,
/// and verifies that rows are correctly filtered during decoding.
fn test_array_predicate_pushdown(
func_name: &str,
predicate_expr: Expr,
expected_pruned: usize,
expected_matched: usize,
expect_list_support: bool,
) {
let item_field = Arc::new(Field::new("item", DataType::Utf8, true));
let schema = Arc::new(Schema::new(vec![Field::new(
"letters",
DataType::List(item_field),
true,
)]));
let mut builder = ListBuilder::new(StringBuilder::new());
// Row 0: ["a", "b"]
builder.values().append_value("a");
builder.values().append_value("b");
builder.append(true);
// Row 1: ["c"]
builder.values().append_value("c");
builder.append(true);
// Row 2: ["c", "d"]
builder.values().append_value("c");
builder.values().append_value("d");
builder.append(true);
let batch =
RecordBatch::try_new(schema.clone(), vec![Arc::new(builder.finish())])
.expect("record batch");
let file = NamedTempFile::new().expect("temp file");
let mut writer =
ArrowWriter::try_new(file.reopen().unwrap(), schema, None).expect("writer");
writer.write(&batch).expect("write batch");
writer.close().expect("close writer");
let reader_file = file.reopen().expect("reopen file");
let parquet_reader_builder =
ParquetRecordBatchReaderBuilder::try_new(reader_file)
.expect("reader builder");
let metadata = parquet_reader_builder.metadata().clone();
let file_schema = parquet_reader_builder.schema().clone();
let expr = logical2physical(&predicate_expr, &file_schema);
if expect_list_support {
assert!(supports_list_predicates(&expr));
}
let metrics = ExecutionPlanMetricsSet::new();
let file_metrics =
ParquetFileMetrics::new(0, &format!("{func_name}.parquet"), &metrics);
let row_filter =
build_row_filter(&expr, &file_schema, &metadata, false, &file_metrics)
.expect("building row filter")
.expect("row filter should exist");
let reader = parquet_reader_builder
.with_row_filter(row_filter)
.build()
.expect("build reader");
let mut total_rows = 0;
for batch in reader {
let batch = batch.expect("record batch");
total_rows += batch.num_rows();
}
assert_eq!(
file_metrics.pushdown_rows_pruned.value(),
expected_pruned,
"{func_name}: expected {expected_pruned} pruned rows"
);
assert_eq!(
file_metrics.pushdown_rows_matched.value(),
expected_matched,
"{func_name}: expected {expected_matched} matched rows"
);
assert_eq!(
total_rows, expected_matched,
"{func_name}: expected {expected_matched} total rows"
);
}
#[test]
fn array_has_pushdown_filters_rows() {
// Test array_has: checks if "c" is in the array
// Rows with "c": row 1 and row 2
let expr = array_has(
col("letters"),
Expr::Literal(ScalarValue::Utf8(Some("c".to_string())), None),
);
test_array_predicate_pushdown("array_has", expr, 1, 2, true);
}
#[test]
fn array_has_any_pushdown_filters_rows() {
// Test array_has_any: checks if array contains any of ["a", "d"]
// Row 0 has "a", row 2 has "d" - both should match
let expr = array_has_any(
col("letters"),
make_array(vec![
Expr::Literal(ScalarValue::Utf8(Some("a".to_string())), None),
Expr::Literal(ScalarValue::Utf8(Some("d".to_string())), None),
]),
);
test_array_predicate_pushdown("array_has_any", expr, 1, 2, true);
}
#[test]
fn array_has_udf_pushdown_filters_rows() {
let expr = array_has_udf().call(vec![
col("letters"),
Expr::Literal(ScalarValue::Utf8(Some("c".to_string())), None),
]);
test_array_predicate_pushdown("array_has_udf", expr, 1, 2, true);
}
#[test]
fn array_has_all_udf_pushdown_filters_rows() {
let expr = array_has_all_udf().call(vec![
col("letters"),
make_array(vec![Expr::Literal(
ScalarValue::Utf8(Some("c".to_string())),
None,
)]),
]);