Skip to content
Prev Previous commit
Next Next commit
review: rename API evaluate_pruning() --> propagate_statistics_vector…
…ized()
  • Loading branch information
2010YOUY01 committed Jan 12, 2026
commit 9d256a58320bf915335591c0c741dfb7993e369f
26 changes: 9 additions & 17 deletions datafusion/physical-expr-common/src/physical_expr.rs
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ use itertools::izip;
mod pruning;

pub use pruning::{
ColumnStats, NullPresence, NullStats, PruningContext, PruningIntermediate,
ColumnStats, NullPresence, NullStats, PropagatedIntermediate, PruningContext,
PruningOutcome, PruningResults, RangeStats,
};

Expand Down Expand Up @@ -438,33 +438,25 @@ pub trait PhysicalExpr: Any + Send + Sync + Display + Debug + DynEq + DynHash {
false
}

/// Evaluates pruning statistics via propagation. See the pruning module
/// docs for background.
/// Evaluates statistics propagation in a vectorized way.
///
/// This is mainly used for predicate pruning now. See the pruning module docs
/// for backgrouds.
///
/// This default implementation is for `PhysicalExpr`s that have not yet
/// implemented pruning; returning `None` signals that no pruning statistics
/// are available.
///
/// In the future, propagation may expose dedicated APIs such as:
/// ```text
/// trait PhysicalExpr {
/// fn propagate_range_stats()...
/// fn propagate_set_stats()...
/// fn propagate_null_stats()...
/// }
/// ```
/// with `evaluate_pruning` combining the individual statistic types.
///
/// # Returns
/// - `None` if the `PhysicalExpr` has not implemented the statistics propagation
/// - For predicate expressions (boolean outputs), implementations should return
/// `Some(PruningIntermediate::PruningResults)`
/// `Some(PropagatedIntermediate::IntermediateResult)`
/// - For arithmetic expressions, implementations should propagate stats and
/// return `Some(PruningIntermediate::IntermediateStats)`
fn evaluate_pruning(
/// return `Some(PropagatedIntermediate::IntermediateStats)`
fn evaluate_statistics_vectorized(
&self,
_ctx: Arc<PruningContext>,
) -> Result<Option<PruningIntermediate>> {
) -> Result<Option<PropagatedIntermediate>> {
Ok(None)
}
}
Expand Down
13 changes: 7 additions & 6 deletions datafusion/physical-expr-common/src/physical_expr/pruning.rs
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,7 @@
//!
//! The key structures involved in pruning are:
//! - [`PruningStatistics`]: the input source statistics for all containers
//! - [`super::PhysicalExpr::evaluate_pruning()`]: evaluates pruning behavior for predicates
//! - [`super::PhysicalExpr::evaluate_statistics_vectorized()`]: evaluates pruning behavior for predicates
//! - [`PruningIntermediate`]: the intermediate result produced during statistics propagation for pruning. Its internal representation uses Arrow Arrays, enabling vectorized evaluation for performance.

use std::{iter::repeat_n, sync::Arc};
Expand Down Expand Up @@ -509,16 +509,17 @@ impl ColumnStats {
}
}

/// Pruning intermediate type propagated through `PhysicalExpr` nodes.
/// Intermediate result used in `PhysicalExpr::evaluate_statistics_vectorized()`
///
/// Holds intermediate results for multiple input micro-partitions/containers,
/// stored in a vectorized Arrow array form.
#[derive(Debug, Clone)]
pub enum PruningIntermediate {
pub enum PropagatedIntermediate {
IntermediateStats(ColumnStats),
IntermediateResult(PruningResults),
}

impl PruningIntermediate {
impl PropagatedIntermediate {
/// Create an `IntermediateStats` variant with no range or null statistics.
pub fn empty_stats() -> Self {
Self::IntermediateStats(ColumnStats::new(None, None, 0))
Expand All @@ -527,8 +528,8 @@ impl PruningIntermediate {
/// Returns the number of containers inside the current `PruningIntermediate`
pub fn len(&self) -> usize {
match self {
PruningIntermediate::IntermediateStats(column_stats) => column_stats.len(),
PruningIntermediate::IntermediateResult(pruning_results) => {
PropagatedIntermediate::IntermediateStats(column_stats) => column_stats.len(),
PropagatedIntermediate::IntermediateResult(pruning_results) => {
pruning_results.len()
}
}
Expand Down
36 changes: 18 additions & 18 deletions datafusion/physical-expr/src/expressions/binary.rs
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ use datafusion_expr::statistics::{
use datafusion_expr::{ColumnarValue, Operator};
use datafusion_physical_expr_common::datum::{apply, apply_cmp};
use datafusion_physical_expr_common::physical_expr::{
ColumnStats, NullStats, PruningContext, PruningIntermediate, PruningOutcome,
ColumnStats, NullStats, PropagatedIntermediate, PruningContext, PruningOutcome,
PruningResults,
};
use std::cmp::Ordering;
Expand Down Expand Up @@ -304,18 +304,18 @@ impl PhysicalExpr for BinaryExpr {
.map(ColumnarValue::Array)
}

fn evaluate_pruning(
fn evaluate_statistics_vectorized(
&self,
ctx: Arc<PruningContext>,
) -> Result<Option<PruningIntermediate>> {
) -> Result<Option<PropagatedIntermediate>> {
use Operator::*;

let is_cmp = matches!(self.op, Eq | NotEq | Lt | LtEq | Gt | GtEq);
if is_cmp {
return self.evaluate_cmp_pruning(ctx);
}

self.evaluate_pruning_binary_op_default(&ctx)
self.evaluate_statistics_vectorized_binary_op_default(&ctx)
}

fn children(&self) -> Vec<&Arc<dyn PhysicalExpr>> {
Expand Down Expand Up @@ -660,19 +660,19 @@ impl BinaryExpr {
fn evaluate_cmp_pruning(
&self,
ctx: Arc<PruningContext>,
) -> Result<Option<PruningIntermediate>> {
let left = self.left.evaluate_pruning(Arc::clone(&ctx))?;
let right = self.right.evaluate_pruning(ctx)?;
) -> Result<Option<PropagatedIntermediate>> {
let left = self.left.evaluate_statistics_vectorized(Arc::clone(&ctx))?;
let right = self.right.evaluate_statistics_vectorized(ctx)?;

let (left_stats, right_stats) = match (left, right) {
(
Some(PruningIntermediate::IntermediateStats(ls)),
Some(PruningIntermediate::IntermediateStats(rs)),
Some(PropagatedIntermediate::IntermediateStats(ls)),
Some(PropagatedIntermediate::IntermediateStats(rs)),
) => (ls, rs),
(None, _) | (_, None) => return Ok(None),
(left_child, right_child) => {
return internal_err!(
"For cmp operators (e.g. >), both of its child must be `PruningIntermediate::intermediateStats`, got left child {:?}, right child {:?}",
"For cmp operators (e.g. >), both of its child must be `PropagatedIntermediate::IntermediateStats`, got left child {:?}, right child {:?}",
left_child,
right_child
);
Expand All @@ -697,12 +697,12 @@ impl BinaryExpr {
// If either side has missing range stats, the result stat should also be
// missing for all containers
let Some(left_range) = left_stats.range_stats() else {
return Ok(Some(PruningIntermediate::IntermediateResult(
return Ok(Some(PropagatedIntermediate::IntermediateResult(
PruningResults::unknown(num_containers),
)));
};
let Some(right_range) = right_stats.range_stats() else {
return Ok(Some(PruningIntermediate::IntermediateResult(
return Ok(Some(PropagatedIntermediate::IntermediateResult(
PruningResults::unknown(num_containers),
)));
};
Expand All @@ -715,7 +715,7 @@ impl BinaryExpr {
let (Some(l_mins), Some(l_maxs), Some(r_mins), Some(r_maxs)) =
(l_mins, l_maxs, r_mins, r_maxs)
else {
return Ok(Some(PruningIntermediate::IntermediateResult(
return Ok(Some(PropagatedIntermediate::IntermediateResult(
PruningResults::unknown(num_containers),
)));
};
Expand Down Expand Up @@ -805,17 +805,17 @@ impl BinaryExpr {
and_kleene(&range_results, &unknown_nulls)?
};

Ok(Some(PruningIntermediate::IntermediateResult(
Ok(Some(PropagatedIntermediate::IntermediateResult(
PruningResults::new(combined_results),
)))
}

/// For all not-yet supported operators, return the default result (Unknown stat
/// for all containers)
fn evaluate_pruning_binary_op_default(
fn evaluate_statistics_vectorized_binary_op_default(
&self,
ctx: &Arc<PruningContext>,
) -> Result<Option<PruningIntermediate>> {
) -> Result<Option<PropagatedIntermediate>> {
use Operator::*;

let num_containers = ctx.statistics().num_containers();
Expand All @@ -840,12 +840,12 @@ impl BinaryExpr {
}

if is_predicate {
return Ok(Some(PruningIntermediate::IntermediateResult(
return Ok(Some(PropagatedIntermediate::IntermediateResult(
PruningResults::unknown(num_containers),
)));
}

Ok(Some(PruningIntermediate::IntermediateStats(
Ok(Some(PropagatedIntermediate::IntermediateStats(
ColumnStats::new(None, None, num_containers),
)))
}
Expand Down
10 changes: 5 additions & 5 deletions datafusion/physical-expr/src/expressions/column.rs
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ use datafusion_common::tree_node::{Transformed, TreeNode};
use datafusion_common::{Column as DFColumn, Result, internal_err, plan_err};
use datafusion_expr::ColumnarValue;
use datafusion_physical_expr_common::physical_expr::{
ColumnStats, NullStats, PruningContext, PruningIntermediate, RangeStats,
ColumnStats, NullStats, PruningContext, PropagatedIntermediate, RangeStats,
};

/// Represents the column at a given index in a RecordBatch
Expand Down Expand Up @@ -148,15 +148,15 @@ impl PhysicalExpr for Column {
}

/// Read the column statistics from `PruningContext`, transform the input stat
/// to the stats in the internal representation (`PruningIntermediate`)
/// to the stats in the internal representation (`PropagatedIntermediate`)
///
/// The implemented statistics types are:
/// - Range stat (min/max)
/// - Null stat
fn evaluate_pruning(
fn evaluate_statistics_vectorized(
&self,
ctx: Arc<PruningContext>,
) -> Result<Option<PruningIntermediate>> {
) -> Result<Option<PropagatedIntermediate>> {
let pruning_column = DFColumn::from_name(self.name());
let stats = ctx.statistics();
let num_containers = stats.num_containers();
Expand All @@ -181,7 +181,7 @@ impl PhysicalExpr for Column {
.and_then(|arr| arr.as_any().downcast_ref::<UInt64Array>());
let null_stats = NullStats::new(null_counts, row_counts)?;

Ok(Some(PruningIntermediate::IntermediateStats(
Ok(Some(PropagatedIntermediate::IntermediateStats(
ColumnStats::new(range_stats, null_stats, num_containers),
)))
}
Expand Down
8 changes: 4 additions & 4 deletions datafusion/physical-expr/src/expressions/literal.rs
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ use datafusion_expr_common::columnar_value::ColumnarValue;
use datafusion_expr_common::interval_arithmetic::Interval;
use datafusion_expr_common::sort_properties::{ExprProperties, SortProperties};
use datafusion_physical_expr_common::physical_expr::{
ColumnStats, NullPresence, NullStats, PruningContext, PruningIntermediate, RangeStats,
ColumnStats, NullPresence, NullStats, PruningContext, PropagatedIntermediate, RangeStats,
};

/// Represents a literal value
Expand Down Expand Up @@ -115,10 +115,10 @@ impl PhysicalExpr for Literal {
Ok(ColumnarValue::Scalar(self.value.clone()))
}

fn evaluate_pruning(
fn evaluate_statistics_vectorized(
&self,
ctx: Arc<PruningContext>,
) -> Result<Option<PruningIntermediate>> {
) -> Result<Option<PropagatedIntermediate>> {
let length = ctx.statistics().num_containers();
let range = RangeStats::new_constant(self.value.clone(), length)?;

Expand All @@ -127,7 +127,7 @@ impl PhysicalExpr for Literal {
false => NullStats::from_uniform_presence(NullPresence::NoNull, length),
};

Ok(Some(PruningIntermediate::IntermediateStats(
Ok(Some(PropagatedIntermediate::IntermediateStats(
ColumnStats::new(Some(range), Some(null), length),
)))
}
Expand Down
Loading