Skip to content

Commit 85bc5ef

Browse files
authored
fix: Projection stats Absent for columns referenced >1 time (apache#22679)
## Which issue does this PR close? - Closes apache#22678. ## Rationale for this change `ProjectionExprs::project_statistics` uses `std::mem::take` to move an input column's `ColumnStatistics` into the output when given a direct column reference. This means if the column is referenced again (either directly or in a `CAST` expression), the statistics are `Absent`. The simple fix is to just `clone` instead of `take`. This pattern crops up in TPC-DS q54, which includes a CTE that projects both `d_date_sk` and `CAST(d_date_sk AS Float64)`, but it's a more general bug. ## What changes are included in this PR? * Fix bug * Add unit tests ## Are these changes tested? Yes; new test added. ## Are there any user-facing changes? No.
1 parent 73e3c2a commit 85bc5ef

1 file changed

Lines changed: 51 additions & 1 deletion

File tree

‎datafusion/physical-expr/src/projection.rs‎

Lines changed: 51 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -661,7 +661,7 @@ impl ProjectionExprs {
661661
for proj_expr in self.exprs.iter() {
662662
let expr = &proj_expr.expr;
663663
let col_stats = if let Some(col) = expr.downcast_ref::<Column>() {
664-
std::mem::take(&mut stats.column_statistics[col.index()])
664+
stats.column_statistics[col.index()].clone()
665665
} else if let Some(literal) = expr.downcast_ref::<Literal>() {
666666
// Handle literal expressions (constants) by calculating proper statistics
667667
let data_type = expr.data_type(output_schema)?;
@@ -2866,6 +2866,56 @@ pub(crate) mod tests {
28662866
Ok(())
28672867
}
28682868

2869+
#[test]
2870+
fn test_project_statistics_duplicate_column() -> Result<()> {
2871+
let input_stats = get_stats();
2872+
let col0 = input_stats.column_statistics[0].clone();
2873+
let projection = ProjectionExprs::new([
2874+
ProjectionExpr::new(Arc::new(Column::new("col0", 0)), "a"),
2875+
ProjectionExpr::new(Arc::new(Column::new("col0", 0)), "b"),
2876+
]);
2877+
2878+
let output_schema = projection.project_schema(&get_schema())?;
2879+
let output_stats = projection.project_statistics(input_stats, &output_schema)?;
2880+
2881+
assert_eq!(output_stats.column_statistics, vec![col0.clone(), col0]);
2882+
Ok(())
2883+
}
2884+
2885+
#[test]
2886+
fn test_project_statistics_column_and_cast() -> Result<()> {
2887+
let input_stats = get_stats();
2888+
let col0 = input_stats.column_statistics[0].clone();
2889+
let projection = ProjectionExprs::new([
2890+
ProjectionExpr::new(Arc::new(Column::new("col0", 0)), "num"),
2891+
ProjectionExpr::new(
2892+
Arc::new(CastExpr::new(
2893+
Arc::new(Column::new("col0", 0)),
2894+
DataType::Int32,
2895+
None,
2896+
)),
2897+
"casted",
2898+
),
2899+
]);
2900+
2901+
let output_schema = projection.project_schema(&get_schema())?;
2902+
let output_stats = projection.project_statistics(input_stats, &output_schema)?;
2903+
2904+
assert_eq!(output_stats.column_statistics[0], col0);
2905+
assert_eq!(
2906+
output_stats.column_statistics[1],
2907+
ColumnStatistics {
2908+
min_value: Precision::Exact(ScalarValue::Int32(Some(-4))),
2909+
max_value: Precision::Exact(ScalarValue::Int32(Some(21))),
2910+
distinct_count: Precision::Exact(5),
2911+
null_count: Precision::Exact(0),
2912+
sum_value: Precision::Absent,
2913+
byte_size: Precision::Absent,
2914+
}
2915+
);
2916+
Ok(())
2917+
}
2918+
28692919
#[test]
28702920
fn test_project_statistics_primitive_width_only() -> Result<()> {
28712921
let input_stats = get_stats();

0 commit comments

Comments
 (0)