Skip to content

Commit 8c7f9b3

Browse files
committed
fix: keep a literal out of scan equivalence classes for complex_expr = literal filters
`FileScanConfig::add_filter_equivalence_info` turned every equality of the source's exact filter into an equivalence class. For `complex_expr = literal` that put the literal in a class, and normalization then replaced the literal inside `complex_expr` with the class's canonical expression. A partial aggregate that relied on the constant group key then failed `SanityCheckPlan`. Follow `FilterExec`: only an equality with a column on one side becomes an equivalence class, and any other equality is recorded as a constant. Closes apache#27287. Claude-Session: https://claude.ai/code/session_01USkuh48jV15LVKgyqXmCod
1 parent c8265b6 commit 8c7f9b3

2 files changed

Lines changed: 65 additions & 15 deletions

File tree

‎datafusion/datasource/src/file_scan_config/mod.rs‎

Lines changed: 31 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ use datafusion_physical_expr::expressions::{BinaryExpr, Column};
4040
use datafusion_physical_expr::projection::{ProjectionExprs, ProjectionMapping};
4141
use datafusion_physical_expr::utils::reassign_expr_columns;
4242
use datafusion_physical_expr::{
43-
EquivalenceProperties, Partitioning, RangePartitioning, split_conjunction,
43+
ConstExpr, EquivalenceProperties, Partitioning, RangePartitioning, split_conjunction,
4444
};
4545
use datafusion_physical_expr_adapter::PhysicalExprAdapterFactory;
4646
use datafusion_physical_expr_common::physical_expr::{PhysicalExpr, is_volatile};
@@ -1384,23 +1384,39 @@ impl FileScanConfig {
13841384
eq_properties: &mut EquivalenceProperties,
13851385
schema: &Schema,
13861386
) -> Result<()> {
1387-
// Gather valid equality pairs from the filter expression
1388-
let equal_pairs = split_conjunction(filter).into_iter().filter_map(|expr| {
1387+
let input_eq_properties = eq_properties.clone();
1388+
let mut constants = vec![];
1389+
for expr in split_conjunction(filter) {
13891390
// Ignore any binary expressions that reference non-existent columns in the current schema
13901391
// (e.g. due to unnecessary projections being removed)
1391-
reassign_expr_columns(Arc::clone(expr), schema)
1392-
.ok()
1393-
.and_then(|expr| match expr.downcast_ref::<BinaryExpr>() {
1394-
Some(expr) if expr.op() == &Operator::Eq => {
1395-
Some((Arc::clone(expr.left()), Arc::clone(expr.right())))
1396-
}
1397-
_ => None,
1398-
})
1399-
});
1400-
1401-
for (lhs, rhs) in equal_pairs {
1402-
eq_properties.add_equal_conditions(lhs, rhs)?
1392+
let Ok(expr) = reassign_expr_columns(Arc::clone(expr), schema) else {
1393+
continue;
1394+
};
1395+
let Some(binary) = expr.downcast_ref::<BinaryExpr>() else {
1396+
continue;
1397+
};
1398+
if binary.op() != &Operator::Eq {
1399+
continue;
1400+
}
1401+
// Follow `FilterExec`: only an equality with a column on one side
1402+
// becomes an equivalence class. A pair like `complex_expr = literal`
1403+
// would put the literal in a class, and normalization would then
1404+
// replace that literal wherever it appears (for example inside
1405+
// `complex_expr` itself) with the canonical expression of the
1406+
// class. Such a pair still makes `complex_expr` a constant.
1407+
if binary.left().is::<Column>() || binary.right().is::<Column>() {
1408+
eq_properties.add_equal_conditions(
1409+
Arc::clone(binary.left()),
1410+
Arc::clone(binary.right()),
1411+
)?;
1412+
} else {
1413+
constants.extend(ConstExpr::collect_predicate_constants(
1414+
&input_eq_properties,
1415+
&expr,
1416+
));
1417+
}
14031418
}
1419+
eq_properties.add_constants(constants)?;
14041420

14051421
Ok(())
14061422
}

‎datafusion/sqllogictest/test_files/push_down_filter_regression.slt‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1019,3 +1019,37 @@ drop table window_ab;
10191019

10201020
statement ok
10211021
set datafusion.explain.logical_plan_only = false;
1022+
1023+
# Regression test for https://github.com/apache/datafusion/issues/27287
1024+
# A scan's exact filter `complex_expr = literal` must not put the literal in an
1025+
# equivalence class. Otherwise normalization rewrites the literal inside
1026+
# `complex_expr`, and an aggregate that relies on the constant group key fails
1027+
# `SanityCheckPlan`.
1028+
1029+
statement ok
1030+
set datafusion.execution.parquet.pushdown_filters = true;
1031+
1032+
statement ok
1033+
COPY (
1034+
SELECT * FROM (VALUES ('x', 'y'), ('x', 'z'), ('w', 'y')) AS v(a, b)
1035+
) TO 'test_files/scratch/push_down_filter_regression/case_key_pushdown.parquet'
1036+
STORED AS PARQUET;
1037+
1038+
statement ok
1039+
CREATE EXTERNAL TABLE case_key_pushdown (a VARCHAR, b VARCHAR)
1040+
STORED AS PARQUET
1041+
LOCATION 'test_files/scratch/push_down_filter_regression/case_key_pushdown.parquet';
1042+
1043+
query TI
1044+
SELECT CASE WHEN a = 'x' AND b = 'y' THEN b WHEN a = 'x' THEN 'z' END AS e, count(*)
1045+
FROM case_key_pushdown
1046+
GROUP BY e
1047+
HAVING e = 'y';
1048+
----
1049+
y 1
1050+
1051+
statement ok
1052+
drop table case_key_pushdown;
1053+
1054+
statement ok
1055+
reset datafusion.execution.parquet.pushdown_filters;

0 commit comments

Comments
 (0)