Skip to content

Commit 994947b

Browse files
cetra3claude
andcommitted
fix: keep a constant sort key when a projection reads a duplicate alias
When a projection names one expression twice, under two aliases, an equivalence class is recorded with the first alias only. A projection source that reads the second alias is equal to a member of that class only after normalization, but `augment_projection_mapping` looked up the class of the source as written, so the class and its constant were lost. `EnsureRequirements` and a later `ProjectionPushdown` then disagreed on whether a sort key was constant, and `SanityCheckPlan` rejected the plan. Fall back to the class of the normalized source when the source as written is in no class. Closes apache#26391. Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent af752e5 commit 994947b

2 files changed

Lines changed: 146 additions & 1 deletion

File tree

‎datafusion/physical-expr/src/equivalence/class.rs‎

Lines changed: 111 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -670,7 +670,17 @@ impl EquivalenceGroup {
670670
mapping
671671
.iter()
672672
.map(|(k, v)| {
673-
let eq_class = self.get_equivalence_class(k);
673+
// A source expression may only be in a class in its normalized
674+
// form. For example, if `a == b`, `a + c` is in a class, and
675+
// the mapping has `b + c`, then the class of `b + c` is found
676+
// only through `a + c`. A leaf that is in no class normalizes
677+
// to itself, so only expressions with children are normalized.
678+
let eq_class = self.get_equivalence_class(k).or_else(|| {
679+
if k.children().is_empty() {
680+
return None;
681+
}
682+
self.get_equivalence_class(&self.normalize_expr(Arc::clone(k)))
683+
});
674684
(k, (v, eq_class))
675685
})
676686
.collect()
@@ -1276,6 +1286,106 @@ mod tests {
12761286
Ok(())
12771287
}
12781288

1289+
#[test]
1290+
fn test_project_constant_of_equivalent_source() -> Result<()> {
1291+
// - columns: [a, b, c].
1292+
// - "a" and "b" in the same equivalence class, and a + c = 7.
1293+
// - then after a b + c projection, col(0) must be the constant 7.
1294+
let schema = Arc::new(Schema::new(vec![
1295+
Field::new("a", DataType::Int32, false),
1296+
Field::new("b", DataType::Int32, false),
1297+
Field::new("c", DataType::Int32, false),
1298+
]));
1299+
let mut group = EquivalenceGroup::default();
1300+
group.add_equal_conditions(col("a", &schema)?, col("b", &schema)?);
1301+
group.add_equal_conditions(
1302+
binary(
1303+
col("a", &schema)?,
1304+
Operator::Plus,
1305+
col("c", &schema)?,
1306+
&schema,
1307+
)?,
1308+
lit(7),
1309+
);
1310+
1311+
let projected_schema =
1312+
Arc::new(Schema::new(vec![Field::new("b+c", DataType::Int32, false)]));
1313+
let mapping = std::iter::once((
1314+
binary(
1315+
col("b", &schema)?,
1316+
Operator::Plus,
1317+
col("c", &schema)?,
1318+
&schema,
1319+
)?,
1320+
vec![(col("b+c", &projected_schema)?, 0)].into(),
1321+
))
1322+
.collect::<ProjectionMapping>();
1323+
1324+
let projected = group.project(&mapping);
1325+
1326+
assert_eq!(
1327+
projected.is_expr_constant(&col("b+c", &projected_schema)?),
1328+
Some(AcrossPartitions::Uniform(Some(ScalarValue::Int32(Some(7)))))
1329+
);
1330+
1331+
Ok(())
1332+
}
1333+
1334+
#[test]
1335+
fn test_project_class_of_equivalent_source() -> Result<()> {
1336+
// - columns: [a, b, c, d].
1337+
// - "a" and "b" in the same equivalence class, and a + c = d * 2.
1338+
// - then after a b + c, d projection, col(0) and col(1) * 2 must be
1339+
// in the same class.
1340+
let schema = abc_schema();
1341+
let mut group = EquivalenceGroup::default();
1342+
group.add_equal_conditions(col("a", &schema)?, col("b", &schema)?);
1343+
group.add_equal_conditions(
1344+
binary(
1345+
col("a", &schema)?,
1346+
Operator::Plus,
1347+
col("c", &schema)?,
1348+
&schema,
1349+
)?,
1350+
binary(col("d", &schema)?, Operator::Multiply, lit(2), &schema)?,
1351+
);
1352+
1353+
let projected_schema = Arc::new(Schema::new(vec![
1354+
Field::new("b+c", DataType::Int32, false),
1355+
Field::new("d", DataType::Int32, false),
1356+
]));
1357+
let mapping = [
1358+
(
1359+
binary(
1360+
col("b", &schema)?,
1361+
Operator::Plus,
1362+
col("c", &schema)?,
1363+
&schema,
1364+
)?,
1365+
vec![(col("b+c", &projected_schema)?, 0)].into(),
1366+
),
1367+
(
1368+
col("d", &schema)?,
1369+
vec![(col("d", &projected_schema)?, 1)].into(),
1370+
),
1371+
]
1372+
.into_iter()
1373+
.collect::<ProjectionMapping>();
1374+
1375+
let projected = group.project(&mapping);
1376+
1377+
let first_normalized = projected.normalize_expr(col("b+c", &projected_schema)?);
1378+
let second_normalized = projected.normalize_expr(binary(
1379+
col("d", &projected_schema)?,
1380+
Operator::Multiply,
1381+
lit(2),
1382+
&projected_schema,
1383+
)?);
1384+
assert!(first_normalized.eq(&second_normalized));
1385+
1386+
Ok(())
1387+
}
1388+
12791389
/// Builds a group from a list of equated column pairs.
12801390
fn group_of(schema: &SchemaRef, pairs: &[(&str, &str)]) -> Result<EquivalenceGroup> {
12811391
let mut group = EquivalenceGroup::default();

‎datafusion/sqllogictest/test_files/projection_pushdown.slt‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2622,3 +2622,38 @@ DROP TABLE union_leaf_c;
26222622

26232623
statement ok
26242624
DROP TABLE union_leaf_d;
2625+
2626+
#####################
2627+
# A sort key that a filter makes constant, read through a duplicate extraction alias
2628+
# See https://github.com/apache/datafusion/issues/26391
2629+
#####################
2630+
2631+
# The filter and the projection each extract `s['k']`, under two aliases. The
2632+
# filter moves into the scan, and the scan computes both aliases. The sort key
2633+
# `c` reads the second alias, and the filter's constant reads the first. The
2634+
# planner must still know that `c` is a constant after the projections merge.
2635+
statement ok
2636+
SET datafusion.execution.parquet.pushdown_filters = true;
2637+
2638+
statement ok
2639+
COPY (SELECT * FROM (VALUES ({k: 'a'}, 1), ({k: 'b'}, 2)) AS v(s, id))
2640+
TO 'test_files/scratch/projection_pushdown/duplicate_alias.parquet'
2641+
STORED AS PARQUET;
2642+
2643+
statement ok
2644+
CREATE EXTERNAL TABLE duplicate_alias STORED AS PARQUET
2645+
LOCATION 'test_files/scratch/projection_pushdown/duplicate_alias.parquet';
2646+
2647+
query TI
2648+
SELECT upper(s['k']) AS c, synth
2649+
FROM (SELECT s, id + 1 AS synth FROM duplicate_alias) AS v
2650+
WHERE upper(s['k']) = 'A'
2651+
ORDER BY synth, c;
2652+
----
2653+
A 2
2654+
2655+
statement ok
2656+
DROP TABLE duplicate_alias;
2657+
2658+
statement ok
2659+
RESET datafusion.execution.parquet.pushdown_filters;

0 commit comments

Comments
 (0)