Skip to content

Commit 6365f90

Browse files
cetra3adriangb
authored andcommitted
fix: keep a constant sort key when a projection reads a duplicate alias (apache#26397)
When a projection names one expression twice, under two aliases, an equivalence class is recorded with the first alias only. A projection source that reads the second alias is equal to a member of that class only after normalization, but `augment_projection_mapping` looked up the class of the source as written, so the class and its constant were lost. `EnsureRequirements` and a later `ProjectionPushdown` then disagreed on whether a sort key was constant, and `SanityCheckPlan` rejected the plan. Fall back to the class of the normalized source when the source as written is in no class. Closes apache#26391. Co-Authored-By: Claude Opus 5.5 <[email protected]> Carried from apache#26397 (open).
1 parent 905feee commit 6365f90

2 files changed

Lines changed: 146 additions & 1 deletion

File tree

‎datafusion/physical-expr/src/equivalence/class.rs‎

Lines changed: 111 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -681,7 +681,17 @@ impl EquivalenceGroup {
681681
mapping
682682
.iter()
683683
.map(|(k, v)| {
684-
let eq_class = self.get_equivalence_class(k);
684+
// A source expression may only be in a class in its normalized
685+
// form. For example, if `a == b`, `a + c` is in a class, and
686+
// the mapping has `b + c`, then the class of `b + c` is found
687+
// only through `a + c`. A leaf that is in no class normalizes
688+
// to itself, so only expressions with children are normalized.
689+
let eq_class = self.get_equivalence_class(k).or_else(|| {
690+
if k.children().is_empty() {
691+
return None;
692+
}
693+
self.get_equivalence_class(&self.normalize_expr(Arc::clone(k)))
694+
});
685695
(k, (v, eq_class))
686696
})
687697
.collect()
@@ -1278,6 +1288,106 @@ mod tests {
12781288
Ok(())
12791289
}
12801290

1291+
#[test]
1292+
fn test_project_constant_of_equivalent_source() -> Result<()> {
1293+
// - columns: [a, b, c].
1294+
// - "a" and "b" in the same equivalence class, and a + c = 7.
1295+
// - then after a b + c projection, col(0) must be the constant 7.
1296+
let schema = Arc::new(Schema::new(vec![
1297+
Field::new("a", DataType::Int32, false),
1298+
Field::new("b", DataType::Int32, false),
1299+
Field::new("c", DataType::Int32, false),
1300+
]));
1301+
let mut group = EquivalenceGroup::default();
1302+
group.add_equal_conditions(col("a", &schema)?, col("b", &schema)?);
1303+
group.add_equal_conditions(
1304+
binary(
1305+
col("a", &schema)?,
1306+
Operator::Plus,
1307+
col("c", &schema)?,
1308+
&schema,
1309+
)?,
1310+
lit(7),
1311+
);
1312+
1313+
let projected_schema =
1314+
Arc::new(Schema::new(vec![Field::new("b+c", DataType::Int32, false)]));
1315+
let mapping = std::iter::once((
1316+
binary(
1317+
col("b", &schema)?,
1318+
Operator::Plus,
1319+
col("c", &schema)?,
1320+
&schema,
1321+
)?,
1322+
vec![(col("b+c", &projected_schema)?, 0)].into(),
1323+
))
1324+
.collect::<ProjectionMapping>();
1325+
1326+
let projected = group.project(&mapping);
1327+
1328+
assert_eq!(
1329+
projected.is_expr_constant(&col("b+c", &projected_schema)?),
1330+
Some(AcrossPartitions::Uniform(Some(ScalarValue::Int32(Some(7)))))
1331+
);
1332+
1333+
Ok(())
1334+
}
1335+
1336+
#[test]
1337+
fn test_project_class_of_equivalent_source() -> Result<()> {
1338+
// - columns: [a, b, c, d].
1339+
// - "a" and "b" in the same equivalence class, and a + c = d * 2.
1340+
// - then after a b + c, d projection, col(0) and col(1) * 2 must be
1341+
// in the same class.
1342+
let schema = abc_schema();
1343+
let mut group = EquivalenceGroup::default();
1344+
group.add_equal_conditions(col("a", &schema)?, col("b", &schema)?);
1345+
group.add_equal_conditions(
1346+
binary(
1347+
col("a", &schema)?,
1348+
Operator::Plus,
1349+
col("c", &schema)?,
1350+
&schema,
1351+
)?,
1352+
binary(col("d", &schema)?, Operator::Multiply, lit(2), &schema)?,
1353+
);
1354+
1355+
let projected_schema = Arc::new(Schema::new(vec![
1356+
Field::new("b+c", DataType::Int32, false),
1357+
Field::new("d", DataType::Int32, false),
1358+
]));
1359+
let mapping = [
1360+
(
1361+
binary(
1362+
col("b", &schema)?,
1363+
Operator::Plus,
1364+
col("c", &schema)?,
1365+
&schema,
1366+
)?,
1367+
vec![(col("b+c", &projected_schema)?, 0)].into(),
1368+
),
1369+
(
1370+
col("d", &schema)?,
1371+
vec![(col("d", &projected_schema)?, 1)].into(),
1372+
),
1373+
]
1374+
.into_iter()
1375+
.collect::<ProjectionMapping>();
1376+
1377+
let projected = group.project(&mapping);
1378+
1379+
let first_normalized = projected.normalize_expr(col("b+c", &projected_schema)?);
1380+
let second_normalized = projected.normalize_expr(binary(
1381+
col("d", &projected_schema)?,
1382+
Operator::Multiply,
1383+
lit(2),
1384+
&projected_schema,
1385+
)?);
1386+
assert!(first_normalized.eq(&second_normalized));
1387+
1388+
Ok(())
1389+
}
1390+
12811391
/// Builds a group from a list of equated column pairs.
12821392
fn group_of(schema: &SchemaRef, pairs: &[(&str, &str)]) -> Result<EquivalenceGroup> {
12831393
let mut group = EquivalenceGroup::default();

‎datafusion/sqllogictest/test_files/projection_pushdown.slt‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2560,3 +2560,38 @@ RESET datafusion.optimizer.max_passes;
25602560

25612561
statement ok
25622562
SET datafusion.execution.target_partitions = 4;
2563+
2564+
#####################
2565+
# A sort key that a filter makes constant, read through a duplicate extraction alias
2566+
# See https://github.com/apache/datafusion/issues/26391
2567+
#####################
2568+
2569+
# The filter and the projection each extract `s['k']`, under two aliases. The
2570+
# filter moves into the scan, and the scan computes both aliases. The sort key
2571+
# `c` reads the second alias, and the filter's constant reads the first. The
2572+
# planner must still know that `c` is a constant after the projections merge.
2573+
statement ok
2574+
SET datafusion.execution.parquet.pushdown_filters = true;
2575+
2576+
statement ok
2577+
COPY (SELECT * FROM (VALUES ({k: 'a'}, 1), ({k: 'b'}, 2)) AS v(s, id))
2578+
TO 'test_files/scratch/projection_pushdown/duplicate_alias.parquet'
2579+
STORED AS PARQUET;
2580+
2581+
statement ok
2582+
CREATE EXTERNAL TABLE duplicate_alias STORED AS PARQUET
2583+
LOCATION 'test_files/scratch/projection_pushdown/duplicate_alias.parquet';
2584+
2585+
query TI
2586+
SELECT upper(s['k']) AS c, synth
2587+
FROM (SELECT s, id + 1 AS synth FROM duplicate_alias) AS v
2588+
WHERE upper(s['k']) = 'A'
2589+
ORDER BY synth, c;
2590+
----
2591+
A 2
2592+
2593+
statement ok
2594+
DROP TABLE duplicate_alias;
2595+
2596+
statement ok
2597+
RESET datafusion.execution.parquet.pushdown_filters;

0 commit comments

Comments
 (0)