Skip to content

Commit 56b9306

Browse files
aaron-perpetualcetra3
authored andcommitted
fix: preserve compound expression equivalences through projection (apache#25922)
## Which issue does this PR close? - Closes apache#25921. ## Rationale for this change Valid queries that sort by a struct field after a left join can fail physical plan validation because a projection loses the equivalence between the field access and an extracted column. ## What changes are included in this PR? Reorder `EquivalenceGroup::project_expr_indirect` to reconstruct an expression from its projected children before substituting an equivalent expression. This preserves equivalences such as `get_field(n, 'label') = extracted_label` when both representations survive the projection. ## What is the testing strategy for this PR? Add SQL regression tests in `joins.slt` covering a left join followed by sorting on a struct field, both with and without an explicitly projected scalar alias. The tests use two partitions to exercise the local sort and sort-preserving merge, and verify the ordered results. The extended workspace tests, physical expression tests, and full lint suite pass. ## Are there any user-facing changes? Affected queries now execute successfully instead of failing physical plan validation. No public API changes.
1 parent ae39a99 commit 56b9306

2 files changed

Lines changed: 88 additions & 25 deletions

File tree

‎datafusion/physical-expr/src/equivalence/class.rs‎

Lines changed: 29 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -624,8 +624,8 @@ impl EquivalenceGroup {
624624
.filter(|req| !self.is_uniform_constant(&req.expr))
625625
}
626626

627-
/// Perform an indirect projection of `expr` by consulting the equivalence
628-
/// classes.
627+
/// Projects `expr` through its children or an equivalent expression in the
628+
/// mapping, preferring to preserve its structure.
629629
fn project_expr_indirect(
630630
aug_mapping: &AugmentedMapping,
631631
expr: &Arc<dyn PhysicalExpr>,
@@ -635,8 +635,32 @@ impl EquivalenceGroup {
635635
return Some(Arc::clone(expr));
636636
}
637637

638-
// The given expression is not inside the mapping, so we try to project
639-
// indirectly using equivalence classes.
638+
// Preserve expression structure when its children can be projected.
639+
// Otherwise, projecting a class such as a + 1 = b can collapse it to
640+
// b = b, losing the equivalence even when a survives the projection.
641+
let children = expr.children();
642+
if !children.is_empty() {
643+
let projected_children = children
644+
.into_iter()
645+
.map(|child| {
646+
// First, we try to project children with an exact match. If
647+
// we are unable to do this, we consult equivalence classes.
648+
if let Some((targets, _)) = aug_mapping.get(child) {
649+
// If we match the source, we can project directly:
650+
let (target, _) = targets.first();
651+
Some(Arc::clone(target))
652+
} else {
653+
Self::project_expr_indirect(aug_mapping, child)
654+
}
655+
})
656+
.collect::<Option<Vec<_>>>();
657+
if let Some(children) = projected_children {
658+
return Some(Arc::clone(expr).with_new_children(children).unwrap());
659+
}
660+
}
661+
662+
// Project expressions whose children could not be projected by consulting
663+
// equivalence classes.
640664
for (targets, eq_class) in aug_mapping.values() {
641665
// If we match an equivalent expression to a source expression in
642666
// the mapping, then we can project. For example, if we have the
@@ -647,27 +671,7 @@ impl EquivalenceGroup {
647671
return Some(Arc::clone(target));
648672
}
649673
}
650-
// Project a non-leaf expression by projecting its children.
651-
let children = expr.children();
652-
if children.is_empty() {
653-
// A leaf expression should be inside the mapping.
654-
return None;
655-
}
656-
children
657-
.into_iter()
658-
.map(|child| {
659-
// First, we try to project children with an exact match. If
660-
// we are unable to do this, we consult equivalence classes.
661-
if let Some((targets, _)) = aug_mapping.get(child) {
662-
// If we match the source, we can project directly:
663-
let (target, _) = targets.first();
664-
Some(Arc::clone(target))
665-
} else {
666-
Self::project_expr_indirect(aug_mapping, child)
667-
}
668-
})
669-
.collect::<Option<Vec<_>>>()
670-
.map(|children| Arc::clone(expr).with_new_children(children).unwrap())
674+
None
671675
}
672676

673677
fn augment_projection_mapping<'a>(

‎datafusion/sqllogictest/test_files/joins.slt‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6299,3 +6299,62 @@ reset datafusion.optimizer.repartition_file_scans;
62996299

63006300
statement ok
63016301
set datafusion.execution.target_partitions = 4;
6302+
6303+
# Projecting away a temporary join-filter column must preserve the equivalence
6304+
# between a struct field and its extracted scalar column. The parallel sort and
6305+
# merge can use different members of that equivalence class.
6306+
statement ok
6307+
set datafusion.execution.target_partitions = 2;
6308+
6309+
statement ok
6310+
CREATE TABLE projection_events (id BIGINT) AS VALUES (1), (2), (3);
6311+
6312+
# UNION ALL gives the right input two partitions, so ordering needs a merge.
6313+
statement ok
6314+
CREATE TABLE projection_networks AS
6315+
SELECT 'b' AS label UNION ALL SELECT 'a' AS label;
6316+
6317+
query TT
6318+
EXPLAIN SELECT * FROM projection_networks;
6319+
----
6320+
logical_plan TableScan: projection_networks projection=[label]
6321+
physical_plan DataSourceExec: partitions=2, partition_sizes=[1, 1]
6322+
6323+
query I?
6324+
SELECT id, n
6325+
FROM projection_events
6326+
LEFT JOIN (
6327+
SELECT named_struct('label', label) AS n FROM projection_networks
6328+
) ON id > 1
6329+
ORDER BY id, n['label'];
6330+
----
6331+
1 NULL
6332+
2 {label: a}
6333+
2 {label: b}
6334+
3 {label: a}
6335+
3 {label: b}
6336+
6337+
# Also exercise the equivalence with an explicitly projected scalar alias.
6338+
query I?
6339+
SELECT id, n
6340+
FROM projection_events
6341+
LEFT JOIN (
6342+
SELECT named_struct('label', label) AS n, label AS extracted
6343+
FROM projection_networks
6344+
) ON id > 1
6345+
ORDER BY id, extracted;
6346+
----
6347+
1 NULL
6348+
2 {label: a}
6349+
2 {label: b}
6350+
3 {label: a}
6351+
3 {label: b}
6352+
6353+
statement ok
6354+
DROP TABLE projection_events;
6355+
6356+
statement ok
6357+
DROP TABLE projection_networks;
6358+
6359+
statement ok
6360+
set datafusion.execution.target_partitions = 4;

0 commit comments

Comments
 (0)