Skip to content

Commit 29dcae6

Browse files
kumarUjjawalalamb
authored andcommitted
fix: optimize_projections failure with struct-field join keys (apache#22903)
## Which issue does this PR close? <!-- We generally require a GitHub issue to be filed for all bug fixes and enhancements and this helps us generate change logs for our releases. You can link an issue to this PR using the GitHub syntax. For example `Closes apache#123` indicates that this PR will close issue apache#123. --> - Closes apache#22895. ## Rationale for this change Join-key extraction can add helper projections and change a child plan's output schema. Some parent nodes cache their schema, so after the child rewrite they can still expose stale fields. Later projection pruning then uses the stale schema and fails with missing-column errors. <!-- Why are you proposing this change? If this is already explained clearly in the issue then this section is not needed. Explaining clearly why changes are proposed helps reviewers understand your changes and offer better suggestions for fixes. --> ## What changes are included in this PR? - Refresh a parent schema only when a rewritten child reports an actual schema change. - Re-check whether the plan has subqueries before each rule, so plans that are decorrelated by earlier rules can use the in-place path in the same optimizer pass. - Add regression coverage for the reported struct-field join-key failures. - Add a regression case for union output labels while union-to-filter rewriting is enabled. - Add a focused optimizer unit test for parent schema refresh after child schema changes. - Update one existing optimized-plan expectation. The new plan removes a redundant dedup step; the surrounding semi joins already preserve distinctness. <!-- There is no need to duplicate the description in the issue here but it is sometimes worth providing a summary of the individual changes in this PR. --> ## Are these changes tested? Yes <!-- We typically require tests for all PRs in order to: 1. Prevent the code from being accidentally broken by subsequent changes 2. Serve as another way to document the expected behavior of the code If tests are not included in your PR, please explain why (for example, are they covered by existing tests)? --> ## Are there any user-facing changes? No <!-- If there are user-facing changes then we may require documentation to be updated before approving the PR. --> <!-- If there are any breaking changes to public APIs, please add the `api change` label. --> --------- Co-authored-by: Andrew Lamb <[email protected]>
1 parent 911df8a commit 29dcae6

4 files changed

Lines changed: 138 additions & 19 deletions

File tree

‎datafusion/optimizer/src/optimizer.rs‎

Lines changed: 89 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -518,10 +518,23 @@ fn rewrite_plan_in_place(
518518
}
519519
}
520520

521-
// Recurse into children using Arc::make_mut (zero-cost when refcount == 1)
522-
changed |= map_children_mut(plan, |child| {
523-
rewrite_plan_in_place(child, apply_order, rule, config)
521+
let mut child_schema_changed = false;
522+
let children_changed = map_children_mut(plan, |child| {
523+
let old_schema = Arc::clone(child.schema());
524+
let child_changed = rewrite_plan_in_place(child, apply_order, rule, config)?;
525+
if child_changed && old_schema.as_ref() != child.schema().as_ref() {
526+
child_schema_changed = true;
527+
}
528+
Ok(child_changed)
524529
})?;
530+
changed |= children_changed;
531+
532+
if child_schema_changed {
533+
// Child rewrites can change their output schemas. Recompute the current
534+
// node before later rules use positional requirements from that schema.
535+
let owned = std::mem::take(plan);
536+
*plan = owned.recompute_schema()?;
537+
}
525538

526539
// f_up phase
527540
if apply_order == ApplyOrder::BottomUp {
@@ -604,13 +617,11 @@ impl Optimizer {
604617
while i < options.optimizer.max_passes {
605618
log_plan(&format!("Optimizer input (pass {i})"), &new_plan);
606619

607-
// Check once per pass whether the plan contains subquery
608-
// expressions. When there are no subqueries, we use the
609-
// cheaper `rewrite` traversal instead of
610-
// `rewrite_with_subqueries`, avoiding the per-node
611-
// map_subqueries call that walks all expression trees
612-
// via ownership-based transform_down.
613-
let has_subqueries = plan_has_subqueries(&new_plan);
620+
// Track subquery presence across the pass. Refresh after changed
621+
// rules so decorrelation can move later rules onto the in-place
622+
// path; that path refreshes parent schemas after child schemas
623+
// change.
624+
let mut has_subqueries = plan_has_subqueries(&new_plan);
614625

615626
for rule in &self.rules {
616627
// If skipping failed rules, copy plan before attempting to rewrite
@@ -690,6 +701,7 @@ impl Optimizer {
690701
new_plan = data;
691702
observer(&new_plan, rule.as_ref());
692703
if transformed {
704+
has_subqueries = plan_has_subqueries(&new_plan);
693705
log_plan(rule.name(), &new_plan);
694706
} else {
695707
debug!(
@@ -773,13 +785,15 @@ mod tests {
773785

774786
use datafusion_common::tree_node::Transformed;
775787
use datafusion_common::{
776-
DFSchema, DFSchemaRef, DataFusionError, Result, assert_contains, plan_err,
788+
Column, DFSchema, DFSchemaRef, DataFusionError, Result, assert_contains, plan_err,
777789
};
778790
use datafusion_expr::logical_plan::EmptyRelation;
779-
use datafusion_expr::{LogicalPlan, LogicalPlanBuilder, Projection, col, lit};
791+
use datafusion_expr::{
792+
Expr, JoinType, LogicalPlan, LogicalPlanBuilder, Projection, col, lit,
793+
};
780794

781795
use crate::optimizer::Optimizer;
782-
use crate::test::test_table_scan;
796+
use crate::test::{test_table_scan, test_table_scan_with_name};
783797
use crate::{OptimizerConfig, OptimizerContext, OptimizerRule};
784798

785799
use super::ApplyOrder;
@@ -863,6 +877,34 @@ mod tests {
863877
Ok(())
864878
}
865879

880+
#[test]
881+
fn in_place_rewrite_recomputes_parent_schema_when_child_schema_changes() -> Result<()>
882+
{
883+
let left = LogicalPlanBuilder::from(test_table_scan_with_name("left")?)
884+
.project(vec![col("left.a"), col("left.b"), col("left.c")])?
885+
.build()?;
886+
let right = LogicalPlanBuilder::from(test_table_scan_with_name("right")?)
887+
.project(vec![col("right.a"), col("right.b"), col("right.c")])?
888+
.build()?;
889+
let mut plan = LogicalPlanBuilder::from(left)
890+
.join_on(right, JoinType::Inner, [col("left.a").eq(col("right.a"))])?
891+
.build()?;
892+
893+
assert_eq!(plan.schema().fields().len(), 6);
894+
895+
let changed = super::rewrite_plan_in_place(
896+
&mut plan,
897+
ApplyOrder::TopDown,
898+
&KeepOnlyAProjectionRule {},
899+
&OptimizerContext::new(),
900+
)?;
901+
902+
assert!(changed);
903+
assert_eq!(plan.schema().fields().len(), 2);
904+
assert!(plan.schema().has_column_with_unqualified_name("a"));
905+
Ok(())
906+
}
907+
866908
#[test]
867909
fn optimizer_detects_plan_equal_to_the_initial() -> Result<()> {
868910
// Run a goofy optimizer, which rotates projection columns
@@ -980,6 +1022,40 @@ mod tests {
9801022
}
9811023
}
9821024

1025+
#[derive(Default, Debug)]
1026+
struct KeepOnlyAProjectionRule {}
1027+
1028+
impl OptimizerRule for KeepOnlyAProjectionRule {
1029+
fn name(&self) -> &str {
1030+
"keep_only_a_projection"
1031+
}
1032+
1033+
fn apply_order(&self) -> Option<ApplyOrder> {
1034+
Some(ApplyOrder::TopDown)
1035+
}
1036+
1037+
fn supports_rewrite(&self) -> bool {
1038+
true
1039+
}
1040+
1041+
fn rewrite(
1042+
&self,
1043+
plan: LogicalPlan,
1044+
_config: &dyn OptimizerConfig,
1045+
) -> Result<Transformed<LogicalPlan>> {
1046+
let projection = match plan {
1047+
LogicalPlan::Projection(p) => p,
1048+
_ => return Ok(Transformed::no(plan)),
1049+
};
1050+
1051+
let expr = Expr::from(Column::from(projection.schema.qualified_field(0)));
1052+
1053+
Ok(Transformed::yes(LogicalPlan::Projection(
1054+
Projection::try_new(vec![expr], Arc::clone(&projection.input))?,
1055+
)))
1056+
}
1057+
}
1058+
9831059
/// A goofy rule doing rotation of columns in all projections.
9841060
///
9851061
/// Useful to test cycle detection.

‎datafusion/optimizer/tests/optimizer_integration.rs‎

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -275,13 +275,12 @@ fn intersect() -> Result<()> {
275275
format!("{plan}"),
276276
@r"
277277
LeftSemi Join: left.col_int32 = test.col_int32, left.col_utf8 = test.col_utf8
278-
Aggregate: groupBy=[[left.col_int32, left.col_utf8]], aggr=[[]]
279-
LeftSemi Join: left.col_int32 = right.col_int32, left.col_utf8 = right.col_utf8
280-
Aggregate: groupBy=[[left.col_int32, left.col_utf8]], aggr=[[]]
281-
SubqueryAlias: left
282-
TableScan: test projection=[col_int32, col_utf8]
283-
SubqueryAlias: right
278+
LeftSemi Join: left.col_int32 = right.col_int32, left.col_utf8 = right.col_utf8
279+
Aggregate: groupBy=[[left.col_int32, left.col_utf8]], aggr=[[]]
280+
SubqueryAlias: left
284281
TableScan: test projection=[col_int32, col_utf8]
282+
SubqueryAlias: right
283+
TableScan: test projection=[col_int32, col_utf8]
285284
TableScan: test projection=[col_int32, col_utf8]
286285
"
287286
);

‎datafusion/sqllogictest/test_files/projection_pushdown.slt‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2071,6 +2071,42 @@ SELECT s, id FROM simple_struct WHERE s['value'] > 100 AND id < 4;
20712071
{value: 200, label: beta} 2
20722072
{value: 150, label: gamma} 3
20732073

2074+
#####################
2075+
# Section 9: Join key extraction with pruned outputs
2076+
#####################
2077+
2078+
statement ok
2079+
CREATE TABLE issue_22895_rt2 AS SELECT * FROM (VALUES
2080+
(named_struct('msg','user auth failed','sid','a'), 1, 'svc1'),
2081+
(named_struct('msg','login token','sid','b'), 2, 'svc2')
2082+
) v(attributes, id, name);
2083+
2084+
query IT
2085+
SELECT a.id, b.name
2086+
FROM issue_22895_rt2 a JOIN issue_22895_rt2 b
2087+
ON a.attributes['sid'] = b.attributes['sid']
2088+
WHERE a.attributes['msg'] LIKE '%auth%'
2089+
ORDER BY a.id, b.name;
2090+
----
2091+
1 svc1
2092+
2093+
statement ok
2094+
CREATE TABLE issue_22895_rt AS SELECT * FROM (VALUES
2095+
(named_struct('uid','u1','t','t1'), TIMESTAMP '2026-06-08T10:00:00', 'a'),
2096+
(named_struct('uid','u2','t','t2'), TIMESTAMP '2026-06-08T11:00:00', 'b')
2097+
) v(attributes, start_timestamp, span_name);
2098+
2099+
query P
2100+
SELECT r.start_timestamp
2101+
FROM issue_22895_rt r
2102+
JOIN (SELECT attributes['uid'] AS uid FROM issue_22895_rt) f
2103+
ON f.uid = r.attributes['uid']
2104+
WHERE r.attributes['t'] IN (SELECT attributes['t'] FROM issue_22895_rt)
2105+
ORDER BY r.start_timestamp;
2106+
----
2107+
2026-06-08T10:00:00
2108+
2026-06-08T11:00:00
2109+
20742110
# Config reset
20752111

20762112
# The SLT runner sets `target_partitions` to 4 instead of using the default, so

‎datafusion/sqllogictest/test_files/union.slt‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -341,6 +341,14 @@ physical_plan
341341
05)--------FilterExec: id@0 = 1 OR id@0 = 2
342342
06)----------DataSourceExec: partitions=1, partition_sizes=[1]
343343

344+
# Regression: schema recomputation must preserve the unqualified UNION
345+
# output labels while unions_to_filter is enabled.
346+
query IT rowsort
347+
SELECT id, name FROM t1 WHERE id = 1 UNION SELECT id, name FROM t1 WHERE id = 2
348+
----
349+
1 Alex
350+
2 Bob
351+
344352
statement ok
345353
set datafusion.optimizer.enable_unions_to_filter = false;
346354

0 commit comments

Comments
 (0)