Skip to content

Commit 29096cf

Browse files
Dandandanclaude
andcommitted
Simplify no-op transformed reporting
- Move duplicated `transformed_if_changed` into `utils.rs` and use it from `decorrelate_predicate_subquery`, `eliminate_cross_join`, `optimize_unions`, and `push_down_join`. - `eliminate_duplicated_expr`: track change inline via length comparison and skip the `Aggregate::try_new`/`Sort` rebuild when no duplicates are present — avoids both the helper call and the schema-recompute. - `optimize_projections`: avoid cloning the whole `Window`/`TableScan` when nothing changes; OR `aggregate_changed` into the recursive result so the `Aggregate` branch reports `Transformed::yes` when FD pruning reduces group_expr/aggr_expr counts. - `optimizer.rs`: extract the `Transformed::yes`/`no` consistency check into `assert_transformed_matches_plan` so the loop stays readable. - `push_down_filter`: revert the simplify-predicate check to a length-based comparison (matches `simplify_predicates` actual semantics — it only changes content via merging, not reordering). Co-Authored-By: Claude Opus 4.7 (1M context) <[email protected]>
1 parent ab1dbb2 commit 29096cf

10 files changed

Lines changed: 137 additions & 126 deletions

File tree

‎datafusion/optimizer/src/decorrelate_predicate_subquery.rs‎

Lines changed: 1 addition & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ use std::sync::Arc;
2222

2323
use crate::decorrelate::PullUpCorrelatedExpr;
2424
use crate::optimizer::ApplyOrder;
25-
use crate::utils::replace_qualified_name;
25+
use crate::utils::{replace_qualified_name, transformed_if_changed};
2626
use crate::{OptimizerConfig, OptimizerRule};
2727

2828
use datafusion_common::alias::AliasGenerator;
@@ -139,17 +139,6 @@ impl OptimizerRule for DecorrelatePredicateSubquery {
139139
}
140140
}
141141

142-
fn transformed_if_changed(
143-
original_plan: LogicalPlan,
144-
new_plan: LogicalPlan,
145-
) -> Transformed<LogicalPlan> {
146-
if new_plan == original_plan {
147-
Transformed::no(original_plan)
148-
} else {
149-
Transformed::yes(new_plan)
150-
}
151-
}
152-
153142
fn rewrite_inner_subqueries(
154143
outer: LogicalPlan,
155144
expr: Expr,

‎datafusion/optimizer/src/eliminate_cross_join.rs‎

Lines changed: 1 addition & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
// under the License.
1717

1818
//! [`EliminateCrossJoin`] converts `CROSS JOIN` to `INNER JOIN` if join predicates are available.
19+
use crate::utils::transformed_if_changed;
1920
use crate::{OptimizerConfig, OptimizerRule};
2021
use std::sync::Arc;
2122

@@ -212,17 +213,6 @@ impl OptimizerRule for EliminateCrossJoin {
212213
}
213214
}
214215

215-
fn transformed_if_changed(
216-
original_plan: LogicalPlan,
217-
new_plan: LogicalPlan,
218-
) -> Transformed<LogicalPlan> {
219-
if new_plan == original_plan {
220-
Transformed::no(original_plan)
221-
} else {
222-
Transformed::yes(new_plan)
223-
}
224-
}
225-
226216
fn rewrite_children(
227217
optimizer: &impl OptimizerRule,
228218
plan: LogicalPlan,

‎datafusion/optimizer/src/eliminate_duplicated_expr.rs‎

Lines changed: 32 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
2020
use crate::optimizer::ApplyOrder;
2121
use crate::{OptimizerConfig, OptimizerRule};
22+
use datafusion_common::HashSet;
2223
use datafusion_common::tree_node::Transformed;
2324
use datafusion_common::{Result, get_required_sort_exprs_indices, internal_err};
2425
use datafusion_expr::logical_plan::LogicalPlan;
@@ -66,32 +67,41 @@ impl OptimizerRule for EliminateDuplicatedExpr {
6667
) -> Result<Transformed<LogicalPlan>> {
6768
match plan {
6869
LogicalPlan::Sort(sort) => {
69-
let original_plan = LogicalPlan::Sort(sort.clone());
70-
let unique_exprs: Vec<_> = sort
70+
let original_len = sort.expr.len();
71+
let dedup_exprs: Vec<_> = sort
7172
.expr
72-
.into_iter()
73+
.iter()
74+
.cloned()
7375
.map(SortExprWrapper)
7476
.collect::<IndexSet<_>>()
7577
.into_iter()
7678
.map(|wrapper| wrapper.0)
7779
.collect();
80+
let dedupe_changed = dedup_exprs.len() != original_len;
7881

79-
let sort_expr_names = unique_exprs
82+
let sort_expr_names = dedup_exprs
8083
.iter()
8184
.map(|sort_expr| sort_expr.expr.schema_name().to_string())
8285
.collect::<Vec<_>>();
8386
let required_indices = get_required_sort_exprs_indices(
8487
sort.input.schema().as_ref(),
8588
&sort_expr_names,
8689
);
90+
let fd_will_prune = required_indices.len() < dedup_exprs.len();
91+
92+
if !dedupe_changed && !fd_will_prune {
93+
// No duplicates and no FD pruning; return original sort
94+
// unchanged so we don't disturb its schema.
95+
return Ok(Transformed::no(LogicalPlan::Sort(sort)));
96+
}
8797

88-
let unique_exprs = if required_indices.len() < unique_exprs.len() {
98+
let unique_exprs = if fd_will_prune {
8999
required_indices
90100
.into_iter()
91-
.map(|idx| unique_exprs[idx].clone())
101+
.map(|idx| dedup_exprs[idx].clone())
92102
.collect()
93103
} else {
94-
unique_exprs
104+
dedup_exprs
95105
};
96106

97107
if unique_exprs.is_empty() {
@@ -100,16 +110,25 @@ impl OptimizerRule for EliminateDuplicatedExpr {
100110
);
101111
}
102112

103-
let new_plan = LogicalPlan::Sort(Sort {
113+
Ok(Transformed::yes(LogicalPlan::Sort(Sort {
104114
expr: unique_exprs,
105115
input: sort.input,
106116
fetch: sort.fetch,
107-
});
108-
109-
Ok(transformed_if_changed(original_plan, new_plan))
117+
})))
110118
}
111119
LogicalPlan::Aggregate(agg) => {
112-
let original_plan = LogicalPlan::Aggregate(agg.clone());
120+
let has_duplicate = {
121+
let mut seen = HashSet::with_capacity(agg.group_expr.len());
122+
agg.group_expr.iter().any(|e| !seen.insert(e))
123+
};
124+
125+
if !has_duplicate {
126+
// Returning the original aggregate preserves its schema —
127+
// `Aggregate::try_new` would recompute it and may produce a
128+
// differing (but semantically equivalent) plan.
129+
return Ok(Transformed::no(LogicalPlan::Aggregate(agg)));
130+
}
131+
113132
let unique_exprs: Vec<Expr> = agg
114133
.group_expr
115134
.into_iter()
@@ -119,7 +138,7 @@ impl OptimizerRule for EliminateDuplicatedExpr {
119138

120139
Aggregate::try_new(agg.input, unique_exprs, agg.aggr_expr)
121140
.map(LogicalPlan::Aggregate)
122-
.map(|plan| transformed_if_changed(original_plan, plan))
141+
.map(Transformed::yes)
123142
}
124143
_ => Ok(Transformed::no(plan)),
125144
}
@@ -129,17 +148,6 @@ impl OptimizerRule for EliminateDuplicatedExpr {
129148
}
130149
}
131150

132-
fn transformed_if_changed(
133-
original_plan: LogicalPlan,
134-
new_plan: LogicalPlan,
135-
) -> Transformed<LogicalPlan> {
136-
if new_plan == original_plan {
137-
Transformed::no(original_plan)
138-
} else {
139-
Transformed::yes(new_plan)
140-
}
141-
}
142-
143151
#[cfg(test)]
144152
mod tests {
145153
use super::*;

‎datafusion/optimizer/src/optimize_projections/mod.rs‎

Lines changed: 46 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -148,6 +148,9 @@ fn optimize_projections(
148148
// `aggregate.aggr_expr`:
149149
let (group_by_reqs, aggregate_reqs) = indices.split_off(n_group_exprs);
150150

151+
let original_group_expr_len = aggregate.group_expr.len();
152+
let original_aggr_expr_len = aggregate.aggr_expr.len();
153+
151154
// Get absolutely necessary GROUP BY fields.
152155
//
153156
// When the input has no functional dependencies, we can
@@ -197,13 +200,16 @@ fn optimize_projections(
197200
)));
198201
}
199202

203+
let aggregate_changed = new_group_bys.len() != original_group_expr_len
204+
|| new_aggr_expr.len() != original_aggr_expr_len;
205+
200206
let all_exprs_iter = new_group_bys.iter().chain(new_aggr_expr.iter());
201207
let schema = aggregate.input.schema();
202208
let necessary_indices =
203209
RequiredIndices::new().with_exprs(schema, all_exprs_iter);
204210
let necessary_exprs = necessary_indices.get_required_exprs(schema);
205211

206-
return optimize_projections(
212+
let rebuilt = optimize_projections(
207213
Arc::unwrap_or_clone(aggregate.input),
208214
config,
209215
necessary_indices,
@@ -224,11 +230,15 @@ fn optimize_projections(
224230
new_aggr_expr,
225231
)
226232
.map(LogicalPlan::Aggregate)
227-
})?
228-
.transform_data(|plan| optimize_subqueries(plan, config));
233+
})?;
234+
235+
let combined = Transformed::new_transformed(
236+
rebuilt.data,
237+
rebuilt.transformed || aggregate_changed,
238+
);
239+
return combined.transform_data(|plan| optimize_subqueries(plan, config));
229240
}
230241
LogicalPlan::Window(window) => {
231-
let original_window = window.clone();
232242
let input_schema = Arc::clone(window.input.schema());
233243
// Split parent requirements to child and window expression sections:
234244
let n_input_fields = input_schema.fields().len();
@@ -245,8 +255,14 @@ fn optimize_projections(
245255
// parent or window expression requirements.
246256
let required_indices = child_reqs.with_exprs(&input_schema, &new_window_expr);
247257

258+
let Window {
259+
input,
260+
window_expr: original_window_expr,
261+
schema: original_schema,
262+
} = window;
263+
248264
let transformed_input = optimize_projections(
249-
Arc::unwrap_or_clone(window.input),
265+
Arc::unwrap_or_clone(input),
250266
config,
251267
required_indices.clone(),
252268
)?;
@@ -270,36 +286,45 @@ fn optimize_projections(
270286
.map(LogicalPlan::Window)
271287
.map(Transformed::yes)?
272288
} else {
273-
Transformed::no(LogicalPlan::Window(original_window))
289+
Transformed::no(LogicalPlan::Window(Window {
290+
input: Arc::new(projected_input.data),
291+
window_expr: original_window_expr,
292+
schema: original_schema,
293+
}))
274294
}
275295
};
276296

277297
return transformed_plan
278298
.transform_data(|plan| optimize_subqueries(plan, config));
279299
}
280300
LogicalPlan::TableScan(table_scan) => {
281-
let original_scan = table_scan.clone();
282-
let TableScan {
283-
table_name,
284-
source,
285-
projection,
286-
filters,
287-
fetch,
288-
projected_schema: _,
289-
} = table_scan;
290-
291301
// Get indices referred to in the original (schema with all fields)
292302
// given projected indices.
293-
let projection = match &projection {
303+
let new_projection = match &table_scan.projection {
294304
Some(projection) => indices.into_mapped_indices(|idx| projection[idx]),
295305
None => indices.into_inner(),
296306
};
297-
let new_scan =
298-
TableScan::try_new(table_name, source, Some(projection), filters, fetch)?;
299-
if new_scan == original_scan {
300-
return Ok(Transformed::no(LogicalPlan::TableScan(original_scan)));
307+
if table_scan.projection.as_ref() == Some(&new_projection) {
308+
// Projection unchanged; return the original scan so we preserve
309+
// its `projected_schema` (which may carry metadata a fresh
310+
// `try_new` would not reproduce).
311+
return Ok(Transformed::no(LogicalPlan::TableScan(table_scan)));
301312
}
302313

314+
let TableScan {
315+
table_name,
316+
source,
317+
filters,
318+
fetch,
319+
..
320+
} = table_scan;
321+
let new_scan = TableScan::try_new(
322+
table_name,
323+
source,
324+
Some(new_projection),
325+
filters,
326+
fetch,
327+
)?;
303328
return Transformed::yes(LogicalPlan::TableScan(new_scan))
304329
.transform_data(|plan| optimize_subqueries(plan, config));
305330
}

‎datafusion/optimizer/src/optimize_unions.rs‎

Lines changed: 1 addition & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717

1818
//! [`OptimizeUnions`]: removes `Union` nodes in the logical plan.
1919
use crate::optimizer::ApplyOrder;
20+
use crate::utils::transformed_if_changed;
2021
use crate::{OptimizerConfig, OptimizerRule};
2122
use datafusion_common::Result;
2223
use datafusion_common::tree_node::Transformed;
@@ -102,17 +103,6 @@ impl OptimizerRule for OptimizeUnions {
102103
}
103104
}
104105

105-
fn transformed_if_changed(
106-
original_plan: LogicalPlan,
107-
new_plan: LogicalPlan,
108-
) -> Transformed<LogicalPlan> {
109-
if new_plan == original_plan {
110-
Transformed::no(original_plan)
111-
} else {
112-
Transformed::yes(new_plan)
113-
}
114-
}
115-
116106
fn extract_plans_from_union(plan: Arc<LogicalPlan>) -> Vec<LogicalPlan> {
117107
match Arc::unwrap_or_clone(plan) {
118108
LogicalPlan::Union(Union { inputs, .. }) => inputs

‎datafusion/optimizer/src/optimizer.rs‎

Lines changed: 28 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -681,24 +681,7 @@ impl Optimizer {
681681
}
682682
.and_then(|tnr| {
683683
#[cfg(debug_assertions)]
684-
{
685-
let plan_changed = input_plan.ne(&tnr.data);
686-
if tnr.transformed {
687-
debug_assert!(
688-
plan_changed,
689-
"Optimizer rule '{}' returned Transformed::yes but did not change the plan\ninput:\n{input_plan}\noutput:\n{}",
690-
rule.name(),
691-
tnr.data,
692-
);
693-
} else {
694-
debug_assert!(
695-
!plan_changed,
696-
"Optimizer rule '{}' returned Transformed::no but changed the plan\ninput:\n{input_plan}\noutput:\n{}",
697-
rule.name(),
698-
tnr.data,
699-
);
700-
}
701-
}
684+
assert_transformed_matches_plan(rule.name(), &input_plan, &tnr);
702685

703686
// run checks optimizer invariant checks, per optimizer rule applied
704687
assert_valid_optimization(&tnr.data, &starting_schema)
@@ -805,6 +788,33 @@ fn assert_valid_optimization(
805788
Ok(())
806789
}
807790

791+
/// Debug-only check that the rule's `Transformed::yes`/`no` flag matches
792+
/// whether the plan was actually changed.
793+
///
794+
/// The no-op skip mechanism in `Optimizer::optimize` relies on this contract;
795+
/// a lying rule will silently produce incorrect plans.
796+
#[cfg(debug_assertions)]
797+
fn assert_transformed_matches_plan(
798+
rule_name: &str,
799+
input_plan: &LogicalPlan,
800+
result: &Transformed<LogicalPlan>,
801+
) {
802+
let plan_changed = input_plan.ne(&result.data);
803+
if result.transformed {
804+
debug_assert!(
805+
plan_changed,
806+
"Optimizer rule '{rule_name}' returned Transformed::yes but did not change the plan\ninput:\n{input_plan}\noutput:\n{}",
807+
result.data,
808+
);
809+
} else {
810+
debug_assert!(
811+
!plan_changed,
812+
"Optimizer rule '{rule_name}' returned Transformed::no but changed the plan\ninput:\n{input_plan}\noutput:\n{}",
813+
result.data,
814+
);
815+
}
816+
}
817+
808818
#[cfg(test)]
809819
mod tests {
810820
use std::sync::{Arc, Mutex};

0 commit comments

Comments
 (0)