Skip to content

Commit 7a1f468

Browse files
authored
fix: preserve nested projection expressions (apache#24686)
## Which issue does this PR close? - No issue has been filed. ## Rationale for this change Anonymous nested projections that reuse the same output name can return wrong results. For example, each `i + 1 AS i` layer must be evaluated independently, but the projection optimizer could drop one layer when two consecutive projection expression vectors were structurally equal. Structural equality does not imply that a projection is safe to elide: repeated computations such as `i + 1 AS i` have the same expression shape but must still be evaluated twice. ## What changes are included in this PR? - Remove the structural-equality fast path that directly elided one of two consecutive projections. - Keep the existing iterative whole-chain merge, so deep projection chains still collapse within one optimizer rule invocation. - Continue using the normal projection rewrite path, which composes repeated expressions and preserves aliases and field metadata. - Add focused regressions for repeated non-idempotent projections, one-pass collapse of a 12-level chain, and metadata-bearing aliases. - Add an execution-level SQLLogicTest with six anonymous `i + 1 AS i` layers under both `max_passes = 1` and the default optimizer configuration. ## What is the testing strategy for this PR? The focused unit tests verify that: - two structurally equal `i + 1 AS i` projections retain both additions; - a 12-level chain preserves all 12 additions and collapses to one `Projection` with `max_passes = 1`; - a metadata-bearing `Alias(Column)` is merged without losing field metadata. The SQLLogicTest executes a six-level anonymous projection chain against a temporary table with both one optimizer pass and the default pass count. Verified with: ```text cargo fmt --all --check cargo test -p datafusion-optimizer optimize_projections # 58 passed; 0 failed cargo test -p datafusion-optimizer --test optimizer_integration # 26 passed; 0 failed cargo test --profile ci -p datafusion-sqllogictest --test sqllogictests -- projection.slt # 1/1 files completed; 0 failures cargo clippy -p datafusion-optimizer --all-targets --all-features -- -D warnings # passed git diff --check # passed ``` ## Are there any user-facing changes? Yes. Deep anonymous nested projections that reuse an output name now preserve every projection expression and return the correct result. There are no public API or configuration changes. --------- Signed-off-by: discord9 <[email protected]> Signed-off-by: discord9 <[email protected]>
1 parent d02bd78 commit 7a1f468

2 files changed

Lines changed: 131 additions & 11 deletions

File tree

‎datafusion/optimizer/src/optimize_projections/mod.rs‎

Lines changed: 63 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -556,17 +556,6 @@ fn merge_consecutive_projections_one_level(
556556
return Projection::try_new_with_schema(expr, input, schema).map(Transformed::no);
557557
};
558558

559-
// A fast path: if the previous projection is same as the current projection
560-
// we can directly remove the current projection and return child projection.
561-
if prev_projection.expr == expr {
562-
return Projection::try_new_with_schema(
563-
expr,
564-
Arc::clone(&prev_projection.input),
565-
schema,
566-
)
567-
.map(Transformed::yes);
568-
}
569-
570559
// Count usages (referrals) of each projection expression in its input fields:
571560
let mut column_referral_map = HashMap::<&Column, usize>::new();
572561
for expr in &expr {
@@ -1210,6 +1199,69 @@ mod tests {
12101199
)
12111200
}
12121201

1202+
#[test]
1203+
fn merge_structurally_equal_non_idempotent_projections() -> Result<()> {
1204+
let schema = Schema::new(vec![Field::new("i", DataType::Int32, false)]);
1205+
let projection = || col("i").add(lit(1)).alias("i");
1206+
let plan = table_scan(TableReference::none(), &schema, None)?
1207+
.project(vec![projection()])?
1208+
.project(vec![projection()])?
1209+
.build()?;
1210+
1211+
assert_optimized_plan_equal!(
1212+
plan,
1213+
@r"
1214+
Projection: ?table?.i + Int32(1) + Int32(1) AS i
1215+
TableScan: ?table? projection=[i]
1216+
"
1217+
)
1218+
}
1219+
1220+
#[test]
1221+
fn merge_deep_projection_chain_in_one_pass() -> Result<()> {
1222+
let schema = Schema::new(vec![Field::new("i", DataType::Int32, false)]);
1223+
let mut plan = table_scan(TableReference::none(), &schema, None)?
1224+
.project(vec![col("i").add(lit(1)).alias("i")])?
1225+
.build()?;
1226+
for _ in 1..12 {
1227+
plan = LogicalPlanBuilder::from(plan)
1228+
.project(vec![col("i").add(lit(1)).alias("i")])?
1229+
.build()?;
1230+
}
1231+
1232+
let optimizer = Optimizer::with_rules(vec![Arc::new(OptimizeProjections::new())]);
1233+
let optimized = optimizer.optimize(
1234+
plan,
1235+
&OptimizerContext::new().with_max_passes(1),
1236+
observe,
1237+
)?;
1238+
let plan_string = format!("{optimized}");
1239+
assert_eq!(12, plan_string.matches("Int32(1)").count());
1240+
assert_eq!(1, plan_string.matches("Projection:").count());
1241+
Ok(())
1242+
}
1243+
1244+
#[test]
1245+
fn merge_columns_and_metadata_alias() -> Result<()> {
1246+
let metadata =
1247+
datafusion_common::metadata::FieldMetadata::from(HashMap::from([(
1248+
"key".to_string(),
1249+
"value".to_string(),
1250+
)]));
1251+
let plan = LogicalPlanBuilder::from(test_table_scan()?)
1252+
.project(vec![col("a")])?
1253+
.project(vec![col("a").alias_with_metadata("a", Some(metadata))])?
1254+
.build()?;
1255+
1256+
let optimized = optimize(plan)?;
1257+
assert_eq!(
1258+
"value",
1259+
optimized.schema().field(0).metadata().get("key").unwrap()
1260+
);
1261+
assert_eq!(1, format!("{optimized}").matches("Projection:").count());
1262+
Ok(())
1263+
}
1264+
12131265
#[test]
12141266
fn merge_alias() -> Result<()> {
12151267
let table_scan = test_table_scan()?;

‎datafusion/sqllogictest/test_files/projection.slt‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -217,6 +217,74 @@ SELECT column1 as a from (values (1), (2)) f where f.column1 = 2;
217217
----
218218
2
219219

220+
# Regression: preserve deep anonymous nested projections with one and default passes.
221+
statement ok
222+
CREATE TABLE nested_projection(i INT);
223+
224+
statement ok
225+
INSERT INTO nested_projection VALUES (3), (4), (5);
226+
227+
statement ok
228+
SET datafusion.optimizer.max_passes = 1;
229+
230+
query I rowsort
231+
SELECT i
232+
FROM (
233+
SELECT i + 1 AS i
234+
FROM (
235+
SELECT i + 1 AS i
236+
FROM (
237+
SELECT i + 1 AS i
238+
FROM (
239+
SELECT i + 1 AS i
240+
FROM (
241+
SELECT i + 1 AS i
242+
FROM (
243+
SELECT i + 1 AS i
244+
FROM nested_projection
245+
)
246+
)
247+
)
248+
)
249+
)
250+
)
251+
----
252+
10
253+
11
254+
9
255+
256+
statement ok
257+
RESET datafusion.optimizer.max_passes;
258+
259+
query I rowsort
260+
SELECT i
261+
FROM (
262+
SELECT i + 1 AS i
263+
FROM (
264+
SELECT i + 1 AS i
265+
FROM (
266+
SELECT i + 1 AS i
267+
FROM (
268+
SELECT i + 1 AS i
269+
FROM (
270+
SELECT i + 1 AS i
271+
FROM (
272+
SELECT i + 1 AS i
273+
FROM nested_projection
274+
)
275+
)
276+
)
277+
)
278+
)
279+
)
280+
----
281+
10
282+
11
283+
9
284+
285+
statement ok
286+
DROP TABLE nested_projection;
287+
220288
# clean data
221289
statement ok
222290
DROP TABLE aggregate_simple;

0 commit comments

Comments
 (0)