Skip to content

Commit cbc04dd

Browse files
discord9lukekim
authored andcommitted
fix: preserve nested projection expressions (apache#24686)
## Which issue does this PR close? - No issue has been filed. ## Rationale for this change Anonymous nested projections that reuse the same output name can return wrong results. For example, each `i + 1 AS i` layer must be evaluated independently, but the projection optimizer could drop one layer when two consecutive projection expression vectors were structurally equal. Structural equality does not imply that a projection is safe to elide: repeated computations such as `i + 1 AS i` have the same expression shape but must still be evaluated twice. ## What changes are included in this PR? - Remove the structural-equality fast path that directly elided one of two consecutive projections. - Keep the existing iterative whole-chain merge, so deep projection chains still collapse within one optimizer rule invocation. - Continue using the normal projection rewrite path, which composes repeated expressions and preserves aliases and field metadata. - Add focused regressions for repeated non-idempotent projections, one-pass collapse of a 12-level chain, and metadata-bearing aliases. - Add an execution-level SQLLogicTest with six anonymous `i + 1 AS i` layers under both `max_passes = 1` and the default optimizer configuration. ## What is the testing strategy for this PR? The focused unit tests verify that: - two structurally equal `i + 1 AS i` projections retain both additions; - a 12-level chain preserves all 12 additions and collapses to one `Projection` with `max_passes = 1`; - a metadata-bearing `Alias(Column)` is merged without losing field metadata. The SQLLogicTest executes a six-level anonymous projection chain against a temporary table with both one optimizer pass and the default pass count. Verified with: ```text cargo fmt --all --check cargo test -p datafusion-optimizer optimize_projections # 58 passed; 0 failed cargo test -p datafusion-optimizer --test optimizer_integration # 26 passed; 0 failed cargo test --profile ci -p datafusion-sqllogictest --test sqllogictests -- projection.slt # 1/1 files completed; 0 failures cargo clippy -p datafusion-optimizer --all-targets --all-features -- -D warnings # passed git diff --check # passed ``` ## Are there any user-facing changes? Yes. Deep anonymous nested projections that reuse an output name now preserve every projection expression and return the correct result. There are no public API or configuration changes. --------- Signed-off-by: discord9 <[email protected]> Signed-off-by: discord9 <[email protected]> (cherry picked from commit 7a1f468)
1 parent 53e60cd commit cbc04dd

2 files changed

Lines changed: 131 additions & 11 deletions

File tree

‎datafusion/optimizer/src/optimize_projections/mod.rs‎

Lines changed: 63 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -556,17 +556,6 @@ fn merge_consecutive_projections(proj: Projection) -> Result<Transformed<Project
556556
return Projection::try_new_with_schema(expr, input, schema).map(Transformed::no);
557557
};
558558

559-
// A fast path: if the previous projection is same as the current projection
560-
// we can directly remove the current projection and return child projection.
561-
if prev_projection.expr == expr {
562-
return Projection::try_new_with_schema(
563-
expr,
564-
Arc::clone(&prev_projection.input),
565-
schema,
566-
)
567-
.map(Transformed::yes);
568-
}
569-
570559
// Count usages (referrals) of each projection expression in its input fields:
571560
let mut column_referral_map = HashMap::<&Column, usize>::new();
572561
expr.iter()
@@ -1267,6 +1256,69 @@ mod tests {
12671256
)
12681257
}
12691258

1259+
#[test]
1260+
fn merge_structurally_equal_non_idempotent_projections() -> Result<()> {
1261+
let schema = Schema::new(vec![Field::new("i", DataType::Int32, false)]);
1262+
let projection = || col("i").add(lit(1)).alias("i");
1263+
let plan = table_scan(TableReference::none(), &schema, None)?
1264+
.project(vec![projection()])?
1265+
.project(vec![projection()])?
1266+
.build()?;
1267+
1268+
assert_optimized_plan_equal!(
1269+
plan,
1270+
@r"
1271+
Projection: ?table?.i + Int32(1) + Int32(1) AS i
1272+
TableScan: ?table? projection=[i]
1273+
"
1274+
)
1275+
}
1276+
1277+
#[test]
1278+
fn merge_deep_projection_chain_in_one_pass() -> Result<()> {
1279+
let schema = Schema::new(vec![Field::new("i", DataType::Int32, false)]);
1280+
let mut plan = table_scan(TableReference::none(), &schema, None)?
1281+
.project(vec![col("i").add(lit(1)).alias("i")])?
1282+
.build()?;
1283+
for _ in 1..12 {
1284+
plan = LogicalPlanBuilder::from(plan)
1285+
.project(vec![col("i").add(lit(1)).alias("i")])?
1286+
.build()?;
1287+
}
1288+
1289+
let optimizer = Optimizer::with_rules(vec![Arc::new(OptimizeProjections::new())]);
1290+
let optimized = optimizer.optimize(
1291+
plan,
1292+
&OptimizerContext::new().with_max_passes(1),
1293+
observe,
1294+
)?;
1295+
let plan_string = format!("{optimized}");
1296+
assert_eq!(12, plan_string.matches("Int32(1)").count());
1297+
assert_eq!(1, plan_string.matches("Projection:").count());
1298+
Ok(())
1299+
}
1300+
1301+
#[test]
1302+
fn merge_columns_and_metadata_alias() -> Result<()> {
1303+
let metadata =
1304+
datafusion_common::metadata::FieldMetadata::from(HashMap::from([(
1305+
"key".to_string(),
1306+
"value".to_string(),
1307+
)]));
1308+
let plan = LogicalPlanBuilder::from(test_table_scan()?)
1309+
.project(vec![col("a")])?
1310+
.project(vec![col("a").alias_with_metadata("a", Some(metadata))])?
1311+
.build()?;
1312+
1313+
let optimized = optimize(plan)?;
1314+
assert_eq!(
1315+
"value",
1316+
optimized.schema().field(0).metadata().get("key").unwrap()
1317+
);
1318+
assert_eq!(1, format!("{optimized}").matches("Projection:").count());
1319+
Ok(())
1320+
}
1321+
12701322
#[test]
12711323
fn merge_alias() -> Result<()> {
12721324
let table_scan = test_table_scan()?;

‎datafusion/sqllogictest/test_files/projection.slt‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -217,6 +217,74 @@ SELECT column1 as a from (values (1), (2)) f where f.column1 = 2;
217217
----
218218
2
219219

220+
# Regression: preserve deep anonymous nested projections with one and default passes.
221+
statement ok
222+
CREATE TABLE nested_projection(i INT);
223+
224+
statement ok
225+
INSERT INTO nested_projection VALUES (3), (4), (5);
226+
227+
statement ok
228+
SET datafusion.optimizer.max_passes = 1;
229+
230+
query I rowsort
231+
SELECT i
232+
FROM (
233+
SELECT i + 1 AS i
234+
FROM (
235+
SELECT i + 1 AS i
236+
FROM (
237+
SELECT i + 1 AS i
238+
FROM (
239+
SELECT i + 1 AS i
240+
FROM (
241+
SELECT i + 1 AS i
242+
FROM (
243+
SELECT i + 1 AS i
244+
FROM nested_projection
245+
)
246+
)
247+
)
248+
)
249+
)
250+
)
251+
----
252+
10
253+
11
254+
9
255+
256+
statement ok
257+
RESET datafusion.optimizer.max_passes;
258+
259+
query I rowsort
260+
SELECT i
261+
FROM (
262+
SELECT i + 1 AS i
263+
FROM (
264+
SELECT i + 1 AS i
265+
FROM (
266+
SELECT i + 1 AS i
267+
FROM (
268+
SELECT i + 1 AS i
269+
FROM (
270+
SELECT i + 1 AS i
271+
FROM (
272+
SELECT i + 1 AS i
273+
FROM nested_projection
274+
)
275+
)
276+
)
277+
)
278+
)
279+
)
280+
----
281+
10
282+
11
283+
9
284+
285+
statement ok
286+
DROP TABLE nested_projection;
287+
220288
# clean data
221289
statement ok
222290
DROP TABLE aggregate_simple;

0 commit comments

Comments
 (0)