Skip to content

Commit b69dd03

Browse files
cool9850311claude
andcommitted
fix: rebase lambda parameters and keep metadata when splitting a projection
Two regressions in the projection split, both reported in review. Each query below works on `main` and failed with the split. Lambda parameters are indexed after the input columns, so widening the input leaves them pointing at the wrong column: SELECT array_transform(arr, x -> x + 1), s['value'], s['label'] FROM t; `remap_filter_expr` in this file already rebases parameters for the join filter pushdown, so the node-level shift moves into `rebase_lambda_variable` and both callers use it rather than carrying two copies. The symptom depends on where the stale index lands. Out of range it is dropped and the query happens to be correct; in range it fails `LambdaVariable`'s field check; in range with a matching name and type it reads the wrong column and returns wrong values with no error. The last case is covered, since the field check is a backstop rather than the guarantee. `ProjectionExec::try_new` derives the output schema from the expressions alone and drops the field metadata the planner attached through `try_new_with_schema_metadata`, which the rule's own `schema_check` then rejects: PhysicalOptimizer rule 'ProjectionPushdown' failed. Schema mismatch. The split now rebuilds the projection with that constructor, taking the metadata from the projection it replaces. Tests. The value queries live in `projection_pushdown.slt`; the fixture gains `arr` and `tags`, appended after `s` so the existing `get_field(s@1, ...)` assertions do not move. A nested lambda needs two leaf expressions: with one the split is width-neutral, the parameter does not move, and the query passes against unfixed code. The metadata test is a Rust test because no SQL syntax attaches field metadata; it asserts on the physical plan's schema, not the DataFrame's, and uses Parquet because only a source that absorbs the leaf projection makes the split fire. Part of apache#27131 Co-Authored-By: Claude Opus 5 <[email protected]>
1 parent 3545ba6 commit b69dd03

3 files changed

Lines changed: 149 additions & 20 deletions

File tree

‎datafusion/core/tests/physical_optimizer/projection_pushdown.rs‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
33
// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
44

5+
use std::collections::HashMap;
56
use std::sync::Arc;
67

78
use arrow::compute::SortOptions;
@@ -10,7 +11,9 @@ use datafusion::datasource::listing::PartitionedFile;
1011
use datafusion::datasource::memory::MemorySourceConfig;
1112
use datafusion::datasource::physical_plan::CsvSource;
1213
use datafusion::datasource::source::DataSourceExec;
14+
use datafusion::prelude::{ParquetReadOptions, SessionContext};
1315
use datafusion_common::config::{ConfigOptions, CsvOptions};
16+
use datafusion_common::metadata::FieldMetadata;
1417
use datafusion_common::{JoinSide, JoinType, NullEquality, Result, ScalarValue};
1518
use datafusion_datasource::TableSchemaBuilder;
1619
use datafusion_datasource::file_scan_config::FileScanConfigBuilder;
@@ -20,6 +23,7 @@ use datafusion_expr::{
2023
Operator, ScalarFunctionArgs, ScalarUDF, ScalarUDFImpl, Signature, Volatility,
2124
};
2225
use datafusion_expr_common::columnar_value::ColumnarValue;
26+
use datafusion_functions::core::expr_fn::get_field;
2327
use datafusion_physical_expr::expressions::{
2428
BinaryExpr, CaseExpr, CastExpr, Column, Literal, NegativeExpr, binary, cast, col,
2529
};
@@ -1905,3 +1909,51 @@ fn test_sort_merge_join_interleaved_projection_embeds() -> Result<()> {
19051909

19061910
Ok(())
19071911
}
1912+
1913+
/// Splitting a projection rebuilds it, so it has to carry over the metadata the
1914+
/// planner attached; otherwise the rule's schema check fails the whole query.
1915+
/// Mirrors `keeps_metadata_of_constant_input` in `eliminate_constant_join`, but
1916+
/// over Parquet, because only a source that absorbs the leaf projection makes
1917+
/// the split fire at all.
1918+
#[tokio::test]
1919+
async fn split_keeps_field_metadata() -> Result<()> {
1920+
let dir = tempfile::tempdir()?;
1921+
let path = dir.path().join("leaf_split_metadata.parquet");
1922+
let path = path.to_str().expect("temp path is not utf-8");
1923+
1924+
let ctx = SessionContext::new();
1925+
// Below the five rows, so `EnsureRequirements` inserts the repartition that
1926+
// the split needs.
1927+
ctx.sql("SET datafusion.execution.batch_size = 2").await?;
1928+
ctx.sql("SET datafusion.execution.target_partitions = 4")
1929+
.await?;
1930+
ctx.sql(&format!(
1931+
"COPY (SELECT column1 AS id, column2 AS s FROM VALUES \
1932+
(1, {{value: 100}}), (2, {{value: 200}}), (3, {{value: 150}}), \
1933+
(4, {{value: 300}}), (5, {{value: 250}})) \
1934+
TO '{path}' STORED AS PARQUET"
1935+
))
1936+
.await?
1937+
.collect()
1938+
.await?;
1939+
ctx.register_parquet("t", path, ParquetReadOptions::default())
1940+
.await?;
1941+
1942+
let metadata =
1943+
FieldMetadata::from(HashMap::from([("k".to_string(), "v".to_string())]));
1944+
let df = ctx.table("t").await?.select(vec![
1945+
get_field(datafusion_expr::col("s"), "value")
1946+
.alias_with_metadata("v", Some(metadata)),
1947+
(datafusion_expr::col("id") + datafusion_expr::lit(1i64)).alias("w"),
1948+
])?;
1949+
1950+
// The physical plan is what the rule rewrites, so the assertion has to come
1951+
// from it rather than from the DataFrame's logical schema.
1952+
let plan = df.create_physical_plan().await?;
1953+
assert_eq!(
1954+
plan.schema().field(0).metadata().get("k"),
1955+
Some(&"v".to_string())
1956+
);
1957+
1958+
Ok(())
1959+
}

‎datafusion/physical-optimizer/src/projection_pushdown.rs‎

Lines changed: 42 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -451,21 +451,32 @@ fn remap_filter_expr(
451451
} else if let Some(variable) = expr.downcast_ref::<LambdaVariable>()
452452
&& old_input_len != new_input_len
453453
{
454-
let offset = variable.index().checked_sub(old_input_len).ok_or_else(|| {
455-
internal_datafusion_err!(
456-
"Lambda parameter {variable} precedes the {old_input_len} input columns"
457-
)
458-
})?;
459-
return Ok(Transformed::yes(Arc::new(LambdaVariable::new(
460-
new_input_len + offset,
461-
Arc::clone(variable.field()),
462-
)) as _));
454+
let rebased = rebase_lambda_variable(variable, old_input_len, new_input_len)?;
455+
return Ok(Transformed::yes(rebased));
463456
}
464457
Ok(Transformed::no(expr))
465458
})
466459
.data()
467460
}
468461

462+
/// Shifts a lambda parameter by the change in input width, for the reason
463+
/// [`remap_filter_expr`] documents.
464+
fn rebase_lambda_variable(
465+
variable: &LambdaVariable,
466+
old_input_len: usize,
467+
new_input_len: usize,
468+
) -> Result<Arc<dyn PhysicalExpr>> {
469+
let offset = variable.index().checked_sub(old_input_len).ok_or_else(|| {
470+
internal_datafusion_err!(
471+
"Lambda parameter {variable} precedes the {old_input_len} input columns"
472+
)
473+
})?;
474+
Ok(Arc::new(LambdaVariable::new(
475+
new_input_len + offset,
476+
Arc::clone(variable.field()),
477+
)))
478+
}
479+
469480
/// Prefix for the columns this rule adds when it splits a projection.
470481
const LEAF_COLUMN_PREFIX: &str = "__datafusion_leaf";
471482

@@ -547,6 +558,7 @@ fn try_split_leaf_projection(
547558
Arc::clone(&proj_expr.expr),
548559
&leaves,
549560
&passthrough_indices,
561+
child_schema.fields().len(),
550562
)?,
551563
alias: proj_expr.alias.clone(),
552564
})
@@ -559,10 +571,15 @@ fn try_split_leaf_projection(
559571
vec![new_child],
560572
ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),
561573
)?;
562-
Ok(Some(Arc::new(ProjectionExec::try_new(
563-
new_exprs,
564-
new_repartition,
565-
)?)))
574+
// `try_new` would derive the output schema from the expressions alone and
575+
// drop the metadata the planner set, which `schema_check` then rejects.
576+
Ok(Some(Arc::new(
577+
ProjectionExec::try_new_with_schema_metadata(
578+
new_exprs,
579+
new_repartition,
580+
projection.schema().as_ref(),
581+
)?,
582+
)))
566583
}
567584

568585
/// Collects the distinct outermost [`ExpressionPlacement::MoveTowardsLeafNodes`]
@@ -614,14 +631,17 @@ fn collect_outer_columns(
614631

615632
/// Rewrites an outer projection expression against the split child schema:
616633
/// every leaf sub-expression becomes a reference to the column the split added
617-
/// for it, and every surviving column is remapped to its new index.
634+
/// for it, and every surviving column or lambda parameter is remapped to its
635+
/// new index. `old_input_len` is the child's width before the split.
618636
fn rewrite_outer_expr(
619637
expr: Arc<dyn PhysicalExpr>,
620638
leaves: &[Arc<dyn PhysicalExpr>],
621639
passthrough_indices: &[usize],
640+
old_input_len: usize,
622641
) -> Result<Arc<dyn PhysicalExpr>> {
623642
// The leaf columns were appended after the passthrough ones.
624643
let first_leaf = passthrough_indices.len();
644+
let new_input_len = first_leaf + leaves.len();
625645
expr.transform_down(|node| {
626646
if let Some(pos) = leaves.iter().position(|leaf| leaf == &node) {
627647
let column = Column::new(&leaf_column_name(pos), first_leaf + pos);
@@ -646,6 +666,14 @@ fn rewrite_outer_expr(
646666
return Ok(Transformed::yes(Arc::new(remapped) as Arc<dyn PhysicalExpr>));
647667
}
648668
}
669+
// The split widens the input, which moves the lambda parameters indexed
670+
// after it; see `remap_filter_expr`.
671+
if let Some(variable) = node.downcast_ref::<LambdaVariable>()
672+
&& old_input_len != new_input_len
673+
{
674+
let rebased = rebase_lambda_variable(variable, old_input_len, new_input_len)?;
675+
return Ok(Transformed::yes(rebased));
676+
}
649677
Ok(Transformed::no(node))
650678
})
651679
.data()

‎datafusion/sqllogictest/test_files/projection_pushdown.slt‎

Lines changed: 55 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -2689,15 +2689,17 @@ DROP TABLE duplicate_alias;
26892689
# `RESET` puts back the global default instead of the harness value, and the
26902690
# harness rejects a file that leaves configuration modified.
26912691

2692+
# `arr` and `tags` are only read by the lambda queries further down. They are
2693+
# appended after `s` so the `get_field(s@1, ...)` indices below do not move.
26922694
statement ok
26932695
COPY (
2694-
SELECT column1 AS id, column2 AS s
2696+
SELECT column1 AS id, column2 AS s, column3 AS arr, column4 AS tags
26952697
FROM VALUES
2696-
(1, {value: 100, label: 'alpha'}),
2697-
(2, {value: 200, label: 'beta'}),
2698-
(3, {value: 150, label: 'gamma'}),
2699-
(4, {value: 300, label: 'delta'}),
2700-
(5, {value: 250, label: 'epsilon'})
2698+
(1, {value: 100, label: 'alpha'}, [1, 2, 3], ['a', 'b']),
2699+
(2, {value: 200, label: 'beta'}, [4, 5], ['c']),
2700+
(3, {value: 150, label: 'gamma'}, [6], ['d', 'e', 'f']),
2701+
(4, {value: 300, label: 'delta'}, [7, 8, 9], ['g']),
2702+
(5, {value: 250, label: 'epsilon'}, [10], ['h', 'i'])
27012703
) TO 'test_files/scratch/projection_pushdown/leaf_split.parquet'
27022704
STORED AS PARQUET;
27032705

@@ -2779,6 +2781,53 @@ SELECT s['value'] + 1 AS v FROM leaf_split ORDER BY v;
27792781
251
27802782
301
27812783

2784+
# A lambda parameter is indexed after the input columns, so the split moves it;
2785+
# see `rewrite_outer_expr`. These are value queries because an index left behind
2786+
# is only sometimes an error, see the third one.
2787+
2788+
statement ok
2789+
SET datafusion.sql_parser.dialect = databricks;
2790+
2791+
query ?IT
2792+
SELECT array_transform(arr, x -> x + 1) AS a, s['value'] AS v, s['label'] AS l
2793+
FROM leaf_split ORDER BY v;
2794+
----
2795+
[2, 3, 4] 100 alpha
2796+
[7] 150 gamma
2797+
[5, 6] 200 beta
2798+
[11] 250 epsilon
2799+
[8, 9, 10] 300 delta
2800+
2801+
# Nested lambdas need two leaves: with one the split is width-neutral, the
2802+
# parameter does not move, and the query passes against broken code.
2803+
query ?IT
2804+
SELECT array_transform(arr, x -> x + array_length(array_transform(arr, y -> y * 2))) AS a,
2805+
s['value'] AS v, s['label'] AS l
2806+
FROM leaf_split ORDER BY v;
2807+
----
2808+
[4, 5, 6] 100 alpha
2809+
[7] 150 gamma
2810+
[6, 7] 200 beta
2811+
[11] 250 epsilon
2812+
[10, 11, 12] 300 delta
2813+
2814+
# Naming the parameter after the column a stale index would land on makes
2815+
# `LambdaVariable`'s field check match, so the wrong column is read with no
2816+
# error at all. The name has to track `leaf_column_name` to stay adversarial.
2817+
query ?ITI
2818+
SELECT array_transform(tags, __datafusion_leaf_1 -> concat(__datafusion_leaf_1, '!')) AS a,
2819+
s['value'] AS v, s['label'] AS l, id
2820+
FROM leaf_split ORDER BY v;
2821+
----
2822+
[a!, b!] 100 alpha 1
2823+
[d!, e!, f!] 150 gamma 3
2824+
[c!] 200 beta 2
2825+
[h!, i!] 250 epsilon 5
2826+
[g!] 300 delta 4
2827+
2828+
statement ok
2829+
SET datafusion.sql_parser.dialect = generic;
2830+
27822831
statement ok
27832832
SET datafusion.execution.batch_size = 8192;
27842833

0 commit comments

Comments
 (0)