Skip to content

Commit a0124a4

Browse files
authored
fix: preserve projection field metadata during physical planning (apache#23981)
## Which issue does this PR close? - Closes apache#23529. ## Rationale for this change Logical projection aliases can attach field metadata, but physical projection planning derived its output schema only from the unwrapped physical expressions. This caused schema validation to reject aggregates above such projections. ## What changes are included in this PR? - Apply logical projection schema and field metadata when constructing `ProjectionExec`. - Preserve the existing physical derivation of field names, types, and nullability. - Add metadata-aware constructors across the physical-expression and physical-plan crate boundary. - Cover projections with alias metadata followed by aggregation. ## Are these changes tested? Yes. A focused physical planner regression test reproduces the original schema mismatch and passes with the fix. The relevant core and physical-plan unit suites, targeted all-targets/all-features clippy, formatting, and diff checks also pass. ## Are there any user-facing changes? Projection field metadata is now preserved in physical plans. Two additive constructor methods support passing projection schema metadata across crate boundaries; no existing API is changed.
1 parent e7304ee commit a0124a4

3 files changed

Lines changed: 171 additions & 4 deletions

File tree

‎datafusion/core/src/physical_planner.rs‎

Lines changed: 52 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1106,6 +1106,7 @@ impl DefaultPhysicalPlanner {
11061106
children.one()?,
11071107
input,
11081108
expr,
1109+
node.schema(),
11091110
)?,
11101111
LogicalPlan::Filter(Filter {
11111112
predicate, input, ..
@@ -1329,6 +1330,7 @@ impl DefaultPhysicalPlanner {
13291330
physical_left,
13301331
input,
13311332
expr,
1333+
left.schema(),
13321334
)?,
13331335
_ => physical_left,
13341336
};
@@ -1343,6 +1345,7 @@ impl DefaultPhysicalPlanner {
13431345
physical_right,
13441346
input,
13451347
expr,
1348+
right.schema(),
13461349
)?,
13471350
_ => physical_right,
13481351
};
@@ -1727,6 +1730,7 @@ impl DefaultPhysicalPlanner {
17271730
join,
17281731
input,
17291732
expr,
1733+
new_logical.schema(),
17301734
)?
17311735
} else {
17321736
join
@@ -2958,6 +2962,7 @@ impl DefaultPhysicalPlanner {
29582962
input_exec: Arc<dyn ExecutionPlan>,
29592963
input: &Arc<LogicalPlan>,
29602964
expr: &[Expr],
2965+
output_schema: &DFSchema,
29612966
) -> Result<Arc<dyn ExecutionPlan>> {
29622967
let input_logical_schema = input.as_ref().schema();
29632968
let input_physical_schema = input_exec.schema();
@@ -3015,7 +3020,11 @@ impl DefaultPhysicalPlanner {
30153020
.into_iter()
30163021
.map(|(expr, alias)| ProjectionExpr { expr, alias })
30173022
.collect();
3018-
Ok(Arc::new(ProjectionExec::try_new(proj_exprs, input_exec)?))
3023+
Ok(Arc::new(ProjectionExec::try_new_with_schema_metadata(
3024+
proj_exprs,
3025+
input_exec,
3026+
output_schema.as_arrow(),
3027+
)?))
30193028
}
30203029
PlanAsyncExpr::Async(
30213030
async_map,
@@ -3027,8 +3036,11 @@ impl DefaultPhysicalPlanner {
30273036
.into_iter()
30283037
.map(|(expr, alias)| ProjectionExpr { expr, alias })
30293038
.collect();
3030-
let new_proj_exec =
3031-
ProjectionExec::try_new(proj_exprs, Arc::new(async_exec))?;
3039+
let new_proj_exec = ProjectionExec::try_new_with_schema_metadata(
3040+
proj_exprs,
3041+
Arc::new(async_exec),
3042+
output_schema.as_arrow(),
3043+
)?;
30323044
Ok(Arc::new(new_proj_exec))
30333045
}
30343046
_ => internal_err!("Unexpected PlanAsyncExpressions variant"),
@@ -3579,6 +3591,43 @@ mod tests {
35793591
Ok(())
35803592
}
35813593

3594+
#[tokio::test]
3595+
async fn test_projection_preserves_field_metadata_for_aggregate() -> Result<()> {
3596+
use datafusion_common::metadata::FieldMetadata;
3597+
use datafusion_expr::expr::AggregateFunction;
3598+
use datafusion_functions_aggregate::min_max::max_udaf;
3599+
3600+
let schema = Schema::new(vec![Field::new("value", DataType::Utf8, false)]);
3601+
let input = LogicalPlan::EmptyRelation(EmptyRelation {
3602+
produce_one_row: false,
3603+
schema: Arc::new(schema.to_dfschema()?),
3604+
});
3605+
let metadata =
3606+
FieldMetadata::from(HashMap::from([("foo".to_string(), "bar".to_string())]));
3607+
let projection = LogicalPlan::Projection(Projection::try_new(
3608+
vec![col("value").alias_with_metadata("value", Some(metadata))],
3609+
Arc::new(input),
3610+
)?);
3611+
let aggregate = LogicalPlan::Aggregate(Aggregate::try_new(
3612+
Arc::new(projection),
3613+
vec![],
3614+
vec![Expr::AggregateFunction(AggregateFunction::new_udf(
3615+
max_udaf(),
3616+
vec![col("value")],
3617+
false,
3618+
None,
3619+
vec![],
3620+
None,
3621+
))],
3622+
)?);
3623+
3624+
DefaultPhysicalPlanner::default()
3625+
.create_physical_plan(&aggregate, &SessionContext::new().state())
3626+
.await?;
3627+
3628+
Ok(())
3629+
}
3630+
35823631
#[derive(Debug, Default)]
35833632
struct NullAccumulator;
35843633

‎datafusion/physical-expr/src/projection.rs‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -539,6 +539,56 @@ impl ProjectionExprs {
539539
})
540540
}
541541

542+
/// Create a new [`Projector`] using field and schema metadata from
543+
/// `projected_schema`.
544+
///
545+
/// Field names, data types, and nullability are still derived from the physical
546+
/// projection expressions and `input_schema`; only field and schema metadata are
547+
/// taken from `projected_schema`.
548+
///
549+
/// # Errors
550+
///
551+
/// Returns an error if the projection cannot be applied to `input_schema`, or if
552+
/// `projected_schema` has a different number of fields than the projection.
553+
pub fn make_projector_with_schema_metadata(
554+
&self,
555+
input_schema: &Schema,
556+
projected_schema: &Schema,
557+
) -> Result<Projector> {
558+
let output_schema = self.project_schema(input_schema)?;
559+
if output_schema.fields().len() != projected_schema.fields().len() {
560+
return Err(internal_datafusion_err!(
561+
"Projection has {} output fields but metadata schema has {} fields",
562+
output_schema.fields().len(),
563+
projected_schema.fields().len()
564+
));
565+
}
566+
567+
let fields = output_schema
568+
.fields()
569+
.iter()
570+
.zip(projected_schema.fields())
571+
.map(|(field, projected_field)| {
572+
Arc::new(
573+
field
574+
.as_ref()
575+
.clone()
576+
.with_metadata(projected_field.metadata().clone()),
577+
)
578+
})
579+
.collect::<Vec<_>>();
580+
let output_schema = Arc::new(Schema::new_with_metadata(
581+
fields,
582+
projected_schema.metadata().clone(),
583+
));
584+
585+
Ok(Projector {
586+
projection: self.clone(),
587+
output_schema,
588+
expression_metrics: None,
589+
})
590+
}
591+
542592
pub fn create_expression_metrics(
543593
&self,
544594
metrics: &ExecutionPlanMetricsSet,

‎datafusion/physical-plan/src/projection.rs‎

Lines changed: 69 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ use std::pin::Pin;
4040
use std::sync::Arc;
4141
use std::task::{Context, Poll};
4242

43-
use arrow::datatypes::SchemaRef;
43+
use arrow::datatypes::{Schema, SchemaRef};
4444
use arrow::record_batch::RecordBatch;
4545
use datafusion_common::config::ConfigOptions;
4646
use datafusion_common::tree_node::{
@@ -143,6 +143,34 @@ impl ProjectionExec {
143143
Self::try_from_projector(projector, input)
144144
}
145145

146+
/// Create a projection using field and schema metadata from
147+
/// `projected_schema`.
148+
///
149+
/// Field names, data types, and nullability are still derived from the physical
150+
/// projection expressions and the input plan; only field and schema metadata are
151+
/// taken from `projected_schema`.
152+
///
153+
/// # Errors
154+
///
155+
/// Returns an error if the projection cannot be applied to the input plan, or if
156+
/// `projected_schema` has a different number of fields than the projection.
157+
pub fn try_new_with_schema_metadata<I, E>(
158+
expr: I,
159+
input: Arc<dyn ExecutionPlan>,
160+
projected_schema: &Schema,
161+
) -> Result<Self>
162+
where
163+
I: IntoIterator<Item = E>,
164+
E: Into<ProjectionExpr>,
165+
{
166+
let input_schema = input.schema();
167+
let expr_arc = expr.into_iter().map(Into::into).collect::<Arc<_>>();
168+
let projection = ProjectionExprs::from_expressions(expr_arc);
169+
let projector = projection
170+
.make_projector_with_schema_metadata(&input_schema, projected_schema)?;
171+
Self::try_from_projector(projector, input)
172+
}
173+
146174
fn try_from_projector(
147175
projector: Projector,
148176
input: Arc<dyn ExecutionPlan>,
@@ -1395,6 +1423,46 @@ mod tests {
13951423
BinaryExpr, Column, DynamicFilterPhysicalExpr, Literal, binary, col, lit,
13961424
};
13971425

1426+
#[test]
1427+
fn test_try_new_with_schema_metadata_only_replaces_metadata() -> Result<()> {
1428+
let input_schema = Arc::new(Schema::new(vec![Field::new(
1429+
"input",
1430+
DataType::Int32,
1431+
false,
1432+
)]));
1433+
let input: Arc<dyn ExecutionPlan> = Arc::new(EmptyExec::new(input_schema));
1434+
let field_metadata =
1435+
HashMap::from([("field-key".to_string(), "field-value".to_string())]);
1436+
let schema_metadata =
1437+
HashMap::from([("schema-key".to_string(), "schema-value".to_string())]);
1438+
let metadata_schema = Schema::new_with_metadata(
1439+
vec![
1440+
Field::new("ignored", DataType::Utf8, true)
1441+
.with_metadata(field_metadata.clone()),
1442+
],
1443+
schema_metadata.clone(),
1444+
);
1445+
1446+
let projection = ProjectionExec::try_new_with_schema_metadata(
1447+
[ProjectionExpr {
1448+
expr: Arc::new(Column::new("input", 0)),
1449+
alias: "output".to_string(),
1450+
}],
1451+
input,
1452+
&metadata_schema,
1453+
)?;
1454+
1455+
let expected_schema = Arc::new(Schema::new_with_metadata(
1456+
vec![
1457+
Field::new("output", DataType::Int32, false)
1458+
.with_metadata(field_metadata),
1459+
],
1460+
schema_metadata,
1461+
));
1462+
assert_eq!(projection.schema(), expected_schema);
1463+
Ok(())
1464+
}
1465+
13981466
#[test]
13991467
fn test_collect_column_indices() -> Result<()> {
14001468
let expr = Arc::new(BinaryExpr::new(

0 commit comments

Comments
 (0)