Skip to content

Commit 5c0e73f

Browse files
authored
fix: preserve ProjectionExec metadata during serialization (apache#25009)
## Which issue does this PR close? - Closes apache#24695. ## Rationale for this change Physical-plan serialization can lose projection field and schema metadata. The projection payload should preserve its output metadata, including inherited metadata, without relying on the child to reconstruct it. ## What changes are included in this PR? - Add an optional schema to `ProjectionExecNode`, emitted when the projection has schema or output-field metadata, or explicitly clears input metadata. - Restore metadata through `try_new_with_schema_metadata`, while deriving field names, data types, and nullability from the expressions. - Keep older payloads without the optional field readable. - Use explicit metadata maps in tests for compatibility with both Arrow 59 and Arrow 60. This preserves the projection's metadata independently of child metadata. It does not make the entire execution plan independent of its children. Omitting duplicated inherited metadata is left for a follow-up once the serialization contract is agreed. ## What is the testing strategy for this PR? Binary and JSON round trips cover inherited and overridden metadata, schema-only and field-only metadata, extension metadata, explicit clearing, metadata-free projections, and older payloads. Child-independent reconstruction and field-count validation remain covered. - Reproduced the Arrow 60 metadata type-inference compilation failure before the test fix. All 272 protobuf integration tests pass with the fix applied to the current-main integration. - The exact PR branch passes formatting, all-target/all-feature Clippy with warnings denied, and `./dev/rust_lint.sh`. - The exact PR branch passes the extended workspace suite and all 518 SQL logic-test files. ## Are there any user-facing changes? Projection schema and output-field metadata survive physical-plan serialization. Nested field definitions remain part of the expression-derived data types. The generated `ProjectionExecNode` Rust struct gains an optional `schema` field, affecting exhaustive struct literals. New readers accept older payloads. This targets main, not a 55.1 backport. --------- Signed-off-by: goutamadwant <[email protected]>
1 parent 3ed377a commit 5c0e73f

5 files changed

Lines changed: 350 additions & 4 deletions

File tree

‎datafusion/physical-plan/src/projection.rs‎

Lines changed: 25 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -652,20 +652,33 @@ impl ExecutionPlan for ProjectionExec {
652652
metrics: _,
653653
// Derived plan properties, recomputed on decode.
654654
cache: _,
655-
// Derived metadata comparison, recomputed with the projector.
656-
overrides_metadata: _,
655+
overrides_metadata,
657656
} = self;
658657
let projection_exprs = projector.projection().as_ref();
659658
let input = ctx.encode_child(input)?;
660659
let expr = ctx.encode_expressions(projection_exprs.iter().map(|p| &p.expr))?;
661660
let expr_name = projection_exprs.iter().map(|p| p.alias.clone()).collect();
661+
let output_schema = projector.output_schema();
662+
// Preserve explicit overrides and field metadata that cannot be
663+
// reconstructed from the child. Inherited schema metadata can be.
664+
let schema = if *overrides_metadata
665+
|| output_schema
666+
.fields()
667+
.iter()
668+
.any(|field| !field.metadata().is_empty())
669+
{
670+
Some(output_schema.as_ref().try_into()?)
671+
} else {
672+
None
673+
};
662674
Ok(Some(protobuf::PhysicalPlanNode {
663675
physical_plan_type: Some(
664676
protobuf::physical_plan_node::PhysicalPlanType::Projection(Box::new(
665677
protobuf::ProjectionExecNode {
666678
input: Some(Box::new(input)),
667679
expr,
668680
expr_name,
681+
schema,
669682
},
670683
)),
671684
),
@@ -701,6 +714,7 @@ impl ProjectionExec {
701714
input,
702715
expr,
703716
expr_name,
717+
schema,
704718
} = &**projection;
705719
let input =
706720
ctx.decode_required_child(input.as_deref(), "ProjectionExec", "input")?;
@@ -715,7 +729,15 @@ impl ProjectionExec {
715729
})
716730
})
717731
.collect::<Result<Vec<ProjectionExpr>>>()?;
718-
Ok(Arc::new(ProjectionExec::try_new(exprs, input)?))
732+
let projection = match schema {
733+
Some(schema) => ProjectionExec::try_new_with_schema_metadata(
734+
exprs,
735+
input,
736+
&Schema::try_from(schema)?,
737+
)?,
738+
None => ProjectionExec::try_new(exprs, input)?,
739+
};
740+
Ok(Arc::new(projection))
719741
}
720742
}
721743

‎datafusion/proto-models/proto/datafusion.proto‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1526,6 +1526,11 @@ message ProjectionExecNode {
15261526
PhysicalPlanNode input = 1;
15271527
repeated PhysicalExprNode expr = 2;
15281528
repeated string expr_name = 3;
1529+
// Only field and schema metadata are used; output types are derived from expr.
1530+
// Present when the projection output has metadata or explicitly overrides
1531+
// inherited metadata, including an explicit clear. Absent for older plans
1532+
// and metadata-free projections.
1533+
datafusion_common.Schema schema = 4;
15291534
}
15301535

15311536
enum AggregateMode {

‎datafusion/proto-models/src/generated/pbjson.rs‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24658,6 +24658,9 @@ impl serde::Serialize for ProjectionExecNode {
2465824658
if !self.expr_name.is_empty() {
2465924659
len += 1;
2466024660
}
24661+
if self.schema.is_some() {
24662+
len += 1;
24663+
}
2466124664
let mut struct_ser = serializer.serialize_struct("datafusion.ProjectionExecNode", len)?;
2466224665
if let Some(v) = self.input.as_ref() {
2466324666
struct_ser.serialize_field("input", v)?;
@@ -24668,6 +24671,9 @@ impl serde::Serialize for ProjectionExecNode {
2466824671
if !self.expr_name.is_empty() {
2466924672
struct_ser.serialize_field("exprName", &self.expr_name)?;
2467024673
}
24674+
if let Some(v) = self.schema.as_ref() {
24675+
struct_ser.serialize_field("schema", v)?;
24676+
}
2467124677
struct_ser.end()
2467224678
}
2467324679
}
@@ -24682,13 +24688,15 @@ impl<'de> serde::Deserialize<'de> for ProjectionExecNode {
2468224688
"expr",
2468324689
"expr_name",
2468424690
"exprName",
24691+
"schema",
2468524692
];
2468624693

2468724694
#[allow(clippy::enum_variant_names)]
2468824695
enum GeneratedField {
2468924696
Input,
2469024697
Expr,
2469124698
ExprName,
24699+
Schema,
2469224700
}
2469324701
impl<'de> serde::Deserialize<'de> for GeneratedField {
2469424702
fn deserialize<D>(deserializer: D) -> std::result::Result<GeneratedField, D::Error>
@@ -24713,6 +24721,7 @@ impl<'de> serde::Deserialize<'de> for ProjectionExecNode {
2471324721
"input" => Ok(GeneratedField::Input),
2471424722
"expr" => Ok(GeneratedField::Expr),
2471524723
"exprName" | "expr_name" => Ok(GeneratedField::ExprName),
24724+
"schema" => Ok(GeneratedField::Schema),
2471624725
_ => Err(serde::de::Error::unknown_field(value, FIELDS)),
2471724726
}
2471824727
}
@@ -24735,6 +24744,7 @@ impl<'de> serde::Deserialize<'de> for ProjectionExecNode {
2473524744
let mut input__ = None;
2473624745
let mut expr__ = None;
2473724746
let mut expr_name__ = None;
24747+
let mut schema__ = None;
2473824748
while let Some(k) = map_.next_key()? {
2473924749
match k {
2474024750
GeneratedField::Input => {
@@ -24755,12 +24765,19 @@ impl<'de> serde::Deserialize<'de> for ProjectionExecNode {
2475524765
}
2475624766
expr_name__ = Some(map_.next_value()?);
2475724767
}
24768+
GeneratedField::Schema => {
24769+
if schema__.is_some() {
24770+
return Err(serde::de::Error::duplicate_field("schema"));
24771+
}
24772+
schema__ = map_.next_value()?;
24773+
}
2475824774
}
2475924775
}
2476024776
Ok(ProjectionExecNode {
2476124777
input: input__,
2476224778
expr: expr__.unwrap_or_default(),
2476324779
expr_name: expr_name__.unwrap_or_default(),
24780+
schema: schema__,
2476424781
})
2476524782
}
2476624783
}

‎datafusion/proto-models/src/generated/prost.rs‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2302,6 +2302,12 @@ pub struct ProjectionExecNode {
23022302
pub expr: ::prost::alloc::vec::Vec<PhysicalExprNode>,
23032303
#[prost(string, repeated, tag = "3")]
23042304
pub expr_name: ::prost::alloc::vec::Vec<::prost::alloc::string::String>,
2305+
/// Only field and schema metadata are used; output types are derived from expr.
2306+
/// Present when the projection output has metadata or explicitly overrides
2307+
/// inherited metadata, including an explicit clear. Absent for older plans
2308+
/// and metadata-free projections.
2309+
#[prost(message, optional, tag = "4")]
2310+
pub schema: ::core::option::Option<super::datafusion_common::Schema>,
23052311
}
23062312
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
23072313
pub struct PartiallySortedInputOrderMode {

0 commit comments

Comments
 (0)