Skip to content

Commit 449dee2

Browse files
buraksennkosiew
authored andcommitted
refactor(proto): migrate UnionExec and InterleaveExec serde (apache#23782)
## Which issue does this PR close? - Closes apache#23503. ## Rationale for this change Part of epic apache#23494. Moves `UnionExec` and `InterleaveExec` protobuf serialization from central dispatch ## What changes are included in this PR? add new proto functions in physical plan and deprecate proto ones ## Are these changes tested? yes existing round trip tests pass ## Are there any user-facing changes? existing methods are deprecated with no immediate api change
1 parent c07dd39 commit 449dee2

2 files changed

Lines changed: 126 additions & 60 deletions

File tree

‎datafusion/physical-plan/src/union.rs‎

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -473,6 +473,42 @@ impl ExecutionPlan for UnionExec {
473473
// on all children (either pushed down or via FilterExec)
474474
Ok(propagation)
475475
}
476+
#[cfg(feature = "proto")]
477+
fn try_to_proto(
478+
&self,
479+
ctx: &crate::proto::ExecutionPlanEncodeCtx<'_>,
480+
) -> Result<Option<datafusion_proto_models::protobuf::PhysicalPlanNode>> {
481+
use datafusion_proto_models::protobuf;
482+
let inputs = ctx.encode_children(self.inputs())?;
483+
Ok(Some(protobuf::PhysicalPlanNode {
484+
physical_plan_type: Some(
485+
protobuf::physical_plan_node::PhysicalPlanType::Union(
486+
protobuf::UnionExecNode { inputs },
487+
),
488+
),
489+
}))
490+
}
491+
}
492+
493+
#[cfg(feature = "proto")]
494+
impl UnionExec {
495+
pub fn try_from_proto(
496+
node: &datafusion_proto_models::protobuf::PhysicalPlanNode,
497+
ctx: &crate::proto::ExecutionPlanDecodeCtx<'_>,
498+
) -> Result<Arc<dyn ExecutionPlan>> {
499+
use datafusion_proto_models::protobuf;
500+
let union = crate::expect_plan_variant!(
501+
node,
502+
protobuf::physical_plan_node::PhysicalPlanType::Union,
503+
"UnionExec",
504+
);
505+
let inputs = union
506+
.inputs
507+
.iter()
508+
.map(|input| ctx.decode_child(input))
509+
.collect::<Result<Vec<_>>>()?;
510+
UnionExec::try_new(inputs)
511+
}
476512
}
477513

478514
/// Combines multiple input streams by interleaving them.
@@ -683,6 +719,42 @@ impl ExecutionPlan for InterleaveExec {
683719
fn benefits_from_input_partitioning(&self) -> Vec<bool> {
684720
vec![false; self.children().len()]
685721
}
722+
#[cfg(feature = "proto")]
723+
fn try_to_proto(
724+
&self,
725+
ctx: &crate::proto::ExecutionPlanEncodeCtx<'_>,
726+
) -> Result<Option<datafusion_proto_models::protobuf::PhysicalPlanNode>> {
727+
use datafusion_proto_models::protobuf;
728+
let inputs = ctx.encode_children(self.inputs())?;
729+
Ok(Some(protobuf::PhysicalPlanNode {
730+
physical_plan_type: Some(
731+
protobuf::physical_plan_node::PhysicalPlanType::Interleave(
732+
protobuf::InterleaveExecNode { inputs },
733+
),
734+
),
735+
}))
736+
}
737+
}
738+
739+
#[cfg(feature = "proto")]
740+
impl InterleaveExec {
741+
pub fn try_from_proto(
742+
node: &datafusion_proto_models::protobuf::PhysicalPlanNode,
743+
ctx: &crate::proto::ExecutionPlanDecodeCtx<'_>,
744+
) -> Result<Arc<dyn ExecutionPlan>> {
745+
use datafusion_proto_models::protobuf;
746+
let interleave = crate::expect_plan_variant!(
747+
node,
748+
protobuf::physical_plan_node::PhysicalPlanType::Interleave,
749+
"InterleaveExec",
750+
);
751+
let inputs = interleave
752+
.inputs
753+
.iter()
754+
.map(|input| ctx.decode_child(input))
755+
.collect::<Result<Vec<_>>>()?;
756+
Ok(Arc::new(InterleaveExec::try_new(inputs)?))
757+
}
686758
}
687759

688760
/// Returns true if all inputs have the same [`Partitioning::Hash`] or [`Partitioning::Range`]

‎datafusion/proto/src/physical_plan/mod.rs‎

Lines changed: 54 additions & 60 deletions
Original file line numberDiff line numberDiff line change
@@ -808,11 +808,11 @@ pub trait PhysicalPlanNodeExt: Sized {
808808
ctx,
809809
proto_converter,
810810
),
811-
PhysicalPlanType::Union(union) => {
812-
self.try_into_union_physical_plan(union, ctx, proto_converter)
811+
PhysicalPlanType::Union(_) => {
812+
UnionExec::try_from_proto(self.node(), &decode_ctx)
813813
}
814-
PhysicalPlanType::Interleave(interleave) => {
815-
self.try_into_interleave_physical_plan(interleave, ctx, proto_converter)
814+
PhysicalPlanType::Interleave(_) => {
815+
InterleaveExec::try_from_proto(self.node(), &decode_ctx)
816816
}
817817
PhysicalPlanType::CrossJoin(_) => {
818818
CrossJoinExec::try_from_proto(self.node(), &decode_ctx)
@@ -956,22 +956,6 @@ pub trait PhysicalPlanNodeExt: Sized {
956956
return Ok(node);
957957
}
958958

959-
if let Some(union) = plan.downcast_ref::<UnionExec>() {
960-
return protobuf::PhysicalPlanNode::try_from_union_exec(
961-
union,
962-
codec,
963-
proto_converter,
964-
);
965-
}
966-
967-
if let Some(interleave) = plan.downcast_ref::<InterleaveExec>() {
968-
return protobuf::PhysicalPlanNode::try_from_interleave_exec(
969-
interleave,
970-
codec,
971-
proto_converter,
972-
);
973-
}
974-
975959
if let Some(exec) = plan.downcast_ref::<WindowAggExec>() {
976960
return protobuf::PhysicalPlanNode::try_from_window_agg_exec(
977961
exec,
@@ -1998,30 +1982,46 @@ pub trait PhysicalPlanNodeExt: Sized {
19981982
.map(|e| Arc::new(e) as _)
19991983
}
20001984

1985+
#[deprecated(
1986+
since = "55.0.0",
1987+
note = "unused by DataFusion; `UnionExec` deserializes itself via `UnionExec::try_from_proto`"
1988+
)]
20011989
fn try_into_union_physical_plan(
20021990
&self,
20031991
union: &protobuf::UnionExecNode,
20041992
ctx: &PhysicalPlanDecodeContext<'_>,
20051993
proto_converter: &dyn PhysicalProtoConverterExtension,
20061994
) -> Result<Arc<dyn ExecutionPlan>> {
2007-
let mut inputs: Vec<Arc<dyn ExecutionPlan>> = vec![];
2008-
for input in &union.inputs {
2009-
inputs.push(proto_converter.proto_to_execution_plan(input, ctx)?);
2010-
}
2011-
UnionExec::try_new(inputs)
1995+
let node = protobuf::PhysicalPlanNode {
1996+
physical_plan_type: Some(PhysicalPlanType::Union(union.clone())),
1997+
};
1998+
let decoder = ConverterPlanDecoder {
1999+
ctx,
2000+
proto_converter,
2001+
};
2002+
let decode_ctx = ExecutionPlanDecodeCtx::new(&decoder);
2003+
UnionExec::try_from_proto(&node, &decode_ctx)
20122004
}
20132005

2006+
#[deprecated(
2007+
since = "55.0.0",
2008+
note = "unused by DataFusion; `InterleaveExec` deserializes itself via `InterleaveExec::try_from_proto`"
2009+
)]
20142010
fn try_into_interleave_physical_plan(
20152011
&self,
20162012
interleave: &protobuf::InterleaveExecNode,
20172013
ctx: &PhysicalPlanDecodeContext<'_>,
20182014
proto_converter: &dyn PhysicalProtoConverterExtension,
20192015
) -> Result<Arc<dyn ExecutionPlan>> {
2020-
let mut inputs: Vec<Arc<dyn ExecutionPlan>> = vec![];
2021-
for input in &interleave.inputs {
2022-
inputs.push(proto_converter.proto_to_execution_plan(input, ctx)?);
2023-
}
2024-
Ok(Arc::new(InterleaveExec::try_new(inputs)?))
2016+
let node = protobuf::PhysicalPlanNode {
2017+
physical_plan_type: Some(PhysicalPlanType::Interleave(interleave.clone())),
2018+
};
2019+
let decoder = ConverterPlanDecoder {
2020+
ctx,
2021+
proto_converter,
2022+
};
2023+
let decode_ctx = ExecutionPlanDecodeCtx::new(&decoder);
2024+
InterleaveExec::try_from_proto(&node, &decode_ctx)
20252025
}
20262026

20272027
#[deprecated(
@@ -3329,48 +3329,42 @@ pub trait PhysicalPlanNodeExt: Sized {
33293329
.ok_or_else(|| internal_datafusion_err!("SortExec is not serializable"))
33303330
}
33313331

3332+
#[deprecated(
3333+
since = "55.0.0",
3334+
note = "unused by DataFusion; `UnionExec` serializes itself via `ExecutionPlan::try_to_proto`"
3335+
)]
33323336
fn try_from_union_exec(
33333337
union: &UnionExec,
33343338
codec: &dyn PhysicalExtensionCodec,
33353339
proto_converter: &dyn PhysicalProtoConverterExtension,
33363340
) -> Result<protobuf::PhysicalPlanNode> {
3337-
let mut inputs: Vec<protobuf::PhysicalPlanNode> = vec![];
3338-
for input in union.inputs() {
3339-
inputs.push(
3340-
protobuf::PhysicalPlanNode::try_from_physical_plan_with_converter(
3341-
input.to_owned(),
3342-
codec,
3343-
proto_converter,
3344-
)?,
3345-
);
3346-
}
3347-
Ok(protobuf::PhysicalPlanNode {
3348-
physical_plan_type: Some(PhysicalPlanType::Union(protobuf::UnionExecNode {
3349-
inputs,
3350-
})),
3351-
})
3341+
let encoder = ConverterPlanEncoder {
3342+
codec,
3343+
proto_converter,
3344+
};
3345+
let encode_ctx = ExecutionPlanEncodeCtx::new(&encoder);
3346+
union
3347+
.try_to_proto(&encode_ctx)?
3348+
.ok_or_else(|| internal_datafusion_err!("UnionExec is not serializable"))
33523349
}
33533350

3351+
#[deprecated(
3352+
since = "55.0.0",
3353+
note = "unused by DataFusion; `InterleaveExec` serializes itself via `ExecutionPlan::try_to_proto`"
3354+
)]
33543355
fn try_from_interleave_exec(
33553356
interleave: &InterleaveExec,
33563357
codec: &dyn PhysicalExtensionCodec,
33573358
proto_converter: &dyn PhysicalProtoConverterExtension,
33583359
) -> Result<protobuf::PhysicalPlanNode> {
3359-
let mut inputs: Vec<protobuf::PhysicalPlanNode> = vec![];
3360-
for input in interleave.inputs() {
3361-
inputs.push(
3362-
protobuf::PhysicalPlanNode::try_from_physical_plan_with_converter(
3363-
input.to_owned(),
3364-
codec,
3365-
proto_converter,
3366-
)?,
3367-
);
3368-
}
3369-
Ok(protobuf::PhysicalPlanNode {
3370-
physical_plan_type: Some(PhysicalPlanType::Interleave(
3371-
protobuf::InterleaveExecNode { inputs },
3372-
)),
3373-
})
3360+
let encoder = ConverterPlanEncoder {
3361+
codec,
3362+
proto_converter,
3363+
};
3364+
let encode_ctx = ExecutionPlanEncodeCtx::new(&encoder);
3365+
interleave
3366+
.try_to_proto(&encode_ctx)?
3367+
.ok_or_else(|| internal_datafusion_err!("InterleaveExec is not serializable"))
33743368
}
33753369

33763370
#[deprecated(

0 commit comments

Comments
 (0)