Skip to content

Commit 04a2a38

Browse files
committed
refactor(proto): migrate UnnestExec serde
The central protobuf dispatcher must know every built-in execution plan, which separates UnnestExec serialization from the type that owns it. Move encoding and decoding to plan-local hooks while retaining the deprecated helpers as compatibility wrappers. Extend roundtrip coverage to include non-default unnest options. CLOSES apache#23510 Signed-off-by: Jiawei Zhao <[email protected]>
1 parent 9a50c67 commit 04a2a38

3 files changed

Lines changed: 165 additions & 64 deletions

File tree

‎datafusion/physical-plan/src/unnest.rs‎

Lines changed: 121 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -283,6 +283,127 @@ impl ExecutionPlan for UnnestExec {
283283
fn metrics(&self) -> Option<MetricsSet> {
284284
Some(self.metrics.clone_inner())
285285
}
286+
287+
#[cfg(feature = "proto")]
288+
fn try_to_proto(
289+
&self,
290+
ctx: &crate::proto::ExecutionPlanEncodeCtx<'_>,
291+
) -> Result<Option<datafusion_proto_models::protobuf::PhysicalPlanNode>> {
292+
use datafusion_proto_models::protobuf;
293+
294+
let input = ctx.encode_child(self.input())?;
295+
let schema = self.schema().as_ref().try_into()?;
296+
let list_type_columns = self
297+
.list_column_indices()
298+
.iter()
299+
.map(|column| protobuf::ListUnnest {
300+
index_in_input_schema: column.index_in_input_schema as _,
301+
depth: column.depth as _,
302+
})
303+
.collect();
304+
let struct_type_columns = self
305+
.struct_column_indices()
306+
.iter()
307+
.map(|index| *index as _)
308+
.collect();
309+
let options = protobuf::UnnestOptions {
310+
preserve_nulls: self.options().preserve_nulls,
311+
recursions: self
312+
.options()
313+
.recursions
314+
.iter()
315+
.map(|recursion| protobuf::RecursionUnnestOption {
316+
input_column: Some((&recursion.input_column).into()),
317+
output_column: Some((&recursion.output_column).into()),
318+
depth: recursion.depth as _,
319+
})
320+
.collect(),
321+
};
322+
323+
Ok(Some(protobuf::PhysicalPlanNode {
324+
physical_plan_type: Some(
325+
protobuf::physical_plan_node::PhysicalPlanType::Unnest(Box::new(
326+
protobuf::UnnestExecNode {
327+
input: Some(Box::new(input)),
328+
schema: Some(schema),
329+
list_type_columns,
330+
struct_type_columns,
331+
options: Some(options),
332+
},
333+
)),
334+
),
335+
}))
336+
}
337+
}
338+
339+
#[cfg(feature = "proto")]
340+
impl UnnestExec {
341+
/// Reconstruct an [`UnnestExec`] from its protobuf representation.
342+
///
343+
/// The exact inverse of [`ExecutionPlan::try_to_proto`].
344+
///
345+
/// [`ExecutionPlan::try_to_proto`]: crate::ExecutionPlan::try_to_proto
346+
pub fn try_from_proto(
347+
node: &datafusion_proto_models::protobuf::PhysicalPlanNode,
348+
ctx: &crate::proto::ExecutionPlanDecodeCtx<'_>,
349+
) -> Result<Arc<dyn ExecutionPlan>> {
350+
use datafusion_proto_models::protobuf;
351+
352+
let unnest = crate::expect_plan_variant!(
353+
node,
354+
protobuf::physical_plan_node::PhysicalPlanType::Unnest,
355+
"UnnestExec",
356+
);
357+
let input =
358+
ctx.decode_required_child(unnest.input.as_deref(), "UnnestExec", "input")?;
359+
let schema: Schema = unnest
360+
.schema
361+
.as_ref()
362+
.ok_or_else(|| {
363+
datafusion_common::internal_datafusion_err!(
364+
"UnnestExec is missing required field 'schema'"
365+
)
366+
})?
367+
.try_into()?;
368+
let list_column_indices = unnest
369+
.list_type_columns
370+
.iter()
371+
.map(|column| ListUnnest {
372+
index_in_input_schema: column.index_in_input_schema as _,
373+
depth: column.depth as _,
374+
})
375+
.collect();
376+
let struct_column_indices = unnest
377+
.struct_type_columns
378+
.iter()
379+
.map(|index| *index as _)
380+
.collect();
381+
let options = unnest.options.as_ref().ok_or_else(|| {
382+
datafusion_common::internal_datafusion_err!(
383+
"UnnestExec is missing required field 'options'"
384+
)
385+
})?;
386+
let options = UnnestOptions {
387+
preserve_nulls: options.preserve_nulls,
388+
recursions: options
389+
.recursions
390+
.iter()
391+
.map(|recursion| datafusion_common::RecursionUnnestOption {
392+
input_column: recursion.input_column.as_ref().unwrap().into(),
393+
output_column: recursion.output_column.as_ref().unwrap().into(),
394+
depth: recursion.depth as _,
395+
})
396+
.collect(),
397+
};
398+
399+
Ok(Arc::new(UnnestExec::new(
400+
input,
401+
list_column_indices,
402+
struct_column_indices,
403+
Arc::new(schema),
404+
options,
405+
)?))
406+
}
286407
}
287408

288409
#[derive(Clone, Debug)]

‎datafusion/proto/src/physical_plan/mod.rs‎

Lines changed: 26 additions & 61 deletions
Original file line numberDiff line numberDiff line change
@@ -100,7 +100,7 @@ use datafusion_physical_plan::scalar_subquery::{ScalarSubqueryExec, ScalarSubque
100100
use datafusion_physical_plan::sorts::sort::SortExec;
101101
use datafusion_physical_plan::sorts::sort_preserving_merge::SortPreservingMergeExec;
102102
use datafusion_physical_plan::union::{InterleaveExec, UnionExec};
103-
use datafusion_physical_plan::unnest::{ListUnnest, UnnestExec};
103+
use datafusion_physical_plan::unnest::UnnestExec;
104104
use datafusion_physical_plan::windows::{BoundedWindowAggExec, WindowAggExec};
105105
use datafusion_physical_plan::{ExecutionPlan, InputOrderMode, PhysicalExpr, WindowExpr};
106106
use prost::Message;
@@ -124,10 +124,7 @@ use crate::physical_plan::to_proto::{
124124
use crate::protobuf::physical_aggregate_expr_node::AggregateFunction;
125125
use crate::protobuf::physical_expr_node::ExprType;
126126
use crate::protobuf::physical_plan_node::PhysicalPlanType;
127-
use crate::protobuf::{
128-
self, ListUnnest as ProtoListUnnest, SortMergeJoinExecNode, proto_error,
129-
window_agg_exec_node,
130-
};
127+
use crate::protobuf::{self, SortMergeJoinExecNode, proto_error, window_agg_exec_node};
131128

132129
pub mod from_proto;
133130
pub mod to_proto;
@@ -853,8 +850,8 @@ pub trait PhysicalPlanNodeExt: Sized {
853850
PhysicalPlanType::ParquetSink(sink) => {
854851
self.try_into_parquet_sink_physical_plan(sink, ctx, proto_converter)
855852
}
856-
PhysicalPlanType::Unnest(unnest) => {
857-
self.try_into_unnest_physical_plan(unnest, ctx, proto_converter)
853+
PhysicalPlanType::Unnest(_) => {
854+
UnnestExec::try_from_proto(self.node(), &decode_ctx)
858855
}
859856
PhysicalPlanType::Cooperative(_) => {
860857
CooperativeExec::try_from_proto(self.node(), &decode_ctx)
@@ -1059,14 +1056,6 @@ pub trait PhysicalPlanNodeExt: Sized {
10591056
return Ok(node);
10601057
}
10611058

1062-
if let Some(exec) = plan.downcast_ref::<UnnestExec>() {
1063-
return protobuf::PhysicalPlanNode::try_from_unnest_exec(
1064-
exec,
1065-
codec,
1066-
proto_converter,
1067-
);
1068-
}
1069-
10701059
if let Some(exec) = plan.downcast_ref::<LazyMemoryExec>()
10711060
&& let Some(node) =
10721061
protobuf::PhysicalPlanNode::try_from_lazy_memory_exec(exec)?
@@ -2504,32 +2493,25 @@ pub trait PhysicalPlanNodeExt: Sized {
25042493
panic!("Trying to use ParquetSink without `parquet` feature enabled");
25052494
}
25062495

2496+
#[deprecated(
2497+
since = "55.0.0",
2498+
note = "unused by DataFusion; `UnnestExec` deserializes itself via `UnnestExec::try_from_proto`"
2499+
)]
25072500
fn try_into_unnest_physical_plan(
25082501
&self,
25092502
unnest: &protobuf::UnnestExecNode,
25102503
ctx: &PhysicalPlanDecodeContext<'_>,
25112504
proto_converter: &dyn PhysicalProtoConverterExtension,
25122505
) -> Result<Arc<dyn ExecutionPlan>> {
2513-
let input = into_physical_plan(&unnest.input, ctx, proto_converter)?;
2514-
2515-
Ok(Arc::new(UnnestExec::new(
2516-
input,
2517-
unnest
2518-
.list_type_columns
2519-
.iter()
2520-
.map(|c| ListUnnest {
2521-
index_in_input_schema: c.index_in_input_schema as _,
2522-
depth: c.depth as _,
2523-
})
2524-
.collect(),
2525-
unnest.struct_type_columns.iter().map(|c| *c as _).collect(),
2526-
Arc::new(convert_required!(unnest.schema)?),
2527-
unnest
2528-
.options
2529-
.as_ref()
2530-
.map(datafusion_common::UnnestOptions::from_proto)
2531-
.ok_or_else(|| proto_error("Missing required field in protobuf"))?,
2532-
)?))
2506+
let node = protobuf::PhysicalPlanNode {
2507+
physical_plan_type: Some(PhysicalPlanType::Unnest(Box::new(unnest.clone()))),
2508+
};
2509+
let decoder = ConverterPlanDecoder {
2510+
ctx,
2511+
proto_converter,
2512+
};
2513+
let decode_ctx = ExecutionPlanDecodeCtx::new(&decoder);
2514+
UnnestExec::try_from_proto(&node, &decode_ctx)
25332515
}
25342516

25352517
fn generate_series_name_to_str(name: protobuf::GenerateSeriesName) -> &'static str {
@@ -3892,39 +3874,22 @@ pub trait PhysicalPlanNodeExt: Sized {
38923874
Ok(None)
38933875
}
38943876

3877+
#[deprecated(
3878+
since = "55.0.0",
3879+
note = "unused by DataFusion; `UnnestExec` serializes itself via `ExecutionPlan::try_to_proto`"
3880+
)]
38953881
fn try_from_unnest_exec(
38963882
exec: &UnnestExec,
38973883
codec: &dyn PhysicalExtensionCodec,
38983884
proto_converter: &dyn PhysicalProtoConverterExtension,
38993885
) -> Result<protobuf::PhysicalPlanNode> {
3900-
let input = protobuf::PhysicalPlanNode::try_from_physical_plan_with_converter(
3901-
exec.input().to_owned(),
3886+
let encoder = ConverterPlanEncoder {
39023887
codec,
39033888
proto_converter,
3904-
)?;
3905-
3906-
Ok(protobuf::PhysicalPlanNode {
3907-
physical_plan_type: Some(PhysicalPlanType::Unnest(Box::new(
3908-
protobuf::UnnestExecNode {
3909-
input: Some(Box::new(input)),
3910-
schema: Some(exec.schema().try_into()?),
3911-
list_type_columns: exec
3912-
.list_column_indices()
3913-
.iter()
3914-
.map(|c| ProtoListUnnest {
3915-
index_in_input_schema: c.index_in_input_schema as _,
3916-
depth: c.depth as _,
3917-
})
3918-
.collect(),
3919-
struct_type_columns: exec
3920-
.struct_column_indices()
3921-
.iter()
3922-
.map(|c| *c as _)
3923-
.collect(),
3924-
options: Some(protobuf::UnnestOptions::from_proto(exec.options())),
3925-
},
3926-
))),
3927-
})
3889+
};
3890+
let encode_ctx = ExecutionPlanEncodeCtx::new(&encoder);
3891+
exec.try_to_proto(&encode_ctx)?
3892+
.ok_or_else(|| internal_datafusion_err!("UnnestExec is not serializable"))
39283893
}
39293894

39303895
#[deprecated(

‎datafusion/proto/tests/cases/roundtrip_physical_plan.rs‎

Lines changed: 18 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2251,7 +2251,14 @@ fn roundtrip_unnest() -> Result<()> {
22512251
let output_schema =
22522252
Arc::new(Schema::new(vec![fa, fb0, fc1, fc2, fd0, fe1, fe2, fe3]));
22532253
let input = Arc::new(EmptyExec::new(input_schema));
2254-
let options = UnnestOptions::default();
2254+
let options = UnnestOptions {
2255+
preserve_nulls: false,
2256+
recursions: vec![datafusion_common::RecursionUnnestOption {
2257+
input_column: datafusion_common::Column::new_unqualified("b"),
2258+
output_column: datafusion_common::Column::new_unqualified("b"),
2259+
depth: 2,
2260+
}],
2261+
};
22552262
let unnest = UnnestExec::new(
22562263
input,
22572264
vec![
@@ -2270,9 +2277,17 @@ fn roundtrip_unnest() -> Result<()> {
22702277
],
22712278
vec![2, 4],
22722279
output_schema,
2273-
options,
2280+
options.clone(),
22742281
)?;
2275-
roundtrip_test(Arc::new(unnest))
2282+
let ctx = SessionContext::new();
2283+
let codec = DefaultPhysicalExtensionCodec {};
2284+
let proto_converter = DefaultPhysicalProtoConverter {};
2285+
let result =
2286+
roundtrip_test_and_return(Arc::new(unnest), &ctx, &codec, &proto_converter)?;
2287+
let result = result.downcast_ref::<UnnestExec>().unwrap();
2288+
assert_eq!(result.options(), &options);
2289+
2290+
Ok(())
22762291
}
22772292

22782293
#[tokio::test]

0 commit comments

Comments
 (0)