Skip to content

Commit 832d5b8

Browse files
buraksennkosiew
authored andcommitted
refactor(proto): migrate GlobalLimitExec and LocalLimitExec serde (apache#23791)
## Which issue does this PR close? - Closes apache#23502. ## Rationale for this change Part of epic apache#23494. Moves `GlobalLimitExec` and `LocalLimitExec` protobuf serialization from central dispatch. ## What changes are included in this PR? Add new proto functions in physical plan and deprecate proto ones. ## Are these changes tested? Yes, existing round trip tests pass. ## Are there any user-facing changes? Existing methods are deprecated with no immediate API change.
1 parent 769bfc0 commit 832d5b8

2 files changed

Lines changed: 142 additions & 60 deletions

File tree

‎datafusion/physical-plan/src/limit.rs‎

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -244,6 +244,59 @@ impl ExecutionPlan for GlobalLimitExec {
244244
fn supports_limit_pushdown(&self) -> bool {
245245
true
246246
}
247+
248+
#[cfg(feature = "proto")]
249+
fn try_to_proto(
250+
&self,
251+
ctx: &crate::proto::ExecutionPlanEncodeCtx<'_>,
252+
) -> Result<Option<datafusion_proto_models::protobuf::PhysicalPlanNode>> {
253+
use datafusion_proto_models::protobuf;
254+
let input = ctx.encode_child(self.input())?;
255+
Ok(Some(protobuf::PhysicalPlanNode {
256+
physical_plan_type: Some(
257+
protobuf::physical_plan_node::PhysicalPlanType::GlobalLimit(Box::new(
258+
protobuf::GlobalLimitExecNode {
259+
input: Some(Box::new(input)),
260+
skip: self.skip() as u32,
261+
fetch: match self.fetch() {
262+
Some(n) => n as i64,
263+
_ => -1, // no limit
264+
},
265+
},
266+
)),
267+
),
268+
}))
269+
}
270+
}
271+
272+
#[cfg(feature = "proto")]
273+
impl GlobalLimitExec {
274+
pub fn try_from_proto(
275+
node: &datafusion_proto_models::protobuf::PhysicalPlanNode,
276+
ctx: &crate::proto::ExecutionPlanDecodeCtx<'_>,
277+
) -> Result<Arc<dyn ExecutionPlan>> {
278+
use datafusion_proto_models::protobuf;
279+
let limit = crate::expect_plan_variant!(
280+
node,
281+
protobuf::physical_plan_node::PhysicalPlanType::GlobalLimit,
282+
"GlobalLimitExec",
283+
);
284+
let input = ctx.decode_required_child(
285+
limit.input.as_deref(),
286+
"GlobalLimitExec",
287+
"input",
288+
)?;
289+
let fetch = if limit.fetch >= 0 {
290+
Some(limit.fetch as usize)
291+
} else {
292+
None
293+
};
294+
Ok(Arc::new(GlobalLimitExec::new(
295+
input,
296+
limit.skip as usize,
297+
fetch,
298+
)))
299+
}
247300
}
248301

249302
/// LocalLimitExec applies a limit to a single partition
@@ -419,6 +472,43 @@ impl ExecutionPlan for LocalLimitExec {
419472
fn cardinality_effect(&self) -> CardinalityEffect {
420473
CardinalityEffect::LowerEqual
421474
}
475+
476+
#[cfg(feature = "proto")]
477+
fn try_to_proto(
478+
&self,
479+
ctx: &crate::proto::ExecutionPlanEncodeCtx<'_>,
480+
) -> Result<Option<datafusion_proto_models::protobuf::PhysicalPlanNode>> {
481+
use datafusion_proto_models::protobuf;
482+
let input = ctx.encode_child(self.input())?;
483+
Ok(Some(protobuf::PhysicalPlanNode {
484+
physical_plan_type: Some(
485+
protobuf::physical_plan_node::PhysicalPlanType::LocalLimit(Box::new(
486+
protobuf::LocalLimitExecNode {
487+
input: Some(Box::new(input)),
488+
fetch: self.fetch() as u32,
489+
},
490+
)),
491+
),
492+
}))
493+
}
494+
}
495+
496+
#[cfg(feature = "proto")]
497+
impl LocalLimitExec {
498+
pub fn try_from_proto(
499+
node: &datafusion_proto_models::protobuf::PhysicalPlanNode,
500+
ctx: &crate::proto::ExecutionPlanDecodeCtx<'_>,
501+
) -> Result<Arc<dyn ExecutionPlan>> {
502+
use datafusion_proto_models::protobuf;
503+
let limit = crate::expect_plan_variant!(
504+
node,
505+
protobuf::physical_plan_node::PhysicalPlanType::LocalLimit,
506+
"LocalLimitExec",
507+
);
508+
let input =
509+
ctx.decode_required_child(limit.input.as_deref(), "LocalLimitExec", "input")?;
510+
Ok(Arc::new(LocalLimitExec::new(input, limit.fetch as usize)))
511+
}
422512
}
423513

424514
/// A Limit stream skips `skip` rows, and then fetch up to `fetch` rows.

‎datafusion/proto/src/physical_plan/mod.rs‎

Lines changed: 52 additions & 60 deletions
Original file line numberDiff line numberDiff line change
@@ -789,11 +789,11 @@ pub trait PhysicalPlanNodeExt: Sized {
789789
PhysicalPlanType::Repartition(repart) => {
790790
self.try_into_repartition_physical_plan(repart, ctx, proto_converter)
791791
}
792-
PhysicalPlanType::GlobalLimit(limit) => {
793-
self.try_into_global_limit_physical_plan(limit, ctx, proto_converter)
792+
PhysicalPlanType::GlobalLimit(_) => {
793+
GlobalLimitExec::try_from_proto(self.node(), &decode_ctx)
794794
}
795-
PhysicalPlanType::LocalLimit(limit) => {
796-
self.try_into_local_limit_physical_plan(limit, ctx, proto_converter)
795+
PhysicalPlanType::LocalLimit(_) => {
796+
LocalLimitExec::try_from_proto(self.node(), &decode_ctx)
797797
}
798798
PhysicalPlanType::Window(window_agg) => {
799799
self.try_into_window_physical_plan(window_agg, ctx, proto_converter)
@@ -914,22 +914,6 @@ pub trait PhysicalPlanNodeExt: Sized {
914914
);
915915
}
916916

917-
if let Some(limit) = plan.downcast_ref::<GlobalLimitExec>() {
918-
return protobuf::PhysicalPlanNode::try_from_global_limit_exec(
919-
limit,
920-
codec,
921-
proto_converter,
922-
);
923-
}
924-
925-
if let Some(limit) = plan.downcast_ref::<LocalLimitExec>() {
926-
return protobuf::PhysicalPlanNode::try_from_local_limit_exec(
927-
limit,
928-
codec,
929-
proto_converter,
930-
);
931-
}
932-
933917
if let Some(exec) = plan.downcast_ref::<HashJoinExec>() {
934918
return protobuf::PhysicalPlanNode::try_from_hash_join_exec(
935919
exec,
@@ -1469,35 +1453,50 @@ pub trait PhysicalPlanNodeExt: Sized {
14691453
Ok(Arc::new(repart_exec))
14701454
}
14711455

1456+
#[deprecated(
1457+
since = "55.0.0",
1458+
note = "unused by DataFusion; `GlobalLimitExec` deserializes itself via `GlobalLimitExec::try_from_proto`"
1459+
)]
14721460
fn try_into_global_limit_physical_plan(
14731461
&self,
14741462
limit: &protobuf::GlobalLimitExecNode,
14751463
ctx: &PhysicalPlanDecodeContext<'_>,
14761464
proto_converter: &dyn PhysicalProtoConverterExtension,
14771465
) -> Result<Arc<dyn ExecutionPlan>> {
1478-
let input: Arc<dyn ExecutionPlan> =
1479-
into_physical_plan(&limit.input, ctx, proto_converter)?;
1480-
let fetch = if limit.fetch >= 0 {
1481-
Some(limit.fetch as usize)
1482-
} else {
1483-
None
1466+
let node = protobuf::PhysicalPlanNode {
1467+
physical_plan_type: Some(PhysicalPlanType::GlobalLimit(Box::new(
1468+
limit.clone(),
1469+
))),
14841470
};
1485-
Ok(Arc::new(GlobalLimitExec::new(
1486-
input,
1487-
limit.skip as usize,
1488-
fetch,
1489-
)))
1471+
let decoder = ConverterPlanDecoder {
1472+
ctx,
1473+
proto_converter,
1474+
};
1475+
let decode_ctx = ExecutionPlanDecodeCtx::new(&decoder);
1476+
GlobalLimitExec::try_from_proto(&node, &decode_ctx)
14901477
}
14911478

1479+
#[deprecated(
1480+
since = "55.0.0",
1481+
note = "unused by DataFusion; `LocalLimitExec` deserializes itself via `LocalLimitExec::try_from_proto`"
1482+
)]
14921483
fn try_into_local_limit_physical_plan(
14931484
&self,
14941485
limit: &protobuf::LocalLimitExecNode,
14951486
ctx: &PhysicalPlanDecodeContext<'_>,
14961487
proto_converter: &dyn PhysicalProtoConverterExtension,
14971488
) -> Result<Arc<dyn ExecutionPlan>> {
1498-
let input: Arc<dyn ExecutionPlan> =
1499-
into_physical_plan(&limit.input, ctx, proto_converter)?;
1500-
Ok(Arc::new(LocalLimitExec::new(input, limit.fetch as usize)))
1489+
let node = protobuf::PhysicalPlanNode {
1490+
physical_plan_type: Some(PhysicalPlanType::LocalLimit(Box::new(
1491+
limit.clone(),
1492+
))),
1493+
};
1494+
let decoder = ConverterPlanDecoder {
1495+
ctx,
1496+
proto_converter,
1497+
};
1498+
let decode_ctx = ExecutionPlanDecodeCtx::new(&decoder);
1499+
LocalLimitExec::try_from_proto(&node, &decode_ctx)
15011500
}
15021501

15031502
fn try_into_window_physical_plan(
@@ -2717,49 +2716,42 @@ pub trait PhysicalPlanNodeExt: Sized {
27172716
.ok_or_else(|| internal_datafusion_err!("FilterExec is not serializable"))
27182717
}
27192718

2719+
#[deprecated(
2720+
since = "55.0.0",
2721+
note = "unused by DataFusion; `GlobalLimitExec` serializes itself via `ExecutionPlan::try_to_proto`"
2722+
)]
27202723
fn try_from_global_limit_exec(
27212724
limit: &GlobalLimitExec,
27222725
codec: &dyn PhysicalExtensionCodec,
27232726
proto_converter: &dyn PhysicalProtoConverterExtension,
27242727
) -> Result<protobuf::PhysicalPlanNode> {
2725-
let input = protobuf::PhysicalPlanNode::try_from_physical_plan_with_converter(
2726-
limit.input().to_owned(),
2728+
let encoder = ConverterPlanEncoder {
27272729
codec,
27282730
proto_converter,
2729-
)?;
2730-
2731-
Ok(protobuf::PhysicalPlanNode {
2732-
physical_plan_type: Some(PhysicalPlanType::GlobalLimit(Box::new(
2733-
protobuf::GlobalLimitExecNode {
2734-
input: Some(Box::new(input)),
2735-
skip: limit.skip() as u32,
2736-
fetch: match limit.fetch() {
2737-
Some(n) => n as i64,
2738-
_ => -1, // no limit
2739-
},
2740-
},
2741-
))),
2731+
};
2732+
let encode_ctx = ExecutionPlanEncodeCtx::new(&encoder);
2733+
limit.try_to_proto(&encode_ctx)?.ok_or_else(|| {
2734+
internal_datafusion_err!("GlobalLimitExec is not serializable")
27422735
})
27432736
}
27442737

2738+
#[deprecated(
2739+
since = "55.0.0",
2740+
note = "unused by DataFusion; `LocalLimitExec` serializes itself via `ExecutionPlan::try_to_proto`"
2741+
)]
27452742
fn try_from_local_limit_exec(
27462743
limit: &LocalLimitExec,
27472744
codec: &dyn PhysicalExtensionCodec,
27482745
proto_converter: &dyn PhysicalProtoConverterExtension,
27492746
) -> Result<protobuf::PhysicalPlanNode> {
2750-
let input = protobuf::PhysicalPlanNode::try_from_physical_plan_with_converter(
2751-
limit.input().to_owned(),
2747+
let encoder = ConverterPlanEncoder {
27522748
codec,
27532749
proto_converter,
2754-
)?;
2755-
Ok(protobuf::PhysicalPlanNode {
2756-
physical_plan_type: Some(PhysicalPlanType::LocalLimit(Box::new(
2757-
protobuf::LocalLimitExecNode {
2758-
input: Some(Box::new(input)),
2759-
fetch: limit.fetch() as u32,
2760-
},
2761-
))),
2762-
})
2750+
};
2751+
let encode_ctx = ExecutionPlanEncodeCtx::new(&encoder);
2752+
limit
2753+
.try_to_proto(&encode_ctx)?
2754+
.ok_or_else(|| internal_datafusion_err!("LocalLimitExec is not serializable"))
27632755
}
27642756

27652757
fn try_from_hash_join_exec(

0 commit comments

Comments
 (0)