Skip to content

Commit 63c5c10

Browse files
buraksennkosiew
authored andcommitted
refactor(proto): migrate CrossJoinExec and NestedLoopJoinExec serde (apache#23834)
## Which issue does this PR close? - Closes apache#23506. ## Rationale for this change Part of epic apache#23494. Moves `CrossJoinExec` and `NestedLoopJoinExec` protobuf serialization from central dispatch. ## What changes are included in this PR? Add new proto functions in physical plan and deprecate proto ones. ## Are these changes tested? Yes, existing round trip tests pass. ## Are there any user-facing changes? Existing methods are deprecated with no immediate API change.
1 parent 7274252 commit 63c5c10

6 files changed

Lines changed: 365 additions & 269 deletions

File tree

‎datafusion/physical-plan/src/joins/cross_join.rs‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -457,6 +457,56 @@ impl ExecutionPlan for CrossJoinExec {
457457
Arc::new(new_right),
458458
))))
459459
}
460+
#[cfg(feature = "proto")]
461+
fn try_to_proto(
462+
&self,
463+
ctx: &crate::proto::ExecutionPlanEncodeCtx<'_>,
464+
) -> Result<Option<datafusion_proto_models::protobuf::PhysicalPlanNode>> {
465+
use datafusion_proto_models::protobuf;
466+
467+
let left = ctx.encode_child(self.left())?;
468+
let right = ctx.encode_child(self.right())?;
469+
470+
Ok(Some(protobuf::PhysicalPlanNode {
471+
physical_plan_type: Some(
472+
protobuf::physical_plan_node::PhysicalPlanType::CrossJoin(Box::new(
473+
protobuf::CrossJoinExecNode {
474+
left: Some(Box::new(left)),
475+
right: Some(Box::new(right)),
476+
},
477+
)),
478+
),
479+
}))
480+
}
481+
}
482+
483+
#[cfg(feature = "proto")]
484+
impl CrossJoinExec {
485+
pub fn try_from_proto(
486+
node: &datafusion_proto_models::protobuf::PhysicalPlanNode,
487+
ctx: &crate::proto::ExecutionPlanDecodeCtx<'_>,
488+
) -> Result<Arc<dyn ExecutionPlan>> {
489+
use datafusion_proto_models::protobuf;
490+
491+
let crossjoin = crate::expect_plan_variant!(
492+
node,
493+
protobuf::physical_plan_node::PhysicalPlanType::CrossJoin,
494+
"CrossJoinExec",
495+
);
496+
497+
let left = ctx.decode_required_child(
498+
crossjoin.left.as_deref(),
499+
"CrossJoinExec",
500+
"left",
501+
)?;
502+
let right = ctx.decode_required_child(
503+
crossjoin.right.as_deref(),
504+
"CrossJoinExec",
505+
"right",
506+
)?;
507+
508+
Ok(Arc::new(CrossJoinExec::new(left, right)))
509+
}
460510
}
461511

462512
/// [left/right]_col_count are required in case the column statistics are None

‎datafusion/physical-plan/src/joins/mod.rs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,8 @@ mod cross_join;
3434
mod hash_join;
3535
mod nested_loop_join;
3636
mod piecewise_merge_join;
37+
#[cfg(feature = "proto")]
38+
mod proto;
3739
mod sort_merge_join;
3840
mod stream_join_utils;
3941
mod symmetric_hash_join;

‎datafusion/physical-plan/src/joins/nested_loop_join.rs‎

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -768,6 +768,91 @@ impl ExecutionPlan for NestedLoopJoinExec {
768768
try_embed_projection(projection, self)
769769
}
770770
}
771+
#[cfg(feature = "proto")]
772+
fn try_to_proto(
773+
&self,
774+
ctx: &crate::proto::ExecutionPlanEncodeCtx<'_>,
775+
) -> Result<Option<datafusion_proto_models::protobuf::PhysicalPlanNode>> {
776+
use datafusion_proto_models::protobuf;
777+
778+
let left = ctx.encode_child(self.left())?;
779+
let right = ctx.encode_child(self.right())?;
780+
781+
let join_type = crate::joins::proto::join_type_to_proto(*self.join_type());
782+
783+
let filter = self
784+
.filter()
785+
.map(|f| crate::joins::proto::join_filter_to_proto(f, ctx))
786+
.transpose()?;
787+
788+
Ok(Some(protobuf::PhysicalPlanNode {
789+
physical_plan_type: Some(
790+
protobuf::physical_plan_node::PhysicalPlanType::NestedLoopJoin(Box::new(
791+
protobuf::NestedLoopJoinExecNode {
792+
left: Some(Box::new(left)),
793+
right: Some(Box::new(right)),
794+
join_type: join_type.into(),
795+
filter,
796+
projection: match self.projection.as_ref() {
797+
None => Vec::new(),
798+
Some(v) if v.is_empty() => vec![u32::MAX],
799+
Some(v) => v.iter().map(|x| *x as u32).collect(),
800+
},
801+
},
802+
)),
803+
),
804+
}))
805+
}
806+
}
807+
808+
#[cfg(feature = "proto")]
809+
impl NestedLoopJoinExec {
810+
pub fn try_from_proto(
811+
node: &datafusion_proto_models::protobuf::PhysicalPlanNode,
812+
ctx: &crate::proto::ExecutionPlanDecodeCtx<'_>,
813+
) -> Result<Arc<dyn ExecutionPlan>> {
814+
use datafusion_proto_models::protobuf;
815+
816+
let join = crate::expect_plan_variant!(
817+
node,
818+
protobuf::physical_plan_node::PhysicalPlanType::NestedLoopJoin,
819+
"NestedLoopJoinExec",
820+
);
821+
822+
let left = ctx.decode_required_child(
823+
join.left.as_deref(),
824+
"NestedLoopJoinExec",
825+
"left",
826+
)?;
827+
let right = ctx.decode_required_child(
828+
join.right.as_deref(),
829+
"NestedLoopJoinExec",
830+
"right",
831+
)?;
832+
833+
let join_type = crate::joins::proto::join_type_from_proto(
834+
join.join_type,
835+
"NestedLoopJoinExec",
836+
)?;
837+
838+
let filter = join
839+
.filter
840+
.as_ref()
841+
.map(|f| {
842+
crate::joins::proto::join_filter_from_proto(f, ctx, "NestedLoopJoinExec")
843+
})
844+
.transpose()?;
845+
846+
let projection = match join.projection.as_slice() {
847+
[] => None,
848+
[u32::MAX] => Some(Vec::new()),
849+
indices => Some(indices.iter().map(|i| *i as usize).collect()),
850+
};
851+
852+
Ok(Arc::new(NestedLoopJoinExec::try_new(
853+
left, right, filter, &join_type, projection,
854+
)?))
855+
}
771856
}
772857

773858
impl EmbeddedProjection for NestedLoopJoinExec {
Lines changed: 161 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,161 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
//! Protobuf conversions shared by the join operators' `try_to_proto` /
19+
//! `try_from_proto` implementations.
20+
//!
21+
//! The enum conversions are by-name exhaustive matches on purpose: the proto
22+
//! enums and the `datafusion_common` enums are numbered differently, so a
23+
//! numeric cast would silently corrupt them.
24+
25+
use std::sync::Arc;
26+
27+
use arrow::datatypes::Schema;
28+
use datafusion_common::{
29+
JoinSide, JoinType, NullEquality, Result, internal_datafusion_err,
30+
};
31+
use datafusion_proto_models::protobuf;
32+
33+
use crate::joins::utils::{ColumnIndex, JoinFilter};
34+
use crate::proto::{ExecutionPlanDecodeCtx, ExecutionPlanEncodeCtx};
35+
36+
pub(crate) fn join_type_to_proto(join_type: JoinType) -> protobuf::JoinType {
37+
match join_type {
38+
JoinType::Inner => protobuf::JoinType::Inner,
39+
JoinType::Left => protobuf::JoinType::Left,
40+
JoinType::Right => protobuf::JoinType::Right,
41+
JoinType::Full => protobuf::JoinType::Full,
42+
JoinType::LeftSemi => protobuf::JoinType::Leftsemi,
43+
JoinType::RightSemi => protobuf::JoinType::Rightsemi,
44+
JoinType::LeftAnti => protobuf::JoinType::Leftanti,
45+
JoinType::RightAnti => protobuf::JoinType::Rightanti,
46+
JoinType::LeftMark => protobuf::JoinType::Leftmark,
47+
JoinType::RightMark => protobuf::JoinType::Rightmark,
48+
}
49+
}
50+
51+
pub(crate) fn join_type_from_proto(value: i32, plan_name: &str) -> Result<JoinType> {
52+
let join_type = protobuf::JoinType::try_from(value)
53+
.map_err(|_| internal_datafusion_err!("{plan_name}: unknown JoinType {value}"))?;
54+
Ok(match join_type {
55+
protobuf::JoinType::Inner => JoinType::Inner,
56+
protobuf::JoinType::Left => JoinType::Left,
57+
protobuf::JoinType::Right => JoinType::Right,
58+
protobuf::JoinType::Full => JoinType::Full,
59+
protobuf::JoinType::Leftsemi => JoinType::LeftSemi,
60+
protobuf::JoinType::Rightsemi => JoinType::RightSemi,
61+
protobuf::JoinType::Leftanti => JoinType::LeftAnti,
62+
protobuf::JoinType::Rightanti => JoinType::RightAnti,
63+
protobuf::JoinType::Leftmark => JoinType::LeftMark,
64+
protobuf::JoinType::Rightmark => JoinType::RightMark,
65+
})
66+
}
67+
68+
pub(crate) fn join_side_to_proto(side: JoinSide) -> protobuf::JoinSide {
69+
match side {
70+
JoinSide::Left => protobuf::JoinSide::LeftSide,
71+
JoinSide::Right => protobuf::JoinSide::RightSide,
72+
JoinSide::None => protobuf::JoinSide::None,
73+
}
74+
}
75+
76+
pub(crate) fn join_side_from_proto(value: i32, plan_name: &str) -> Result<JoinSide> {
77+
let side = protobuf::JoinSide::try_from(value)
78+
.map_err(|_| internal_datafusion_err!("{plan_name}: unknown JoinSide {value}"))?;
79+
Ok(match side {
80+
protobuf::JoinSide::LeftSide => JoinSide::Left,
81+
protobuf::JoinSide::RightSide => JoinSide::Right,
82+
protobuf::JoinSide::None => JoinSide::None,
83+
})
84+
}
85+
86+
pub(crate) fn null_equality_to_proto(
87+
null_equality: NullEquality,
88+
) -> protobuf::NullEquality {
89+
match null_equality {
90+
NullEquality::NullEqualsNothing => protobuf::NullEquality::NullEqualsNothing,
91+
NullEquality::NullEqualsNull => protobuf::NullEquality::NullEqualsNull,
92+
}
93+
}
94+
95+
pub(crate) fn null_equality_from_proto(
96+
value: i32,
97+
plan_name: &str,
98+
) -> Result<NullEquality> {
99+
let null_equality = protobuf::NullEquality::try_from(value).map_err(|_| {
100+
internal_datafusion_err!("{plan_name}: unknown NullEquality {value}")
101+
})?;
102+
Ok(match null_equality {
103+
protobuf::NullEquality::NullEqualsNothing => NullEquality::NullEqualsNothing,
104+
protobuf::NullEquality::NullEqualsNull => NullEquality::NullEqualsNull,
105+
})
106+
}
107+
108+
pub(crate) fn join_filter_to_proto(
109+
filter: &JoinFilter,
110+
ctx: &ExecutionPlanEncodeCtx<'_>,
111+
) -> Result<protobuf::JoinFilter> {
112+
let expression = ctx.encode_expr(filter.expression())?;
113+
let column_indices = filter
114+
.column_indices()
115+
.iter()
116+
.map(|column_index| protobuf::ColumnIndex {
117+
index: column_index.index as u32,
118+
side: join_side_to_proto(column_index.side).into(),
119+
})
120+
.collect();
121+
Ok(protobuf::JoinFilter {
122+
expression: Some(expression),
123+
column_indices,
124+
schema: Some(filter.schema().as_ref().try_into()?),
125+
})
126+
}
127+
128+
pub(crate) fn join_filter_from_proto(
129+
filter: &protobuf::JoinFilter,
130+
ctx: &ExecutionPlanDecodeCtx<'_>,
131+
plan_name: &str,
132+
) -> Result<JoinFilter> {
133+
let schema: Schema = filter
134+
.schema
135+
.as_ref()
136+
.ok_or_else(|| {
137+
internal_datafusion_err!("{plan_name}: JoinFilter missing schema")
138+
})?
139+
.try_into()?;
140+
let expression = ctx.decode_required_expr(
141+
filter.expression.as_ref(),
142+
&schema,
143+
plan_name,
144+
"filter.expression",
145+
)?;
146+
let column_indices = filter
147+
.column_indices
148+
.iter()
149+
.map(|column_index| {
150+
Ok(ColumnIndex {
151+
index: column_index.index as usize,
152+
side: join_side_from_proto(column_index.side, plan_name)?,
153+
})
154+
})
155+
.collect::<Result<Vec<_>>>()?;
156+
Ok(JoinFilter::new(
157+
expression,
158+
column_indices,
159+
Arc::new(schema),
160+
))
161+
}

0 commit comments

Comments
 (0)