Skip to content

Commit 9a50c67

Browse files
refactor(proto): migrate sort merge join serde (apache#23712)
## Which issue does this PR close? <!-- We generally require a GitHub issue to be filed for all bug fixes and enhancements and this helps us generate change logs for our releases. You can link an issue to this PR using the GitHub syntax. For example `Closes apache#123` indicates that this PR will close issue apache#123. --> - Closes apache#23508 . ## Rationale for this change <!-- Why are you proposing this change? If this is already explained clearly in the issue then this section is not needed. Explaining clearly why changes are proposed helps reviewers understand your changes and offer better suggestions for fixes. --> `SortMergeJoinExec` serialization currently depends on centralized encoding and decoding logic in `datafusion-proto`. This keeps protobuf ownership separate from the execution plan and contributes to the central type-dispatch chain tracked by apache#23494. Moving serialization into `SortMergeJoinExec` keeps the implementation with the plan that owns the relevant state and aligns it with the new per-plan serialization model. ## What changes are included in this PR? <!-- There is no need to duplicate the description in the issue here but it is sometimes worth providing a summary of the individual changes in this PR. --> - Add `ExecutionPlan::try_to_proto` for `SortMergeJoinExec`. - Add `SortMergeJoinExec::try_from_proto`. - Preserve serialization of: - Both child plans. - Join-key expressions. - Join type and null-equality behavior. - Optional join filters and column origins. - Per-key sort options. - Convert `JoinType`, `JoinSide`, and `NullEquality` through exhaustive by-name matches. - Route decoding through the plan-local implementation and remove the centralized encoding downcast. - Retain the old public conversion helpers as deprecated compatibility APIs. - Preserve the existing protobuf wire representation. - Expand roundtrip coverage for every join type, both null-equality modes, filters, column origins, and non-default sort options. ## Are these changes tested? Yes <!-- We typically require tests for all PRs in order to: 1. Prevent the code from being accidentally broken by subsequent changes 2. Serve as another way to document the expected behavior of the code If tests are not included in your PR, please explain why (for example, are they covered by existing tests)? --> ## Are there any user-facing changes? <!-- If there are user-facing changes then we may require documentation to be updated before approving the PR. --> <!-- If there are any breaking changes to public APIs, please add the `api change` label. --> There are no runtime behavior or protobuf wire-format changes. The old `PhysicalPlanNodeExt` conversion helpers remain available but are deprecated in favor of the plan-local serialization hooks. --------- Signed-off-by: Jiawei Zhao <[email protected]>
1 parent f3f381c commit 9a50c67

3 files changed

Lines changed: 317 additions & 209 deletions

File tree

‎datafusion/physical-plan/src/joins/sort_merge_join/exec.rs‎

Lines changed: 242 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -652,4 +652,246 @@ impl ExecutionPlan for SortMergeJoinExec {
652652
self.null_equality,
653653
)?)))
654654
}
655+
656+
#[cfg(feature = "proto")]
657+
fn try_to_proto(
658+
&self,
659+
ctx: &crate::proto::ExecutionPlanEncodeCtx<'_>,
660+
) -> Result<Option<datafusion_proto_models::protobuf::PhysicalPlanNode>> {
661+
use datafusion_proto_models::protobuf;
662+
663+
let left = ctx.encode_child(self.left())?;
664+
let right = ctx.encode_child(self.right())?;
665+
let on = self
666+
.on()
667+
.iter()
668+
.map(|(left, right)| {
669+
Ok(protobuf::JoinOn {
670+
left: Some(ctx.encode_expr(left)?),
671+
right: Some(ctx.encode_expr(right)?),
672+
})
673+
})
674+
.collect::<Result<Vec<_>>>()?;
675+
676+
let join_type = match self.join_type() {
677+
JoinType::Inner => protobuf::JoinType::Inner,
678+
JoinType::Left => protobuf::JoinType::Left,
679+
JoinType::Right => protobuf::JoinType::Right,
680+
JoinType::Full => protobuf::JoinType::Full,
681+
JoinType::LeftSemi => protobuf::JoinType::Leftsemi,
682+
JoinType::RightSemi => protobuf::JoinType::Rightsemi,
683+
JoinType::LeftAnti => protobuf::JoinType::Leftanti,
684+
JoinType::RightAnti => protobuf::JoinType::Rightanti,
685+
JoinType::LeftMark => protobuf::JoinType::Leftmark,
686+
JoinType::RightMark => protobuf::JoinType::Rightmark,
687+
};
688+
let null_equality = match self.null_equality() {
689+
NullEquality::NullEqualsNothing => protobuf::NullEquality::NullEqualsNothing,
690+
NullEquality::NullEqualsNull => protobuf::NullEquality::NullEqualsNull,
691+
};
692+
let filter = self
693+
.filter()
694+
.as_ref()
695+
.map(|filter| -> Result<protobuf::JoinFilter> {
696+
let expression = ctx.encode_expr(filter.expression())?;
697+
let column_indices = filter
698+
.column_indices()
699+
.iter()
700+
.map(|column_index| {
701+
let side = match column_index.side {
702+
JoinSide::Left => protobuf::JoinSide::LeftSide,
703+
JoinSide::Right => protobuf::JoinSide::RightSide,
704+
JoinSide::None => protobuf::JoinSide::None,
705+
};
706+
protobuf::ColumnIndex {
707+
index: column_index.index as u32,
708+
side: side.into(),
709+
}
710+
})
711+
.collect();
712+
Ok(protobuf::JoinFilter {
713+
expression: Some(expression),
714+
column_indices,
715+
schema: Some(filter.schema().as_ref().try_into()?),
716+
})
717+
})
718+
.transpose()?;
719+
let sort_options = self
720+
.sort_options()
721+
.iter()
722+
.map(|options| protobuf::SortExprNode {
723+
expr: None,
724+
asc: !options.descending,
725+
nulls_first: options.nulls_first,
726+
})
727+
.collect();
728+
729+
Ok(Some(protobuf::PhysicalPlanNode {
730+
physical_plan_type: Some(
731+
protobuf::physical_plan_node::PhysicalPlanType::SortMergeJoin(Box::new(
732+
protobuf::SortMergeJoinExecNode {
733+
left: Some(Box::new(left)),
734+
right: Some(Box::new(right)),
735+
on,
736+
join_type: join_type.into(),
737+
filter,
738+
sort_options,
739+
null_equality: null_equality.into(),
740+
},
741+
)),
742+
),
743+
}))
744+
}
745+
}
746+
747+
#[cfg(feature = "proto")]
748+
impl SortMergeJoinExec {
749+
/// Reconstruct a [`SortMergeJoinExec`] from its protobuf representation.
750+
///
751+
/// The exact inverse of [`ExecutionPlan::try_to_proto`].
752+
///
753+
/// [`ExecutionPlan::try_to_proto`]: crate::ExecutionPlan::try_to_proto
754+
pub fn try_from_proto(
755+
node: &datafusion_proto_models::protobuf::PhysicalPlanNode,
756+
ctx: &crate::proto::ExecutionPlanDecodeCtx<'_>,
757+
) -> Result<Arc<dyn ExecutionPlan>> {
758+
use crate::joins::utils::ColumnIndex;
759+
use arrow::datatypes::Schema;
760+
use datafusion_common::internal_datafusion_err;
761+
use datafusion_proto_models::protobuf;
762+
763+
let sort_join = crate::expect_plan_variant!(
764+
node,
765+
protobuf::physical_plan_node::PhysicalPlanType::SortMergeJoin,
766+
"SortMergeJoinExec",
767+
);
768+
let left = ctx.decode_required_child(
769+
sort_join.left.as_deref(),
770+
"SortMergeJoinExec",
771+
"left",
772+
)?;
773+
let right = ctx.decode_required_child(
774+
sort_join.right.as_deref(),
775+
"SortMergeJoinExec",
776+
"right",
777+
)?;
778+
let left_schema = left.schema();
779+
let right_schema = right.schema();
780+
let on = sort_join
781+
.on
782+
.iter()
783+
.map(|columns| {
784+
let left = ctx.decode_required_expr(
785+
columns.left.as_ref(),
786+
left_schema.as_ref(),
787+
"SortMergeJoinExec",
788+
"on.left",
789+
)?;
790+
let right = ctx.decode_required_expr(
791+
columns.right.as_ref(),
792+
right_schema.as_ref(),
793+
"SortMergeJoinExec",
794+
"on.right",
795+
)?;
796+
Ok((left, right))
797+
})
798+
.collect::<Result<JoinOn>>()?;
799+
800+
let join_type =
801+
match protobuf::JoinType::try_from(sort_join.join_type).map_err(|_| {
802+
internal_datafusion_err!(
803+
"SortMergeJoinExec: unknown JoinType {}",
804+
sort_join.join_type
805+
)
806+
})? {
807+
protobuf::JoinType::Inner => JoinType::Inner,
808+
protobuf::JoinType::Left => JoinType::Left,
809+
protobuf::JoinType::Right => JoinType::Right,
810+
protobuf::JoinType::Full => JoinType::Full,
811+
protobuf::JoinType::Leftsemi => JoinType::LeftSemi,
812+
protobuf::JoinType::Rightsemi => JoinType::RightSemi,
813+
protobuf::JoinType::Leftanti => JoinType::LeftAnti,
814+
protobuf::JoinType::Rightanti => JoinType::RightAnti,
815+
protobuf::JoinType::Leftmark => JoinType::LeftMark,
816+
protobuf::JoinType::Rightmark => JoinType::RightMark,
817+
};
818+
let null_equality = match protobuf::NullEquality::try_from(
819+
sort_join.null_equality,
820+
)
821+
.map_err(|_| {
822+
internal_datafusion_err!(
823+
"SortMergeJoinExec: unknown NullEquality {}",
824+
sort_join.null_equality
825+
)
826+
})? {
827+
protobuf::NullEquality::NullEqualsNothing => NullEquality::NullEqualsNothing,
828+
protobuf::NullEquality::NullEqualsNull => NullEquality::NullEqualsNull,
829+
};
830+
let filter = sort_join
831+
.filter
832+
.as_ref()
833+
.map(|filter| -> Result<JoinFilter> {
834+
let schema: Schema = filter
835+
.schema
836+
.as_ref()
837+
.ok_or_else(|| {
838+
internal_datafusion_err!(
839+
"SortMergeJoinExec: JoinFilter missing schema"
840+
)
841+
})?
842+
.try_into()?;
843+
let expression = ctx.decode_required_expr(
844+
filter.expression.as_ref(),
845+
&schema,
846+
"SortMergeJoinExec",
847+
"filter.expression",
848+
)?;
849+
let column_indices = filter
850+
.column_indices
851+
.iter()
852+
.map(|column_index| {
853+
let side = protobuf::JoinSide::try_from(column_index.side)
854+
.map_err(|_| {
855+
internal_datafusion_err!(
856+
"SortMergeJoinExec: unknown JoinSide {}",
857+
column_index.side
858+
)
859+
})?;
860+
let side = match side {
861+
protobuf::JoinSide::LeftSide => JoinSide::Left,
862+
protobuf::JoinSide::RightSide => JoinSide::Right,
863+
protobuf::JoinSide::None => JoinSide::None,
864+
};
865+
Ok(ColumnIndex {
866+
index: column_index.index as usize,
867+
side,
868+
})
869+
})
870+
.collect::<Result<Vec<_>>>()?;
871+
Ok(JoinFilter::new(
872+
expression,
873+
column_indices,
874+
Arc::new(schema),
875+
))
876+
})
877+
.transpose()?;
878+
let sort_options = sort_join
879+
.sort_options
880+
.iter()
881+
.map(|options| SortOptions {
882+
descending: !options.asc,
883+
nulls_first: options.nulls_first,
884+
})
885+
.collect();
886+
887+
Ok(Arc::new(Self::try_new(
888+
left,
889+
right,
890+
on,
891+
filter,
892+
join_type,
893+
sort_options,
894+
null_equality,
895+
)?))
896+
}
655897
}

0 commit comments

Comments
 (0)