Skip to content

Commit a61a491

Browse files
committed
fix(physical-plan): preserve sort-merge join API compatibility
Restore the seven-argument SortMergeJoinExec::try_new constructor and enable null-aware semantics through the validated with_null_aware builder. Preserve the flag when rebuilding plans. Restore the original protobuf schema and generated types. Reject serialization of null-aware sort-merge joins rather than silently losing their semantics. Restore ordinary join test configuration and update direct stream callers to pass no null-aware mode.
1 parent d9f95b6 commit a61a491

5 files changed

Lines changed: 45 additions & 90 deletions

File tree

‎datafusion/physical-plan/src/joins/sort_merge_join/exec.rs‎

Lines changed: 35 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -213,7 +213,6 @@ impl SortMergeJoinExec {
213213
/// The inputs are sorted using `sort_options` are applied to the columns in the `on`
214214
/// # Error
215215
/// This function errors when it is not possible to join the left and right sides on keys `on`.
216-
#[expect(clippy::too_many_arguments)]
217216
pub fn try_new(
218217
left: Arc<dyn ExecutionPlan>,
219218
right: Arc<dyn ExecutionPlan>,
@@ -222,7 +221,6 @@ impl SortMergeJoinExec {
222221
join_type: JoinType,
223222
sort_options: Vec<SortOptions>,
224223
null_equality: NullEquality,
225-
null_aware: bool,
226224
) -> Result<Self> {
227225
let left_schema = left.schema();
228226
let right_schema = right.schema();
@@ -278,12 +276,23 @@ impl SortMergeJoinExec {
278276
right_sort_exprs,
279277
sort_options,
280278
null_equality,
281-
null_aware,
279+
null_aware: false,
282280
projection: None,
283281
cache: Arc::new(cache),
284282
})
285283
}
286284

285+
/// Enables or disables null-aware (`NOT IN`) semantics.
286+
///
287+
/// Enabling requires an uncorrelated `LeftAnti` join with one key, no
288+
/// filter, [`NullEquality::NullEqualsNothing`], and NULLS FIRST ordering.
289+
/// Both inputs must have a single partition when executed.
290+
pub fn with_null_aware(mut self, null_aware: bool) -> Result<Self> {
291+
self.null_aware = null_aware;
292+
self.null_aware_mode()?;
293+
Ok(self)
294+
}
295+
287296
/// Returns this join emitting only the columns in `projection`, in that order.
288297
/// The indices address the join's own schema, before any projection.
289298
pub fn with_projection(&self, projection: Option<Vec<usize>>) -> Result<Self> {
@@ -452,8 +461,8 @@ impl SortMergeJoinExec {
452461
self.join_type().swap(),
453462
self.sort_options.clone(),
454463
self.null_equality,
455-
self.null_aware,
456464
)?
465+
.with_null_aware(self.null_aware)?
457466
.with_projection(swap_join_projection(
458467
left.schema().fields().len(),
459468
right.schema().fields().len(),
@@ -661,8 +670,8 @@ impl ExecutionPlan for SortMergeJoinExec {
661670
self.join_type,
662671
self.sort_options.clone(),
663672
self.null_equality,
664-
self.null_aware,
665673
)?
674+
.with_null_aware(self.null_aware)?
666675
.with_projection(self.projection.clone())?,
667676
)),
668677
_ => internal_err!("SortMergeJoin wrong number of children"),
@@ -821,16 +830,18 @@ impl ExecutionPlan for SortMergeJoinExec {
821830
self.filter().as_ref(),
822831
&column_indices,
823832
)? {
824-
Ok(Some(Arc::new(SortMergeJoinExec::try_new(
825-
Arc::new(projected_left_child),
826-
Arc::new(projected_right_child),
827-
join_on,
828-
join_filter,
829-
self.join_type,
830-
self.sort_options.clone(),
831-
self.null_equality,
832-
self.null_aware,
833-
)?)))
833+
Ok(Some(Arc::new(
834+
SortMergeJoinExec::try_new(
835+
Arc::new(projected_left_child),
836+
Arc::new(projected_right_child),
837+
join_on,
838+
join_filter,
839+
self.join_type,
840+
self.sort_options.clone(),
841+
self.null_equality,
842+
)?
843+
.with_null_aware(self.null_aware)?,
844+
)))
834845
} else {
835846
try_embed_projection(projection, self)
836847
}
@@ -843,6 +854,14 @@ impl ExecutionPlan for SortMergeJoinExec {
843854
) -> Result<Option<datafusion_proto_models::protobuf::PhysicalPlanNode>> {
844855
use datafusion_proto_models::protobuf;
845856

857+
// The existing protobuf node cannot represent null-aware semantics.
858+
// Reject encoding rather than silently decoding as an ordinary anti join.
859+
if self.null_aware {
860+
return datafusion_common::not_impl_err!(
861+
"Serializing null-aware SortMergeJoinExec is not supported"
862+
);
863+
}
864+
846865
// Destructure exhaustively (no `..`) so that a newly added field is a
847866
// compile error here instead of being silently left out of the proto.
848867
let Self {
@@ -853,7 +872,7 @@ impl ExecutionPlan for SortMergeJoinExec {
853872
join_type,
854873
sort_options,
855874
null_equality,
856-
null_aware,
875+
null_aware: _,
857876
projection,
858877
// derived from the children's schemas by `try_new` on decode
859878
schema: _,
@@ -881,7 +900,6 @@ impl ExecutionPlan for SortMergeJoinExec {
881900

882901
let join_type = crate::joins::proto::join_type_to_proto(*join_type);
883902
let null_equality = crate::joins::proto::null_equality_to_proto(*null_equality);
884-
let null_aware = *null_aware;
885903
let filter = filter
886904
.as_ref()
887905
.map(|filter| crate::joins::proto::join_filter_to_proto(filter, ctx))
@@ -920,7 +938,6 @@ impl ExecutionPlan for SortMergeJoinExec {
920938
indices.iter().map(|index| *index as u32).collect()
921939
}
922940
},
923-
null_aware,
924941
},
925942
)),
926943
),
@@ -957,7 +974,6 @@ impl SortMergeJoinExec {
957974
sort_options,
958975
null_equality,
959976
projection,
960-
null_aware,
961977
} = &**sort_join;
962978

963979
let left =
@@ -992,8 +1008,6 @@ impl SortMergeJoinExec {
9921008
"SortMergeJoinExec",
9931009
)?;
9941010

995-
let null_aware = *null_aware;
996-
9971011
let filter = filter
9981012
.as_ref()
9991013
.map(|filter| {
@@ -1028,7 +1042,6 @@ impl SortMergeJoinExec {
10281042
join_type,
10291043
sort_options,
10301044
null_equality,
1031-
null_aware,
10321045
)?
10331046
.with_projection(projection)?,
10341047
))

0 commit comments

Comments
 (0)