Skip to content

Commit 23c9f41

Browse files
Sunninskyjeffw13claude
authored
[branch-55] fix(smj): handle floating-point keys across batch boundaries (#25134) (#27353)
## Which issue does this PR close? - Part of #26868 (the 55.3.0 patch release tracking issue). - Backport of #25134 to `branch-55`. #25134 fixed #25133. ## Rationale for this change On 55.2.0, a mark, semi or anti sort-merge join on a floating-point key fails when a key group ends at the end of an input batch: a Float32 or Float64 key that is NaN panics with ``called `Option::unwrap()` on a `None` value``, and a Float16 key returns `Unsupported data type in sort merge join comparator: Float16`. The main merge scan handles the same keys. These are failed queries, which makes this eligible for backport as a correctness fix. #25134 fixed it on `main` (labelled `v56.0.0`); it is on `branch-56` but not on `branch-55`. It was requested for 55.3.0 on #26868 (#26868 (comment)). ## What changes are included in this PR? - A `git cherry-pick -x` of 2abf69d (#25134, by @jeffw13). It applied without conflicts and this PR changes nothing else. - `datafusion/physical-plan/src/joins/sort_merge_join/bitwise_stream.rs`: `keys_match` uses the existing `JoinKeyComparator` for floating-point keys, matching the merge scan's equality (including signed zero), instead of the scalar comparator whose `partial_cmp` panics on NaN. Integer-only comparisons keep the scalar path. - `datafusion/physical-plan/src/joins/sort_merge_join/tests.rs`: `join_float_key_batch_boundaries`, 144 cases (Float16/32/64, left/right mark/semi/anti, both null-equality modes, ascending/descending, one- and two-row input batches). ## Are these changes tested? Yes, by the 144 cases that #25134 added. On this branch (Rust 1.95.0) I ran: - `cargo test -p datafusion-physical-plan --lib`: 1844 passed. - With the change in `bitwise_stream.rs` reverted and the tests kept, `cargo test -p datafusion-physical-plan --lib sort_merge_join` fails exactly the 144 new cases (96 NaN panics and 48 Float16 errors). - `cargo fmt --all -- --check` and `cargo clippy -p datafusion-physical-plan --all-targets --all-features -- -D warnings` pass. ## Are there any user-facing changes? Affected sort-merge joins complete instead of failing at floating-point batch boundaries. No public API change. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-authored-by: Jeff Wang <[email protected]> Co-authored-by: Claude Sonnet 5.5 <[email protected]>
1 parent 4f4d73f commit 23c9f41

2 files changed

Lines changed: 129 additions & 0 deletions

File tree

‎datafusion/physical-plan/src/joins/sort_merge_join/bitwise_stream.rs‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1178,6 +1178,21 @@ fn keys_match(
11781178
null_equality: NullEquality,
11791179
) -> Result<bool> {
11801180
debug_assert!(left_arrays.iter().all(|a| a.len() == 1));
1181+
if left_arrays
1182+
.iter()
1183+
.any(|array| array.data_type().is_floating())
1184+
{
1185+
// The scalar comparator's partial_cmp panics on NaN. Match the merge
1186+
// scan's floating-point equality, including its signed-zero handling.
1187+
let right_keys = slice_keys(right_arrays, 0);
1188+
return Ok(JoinKeyComparator::new(
1189+
left_arrays,
1190+
&right_keys,
1191+
sort_options,
1192+
null_equality,
1193+
)?
1194+
.is_equal(0, 0));
1195+
}
11811196
let cmp = compare_join_arrays(
11821197
left_arrays,
11831198
0,

‎datafusion/physical-plan/src/joins/sort_merge_join/tests.rs‎

Lines changed: 114 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4029,6 +4029,120 @@ fn columns(schema: &Schema) -> Vec<String> {
40294029
schema.fields().iter().map(|f| f.name().clone()).collect()
40304030
}
40314031

4032+
// Floating-point key groups must compare consistently within and across batches.
4033+
#[rstest::rstest]
4034+
#[tokio::test]
4035+
async fn join_float_key_batch_boundaries(
4036+
#[values(DataType::Float16, DataType::Float32, DataType::Float64)] key_type: DataType,
4037+
#[values(LeftMark, RightMark, LeftSemi, RightSemi, LeftAnti, RightAnti)]
4038+
join_type: JoinType,
4039+
#[values(NullEquality::NullEqualsNothing, NullEquality::NullEqualsNull)]
4040+
null_equality: NullEquality,
4041+
#[values(false, true)] descending: bool,
4042+
#[values(1, 2)] input_batch_size: usize,
4043+
) -> Result<()> {
4044+
let input = |unmatched_key| -> Result<Arc<dyn ExecutionPlan>> {
4045+
let mut keys = vec![
4046+
None,
4047+
Some(f64::NEG_INFINITY),
4048+
Some(-0.0),
4049+
Some(0.0),
4050+
Some(unmatched_key),
4051+
Some(f64::INFINITY),
4052+
Some(f64::NAN),
4053+
Some(f64::NAN),
4054+
Some(f64::NAN),
4055+
];
4056+
let mut ids: Vec<i32> = (0..keys.len() as i32).collect();
4057+
if descending {
4058+
keys.reverse();
4059+
ids.reverse();
4060+
}
4061+
let batch = RecordBatch::try_from_iter(vec![
4062+
(
4063+
"id",
4064+
Arc::new(Int32Array::from(ids)) as arrow::array::ArrayRef,
4065+
),
4066+
(
4067+
"key",
4068+
arrow::compute::cast(&arrow::array::Float64Array::from(keys), &key_type)?,
4069+
),
4070+
])?;
4071+
Ok(build_table_from_batches(
4072+
(0..batch.num_rows())
4073+
.step_by(input_batch_size)
4074+
.map(|offset| {
4075+
batch.slice(offset, input_batch_size.min(batch.num_rows() - offset))
4076+
})
4077+
.collect(),
4078+
))
4079+
};
4080+
let on: JoinOn = vec![(
4081+
Arc::new(Column::new("key", 1)),
4082+
Arc::new(Column::new("key", 1)),
4083+
)];
4084+
let (_, output) = join_collect_with_options(
4085+
input(1.0)?,
4086+
input(2.0)?,
4087+
on,
4088+
join_type,
4089+
vec![SortOptions {
4090+
descending,
4091+
nulls_first: !descending,
4092+
}],
4093+
null_equality,
4094+
)
4095+
.await?;
4096+
4097+
let is_mark = matches!(join_type, LeftMark | RightMark);
4098+
let is_anti = matches!(join_type, LeftAnti | RightAnti);
4099+
let mut expected: Vec<_> = (0..9)
4100+
.map(|id| {
4101+
let matched =
4102+
id != 4 && (id != 0 || null_equality == NullEquality::NullEqualsNull);
4103+
(id, matched)
4104+
})
4105+
.filter(|(_, matched)| is_mark || *matched != is_anti)
4106+
.collect();
4107+
if descending {
4108+
expected.reverse();
4109+
}
4110+
let mut ids = vec![];
4111+
let mut marks = vec![];
4112+
for batch in output {
4113+
assert_eq!(batch.schema().field(1).data_type(), &key_type);
4114+
ids.extend_from_slice(
4115+
batch
4116+
.column(0)
4117+
.as_any()
4118+
.downcast_ref::<Int32Array>()
4119+
.unwrap()
4120+
.values(),
4121+
);
4122+
if is_mark {
4123+
marks.extend(
4124+
batch
4125+
.column(2)
4126+
.as_any()
4127+
.downcast_ref::<BooleanArray>()
4128+
.unwrap()
4129+
.iter(),
4130+
);
4131+
}
4132+
}
4133+
assert_eq!(ids, expected.iter().map(|(id, _)| *id).collect::<Vec<_>>());
4134+
if is_mark {
4135+
assert_eq!(
4136+
marks,
4137+
expected
4138+
.iter()
4139+
.map(|(_, matched)| Some(*matched))
4140+
.collect::<Vec<_>>()
4141+
);
4142+
}
4143+
Ok(())
4144+
}
4145+
40324146
// ==================== BitwiseSortMergeJoinStream direct tests ====================
40334147
//
40344148
// These tests construct a BitwiseSortMergeJoinStream directly (bypassing exec)

0 commit comments

Comments
 (0)