Skip to content

Commit 93a29e0

Browse files
SunninskyCuteChuanChuanclaude
authored
[branch-55] fix: allow NULL-padded rows in sort-merge join filter evaluation (#26027) (#27354)
## Which issue does this PR close? - Part of #26868 (the 55.3.0 patch release tracking issue). - Backport of #26027 to `branch-55`. #26027 fixed #26001. ## Rationale for this change With a sort-merge join (`datafusion.optimizer.prefer_hash_join = false`), a `LEFT`, `RIGHT` or `FULL` join with an extra join filter fails with `Arrow error: Invalid argument error: Column '...' is declared as non-nullable but contains null values` when the NULL-padded side has `NOT NULL` columns referenced by the filter. Hash join returns the correct result for the same queries. This is a failed query, which makes it eligible for backport as a correctness fix. When a streamed row has no match but the buffered side still holds a batch, the NULL-padded row is materialized in the same chunk as matched rows, so its NULL buffered-side columns reach filter evaluation, whose intermediate schema declares them non-nullable. #26027 fixed it on `main` (labelled `v56.0.0`); it is on `branch-56` but not on `branch-55`, so it is in neither 55.1.0 nor 55.2.0. It was requested for 55.3.0 on #26868 (#26868 (comment)). ## What changes are included in this PR? - A `git cherry-pick -x` of 8248a57 (#26027, by @CuteChuanChuan). - `datafusion/physical-plan/src/joins/sort_merge_join/materializing_stream.rs`: `MaterializingSortMergeJoinStream::try_new` builds the filter schema once with the buffered-side fields marked nullable, and filter evaluation uses it. It is only built when the join needs deferred filtering (outer join with a filter); inner joins keep the original schema. The filter result of a NULL-padded row does not affect the output, so this does not change results. - `datafusion/sqllogictest/test_files/sort_merge_join.slt`: the `LEFT`, `FULL` and `RIGHT` cases from the issue, and a `LEFT` case whose filter divides by a buffered-side column. - One conflict, in `materializing_stream.rs`: on this branch `needs_deferred_filtering` takes `&Option<JoinFilter>` (on `main` it takes `Option<&JoinFilter>`), so the new local passes `&filter`. The rest is as upstream. ## Are these changes tested? Yes. On this branch (Rust 1.95.0) I ran: - `cargo test -p datafusion-sqllogictest --test sqllogictests -- join`: all 16 matching files pass, including `sort_merge_join.slt` with the new cases. - With the change in `materializing_stream.rs` reverted and the new slt cases kept, the four new queries fail with the Arrow error above (`sort_merge_join.slt` lines 990, 997, 1004 and 1013). - `cargo test -p datafusion-physical-plan --lib`: 1700 passed. - `cargo fmt --all -- --check` and `cargo clippy -p datafusion-physical-plan --all-targets --all-features -- -D warnings` pass. ## Are there any user-facing changes? No. Bug fix only; no public API change. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-authored-by: Yu-Chuan Hung <[email protected]> Co-authored-by: Claude Sonnet 5.5 <[email protected]>
1 parent 23c9f41 commit 93a29e0

2 files changed

Lines changed: 92 additions & 4 deletions

File tree

‎datafusion/physical-plan/src/joins/sort_merge_join/materializing_stream.rs‎

Lines changed: 34 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -45,11 +45,11 @@ use arrow::compute::{
4545
self, BatchCoalescer, SortOptions, concat_batches, filter_record_batch, interleave,
4646
take_arrays,
4747
};
48-
use arrow::datatypes::SchemaRef;
48+
use arrow::datatypes::{Schema, SchemaRef};
4949
use datafusion_common::cast::as_uint64_array;
5050
use datafusion_common::instant::Instant;
5151
use datafusion_common::{
52-
DataFusionError, JoinType, NullEquality, Result, exec_err, internal_err,
52+
DataFusionError, JoinSide, JoinType, NullEquality, Result, exec_err, internal_err,
5353
};
5454
use datafusion_execution::memory_pool::MemoryReservation;
5555
use datafusion_execution::runtime_env::RuntimeEnv;
@@ -295,6 +295,10 @@ pub(super) struct MaterializingSortMergeJoinStream {
295295
/// Cached `needs_deferred_filtering(filter, join_type)` — both inputs
296296
/// are fixed at construction time.
297297
pub deferred_filtering: bool,
298+
/// The join filter's schema with buffered-side fields marked nullable,
299+
/// since NULL-padded rows reach filter evaluation. `None` unless
300+
/// `deferred_filtering`.
301+
pub filter_schema: Option<SchemaRef>,
298302
/// Target output batch size
299303
pub batch_size: usize,
300304

@@ -553,6 +557,30 @@ impl MaterializingSortMergeJoinStream {
553557
semi/anti/mark joins use BitwiseSortMergeJoinStream"
554558
);
555559
let join_time = join_metrics.join_time();
560+
let buffered_side = if join_type == JoinType::Right {
561+
JoinSide::Left
562+
} else {
563+
JoinSide::Right
564+
};
565+
let deferred_filtering = needs_deferred_filtering(&filter, join_type);
566+
let filter_schema = if deferred_filtering && let Some(f) = &filter {
567+
let mut fields = Vec::with_capacity(f.schema().fields().len());
568+
for (column_index, field) in
569+
f.column_indices().iter().zip(f.schema().fields().iter())
570+
{
571+
if column_index.side == buffered_side {
572+
fields.push(Arc::new(field.as_ref().clone().with_nullable(true)));
573+
} else {
574+
fields.push(Arc::clone(field));
575+
}
576+
}
577+
Some(Arc::new(Schema::new_with_metadata(
578+
fields,
579+
f.schema().metadata().clone(),
580+
)))
581+
} else {
582+
None
583+
};
556584
let mut this = Self {
557585
sort_options,
558586
null_equality,
@@ -568,7 +596,8 @@ impl MaterializingSortMergeJoinStream {
568596
buffered_exhausted: false,
569597
on_streamed,
570598
on_buffered,
571-
deferred_filtering: needs_deferred_filtering(&filter, join_type),
599+
deferred_filtering,
600+
filter_schema,
572601
filter,
573602
joined_record_batches: JoinedRecordBatches {
574603
joined_batches: BatchCoalescer::new(Arc::clone(&schema), batch_size)
@@ -1560,8 +1589,9 @@ impl MaterializingSortMergeJoinStream {
15601589

15611590
if !filter_columns.is_empty() {
15621591
if let Some(f) = &self.filter {
1592+
let schema = self.filter_schema.as_ref().unwrap_or_else(|| f.schema());
15631593
let filter_batch =
1564-
RecordBatch::try_new(Arc::clone(f.schema()), filter_columns)?;
1594+
RecordBatch::try_new(Arc::clone(schema), filter_columns)?;
15651595
let filter_result = f
15661596
.expression()
15671597
.evaluate(&filter_batch)?

‎datafusion/sqllogictest/test_files/sort_merge_join.slt‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -971,6 +971,64 @@ WHERE t1_sorted.data < 0
971971
----
972972
100
973973

974+
# Regression test for #26001: outer SMJ with a join filter must not fail
975+
# when the NULL-padded side has NOT NULL columns. target_partitions = 2
976+
# is needed to reproduce; with 1 partition the bug does not trigger.
977+
978+
statement ok
979+
set datafusion.execution.target_partitions = 2;
980+
981+
statement ok
982+
CREATE TABLE a (k INT NOT NULL, v INT NOT NULL) AS VALUES (1, 10), (2, 20), (3, 30);
983+
984+
statement ok
985+
CREATE TABLE b (k INT NOT NULL, w INT NOT NULL) AS VALUES (1, 100), (3, 300);
986+
987+
statement ok
988+
CREATE TABLE c (k INT NOT NULL, w INT NOT NULL) AS VALUES (1, 100), (2, 200), (3, 300);
989+
990+
query IIII
991+
SELECT * FROM a LEFT JOIN b ON a.k = b.k AND a.v < b.w ORDER BY a.k
992+
----
993+
1 10 1 100
994+
2 20 NULL NULL
995+
3 30 3 300
996+
997+
query IIII
998+
SELECT * FROM a FULL JOIN b ON a.k = b.k AND a.v < b.w ORDER BY a.k
999+
----
1000+
1 10 1 100
1001+
2 20 NULL NULL
1002+
3 30 3 300
1003+
1004+
query IIII
1005+
SELECT * FROM b RIGHT JOIN c ON b.k = c.k AND b.w <= c.w ORDER BY c.k
1006+
----
1007+
1 100 1 100
1008+
NULL NULL 2 200
1009+
3 300 3 300
1010+
1011+
# NULL-padded rows reach the filter; their zero-valued slots must not
1012+
# cause divide-by-zero errors
1013+
query IIII
1014+
SELECT * FROM a LEFT JOIN b ON a.k = b.k AND a.v / b.w < 1 ORDER BY a.k
1015+
----
1016+
1 10 1 100
1017+
2 20 NULL NULL
1018+
3 30 3 300
1019+
1020+
statement ok
1021+
DROP TABLE a;
1022+
1023+
statement ok
1024+
DROP TABLE b;
1025+
1026+
statement ok
1027+
DROP TABLE c;
1028+
1029+
statement ok
1030+
set datafusion.execution.target_partitions = 4;
1031+
9741032
# Config reset
9751033
statement ok
9761034
RESET datafusion.optimizer.prefer_hash_join;

0 commit comments

Comments
 (0)