Skip to content

Commit 67425b4

Browse files
authored
fix: preserve no-filter SMJ matches across pending outer batches (apache#23049)
## Which issue does this PR close? - Closes apache#23048 ## Rationale for this change When the no-filter bitwise sort-merge join path finds a matching key, it advances the inner cursor past that key before marking all matching outer rows. If the outer key group continues into the next outer batch and polling that batch returns `Pending`, `poll_join` resumes from its top-level state with the inner cursor already past the matched key. On resume, the stream still retained the already-emitted outer batch. That stale batch caused the pending boundary state to be applied to the wrong batch and then discarded. When the actual next outer batch was loaded later, rows continuing the matched key could compare as `Less` than the current inner key and be incorrectly treated as unmatched. Fix this by clearing `outer_batch` after emitting the fully consumed batch and before polling for the next outer batch. This makes the resumed top-level state load the actual next outer batch before applying `resume_boundary`, matching the filtered code path. ## What changes are included in this PR? * Bug fix for SMJ semi/anti-join behavior across outer batches * Add unit test ## Are these changes tested? Yes. ## Are there any user-facing changes? No.
1 parent 7ced0ac commit 67425b4

2 files changed

Lines changed: 101 additions & 1 deletion

File tree

‎datafusion/physical-plan/src/joins/sort_merge_join/bitwise_stream.rs‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1177,12 +1177,13 @@ impl BitwiseSortMergeJoinStream {
11771177
self.emit_outer_batch()?;
11781178
self.pending_boundary =
11791179
Some(PendingBoundary::NoFilter { saved_keys });
1180+
// Clear stale batch before polling
1181+
self.outer_batch = None;
11801182

11811183
match ready!(self.poll_next_outer_batch(cx)) {
11821184
Err(e) => return Poll::Ready(Err(e)),
11831185
Ok(false) => {
11841186
self.pending_boundary = None;
1185-
self.outer_batch = None;
11861187
break;
11871188
}
11881189
Ok(true) => {

‎datafusion/physical-plan/src/joins/sort_merge_join/tests.rs‎

Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4342,6 +4342,105 @@ async fn no_filter_boundary_pending_loses_outer_rows() -> Result<()> {
43424342
Ok(())
43434343
}
43444344

4345+
/// Verifies no-filter semi/anti joins when a matching outer key group spans
4346+
/// multiple batches and the next outer batch is temporarily unavailable.
4347+
///
4348+
/// The outer input has an unmatched prefix row followed by a matching key
4349+
/// group that continues in the next batch. Both rows with key=1 should be
4350+
/// treated as matched. Returning `Pending` before the second batch forces
4351+
/// `poll_join` to return and later resume from its top-level state, rather
4352+
/// than continuing the same in-progress boundary loop.
4353+
#[tokio::test]
4354+
async fn no_filter_boundary_pending_with_unmatched_prefix() -> Result<()> {
4355+
let left_schema = Arc::new(Schema::new(vec![
4356+
Field::new("a1", DataType::Int32, false),
4357+
Field::new("b1", DataType::Int32, false),
4358+
Field::new("c1", DataType::Int32, false),
4359+
]));
4360+
let right_schema = Arc::new(Schema::new(vec![
4361+
Field::new("a2", DataType::Int32, false),
4362+
Field::new("b1", DataType::Int32, false),
4363+
Field::new("c2", DataType::Int32, false),
4364+
]));
4365+
4366+
// Key=0 is unmatched. Key=1 matches inner and spans the batch boundary.
4367+
let outer_batch1 = RecordBatch::try_new(
4368+
Arc::clone(&left_schema),
4369+
vec![
4370+
Arc::new(Int32Array::from(vec![0, 1])),
4371+
Arc::new(Int32Array::from(vec![0, 1])),
4372+
Arc::new(Int32Array::from(vec![0, 10])),
4373+
],
4374+
)?;
4375+
let outer_batch2 = RecordBatch::try_new(
4376+
Arc::clone(&left_schema),
4377+
vec![
4378+
Arc::new(Int32Array::from(vec![2])),
4379+
Arc::new(Int32Array::from(vec![1])), // same key
4380+
Arc::new(Int32Array::from(vec![20])),
4381+
],
4382+
)?;
4383+
4384+
// Key=1 matches two outer rows. Key=2 keeps the inner input non-exhausted.
4385+
let inner_batch = RecordBatch::try_new(
4386+
Arc::clone(&right_schema),
4387+
vec![
4388+
Arc::new(Int32Array::from(vec![100, 200])),
4389+
Arc::new(Int32Array::from(vec![1, 2])),
4390+
Arc::new(Int32Array::from(vec![50, 60])),
4391+
],
4392+
)?;
4393+
4394+
let on_outer: Vec<PhysicalExprRef> = vec![Arc::new(Column::new("b1", 1))];
4395+
let on_inner: Vec<PhysicalExprRef> = vec![Arc::new(Column::new("b1", 1))];
4396+
4397+
for (join_type, expected_a1) in [(LeftSemi, vec![1, 2]), (LeftAnti, vec![0])] {
4398+
let outer: SendableRecordBatchStream = Box::pin(PendingStream::new(
4399+
vec![outer_batch1.clone(), outer_batch2.clone()],
4400+
vec![false, true], // Pending before 2nd outer batch
4401+
));
4402+
let inner: SendableRecordBatchStream =
4403+
Box::pin(PendingStream::new(vec![inner_batch.clone()], vec![false]));
4404+
4405+
let metrics = ExecutionPlanMetricsSet::new();
4406+
let inner_schema = inner.schema();
4407+
let (reservation, spill_manager, runtime_env) =
4408+
test_stream_resources(inner_schema, &metrics);
4409+
let stream = BitwiseSortMergeJoinStream::try_new(
4410+
Arc::clone(&left_schema),
4411+
vec![SortOptions::default()],
4412+
NullEquality::NullEqualsNothing,
4413+
outer,
4414+
inner,
4415+
on_outer.clone(),
4416+
on_inner.clone(),
4417+
None, // no filter
4418+
join_type,
4419+
8192,
4420+
0,
4421+
&metrics,
4422+
reservation,
4423+
spill_manager,
4424+
runtime_env,
4425+
)?;
4426+
4427+
let batches = collect_stream(stream).await?;
4428+
let actual_a1 = batches
4429+
.iter()
4430+
.flat_map(|batch| {
4431+
let values = batch
4432+
.column(0)
4433+
.as_any()
4434+
.downcast_ref::<Int32Array>()
4435+
.unwrap();
4436+
(0..batch.num_rows()).map(|row| values.value(row))
4437+
})
4438+
.collect::<Vec<_>>();
4439+
assert_eq!(actual_a1, expected_a1, "{join_type:?}");
4440+
}
4441+
Ok(())
4442+
}
4443+
43454444
/// Tests the filtered boundary Pending re-entry: outer key group spans
43464445
/// batches with a filter, and poll_next_outer_batch returns Pending.
43474446
///

0 commit comments

Comments
 (0)