Skip to content

Commit 887b065

Browse files
[branch-54] fix: preserve no-filter SMJ matches across pending outer batches (backport #23049) (#23574)
## Which issue does this PR close? - Backport of #23049 to `branch-54` (for 54.1.0, tracked in #22547). - Fixes wrong results #23048 (regression introduced in 54.0.0 by the SMJ bitwise-path rewrite #20806). ## Rationale for this change Sort-merge join semi/anti drops matches across pending outer batches, producing wrong results in 54.0.0. No API changes, correctness-only fix, so it fits the backport criteria. ## What changes are included in this PR? Clean cherry-pick of #23049. No adaptation required. ## Are these changes tested? Yes. Carries the original SMJ regression tests, all tests pass. ## Are there any user-facing changes? Semi/anti sort-merge joins now return the correct rows. No API changes. Co-authored-by: Neil Conway <[email protected]>
1 parent 06c4d41 commit 887b065

2 files changed

Lines changed: 101 additions & 1 deletion

File tree

‎datafusion/physical-plan/src/joins/sort_merge_join/bitwise_stream.rs‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1121,12 +1121,13 @@ impl BitwiseSortMergeJoinStream {
11211121
self.emit_outer_batch()?;
11221122
self.pending_boundary =
11231123
Some(PendingBoundary::NoFilter { saved_keys });
1124+
// Clear stale batch before polling
1125+
self.outer_batch = None;
11241126

11251127
match ready!(self.poll_next_outer_batch(cx)) {
11261128
Err(e) => return Poll::Ready(Err(e)),
11271129
Ok(false) => {
11281130
self.pending_boundary = None;
1129-
self.outer_batch = None;
11301131
break;
11311132
}
11321133
Ok(true) => {

‎datafusion/physical-plan/src/joins/sort_merge_join/tests.rs‎

Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4323,6 +4323,105 @@ async fn no_filter_boundary_pending_loses_outer_rows() -> Result<()> {
43234323
Ok(())
43244324
}
43254325

4326+
/// Verifies no-filter semi/anti joins when a matching outer key group spans
4327+
/// multiple batches and the next outer batch is temporarily unavailable.
4328+
///
4329+
/// The outer input has an unmatched prefix row followed by a matching key
4330+
/// group that continues in the next batch. Both rows with key=1 should be
4331+
/// treated as matched. Returning `Pending` before the second batch forces
4332+
/// `poll_join` to return and later resume from its top-level state, rather
4333+
/// than continuing the same in-progress boundary loop.
4334+
#[tokio::test]
4335+
async fn no_filter_boundary_pending_with_unmatched_prefix() -> Result<()> {
4336+
let left_schema = Arc::new(Schema::new(vec![
4337+
Field::new("a1", DataType::Int32, false),
4338+
Field::new("b1", DataType::Int32, false),
4339+
Field::new("c1", DataType::Int32, false),
4340+
]));
4341+
let right_schema = Arc::new(Schema::new(vec![
4342+
Field::new("a2", DataType::Int32, false),
4343+
Field::new("b1", DataType::Int32, false),
4344+
Field::new("c2", DataType::Int32, false),
4345+
]));
4346+
4347+
// Key=0 is unmatched. Key=1 matches inner and spans the batch boundary.
4348+
let outer_batch1 = RecordBatch::try_new(
4349+
Arc::clone(&left_schema),
4350+
vec![
4351+
Arc::new(Int32Array::from(vec![0, 1])),
4352+
Arc::new(Int32Array::from(vec![0, 1])),
4353+
Arc::new(Int32Array::from(vec![0, 10])),
4354+
],
4355+
)?;
4356+
let outer_batch2 = RecordBatch::try_new(
4357+
Arc::clone(&left_schema),
4358+
vec![
4359+
Arc::new(Int32Array::from(vec![2])),
4360+
Arc::new(Int32Array::from(vec![1])), // same key
4361+
Arc::new(Int32Array::from(vec![20])),
4362+
],
4363+
)?;
4364+
4365+
// Key=1 matches two outer rows. Key=2 keeps the inner input non-exhausted.
4366+
let inner_batch = RecordBatch::try_new(
4367+
Arc::clone(&right_schema),
4368+
vec![
4369+
Arc::new(Int32Array::from(vec![100, 200])),
4370+
Arc::new(Int32Array::from(vec![1, 2])),
4371+
Arc::new(Int32Array::from(vec![50, 60])),
4372+
],
4373+
)?;
4374+
4375+
let on_outer: Vec<PhysicalExprRef> = vec![Arc::new(Column::new("b1", 1))];
4376+
let on_inner: Vec<PhysicalExprRef> = vec![Arc::new(Column::new("b1", 1))];
4377+
4378+
for (join_type, expected_a1) in [(LeftSemi, vec![1, 2]), (LeftAnti, vec![0])] {
4379+
let outer: SendableRecordBatchStream = Box::pin(PendingStream::new(
4380+
vec![outer_batch1.clone(), outer_batch2.clone()],
4381+
vec![false, true], // Pending before 2nd outer batch
4382+
));
4383+
let inner: SendableRecordBatchStream =
4384+
Box::pin(PendingStream::new(vec![inner_batch.clone()], vec![false]));
4385+
4386+
let metrics = ExecutionPlanMetricsSet::new();
4387+
let inner_schema = inner.schema();
4388+
let (reservation, spill_manager, runtime_env) =
4389+
test_stream_resources(inner_schema, &metrics);
4390+
let stream = BitwiseSortMergeJoinStream::try_new(
4391+
Arc::clone(&left_schema),
4392+
vec![SortOptions::default()],
4393+
NullEquality::NullEqualsNothing,
4394+
outer,
4395+
inner,
4396+
on_outer.clone(),
4397+
on_inner.clone(),
4398+
None, // no filter
4399+
join_type,
4400+
8192,
4401+
0,
4402+
&metrics,
4403+
reservation,
4404+
spill_manager,
4405+
runtime_env,
4406+
)?;
4407+
4408+
let batches = collect_stream(stream).await?;
4409+
let actual_a1 = batches
4410+
.iter()
4411+
.flat_map(|batch| {
4412+
let values = batch
4413+
.column(0)
4414+
.as_any()
4415+
.downcast_ref::<Int32Array>()
4416+
.unwrap();
4417+
(0..batch.num_rows()).map(|row| values.value(row))
4418+
})
4419+
.collect::<Vec<_>>();
4420+
assert_eq!(actual_a1, expected_a1, "{join_type:?}");
4421+
}
4422+
Ok(())
4423+
}
4424+
43264425
/// Tests the filtered boundary Pending re-entry: outer key group spans
43274426
/// batches with a filter, and poll_next_outer_batch returns Pending.
43284427
///

0 commit comments

Comments
 (0)