Skip to content

Commit 8d54633

Browse files
buraksennimtherealnaska
authored andcommitted
fix: drain PiecewiseMergeJoin output before state transitions (apache#24298)
## Which issue does this PR close? - Closes apache#24297 ## Rationale for this change `ClassicPWMJStream` transitions phases while its output `BatchCoalescer` still holds completed batches. The Left/Full unmatched pass falls back into its producer when the drained queue is an exact multiple of `batch_size` — emitting duplicate rows forever — and queued unmatched Right/Full batches are dropped at terminal transitions. A zero-row placeholder batch also escapes as stream output on empty results. ## What changes are included in this PR? - Add `BatchProcessState::next_drained_batch()` (`finish_buffered_batch` + `next_completed_batch`); `None` guarantees the coalescer is empty - `process_stream_batch` and `process_unmatched_buffered_batch` drain one batch per poll and transition only after a confirmed-empty pop, so a finished producer can never re-run and no queued batch is lost - Scan completion returns `StatefulStreamResult::Continue` instead of the empty placeholder batch; empty joins now yield no batches ## Are these changes tested? Yes. The existing unmatched Left/Right tests now run with `batch_size = 1` (the Left one bounded with `take(6)` so the old duplicate loop fails as a snapshot mismatch instead of hanging) — both fail on main and pass here. The empty-join tests assert no batches are emitted. ## Are there any user-facing changes? No (PWMJ is experimental and off by default).
1 parent 095276a commit 8d54633

1 file changed

Lines changed: 49 additions & 74 deletions

File tree

‎datafusion/physical-plan/src/joins/piecewise_merge_join/classic_join.rs‎

Lines changed: 49 additions & 74 deletions
Original file line numberDiff line numberDiff line change
@@ -277,6 +277,18 @@ impl ClassicPWMJStream {
277277
return Ok(StatefulStreamResult::Ready(Some(batch)));
278278
}
279279

280+
// A finished scan can leave several completed batches queued; emit
281+
// them one per poll and transition only once the queue is empty, so
282+
// no output is lost.
283+
if !self.batch_process_state.continue_process {
284+
if let Some(batch) = self.batch_process_state.next_drained_batch()? {
285+
return Ok(StatefulStreamResult::Ready(Some(batch)));
286+
}
287+
288+
self.state = PiecewiseMergeJoinStreamState::FetchStreamBatch;
289+
return Ok(StatefulStreamResult::Continue);
290+
}
291+
280292
// Produce more work
281293
let batch = resolve_classic_join(
282294
buffered_side,
@@ -289,25 +301,8 @@ impl ClassicPWMJStream {
289301
)?;
290302

291303
if !self.batch_process_state.continue_process {
292-
// We finished scanning this stream batch.
293-
self.batch_process_state
294-
.output_batches
295-
.finish_buffered_batch()?;
296-
if let Some(b) = self
297-
.batch_process_state
298-
.output_batches
299-
.next_completed_batch()
300-
{
301-
self.state = PiecewiseMergeJoinStreamState::FetchStreamBatch;
302-
return Ok(StatefulStreamResult::Ready(Some(b)));
303-
}
304-
305-
// Nothing pending; hand back whatever `resolve` returned (often empty) and move on.
306-
if self.batch_process_state.output_batches.is_empty() {
307-
self.state = PiecewiseMergeJoinStreamState::FetchStreamBatch;
308-
309-
return Ok(StatefulStreamResult::Ready(Some(batch)));
310-
}
304+
// Scan finished; re-enter through the drain guard above.
305+
return Ok(StatefulStreamResult::Continue);
311306
}
312307

313308
Ok(StatefulStreamResult::Ready(Some(batch)))
@@ -324,25 +319,13 @@ impl ClassicPWMJStream {
324319
}
325320

326321
if !self.batch_process_state.continue_process {
327-
if let Some(batch) = self
328-
.batch_process_state
329-
.output_batches
330-
.next_completed_batch()
331-
{
322+
if let Some(batch) = self.batch_process_state.next_drained_batch()? {
332323
return Ok(StatefulStreamResult::Ready(Some(batch)));
333324
}
334325

335-
self.batch_process_state
336-
.output_batches
337-
.finish_buffered_batch()?;
338-
if let Some(batch) = self
339-
.batch_process_state
340-
.output_batches
341-
.next_completed_batch()
342-
{
343-
self.state = PiecewiseMergeJoinStreamState::Completed;
344-
return Ok(StatefulStreamResult::Ready(Some(batch)));
345-
}
326+
// Fully drained; finish instead of re-running the pass.
327+
self.state = PiecewiseMergeJoinStreamState::Completed;
328+
return Ok(StatefulStreamResult::Continue);
346329
}
347330

348331
let buffered_data =
@@ -372,29 +355,8 @@ impl ClassicPWMJStream {
372355
self.batch_process_state.output_batches.push_batch(batch)?;
373356

374357
self.batch_process_state.continue_process = false;
375-
if let Some(batch) = self
376-
.batch_process_state
377-
.output_batches
378-
.next_completed_batch()
379-
{
380-
return Ok(StatefulStreamResult::Ready(Some(batch)));
381-
}
382-
383-
self.batch_process_state
384-
.output_batches
385-
.finish_buffered_batch()?;
386-
if let Some(batch) = self
387-
.batch_process_state
388-
.output_batches
389-
.next_completed_batch()
390-
{
391-
self.state = PiecewiseMergeJoinStreamState::Completed;
392-
return Ok(StatefulStreamResult::Ready(Some(batch)));
393-
}
394-
395-
self.state = PiecewiseMergeJoinStreamState::Completed;
396-
self.batch_process_state.reset();
397-
Ok(StatefulStreamResult::Ready(None))
358+
// Re-enter through the drain guard above.
359+
Ok(StatefulStreamResult::Continue)
398360
}
399361
}
400362

@@ -438,6 +400,13 @@ impl BatchProcessState {
438400
self.continue_process = true;
439401
self.processed_null_count = false;
440402
}
403+
404+
// `None` guarantees the coalescer holds no pending rows, so the caller
405+
// may safely transition state without losing output.
406+
fn next_drained_batch(&mut self) -> Result<Option<RecordBatch>> {
407+
self.output_batches.finish_buffered_batch()?;
408+
Ok(self.output_batches.next_completed_batch())
409+
}
441410
}
442411

443412
impl Stream for ClassicPWMJStream {
@@ -674,7 +643,9 @@ mod tests {
674643
use arrow_schema::{DataType, Field};
675644
use datafusion_common::test_util::batches_to_string;
676645
use datafusion_execution::TaskContext;
646+
use datafusion_execution::config::SessionConfig;
677647
use datafusion_physical_expr::{PhysicalExpr, expressions::Column};
648+
use futures::TryStreamExt;
678649
use insta::assert_snapshot;
679650
use std::sync::Arc;
680651

@@ -964,12 +935,8 @@ mod tests {
964935
);
965936
let (_, batches) =
966937
join_collect(left, right, on, Operator::LtEq, JoinType::Inner).await?;
967-
assert_snapshot!(batches_to_string(&batches), @r"
968-
+----+----+----+----+----+----+
969-
| a1 | b1 | c1 | a2 | b1 | c2 |
970-
+----+----+----+----+----+----+
971-
+----+----+----+----+----+----+
972-
");
938+
// An empty join result produces no batches at all, not an empty batch.
939+
assert!(batches.is_empty());
973940
Ok(())
974941
}
975942

@@ -1298,8 +1265,16 @@ mod tests {
12981265
Arc::new(Column::new_with_schema("b1", &right.schema())?) as _,
12991266
);
13001267

1301-
let (_, batches) =
1302-
join_collect(left, right, on, Operator::LtEq, JoinType::Left).await?;
1268+
let task_ctx = Arc::new(
1269+
TaskContext::default()
1270+
.with_session_config(SessionConfig::new().with_batch_size(1)),
1271+
);
1272+
// Bound collection so the old duplicate loop becomes a snapshot mismatch.
1273+
let batches = join(left, right, on, Operator::LtEq, JoinType::Left)?
1274+
.execute(0, task_ctx)?
1275+
.take(6)
1276+
.try_collect::<Vec<_>>()
1277+
.await?;
13031278

13041279
assert_snapshot!(batches_to_string(&batches), @r"
13051280
+----+----+----+----+----+----+
@@ -1344,8 +1319,12 @@ mod tests {
13441319
Arc::new(Column::new_with_schema("b1", &right.schema())?) as _,
13451320
);
13461321

1347-
let (_, batches) =
1348-
join_collect(left, right, on, Operator::GtEq, JoinType::Right).await?;
1322+
let task_ctx = Arc::new(
1323+
TaskContext::default()
1324+
.with_session_config(SessionConfig::new().with_batch_size(1)),
1325+
);
1326+
let join = join(left, right, on, Operator::GtEq, JoinType::Right)?;
1327+
let batches = common::collect(join.execute(0, task_ctx)?).await?;
13491328

13501329
assert_snapshot!(batches_to_string(&batches), @r"
13511330
+----+----+----+----+----+----+
@@ -1408,12 +1387,8 @@ mod tests {
14081387
let (_, batches) =
14091388
join_collect(left, right, on, Operator::Gt, JoinType::Inner).await?;
14101389

1411-
assert_snapshot!(batches_to_string(&batches), @r"
1412-
+----+----+----+----+----+----+
1413-
| a1 | b1 | c1 | a2 | b1 | c2 |
1414-
+----+----+----+----+----+----+
1415-
+----+----+----+----+----+----+
1416-
");
1390+
// An empty join result produces no batches at all, not an empty batch.
1391+
assert!(batches.is_empty());
14171392
Ok(())
14181393
}
14191394

0 commit comments

Comments
 (0)