Skip to content

Commit 5bab8f5

Browse files
committed
comments
1 parent fc208f5 commit 5bab8f5

1 file changed

Lines changed: 31 additions & 27 deletions

File tree

‎datafusion/physical-plan/src/joins/sort_merge_join/materializing_stream.rs‎

Lines changed: 31 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -601,30 +601,32 @@ impl MaterializingSortMergeJoinStream {
601601
/// contiguous rows sharing a key) at a time:
602602
///
603603
/// ```text
604-
/// load the first streamed row and the first buffered key group
605-
/// while either input still has rows:
606-
/// compare the join keys at both cursors
607-
/// Less → the streamed row can never match:
608-
/// null-join it (outer joins), advance streamed
609-
/// Greater → the buffered group can never match again:
610-
/// null-join it if nothing matched it (FULL join),
611-
/// advance to the next buffered group
612-
/// Equal → pair the streamed row with the whole group, advance
613-
/// streamed (the group stays — the next streamed row
614-
/// may share its key)
615-
/// materialize ("freeze") pairs once batch_size of them accumulate
616-
/// emit completed output batches
617-
/// flush everything that remains
604+
/// 1. load the first streamed row and the first buffered key group
605+
/// 2. while either input still has rows:
606+
/// 3. compare the join keys at both cursors
607+
/// 3a. Less → the streamed row can never match:
608+
/// null-join it (outer joins), advance streamed
609+
/// 3b. Greater → the buffered group can never match again:
610+
/// null-join it if nothing matched it (FULL join),
611+
/// advance to the next buffered group
612+
/// 3c. Equal → pair the streamed row with the whole group,
613+
/// advance streamed (the group stays — the next
614+
/// streamed row may share its key)
615+
/// (materialize ("freeze") pairs once batch_size accumulate)
616+
/// 4. emit completed output batches
617+
/// 5. flush everything that remains
618618
/// ```
619619
async fn join(
620620
mut self,
621621
emitter: &mut TryEmitter<RecordBatch, DataFusionError>,
622622
) -> Result<()> {
623623
// TODO - add join time metric
624624

625+
// 1. Load the first streamed row and the first buffered key group.
625626
self.load_next_streamed_batch().await?;
626627
self.advance_buffered_group().await?;
627628

629+
// 2. Merge-scan while either input still has rows.
628630
while !(self.streamed_exhausted && self.buffered_exhausted) {
629631
// Flush the deferred-filtering pipeline once a full batch of
630632
// rows accumulated (filtered outer joins output through it).
@@ -634,11 +636,12 @@ impl MaterializingSortMergeJoinStream {
634636
self.emit_deferred_output(emitter).await?;
635637
}
636638

637-
// An exhausted side compares as the larger one, so the other
638-
// side keeps draining through its own arm.
639+
// 3. Compare the join keys at both cursors. An exhausted side
640+
// compares as the larger one, so the other side keeps
641+
// draining through its own arm.
639642
match self.compare_streamed_buffered()? {
640-
// The streamed row can never match: null-join it (outer
641-
// joins emit it; inner joins drop it), then advance.
643+
// 3a. The streamed row can never match: null-join it (outer
644+
// joins emit it; inner joins drop it), then advance.
642645
Ordering::Less => {
643646
self.null_join_streamed_row();
644647
if self.num_unfrozen_pairs() >= self.batch_size {
@@ -648,19 +651,19 @@ impl MaterializingSortMergeJoinStream {
648651
self.load_next_streamed_batch().await?;
649652
}
650653
}
651-
// The buffered group can never match again: null-join it
652-
// if nothing matched it (FULL join), then advance to the
653-
// next key group.
654+
// 3b. The buffered group can never match again: null-join
655+
// it if nothing matched it (FULL join), then advance to
656+
// the next key group.
654657
Ordering::Greater => {
655658
self.null_join_buffered_group();
656659
if !self.try_advance_buffered_group()? {
657660
self.advance_buffered_group().await?;
658661
}
659662
}
660-
// Match: pair the streamed row with the whole group —
661-
// materializing ("freezing") mid-scan whenever a full
662-
// batch of pairs accumulates — then advance streamed.
663-
// The group stays for the next streamed row.
663+
// 3c. Match: pair the streamed row with the whole group —
664+
// materializing ("freezing") mid-scan whenever a full
665+
// batch of pairs accumulates — then advance streamed.
666+
// The group stays for the next streamed row.
664667
Ordering::Equal => {
665668
while !self.pair_streamed_row_with_group() {
666669
self.freeze_full_batch(emitter).await?;
@@ -671,8 +674,8 @@ impl MaterializingSortMergeJoinStream {
671674
}
672675
}
673676

674-
// Emit completed output batches (filtered joins emit through
675-
// the deferred-filtering pipeline above instead).
677+
// 4. Emit completed output batches (filtered joins emit
678+
// through the deferred-filtering pipeline above instead).
676679
if !self.deferred_filtering
677680
&& self
678681
.joined_record_batches
@@ -683,6 +686,7 @@ impl MaterializingSortMergeJoinStream {
683686
}
684687
}
685688

689+
// 5. Flush everything that remains.
686690
self.on_children_exhausted(emitter).await
687691
}
688692

0 commit comments

Comments
 (0)