Skip to content

Commit c6d7224

Browse files
committed
refactor: keep the join key comparator on the probe batch state
The comparator is only valid for one probe batch. Storing it on ProcessProbeBatchState ties its lifetime to the batch, so the manual resets in fetch_probe_batch and on the limit path go away and the probe arrays are released as soon as the batch finishes.
1 parent 71de216 commit c6d7224

2 files changed

Lines changed: 13 additions & 10 deletions

File tree

‎datafusion/physical-plan/src/joins/hash_join/stream.rs‎

Lines changed: 7 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -130,7 +130,7 @@ impl BuildSide {
130130
/// [`need_produce_result_in_final`]), and only in the partition that finished
131131
/// probing last. That state re-enters itself once per emitted chunk of at most
132132
/// `batch_size` rows.
133-
#[derive(Debug, Clone)]
133+
#[derive(Debug)]
134134
pub(super) enum HashJoinStreamState {
135135
/// Initial state for HashJoinStream indicating that build-side data not collected yet
136136
WaitBuildSide,
@@ -175,7 +175,7 @@ impl HashJoinStreamState {
175175
}
176176

177177
/// Container for HashJoinStreamState::ProcessProbeBatch related data
178-
#[derive(Debug, Clone)]
178+
#[derive(Debug)]
179179
pub(super) struct ProcessProbeBatchState {
180180
/// Current probe-side batch
181181
batch: RecordBatch,
@@ -194,6 +194,9 @@ pub(super) struct ProcessProbeBatchState {
194194
/// `probe_hit_rate` and `avg_fanout` count a probe row whose matches span
195195
/// several chunks only once.
196196
matched_probe_idx: Option<u32>,
197+
/// Key comparator for this batch, built on first use and reused by every
198+
/// chunk
199+
key_comparator: Option<JoinKeyComparator>,
197200
}
198201

199202
impl ProcessProbeBatchState {
@@ -388,9 +391,6 @@ pub(super) struct HashJoinStream {
388391
probe_indices_buffer: Vec<u32>,
389392
/// Scratch space for build indices during hash lookup
390393
build_indices_buffer: Vec<u64>,
391-
/// Key comparator for the current probe batch, built on first use and
392-
/// reused by every chunk of that batch
393-
probe_key_comparator: Option<JoinKeyComparator>,
394394

395395
/// Scratch space for scope-key hashes in the null-aware mark pass, reused
396396
/// across probe batches. Separate from `hashes_buffer`, which still holds
@@ -585,7 +585,6 @@ impl HashJoinStream {
585585
hashes_buffer,
586586
probe_indices_buffer: Vec::with_capacity(batch_size),
587587
build_indices_buffer: Vec::with_capacity(batch_size),
588-
probe_key_comparator: None,
589588
// Left unallocated: only correlated null-aware joins ever use
590589
// these, and they grow them on first use.
591590
null_mark_hashes_buffer: Vec::new(),
@@ -775,8 +774,6 @@ impl HashJoinStream {
775774
&mut self,
776775
cx: &mut std::task::Context<'_>,
777776
) -> Poll<Result<StatefulStreamResult<Option<RecordBatch>>>> {
778-
// The comparator refers to the previous probe batch's key arrays.
779-
self.probe_key_comparator = None;
780777
match ready!(self.right.poll_next_unpin(cx)) {
781778
None => {
782779
// Release the probe-side input pipeline's resources. The schema
@@ -816,6 +813,7 @@ impl HashJoinStream {
816813
offset: (0, None),
817814
joined_probe_idx: None,
818815
matched_probe_idx: None,
816+
key_comparator: None,
819817
});
820818
}
821819
Some(Err(err)) => return Poll::Ready(Err(err)),
@@ -907,7 +905,7 @@ impl HashJoinStream {
907905
state.offset,
908906
&mut self.probe_indices_buffer,
909907
&mut self.build_indices_buffer,
910-
&mut self.probe_key_comparator,
908+
&mut state.key_comparator,
911909
)?,
912910
Map::ArrayMap(array_map) => {
913911
let next_offset = array_map.get_matched_indices_with_limit_offset(
@@ -1041,7 +1039,6 @@ impl HashJoinStream {
10411039
// If limit reached, finish and move to Completed state
10421040
if push_status == PushBatchStatus::LimitReached {
10431041
self.output_buffer.finish()?;
1044-
self.probe_key_comparator = None;
10451042
self.state = HashJoinStreamState::Completed;
10461043
return Ok(StatefulStreamResult::Continue);
10471044
}

‎datafusion/physical-plan/src/joins/utils.rs‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2478,6 +2478,12 @@ pub struct JoinKeyComparator {
24782478
rest: Vec<DynComparator>,
24792479
}
24802480

2481+
impl Debug for JoinKeyComparator {
2482+
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2483+
f.debug_struct("JoinKeyComparator").finish_non_exhaustive()
2484+
}
2485+
}
2486+
24812487
impl JoinKeyComparator {
24822488
/// Build comparators for each join key column pair.
24832489
pub fn new(

0 commit comments

Comments
 (0)