Skip to content

Commit 62a78ee

Browse files
committed
fix: reuse evaluated hash join build keys
Avoid evaluating build keys again just to check for nulls. CLOSES apache#26806
1 parent c8265b6 commit 62a78ee

1 file changed

Lines changed: 12 additions & 19 deletions

File tree

  • datafusion/physical-plan/src/joins/hash_join

‎datafusion/physical-plan/src/joins/hash_join/exec.rs‎

Lines changed: 12 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -160,30 +160,25 @@ fn subtree_max_len(data: &ArrayData) -> usize {
160160
///
161161
/// On `Some`, the memory of the [`ArrayMap`] has already been added to
162162
/// `reservation`; the caller creates the map with [`ArrayMap::try_new`].
163-
#[expect(clippy::too_many_arguments)]
164163
fn array_map_key_range(
165164
bounds: Option<&PartitionBounds>,
166-
schema: &SchemaRef,
167-
batches: &[RecordBatch],
168-
on_left: &[PhysicalExprRef],
165+
keys: &[ArrayRef],
169166
reservation: &mut MemoryReservation,
170167
perfect_hash_join_small_build_threshold: usize,
171168
perfect_hash_join_min_key_density: f64,
172169
null_equality: NullEquality,
173170
) -> Result<Option<(u64, u64)>> {
171+
let [key] = keys else {
172+
return Ok(None);
173+
};
174174
// `bounds` are also collected for dynamic filters, on any key type, so the
175175
// key type cannot be inferred from their presence.
176-
if !is_perfect_hash_join_candidate(on_left, schema)? {
176+
if !ArrayMap::is_supported_type(key.data_type()) {
177177
return Ok(None);
178178
}
179179

180-
if null_equality == NullEquality::NullEqualsNull {
181-
for batch in batches.iter() {
182-
let arrays = evaluate_expressions_to_arrays(on_left, batch)?;
183-
if arrays[0].null_count() > 0 {
184-
return Ok(None);
185-
}
186-
}
180+
if null_equality == NullEquality::NullEqualsNull && key.null_count() > 0 {
181+
return Ok(None);
187182
}
188183

189184
let (min_val, max_val) = if let Some(bounds) = bounds {
@@ -213,10 +208,10 @@ fn array_map_key_range(
213208
};
214209

215210
let range = ArrayMap::calculate_range(min_val, max_val);
216-
let num_row: usize = batches.iter().map(|x| x.num_rows()).sum();
211+
let num_rows = key.len();
217212

218213
// TODO: support create ArrayMap<u64>
219-
if num_row >= u32::MAX as usize {
214+
if num_rows >= u32::MAX as usize {
220215
return Ok(None);
221216
}
222217

@@ -226,15 +221,15 @@ fn array_map_key_range(
226221
return Ok(None);
227222
}
228223

229-
let dense_ratio = (num_row as f64) / ((range + 1) as f64);
224+
let dense_ratio = (num_rows as f64) / ((range + 1) as f64);
230225

231226
if range >= perfect_hash_join_small_build_threshold as u64
232227
&& dense_ratio <= perfect_hash_join_min_key_density
233228
{
234229
return Ok(None);
235230
}
236231

237-
let mem_size = ArrayMap::estimate_memory_size(min_val, max_val, num_row);
232+
let mem_size = ArrayMap::estimate_memory_size(min_val, max_val, num_rows);
238233
reservation.try_grow(mem_size)?;
239234

240235
Ok(Some((min_val, max_val)))
@@ -3269,9 +3264,7 @@ fn build_in_memory(
32693264

32703265
let join_hash_map = if let Some((min_val, max_val)) = array_map_key_range(
32713266
bounds.as_ref(),
3272-
schema,
3273-
std::slice::from_ref(&batch),
3274-
on_left,
3267+
&left_values,
32753268
&mut map_reservation,
32763269
config.execution.perfect_hash_join_small_build_threshold,
32773270
config.execution.perfect_hash_join_min_key_density,

0 commit comments

Comments
 (0)