Skip to content

Commit 4ede583

Browse files
committed
refactor: expose hash helpers for dictionary caching
1 parent acbad14 commit 4ede583

1 file changed

Lines changed: 24 additions & 8 deletions

File tree

‎datafusion/common/src/hash_utils.rs‎

Lines changed: 24 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -607,13 +607,25 @@ fn hash_dictionary_scatter<
607607
}
608608
}
609609

610+
/// Expands precomputed dictionary value hashes to row hashes.
611+
///
612+
/// Null keys and null dictionary values leave the corresponding row hash
613+
/// unchanged. If `multi_col` is true, the value hash is combined with the
614+
/// existing row hash; otherwise, it replaces it.
615+
///
616+
/// # Panics
617+
///
618+
/// Panics if there is not exactly one hash per dictionary value, or if the
619+
/// output length differs from the dictionary length.
610620
#[cfg(not(feature = "force_hash_collisions"))]
611-
fn dispatch_dictionary_scatter<K: ArrowDictionaryKeyType>(
621+
pub fn scatter_dictionary_hashes<K: ArrowDictionaryKeyType>(
612622
array: &DictionaryArray<K>,
613623
dict_hashes: &[u64],
614624
hashes_buffer: &mut [u64],
615625
multi_col: bool,
616626
) {
627+
assert_eq!(dict_hashes.len(), array.values().len());
628+
assert_eq!(hashes_buffer.len(), array.len());
617629
let has_null_keys = array.keys().null_count() != 0;
618630
let has_null_values = array.values().null_count() != 0;
619631

@@ -675,7 +687,7 @@ fn hash_dictionary<K: ArrowDictionaryKeyType>(
675687
let dict_values = array.values();
676688
let mut dict_hashes = vec![0; dict_values.len()];
677689
create_hashes([dict_values], random_state, &mut dict_hashes)?;
678-
dispatch_dictionary_scatter(array, &dict_hashes, hashes_buffer, multi_col);
690+
scatter_dictionary_hashes(array, &dict_hashes, hashes_buffer, multi_col);
679691
Ok(())
680692
}
681693

@@ -689,7 +701,7 @@ fn hash_dictionary_with_child_hashing<K: ArrowDictionaryKeyType>(
689701
let dict_values = array.values();
690702
let mut dict_hashes = vec![0; dict_values.len()];
691703
child_hashing.create_hashes([dict_values], &mut dict_hashes)?;
692-
dispatch_dictionary_scatter(array, &dict_hashes, hashes_buffer, multi_col);
704+
scatter_dictionary_hashes(array, &dict_hashes, hashes_buffer, multi_col);
693705
Ok(())
694706
}
695707

@@ -1141,10 +1153,13 @@ fn hash_run_array<R: RunEndIndexType>(
11411153
}
11421154
}
11431155

1144-
/// Internal helper function that hashes a single array and either initializes or combines
1145-
/// the hash values in the buffer.
1156+
/// Hashes one array, either initializing `hashes_buffer` or combining with it.
1157+
///
1158+
/// This is an internal API for physical operators that handle dictionary
1159+
/// columns separately with [`scatter_dictionary_hashes`].
1160+
#[doc(hidden)]
11461161
#[cfg(not(feature = "force_hash_collisions"))]
1147-
fn hash_single_array(
1162+
pub fn hash_single_array(
11481163
array: &dyn Array,
11491164
random_state: &impl HashState,
11501165
hashes_buffer: &mut [u64],
@@ -1244,9 +1259,10 @@ fn hash_single_array(
12441259
Ok(())
12451260
}
12461261

1247-
/// Test version of `hash_single_array` that forces all hashes to collide to zero.
1262+
/// Test version of [`hash_single_array`] that forces all hashes to collide to zero.
1263+
#[doc(hidden)]
12481264
#[cfg(feature = "force_hash_collisions")]
1249-
fn hash_single_array(
1265+
pub fn hash_single_array(
12501266
_array: &dyn Array,
12511267
_random_state: &impl HashState,
12521268
hashes_buffer: &mut [u64],

0 commit comments

Comments
 (0)