Skip to content

Commit 8caaa35

Browse files
committed
fix(aggregate): keep cached dictionary groups consistent
1 parent 8f0e9b8 commit 8caaa35

3 files changed

Lines changed: 194 additions & 68 deletions

File tree

‎datafusion/common/src/hash_utils.rs‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -696,6 +696,25 @@ pub fn scatter_dictionary_hashes<K: ArrowDictionaryKeyType>(
696696
Ok(())
697697
}
698698

699+
/// Hashes one column into `hashes_buffer` exactly as [`create_hashes`] does
700+
/// for that column; `rehash` is `true` for every column after the first.
701+
///
702+
/// # Public Only for Internal Use:
703+
///
704+
/// This is not a public API and is for internal use only; see [API policy] for details.
705+
///
706+
/// [API policy]: https://datafusion.apache.org/contributor-guide/api-health.html#internal-public-apis
707+
#[doc(hidden)]
708+
#[cfg(not(feature = "force_hash_collisions"))]
709+
pub fn hash_column(
710+
array: &dyn Array,
711+
random_state: &impl HashState,
712+
hashes_buffer: &mut [u64],
713+
rehash: bool,
714+
) -> Result<()> {
715+
hash_single_array(array, random_state, hashes_buffer, rehash)
716+
}
717+
699718
/// Hash the values in a dictionary array.
700719
#[cfg(not(feature = "force_hash_collisions"))]
701720
fn hash_dictionary<K: ArrowDictionaryKeyType>(

‎datafusion/physical-plan/src/aggregates/group_values/multi_group_by/dictionary.rs‎

Lines changed: 15 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,8 @@ pub struct DictionaryGroupValuesColumn<K: ArrowDictionaryKeyType + Send + Sync>
5757
/// Whether the outer table guarantees each append creates a new group, so
5858
/// inner slots and group indices are identical.
5959
values_are_group_unique: bool,
60+
#[cfg(test)]
61+
hash_builds: usize,
6062
_phantom: PhantomData<K>,
6163
}
6264

@@ -75,6 +77,8 @@ impl<K: ArrowDictionaryKeyType + Send + Sync> DictionaryGroupValuesColumn<K> {
7577
val_hashes: Vec::default(),
7678
cached_values: None,
7779
values_are_group_unique: false,
80+
#[cfg(test)]
81+
hash_builds: 0,
7882
_phantom: PhantomData,
7983
}
8084
}
@@ -161,6 +165,10 @@ impl<K: ArrowDictionaryKeyType + Send + Sync> DictionaryGroupValuesColumn<K> {
161165
}
162166

163167
fn hash_values(&mut self, values: &ArrayRef) {
168+
#[cfg(test)]
169+
{
170+
self.hash_builds += 1;
171+
}
164172
self.cached_values = None;
165173
self.val_hashes.clear();
166174
self.val_hashes.resize(values.len(), 0);
@@ -397,6 +405,11 @@ impl<K: ArrowDictionaryKeyType + Send + Sync> GroupColumn
397405
true
398406
}
399407

408+
#[cfg(test)]
409+
fn dictionary_hash_builds(&self) -> usize {
410+
self.hash_builds
411+
}
412+
400413
fn vectorized_equal_to(
401414
&mut self,
402415
lhs_rows: &[usize],
@@ -622,28 +635,17 @@ impl<K: ArrowDictionaryKeyType + Send + Sync> GroupColumn
622635
let emitted: ArrayRef =
623636
Arc::new(DictionaryArray::<K>::new(emitted_keys, compact_emit_values));
624637

625-
// Null deferred to last so null_inner_slot is always the highest index
626-
// and check_key_overflow can subtract it without a false overflow.
638+
// Preserve group order so inner slots remain equal to group indices
639+
// when this is the only grouping column.
627640
let remaining = self.group_to_inner[n..].to_vec();
628641
let mut old_to_new = vec![usize::MAX; old_inner_len];
629642
let mut new_to_old: Vec<usize> = Vec::new();
630-
let mut null_old_slot: Option<usize> = None;
631643
for &old in &remaining {
632-
if all_inner_values.is_null(old) {
633-
if null_old_slot.is_none() {
634-
null_old_slot = Some(old);
635-
}
636-
continue;
637-
}
638644
if old_to_new[old] == usize::MAX {
639645
old_to_new[old] = new_to_old.len();
640646
new_to_old.push(old);
641647
}
642648
}
643-
if let Some(old) = null_old_slot {
644-
old_to_new[old] = new_to_old.len();
645-
new_to_old.push(old);
646-
}
647649

648650
self.value_dedup = HashTable::new();
649651
self.value_dedup_size = 0;

0 commit comments

Comments
 (0)