Skip to content

Commit 3d1e1c9

Browse files
committed
working version
1 parent 09fcdef commit 3d1e1c9

1 file changed

Lines changed: 14 additions & 24 deletions

File tree

  • datafusion/physical-plan/src/aggregates/group_values/multi_group_by

‎datafusion/physical-plan/src/aggregates/group_values/multi_group_by/dictionary.rs‎

Lines changed: 14 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -33,14 +33,9 @@ pub struct DictionaryGroupValuesColumn<K: ArrowDictionaryKeyType + Send + Sync>
3333
null_array: ArrayRef,
3434
cached_values: Option<ArrayRef>,
3535
cached_combined: Option<ArrayRef>,
36-
/// Scratch space for the low-cardinality path in `vectorized_equal_to`:
37-
/// indexed by lhs group id, caches the equality result per group to avoid
38-
/// redundant `inner.equal_to` calls across the 8k+ rhs rows that map to
39-
/// the same few groups. `Mutex` is needed because the `GroupColumn` trait
40-
/// defines `vectorized_equal_to` as `&self`; interior mutability lets us
41-
/// reuse this allocation across calls without changing every implementor.
42-
/// The lock is always uncontended — aggregation is single-threaded per partition.
43-
equal_to_scratch: Mutex<Vec<Option<bool>>>,
36+
/// Per-call group equality cache for the low-cardinality path. `Mutex`
37+
/// because `vectorized_equal_to` takes `&self`; always uncontended.
38+
group_eq_cache: Mutex<Vec<Option<bool>>>,
4439
_phantom: PhantomData<K>,
4540
}
4641

@@ -52,7 +47,7 @@ impl<K: ArrowDictionaryKeyType + Send + Sync> DictionaryGroupValuesColumn<K> {
5247
null_array,
5348
cached_values: None,
5449
cached_combined: None,
55-
equal_to_scratch: Mutex::new(Vec::new()),
50+
group_eq_cache: Mutex::new(Vec::new()),
5651
_phantom: PhantomData,
5752
}
5853
}
@@ -93,6 +88,7 @@ impl<K: ArrowDictionaryKeyType + Send + Sync> DictionaryGroupValuesColumn<K> {
9388
.collect();
9489
Arc::new(DictionaryArray::<K>::new(keys, values))
9590
}
91+
9692
fn valid_bounds<T: ArrowDictionaryKeyType>(num_values: usize) -> bool {
9793
let max: usize = match T::DATA_TYPE {
9894
DataType::Int8 => i8::MAX as usize,
@@ -135,43 +131,37 @@ impl<K: ArrowDictionaryKeyType + Send + Sync> GroupColumn
135131
rhs_rows: &[usize],
136132
equal_to_results: &mut BooleanBufferBuilder,
137133
) {
138-
// lhs_rows contains group indices de-duplicated by the hash table, so
139-
// its unique-value count reflects the true data cardinality. Low
140-
// cardinality means the same few group slots are visited repeatedly,
141-
// which is where a per-group result cache pays off. High cardinality
142-
// means almost every lhs slot is unique, so the cache adds overhead
143-
// with no reuse benefit.
144-
let unique_lhs: hashbrown::HashSet<usize> = lhs_rows.iter().copied().collect();
145-
let low_cardinality = unique_lhs.len() <= rhs_rows.len() / 5;
134+
let unique_group_count: hashbrown::HashSet<usize> =
135+
lhs_rows.iter().copied().collect();
136+
let low_cardinality = unique_group_count.len() <= rhs_rows.len() / 5;
146137

147138
let dict = array.as_dictionary::<K>();
148139
let values = dict.values();
149140

150141
if low_cardinality {
151-
let mut scratch = self.equal_to_scratch.lock().unwrap();
152-
scratch.resize(lhs_rows.len(), None);
142+
let mut cache = self.group_eq_cache.lock().unwrap();
143+
cache.resize(lhs_rows.len(), None);
153144

154145
for (idx, (&lhs_row, &rhs_row)) in lhs_rows.iter().zip(rhs_rows).enumerate() {
155146
if !equal_to_results.get_bit(idx) {
156147
continue;
157148
}
158-
let is_equal = if let Some(cached) = scratch[lhs_row] {
149+
let is_equal = if let Some(cached) = cache[lhs_row] {
159150
cached
160151
} else {
161152
let result = match dict.key(rhs_row) {
162153
None => self.inner.equal_to(lhs_row, &self.null_array, 0),
163154
Some(key) => self.inner.equal_to(lhs_row, values, key),
164155
};
165-
scratch[lhs_row] = Some(result);
156+
cache[lhs_row] = Some(result);
166157
result
167158
};
168159
if !is_equal {
169160
equal_to_results.set_bit(idx, false);
170161
}
171162
}
172163

173-
// key space is only valid per intern() call, so clear the cache after each call to avoid false positives in future calls
174-
scratch.clear();
164+
cache.clear();
175165
} else {
176166
let keys = dict.keys();
177167

@@ -230,7 +220,7 @@ impl<K: ArrowDictionaryKeyType + Send + Sync> GroupColumn
230220

231221
fn size(&self) -> usize {
232222
self.inner.size()
233-
+ self.equal_to_scratch.lock().unwrap().capacity() * size_of::<Option<bool>>()
223+
+ self.group_eq_cache.lock().unwrap().capacity() * size_of::<Option<bool>>()
234224
}
235225

236226
fn build(self: Box<Self>) -> ArrayRef {

0 commit comments

Comments
 (0)