@@ -33,14 +33,9 @@ pub struct DictionaryGroupValuesColumn<K: ArrowDictionaryKeyType + Send + Sync>
3333 null_array : ArrayRef ,
3434 cached_values : Option < ArrayRef > ,
3535 cached_combined : Option < ArrayRef > ,
36- /// Scratch space for the low-cardinality path in `vectorized_equal_to`:
37- /// indexed by lhs group id, caches the equality result per group to avoid
38- /// redundant `inner.equal_to` calls across the 8k+ rhs rows that map to
39- /// the same few groups. `Mutex` is needed because the `GroupColumn` trait
40- /// defines `vectorized_equal_to` as `&self`; interior mutability lets us
41- /// reuse this allocation across calls without changing every implementor.
42- /// The lock is always uncontended — aggregation is single-threaded per partition.
43- equal_to_scratch : Mutex < Vec < Option < bool > > > ,
36+ /// Per-call group equality cache for the low-cardinality path. `Mutex`
37+ /// because `vectorized_equal_to` takes `&self`; always uncontended.
38+ group_eq_cache : Mutex < Vec < Option < bool > > > ,
4439 _phantom : PhantomData < K > ,
4540}
4641
@@ -52,7 +47,7 @@ impl<K: ArrowDictionaryKeyType + Send + Sync> DictionaryGroupValuesColumn<K> {
5247 null_array,
5348 cached_values : None ,
5449 cached_combined : None ,
55- equal_to_scratch : Mutex :: new ( Vec :: new ( ) ) ,
50+ group_eq_cache : Mutex :: new ( Vec :: new ( ) ) ,
5651 _phantom : PhantomData ,
5752 }
5853 }
@@ -93,6 +88,7 @@ impl<K: ArrowDictionaryKeyType + Send + Sync> DictionaryGroupValuesColumn<K> {
9388 . collect ( ) ;
9489 Arc :: new ( DictionaryArray :: < K > :: new ( keys, values) )
9590 }
91+
9692 fn valid_bounds < T : ArrowDictionaryKeyType > ( num_values : usize ) -> bool {
9793 let max: usize = match T :: DATA_TYPE {
9894 DataType :: Int8 => i8:: MAX as usize ,
@@ -135,43 +131,37 @@ impl<K: ArrowDictionaryKeyType + Send + Sync> GroupColumn
135131 rhs_rows : & [ usize ] ,
136132 equal_to_results : & mut BooleanBufferBuilder ,
137133 ) {
138- // lhs_rows contains group indices de-duplicated by the hash table, so
139- // its unique-value count reflects the true data cardinality. Low
140- // cardinality means the same few group slots are visited repeatedly,
141- // which is where a per-group result cache pays off. High cardinality
142- // means almost every lhs slot is unique, so the cache adds overhead
143- // with no reuse benefit.
144- let unique_lhs: hashbrown:: HashSet < usize > = lhs_rows. iter ( ) . copied ( ) . collect ( ) ;
145- let low_cardinality = unique_lhs. len ( ) <= rhs_rows. len ( ) / 5 ;
134+ let unique_group_count: hashbrown:: HashSet < usize > =
135+ lhs_rows. iter ( ) . copied ( ) . collect ( ) ;
136+ let low_cardinality = unique_group_count. len ( ) <= rhs_rows. len ( ) / 5 ;
146137
147138 let dict = array. as_dictionary :: < K > ( ) ;
148139 let values = dict. values ( ) ;
149140
150141 if low_cardinality {
151- let mut scratch = self . equal_to_scratch . lock ( ) . unwrap ( ) ;
152- scratch . resize ( lhs_rows. len ( ) , None ) ;
142+ let mut cache = self . group_eq_cache . lock ( ) . unwrap ( ) ;
143+ cache . resize ( lhs_rows. len ( ) , None ) ;
153144
154145 for ( idx, ( & lhs_row, & rhs_row) ) in lhs_rows. iter ( ) . zip ( rhs_rows) . enumerate ( ) {
155146 if !equal_to_results. get_bit ( idx) {
156147 continue ;
157148 }
158- let is_equal = if let Some ( cached) = scratch [ lhs_row] {
149+ let is_equal = if let Some ( cached) = cache [ lhs_row] {
159150 cached
160151 } else {
161152 let result = match dict. key ( rhs_row) {
162153 None => self . inner . equal_to ( lhs_row, & self . null_array , 0 ) ,
163154 Some ( key) => self . inner . equal_to ( lhs_row, values, key) ,
164155 } ;
165- scratch [ lhs_row] = Some ( result) ;
156+ cache [ lhs_row] = Some ( result) ;
166157 result
167158 } ;
168159 if !is_equal {
169160 equal_to_results. set_bit ( idx, false ) ;
170161 }
171162 }
172163
173- // key space is only valid per intern() call, so clear the cache after each call to avoid false positives in future calls
174- scratch. clear ( ) ;
164+ cache. clear ( ) ;
175165 } else {
176166 let keys = dict. keys ( ) ;
177167
@@ -230,7 +220,7 @@ impl<K: ArrowDictionaryKeyType + Send + Sync> GroupColumn
230220
231221 fn size ( & self ) -> usize {
232222 self . inner . size ( )
233- + self . equal_to_scratch . lock ( ) . unwrap ( ) . capacity ( ) * size_of :: < Option < bool > > ( )
223+ + self . group_eq_cache . lock ( ) . unwrap ( ) . capacity ( ) * size_of :: < Option < bool > > ( )
234224 }
235225
236226 fn build ( self : Box < Self > ) -> ArrayRef {
0 commit comments