@@ -631,10 +631,26 @@ fn hash_dictionary<K: ArrowDictionaryKeyType>(
631631 hashes_buffer : & mut [ u64 ] ,
632632 multi_col : bool ,
633633) -> Result < ( ) > {
634+ let dict_values = array. values ( ) ;
635+
636+ // Slicing a dictionary only slices its keys and retains the complete values
637+ // array. Hashing every retained value can therefore be much more expensive
638+ // than hashing the logical rows in a small slice. Expand only for this
639+ // sparse-backing case; normal dictionaries keep the hash-once-and-scatter
640+ // path below.
641+ if dict_values. len ( ) > array. len ( ) . saturating_mul ( 4 ) {
642+ let logical_values = take ( dict_values. as_ref ( ) , array. keys ( ) , None ) ?;
643+ return hash_single_array (
644+ logical_values. as_ref ( ) ,
645+ random_state,
646+ hashes_buffer,
647+ multi_col,
648+ ) ;
649+ }
650+
634651 // Hash each dictionary value once, and then use that computed
635652 // hash for each key value to avoid a potentially expensive
636653 // redundant hashing for large dictionary elements (e.g. strings)
637- let dict_values = array. values ( ) ;
638654 let mut dict_hashes = vec ! [ 0 ; dict_values. len( ) ] ;
639655 create_hashes ( [ dict_values] , random_state, & mut dict_hashes) ?;
640656 dispatch_dictionary_scatter ( array, & dict_hashes, hashes_buffer, multi_col) ;
@@ -2164,6 +2180,54 @@ mod tests {
21642180 assert_ne ! ( one_col_hashes, two_col_hashes) ;
21652181 }
21662182
2183+ #[ test]
2184+ #[ cfg( not( feature = "force_hash_collisions" ) ) ]
2185+ fn sparse_dictionary_hashes_match_expanded_values ( ) {
2186+ let values = ( 0 ..64 )
2187+ . map ( |i| ( i != 7 ) . then ( || format ! ( "value-{i}" ) ) )
2188+ . collect :: < Vec < _ > > ( ) ;
2189+ let values: ArrayRef =
2190+ Arc :: new ( StringArray :: from_iter ( values. iter ( ) . map ( Option :: as_deref) ) ) ;
2191+ let keys = Int32Array :: from ( vec ! [ Some ( 63 ) , None , Some ( 7 ) , Some ( 2 ) ] ) ;
2192+ let logical_values = take ( values. as_ref ( ) , & keys, None ) . unwrap ( ) ;
2193+ let dictionary: ArrayRef =
2194+ Arc :: new ( DictionaryArray :: < Int32Type > :: new ( keys, Arc :: clone ( & values) ) ) ;
2195+ let prefix: ArrayRef = Arc :: new ( Int32Array :: from ( vec ! [ 1 , 2 , 3 , 4 ] ) ) ;
2196+ let random_state = RandomState :: with_seed ( 0 ) ;
2197+
2198+ let mut dictionary_hashes = vec ! [ 0 ; dictionary. len( ) ] ;
2199+ create_hashes (
2200+ & [ Arc :: clone ( & dictionary) ] ,
2201+ & random_state,
2202+ & mut dictionary_hashes,
2203+ )
2204+ . unwrap ( ) ;
2205+ let mut expanded_hashes = vec ! [ 0 ; logical_values. len( ) ] ;
2206+ create_hashes (
2207+ & [ Arc :: clone ( & logical_values) ] ,
2208+ & random_state,
2209+ & mut expanded_hashes,
2210+ )
2211+ . unwrap ( ) ;
2212+ assert_eq ! ( dictionary_hashes, expanded_hashes) ;
2213+
2214+ dictionary_hashes. fill ( 0 ) ;
2215+ create_hashes (
2216+ & [ Arc :: clone ( & prefix) , dictionary] ,
2217+ & random_state,
2218+ & mut dictionary_hashes,
2219+ )
2220+ . unwrap ( ) ;
2221+ expanded_hashes. fill ( 0 ) ;
2222+ create_hashes (
2223+ & [ prefix, logical_values] ,
2224+ & random_state,
2225+ & mut expanded_hashes,
2226+ )
2227+ . unwrap ( ) ;
2228+ assert_eq ! ( dictionary_hashes, expanded_hashes) ;
2229+ }
2230+
21672231 #[ test]
21682232 fn test_create_hashes_from_arrays ( ) {
21692233 let int_array: ArrayRef = Arc :: new ( Int32Array :: from ( vec ! [ 1 , 2 , 3 , 4 ] ) ) ;
0 commit comments