@@ -39,7 +39,8 @@ type DictionaryHashEntry = (Weak<dyn Array>, Vec<u64>);
3939pub struct CachingArrayHasher {
4040 /// Stored with the cache so value hashes cannot cross hash seeds.
4141 random_state : RandomState ,
42- /// One entry per column. Weak identity avoids retaining dictionary values.
42+ /// One entry per column. Weak identity avoids retaining dictionary values,
43+ /// and keeps their allocation reserved so a new `Arc` can't reuse the address.
4344 dictionaries : Vec < Option < DictionaryHashEntry > > ,
4445}
4546
@@ -117,9 +118,9 @@ impl CachingArrayHasher {
117118 . take( )
118119 . map( |( _, hashes) | hashes)
119120 . unwrap_or_default( ) ;
120- hashes. resize( values. len( ) , 0 ) ;
121121 // Fresh hashing leaves null value slots unchanged.
122- hashes. fill( 0 ) ;
122+ hashes. clear( ) ;
123+ hashes. resize( values. len( ) , 0 ) ;
123124 hash_single_array(
124125 values. as_ref( ) ,
125126 & self . random_state,
@@ -148,6 +149,11 @@ impl CachingArrayHasher {
148149 Some ( ( values. upgrade ( ) ?, hashes) )
149150 }
150151
152+ /// Drops all cached value hashes. Later batches rehash and refill the cache.
153+ pub fn clear ( & mut self ) {
154+ self . dictionaries . fill ( None ) ;
155+ }
156+
151157 /// Returns the allocated size of the cache.
152158 pub fn size ( & self ) -> usize {
153159 self . dictionaries . capacity ( ) * size_of :: < Option < DictionaryHashEntry > > ( )
@@ -1525,11 +1531,9 @@ mod tests {
15251531 hasher : & CachingArrayHasher ,
15261532 column_index : usize ,
15271533 expected_values : & ArrayRef ,
1528- ) -> * const u64 {
1529- let ( cached_values, value_hashes) =
1530- hasher. dictionary_hashes ( column_index) . unwrap ( ) ;
1534+ ) {
1535+ let ( cached_values, _) = hasher. dictionary_hashes ( column_index) . unwrap ( ) ;
15311536 assert ! ( Arc :: ptr_eq( & cached_values, expected_values) ) ;
1532- value_hashes. as_ptr ( )
15331537 }
15341538
15351539 #[ cfg( not( feature = "force_hash_collisions" ) ) ]
@@ -1546,7 +1550,7 @@ mod tests {
15461550 assert_hashes_match ( & mut hasher, & first_batch, & random_state) ?;
15471551
15481552 assert ! ( hasher. dictionary_hashes( 0 ) . is_none( ) ) ;
1549- let value_hash_allocation = assert_cached_values ( & hasher, 1 , & dictionary_values) ;
1553+ assert_cached_values ( & hasher, 1 , & dictionary_values) ;
15501554
15511555 // A later batch has new plain values and dictionary keys, but shares
15521556 // the dictionary values array and therefore its cached hashes.
@@ -1555,10 +1559,7 @@ mod tests {
15551559 string_dictionary( vec![ 1 , 0 ] , & dictionary_values) ,
15561560 ] ;
15571561 assert_hashes_match ( & mut hasher, & second_batch, & random_state) ?;
1558- assert_eq ! (
1559- assert_cached_values( & hasher, 1 , & dictionary_values) ,
1560- value_hash_allocation
1561- ) ;
1562+ assert_cached_values ( & hasher, 1 , & dictionary_values) ;
15621563 Ok ( ( ) )
15631564 }
15641565
@@ -1575,8 +1576,8 @@ mod tests {
15751576 ] ;
15761577 let mut hasher = caching_hasher ( & first_batch, & random_state) ;
15771578 assert_hashes_match ( & mut hasher, & first_batch, & random_state) ?;
1578- let country_hash_allocation = assert_cached_values ( & hasher, 0 , & country_values) ;
1579- let status_hash_allocation = assert_cached_values ( & hasher, 1 , & status_values) ;
1579+ assert_cached_values ( & hasher, 0 , & country_values) ;
1580+ assert_cached_values ( & hasher, 1 , & status_values) ;
15801581
15811582 // Aggregation batches can rebuild one dictionary while another group
15821583 // column continues to share its values array.
@@ -1587,14 +1588,8 @@ mod tests {
15871588 string_dictionary( vec![ 0 , 1 ] , & status_values) ,
15881589 ] ;
15891590 assert_hashes_match ( & mut hasher, & second_batch, & random_state) ?;
1590- assert_eq ! (
1591- assert_cached_values( & hasher, 0 , & new_country_values) ,
1592- country_hash_allocation
1593- ) ;
1594- assert_eq ! (
1595- assert_cached_values( & hasher, 1 , & status_values) ,
1596- status_hash_allocation
1597- ) ;
1591+ assert_cached_values ( & hasher, 0 , & new_country_values) ;
1592+ assert_cached_values ( & hasher, 1 , & status_values) ;
15981593 assert_eq ! ( hasher. dictionary_hashes( 0 ) . unwrap( ) . 1 [ 1 ] , 0 ) ;
15991594
16001595 // Reverse the transition: the first dictionary stays shared while the
@@ -1606,14 +1601,47 @@ mod tests {
16061601 string_dictionary( vec![ 1 , 0 ] , & new_status_values) ,
16071602 ] ;
16081603 assert_hashes_match ( & mut hasher, & third_batch, & random_state) ?;
1609- assert_eq ! (
1610- assert_cached_values( & hasher, 0 , & new_country_values) ,
1611- country_hash_allocation
1612- ) ;
1613- assert_eq ! (
1614- assert_cached_values( & hasher, 1 , & new_status_values) ,
1615- status_hash_allocation
1616- ) ;
1604+ assert_cached_values ( & hasher, 0 , & new_country_values) ;
1605+ assert_cached_values ( & hasher, 1 , & new_status_values) ;
1606+ Ok ( ( ) )
1607+ }
1608+
1609+ #[ cfg( not( feature = "force_hash_collisions" ) ) ]
1610+ #[ test]
1611+ fn caching_array_hasher_skips_rehash_only_for_same_values_arc ( ) -> Result < ( ) > {
1612+ let random_state = RandomState :: with_seed ( 0 ) ;
1613+ let values: ArrayRef = Arc :: new ( StringArray :: from ( vec ! [ "a" , "b" ] ) ) ;
1614+ let batch = vec ! [ string_dictionary( vec![ 0 , 1 ] , & values) ] ;
1615+ let mut hasher = caching_hasher ( & batch, & random_state) ;
1616+ let mut hashes = vec ! [ 0 ; 2 ] ;
1617+ hasher. create_hashes ( & batch, & mut hashes) ?;
1618+ let fresh = hashes. clone ( ) ;
1619+
1620+ // Poison the cached hashes: only a cache hit can return the sentinel.
1621+ const SENTINEL : u64 = 0xDEAD_BEEF ;
1622+ hasher. dictionaries [ 0 ] . as_mut ( ) . unwrap ( ) . 1 . fill ( SENTINEL ) ;
1623+
1624+ let same_arc = vec ! [ string_dictionary( vec![ 1 , 0 ] , & values) ] ;
1625+ hasher. create_hashes ( & same_arc, & mut hashes) ?;
1626+ assert_eq ! ( hashes, vec![ SENTINEL ; 2 ] ) ;
1627+
1628+ // Equal contents behind a different `Arc` is a miss.
1629+ let equal_values: ArrayRef = Arc :: new ( StringArray :: from ( vec ! [ "a" , "b" ] ) ) ;
1630+ let new_arc = vec ! [ string_dictionary( vec![ 0 , 1 ] , & equal_values) ] ;
1631+ hasher. create_hashes ( & new_arc, & mut hashes) ?;
1632+ assert_eq ! ( hashes, fresh) ;
1633+
1634+ // A dropped values array must not be matched by a later allocation.
1635+ hasher. dictionaries [ 0 ] . as_mut ( ) . unwrap ( ) . 1 . fill ( SENTINEL ) ;
1636+ drop ( new_arc) ;
1637+ drop ( equal_values) ;
1638+ for _ in 0 ..1000 {
1639+ let reused: ArrayRef = Arc :: new ( StringArray :: from ( vec ! [ "a" , "b" ] ) ) ;
1640+ let batch = vec ! [ string_dictionary( vec![ 0 , 1 ] , & reused) ] ;
1641+ hasher. create_hashes ( & batch, & mut hashes) ?;
1642+ assert_eq ! ( hashes, fresh) ;
1643+ hasher. dictionaries [ 0 ] . as_mut ( ) . unwrap ( ) . 1 . fill ( SENTINEL ) ;
1644+ }
16171645 Ok ( ( ) )
16181646 }
16191647
0 commit comments