@@ -99,11 +99,18 @@ impl StaticFilter for ArrayStaticFilter {
9999 ) ) ;
100100 }
101101
102- downcast_dictionary_array ! {
103- v => {
104- let values_contains = self . contains( v. values( ) . as_ref( ) , negated) ?;
105- let result = take( & values_contains, v. keys( ) , None ) ?;
106- return Ok ( downcast_array( result. as_ref( ) ) )
102+ // Unwrap dictionary-encoded needles when the value type matches
103+ // in_array, evaluating against distinct values and mapping back
104+ // via keys.
105+ match v. data_type ( ) {
106+ DataType :: Dictionary ( _, value_type)
107+ if value_type. as_ref ( ) == self . in_array . data_type ( ) =>
108+ {
109+ let dict = v. as_any_dictionary_opt ( ) . unwrap ( ) ;
110+ let values_contains =
111+ self . contains ( dict. values ( ) . as_ref ( ) , negated) ?;
112+ let result = take ( & values_contains, dict. keys ( ) , None ) ?;
113+ return Ok ( downcast_array ( result. as_ref ( ) ) ) ;
107114 }
108115 _ => { }
109116 }
@@ -3878,4 +3885,180 @@ mod tests {
38783885 ) ;
38793886 Ok ( ( ) )
38803887 }
3888+
3889+ // -----------------------------------------------------------------------
3890+ // Tests for try_new_from_array covering all (in_array, needle) type
3891+ // combinations that occur in HashJoin dynamic filter pushdown.
3892+ //
3893+ // try_new (used by SQL IN expressions) always produces a non-Dictionary
3894+ // in_array because evaluate_list() flattens Dictionary scalars to their
3895+ // value type. try_new_from_array passes the array directly, so it is
3896+ // the only path that can produce a Dictionary in_array.
3897+ // -----------------------------------------------------------------------
3898+
3899+ fn wrap_in_dict ( array : ArrayRef ) -> ArrayRef {
3900+ let keys = Int32Array :: from ( ( 0 ..array. len ( ) as i32 ) . collect :: < Vec < _ > > ( ) ) ;
3901+ Arc :: new ( DictionaryArray :: new ( keys, array) )
3902+ }
3903+
3904+ fn eval_in_list_from_array (
3905+ needle_type : DataType ,
3906+ needle : ArrayRef ,
3907+ in_array : ArrayRef ,
3908+ ) -> Result < BooleanArray > {
3909+ let schema = Schema :: new ( vec ! [ Field :: new( "a" , needle_type, false ) ] ) ;
3910+ let col_a = col ( "a" , & schema) ?;
3911+ let expr = Arc :: new ( InListExpr :: try_new_from_array (
3912+ col_a, in_array, false ,
3913+ ) ?) as Arc < dyn PhysicalExpr > ;
3914+ let batch = RecordBatch :: try_new ( Arc :: new ( schema) , vec ! [ needle] ) ?;
3915+ let result = expr. evaluate ( & batch) ?. into_array ( batch. num_rows ( ) ) ?;
3916+ Ok ( as_boolean_array ( & result) . clone ( ) )
3917+ }
3918+
3919+ #[ test]
3920+ fn test_in_list_from_array_type_combinations ( ) -> Result < ( ) > {
3921+ use arrow:: compute:: cast;
3922+
3923+ // All cases: needle[0] and needle[2] match, needle[1] does not.
3924+ let expected =
3925+ BooleanArray :: from ( vec ! [ Some ( true ) , Some ( false ) , Some ( true ) ] ) ;
3926+
3927+ // Base arrays cast to each target type
3928+ let base_in = Arc :: new ( Int64Array :: from ( vec ! [ 1i64 , 2 , 3 ] ) ) as ArrayRef ;
3929+ let base_needle = Arc :: new ( Int64Array :: from ( vec ! [ 1i64 , 4 , 2 ] ) ) as ArrayRef ;
3930+
3931+ // Test all specializations in instantiate_static_filter
3932+ let primitive_types = vec ! [
3933+ DataType :: Int8 ,
3934+ DataType :: Int16 ,
3935+ DataType :: Int32 ,
3936+ DataType :: Int64 ,
3937+ DataType :: UInt8 ,
3938+ DataType :: UInt16 ,
3939+ DataType :: UInt32 ,
3940+ DataType :: UInt64 ,
3941+ DataType :: Float32 ,
3942+ DataType :: Float64 ,
3943+ ] ;
3944+
3945+ for dt in & primitive_types {
3946+ let in_array = cast ( & base_in, dt) ?;
3947+ let needle = cast ( & base_needle, dt) ?;
3948+
3949+ // T in_array, T needle
3950+ assert_eq ! ( expected, eval_in_list_from_array(
3951+ dt. clone( ) , Arc :: clone( & needle) , Arc :: clone( & in_array) ,
3952+ ) ?, "same-type failed for {dt:?}" ) ;
3953+
3954+ // T in_array, Dict(Int32, T) needle
3955+ let dict_dt = DataType :: Dictionary (
3956+ Box :: new ( DataType :: Int32 ) ,
3957+ Box :: new ( dt. clone ( ) ) ,
3958+ ) ;
3959+ assert_eq ! ( expected, eval_in_list_from_array(
3960+ dict_dt, wrap_in_dict( needle) , in_array,
3961+ ) ?, "dict-needle failed for {dt:?}" ) ;
3962+ }
3963+
3964+ // Utf8 (falls through to ArrayStaticFilter)
3965+ let utf8_in = Arc :: new ( StringArray :: from ( vec ! [ "a" , "b" , "c" ] ) ) as ArrayRef ;
3966+ let utf8_needle =
3967+ Arc :: new ( StringArray :: from ( vec ! [ "a" , "d" , "b" ] ) ) as ArrayRef ;
3968+ let dict_utf8 = DataType :: Dictionary (
3969+ Box :: new ( DataType :: Int32 ) ,
3970+ Box :: new ( DataType :: Utf8 ) ,
3971+ ) ;
3972+
3973+ // Utf8 in_array, Utf8 needle
3974+ assert_eq ! ( expected, eval_in_list_from_array(
3975+ DataType :: Utf8 ,
3976+ Arc :: clone( & utf8_needle) ,
3977+ Arc :: clone( & utf8_in) ,
3978+ ) ?) ;
3979+
3980+ // Utf8 in_array, Dict(Utf8) needle
3981+ assert_eq ! ( expected, eval_in_list_from_array(
3982+ dict_utf8. clone( ) ,
3983+ wrap_in_dict( Arc :: clone( & utf8_needle) ) ,
3984+ Arc :: clone( & utf8_in) ,
3985+ ) ?) ;
3986+
3987+ // Dict(Utf8) in_array, Dict(Utf8) needle: the #20937 bug
3988+ assert_eq ! ( expected, eval_in_list_from_array(
3989+ dict_utf8,
3990+ wrap_in_dict( Arc :: clone( & utf8_needle) ) ,
3991+ wrap_in_dict( Arc :: clone( & utf8_in) ) ,
3992+ ) ?) ;
3993+
3994+ // Struct in_array, Struct needle: multi-column join
3995+ let struct_fields = Fields :: from ( vec ! [
3996+ Field :: new( "c0" , DataType :: Utf8 , true ) ,
3997+ Field :: new( "c1" , DataType :: Int64 , true ) ,
3998+ ] ) ;
3999+ let struct_type = DataType :: Struct ( struct_fields. clone ( ) ) ;
4000+ let make_struct = |c0 : ArrayRef , c1 : ArrayRef | -> ArrayRef {
4001+ let pairs: Vec < ( FieldRef , ArrayRef ) > =
4002+ struct_fields. iter ( ) . cloned ( ) . zip ( [ c0, c1] ) . collect ( ) ;
4003+ Arc :: new ( StructArray :: from ( pairs) )
4004+ } ;
4005+ assert_eq ! ( expected, eval_in_list_from_array(
4006+ struct_type,
4007+ make_struct(
4008+ Arc :: clone( & utf8_needle) ,
4009+ Arc :: new( Int64Array :: from( vec![ 1 , 4 , 2 ] ) ) ,
4010+ ) ,
4011+ make_struct(
4012+ Arc :: clone( & utf8_in) ,
4013+ Arc :: new( Int64Array :: from( vec![ 1 , 2 , 3 ] ) ) ,
4014+ ) ,
4015+ ) ?) ;
4016+
4017+ // Struct with Dict fields: multi-column Dict join
4018+ let dict_struct_fields = Fields :: from ( vec ! [
4019+ Field :: new( "c0" , DataType :: Dictionary (
4020+ Box :: new( DataType :: Int32 ) ,
4021+ Box :: new( DataType :: Utf8 ) ,
4022+ ) , true ) ,
4023+ Field :: new( "c1" , DataType :: Int64 , true ) ,
4024+ ] ) ;
4025+ let dict_struct_type = DataType :: Struct ( dict_struct_fields. clone ( ) ) ;
4026+ let make_dict_struct = |c0 : ArrayRef , c1 : ArrayRef | -> ArrayRef {
4027+ let pairs: Vec < ( FieldRef , ArrayRef ) > =
4028+ dict_struct_fields. iter ( ) . cloned ( ) . zip ( [ c0, c1] ) . collect ( ) ;
4029+ Arc :: new ( StructArray :: from ( pairs) )
4030+ } ;
4031+ assert_eq ! ( expected, eval_in_list_from_array(
4032+ dict_struct_type,
4033+ make_dict_struct(
4034+ wrap_in_dict( Arc :: clone( & utf8_needle) ) ,
4035+ Arc :: new( Int64Array :: from( vec![ 1 , 4 , 2 ] ) ) ,
4036+ ) ,
4037+ make_dict_struct(
4038+ wrap_in_dict( Arc :: clone( & utf8_in) ) ,
4039+ Arc :: new( Int64Array :: from( vec![ 1 , 2 , 3 ] ) ) ,
4040+ ) ,
4041+ ) ?) ;
4042+
4043+ Ok ( ( ) )
4044+ }
4045+
4046+ #[ test]
4047+ fn test_in_list_from_array_type_mismatch_errors ( ) -> Result < ( ) > {
4048+ // Utf8 needle with Dict(Utf8) in_array: make_comparator does not
4049+ // support cross-type comparison.
4050+ let result = eval_in_list_from_array (
4051+ DataType :: Utf8 ,
4052+ Arc :: new ( StringArray :: from ( vec ! [ "a" , "d" , "b" ] ) ) ,
4053+ wrap_in_dict ( Arc :: new ( StringArray :: from ( vec ! [ "a" , "b" , "c" ] ) ) ) ,
4054+ ) ;
4055+ assert ! ( result. is_err( ) ) ;
4056+ assert ! (
4057+ result
4058+ . unwrap_err( )
4059+ . to_string( )
4060+ . contains( "Can't compare arrays of different types" )
4061+ ) ;
4062+ Ok ( ( ) )
4063+ }
38814064}
0 commit comments