Skip to content

Commit d58cc62

Browse files
committed
fix: InList Dictionary filter pushdown type mismatch
Guard the dictionary unwrap in ArrayStaticFilter::contains() to only fire when the dictionary value type matches in_array's type. This prevents a make_comparator type mismatch when both the needle and in_array are Dictionary-encoded, which occurs in HashJoin dynamic filter pushdown with pushdown_filters enabled. Closes apache#20937
1 parent 3ece9ec commit d58cc62

2 files changed

Lines changed: 194 additions & 6 deletions

File tree

‎datafusion/physical-expr/src/expressions/in_list.rs‎

Lines changed: 188 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -99,11 +99,18 @@ impl StaticFilter for ArrayStaticFilter {
9999
));
100100
}
101101

102-
downcast_dictionary_array! {
103-
v => {
104-
let values_contains = self.contains(v.values().as_ref(), negated)?;
105-
let result = take(&values_contains, v.keys(), None)?;
106-
return Ok(downcast_array(result.as_ref()))
102+
// Unwrap dictionary-encoded needles when the value type matches
103+
// in_array, evaluating against distinct values and mapping back
104+
// via keys.
105+
match v.data_type() {
106+
DataType::Dictionary(_, value_type)
107+
if value_type.as_ref() == self.in_array.data_type() =>
108+
{
109+
let dict = v.as_any_dictionary_opt().unwrap();
110+
let values_contains =
111+
self.contains(dict.values().as_ref(), negated)?;
112+
let result = take(&values_contains, dict.keys(), None)?;
113+
return Ok(downcast_array(result.as_ref()));
107114
}
108115
_ => {}
109116
}
@@ -3878,4 +3885,180 @@ mod tests {
38783885
);
38793886
Ok(())
38803887
}
3888+
3889+
// -----------------------------------------------------------------------
3890+
// Tests for try_new_from_array covering all (in_array, needle) type
3891+
// combinations that occur in HashJoin dynamic filter pushdown.
3892+
//
3893+
// try_new (used by SQL IN expressions) always produces a non-Dictionary
3894+
// in_array because evaluate_list() flattens Dictionary scalars to their
3895+
// value type. try_new_from_array passes the array directly, so it is
3896+
// the only path that can produce a Dictionary in_array.
3897+
// -----------------------------------------------------------------------
3898+
3899+
fn wrap_in_dict(array: ArrayRef) -> ArrayRef {
3900+
let keys = Int32Array::from((0..array.len() as i32).collect::<Vec<_>>());
3901+
Arc::new(DictionaryArray::new(keys, array))
3902+
}
3903+
3904+
fn eval_in_list_from_array(
3905+
needle_type: DataType,
3906+
needle: ArrayRef,
3907+
in_array: ArrayRef,
3908+
) -> Result<BooleanArray> {
3909+
let schema = Schema::new(vec![Field::new("a", needle_type, false)]);
3910+
let col_a = col("a", &schema)?;
3911+
let expr = Arc::new(InListExpr::try_new_from_array(
3912+
col_a, in_array, false,
3913+
)?) as Arc<dyn PhysicalExpr>;
3914+
let batch = RecordBatch::try_new(Arc::new(schema), vec![needle])?;
3915+
let result = expr.evaluate(&batch)?.into_array(batch.num_rows())?;
3916+
Ok(as_boolean_array(&result).clone())
3917+
}
3918+
3919+
#[test]
3920+
fn test_in_list_from_array_type_combinations() -> Result<()> {
3921+
use arrow::compute::cast;
3922+
3923+
// All cases: needle[0] and needle[2] match, needle[1] does not.
3924+
let expected =
3925+
BooleanArray::from(vec![Some(true), Some(false), Some(true)]);
3926+
3927+
// Base arrays cast to each target type
3928+
let base_in = Arc::new(Int64Array::from(vec![1i64, 2, 3])) as ArrayRef;
3929+
let base_needle = Arc::new(Int64Array::from(vec![1i64, 4, 2])) as ArrayRef;
3930+
3931+
// Test all specializations in instantiate_static_filter
3932+
let primitive_types = vec![
3933+
DataType::Int8,
3934+
DataType::Int16,
3935+
DataType::Int32,
3936+
DataType::Int64,
3937+
DataType::UInt8,
3938+
DataType::UInt16,
3939+
DataType::UInt32,
3940+
DataType::UInt64,
3941+
DataType::Float32,
3942+
DataType::Float64,
3943+
];
3944+
3945+
for dt in &primitive_types {
3946+
let in_array = cast(&base_in, dt)?;
3947+
let needle = cast(&base_needle, dt)?;
3948+
3949+
// T in_array, T needle
3950+
assert_eq!(expected, eval_in_list_from_array(
3951+
dt.clone(), Arc::clone(&needle), Arc::clone(&in_array),
3952+
)?, "same-type failed for {dt:?}");
3953+
3954+
// T in_array, Dict(Int32, T) needle
3955+
let dict_dt = DataType::Dictionary(
3956+
Box::new(DataType::Int32),
3957+
Box::new(dt.clone()),
3958+
);
3959+
assert_eq!(expected, eval_in_list_from_array(
3960+
dict_dt, wrap_in_dict(needle), in_array,
3961+
)?, "dict-needle failed for {dt:?}");
3962+
}
3963+
3964+
// Utf8 (falls through to ArrayStaticFilter)
3965+
let utf8_in = Arc::new(StringArray::from(vec!["a", "b", "c"])) as ArrayRef;
3966+
let utf8_needle =
3967+
Arc::new(StringArray::from(vec!["a", "d", "b"])) as ArrayRef;
3968+
let dict_utf8 = DataType::Dictionary(
3969+
Box::new(DataType::Int32),
3970+
Box::new(DataType::Utf8),
3971+
);
3972+
3973+
// Utf8 in_array, Utf8 needle
3974+
assert_eq!(expected, eval_in_list_from_array(
3975+
DataType::Utf8,
3976+
Arc::clone(&utf8_needle),
3977+
Arc::clone(&utf8_in),
3978+
)?);
3979+
3980+
// Utf8 in_array, Dict(Utf8) needle
3981+
assert_eq!(expected, eval_in_list_from_array(
3982+
dict_utf8.clone(),
3983+
wrap_in_dict(Arc::clone(&utf8_needle)),
3984+
Arc::clone(&utf8_in),
3985+
)?);
3986+
3987+
// Dict(Utf8) in_array, Dict(Utf8) needle: the #20937 bug
3988+
assert_eq!(expected, eval_in_list_from_array(
3989+
dict_utf8,
3990+
wrap_in_dict(Arc::clone(&utf8_needle)),
3991+
wrap_in_dict(Arc::clone(&utf8_in)),
3992+
)?);
3993+
3994+
// Struct in_array, Struct needle: multi-column join
3995+
let struct_fields = Fields::from(vec![
3996+
Field::new("c0", DataType::Utf8, true),
3997+
Field::new("c1", DataType::Int64, true),
3998+
]);
3999+
let struct_type = DataType::Struct(struct_fields.clone());
4000+
let make_struct = |c0: ArrayRef, c1: ArrayRef| -> ArrayRef {
4001+
let pairs: Vec<(FieldRef, ArrayRef)> =
4002+
struct_fields.iter().cloned().zip([c0, c1]).collect();
4003+
Arc::new(StructArray::from(pairs))
4004+
};
4005+
assert_eq!(expected, eval_in_list_from_array(
4006+
struct_type,
4007+
make_struct(
4008+
Arc::clone(&utf8_needle),
4009+
Arc::new(Int64Array::from(vec![1, 4, 2])),
4010+
),
4011+
make_struct(
4012+
Arc::clone(&utf8_in),
4013+
Arc::new(Int64Array::from(vec![1, 2, 3])),
4014+
),
4015+
)?);
4016+
4017+
// Struct with Dict fields: multi-column Dict join
4018+
let dict_struct_fields = Fields::from(vec![
4019+
Field::new("c0", DataType::Dictionary(
4020+
Box::new(DataType::Int32),
4021+
Box::new(DataType::Utf8),
4022+
), true),
4023+
Field::new("c1", DataType::Int64, true),
4024+
]);
4025+
let dict_struct_type = DataType::Struct(dict_struct_fields.clone());
4026+
let make_dict_struct = |c0: ArrayRef, c1: ArrayRef| -> ArrayRef {
4027+
let pairs: Vec<(FieldRef, ArrayRef)> =
4028+
dict_struct_fields.iter().cloned().zip([c0, c1]).collect();
4029+
Arc::new(StructArray::from(pairs))
4030+
};
4031+
assert_eq!(expected, eval_in_list_from_array(
4032+
dict_struct_type,
4033+
make_dict_struct(
4034+
wrap_in_dict(Arc::clone(&utf8_needle)),
4035+
Arc::new(Int64Array::from(vec![1, 4, 2])),
4036+
),
4037+
make_dict_struct(
4038+
wrap_in_dict(Arc::clone(&utf8_in)),
4039+
Arc::new(Int64Array::from(vec![1, 2, 3])),
4040+
),
4041+
)?);
4042+
4043+
Ok(())
4044+
}
4045+
4046+
#[test]
4047+
fn test_in_list_from_array_type_mismatch_errors() -> Result<()> {
4048+
// Utf8 needle with Dict(Utf8) in_array: make_comparator does not
4049+
// support cross-type comparison.
4050+
let result = eval_in_list_from_array(
4051+
DataType::Utf8,
4052+
Arc::new(StringArray::from(vec!["a", "d", "b"])),
4053+
wrap_in_dict(Arc::new(StringArray::from(vec!["a", "b", "c"]))),
4054+
);
4055+
assert!(result.is_err());
4056+
assert!(
4057+
result
4058+
.unwrap_err()
4059+
.to_string()
4060+
.contains("Can't compare arrays of different types")
4061+
);
4062+
Ok(())
4063+
}
38814064
}

‎datafusion/sqllogictest/test_files/parquet_filter_pushdown.slt‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -918,13 +918,18 @@ CREATE EXTERNAL TABLE dict_filter_bug
918918
STORED AS PARQUET
919919
LOCATION 'test_files/scratch/parquet_filter_pushdown/dict_filter_bug.parquet';
920920

921-
query error Can't compare arrays of different types
921+
query TR
922922
SELECT t.tag1, t.value
923923
FROM dict_filter_bug t
924924
JOIN (VALUES ('A'), ('B')) AS v(c1)
925925
ON t.tag1 = v.c1
926926
ORDER BY t.tag1, t.value
927927
LIMIT 4;
928+
----
929+
A 0
930+
A 26
931+
A 52
932+
A 78
928933

929934
# Cleanup
930935
statement ok

0 commit comments

Comments
 (0)