Skip to content

Commit 5731228

Browse files
authored
unwrap_cast_in_comparison: fix bug which can find the field for the schema (#3699)
* the unwrap rule: don't throw error when meet unsupported data type * support data type in unwrap cast rule
1 parent d0760f3 commit 5731228

2 files changed

Lines changed: 62 additions & 4 deletions

File tree

‎datafusion/optimizer/src/unwrap_cast_in_comparison.rs‎

Lines changed: 34 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -22,12 +22,13 @@ use crate::{OptimizerConfig, OptimizerRule};
2222
use arrow::datatypes::{
2323
DataType, MAX_DECIMAL_FOR_EACH_PRECISION, MIN_DECIMAL_FOR_EACH_PRECISION,
2424
};
25-
use datafusion_common::{DFSchemaRef, DataFusionError, Result, ScalarValue};
25+
use datafusion_common::{DFSchema, DFSchemaRef, DataFusionError, Result, ScalarValue};
2626
use datafusion_expr::expr_rewriter::{ExprRewritable, ExprRewriter, RewriteRecursion};
2727
use datafusion_expr::utils::from_plan;
2828
use datafusion_expr::{
2929
binary_expr, in_list, lit, Expr, ExprSchemable, LogicalPlan, Operator,
3030
};
31+
use std::sync::Arc;
3132

3233
/// The rule can be used to the numeric binary comparison with literal expr, like below pattern:
3334
/// `cast(left_expr as data_type) comparison_op literal_expr` or `literal_expr comparison_op cast(right_expr as data_type)`.
@@ -79,10 +80,18 @@ fn optimize(plan: &LogicalPlan) -> Result<LogicalPlan> {
7980
.map(|input| optimize(input))
8081
.collect::<Result<Vec<_>>>()?;
8182

82-
let schema = plan.schema();
83+
let mut schema = new_inputs.iter().map(|input| input.schema()).fold(
84+
DFSchema::empty(),
85+
|mut lhs, rhs| {
86+
lhs.merge(rhs);
87+
lhs
88+
},
89+
);
90+
91+
schema.merge(plan.schema());
8392

8493
let mut expr_rewriter = UnwrapCastExprRewriter {
85-
schema: schema.clone(),
94+
schema: Arc::new(schema),
8695
};
8796

8897
let new_exprs = plan
@@ -273,6 +282,11 @@ fn try_cast_literal_to_type(
273282
lit_value: &ScalarValue,
274283
target_type: &DataType,
275284
) -> Result<Option<ScalarValue>> {
285+
let lit_data_type = lit_value.get_datatype();
286+
// the rule just support the signed numeric data type now
287+
if !is_support_data_type(&lit_data_type) || !is_support_data_type(target_type) {
288+
return Ok(None);
289+
}
276290
if lit_value.is_null() {
277291
// null value can be cast to any type of null value
278292
return Ok(Some(ScalarValue::try_from(target_type)?));
@@ -373,7 +387,7 @@ mod tests {
373387
use arrow::datatypes::DataType;
374388
use datafusion_common::{DFField, DFSchema, DFSchemaRef, ScalarValue};
375389
use datafusion_expr::expr_rewriter::ExprRewritable;
376-
use datafusion_expr::{cast, col, lit, try_cast, Expr};
390+
use datafusion_expr::{cast, col, in_list, lit, try_cast, Expr};
377391
use std::collections::HashMap;
378392
use std::sync::Arc;
379393

@@ -585,6 +599,21 @@ mod tests {
585599
assert_eq!(optimize_test(expr_lt, &schema), expected);
586600
}
587601

602+
#[test]
603+
fn test_not_support_data_type() {
604+
// "c6 > 0" will be cast to `cast(c6 as int64) > 0
605+
// but the type of c6 is uint32
606+
// the rewriter will not throw error and just return the original expr
607+
let schema = expr_test_schema();
608+
let expr_input = cast(col("c6"), DataType::Int64).eq(lit(0i64));
609+
assert_eq!(optimize_test(expr_input.clone(), &schema), expr_input);
610+
611+
// inlist for unsupported data type
612+
let expr_input =
613+
in_list(cast(col("c6"), DataType::Int64), vec![lit(0i64)], false);
614+
assert_eq!(optimize_test(expr_input.clone(), &schema), expr_input);
615+
}
616+
588617
fn optimize_test(expr: Expr, schema: &DFSchemaRef) -> Expr {
589618
let mut expr_rewriter = UnwrapCastExprRewriter {
590619
schema: schema.clone(),
@@ -601,6 +630,7 @@ mod tests {
601630
DFField::new(None, "c3", DataType::Decimal128(18, 2), false),
602631
DFField::new(None, "c4", DataType::Decimal128(38, 37), false),
603632
DFField::new(None, "c5", DataType::Float32, false),
633+
DFField::new(None, "c6", DataType::UInt32, false),
604634
],
605635
HashMap::new(),
606636
)

‎datafusion/optimizer/tests/integration-test.rs‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,33 @@ use std::any::Any;
2929
use std::collections::HashMap;
3030
use std::sync::Arc;
3131

32+
#[test]
33+
fn case_when() -> Result<()> {
34+
let sql = "SELECT CASE WHEN col_int32 > 0 THEN 1 ELSE 0 END FROM test";
35+
let plan = test_sql(sql)?;
36+
let expected = "Projection: CASE WHEN #test.col_int32 > Int32(0) THEN Int64(1) ELSE Int64(0) END\
37+
\n TableScan: test projection=[col_int32]";
38+
assert_eq!(expected, format!("{:?}", plan));
39+
40+
let sql = "SELECT CASE WHEN col_uint32 > 0 THEN 1 ELSE 0 END FROM test";
41+
let plan = test_sql(sql)?;
42+
let expected = "Projection: CASE WHEN CAST(#test.col_uint32 AS Int64) > Int64(0) THEN Int64(1) ELSE Int64(0) END\
43+
\n TableScan: test projection=[col_uint32]";
44+
assert_eq!(expected, format!("{:?}", plan));
45+
Ok(())
46+
}
47+
48+
#[test]
49+
fn unsigned_target_type() -> Result<()> {
50+
let sql = "SELECT * FROM test WHERE col_uint32 > 0";
51+
let plan = test_sql(sql)?;
52+
let expected = "Projection: #test.col_int32, #test.col_uint32, #test.col_utf8, #test.col_date32, #test.col_date64\
53+
\n Filter: CAST(#test.col_uint32 AS Int64) > Int64(0)\
54+
\n TableScan: test projection=[col_int32, col_uint32, col_utf8, col_date32, col_date64]";
55+
assert_eq!(expected, format!("{:?}", plan));
56+
Ok(())
57+
}
58+
3259
#[test]
3360
fn distribute_by() -> Result<()> {
3461
// regression test for https://github.com/apache/arrow-datafusion/issues/3234
@@ -114,6 +141,7 @@ impl ContextProvider for MySchemaProvider {
114141
let schema = Schema::new_with_metadata(
115142
vec![
116143
Field::new("col_int32", DataType::Int32, true),
144+
Field::new("col_uint32", DataType::UInt32, true),
117145
Field::new("col_utf8", DataType::Utf8, true),
118146
Field::new("col_date32", DataType::Date32, true),
119147
Field::new("col_date64", DataType::Date64, true),

0 commit comments

Comments
 (0)