Skip to content

Commit aec96ef

Browse files
committed
fix(expr): differentiate aggregate functions with and without explicit cast in output schema (apache#26221)
1 parent 1da3cf0 commit aec96ef

33 files changed

Lines changed: 1233 additions & 165 deletions

File tree

‎datafusion/expr/src/expr.rs‎

Lines changed: 322 additions & 51 deletions
Large diffs are not rendered by default.

‎datafusion/expr/src/expr_rewriter/order_by.rs‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -119,13 +119,23 @@ fn rewrite_in_terms_of_projection(
119119
let (qualifier, field_name) = found.qualified_name();
120120
let col = Expr::Column(Column::new(qualifier, field_name));
121121
return Ok(Transformed::yes(match normalized_expr {
122-
Expr::Cast(Cast { expr: _, field }) => Expr::Cast(Cast {
122+
Expr::Cast(Cast {
123+
expr: _,
124+
field,
125+
is_implicit,
126+
}) => Expr::Cast(Cast {
123127
expr: Box::new(col),
124128
field,
129+
is_implicit,
125130
}),
126-
Expr::TryCast(TryCast { expr: _, field }) => Expr::TryCast(TryCast {
131+
Expr::TryCast(TryCast {
132+
expr: _,
133+
field,
134+
is_implicit,
135+
}) => Expr::TryCast(TryCast {
127136
expr: Box::new(col),
128137
field,
138+
is_implicit,
129139
}),
130140
_ => col,
131141
}));

‎datafusion/expr/src/expr_schema.rs‎

Lines changed: 8 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -665,14 +665,14 @@ impl ExprSchemable for Expr {
665665
func.return_field_from_args(args)
666666
}
667667
// _ => Ok((self.get_type(schema)?, self.nullable(schema)?)),
668-
Expr::Cast(Cast { expr, field }) => expr
668+
Expr::Cast(Cast { expr, field, .. }) => expr
669669
.to_field(schema)
670670
.map(|(_table_ref, src)| cast_output_field(&src, field, false)),
671671
Expr::Placeholder(Placeholder {
672672
id: _,
673673
field: Some(field),
674674
}) => Ok(Arc::clone(field).renamed(&schema_name)),
675-
Expr::TryCast(TryCast { expr, field }) => expr
675+
Expr::TryCast(TryCast { expr, field, .. }) => expr
676676
.to_field(schema)
677677
.map(|(_table_ref, src)| cast_output_field(&src, field, true)),
678678
Expr::LambdaVariable(LambdaVariable {
@@ -773,7 +773,10 @@ impl ExprSchemable for Expr {
773773
Expr::ScalarSubquery(subquery) => {
774774
Ok(Expr::ScalarSubquery(cast_subquery(subquery, cast_to_type)?))
775775
}
776-
_ => Ok(Expr::Cast(Cast::new(Box::new(self), cast_to_type.clone()))),
776+
_ => Ok(Expr::Cast(Cast::new_implicit(
777+
Box::new(self),
778+
cast_to_type.clone(),
779+
))),
777780
}
778781
} else {
779782
plan_err!("Cannot automatically convert {this_type} to {cast_to_type}")
@@ -1526,15 +1529,9 @@ mod tests {
15261529
use_try_cast: bool,
15271530
) -> Expr {
15281531
if use_try_cast {
1529-
Expr::TryCast(TryCast {
1530-
expr: Box::new(expr),
1531-
field: target_field,
1532-
})
1532+
Expr::TryCast(TryCast::new_from_field(Box::new(expr), target_field))
15331533
} else {
1534-
Expr::Cast(Cast {
1535-
expr: Box::new(expr),
1536-
field: target_field,
1537-
})
1534+
Expr::Cast(Cast::new_from_field(Box::new(expr), target_field))
15381535
}
15391536
}
15401537

‎datafusion/expr/src/tree_node.rs‎

Lines changed: 22 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -247,12 +247,28 @@ impl TreeNode for Expr {
247247
.update_data(|(new_expr, new_when_then_expr, new_else_expr)| {
248248
Expr::Case(Case::new(new_expr, new_when_then_expr, new_else_expr))
249249
}),
250-
Expr::Cast(Cast { expr, field }) => expr
251-
.map_elements(f)?
252-
.update_data(|be| Expr::Cast(Cast::new_from_field(be, field))),
253-
Expr::TryCast(TryCast { expr, field }) => expr
254-
.map_elements(f)?
255-
.update_data(|be| Expr::TryCast(TryCast::new_from_field(be, field))),
250+
Expr::Cast(Cast {
251+
expr,
252+
field,
253+
is_implicit,
254+
}) => expr.map_elements(f)?.update_data(|be| {
255+
Expr::Cast(Cast {
256+
expr: be,
257+
field,
258+
is_implicit,
259+
})
260+
}),
261+
Expr::TryCast(TryCast {
262+
expr,
263+
field,
264+
is_implicit,
265+
}) => expr.map_elements(f)?.update_data(|be| {
266+
Expr::TryCast(TryCast {
267+
expr: be,
268+
field,
269+
is_implicit,
270+
})
271+
}),
256272
Expr::ScalarFunction(ScalarFunction { func, args }) => {
257273
args.map_elements(f)?.map_data(|new_args| {
258274
Ok(Expr::ScalarFunction(ScalarFunction::new_udf(

‎datafusion/expr/src/udaf.rs‎

Lines changed: 72 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -33,8 +33,8 @@ use datafusion_physical_expr_common::physical_expr::PhysicalExpr;
3333

3434
use crate::expr::{
3535
AggregateFunction, AggregateFunctionParams, ExprListDisplay, WindowFunctionParams,
36-
schema_name_from_exprs, schema_name_from_exprs_comma_separated_without_space,
37-
schema_name_from_sorts,
36+
schema_name_from_aggregate_args, schema_name_from_exprs,
37+
schema_name_from_function_args, schema_name_from_sorts,
3838
};
3939
use crate::function::{
4040
AccumulatorArgs, AggregateFunctionSimplification, StateFieldsArgs,
@@ -1088,7 +1088,7 @@ impl<'a> UdafSchemaNameBuilder<'a> {
10881088
"{}({}{})",
10891089
name,
10901090
if *distinct { "DISTINCT " } else { "" },
1091-
schema_name_from_exprs_comma_separated_without_space(args)?
1091+
schema_name_from_aggregate_args(args)?
10921092
))?;
10931093

10941094
if let Some(null_treatment) = null_treatment {
@@ -1266,13 +1266,13 @@ impl<'a> UdafWindowFunctionSchemaNameBuilder<'a> {
12661266
schema_name.write_fmt(format_args!(
12671267
"{}(DISTINCT {})",
12681268
name,
1269-
schema_name_from_exprs(args)?
1269+
schema_name_from_function_args(args)?
12701270
))?;
12711271
} else {
12721272
schema_name.write_fmt(format_args!(
12731273
"{}({})",
12741274
name,
1275-
schema_name_from_exprs(args)?
1275+
schema_name_from_function_args(args)?
12761276
))?;
12771277
}
12781278

@@ -1946,4 +1946,71 @@ mod test {
19461946

19471947
Ok(())
19481948
}
1949+
1950+
#[test]
1951+
fn test_udaf_schema_name_with_cast_distinction() -> Result<()> {
1952+
use crate::expr::{AggregateFunctionParams, WindowFunctionParams};
1953+
use crate::{
1954+
UdafSchemaNameBuilder, UdafWindowFunctionSchemaNameBuilder, WindowFrame,
1955+
cast, col,
1956+
};
1957+
1958+
let col_x = col("x");
1959+
let cast_x = cast(col_x.clone(), DataType::Float32);
1960+
1961+
let params1 = AggregateFunctionParams {
1962+
args: vec![col_x.clone()],
1963+
distinct: false,
1964+
filter: None,
1965+
order_by: vec![],
1966+
null_treatment: None,
1967+
};
1968+
let params2 = AggregateFunctionParams {
1969+
args: vec![cast_x.clone()],
1970+
distinct: false,
1971+
filter: None,
1972+
order_by: vec![],
1973+
null_treatment: None,
1974+
};
1975+
1976+
let name1 = UdafSchemaNameBuilder::new("sum", &params1).build()?;
1977+
let name2 = UdafSchemaNameBuilder::new("sum", &params2).build()?;
1978+
assert_eq!(name1, "sum(x)");
1979+
assert_eq!(name2, "sum(CAST(x AS Float32))");
1980+
assert_ne!(name1, name2);
1981+
1982+
let win_params1 = WindowFunctionParams {
1983+
args: vec![col_x],
1984+
partition_by: vec![],
1985+
order_by: vec![],
1986+
window_frame: WindowFrame::new(None),
1987+
filter: None,
1988+
null_treatment: None,
1989+
distinct: false,
1990+
};
1991+
let win_params2 = WindowFunctionParams {
1992+
args: vec![cast_x],
1993+
partition_by: vec![],
1994+
order_by: vec![],
1995+
window_frame: WindowFrame::new(None),
1996+
filter: None,
1997+
null_treatment: None,
1998+
distinct: false,
1999+
};
2000+
let win_name1 =
2001+
UdafWindowFunctionSchemaNameBuilder::new("sum", &win_params1).build()?;
2002+
let win_name2 =
2003+
UdafWindowFunctionSchemaNameBuilder::new("sum", &win_params2).build()?;
2004+
assert_eq!(
2005+
win_name1,
2006+
"sum(x) ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING"
2007+
);
2008+
assert_eq!(
2009+
win_name2,
2010+
"sum(CAST(x AS Float32)) ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING"
2011+
);
2012+
assert_ne!(win_name1, win_name2);
2013+
2014+
Ok(())
2015+
}
19492016
}

‎datafusion/functions/src/core/arrow_cast.rs‎

Lines changed: 10 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -20,9 +20,8 @@
2020
use arrow::datatypes::{DataType, Field, FieldRef};
2121
use arrow::error::ArrowError;
2222
use datafusion_common::{
23-
Result, ScalarValue, arrow_datafusion_err, datatype::DataTypeExt,
24-
exec_datafusion_err, exec_err, internal_err, types::logical_string,
25-
utils::take_function_args,
23+
Result, ScalarValue, arrow_datafusion_err, exec_datafusion_err, exec_err,
24+
internal_err, types::logical_string, utils::take_function_args,
2625
};
2726

2827
use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext};
@@ -174,20 +173,20 @@ impl ScalarUDFImpl for ArrowCastFunc {
174173
.contains_key("ARROW:extension:name");
175174
if has_extension_metadata {
176175
// Need to create a cast to strip extension metadata
177-
Expr::Cast(datafusion_expr::Cast {
178-
expr: Box::new(source_arg),
179-
field: target_type.into_nullable_field_ref(),
180-
})
176+
Expr::Cast(datafusion_expr::Cast::new(
177+
Box::new(source_arg),
178+
target_type,
179+
))
181180
} else {
182181
// the argument's data type is already the correct type
183182
source_arg
184183
}
185184
} else {
186185
// Use an actual cast to get the correct type
187-
Expr::Cast(datafusion_expr::Cast {
188-
expr: Box::new(source_arg),
189-
field: target_type.into_nullable_field_ref(),
190-
})
186+
Expr::Cast(datafusion_expr::Cast::new(
187+
Box::new(source_arg),
188+
target_type,
189+
))
191190
};
192191
// return the newly written argument to DataFusion
193192
Ok(ExprSimplifyResult::Simplified(new_expr))

‎datafusion/functions/src/core/arrow_try_cast.rs‎

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -20,8 +20,8 @@
2020
use arrow::datatypes::{DataType, Field, FieldRef};
2121
use arrow::error::ArrowError;
2222
use datafusion_common::{
23-
Result, arrow_datafusion_err, datatype::DataTypeExt, exec_datafusion_err, exec_err,
24-
internal_err, types::logical_string, utils::take_function_args,
23+
Result, arrow_datafusion_err, exec_datafusion_err, exec_err, internal_err,
24+
types::logical_string, utils::take_function_args,
2525
};
2626

2727
use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext};
@@ -147,18 +147,18 @@ impl ScalarUDFImpl for ArrowTryCastFunc {
147147
.contains_key("ARROW:extension:name");
148148
if has_extension_metadata {
149149
// Need to create a try_cast to strip extension metadata
150-
Expr::TryCast(datafusion_expr::TryCast {
151-
expr: Box::new(source_arg),
152-
field: target_type.into_nullable_field_ref(),
153-
})
150+
Expr::TryCast(datafusion_expr::TryCast::new(
151+
Box::new(source_arg),
152+
target_type,
153+
))
154154
} else {
155155
source_arg
156156
}
157157
} else {
158-
Expr::TryCast(datafusion_expr::TryCast {
159-
expr: Box::new(source_arg),
160-
field: target_type.into_nullable_field_ref(),
161-
})
158+
Expr::TryCast(datafusion_expr::TryCast::new(
159+
Box::new(source_arg),
160+
target_type,
161+
))
162162
};
163163
Ok(ExprSimplifyResult::Simplified(new_expr))
164164
}

‎datafusion/functions/src/core/cast_to_type.rs‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -132,10 +132,10 @@ impl ScalarUDFImpl for CastToTypeFunc {
132132
} else {
133133
let nullable = info.nullable(&source_arg)? || target_type == DataType::Null;
134134
// Use an actual cast to get the correct type
135-
Expr::Cast(datafusion_expr::Cast {
136-
expr: Box::new(source_arg),
137-
field: Field::new("", target_type, nullable).into(),
138-
})
135+
Expr::Cast(datafusion_expr::Cast::new_from_field(
136+
Box::new(source_arg),
137+
Field::new("", target_type, nullable).into(),
138+
))
139139
};
140140
Ok(ExprSimplifyResult::Simplified(new_expr))
141141
}

‎datafusion/functions/src/core/try_cast_to_type.rs‎

Lines changed: 5 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -18,9 +18,7 @@
1818
//! [`TryCastToTypeFunc`]: Implementation of the `try_cast_to_type` function
1919
2020
use arrow::datatypes::{DataType, Field, FieldRef};
21-
use datafusion_common::{
22-
Result, datatype::DataTypeExt, internal_err, utils::take_function_args,
23-
};
21+
use datafusion_common::{Result, internal_err, utils::take_function_args};
2422
use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext};
2523
use datafusion_expr::{
2624
Coercion, ColumnarValue, Documentation, Expr, ReturnFieldArgs, ScalarFunctionArgs,
@@ -116,10 +114,10 @@ impl ScalarUDFImpl for TryCastToTypeFunc {
116114
let new_expr = if source_type == target_type {
117115
source_arg
118116
} else {
119-
Expr::TryCast(datafusion_expr::TryCast {
120-
expr: Box::new(source_arg),
121-
field: target_type.into_nullable_field_ref(),
122-
})
117+
Expr::TryCast(datafusion_expr::TryCast::new(
118+
Box::new(source_arg),
119+
target_type,
120+
))
123121
};
124122
Ok(ExprSimplifyResult::Simplified(new_expr))
125123
}

‎datafusion/optimizer/src/analyzer/type_coercion.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -589,7 +589,7 @@ impl<'a> TypeCoercionRewriter<'a> {
589589
use DataType::*;
590590

591591
fn cast(expr: Expr, target_type: DataType) -> Expr {
592-
Expr::Cast(Cast::new(Box::new(expr), target_type))
592+
Expr::Cast(Cast::new_implicit(Box::new(expr), target_type))
593593
}
594594

595595
fn time_to_nanos(

0 commit comments

Comments
 (0)