Skip to content

Commit c857382

Browse files
yinli-systemsKevin-Li-2025Jefffrey
authored
Align scalar UDF literal args with coerced return fields (apache#23232)
## Which issue does this PR close? - Closes apache#19982. ## Rationale for this change `ReturnFieldArgs` exposes both coerced `arg_fields` and optional scalar literal arguments. A coercible UDF could receive an `Int16` field but an original `Int64` literal, or lose the literal behind an implicit cast. That makes return-field decisions inconsistent and forced UDF implementations to compensate for planner internals. ## What changes are included in this PR? - Materialize successful implicit casts of scalar-function literals during type coercion, so literal values and `arg_fields` always have the same type. - Keep user-written `Cast` and `TryCast` expressions as expressions rather than reporting them as literals. - Preserve execution-time errors for invalid literal casts. - Simplify return-field scalar extraction to accept only genuine post-coercion literals. - Update `round` to use the strengthened contract and document it on `ReturnFieldArgs`. - Add unit and integration regressions for coerced integers, typed nulls, list literals, invalid values, explicit casts, expression arguments, and `arrow_cast`'s required literal type argument. This follows the approach discussed in apache#20012: materialize implicit literal coercion in the type-coercion pass without changing explicit-cast semantics. ## Are these changes tested? - `cargo fmt --all --check` - `cargo test -p datafusion --test user_defined_integration test_return_field_args_scalar_argument_types_match_arg_fields` (1 passed) - `cargo test -p datafusion --test core_integration test_fn_arrow_cast_requires_literal_type_arg` (1 passed) - `cargo test -p datafusion-expr scalar_arguments_` (2 passed) - `cargo clippy --all-targets --all-features -- -D warnings` - `git diff --check` The validation process caught and fixed two semantic regressions before push: explicit casts being folded into scalar literals, and `arrow_cast` reporting its existing error during planning rather than collection. ## AI assistance AI tooling assisted with implementation and test iteration. I reviewed the final diff, the related apache#20012 design discussion, and the scalar-function coercion/return-field flow end to end, and ran the validations listed above. --------- Signed-off-by: Kevin-Li-2025 <[email protected]> Signed-off-by: Kevin-Li-2025 <[email protected]> Co-authored-by: Kevin-Li-2025 <[email protected]> Co-authored-by: Jeffrey Vo <[email protected]>
1 parent ad8a147 commit c857382

5 files changed

Lines changed: 248 additions & 28 deletions

File tree

‎datafusion/core/tests/user_defined/user_defined_scalar_functions.rs‎

Lines changed: 92 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -20,8 +20,8 @@ use std::hash::{Hash, Hasher};
2020
use std::sync::Arc;
2121

2222
use arrow::array::{
23-
Array, ArrayRef, Float32Array, Float64Array, Int32Array, RecordBatch, StringArray,
24-
builder::BooleanBuilder, cast::AsArray,
23+
Array, ArrayRef, Float32Array, Float64Array, Int16Array, Int32Array, RecordBatch,
24+
StringArray, builder::BooleanBuilder, cast::AsArray,
2525
};
2626
use arrow::array::{Int8Array, UInt64Array, as_string_array, create_array, record_batch};
2727
use arrow::compute::kernels::numeric::add;
@@ -40,13 +40,15 @@ use datafusion_common::{
4040
DFSchema, DataFusionError, Result, ScalarValue, assert_batches_eq,
4141
assert_batches_sorted_eq, assert_contains, exec_datafusion_err, exec_err,
4242
not_impl_err, plan_err,
43+
types::{NativeType, logical_int16},
4344
};
4445
use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext};
4546
use datafusion_expr::{
4647
Accumulator, ColumnarValue, CreateFunction, CreateFunctionBody, LogicalPlanBuilder,
4748
OperateFunctionArg, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDF, ScalarUDFImpl,
48-
Signature, Volatility, lit_with_metadata,
49+
Signature, TypeSignatureClass, Volatility, lit_with_metadata,
4950
};
51+
use datafusion_expr_common::signature::Coercion;
5052
use datafusion_expr_common::signature::TypeSignature;
5153
use datafusion_functions_nested::range::range_udf;
5254
use parking_lot::Mutex;
@@ -2073,6 +2075,93 @@ AS t(string, extension)
20732075
Ok(())
20742076
}
20752077

2078+
/// https://github.com/apache/datafusion/issues/19982
2079+
#[tokio::test]
2080+
async fn test_return_field_args_scalar_argument_types_match_arg_fields() -> Result<()> {
2081+
#[derive(Debug, PartialEq, Eq, Hash)]
2082+
struct TestUdf {
2083+
name: &'static str,
2084+
expect_literal: bool,
2085+
signature: Signature,
2086+
}
2087+
2088+
impl TestUdf {
2089+
fn new(name: &'static str, expect_literal: bool) -> Self {
2090+
Self {
2091+
name,
2092+
expect_literal,
2093+
signature: Signature::coercible(
2094+
vec![Coercion::new_implicit(
2095+
TypeSignatureClass::Native(logical_int16()),
2096+
vec![TypeSignatureClass::Numeric],
2097+
NativeType::Int16,
2098+
)],
2099+
Volatility::Immutable,
2100+
),
2101+
}
2102+
}
2103+
}
2104+
2105+
impl ScalarUDFImpl for TestUdf {
2106+
fn name(&self) -> &str {
2107+
self.name
2108+
}
2109+
2110+
fn signature(&self) -> &Signature {
2111+
&self.signature
2112+
}
2113+
2114+
fn return_type(&self, _arg_types: &[DataType]) -> Result<DataType> {
2115+
unreachable!("return_field_from_args is implemented")
2116+
}
2117+
2118+
fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result<FieldRef> {
2119+
assert_eq!(args.arg_fields.len(), 1);
2120+
assert_eq!(args.scalar_arguments.len(), 1);
2121+
assert_eq!(
2122+
args.scalar_arguments[0].is_some(),
2123+
self.expect_literal,
2124+
"unexpected scalar argument for {}",
2125+
self.name
2126+
);
2127+
if self.expect_literal {
2128+
let scalar = args.scalar_arguments[0].unwrap();
2129+
assert_eq!(args.arg_fields[0].data_type(), &scalar.data_type());
2130+
}
2131+
Ok(
2132+
Field::new(self.name(), args.arg_fields[0].data_type().clone(), true)
2133+
.into(),
2134+
)
2135+
}
2136+
2137+
fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {
2138+
assert_eq!(args.args[0].data_type(), DataType::Int16);
2139+
Ok(args.args[0].clone())
2140+
}
2141+
}
2142+
2143+
let ctx = SessionContext::new();
2144+
let coerced_literal_udf: ScalarUDF = TestUdf::new("coerced_literal_udf", true).into();
2145+
let expression_arg_udf: ScalarUDF = TestUdf::new("expression_arg_udf", false).into();
2146+
ctx.register_udf(coerced_literal_udf);
2147+
ctx.register_udf(expression_arg_udf.clone());
2148+
2149+
ctx.sql("select coerced_literal_udf(1), coerced_literal_udf(NULL)")
2150+
.await?
2151+
.collect()
2152+
.await?;
2153+
let batch = RecordBatch::try_new(
2154+
Arc::new(Schema::new(vec![Field::new("a", DataType::Int16, false)])),
2155+
vec![Arc::new(Int16Array::from(vec![1]))],
2156+
)?;
2157+
ctx.read_batch(batch)?
2158+
.select(vec![expression_arg_udf.call(vec![col("a")])])?
2159+
.collect()
2160+
.await?;
2161+
2162+
Ok(())
2163+
}
2164+
20762165
/// https://github.com/apache/datafusion/issues/17422
20772166
#[tokio::test]
20782167
async fn test_extension_metadata_preserve_in_subquery() -> Result<()> {

‎datafusion/expr/src/expr_schema.rs‎

Lines changed: 78 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,26 @@ fn cast_output_field(
8787
Arc::new(f)
8888
}
8989

90+
fn scalar_arguments_for_fields(
91+
args: &[Expr],
92+
arg_fields: &[FieldRef],
93+
) -> Vec<Option<ScalarValue>> {
94+
args.iter()
95+
.zip(arg_fields)
96+
.map(|(expr, field)| scalar_argument_for_field(expr, field))
97+
.collect()
98+
}
99+
100+
fn scalar_argument_for_field(expr: &Expr, arg_field: &FieldRef) -> Option<ScalarValue> {
101+
match expr {
102+
Expr::Literal(sv, _) => Some(
103+
sv.cast_to(arg_field.data_type())
104+
.unwrap_or_else(|_| sv.clone()),
105+
),
106+
_ => None,
107+
}
108+
}
109+
90110
impl ExprSchemable for Expr {
91111
/// Returns the [arrow::datatypes::DataType] of the expression
92112
/// based on [ExprSchema]
@@ -585,16 +605,12 @@ impl ExprSchemable for Expr {
585605
.collect::<Result<Vec<_>>>()?;
586606
let new_fields = verify_function_arguments(func.as_ref(), &fields)?;
587607

588-
let arguments = args
589-
.iter()
590-
.map(|e| match e {
591-
Expr::Literal(sv, _) => Some(sv),
592-
_ => None,
593-
})
594-
.collect::<Vec<_>>();
608+
let arguments = scalar_arguments_for_fields(args, &new_fields);
609+
let argument_refs =
610+
arguments.iter().map(Option::as_ref).collect::<Vec<_>>();
595611
let args = ReturnFieldArgs {
596612
arg_fields: &new_fields,
597-
scalar_arguments: &arguments,
613+
scalar_arguments: &argument_refs,
598614
};
599615

600616
func.return_field_from_args(args)
@@ -827,6 +843,60 @@ mod tests {
827843
}};
828844
}
829845

846+
#[test]
847+
fn scalar_arguments_match_coerced_fields() {
848+
let int16_field: FieldRef = Field::new("arg", DataType::Int16, true).into();
849+
850+
assert_eq!(
851+
scalar_argument_for_field(&lit(1_i64), &int16_field),
852+
Some(ScalarValue::Int16(Some(1)))
853+
);
854+
assert_eq!(
855+
scalar_argument_for_field(&lit(ScalarValue::Null), &int16_field),
856+
Some(ScalarValue::Int16(None))
857+
);
858+
859+
let int32_list = ScalarValue::List(ScalarValue::new_list(
860+
&[ScalarValue::Int32(Some(1))],
861+
&DataType::Int32,
862+
true,
863+
));
864+
let int64_list_type = DataType::new_list(DataType::Int64, true);
865+
let int64_list_field: FieldRef =
866+
Field::new("arg", int64_list_type.clone(), true).into();
867+
assert_eq!(
868+
scalar_argument_for_field(&lit(int32_list.clone()), &int64_list_field),
869+
Some(int32_list.cast_to(&int64_list_type).unwrap())
870+
);
871+
}
872+
873+
#[test]
874+
fn scalar_arguments_exclude_expression_casts_and_preserve_invalid_values() {
875+
let int16_field: FieldRef = Field::new("arg", DataType::Int16, true).into();
876+
let int32_field: FieldRef = Field::new("arg", DataType::Int32, true).into();
877+
let explicit_cast = Expr::Cast(Cast::new(Box::new(lit(1_i64)), DataType::Int16));
878+
let explicit_try_cast =
879+
Expr::TryCast(TryCast::new(Box::new(lit(1_i64)), DataType::Int16));
880+
let out_of_i32_range = i64::from(i32::MAX) + 1;
881+
882+
assert_eq!(
883+
scalar_argument_for_field(&explicit_cast, &int16_field),
884+
None
885+
);
886+
assert_eq!(
887+
scalar_argument_for_field(&explicit_try_cast, &int16_field),
888+
None
889+
);
890+
assert_eq!(
891+
scalar_argument_for_field(&lit("not an integer"), &int16_field),
892+
Some(ScalarValue::Utf8(Some("not an integer".to_string())))
893+
);
894+
assert_eq!(
895+
scalar_argument_for_field(&lit(out_of_i32_range), &int32_field),
896+
Some(ScalarValue::Int64(Some(out_of_i32_range)))
897+
);
898+
}
899+
830900
#[test]
831901
fn expr_schema_nullability() {
832902
let expr = col("foo").eq(lit(1));

‎datafusion/functions/src/math/round.rs‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -241,9 +241,6 @@ impl ScalarUDFImpl for RoundFunc {
241241

242242
// If decimal_places is a scalar literal, we can incorporate it into the output type
243243
// (scale reduction). Otherwise, keep the input scale as we can't pick a per-row scale.
244-
//
245-
// Note: `scalar_arguments` contains the original literal values (pre-coercion), so
246-
// integer literals may appear as Int64 even though the signature coerces them to Int32.
247244
let decimal_places: Option<i32> = match args.scalar_arguments.get(1) {
248245
None => Some(0), // No dp argument means default to 0
249246
Some(None) => None, // dp is not a literal (e.g. column)

‎datafusion/optimizer/src/analyzer/type_coercion.rs‎

Lines changed: 74 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -813,8 +813,11 @@ impl TreeNodeRewriter for TypeCoercionRewriter<'_> {
813813
Ok(Transformed::yes(Expr::Case(case)))
814814
}
815815
Expr::ScalarFunction(ScalarFunction { func, args }) => {
816-
let new_expr =
817-
coerce_arguments_for_signature(args, self.schema, func.as_ref())?;
816+
let new_expr = coerce_scalar_function_arguments_for_signature(
817+
args,
818+
self.schema,
819+
func.as_ref(),
820+
)?;
818821
Ok(Transformed::yes(Expr::ScalarFunction(
819822
ScalarFunction::new_udf(func, new_expr),
820823
)))
@@ -1147,23 +1150,80 @@ fn coerce_arguments_for_signature<F: UDFCoercionExt>(
11471150
schema: &DFSchema,
11481151
func: &F,
11491152
) -> Result<Vec<Expr>> {
1150-
let current_fields = expressions
1151-
.iter()
1152-
.map(|e| e.to_field(schema).map(|(_, f)| f))
1153-
.collect::<Result<Vec<_>>>()?;
1153+
let coerced_types = coerced_argument_types(&expressions, schema, func)?;
11541154

1155-
let coerced_types = fields_with_udf(&current_fields, func)?
1155+
expressions
11561156
.into_iter()
1157-
.map(|f| f.data_type().clone())
1158-
.collect::<Vec<_>>();
1157+
.zip(coerced_types)
1158+
.map(|(expr, data_type)| expr.cast_to(&data_type, schema))
1159+
.collect()
1160+
}
1161+
1162+
/// Coerces scalar function arguments while materializing successful implicit
1163+
/// casts of literals. This preserves the literal for subsequent calls to
1164+
/// `return_field_from_args` without treating user-written `Cast` or `TryCast`
1165+
/// expressions as scalar arguments.
1166+
fn coerce_scalar_function_arguments_for_signature<F: UDFCoercionExt>(
1167+
expressions: Vec<Expr>,
1168+
schema: &DFSchema,
1169+
func: &F,
1170+
) -> Result<Vec<Expr>> {
1171+
let coerced_types = coerced_argument_types(&expressions, schema, func)?;
11591172

11601173
expressions
11611174
.into_iter()
1162-
.enumerate()
1163-
.map(|(i, expr)| expr.cast_to(&coerced_types[i], schema))
1175+
.zip(coerced_types)
1176+
.map(|(expr, data_type)| {
1177+
coerce_scalar_function_argument(expr, &data_type, schema)
1178+
})
11641179
.collect()
11651180
}
11661181

1182+
fn coerced_argument_types<F: UDFCoercionExt>(
1183+
expressions: &[Expr],
1184+
schema: &DFSchema,
1185+
func: &F,
1186+
) -> Result<Vec<DataType>> {
1187+
let current_fields = expressions
1188+
.iter()
1189+
.map(|e| e.to_field(schema).map(|(_, f)| f))
1190+
.collect::<Result<Vec<_>>>()?;
1191+
1192+
fields_with_udf(&current_fields, func).map(|fields| {
1193+
fields
1194+
.into_iter()
1195+
.map(|field| field.data_type().clone())
1196+
.collect()
1197+
})
1198+
}
1199+
1200+
fn coerce_scalar_function_argument(
1201+
expr: Expr,
1202+
data_type: &DataType,
1203+
schema: &DFSchema,
1204+
) -> Result<Expr> {
1205+
if matches!(&expr, Expr::Cast(_) | Expr::TryCast(_))
1206+
&& expr.get_type(schema)? == *data_type
1207+
{
1208+
return Ok(expr);
1209+
}
1210+
1211+
let Expr::Literal(value, metadata) = expr else {
1212+
return expr.cast_to(data_type, schema);
1213+
};
1214+
1215+
if value.data_type() != *data_type
1216+
&& let Ok(value) = value.cast_to(data_type)
1217+
{
1218+
return Ok(Expr::Literal(value, metadata));
1219+
}
1220+
1221+
// A failed value cast remains an expression cast so execution produces the
1222+
// same error as before. Since it is no longer a literal at the coerced type,
1223+
// it is reported as `None` in `ReturnFieldArgs::scalar_arguments`.
1224+
Expr::Literal(value, metadata).cast_to(data_type, schema)
1225+
}
1226+
11671227
fn coerce_case_expression(case: Case, schema: &DFSchema) -> Result<Case> {
11681228
// Given expressions like:
11691229
//
@@ -2034,7 +2094,7 @@ mod test {
20342094
assert_analyzed_plan_eq!(
20352095
plan,
20362096
@r"
2037-
Projection: TestScalarUDF(CAST(Int32(123) AS Float32))
2097+
Projection: TestScalarUDF(Float32(123)) AS TestScalarUDF(Int32(123))
20382098
EmptyRelation: rows=0
20392099
"
20402100
)
@@ -2071,7 +2131,7 @@ mod test {
20712131
assert_analyzed_plan_eq!(
20722132
plan,
20732133
@r"
2074-
Projection: TestScalarUDF(CAST(Int64(10) AS Float32))
2134+
Projection: TestScalarUDF(Float32(10)) AS TestScalarUDF(Int64(10))
20752135
EmptyRelation: rows=0
20762136
"
20772137
)
@@ -2611,7 +2671,7 @@ mod test {
26112671
assert_analyzed_plan_eq!(
26122672
plan,
26132673
@r#"
2614-
Projection: TestScalarUDF(a, Utf8("b"), CAST(Boolean(true) AS Utf8), CAST(Boolean(false) AS Utf8), CAST(Int32(13) AS Utf8))
2674+
Projection: TestScalarUDF(a, Utf8("b"), Utf8("true"), Utf8("false"), Utf8("13")) AS TestScalarUDF(a,Utf8("b"),Boolean(true),Boolean(false),Int32(13))
26152675
EmptyRelation: rows=0
26162676
"#
26172677
)

‎datafusion/sqllogictest/test_files/string_numeric_coercion.slt‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -573,6 +573,10 @@ SELECT arrow_cast('hello', 'RunEndEncoded("run_ends": non-null Int32, "values":
573573
----
574574
true
575575

576+
# The type argument to arrow_cast must remain a literal after planning.
577+
statement error arrow_cast requires its second argument
578+
SELECT arrow_cast(1, cast('Utf8' as varchar));
579+
576580
# -------------------------------------------------
577581
# Cleanup
578582
# -------------------------------------------------

0 commit comments

Comments
 (0)