Skip to content

Commit fcbc9bd

Browse files
authored
[branch-54] Handle nulls in type coercion of higher-order UDFs, map_extract, spark array_repeat (backport #23071) (#23629)
## Which issue does this PR close? - Backport of #23071 (clean cherry-pick of non-merge commits) - Closes #23068 . ## Rationale for this change Unresolved lambda variables and unspecified placeholders report a `DataType::Null` data type, which causes an error when used as argument of a function that doesn't handle nulls during type coercion ## What changes are included in this PR? Handle null in `coerce_single_list_arg` for higher-order functions, `map_extract` and in spark `array_repeat`. All other functions (scalar, window, agg and higher-order) that uses user defined type coercion have been checked to currently handle null args. ## Are these changes tested? Test passing `array_filter` with an unresolved lambda variable to `Dataframe::with_column` (which indirectly calls `value_fields_with_higher_order_udf_and_lambdas` coercion via `Projection::try_new`, `projection_schema`, `exprlist_to_fields`, `Expr::to_field` ), and sqllogictests ## Are there any user-facing changes? No
1 parent 61b0f76 commit fcbc9bd

10 files changed

Lines changed: 98 additions & 30 deletions

File tree

‎datafusion/core/tests/dataframe/mod.rs‎

Lines changed: 48 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@ use datafusion_functions_aggregate::expr_fn::{
3939
array_agg, avg, avg_distinct, count, count_distinct, max, median, min, sum,
4040
sum_distinct,
4141
};
42+
use datafusion_functions_nested::expr_fn::{array_filter, array_transform, make_array};
4243
use datafusion_functions_nested::make_array::make_array_udf;
4344
use datafusion_functions_window::expr_fn::{first_value, lead, row_number};
4445
use insta::assert_snapshot;
@@ -78,8 +79,8 @@ use datafusion_expr::{
7879
CreateMemoryTable, CreateView, DdlStatement, Expr, ExprFunctionExt, ExprSchemable,
7980
LogicalPlan, LogicalPlanBuilder, ScalarFunctionImplementation, SortExpr, TableType,
8081
WindowFrame, WindowFrameBound, WindowFrameUnits, WindowFunctionDefinition, cast, col,
81-
create_udf, exists, in_subquery, lit, out_ref_col, placeholder, scalar_subquery,
82-
when, wildcard,
82+
create_udf, exists, in_subquery, lambda, lambda_var, lit, out_ref_col, placeholder,
83+
scalar_subquery, when, wildcard,
8384
};
8485
use datafusion_physical_expr::Partitioning;
8586
use datafusion_physical_expr::aggregate::AggregateExprBuilder;
@@ -90,7 +91,9 @@ use datafusion_physical_plan::aggregates::{
9091
AggregateExec, AggregateMode, PhysicalGroupBy,
9192
};
9293
use datafusion_physical_plan::empty::EmptyExec;
93-
use datafusion_physical_plan::{ExecutionPlan, ExecutionPlanProperties, displayable};
94+
use datafusion_physical_plan::{
95+
ExecutionPlan, ExecutionPlanProperties, collect, displayable,
96+
};
9497

9598
use datafusion::error::Result as DataFusionResult;
9699
use datafusion::execution::options::JsonReadOptions;
@@ -7062,3 +7065,45 @@ async fn test_grouping_with_alias() -> Result<()> {
70627065

70637066
Ok(())
70647067
}
7068+
7069+
#[tokio::test]
7070+
async fn test_unresolved_lambda_variable() -> Result<()> {
7071+
let plan = table_with_mixed_lists()
7072+
.await?
7073+
.with_column(
7074+
"c",
7075+
array_transform(
7076+
make_array(vec![col("list")]),
7077+
lambda(
7078+
["x"],
7079+
array_filter(
7080+
lambda_var("x"),
7081+
lambda(["y"], lambda_var("y").gt_eq(lit(2))),
7082+
),
7083+
),
7084+
),
7085+
)?
7086+
.select_columns(&["list", "c"])?
7087+
.into_unoptimized_plan()
7088+
.resolve_lambda_variables()?
7089+
.data;
7090+
7091+
let session = SessionContext::new();
7092+
let exec = session.state().create_physical_plan(&plan).await?;
7093+
let context = session.task_ctx();
7094+
let results = collect(exec, context).await?;
7095+
7096+
let expected = [
7097+
"+-----------+----------+",
7098+
"| list | c |",
7099+
"+-----------+----------+",
7100+
"| [1, 2, 3] | [[2, 3]] |",
7101+
"| | [] |",
7102+
"| [] | [[]] |",
7103+
"| | [] |",
7104+
"+-----------+----------+",
7105+
];
7106+
assert_batches_eq!(expected, &results);
7107+
7108+
Ok(())
7109+
}

‎datafusion/functions-nested/src/array_any_match.rs‎

Lines changed: 3 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,8 @@ use datafusion_expr::{
3737
use datafusion_macros::user_doc;
3838
use std::{fmt::Debug, sync::Arc};
3939

40+
use crate::lambda_utils::coerce_single_list_arg;
41+
4042
make_higher_order_function_expr_and_func!(
4143
ArrayAnyMatch,
4244
array_any_match,
@@ -120,30 +122,7 @@ impl HigherOrderUDFImpl for ArrayAnyMatch {
120122
}
121123

122124
fn coerce_value_types(&self, arg_types: &[DataType]) -> Result<Vec<DataType>> {
123-
let [list] = arg_types else {
124-
return plan_err!(
125-
"{} function requires 1 value argument, got {}",
126-
self.name(),
127-
arg_types.len()
128-
);
129-
};
130-
131-
let coerced = match list {
132-
DataType::List(_) | DataType::LargeList(_) => list.clone(),
133-
DataType::ListView(field) | DataType::FixedSizeList(field, _) => {
134-
DataType::List(Arc::clone(field))
135-
}
136-
DataType::LargeListView(field) => DataType::LargeList(Arc::clone(field)),
137-
_ => {
138-
return plan_err!(
139-
"{} expected a list as first argument, got {}",
140-
self.name(),
141-
list
142-
);
143-
}
144-
};
145-
146-
Ok(vec![coerced])
125+
coerce_single_list_arg(self.name(), arg_types)
147126
}
148127

149128
fn lambda_parameters(

‎datafusion/functions-nested/src/lambda_utils.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,7 @@ pub(crate) fn coerce_single_list_arg(
6565
DataType::List(Arc::clone(field))
6666
}
6767
DataType::LargeListView(field) => DataType::LargeList(Arc::clone(field)),
68+
DataType::Null => DataType::new_list(DataType::Null, true),
6869
_ => return plan_err!("{name} expected a list as first argument, got {list}"),
6970
};
7071

‎datafusion/functions-nested/src/map_extract.rs‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -105,6 +105,11 @@ impl ScalarUDFImpl for MapExtract {
105105

106106
fn return_type(&self, arg_types: &[DataType]) -> Result<DataType> {
107107
let [map_type, _] = take_function_args(self.name(), arg_types)?;
108+
109+
if map_type.is_null() {
110+
return Ok(DataType::Null);
111+
}
112+
108113
let map_fields = get_map_entry_field(map_type)?;
109114
Ok(DataType::List(Arc::new(Field::new_list_field(
110115
map_fields.last().unwrap().data_type().clone(),
@@ -123,6 +128,10 @@ impl ScalarUDFImpl for MapExtract {
123128
fn coerce_types(&self, arg_types: &[DataType]) -> Result<Vec<DataType>> {
124129
let [map_type, _] = take_function_args(self.name(), arg_types)?;
125130

131+
if map_type.is_null() {
132+
return Ok(arg_types.to_vec());
133+
}
134+
126135
let field = get_map_entry_field(map_type)?;
127136
Ok(vec![
128137
map_type.clone(),
@@ -185,6 +194,7 @@ fn map_extract_inner(args: &[ArrayRef]) -> Result<ArrayRef> {
185194

186195
let map_array = match map_arg.data_type() {
187196
DataType::Map(_, _) => as_map_array(&map_arg)?,
197+
DataType::Null => return Ok(Arc::clone(map_arg)),
188198
_ => return exec_err!("The first argument in map_extract must be a map"),
189199
};
190200

‎datafusion/spark/src/function/array/repeat.rs‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -74,9 +74,11 @@ impl ScalarUDFImpl for SparkArrayRepeat {
7474

7575
// Coerce the second argument to Int64/UInt64 if it's a numeric type
7676
let second = match second_type {
77-
DataType::Int8 | DataType::Int16 | DataType::Int32 | DataType::Int64 => {
78-
DataType::Int64
79-
}
77+
DataType::Int8
78+
| DataType::Int16
79+
| DataType::Int32
80+
| DataType::Int64
81+
| DataType::Null => DataType::Int64,
8082
DataType::UInt8 | DataType::UInt16 | DataType::UInt32 | DataType::UInt64 => {
8183
DataType::UInt64
8284
}

‎datafusion/sqllogictest/test_files/array/array_any_match.slt‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -103,6 +103,12 @@ SELECT list_any_match([1, 2, 3], x -> x > 2);
103103
----
104104
true
105105

106+
# null arg
107+
query B
108+
SELECT array_any_match(NULL, x -> x > 2);
109+
----
110+
NULL
111+
106112
statement ok
107113
drop table t;
108114

‎datafusion/sqllogictest/test_files/array/array_filter.slt‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -204,6 +204,12 @@ SELECT array_transform(array_filter(list, v -> v > 1), v -> v * 3) FROM with_nul
204204
[6]
205205
NULL
206206

207+
# null arg
208+
query ?
209+
SELECT array_filter(NULL, x -> x > 2);
210+
----
211+
NULL
212+
207213
statement ok
208214
drop table t;
209215

‎datafusion/sqllogictest/test_files/array/array_transform.slt‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -393,6 +393,12 @@ physical_plan
393393
02)--ProjectionExec: expr=[text@0 as text, list@1 as list, number@2 as number, CASE WHEN number@2 > 30 THEN array_transform(make_array(make_array(list@1)), (list) -> array_transform(list@3, (list) -> array_transform(list@4, (v) -> number@2 + v@5 + array_element(list@4, 1)))) ELSE array_transform(make_array(make_array(list@1)), (list) -> array_transform(list@3, (list) -> array_transform(list@4, (v) -> number@2 + array_element(list@4, 1)))) END as CASE WHEN t.number > Int64(30) THEN array_transform(make_array(make_array(t.list)),(list) -> array_transform(list,(list) -> array_transform(list,(v) -> t.number + v + list[Int64(1)]))) ELSE array_transform(make_array(make_array(t.list)),(list) -> array_transform(list,(list) -> array_transform(list,(v) -> t.number + list[Int64(1)]))) END]
394394
03)----DataSourceExec: partitions=1, partition_sizes=[1]
395395

396+
# null arg
397+
query ?
398+
SELECT array_transform(NULL, x -> x * 2);
399+
----
400+
NULL
401+
396402
query error
397403
select array_transform();
398404
----

‎datafusion/sqllogictest/test_files/map.slt‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -642,6 +642,12 @@ select map_extract(MAP {1: 1, 2: 2, 3:3}, '1'), map_extract(MAP {1: 1, 2: 2, 3:3
642642
----
643643
[1] [1] [1] [NULL] [1]
644644

645+
# null arg
646+
query ?
647+
select map_extract(NULL, 'a');
648+
----
649+
NULL
650+
645651
# map_extract with columns
646652
query ???
647653
select map_extract(column1, 1), map_extract(column1, 5), map_extract(column1, 7) from map_array_table_1;

‎datafusion/sqllogictest/test_files/spark/array/array_repeat.slt‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -112,3 +112,10 @@ FROM VALUES
112112
[[123], [123]]
113113
[[], []]
114114
[[NULL], [NULL]]
115+
116+
117+
# null count
118+
query ?
119+
select array_repeat('a', NULL);
120+
----
121+
NULL

0 commit comments

Comments
 (0)