Skip to content

Commit 33032d8

Browse files
kamilk-palhareshkh
andauthored
[branch-54] Expand scalar async UDF results to the batch row count (#25828) (#27372)
- Backport of #25828 to `branch-54` - Part of #23805 Co-authored-by: Haresh Khanna <[email protected]>
1 parent 31cb93f commit 33032d8

2 files changed

Lines changed: 34 additions & 11 deletions

File tree

‎datafusion/core/tests/user_defined/user_defined_async_scalar_functions.rs‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,35 @@ async fn test_async_udf_with_non_modular_batch_size() -> Result<()> {
8888
Ok(())
8989
}
9090

91+
// An async UDF may return a scalar when all of its arguments are scalars. The
92+
// scalar applies to every row, so it must be expanded to the number of rows in
93+
// each chunk rather than to a single row.
94+
#[tokio::test]
95+
async fn test_async_udf_with_scalar_result() -> Result<()> {
96+
let ctx = register_table_and_udf()?;
97+
98+
let df = ctx
99+
.sql("SELECT id, test_async_udf('constant') as result FROM test_table")
100+
.await?;
101+
102+
let result = df.collect().await?;
103+
104+
assert_batches_eq!(
105+
&[
106+
"+----+----------+",
107+
"| id | result |",
108+
"+----+----------+",
109+
"| 0 | constant |",
110+
"| 1 | constant |",
111+
"| 2 | constant |",
112+
"+----+----------+"
113+
],
114+
&result
115+
);
116+
117+
Ok(())
118+
}
119+
91120
// This test checks if metrics are printed for `AsyncFuncExec`
92121
#[tokio::test]
93122
async fn test_async_udf_metrics() -> Result<()> {

‎datafusion/physical-expr/src/async_scalar_function.rs‎

Lines changed: 5 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -165,7 +165,8 @@ impl AsyncFuncExpr {
165165
return_field: Arc::clone(&self.return_field),
166166
config_options: Arc::clone(&config_options),
167167
})
168-
.await?,
168+
.await?
169+
.into_array_of_size(current_batch.num_rows())?,
169170
);
170171
}
171172
} else {
@@ -184,20 +185,13 @@ impl AsyncFuncExpr {
184185
return_field: Arc::clone(&self.return_field),
185186
config_options: Arc::clone(&config_options),
186187
})
187-
.await?,
188+
.await?
189+
.into_array_of_size(batch.num_rows())?,
188190
);
189191
}
190192

191-
let datas = result_batches
192-
.into_iter()
193-
.map(|cv| match cv {
194-
ColumnarValue::Array(arr) => Ok(arr),
195-
ColumnarValue::Scalar(scalar) => Ok(scalar.to_array_of_size(1)?),
196-
})
197-
.collect::<Result<Vec<_>>>()?;
198-
199193
// Get references to the arrays as dyn Array to call concat
200-
let dyn_arrays = datas
194+
let dyn_arrays = result_batches
201195
.iter()
202196
.map(|arr| arr as &dyn arrow::array::Array)
203197
.collect::<Vec<_>>();

0 commit comments

Comments
 (0)