Skip to content

Commit 44a258e

Browse files
kamilk-palhareshkh
andauthored
[branch-55] Expand scalar async UDF results to the batch row count (#25828) (#27373)
- Backport of #25828 to `branch-55` - Part of #26868 Co-authored-by: Haresh Khanna <[email protected]>
1 parent f020c02 commit 44a258e

2 files changed

Lines changed: 34 additions & 11 deletions

File tree

‎datafusion/core/tests/user_defined/user_defined_async_scalar_functions.rs‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,35 @@ async fn test_async_udf_with_non_modular_batch_size() -> Result<()> {
8989
Ok(())
9090
}
9191

92+
// An async UDF may return a scalar when all of its arguments are scalars. The
93+
// scalar applies to every row, so it must be expanded to the number of rows in
94+
// each chunk rather than to a single row.
95+
#[tokio::test]
96+
async fn test_async_udf_with_scalar_result() -> Result<()> {
97+
let ctx = register_table_and_udf()?;
98+
99+
let df = ctx
100+
.sql("SELECT id, test_async_udf('constant') as result FROM test_table")
101+
.await?;
102+
103+
let result = df.collect().await?;
104+
105+
assert_batches_eq!(
106+
&[
107+
"+----+----------+",
108+
"| id | result |",
109+
"+----+----------+",
110+
"| 0 | constant |",
111+
"| 1 | constant |",
112+
"| 2 | constant |",
113+
"+----+----------+"
114+
],
115+
&result
116+
);
117+
118+
Ok(())
119+
}
120+
92121
// This test checks if metrics are printed for `AsyncFuncExec`
93122
#[tokio::test]
94123
async fn test_async_udf_metrics() -> Result<()> {

‎datafusion/physical-expr/src/async_scalar_function.rs‎

Lines changed: 5 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -162,7 +162,8 @@ impl AsyncFuncExpr {
162162
return_field: Arc::clone(&self.return_field),
163163
config_options: Arc::clone(&config_options),
164164
})
165-
.await?,
165+
.await?
166+
.into_array_of_size(current_batch.num_rows())?,
166167
);
167168
}
168169
} else {
@@ -181,20 +182,13 @@ impl AsyncFuncExpr {
181182
return_field: Arc::clone(&self.return_field),
182183
config_options: Arc::clone(&config_options),
183184
})
184-
.await?,
185+
.await?
186+
.into_array_of_size(batch.num_rows())?,
185187
);
186188
}
187189

188-
let datas = result_batches
189-
.into_iter()
190-
.map(|cv| match cv {
191-
ColumnarValue::Array(arr) => Ok(arr),
192-
ColumnarValue::Scalar(scalar) => Ok(scalar.to_array_of_size(1)?),
193-
})
194-
.collect::<Result<Vec<_>>>()?;
195-
196190
// Get references to the arrays as dyn Array to call concat
197-
let dyn_arrays = datas
191+
let dyn_arrays = result_batches
198192
.iter()
199193
.map(|arr| arr as &dyn arrow::array::Array)
200194
.collect::<Vec<_>>();

0 commit comments

Comments
 (0)