Skip to content

Commit 2bc02b9

Browse files
hareshkhOmega359
authored andcommitted
Expand scalar async UDF results to the batch row count (apache#25828)
## Which issue does this PR close? - Closes apache#25875 ## Rationale for this change When an async UDF returns a `ColumnarValue::Scalar` (e.g. all arguments are literals), queries on more than one row fail with `all columns in a record batch must have the same length`. ## What changes are included in this PR? `AsyncFuncExpr::invoke_with_args` expands each result to its chunk's row count via `into_array_of_size`, instead of turning a scalar into a 1-row array. ## What is the testing strategy for this PR? Added `test_async_udf_with_scalar_result` in `user_defined_async_scalar_functions.rs`. It fails without the fix. ## Are there any user-facing changes? No.
1 parent fd0a945 commit 2bc02b9

2 files changed

Lines changed: 34 additions & 11 deletions

File tree

‎datafusion/core/tests/user_defined/user_defined_async_scalar_functions.rs‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,35 @@ async fn test_async_udf_with_non_modular_batch_size() -> Result<()> {
8989
Ok(())
9090
}
9191

92+
// An async UDF may return a scalar when all of its arguments are scalars. The
93+
// scalar applies to every row, so it must be expanded to the number of rows in
94+
// each chunk rather than to a single row.
95+
#[tokio::test]
96+
async fn test_async_udf_with_scalar_result() -> Result<()> {
97+
let ctx = register_table_and_udf()?;
98+
99+
let df = ctx
100+
.sql("SELECT id, test_async_udf('constant') as result FROM test_table")
101+
.await?;
102+
103+
let result = df.collect().await?;
104+
105+
assert_batches_eq!(
106+
&[
107+
"+----+----------+",
108+
"| id | result |",
109+
"+----+----------+",
110+
"| 0 | constant |",
111+
"| 1 | constant |",
112+
"| 2 | constant |",
113+
"+----+----------+"
114+
],
115+
&result
116+
);
117+
118+
Ok(())
119+
}
120+
92121
// This test checks if metrics are printed for `AsyncFuncExec`
93122
#[tokio::test]
94123
async fn test_async_udf_metrics() -> Result<()> {

‎datafusion/physical-expr/src/async_scalar_function.rs‎

Lines changed: 5 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -162,7 +162,8 @@ impl AsyncFuncExpr {
162162
return_field: Arc::clone(&self.return_field),
163163
config_options: Arc::clone(&config_options),
164164
})
165-
.await?,
165+
.await?
166+
.into_array_of_size(current_batch.num_rows())?,
166167
);
167168
}
168169
} else {
@@ -181,20 +182,13 @@ impl AsyncFuncExpr {
181182
return_field: Arc::clone(&self.return_field),
182183
config_options: Arc::clone(&config_options),
183184
})
184-
.await?,
185+
.await?
186+
.into_array_of_size(batch.num_rows())?,
185187
);
186188
}
187189

188-
let datas = result_batches
189-
.into_iter()
190-
.map(|cv| match cv {
191-
ColumnarValue::Array(arr) => Ok(arr),
192-
ColumnarValue::Scalar(scalar) => Ok(scalar.to_array_of_size(1)?),
193-
})
194-
.collect::<Result<Vec<_>>>()?;
195-
196190
// Get references to the arrays as dyn Array to call concat
197-
let dyn_arrays = datas
191+
let dyn_arrays = result_batches
198192
.iter()
199193
.map(|arr| arr as &dyn arrow::array::Array)
200194
.collect::<Vec<_>>();

0 commit comments

Comments
 (0)