Skip to content

Commit a2d853b

Browse files
committed
Add support for calling async UDF as aggregation expression
Fixes apache#17619
1 parent 986cfcd commit a2d853b

3 files changed

Lines changed: 152 additions & 4 deletions

File tree

‎datafusion/core/src/physical_planner.rs‎

Lines changed: 48 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -741,9 +741,54 @@ impl DefaultPhysicalPlanner {
741741
})
742742
.collect::<Result<Vec<_>>>()?;
743743

744-
let (aggregates, filters, _order_bys): (Vec<_>, Vec<_>, Vec<_>) =
744+
let (mut aggregates, filters, _order_bys): (Vec<_>, Vec<_>, Vec<_>) =
745745
multiunzip(agg_filter);
746746

747+
let mut async_exprs = Vec::new();
748+
let num_input_columns = physical_input_schema.fields().len();
749+
750+
for agg_func in &mut aggregates {
751+
match self.try_plan_async_exprs(
752+
num_input_columns,
753+
PlannedExprResult::Expr(agg_func.expressions()),
754+
physical_input_schema.as_ref(),
755+
)? {
756+
PlanAsyncExpr::Async(
757+
async_map,
758+
PlannedExprResult::Expr(physical_exprs),
759+
) => {
760+
async_exprs.extend(async_map.async_exprs);
761+
762+
if let Some(new_agg_func) = agg_func.with_new_expressions(
763+
physical_exprs,
764+
agg_func
765+
.order_bys()
766+
.iter()
767+
.cloned()
768+
.map(|x| x.expr)
769+
.collect(),
770+
) {
771+
*agg_func = Arc::new(new_agg_func);
772+
} else {
773+
return internal_err!("Failed to plan async expression");
774+
}
775+
}
776+
PlanAsyncExpr::Sync(PlannedExprResult::Expr(_)) => {
777+
// Do nothing
778+
}
779+
_ => {
780+
return internal_err!(
781+
"Unexpected result from try_plan_async_exprs"
782+
)
783+
}
784+
}
785+
}
786+
let input_exec = if !async_exprs.is_empty() {
787+
Arc::new(AsyncFuncExec::try_new(async_exprs, input_exec)?)
788+
} else {
789+
input_exec
790+
};
791+
747792
let initial_aggr = Arc::new(AggregateExec::try_new(
748793
AggregateMode::Partial,
749794
groups.clone(),
@@ -2272,11 +2317,13 @@ impl DefaultPhysicalPlanner {
22722317
}
22732318
}
22742319

2320+
#[derive(Debug)]
22752321
enum PlannedExprResult {
22762322
ExprWithName(Vec<(Arc<dyn PhysicalExpr>, String)>),
22772323
Expr(Vec<Arc<dyn PhysicalExpr>>),
22782324
}
22792325

2326+
#[derive(Debug)]
22802327
enum PlanAsyncExpr {
22812328
Sync(PlannedExprResult),
22822329
Async(AsyncMapper, PlannedExprResult),

‎datafusion/sqllogictest/src/test_context.rs‎

Lines changed: 58 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
// specific language governing permissions and limitations
1616
// under the License.
1717

18+
use std::any::Any;
1819
use std::collections::HashMap;
1920
use std::fs::File;
2021
use std::io::Write;
@@ -31,8 +32,13 @@ use arrow::record_batch::RecordBatch;
3132
use datafusion::catalog::{
3233
CatalogProvider, MemoryCatalogProvider, MemorySchemaProvider, Session,
3334
};
34-
use datafusion::common::DataFusionError;
35-
use datafusion::logical_expr::{create_udf, ColumnarValue, Expr, ScalarUDF, Volatility};
35+
use datafusion::common::{not_impl_err, DataFusionError, Result};
36+
use datafusion::functions::math::abs;
37+
use datafusion::logical_expr::async_udf::{AsyncScalarUDF, AsyncScalarUDFImpl};
38+
use datafusion::logical_expr::{
39+
create_udf, ColumnarValue, Expr, ScalarFunctionArgs, ScalarUDF, ScalarUDFImpl,
40+
Signature, Volatility,
41+
};
3642
use datafusion::physical_plan::ExecutionPlan;
3743
use datafusion::prelude::*;
3844
use datafusion::{
@@ -133,6 +139,10 @@ impl TestContext {
133139
info!("Registering table with union column");
134140
register_union_table(test_ctx.session_ctx())
135141
}
142+
"async_udf.slt" => {
143+
info!("Registering dummy async udf");
144+
register_async_abs_udf(test_ctx.session_ctx())
145+
}
136146
_ => {
137147
info!("Using default SessionContext");
138148
}
@@ -235,7 +245,7 @@ pub async fn register_temp_table(ctx: &SessionContext) {
235245

236246
#[async_trait]
237247
impl TableProvider for TestTable {
238-
fn as_any(&self) -> &dyn std::any::Any {
248+
fn as_any(&self) -> &dyn Any {
239249
self
240250
}
241251

@@ -458,3 +468,48 @@ fn register_union_table(ctx: &SessionContext) {
458468

459469
ctx.register_batch("union_table", batch).unwrap();
460470
}
471+
472+
fn register_async_abs_udf(ctx: &SessionContext) {
473+
#[derive(Debug, PartialEq, Eq, Hash)]
474+
struct AsyncAbs {
475+
inner_abs: Arc<ScalarUDF>,
476+
}
477+
impl AsyncAbs {
478+
fn new() -> Self {
479+
AsyncAbs { inner_abs: abs() }
480+
}
481+
}
482+
impl ScalarUDFImpl for AsyncAbs {
483+
fn as_any(&self) -> &dyn Any {
484+
self
485+
}
486+
487+
fn name(&self) -> &str {
488+
"async_abs"
489+
}
490+
491+
fn signature(&self) -> &Signature {
492+
self.inner_abs.signature()
493+
}
494+
495+
fn return_type(&self, arg_types: &[DataType]) -> Result<DataType> {
496+
self.inner_abs.return_type(arg_types)
497+
}
498+
499+
fn invoke_with_args(&self, _args: ScalarFunctionArgs) -> Result<ColumnarValue> {
500+
not_impl_err!("{} can only be called from async contexts", self.name())
501+
}
502+
}
503+
#[async_trait]
504+
impl AsyncScalarUDFImpl for AsyncAbs {
505+
async fn invoke_async_with_args(
506+
&self,
507+
args: ScalarFunctionArgs,
508+
) -> Result<ColumnarValue> {
509+
return self.inner_abs.invoke_with_args(args);
510+
}
511+
}
512+
let async_abs = AsyncAbs::new();
513+
let udf = AsyncScalarUDF::new(Arc::new(async_abs));
514+
ctx.register_udf(udf.into_scalar_udf());
515+
}
Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
2+
# Licensed to the Apache Software Foundation (ASF) under one
3+
# or more contributor license agreements. See the NOTICE file
4+
# distributed with this work for additional information
5+
# regarding copyright ownership. The ASF licenses this file
6+
# to you under the Apache License, Version 2.0 (the
7+
# "License"); you may not use this file except in compliance
8+
# with the License. You may obtain a copy of the License at
9+
10+
# http://www.apache.org/licenses/LICENSE-2.0
11+
12+
# Unless required by applicable law or agreed to in writing,
13+
# software distributed under the License is distributed on an
14+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
# KIND, either express or implied. See the License for the
16+
# specific language governing permissions and limitations
17+
# under the License.
18+
19+
statement ok
20+
create table data(x int) as values (-10), (2);
21+
22+
# Async udf can be used in aggregation
23+
query I
24+
select min(async_abs(x)) from data;
25+
----
26+
2
27+
28+
# Async udf can be used in aggregation with group by
29+
query I rowsort
30+
select min(async_abs(x)) from data group by async_abs(x);
31+
----
32+
10
33+
2
34+
35+
# Async udf can be used in filter
36+
query I
37+
select * from data where async_abs(x) < 5;
38+
----
39+
2
40+
41+
# Async udf can be used in projection
42+
query I rowsort
43+
select async_abs(x) from data;
44+
----
45+
10
46+
2

0 commit comments

Comments
 (0)