Skip to content

Commit 547bcac

Browse files
committed
Add ConfigOptions to ScalarFunctionArgs, refactor AsyncScalarUDF.invoke_async_with_args to remove ConfigOptions arg.
1 parent 1cb7bcb commit 547bcac

78 files changed

Lines changed: 610 additions & 364 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎datafusion-examples/examples/async_udf.rs‎

Lines changed: 1 addition & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,6 @@ use datafusion::common::cast::as_string_view_array;
2929
use datafusion::common::error::Result;
3030
use datafusion::common::not_impl_err;
3131
use datafusion::common::utils::take_function_args;
32-
use datafusion::config::ConfigOptions;
3332
use datafusion::execution::SessionStateBuilder;
3433
use datafusion::logical_expr::async_udf::{AsyncScalarUDF, AsyncScalarUDFImpl};
3534
use datafusion::logical_expr::{
@@ -195,11 +194,7 @@ impl AsyncScalarUDFImpl for AskLLM {
195194
/// is processing the query, so you may wish to make actual network requests
196195
/// on a different `Runtime`, as explained in the `thread_pools.rs` example
197196
/// in this directory.
198-
async fn invoke_async_with_args(
199-
&self,
200-
args: ScalarFunctionArgs,
201-
_option: &ConfigOptions,
202-
) -> Result<ArrayRef> {
197+
async fn invoke_async_with_args(&self, args: ScalarFunctionArgs) -> Result<ArrayRef> {
203198
// in a real UDF you would likely want to special case constant
204199
// arguments to improve performance, but this example converts the
205200
// arguments to arrays for simplicity.

‎datafusion/common/src/config.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -914,7 +914,7 @@ impl<'a> TryInto<arrow::util::display::FormatOptions<'a>> for &'a FormatOptions
914914
}
915915

916916
/// A key value pair, with a corresponding description
917-
#[derive(Debug)]
917+
#[derive(Debug, Hash, PartialEq, Eq)]
918918
pub struct ConfigEntry {
919919
/// A unique string to identify this config value
920920
pub key: String,

‎datafusion/core/tests/fuzz_cases/equivalence/ordering.rs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ use crate::fuzz_cases::equivalence::utils::{
2121
is_table_same_after_sort, TestScalarUDF,
2222
};
2323
use arrow::compute::SortOptions;
24+
use datafusion_common::config::ConfigOptions;
2425
use datafusion_common::Result;
2526
use datafusion_expr::{Operator, ScalarUDF};
2627
use datafusion_physical_expr::equivalence::{
@@ -110,6 +111,7 @@ fn test_ordering_satisfy_with_equivalence_complex_random() -> Result<()> {
110111
Arc::clone(&test_fun),
111112
vec![col_a],
112113
&test_schema,
114+
Arc::new(ConfigOptions::default()),
113115
)?);
114116
let a_plus_b = Arc::new(BinaryExpr::new(
115117
col("a", &test_schema)?,

‎datafusion/core/tests/fuzz_cases/equivalence/projection.rs‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ use crate::fuzz_cases::equivalence::utils::{
2020
is_table_same_after_sort, TestScalarUDF,
2121
};
2222
use arrow::compute::SortOptions;
23+
use datafusion_common::config::ConfigOptions;
2324
use datafusion_common::Result;
2425
use datafusion_expr::{Operator, ScalarUDF};
2526
use datafusion_physical_expr::equivalence::ProjectionMapping;
@@ -49,6 +50,7 @@ fn project_orderings_random() -> Result<()> {
4950
Arc::clone(&test_fun),
5051
vec![col_a],
5152
&test_schema,
53+
Arc::new(ConfigOptions::default()),
5254
)?);
5355
// a + b
5456
let a_plus_b = Arc::new(BinaryExpr::new(
@@ -122,6 +124,7 @@ fn ordering_satisfy_after_projection_random() -> Result<()> {
122124
Arc::clone(&test_fun),
123125
vec![col_a],
124126
&test_schema,
127+
Arc::new(ConfigOptions::default()),
125128
)?) as PhysicalExprRef;
126129
// a + b
127130
let a_plus_b = Arc::new(BinaryExpr::new(

‎datafusion/core/tests/fuzz_cases/equivalence/properties.rs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ use datafusion_physical_expr::expressions::{col, BinaryExpr};
2828
use datafusion_physical_expr::{LexOrdering, ScalarFunctionExpr};
2929
use datafusion_physical_expr_common::sort_expr::PhysicalSortExpr;
3030

31+
use datafusion_common::config::ConfigOptions;
3132
use itertools::Itertools;
3233

3334
#[test]
@@ -49,6 +50,7 @@ fn test_find_longest_permutation_random() -> Result<()> {
4950
Arc::clone(&test_fun),
5051
vec![col_a],
5152
&test_schema,
53+
Arc::new(ConfigOptions::default()),
5254
)?) as _;
5355

5456
let a_plus_b = Arc::new(BinaryExpr::new(

‎datafusion/core/tests/physical_optimizer/projection_pushdown.rs‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -128,6 +128,7 @@ fn test_update_matching_exprs() -> Result<()> {
128128
)),
129129
],
130130
Field::new("f", DataType::Int32, true).into(),
131+
Arc::new(ConfigOptions::default()),
131132
)),
132133
Arc::new(CaseExpr::try_new(
133134
Some(Arc::new(Column::new("d", 2))),
@@ -193,6 +194,7 @@ fn test_update_matching_exprs() -> Result<()> {
193194
)),
194195
],
195196
Field::new("f", DataType::Int32, true).into(),
197+
Arc::new(ConfigOptions::default()),
196198
)),
197199
Arc::new(CaseExpr::try_new(
198200
Some(Arc::new(Column::new("d", 3))),
@@ -261,6 +263,7 @@ fn test_update_projected_exprs() -> Result<()> {
261263
)),
262264
],
263265
Field::new("f", DataType::Int32, true).into(),
266+
Arc::new(ConfigOptions::default()),
264267
)),
265268
Arc::new(CaseExpr::try_new(
266269
Some(Arc::new(Column::new("d", 2))),
@@ -326,6 +329,7 @@ fn test_update_projected_exprs() -> Result<()> {
326329
)),
327330
],
328331
Field::new("f", DataType::Int32, true).into(),
332+
Arc::new(ConfigOptions::default()),
329333
)),
330334
Arc::new(CaseExpr::try_new(
331335
Some(Arc::new(Column::new("d_new", 3))),

‎datafusion/expr/src/async_udf.rs‎

Lines changed: 2 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,6 @@ use crate::{
2222
use arrow::array::ArrayRef;
2323
use arrow::datatypes::{DataType, FieldRef};
2424
use async_trait::async_trait;
25-
use datafusion_common::config::ConfigOptions;
2625
use datafusion_common::error::Result;
2726
use datafusion_common::internal_err;
2827
use datafusion_expr_common::columnar_value::ColumnarValue;
@@ -49,11 +48,7 @@ pub trait AsyncScalarUDFImpl: ScalarUDFImpl {
4948
}
5049

5150
/// Invoke the function asynchronously with the async arguments
52-
async fn invoke_async_with_args(
53-
&self,
54-
args: ScalarFunctionArgs,
55-
option: &ConfigOptions,
56-
) -> Result<ArrayRef>;
51+
async fn invoke_async_with_args(&self, args: ScalarFunctionArgs) -> Result<ArrayRef>;
5752
}
5853

5954
/// A scalar UDF that must be invoked using async methods
@@ -100,9 +95,8 @@ impl AsyncScalarUDF {
10095
pub async fn invoke_async_with_args(
10196
&self,
10297
args: ScalarFunctionArgs,
103-
option: &ConfigOptions,
10498
) -> Result<ArrayRef> {
105-
self.inner.invoke_async_with_args(args, option).await
99+
self.inner.invoke_async_with_args(args).await
106100
}
107101
}
108102

‎datafusion/expr/src/execution_props.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -118,6 +118,6 @@ mod test {
118118
#[test]
119119
fn debug() {
120120
let props = ExecutionProps::new();
121-
assert_eq!("ExecutionProps { query_execution_start_time: 1970-01-01T00:00:00Z, alias_generator: AliasGenerator { next_id: 1 }, var_providers: None }", format!("{props:?}"));
121+
assert_eq!("ExecutionProps { query_execution_start_time: 1970-01-01T00:00:00Z, alias_generator: AliasGenerator { next_id: 1 }, config_options: None, var_providers: None }", format!("{props:?}"));
122122
}
123123
}

‎datafusion/expr/src/udf.rs‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ use crate::simplify::{ExprSimplifyResult, SimplifyInfo};
2323
use crate::sort_properties::{ExprProperties, SortProperties};
2424
use crate::{udf_equals_hash, ColumnarValue, Documentation, Expr, Signature};
2525
use arrow::datatypes::{DataType, Field, FieldRef};
26+
use datafusion_common::config::ConfigOptions;
2627
use datafusion_common::{not_impl_err, ExprSchema, Result, ScalarValue};
2728
use datafusion_expr_common::interval_arithmetic::Interval;
2829
use std::any::Any;
@@ -311,6 +312,8 @@ pub struct ScalarFunctionArgs {
311312
/// or `return_field_from_args`) when creating the physical expression
312313
/// from the logical expression
313314
pub return_field: FieldRef,
315+
/// The config options at execution time
316+
pub config_options: Arc<ConfigOptions>,
314317
}
315318

316319
impl ScalarFunctionArgs {

‎datafusion/ffi/src/udf/mod.rs‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ use arrow::{
3232
ffi::{from_ffi, to_ffi, FFI_ArrowSchema},
3333
};
3434
use arrow_schema::FieldRef;
35+
use datafusion::config::ConfigOptions;
3536
use datafusion::logical_expr::{udf_equals_hash, ReturnFieldArgs};
3637
use datafusion::{
3738
error::DataFusionError,
@@ -207,6 +208,8 @@ unsafe extern "C" fn invoke_with_args_fn_wrapper(
207208
arg_fields,
208209
number_rows,
209210
return_field,
211+
// todo - should the config options go through serdes?
212+
config_options: Arc::new(ConfigOptions::default()),
210213
};
211214

212215
let result = rresult_return!(udf
@@ -378,6 +381,8 @@ impl ScalarUDFImpl for ForeignScalarUDF {
378381
arg_fields,
379382
number_rows,
380383
return_field,
384+
// todo - should the config options go through serdes?
385+
config_options: _config_options,
381386
} = invoke_args;
382387

383388
let args = args

0 commit comments

Comments
 (0)