Skip to content

Commit 34834dc

Browse files
authored
refactor: wrap HigherOrderUDFImpl in a concrete HigherOrderUDF struct (apache#22593)
## Which issue does this PR close? <!-- We generally require a GitHub issue to be filed for all bug fixes and enhancements and this helps us generate change logs for our releases. You can link an issue to this PR using the GitHub syntax. For example `Closes apache#123` indicates that this PR will close issue apache#123. --> Part of apache#21172 ## Rationale for this change `HigherOrderUDF` was the only UDF kind defined as a trait that callers used directly via `Arc<dyn HigherOrderUDF>`. The other UDFs: `ScalarUDF`, `AggregateUDF`, `WindowUDF` — are concrete structs that wrap their respective `*Impl trait`, which makes inherent methods like `with_aliases` ergonomic to call on the function object. With the trait-only setup, adding aliases to an existing higher-order function required an extension trait import or a free helper function. This PR brings higher order functions in line with the other UDFs so the same `with_aliases` pattern works. ## What changes are included in this PR? - Rename the `HigherOrderUDF` trait to `HigherOrderUDFImpl`, matching `ScalarUDFImpl`/`AggregateUDFImpl`. Add a concrete `HigherOrderUDF` struct wrapping `Arc<dyn HigherOrderUDFImpl>`, with the same shape as `ScalarUDF`: new_from_impl, new_from_shared_impl, inner, with_aliases, From<F: HigherOrderUDFImpl>, and delegate methods for every trait method. `with_aliases` is backed by a private `AliasedHigherOrderUDFImpl` decorator (same pattern as `AliasedScalarUDFImpl`). - Update `Expr::HigherOrderFunction`, `FunctionRegistry`, the `create_higher_order! `singleton macro, and all consumer files ( across several crates) to use `Arc<HigherOrderUDF>` instead of `Arc<dyn HigherOrderUDF>`. Existing impls (`ArrayFilter`, `ArrayTransform`, `ArrayAnyMatch`) now implement `HigherOrderUDFImpl`; their public constructors continue to return `Arc<HigherOrderUDF>` so external call sites need no changes. Callers can now write: `array_filter_higher_order_function().with_aliases(["filter"]) ` exactly like the existing scalar pattern: `make_array_udf().as_ref().clone().with_aliases(["array_construct"]) ` ## Are these changes tested? Covered by existing tests ## Are there any user-facing changes? Yes, any code referring to `Arc<dyn HigherOrderUDF>` needs to become `Arc<HigherOrderUDF>`, and any code that wrote `impl HigherOrderUDF for MyHOF` needs to write i`mpl HigherOrderUDFImpl for MyType`. Constructing a HigherOrderUDF from an impl is HigherOrderUDF::new_from_impl(my_impl) (or my_impl.into()).
1 parent a7c2f7d commit 34834dc

36 files changed

Lines changed: 464 additions & 167 deletions

File tree

‎datafusion-examples/examples/sql_ops/frontend.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -154,7 +154,7 @@ impl ContextProvider for MyContextProvider {
154154
None
155155
}
156156

157-
fn get_higher_order_meta(&self, _name: &str) -> Option<Arc<dyn HigherOrderUDF>> {
157+
fn get_higher_order_meta(&self, _name: &str) -> Option<Arc<HigherOrderUDF>> {
158158
None
159159
}
160160

‎datafusion/core/src/bin/print_functions_docs.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -287,7 +287,7 @@ impl DocProvider for WindowUDF {
287287
}
288288
}
289289

290-
impl DocProvider for Arc<dyn HigherOrderUDF> {
290+
impl DocProvider for Arc<HigherOrderUDF> {
291291
fn get_name(&self) -> String {
292292
self.name().to_string()
293293
}

‎datafusion/core/src/datasource/listing_table_factory.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -587,7 +587,7 @@ mod tests {
587587
}
588588
fn higher_order_functions(
589589
&self,
590-
) -> &HashMap<String, Arc<dyn datafusion_expr::HigherOrderUDF>> {
590+
) -> &HashMap<String, Arc<datafusion_expr::HigherOrderUDF>> {
591591
unimplemented!()
592592
}
593593
fn aggregate_functions(

‎datafusion/core/src/execution/context/mod.rs‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1627,7 +1627,7 @@ impl SessionContext {
16271627
/// - `SELECT "my_HIGHER_ORDER_FUNC"(x)` will look for a function named `"my_HIGHER_ORDER_FUNC"`
16281628
///
16291629
/// Any functions registered with the function name or its aliases will be overwritten with this new function
1630-
pub fn register_higher_order_function(&self, f: Arc<dyn HigherOrderUDF>) {
1630+
pub fn register_higher_order_function(&self, f: Arc<HigherOrderUDF>) {
16311631
let mut state = self.state.write();
16321632
state.register_higher_order_function(f).ok();
16331633
}
@@ -2064,7 +2064,7 @@ impl FunctionRegistry for SessionContext {
20642064
self.state.read().udf(name)
20652065
}
20662066

2067-
fn higher_order_function(&self, name: &str) -> Result<Arc<dyn HigherOrderUDF>> {
2067+
fn higher_order_function(&self, name: &str) -> Result<Arc<HigherOrderUDF>> {
20682068
self.state.read().higher_order_function(name)
20692069
}
20702070

@@ -2082,8 +2082,8 @@ impl FunctionRegistry for SessionContext {
20822082

20832083
fn register_higher_order_function(
20842084
&mut self,
2085-
function: Arc<dyn HigherOrderUDF>,
2086-
) -> Result<Option<Arc<dyn HigherOrderUDF>>> {
2085+
function: Arc<HigherOrderUDF>,
2086+
) -> Result<Option<Arc<HigherOrderUDF>>> {
20872087
self.state.write().register_higher_order_function(function)
20882088
}
20892089

@@ -2222,7 +2222,7 @@ pub enum RegisterFunction {
22222222
/// Window user defined function
22232223
Window(Arc<WindowUDF>),
22242224
/// Higher-order user defined function
2225-
HigherOrder(Arc<dyn HigherOrderUDF>),
2225+
HigherOrder(Arc<HigherOrderUDF>),
22262226
/// Table user defined function
22272227
Table(String, Arc<dyn TableFunctionImpl>),
22282228
}

‎datafusion/core/src/execution/session_state.rs‎

Lines changed: 12 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -162,7 +162,7 @@ pub struct SessionState {
162162
/// Scalar functions that are registered with the context
163163
scalar_functions: HashMap<String, Arc<ScalarUDF>>,
164164
/// Higher order functions that are registered with the context
165-
higher_order_functions: HashMap<String, Arc<dyn HigherOrderUDF>>,
165+
higher_order_functions: HashMap<String, Arc<HigherOrderUDF>>,
166166
/// Aggregate functions registered in the context
167167
aggregate_functions: HashMap<String, Arc<AggregateUDF>>,
168168
/// Window functions registered in the context
@@ -286,7 +286,7 @@ impl Session for SessionState {
286286
&self.scalar_functions
287287
}
288288

289-
fn higher_order_functions(&self) -> &HashMap<String, Arc<dyn HigherOrderUDF>> {
289+
fn higher_order_functions(&self) -> &HashMap<String, Arc<HigherOrderUDF>> {
290290
&self.higher_order_functions
291291
}
292292

@@ -936,7 +936,7 @@ impl SessionState {
936936
}
937937

938938
/// Return reference to higher_order_functions
939-
pub fn higher_order_functions(&self) -> &HashMap<String, Arc<dyn HigherOrderUDF>> {
939+
pub fn higher_order_functions(&self) -> &HashMap<String, Arc<HigherOrderUDF>> {
940940
&self.higher_order_functions
941941
}
942942

@@ -1036,7 +1036,7 @@ pub struct SessionStateBuilder {
10361036
catalog_list: Option<Arc<dyn CatalogProviderList>>,
10371037
table_functions: Option<HashMap<String, Arc<TableFunction>>>,
10381038
scalar_functions: Option<Vec<Arc<ScalarUDF>>>,
1039-
higher_order_functions: Option<Vec<Arc<dyn HigherOrderUDF>>>,
1039+
higher_order_functions: Option<Vec<Arc<HigherOrderUDF>>>,
10401040
aggregate_functions: Option<Vec<Arc<AggregateUDF>>>,
10411041
window_functions: Option<Vec<Arc<WindowUDF>>>,
10421042
extension_types: Option<ExtensionTypeRegistryRef>,
@@ -1373,7 +1373,7 @@ impl SessionStateBuilder {
13731373
/// Set the map of [`HigherOrderUDF`]s
13741374
pub fn with_higher_order_functions(
13751375
mut self,
1376-
higher_order_functions: Vec<Arc<dyn HigherOrderUDF>>,
1376+
higher_order_functions: Vec<Arc<HigherOrderUDF>>,
13771377
) -> Self {
13781378
self.higher_order_functions = Some(higher_order_functions);
13791379
self
@@ -1793,9 +1793,7 @@ impl SessionStateBuilder {
17931793
}
17941794

17951795
/// Returns the current scalar_functions value
1796-
pub fn higher_order_functions(
1797-
&mut self,
1798-
) -> &mut Option<Vec<Arc<dyn HigherOrderUDF>>> {
1796+
pub fn higher_order_functions(&mut self) -> &mut Option<Vec<Arc<HigherOrderUDF>>> {
17991797
&mut self.higher_order_functions
18001798
}
18011799

@@ -2018,7 +2016,7 @@ impl ContextProvider for SessionContextProvider<'_> {
20182016
self.state.scalar_functions().get(name).cloned()
20192017
}
20202018

2021-
fn get_higher_order_meta(&self, name: &str) -> Option<Arc<dyn HigherOrderUDF>> {
2019+
fn get_higher_order_meta(&self, name: &str) -> Option<Arc<HigherOrderUDF>> {
20222020
self.state.higher_order_functions().get(name).cloned()
20232021
}
20242022

@@ -2108,7 +2106,7 @@ impl FunctionRegistry for SessionState {
21082106
fn higher_order_function(
21092107
&self,
21102108
name: &str,
2111-
) -> datafusion_common::Result<Arc<dyn HigherOrderUDF>> {
2109+
) -> datafusion_common::Result<Arc<HigherOrderUDF>> {
21122110
self.higher_order_functions
21132111
.get(name)
21142112
.cloned()
@@ -2144,8 +2142,8 @@ impl FunctionRegistry for SessionState {
21442142

21452143
fn register_higher_order_function(
21462144
&mut self,
2147-
function: Arc<dyn HigherOrderUDF>,
2148-
) -> datafusion_common::Result<Option<Arc<dyn HigherOrderUDF>>> {
2145+
function: Arc<HigherOrderUDF>,
2146+
) -> datafusion_common::Result<Option<Arc<HigherOrderUDF>>> {
21492147
function.aliases().iter().for_each(|alias| {
21502148
self.higher_order_functions
21512149
.insert(alias.clone(), Arc::clone(&function));
@@ -2193,7 +2191,7 @@ impl FunctionRegistry for SessionState {
21932191
fn deregister_higher_order_function(
21942192
&mut self,
21952193
name: &str,
2196-
) -> datafusion_common::Result<Option<Arc<dyn HigherOrderUDF>>> {
2194+
) -> datafusion_common::Result<Option<Arc<HigherOrderUDF>>> {
21972195
let function = self.higher_order_functions.remove(name);
21982196
if let Some(function) = &function {
21992197
for alias in function.aliases() {
@@ -2679,7 +2677,7 @@ mod tests {
26792677
self.state.scalar_functions().get(name).cloned()
26802678
}
26812679

2682-
fn get_higher_order_meta(&self, name: &str) -> Option<Arc<dyn HigherOrderUDF>> {
2680+
fn get_higher_order_meta(&self, name: &str) -> Option<Arc<HigherOrderUDF>> {
26832681
self.state.higher_order_functions().get(name).cloned()
26842682
}
26852683

‎datafusion/core/src/execution/session_state_defaults.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -114,7 +114,7 @@ impl SessionStateDefaults {
114114
}
115115

116116
/// returns the list of default [`HigherOrderUDF`]s
117-
pub fn default_higher_order_functions() -> Vec<Arc<dyn HigherOrderUDF>> {
117+
pub fn default_higher_order_functions() -> Vec<Arc<HigherOrderUDF>> {
118118
#[cfg(feature = "nested_expressions")]
119119
return functions_nested::all_default_higher_order_functions();
120120

‎datafusion/core/tests/optimizer/mod.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -216,7 +216,7 @@ impl ContextProvider for MyContextProvider {
216216
self.udfs.get(name).cloned()
217217
}
218218

219-
fn get_higher_order_meta(&self, _name: &str) -> Option<Arc<dyn HigherOrderUDF>> {
219+
fn get_higher_order_meta(&self, _name: &str) -> Option<Arc<HigherOrderUDF>> {
220220
None
221221
}
222222

‎datafusion/datasource-arrow/src/file_format.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -593,7 +593,7 @@ mod tests {
593593
unimplemented!()
594594
}
595595

596-
fn higher_order_functions(&self) -> &HashMap<String, Arc<dyn HigherOrderUDF>> {
596+
fn higher_order_functions(&self) -> &HashMap<String, Arc<HigherOrderUDF>> {
597597
unimplemented!()
598598
}
599599

‎datafusion/datasource/src/url.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1210,7 +1210,7 @@ mod tests {
12101210
unimplemented!()
12111211
}
12121212

1213-
fn higher_order_functions(&self) -> &HashMap<String, Arc<dyn HigherOrderUDF>> {
1213+
fn higher_order_functions(&self) -> &HashMap<String, Arc<HigherOrderUDF>> {
12141214
unimplemented!()
12151215
}
12161216

‎datafusion/execution/src/task.rs‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,7 @@ pub struct TaskContext {
5959
/// Scalar functions associated with this task context
6060
scalar_functions: HashMap<String, Arc<ScalarUDF>>,
6161
/// Higher order functions associated with this task context
62-
higher_order_functions: HashMap<String, Arc<dyn HigherOrderUDF>>,
62+
higher_order_functions: HashMap<String, Arc<HigherOrderUDF>>,
6363
/// Aggregate functions associated with this task context
6464
aggregate_functions: HashMap<String, Arc<AggregateUDF>>,
6565
/// Window functions associated with this task context
@@ -98,7 +98,7 @@ impl TaskContext {
9898
session_id: String,
9999
session_config: SessionConfig,
100100
scalar_functions: HashMap<String, Arc<ScalarUDF>>,
101-
higher_order_functions: HashMap<String, Arc<dyn HigherOrderUDF>>,
101+
higher_order_functions: HashMap<String, Arc<HigherOrderUDF>>,
102102
aggregate_functions: HashMap<String, Arc<AggregateUDF>>,
103103
window_functions: HashMap<String, Arc<WindowUDF>>,
104104
runtime: Arc<RuntimeEnv>,
@@ -144,7 +144,7 @@ impl TaskContext {
144144
&self.scalar_functions
145145
}
146146

147-
pub fn higher_order_functions(&self) -> &HashMap<String, Arc<dyn HigherOrderUDF>> {
147+
pub fn higher_order_functions(&self) -> &HashMap<String, Arc<HigherOrderUDF>> {
148148
&self.higher_order_functions
149149
}
150150

@@ -182,7 +182,7 @@ impl FunctionRegistry for TaskContext {
182182
})
183183
}
184184

185-
fn higher_order_function(&self, name: &str) -> Result<Arc<dyn HigherOrderUDF>> {
185+
fn higher_order_function(&self, name: &str) -> Result<Arc<HigherOrderUDF>> {
186186
let result = self.higher_order_functions.get(name);
187187

188188
result.cloned().ok_or_else(|| {
@@ -236,8 +236,8 @@ impl FunctionRegistry for TaskContext {
236236

237237
fn register_higher_order_function(
238238
&mut self,
239-
function: Arc<dyn HigherOrderUDF>,
240-
) -> Result<Option<Arc<dyn HigherOrderUDF>>> {
239+
function: Arc<HigherOrderUDF>,
240+
) -> Result<Option<Arc<HigherOrderUDF>>> {
241241
function.aliases().iter().for_each(|alias| {
242242
self.higher_order_functions
243243
.insert(alias.clone(), Arc::clone(&function));

0 commit comments

Comments
 (0)