Skip to content

Commit ac279eb

Browse files
fix(ffi): forward distinct_handling through FFI_AggregateUDF (apache#25876)
## Which issue does this PR close? - Closes apache#25374. ## Rationale for this change `AggregateUDFImpl::distinct_handling` (apache#25288) doesn't survive the FFI boundary. `FFI_AggregateUDF` has no slot for it, so `ForeignAggregateUDF` falls back to the trait default and every UDAF loaded from another library reports `Sensitive`. A foreign `min` never gets its `DISTINCT` dropped, and a foreign `stddev`, which doesn't read `is_distinct` at all, tells the planner it deduplicates its own input. ## What changes are included in this PR? Same shape as `order_sensitivity`, all in `datafusion/ffi/src/udaf/mod.rs`: - `FFI_DistinctHandling`, a `#[repr(C)]` enum with `From` impls both ways, next to `FFI_AggregateOrderSensitivity`. - A `distinct_handling` fn pointer on `FFI_AggregateUDF`, added at the end of the struct after `supports_null_handling_clause`. - `ForeignAggregateUDF::distinct_handling` calls through it. `DistinctHandling` is `#[non_exhaustive]`, so the native-to-FFI conversion needs a wildcard arm. I map anything unknown to `Sensitive`, the trait default, which is what the issue suggested. That and appending the field at the end are the two assumptions worth a second look. The arm covers a variant added to `datafusion-expr` before the FFI enum catches up. Mismatched library versions stay unsupported, same as today. ## What is the testing strategy for this PR? - `udaf::tests::test_distinct_handling` wraps `min`, `sum` and `stddev` with the mock foreign marker and checks all three variants come back. On main `min` reports `Sensitive` and the test fails. - `tests/ffi_udaf.rs::test_distinct_handling` does the same through the separately built test library, for `stddev` (`Unsupported`) and `sum` (`Sensitive`). On main's `udaf/mod.rs` it fails with `left: Sensitive, right: Unsupported`. - `test_round_trip_all_distinct_handlings` round-trips each variant, like the existing order sensitivity test. `cargo test -p datafusion-ffi --features integration-tests` passes, as does `cargo clippy -p datafusion-ffi --all-targets --features integration-tests --no-deps -- -D warnings` and `cargo fmt --all -- --check`. ## Are there any user-facing changes? Foreign UDAFs now report the `distinct_handling` they declare, so an `Insensitive` one can have `DISTINCT` eliminated. Adding a field changes the `FFI_AggregateUDF` layout, so this probably wants the `api change` label. This was written with AI assistance. I've read the change end to end and reproduced both failures above myself.
1 parent 9eeb66f commit ac279eb

4 files changed

Lines changed: 159 additions & 6 deletions

File tree

‎datafusion/ffi/integration-tests/tests/cases/udaf.rs‎

Lines changed: 67 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,10 +20,10 @@ mod tests {
2020

2121
use arrow::array::{Float64Array, record_batch};
2222
use datafusion::error::Result;
23-
use datafusion::logical_expr::{AggregateUDF, AggregateUDFImpl};
23+
use datafusion::logical_expr::{AggregateUDF, AggregateUDFImpl, DistinctHandling};
2424
use datafusion::prelude::{SessionContext, col};
2525
use datafusion_catalog::MemTable;
26-
use datafusion_expr::{ScalarUDF, ScalarUDFImpl};
26+
use datafusion_expr::{ExprFunctionExt, ScalarUDF, ScalarUDFImpl};
2727
use datafusion_ffi::tests::utils::get_module;
2828

2929
#[tokio::test]
@@ -79,6 +79,71 @@ mod tests {
7979
Ok(())
8080
}
8181

82+
#[test]
83+
fn test_distinct_handling() -> Result<()> {
84+
let module = get_module()?;
85+
86+
let ffi_stddev_func = (module.create_stddev_udaf)();
87+
let foreign_stddev_func: Arc<dyn AggregateUDFImpl> = (&ffi_stddev_func).into();
88+
assert_eq!(
89+
foreign_stddev_func.distinct_handling(),
90+
DistinctHandling::Unsupported
91+
);
92+
93+
let ffi_sum_func = (module.create_sum_udaf)();
94+
let foreign_sum_func: Arc<dyn AggregateUDFImpl> = (&ffi_sum_func).into();
95+
assert_eq!(
96+
foreign_sum_func.distinct_handling(),
97+
DistinctHandling::Sensitive
98+
);
99+
100+
Ok(())
101+
}
102+
103+
#[tokio::test]
104+
async fn test_foreign_min_distinct_drops_distinct() -> Result<()> {
105+
let module = get_module()?;
106+
107+
let ffi_min_func = (module.create_min_udaf)();
108+
let foreign_min_func: Arc<dyn AggregateUDFImpl> = (&ffi_min_func).into();
109+
110+
let udaf = AggregateUDF::new_from_shared_impl(foreign_min_func);
111+
112+
let ctx = SessionContext::default();
113+
let record_batch = record_batch!(
114+
("a", Int32, vec![1, 2, 2, 4, 4, 4, 4]),
115+
("b", Float64, vec![3.0, 2.0, 2.0, 4.0, 1.0, 1.0, 4.0])
116+
)
117+
.unwrap();
118+
119+
let df = ctx
120+
.read_batch(record_batch)?
121+
.aggregate(
122+
vec![col("a")],
123+
vec![udaf.call(vec![col("b")]).distinct().build()?.alias("min_b")],
124+
)?
125+
.sort_by(vec![col("a")])?;
126+
127+
let plan = df.clone().into_optimized_plan()?;
128+
assert_eq!(
129+
plan.display_indent().to_string(),
130+
"Sort: ?table?.a ASC NULLS LAST\
131+
\n Aggregate: groupBy=[[?table?.a]], aggr=[[min(?table?.b) AS min_b]]\
132+
\n TableScan: ?table? projection=[a, b]"
133+
);
134+
135+
let result = df.collect().await?;
136+
137+
let expected = record_batch!(
138+
("a", Int32, vec![1, 2, 4]),
139+
("min_b", Float64, vec![3.0, 2.0, 1.0])
140+
)?;
141+
142+
assert_eq!(result[0], expected);
143+
144+
Ok(())
145+
}
146+
82147
#[tokio::test]
83148
async fn test_ffi_grouping_udaf() -> Result<()> {
84149
let module = get_module()?;

‎datafusion/ffi/src/tests/mod.rs‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -36,9 +36,9 @@ use datafusion_physical_expr::{
3636
use datafusion_physical_plan::{ExecutionPlan, Partitioning};
3737
use sync_provider::create_sync_table_provider;
3838
use udf_udaf_udwf::{
39-
create_ffi_abs_func, create_ffi_first_value_func, create_ffi_random_func,
40-
create_ffi_rank_func, create_ffi_stddev_func, create_ffi_sum_func,
41-
create_ffi_table_func,
39+
create_ffi_abs_func, create_ffi_first_value_func, create_ffi_min_func,
40+
create_ffi_random_func, create_ffi_rank_func, create_ffi_stddev_func,
41+
create_ffi_sum_func, create_ffi_table_func,
4242
};
4343

4444
use crate::catalog_provider::FFI_CatalogProvider;
@@ -142,6 +142,9 @@ pub struct ForeignLibraryModule {
142142

143143
/// Create an aggregate UDAF using first_value
144144
pub create_first_value_udaf: extern "C" fn() -> FFI_AggregateUDF,
145+
146+
/// Create an aggregate UDAF using min
147+
pub create_min_udaf: extern "C" fn() -> FFI_AggregateUDF,
145148
}
146149

147150
pub fn create_test_schema() -> Arc<Schema> {
@@ -429,5 +432,6 @@ pub extern "C" fn datafusion_ffi_get_module() -> ForeignLibraryModule {
429432
create_query_planner: query_planner::create_query_planner,
430433
version: super::version,
431434
create_first_value_udaf: create_ffi_first_value_func,
435+
create_min_udaf: create_ffi_min_func,
432436
}
433437
}

‎datafusion/ffi/src/tests/udf_udaf_udwf.rs‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ use datafusion_expr::{
2929
use datafusion_functions::math::abs::AbsFunc;
3030
use datafusion_functions::math::random::RandomFunc;
3131
use datafusion_functions_aggregate::first_last::FirstValue;
32+
use datafusion_functions_aggregate::min_max::Min;
3233
use datafusion_functions_aggregate::stddev::Stddev;
3334
use datafusion_functions_aggregate::sum::Sum;
3435
use datafusion_functions_table::generate_series::RangeFunc;
@@ -210,6 +211,12 @@ pub(crate) extern "C" fn create_ffi_stddev_func() -> FFI_AggregateUDF {
210211
udaf.into()
211212
}
212213

214+
pub(crate) extern "C" fn create_ffi_min_func() -> FFI_AggregateUDF {
215+
let udaf: Arc<AggregateUDF> = Arc::new(Min::new().into());
216+
217+
udaf.into()
218+
}
219+
213220
pub(crate) extern "C" fn create_ffi_rank_func() -> FFI_WindowUDF {
214221
let udwf: Arc<WindowUDF> = Arc::new(
215222
Rank::new(

‎datafusion/ffi/src/udaf/mod.rs‎

Lines changed: 78 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,8 @@ use datafusion_common::{DataFusionError, Result, ffi_datafusion_err};
2424
use datafusion_expr::function::AggregateFunctionSimplification;
2525
use datafusion_expr::type_coercion::functions::fields_with_udf;
2626
use datafusion_expr::{
27-
Accumulator, AggregateUDF, AggregateUDFImpl, GroupsAccumulator, Signature,
27+
Accumulator, AggregateUDF, AggregateUDFImpl, DistinctHandling, GroupsAccumulator,
28+
Signature,
2829
};
2930
use datafusion_functions_aggregate_common::accumulator::{
3031
AccumulatorArgs, StateFieldsArgs,
@@ -149,6 +150,10 @@ pub struct FFI_AggregateUDF {
149150
/// FFI equivalent to [`AggregateUDF::supports_null_handling_clause`]
150151
pub supports_null_handling_clause:
151152
unsafe extern "C" fn(udaf: &FFI_AggregateUDF) -> bool,
153+
154+
/// FFI equivalent to [`AggregateUDF::distinct_handling`]
155+
pub distinct_handling:
156+
unsafe extern "C" fn(udaf: &FFI_AggregateUDF) -> FFI_DistinctHandling,
152157
}
153158

154159
unsafe impl Send for FFI_AggregateUDF {}
@@ -337,6 +342,12 @@ unsafe extern "C" fn supports_null_handling_clause_fn_wrapper(
337342
unsafe { udaf.inner().supports_null_handling_clause() }
338343
}
339344

345+
unsafe extern "C" fn distinct_handling_fn_wrapper(
346+
udaf: &FFI_AggregateUDF,
347+
) -> FFI_DistinctHandling {
348+
unsafe { udaf.inner().distinct_handling().into() }
349+
}
350+
340351
unsafe extern "C" fn coerce_types_fn_wrapper(
341352
udaf: &FFI_AggregateUDF,
342353
arg_types: SVec<WrappedSchema>,
@@ -412,6 +423,7 @@ impl From<Arc<AggregateUDF>> for FFI_AggregateUDF {
412423
private_data: Box::into_raw(private_data).cast::<c_void>(),
413424
library_marker_id: crate::get_library_marker_id,
414425
supports_null_handling_clause: supports_null_handling_clause_fn_wrapper,
426+
distinct_handling: distinct_handling_fn_wrapper,
415427
}
416428
}
417429
}
@@ -610,6 +622,10 @@ impl AggregateUDFImpl for ForeignAggregateUDF {
610622
unsafe { (self.udaf.supports_null_handling_clause)(&self.udaf) }
611623
}
612624

625+
fn distinct_handling(&self) -> DistinctHandling {
626+
unsafe { (self.udaf.distinct_handling)(&self.udaf).into() }
627+
}
628+
613629
fn simplify(&self) -> Option<AggregateFunctionSimplification> {
614630
None
615631
}
@@ -655,6 +671,37 @@ impl From<AggregateOrderSensitivity> for FFI_AggregateOrderSensitivity {
655671
}
656672
}
657673

674+
#[repr(C)]
675+
#[derive(Debug)]
676+
pub enum FFI_DistinctHandling {
677+
Insensitive,
678+
Sensitive,
679+
Unsupported,
680+
}
681+
682+
impl From<FFI_DistinctHandling> for DistinctHandling {
683+
fn from(value: FFI_DistinctHandling) -> Self {
684+
match value {
685+
FFI_DistinctHandling::Insensitive => Self::Insensitive,
686+
FFI_DistinctHandling::Sensitive => Self::Sensitive,
687+
FFI_DistinctHandling::Unsupported => Self::Unsupported,
688+
}
689+
}
690+
}
691+
692+
impl From<DistinctHandling> for FFI_DistinctHandling {
693+
fn from(value: DistinctHandling) -> Self {
694+
match value {
695+
DistinctHandling::Insensitive => Self::Insensitive,
696+
DistinctHandling::Sensitive => Self::Sensitive,
697+
DistinctHandling::Unsupported => Self::Unsupported,
698+
// `DistinctHandling` is non_exhaustive, so map any variant added
699+
// later to the default.
700+
_ => Self::Sensitive,
701+
}
702+
}
703+
}
704+
658705
#[cfg(test)]
659706
mod tests {
660707

@@ -798,6 +845,24 @@ mod tests {
798845
Ok(())
799846
}
800847

848+
#[test]
849+
fn test_distinct_handling() -> Result<()> {
850+
let min = create_test_foreign_udaf(
851+
datafusion::functions_aggregate::min_max::Min::new(),
852+
)?;
853+
assert_eq!(min.distinct_handling(), DistinctHandling::Insensitive);
854+
855+
let sum = create_test_foreign_udaf(Sum::new())?;
856+
assert_eq!(sum.distinct_handling(), DistinctHandling::Sensitive);
857+
858+
let stddev = create_test_foreign_udaf(
859+
datafusion::functions_aggregate::stddev::Stddev::new(),
860+
)?;
861+
assert_eq!(stddev.distinct_handling(), DistinctHandling::Unsupported);
862+
863+
Ok(())
864+
}
865+
801866
#[test]
802867
fn test_beneficial_ordering() -> Result<()> {
803868
let foreign_udaf = create_test_foreign_udaf(
@@ -870,6 +935,18 @@ mod tests {
870935
test_round_trip_order_sensitivity(AggregateOrderSensitivity::Beneficial);
871936
}
872937

938+
#[test]
939+
fn test_round_trip_all_distinct_handlings() {
940+
for handling in [
941+
DistinctHandling::Insensitive,
942+
DistinctHandling::Sensitive,
943+
DistinctHandling::Unsupported,
944+
] {
945+
let ffi_handling: FFI_DistinctHandling = handling.into();
946+
assert_eq!(DistinctHandling::from(ffi_handling), handling);
947+
}
948+
}
949+
873950
#[test]
874951
fn test_ffi_udaf_local_bypass() -> Result<()> {
875952
let original_udaf = Sum::new();

0 commit comments

Comments
 (0)