1717
1818use std:: sync:: Arc ;
1919
20- use datafusion_physical_expr:: Partitioning ;
20+ use datafusion_common:: { DataFusionError , ScalarValue , SplitPoint } ;
21+ use datafusion_physical_expr:: {
22+ LexOrdering , Partitioning , PhysicalSortExpr , RangePartitioning ,
23+ } ;
2124use datafusion_physical_expr_common:: physical_expr:: PhysicalExpr ;
2225use stabby:: vec:: Vec as SVec ;
2326
27+ use crate :: arrow_wrappers:: WrappedArray ;
2428use crate :: physical_expr:: FFI_PhysicalExpr ;
29+ use crate :: physical_expr:: sort:: FFI_PhysicalSortExpr ;
30+
31+ /// A stable struct for sharing [`RangePartitioning`] across FFI boundaries.
32+ /// See [`RangePartitioning`] for the descriptions of each field.
33+ #[ repr( C ) ]
34+ #[ derive( Debug ) ]
35+ pub struct FFI_RangePartitioning {
36+ split_points : SVec < SVec < WrappedArray > > ,
37+ ordering : SVec < FFI_PhysicalSortExpr > ,
38+ }
2539
2640/// A stable struct for sharing [`Partitioning`] across FFI boundaries.
27- /// See [' Partitioning' ] for the meaning of each variant.
41+ /// See [` Partitioning` ] for the meaning of each variant.
2842#[ repr( C ) ]
2943#[ derive( Debug ) ]
3044pub enum FFI_Partitioning {
3145 RoundRobinBatch ( usize ) ,
3246 Hash ( SVec < FFI_PhysicalExpr > , usize ) ,
3347 UnknownPartitioning ( usize ) ,
48+ Range ( FFI_RangePartitioning ) ,
3449}
3550
3651impl From < & Partitioning > for FFI_Partitioning {
@@ -45,49 +60,130 @@ impl From<&Partitioning> for FFI_Partitioning {
4560 . collect ( ) ;
4661 Self :: Hash ( exprs, * size)
4762 }
48- // FFI does not yet expose range partition metadata.
49- // See https://github.com/apache/datafusion/issues/22394
5063 Partitioning :: Range ( range) => {
51- Self :: UnknownPartitioning ( range. partition_count ( ) )
64+ // Producer-side conversion should be infallible at ABI boundary
65+ let split_points = range
66+ . split_points ( )
67+ . iter ( )
68+ . map ( |split_point| {
69+ split_point
70+ . values ( )
71+ . iter ( )
72+ . map ( |value| {
73+ WrappedArray :: try_from ( value) . expect (
74+ "ScalarValue in RangePartitioning should convert to WrappedArray" ,
75+ )
76+ } )
77+ . collect ( )
78+ } )
79+ . collect ( ) ;
80+ let ordering = range
81+ . ordering ( )
82+ . iter ( )
83+ . map ( FFI_PhysicalSortExpr :: from)
84+ . collect ( ) ;
85+ Self :: Range ( FFI_RangePartitioning {
86+ split_points,
87+ ordering,
88+ } )
5289 }
5390 Partitioning :: UnknownPartitioning ( size) => Self :: UnknownPartitioning ( * size) ,
5491 }
5592 }
5693}
5794
58- impl From < & FFI_Partitioning > for Partitioning {
59- fn from ( value : & FFI_Partitioning ) -> Self {
60- match value {
95+ impl TryFrom < FFI_Partitioning > for Partitioning {
96+ type Error = DataFusionError ;
97+
98+ fn try_from ( value : FFI_Partitioning ) -> Result < Self , Self :: Error > {
99+ Ok ( match value {
61100 FFI_Partitioning :: RoundRobinBatch ( size) => {
62- Partitioning :: RoundRobinBatch ( * size)
101+ Partitioning :: RoundRobinBatch ( size)
63102 }
64103 FFI_Partitioning :: Hash ( exprs, size) => {
65104 let exprs = exprs. iter ( ) . map ( <Arc < dyn PhysicalExpr > >:: from) . collect ( ) ;
66- Self :: Hash ( exprs, * size)
105+ Self :: Hash ( exprs, size)
106+ }
107+ FFI_Partitioning :: Range ( range) => {
108+ let split_points = range
109+ . split_points
110+ . into_iter ( )
111+ . map ( |split_point| {
112+ split_point
113+ . into_iter ( )
114+ . map ( ScalarValue :: try_from)
115+ . collect :: < Result < Vec < _ > , _ > > ( )
116+ . map ( SplitPoint :: new)
117+ } )
118+ . collect :: < Result < Vec < _ > , _ > > ( ) ?;
119+
120+ let ordering =
121+ LexOrdering :: new ( range. ordering . iter ( ) . map ( PhysicalSortExpr :: from) )
122+ . ok_or_else ( || {
123+ DataFusionError :: Internal (
124+ "FFI Range partitioning ordering must be non-empty"
125+ . to_string ( ) ,
126+ )
127+ } ) ?;
128+
129+ Self :: Range ( RangePartitioning :: try_new ( ordering, split_points) ?)
67130 }
68131 FFI_Partitioning :: UnknownPartitioning ( size) => {
69- Self :: UnknownPartitioning ( * size)
132+ Self :: UnknownPartitioning ( size)
70133 }
71- }
134+ } )
72135 }
73136}
74137
75138#[ cfg( test) ]
76139mod tests {
77- use datafusion_physical_expr:: Partitioning ;
78- use datafusion_physical_expr:: expressions:: lit;
140+ use std:: sync:: Arc ;
141+
142+ use arrow_schema:: SortOptions ;
143+ use datafusion_common:: { Result , ScalarValue , SplitPoint } ;
144+ use datafusion_physical_expr:: expressions:: { Column , lit} ;
145+ use datafusion_physical_expr:: {
146+ LexOrdering , Partitioning , PhysicalSortExpr , RangePartitioning ,
147+ } ;
148+ use datafusion_physical_expr_common:: physical_expr:: PhysicalExpr ;
149+ use stabby:: vec:: Vec as SVec ;
79150
80- use crate :: physical_expr:: partitioning:: FFI_Partitioning ;
151+ use crate :: physical_expr:: partitioning:: { FFI_Partitioning , FFI_RangePartitioning } ;
152+
153+ fn range_partitioning ( ) -> Result < Partitioning > {
154+ let a = Arc :: new ( Column :: new ( "a" , 0 ) ) as Arc < dyn PhysicalExpr > ;
155+ let b = Arc :: new ( Column :: new ( "b" , 1 ) ) as Arc < dyn PhysicalExpr > ;
156+ let ordering = LexOrdering :: new ( [
157+ PhysicalSortExpr :: new ( a, SortOptions :: default ( ) ) ,
158+ PhysicalSortExpr :: new ( b, SortOptions :: new ( true , false ) ) ,
159+ ] )
160+ . expect ( "non-empty ordering" ) ;
161+ let split_points = vec ! [
162+ SplitPoint :: new( vec![
163+ ScalarValue :: Int64 ( Some ( 10 ) ) ,
164+ ScalarValue :: Utf8 ( Some ( "a" . to_string( ) ) ) ,
165+ ] ) ,
166+ SplitPoint :: new( vec![
167+ ScalarValue :: Int64 ( Some ( 20 ) ) ,
168+ ScalarValue :: Utf8 ( Some ( "b" . to_string( ) ) ) ,
169+ ] ) ,
170+ ] ;
171+ Ok ( Partitioning :: Range ( RangePartitioning :: try_new (
172+ ordering,
173+ split_points,
174+ ) ?) )
175+ }
81176
82177 #[ test]
83- fn round_trip_ffi_partitioning ( ) {
178+ fn round_trip_ffi_partitioning ( ) -> Result < ( ) > {
84179 for partitioning in [
85180 Partitioning :: RoundRobinBatch ( 10 ) ,
86181 Partitioning :: Hash ( vec ! [ lit( 1 ) ] , 10 ) ,
87182 Partitioning :: UnknownPartitioning ( 10 ) ,
183+ range_partitioning ( ) ?,
88184 ] {
89185 let ffi_partitioning: FFI_Partitioning = ( & partitioning) . into ( ) ;
90- let returned: Partitioning = ( & ffi_partitioning) . into ( ) ;
186+ let returned: Partitioning = ffi_partitioning. try_into ( ) ? ;
91187
92188 if let Partitioning :: UnknownPartitioning ( return_size) = returned {
93189 let Partitioning :: UnknownPartitioning ( original_size) = partitioning
@@ -99,5 +195,32 @@ mod tests {
99195 assert_eq ! ( partitioning, returned) ;
100196 }
101197 }
198+
199+ Ok ( ( ) )
200+ }
201+
202+ #[ test]
203+ fn round_trip_ffi_range_partitioning_compound_key ( ) -> Result < ( ) > {
204+ let partitioning = range_partitioning ( ) ?;
205+
206+ let ffi_partitioning: FFI_Partitioning = ( & partitioning) . into ( ) ;
207+ let returned: Partitioning = ffi_partitioning. try_into ( ) ?;
208+ assert_eq ! ( partitioning, returned) ;
209+
210+ Ok ( ( ) )
211+ }
212+
213+ #[ test]
214+ fn ffi_range_partitioning_rejects_empty_ordering ( ) {
215+ let ffi_partitioning = FFI_Partitioning :: Range ( FFI_RangePartitioning {
216+ split_points : SVec :: new ( ) ,
217+ ordering : SVec :: new ( ) ,
218+ } ) ;
219+
220+ let err = Partitioning :: try_from ( ffi_partitioning) . unwrap_err ( ) ;
221+ assert ! (
222+ err. to_string( ) . contains( "ordering must be non-empty" ) ,
223+ "{err}"
224+ ) ;
102225 }
103226}
0 commit comments