Skip to content

Commit 69a480b

Browse files
saadtajwartimsaucer
authored andcommitted
feat: Range Partitioning FFI (apache#23520)
## Which issue does this PR close? - Closes apache#22394 ## Rationale for this change Exposing range partition metadata via the FFI for external consumers. ## What changes are included in this PR? - Added FFI mirror struct for `RangePartitioning` and added new enum variant for range in `FFI_Partitioning` - For native -> FFI, added match arm for the new variant, same with FFI -> native but changed the approach of `From` -> `TryFrom` to utilize the validation for `RangePartitioning` and modified `plan_properties` to match - Added tests ## Are these changes tested? Yes ## Are there any user-facing changes? Yes, exposing Range partitioning over FFI. This exposes a new `Range` variant in the `FFI_Partitioning` enum, which may cause consumers of this enum to add another arm to match statements to handle the new enum. New `FFI_RangePartitioning` struct for the `Range` variant. --------- Co-authored-by: Tim Saucer <[email protected]>
1 parent 4b5102c commit 69a480b

2 files changed

Lines changed: 183 additions & 20 deletions

File tree

‎datafusion/ffi/src/physical_expr/partitioning.rs‎

Lines changed: 140 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -17,20 +17,35 @@
1717

1818
use std::sync::Arc;
1919

20-
use datafusion_physical_expr::Partitioning;
20+
use datafusion_common::{DataFusionError, ScalarValue, SplitPoint};
21+
use datafusion_physical_expr::{
22+
LexOrdering, Partitioning, PhysicalSortExpr, RangePartitioning,
23+
};
2124
use datafusion_physical_expr_common::physical_expr::PhysicalExpr;
2225
use stabby::vec::Vec as SVec;
2326

27+
use crate::arrow_wrappers::WrappedArray;
2428
use crate::physical_expr::FFI_PhysicalExpr;
29+
use crate::physical_expr::sort::FFI_PhysicalSortExpr;
30+
31+
/// A stable struct for sharing [`RangePartitioning`] across FFI boundaries.
32+
/// See [`RangePartitioning`] for the descriptions of each field.
33+
#[repr(C)]
34+
#[derive(Debug)]
35+
pub struct FFI_RangePartitioning {
36+
split_points: SVec<SVec<WrappedArray>>,
37+
ordering: SVec<FFI_PhysicalSortExpr>,
38+
}
2539

2640
/// A stable struct for sharing [`Partitioning`] across FFI boundaries.
27-
/// See ['Partitioning'] for the meaning of each variant.
41+
/// See [`Partitioning`] for the meaning of each variant.
2842
#[repr(C)]
2943
#[derive(Debug)]
3044
pub enum FFI_Partitioning {
3145
RoundRobinBatch(usize),
3246
Hash(SVec<FFI_PhysicalExpr>, usize),
3347
UnknownPartitioning(usize),
48+
Range(FFI_RangePartitioning),
3449
}
3550

3651
impl From<&Partitioning> for FFI_Partitioning {
@@ -45,49 +60,130 @@ impl From<&Partitioning> for FFI_Partitioning {
4560
.collect();
4661
Self::Hash(exprs, *size)
4762
}
48-
// FFI does not yet expose range partition metadata.
49-
// See https://github.com/apache/datafusion/issues/22394
5063
Partitioning::Range(range) => {
51-
Self::UnknownPartitioning(range.partition_count())
64+
// Producer-side conversion should be infallible at ABI boundary
65+
let split_points = range
66+
.split_points()
67+
.iter()
68+
.map(|split_point| {
69+
split_point
70+
.values()
71+
.iter()
72+
.map(|value| {
73+
WrappedArray::try_from(value).expect(
74+
"ScalarValue in RangePartitioning should convert to WrappedArray",
75+
)
76+
})
77+
.collect()
78+
})
79+
.collect();
80+
let ordering = range
81+
.ordering()
82+
.iter()
83+
.map(FFI_PhysicalSortExpr::from)
84+
.collect();
85+
Self::Range(FFI_RangePartitioning {
86+
split_points,
87+
ordering,
88+
})
5289
}
5390
Partitioning::UnknownPartitioning(size) => Self::UnknownPartitioning(*size),
5491
}
5592
}
5693
}
5794

58-
impl From<&FFI_Partitioning> for Partitioning {
59-
fn from(value: &FFI_Partitioning) -> Self {
60-
match value {
95+
impl TryFrom<FFI_Partitioning> for Partitioning {
96+
type Error = DataFusionError;
97+
98+
fn try_from(value: FFI_Partitioning) -> Result<Self, Self::Error> {
99+
Ok(match value {
61100
FFI_Partitioning::RoundRobinBatch(size) => {
62-
Partitioning::RoundRobinBatch(*size)
101+
Partitioning::RoundRobinBatch(size)
63102
}
64103
FFI_Partitioning::Hash(exprs, size) => {
65104
let exprs = exprs.iter().map(<Arc<dyn PhysicalExpr>>::from).collect();
66-
Self::Hash(exprs, *size)
105+
Self::Hash(exprs, size)
106+
}
107+
FFI_Partitioning::Range(range) => {
108+
let split_points = range
109+
.split_points
110+
.into_iter()
111+
.map(|split_point| {
112+
split_point
113+
.into_iter()
114+
.map(ScalarValue::try_from)
115+
.collect::<Result<Vec<_>, _>>()
116+
.map(SplitPoint::new)
117+
})
118+
.collect::<Result<Vec<_>, _>>()?;
119+
120+
let ordering =
121+
LexOrdering::new(range.ordering.iter().map(PhysicalSortExpr::from))
122+
.ok_or_else(|| {
123+
DataFusionError::Internal(
124+
"FFI Range partitioning ordering must be non-empty"
125+
.to_string(),
126+
)
127+
})?;
128+
129+
Self::Range(RangePartitioning::try_new(ordering, split_points)?)
67130
}
68131
FFI_Partitioning::UnknownPartitioning(size) => {
69-
Self::UnknownPartitioning(*size)
132+
Self::UnknownPartitioning(size)
70133
}
71-
}
134+
})
72135
}
73136
}
74137

75138
#[cfg(test)]
76139
mod tests {
77-
use datafusion_physical_expr::Partitioning;
78-
use datafusion_physical_expr::expressions::lit;
140+
use std::sync::Arc;
141+
142+
use arrow_schema::SortOptions;
143+
use datafusion_common::{Result, ScalarValue, SplitPoint};
144+
use datafusion_physical_expr::expressions::{Column, lit};
145+
use datafusion_physical_expr::{
146+
LexOrdering, Partitioning, PhysicalSortExpr, RangePartitioning,
147+
};
148+
use datafusion_physical_expr_common::physical_expr::PhysicalExpr;
149+
use stabby::vec::Vec as SVec;
79150

80-
use crate::physical_expr::partitioning::FFI_Partitioning;
151+
use crate::physical_expr::partitioning::{FFI_Partitioning, FFI_RangePartitioning};
152+
153+
fn range_partitioning() -> Result<Partitioning> {
154+
let a = Arc::new(Column::new("a", 0)) as Arc<dyn PhysicalExpr>;
155+
let b = Arc::new(Column::new("b", 1)) as Arc<dyn PhysicalExpr>;
156+
let ordering = LexOrdering::new([
157+
PhysicalSortExpr::new(a, SortOptions::default()),
158+
PhysicalSortExpr::new(b, SortOptions::new(true, false)),
159+
])
160+
.expect("non-empty ordering");
161+
let split_points = vec![
162+
SplitPoint::new(vec![
163+
ScalarValue::Int64(Some(10)),
164+
ScalarValue::Utf8(Some("a".to_string())),
165+
]),
166+
SplitPoint::new(vec![
167+
ScalarValue::Int64(Some(20)),
168+
ScalarValue::Utf8(Some("b".to_string())),
169+
]),
170+
];
171+
Ok(Partitioning::Range(RangePartitioning::try_new(
172+
ordering,
173+
split_points,
174+
)?))
175+
}
81176

82177
#[test]
83-
fn round_trip_ffi_partitioning() {
178+
fn round_trip_ffi_partitioning() -> Result<()> {
84179
for partitioning in [
85180
Partitioning::RoundRobinBatch(10),
86181
Partitioning::Hash(vec![lit(1)], 10),
87182
Partitioning::UnknownPartitioning(10),
183+
range_partitioning()?,
88184
] {
89185
let ffi_partitioning: FFI_Partitioning = (&partitioning).into();
90-
let returned: Partitioning = (&ffi_partitioning).into();
186+
let returned: Partitioning = ffi_partitioning.try_into()?;
91187

92188
if let Partitioning::UnknownPartitioning(return_size) = returned {
93189
let Partitioning::UnknownPartitioning(original_size) = partitioning
@@ -99,5 +195,32 @@ mod tests {
99195
assert_eq!(partitioning, returned);
100196
}
101197
}
198+
199+
Ok(())
200+
}
201+
202+
#[test]
203+
fn round_trip_ffi_range_partitioning_compound_key() -> Result<()> {
204+
let partitioning = range_partitioning()?;
205+
206+
let ffi_partitioning: FFI_Partitioning = (&partitioning).into();
207+
let returned: Partitioning = ffi_partitioning.try_into()?;
208+
assert_eq!(partitioning, returned);
209+
210+
Ok(())
211+
}
212+
213+
#[test]
214+
fn ffi_range_partitioning_rejects_empty_ordering() {
215+
let ffi_partitioning = FFI_Partitioning::Range(FFI_RangePartitioning {
216+
split_points: SVec::new(),
217+
ordering: SVec::new(),
218+
});
219+
220+
let err = Partitioning::try_from(ffi_partitioning).unwrap_err();
221+
assert!(
222+
err.to_string().contains("ordering must be non-empty"),
223+
"{err}"
224+
);
102225
}
103226
}

‎datafusion/ffi/src/plan_properties.rs‎

Lines changed: 43 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ use std::sync::Arc;
2020

2121
use arrow::datatypes::SchemaRef;
2222
use datafusion_common::error::{DataFusionError, Result};
23-
use datafusion_physical_expr::EquivalenceProperties;
23+
use datafusion_physical_expr::{EquivalenceProperties, Partitioning};
2424
use datafusion_physical_expr_common::sort_expr::PhysicalSortExpr;
2525
use datafusion_physical_plan::PlanProperties;
2626
use datafusion_physical_plan::execution_plan::{Boundedness, EmissionType};
@@ -172,6 +172,7 @@ impl TryFrom<FFI_PlanProperties> for PlanProperties {
172172
.unwrap_or_default();
173173

174174
let partitioning = unsafe { (ffi_props.output_partitioning)(&ffi_props) };
175+
let partitioning = Partitioning::try_from(partitioning)?;
175176

176177
let eq_properties = if sort_exprs.is_empty() {
177178
EquivalenceProperties::new(Arc::new(schema))
@@ -187,7 +188,7 @@ impl TryFrom<FFI_PlanProperties> for PlanProperties {
187188

188189
Ok(PlanProperties::new(
189190
eq_properties,
190-
(&partitioning).into(),
191+
partitioning,
191192
emission_type,
192193
boundedness,
193194
))
@@ -260,13 +261,15 @@ impl From<FFI_EmissionType> for EmissionType {
260261

261262
#[cfg(test)]
262263
mod tests {
264+
use arrow::datatypes::{DataType, Field, Schema};
263265
use datafusion::physical_expr::PhysicalSortExpr;
264266
use datafusion::physical_plan::Partitioning;
267+
use datafusion_common::{ScalarValue, SplitPoint};
268+
use datafusion_physical_expr::{LexOrdering, RangePartitioning};
265269

266270
use super::*;
267271

268272
fn create_test_props() -> Result<PlanProperties> {
269-
use arrow::datatypes::{DataType, Field, Schema};
270273
let schema =
271274
Arc::new(Schema::new(vec![Field::new("a", DataType::Float32, false)]));
272275

@@ -282,6 +285,25 @@ mod tests {
282285
))
283286
}
284287

288+
fn create_range_test_props() -> Result<PlanProperties> {
289+
let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int64, false)]));
290+
let col = datafusion::physical_plan::expressions::col("a", &schema)?;
291+
let ordering = LexOrdering::new([PhysicalSortExpr::new_default(col)])
292+
.expect("non-empty ordering");
293+
let split_points = vec![
294+
SplitPoint::new(vec![ScalarValue::Int64(Some(10))]),
295+
SplitPoint::new(vec![ScalarValue::Int64(Some(20))]),
296+
];
297+
let range = RangePartitioning::try_new(ordering, split_points)?;
298+
299+
Ok(PlanProperties::new(
300+
EquivalenceProperties::new(schema),
301+
Partitioning::Range(range),
302+
EmissionType::Incremental,
303+
Boundedness::Bounded,
304+
))
305+
}
306+
285307
#[test]
286308
fn test_round_trip_ffi_plan_properties() -> Result<()> {
287309
let original_props = create_test_props()?;
@@ -314,4 +336,22 @@ mod tests {
314336

315337
Ok(())
316338
}
339+
340+
#[test]
341+
fn test_round_trip_ffi_plan_properties_range_partitioning() -> Result<()> {
342+
let original_props = create_range_test_props()?;
343+
344+
let mut local_props_ptr = FFI_PlanProperties::from(&original_props);
345+
local_props_ptr.library_marker_id = crate::mock_foreign_marker_id;
346+
347+
let foreign_props: PlanProperties = local_props_ptr.try_into()?;
348+
349+
assert_eq!(
350+
format!("{:?}", foreign_props.output_partitioning()),
351+
format!("{:?}", original_props.output_partitioning())
352+
);
353+
assert_eq!(format!("{foreign_props:?}"), format!("{original_props:?}"));
354+
355+
Ok(())
356+
}
317357
}

0 commit comments

Comments
 (0)