Skip to content

Commit a1f56b7

Browse files
authored
feat: logical plan protobuf representation for range repartitioning (apache#23030)
## Which issue does this PR close? <!-- We generally require a GitHub issue to be filed for all bug fixes and enhancements and this helps us generate change logs for our releases. You can link an issue to this PR using the GitHub syntax. For example `Closes apache#123` indicates that this PR will close issue apache#123. --> - Closes apache#22787 ## Rationale for this change <!-- Why are you proposing this change? If this is already explained clearly in the issue then this section is not needed. Explaining clearly why changes are proposed helps reviewers understand your changes and offer better suggestions for fixes. --> The range repartitioning scheme for logical plans does not currently have a protobuf representation. ## What changes are included in this PR? <!-- There is no need to duplicate the description in the issue here but it is sometimes worth providing a summary of the individual changes in this PR. --> A protobuf representation of the `RangeRepartition` struct was added to `datafusion.proto`, and the codegened Rust types were created. Added logic for serializing and deserializing to and from the protobuf representation, and a roundtrip test as well! ## Are these changes tested? <!-- We typically require tests for all PRs in order to: 1. Prevent the code from being accidentally broken by subsequent changes 2. Serve as another way to document the expected behavior of the code If tests are not included in your PR, please explain why (for example, are they covered by existing tests)? --> Yes! Added a test in `roundtrip_logical_plan` ## Are there any user-facing changes? No, adding internal protobuf serialization support for an existing logical plan variant <!-- If there are user-facing changes then we may require documentation to be updated before approving the PR. --> <!-- If there are any breaking changes to public APIs, please add the `api change` label. -->
1 parent 67425b4 commit a1f56b7

7 files changed

Lines changed: 387 additions & 28 deletions

File tree

‎datafusion/proto-models/proto/datafusion.proto‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -148,9 +148,19 @@ message RepartitionNode {
148148
oneof partition_method {
149149
uint64 round_robin = 2;
150150
HashRepartition hash = 3;
151+
RangeRepartition range = 4;
151152
}
152153
}
153154

155+
message RangeSplitPoint {
156+
repeated datafusion_common.ScalarValue value = 1;
157+
}
158+
159+
message RangeRepartition {
160+
repeated SortExprNode sort_expr = 1;
161+
repeated RangeSplitPoint split_point = 2;
162+
}
163+
154164
message HashRepartition {
155165
repeated LogicalExprNode hash_expr = 1;
156166
uint64 partition_count = 2;

‎datafusion/proto-models/src/generated/pbjson.rs‎

Lines changed: 214 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22350,6 +22350,207 @@ impl<'de> serde::Deserialize<'de> for ProjectionNode {
2235022350
deserializer.deserialize_struct("datafusion.ProjectionNode", FIELDS, GeneratedVisitor)
2235122351
}
2235222352
}
22353+
impl serde::Serialize for RangeRepartition {
22354+
#[allow(deprecated)]
22355+
fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
22356+
where
22357+
S: serde::Serializer,
22358+
{
22359+
use serde::ser::SerializeStruct;
22360+
let mut len = 0;
22361+
if !self.sort_expr.is_empty() {
22362+
len += 1;
22363+
}
22364+
if !self.split_point.is_empty() {
22365+
len += 1;
22366+
}
22367+
let mut struct_ser = serializer.serialize_struct("datafusion.RangeRepartition", len)?;
22368+
if !self.sort_expr.is_empty() {
22369+
struct_ser.serialize_field("sortExpr", &self.sort_expr)?;
22370+
}
22371+
if !self.split_point.is_empty() {
22372+
struct_ser.serialize_field("splitPoint", &self.split_point)?;
22373+
}
22374+
struct_ser.end()
22375+
}
22376+
}
22377+
impl<'de> serde::Deserialize<'de> for RangeRepartition {
22378+
#[allow(deprecated)]
22379+
fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>
22380+
where
22381+
D: serde::Deserializer<'de>,
22382+
{
22383+
const FIELDS: &[&str] = &[
22384+
"sort_expr",
22385+
"sortExpr",
22386+
"split_point",
22387+
"splitPoint",
22388+
];
22389+
22390+
#[allow(clippy::enum_variant_names)]
22391+
enum GeneratedField {
22392+
SortExpr,
22393+
SplitPoint,
22394+
}
22395+
impl<'de> serde::Deserialize<'de> for GeneratedField {
22396+
fn deserialize<D>(deserializer: D) -> std::result::Result<GeneratedField, D::Error>
22397+
where
22398+
D: serde::Deserializer<'de>,
22399+
{
22400+
struct GeneratedVisitor;
22401+
22402+
impl serde::de::Visitor<'_> for GeneratedVisitor {
22403+
type Value = GeneratedField;
22404+
22405+
fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
22406+
write!(formatter, "expected one of: {:?}", &FIELDS)
22407+
}
22408+
22409+
#[allow(unused_variables)]
22410+
fn visit_str<E>(self, value: &str) -> std::result::Result<GeneratedField, E>
22411+
where
22412+
E: serde::de::Error,
22413+
{
22414+
match value {
22415+
"sortExpr" | "sort_expr" => Ok(GeneratedField::SortExpr),
22416+
"splitPoint" | "split_point" => Ok(GeneratedField::SplitPoint),
22417+
_ => Err(serde::de::Error::unknown_field(value, FIELDS)),
22418+
}
22419+
}
22420+
}
22421+
deserializer.deserialize_identifier(GeneratedVisitor)
22422+
}
22423+
}
22424+
struct GeneratedVisitor;
22425+
impl<'de> serde::de::Visitor<'de> for GeneratedVisitor {
22426+
type Value = RangeRepartition;
22427+
22428+
fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
22429+
formatter.write_str("struct datafusion.RangeRepartition")
22430+
}
22431+
22432+
fn visit_map<V>(self, mut map_: V) -> std::result::Result<RangeRepartition, V::Error>
22433+
where
22434+
V: serde::de::MapAccess<'de>,
22435+
{
22436+
let mut sort_expr__ = None;
22437+
let mut split_point__ = None;
22438+
while let Some(k) = map_.next_key()? {
22439+
match k {
22440+
GeneratedField::SortExpr => {
22441+
if sort_expr__.is_some() {
22442+
return Err(serde::de::Error::duplicate_field("sortExpr"));
22443+
}
22444+
sort_expr__ = Some(map_.next_value()?);
22445+
}
22446+
GeneratedField::SplitPoint => {
22447+
if split_point__.is_some() {
22448+
return Err(serde::de::Error::duplicate_field("splitPoint"));
22449+
}
22450+
split_point__ = Some(map_.next_value()?);
22451+
}
22452+
}
22453+
}
22454+
Ok(RangeRepartition {
22455+
sort_expr: sort_expr__.unwrap_or_default(),
22456+
split_point: split_point__.unwrap_or_default(),
22457+
})
22458+
}
22459+
}
22460+
deserializer.deserialize_struct("datafusion.RangeRepartition", FIELDS, GeneratedVisitor)
22461+
}
22462+
}
22463+
impl serde::Serialize for RangeSplitPoint {
22464+
#[allow(deprecated)]
22465+
fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
22466+
where
22467+
S: serde::Serializer,
22468+
{
22469+
use serde::ser::SerializeStruct;
22470+
let mut len = 0;
22471+
if !self.value.is_empty() {
22472+
len += 1;
22473+
}
22474+
let mut struct_ser = serializer.serialize_struct("datafusion.RangeSplitPoint", len)?;
22475+
if !self.value.is_empty() {
22476+
struct_ser.serialize_field("value", &self.value)?;
22477+
}
22478+
struct_ser.end()
22479+
}
22480+
}
22481+
impl<'de> serde::Deserialize<'de> for RangeSplitPoint {
22482+
#[allow(deprecated)]
22483+
fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>
22484+
where
22485+
D: serde::Deserializer<'de>,
22486+
{
22487+
const FIELDS: &[&str] = &[
22488+
"value",
22489+
];
22490+
22491+
#[allow(clippy::enum_variant_names)]
22492+
enum GeneratedField {
22493+
Value,
22494+
}
22495+
impl<'de> serde::Deserialize<'de> for GeneratedField {
22496+
fn deserialize<D>(deserializer: D) -> std::result::Result<GeneratedField, D::Error>
22497+
where
22498+
D: serde::Deserializer<'de>,
22499+
{
22500+
struct GeneratedVisitor;
22501+
22502+
impl serde::de::Visitor<'_> for GeneratedVisitor {
22503+
type Value = GeneratedField;
22504+
22505+
fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
22506+
write!(formatter, "expected one of: {:?}", &FIELDS)
22507+
}
22508+
22509+
#[allow(unused_variables)]
22510+
fn visit_str<E>(self, value: &str) -> std::result::Result<GeneratedField, E>
22511+
where
22512+
E: serde::de::Error,
22513+
{
22514+
match value {
22515+
"value" => Ok(GeneratedField::Value),
22516+
_ => Err(serde::de::Error::unknown_field(value, FIELDS)),
22517+
}
22518+
}
22519+
}
22520+
deserializer.deserialize_identifier(GeneratedVisitor)
22521+
}
22522+
}
22523+
struct GeneratedVisitor;
22524+
impl<'de> serde::de::Visitor<'de> for GeneratedVisitor {
22525+
type Value = RangeSplitPoint;
22526+
22527+
fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
22528+
formatter.write_str("struct datafusion.RangeSplitPoint")
22529+
}
22530+
22531+
fn visit_map<V>(self, mut map_: V) -> std::result::Result<RangeSplitPoint, V::Error>
22532+
where
22533+
V: serde::de::MapAccess<'de>,
22534+
{
22535+
let mut value__ = None;
22536+
while let Some(k) = map_.next_key()? {
22537+
match k {
22538+
GeneratedField::Value => {
22539+
if value__.is_some() {
22540+
return Err(serde::de::Error::duplicate_field("value"));
22541+
}
22542+
value__ = Some(map_.next_value()?);
22543+
}
22544+
}
22545+
}
22546+
Ok(RangeSplitPoint {
22547+
value: value__.unwrap_or_default(),
22548+
})
22549+
}
22550+
}
22551+
deserializer.deserialize_struct("datafusion.RangeSplitPoint", FIELDS, GeneratedVisitor)
22552+
}
22553+
}
2235322554
impl serde::Serialize for RecursionUnnestOption {
2235422555
#[allow(deprecated)]
2235522556
fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
@@ -22778,6 +22979,9 @@ impl serde::Serialize for RepartitionNode {
2277822979
repartition_node::PartitionMethod::Hash(v) => {
2277922980
struct_ser.serialize_field("hash", v)?;
2278022981
}
22982+
repartition_node::PartitionMethod::Range(v) => {
22983+
struct_ser.serialize_field("range", v)?;
22984+
}
2278122985
}
2278222986
}
2278322987
struct_ser.end()
@@ -22794,13 +22998,15 @@ impl<'de> serde::Deserialize<'de> for RepartitionNode {
2279422998
"round_robin",
2279522999
"roundRobin",
2279623000
"hash",
23001+
"range",
2279723002
];
2279823003

2279923004
#[allow(clippy::enum_variant_names)]
2280023005
enum GeneratedField {
2280123006
Input,
2280223007
RoundRobin,
2280323008
Hash,
23009+
Range,
2280423010
}
2280523011
impl<'de> serde::Deserialize<'de> for GeneratedField {
2280623012
fn deserialize<D>(deserializer: D) -> std::result::Result<GeneratedField, D::Error>
@@ -22825,6 +23031,7 @@ impl<'de> serde::Deserialize<'de> for RepartitionNode {
2282523031
"input" => Ok(GeneratedField::Input),
2282623032
"roundRobin" | "round_robin" => Ok(GeneratedField::RoundRobin),
2282723033
"hash" => Ok(GeneratedField::Hash),
23034+
"range" => Ok(GeneratedField::Range),
2282823035
_ => Err(serde::de::Error::unknown_field(value, FIELDS)),
2282923036
}
2283023037
}
@@ -22865,6 +23072,13 @@ impl<'de> serde::Deserialize<'de> for RepartitionNode {
2286523072
return Err(serde::de::Error::duplicate_field("hash"));
2286623073
}
2286723074
partition_method__ = map_.next_value::<::std::option::Option<_>>()?.map(repartition_node::PartitionMethod::Hash)
23075+
;
23076+
}
23077+
GeneratedField::Range => {
23078+
if partition_method__.is_some() {
23079+
return Err(serde::de::Error::duplicate_field("range"));
23080+
}
23081+
partition_method__ = map_.next_value::<::std::option::Option<_>>()?.map(repartition_node::PartitionMethod::Range)
2286823082
;
2286923083
}
2287023084
}

‎datafusion/proto-models/src/generated/prost.rs‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -210,7 +210,7 @@ pub struct SortNode {
210210
pub struct RepartitionNode {
211211
#[prost(message, optional, boxed, tag = "1")]
212212
pub input: ::core::option::Option<::prost::alloc::boxed::Box<LogicalPlanNode>>,
213-
#[prost(oneof = "repartition_node::PartitionMethod", tags = "2, 3")]
213+
#[prost(oneof = "repartition_node::PartitionMethod", tags = "2, 3, 4")]
214214
pub partition_method: ::core::option::Option<repartition_node::PartitionMethod>,
215215
}
216216
/// Nested message and enum types in `RepartitionNode`.
@@ -221,9 +221,23 @@ pub mod repartition_node {
221221
RoundRobin(u64),
222222
#[prost(message, tag = "3")]
223223
Hash(super::HashRepartition),
224+
#[prost(message, tag = "4")]
225+
Range(super::RangeRepartition),
224226
}
225227
}
226228
#[derive(Clone, PartialEq, ::prost::Message)]
229+
pub struct RangeSplitPoint {
230+
#[prost(message, repeated, tag = "1")]
231+
pub value: ::prost::alloc::vec::Vec<super::datafusion_common::ScalarValue>,
232+
}
233+
#[derive(Clone, PartialEq, ::prost::Message)]
234+
pub struct RangeRepartition {
235+
#[prost(message, repeated, tag = "1")]
236+
pub sort_expr: ::prost::alloc::vec::Vec<SortExprNode>,
237+
#[prost(message, repeated, tag = "2")]
238+
pub split_point: ::prost::alloc::vec::Vec<RangeSplitPoint>,
239+
}
240+
#[derive(Clone, PartialEq, ::prost::Message)]
227241
pub struct HashRepartition {
228242
#[prost(message, repeated, tag = "1")]
229243
pub hash_expr: ::prost::alloc::vec::Vec<LogicalExprNode>,

‎datafusion/proto/src/logical_plan/from_proto.rs‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ use std::sync::Arc;
2020
use arrow::datatypes::{DataType, Field};
2121
use datafusion_common::datatype::DataTypeExt;
2222
use datafusion_common::{
23-
NullEquality, RecursionUnnestOption, Result, ScalarValue, TableReference,
23+
NullEquality, RecursionUnnestOption, Result, ScalarValue, SplitPoint, TableReference,
2424
UnnestOptions, exec_datafusion_err, internal_err, plan_datafusion_err,
2525
};
2626
use datafusion_execution::TaskContext;
@@ -899,3 +899,14 @@ fn parse_required_expr(
899899
fn proto_error<S: Into<String>>(message: S) -> Error {
900900
Error::General(message.into())
901901
}
902+
903+
pub(super) fn parse_protobuf_range_split_point(
904+
split_point: &protobuf::RangeSplitPoint,
905+
) -> Result<SplitPoint, Error> {
906+
let values = split_point
907+
.value
908+
.iter()
909+
.map(ScalarValue::try_from)
910+
.collect::<Result<_, Error>>()?;
911+
Ok(SplitPoint::new(values))
912+
}

‎datafusion/proto/src/logical_plan/mod.rs‎

Lines changed: 25 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -60,8 +60,8 @@ use datafusion_datasource_json::file_format::{
6060
use datafusion_datasource_parquet::file_format::{ParquetFormat, ParquetFormatFactory};
6161
use datafusion_expr::dml::InsertOp;
6262
use datafusion_expr::{
63-
AggregateUDF, DmlStatement, FetchType, HigherOrderUDF, RecursiveQuery, SkipType,
64-
TableSource, Unnest, WriteOp,
63+
AggregateUDF, DmlStatement, FetchType, HigherOrderUDF, RangePartitioning,
64+
RecursiveQuery, SkipType, TableSource, Unnest, WriteOp,
6565
};
6666
use datafusion_expr::{
6767
DistinctOn, DropView, Expr, JoinConstraint, LogicalPlan, LogicalPlanBuilder,
@@ -76,6 +76,7 @@ use datafusion_expr::{
7676
use datafusion_proto_common::protobuf_common;
7777

7878
use self::to_proto::{serialize_expr, serialize_exprs};
79+
use crate::logical_plan::to_proto::serialize_range_split_point;
7980
use crate::logical_plan::to_proto::serialize_sorts;
8081
use datafusion_catalog::TableProvider;
8182
use datafusion_catalog::default_table_source::{provider_as_source, source_as_provider};
@@ -745,6 +746,16 @@ impl AsLogicalPlan for LogicalPlanNode {
745746
PartitionMethod::RoundRobin(partition_count) => {
746747
Partitioning::RoundRobinBatch(*partition_count as usize)
747748
}
749+
PartitionMethod::Range(protobuf::RangeRepartition {
750+
sort_expr: pb_sort_expr,
751+
split_point,
752+
}) => Partitioning::Range(RangePartitioning::try_new(
753+
from_proto::parse_sorts(pb_sort_expr, ctx, extension_codec)?,
754+
split_point
755+
.iter()
756+
.map(from_proto::parse_protobuf_range_split_point)
757+
.collect::<Result<Vec<_>, _>>()?,
758+
)?),
748759
};
749760

750761
LogicalPlanBuilder::from(input)
@@ -1754,10 +1765,18 @@ impl AsLogicalPlan for LogicalPlanNode {
17541765
Partitioning::RoundRobinBatch(partition_count) => {
17551766
PartitionMethod::RoundRobin(*partition_count as u64)
17561767
}
1757-
Partitioning::Range(_) => {
1758-
// TODO: Support range repartition protobuf serialization.
1759-
// Tracked by https://github.com/apache/datafusion/issues/22787
1760-
return not_impl_err!("Range repartition");
1768+
Partitioning::Range(range_partitioning) => {
1769+
let ordering = range_partitioning.ordering();
1770+
let split_point = range_partitioning
1771+
.split_points()
1772+
.iter()
1773+
.map(serialize_range_split_point)
1774+
.collect::<Result<Vec<_>, _>>()?;
1775+
1776+
PartitionMethod::Range(protobuf::RangeRepartition {
1777+
sort_expr: serialize_sorts(ordering, extension_codec)?,
1778+
split_point,
1779+
})
17611780
}
17621781
Partitioning::DistributeBy(_) => {
17631782
return not_impl_err!("DistributeBy");

0 commit comments

Comments
 (0)