Skip to content

Commit 7274252

Browse files
buraksennkosiew
authored andcommitted
refactor(proto): migrate RepartitionExec serde (apache#23792)
## Which issue does this PR close? - Closes apache#23504. ## Rationale for this change Part of epic apache#23494. Moves `RepartitionExec` protobuf serialization from central dispatch. ## What changes are included in this PR? Add new proto functions in physical plan and deprecate proto ones. ## Are these changes tested? Yes, existing round trip tests pass. ## Are there any user-facing changes? Existing methods are deprecated with no immediate API change.
1 parent 76f2e6e commit 7274252

2 files changed

Lines changed: 205 additions & 38 deletions

File tree

  • datafusion
    • physical-plan/src/repartition
    • proto/src/physical_plan

‎datafusion/physical-plan/src/repartition/mod.rs‎

Lines changed: 181 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1703,6 +1703,187 @@ impl ExecutionPlan for RepartitionExec {
17031703
cache: new_properties.into(),
17041704
})))
17051705
}
1706+
1707+
#[cfg(feature = "proto")]
1708+
fn try_to_proto(
1709+
&self,
1710+
ctx: &crate::proto::ExecutionPlanEncodeCtx<'_>,
1711+
) -> Result<Option<datafusion_proto_models::protobuf::PhysicalPlanNode>> {
1712+
use datafusion_proto_models::protobuf;
1713+
1714+
let input = ctx.encode_child(self.input())?;
1715+
1716+
// Keep the existing protobuf wire representation unchanged.
1717+
let partition_method = match self.partitioning() {
1718+
Partitioning::RoundRobinBatch(n) => {
1719+
protobuf::partitioning::PartitionMethod::RoundRobin(*n as u64)
1720+
}
1721+
Partitioning::Hash(exprs, n) => {
1722+
let hash_expr = ctx.encode_expressions(exprs)?;
1723+
protobuf::partitioning::PartitionMethod::Hash(
1724+
protobuf::PhysicalHashRepartition {
1725+
hash_expr,
1726+
partition_count: *n as u64,
1727+
},
1728+
)
1729+
}
1730+
Partitioning::Range(range) => {
1731+
let sort_expr = range
1732+
.ordering()
1733+
.iter()
1734+
.map(|sort_expr| {
1735+
Ok(protobuf::PhysicalSortExprNode {
1736+
expr: Some(Box::new(ctx.encode_expr(&sort_expr.expr)?)),
1737+
asc: !sort_expr.options.descending,
1738+
nulls_first: sort_expr.options.nulls_first,
1739+
})
1740+
})
1741+
.collect::<Result<Vec<_>>>()?;
1742+
let split_point = range
1743+
.split_points()
1744+
.iter()
1745+
.map(|split_point| {
1746+
let value = split_point
1747+
.values()
1748+
.iter()
1749+
.map(|value| value.try_into().map_err(Into::into))
1750+
.collect::<Result<Vec<_>>>()?;
1751+
Ok(protobuf::PhysicalRangeSplitPoint { value })
1752+
})
1753+
.collect::<Result<Vec<_>>>()?;
1754+
protobuf::partitioning::PartitionMethod::Range(
1755+
protobuf::PhysicalRangePartitioning {
1756+
sort_expr,
1757+
split_point,
1758+
},
1759+
)
1760+
}
1761+
Partitioning::UnknownPartitioning(n) => {
1762+
protobuf::partitioning::PartitionMethod::Unknown(*n as u64)
1763+
}
1764+
};
1765+
1766+
Ok(Some(protobuf::PhysicalPlanNode {
1767+
physical_plan_type: Some(
1768+
protobuf::physical_plan_node::PhysicalPlanType::Repartition(Box::new(
1769+
protobuf::RepartitionExecNode {
1770+
input: Some(Box::new(input)),
1771+
partitioning: Some(protobuf::Partitioning {
1772+
partition_method: Some(partition_method),
1773+
}),
1774+
preserve_order: self.preserve_order(),
1775+
},
1776+
)),
1777+
),
1778+
}))
1779+
}
1780+
}
1781+
1782+
#[cfg(feature = "proto")]
1783+
impl RepartitionExec {
1784+
/// Reconstruct a [`RepartitionExec`] from its protobuf representation.
1785+
pub fn try_from_proto(
1786+
node: &datafusion_proto_models::protobuf::PhysicalPlanNode,
1787+
ctx: &crate::proto::ExecutionPlanDecodeCtx<'_>,
1788+
) -> Result<Arc<dyn ExecutionPlan>> {
1789+
use datafusion_proto_models::protobuf;
1790+
1791+
let repart = crate::expect_plan_variant!(
1792+
node,
1793+
protobuf::physical_plan_node::PhysicalPlanType::Repartition,
1794+
"RepartitionExec",
1795+
);
1796+
let input = ctx.decode_required_child(
1797+
repart.input.as_deref(),
1798+
"RepartitionExec",
1799+
"input",
1800+
)?;
1801+
let input_schema = input.schema();
1802+
1803+
let partition_method = repart
1804+
.partitioning
1805+
.as_ref()
1806+
.and_then(|p| p.partition_method.as_ref())
1807+
.ok_or_else(|| {
1808+
datafusion_common::internal_datafusion_err!(
1809+
"RepartitionExec is missing required field 'partitioning'"
1810+
)
1811+
})?;
1812+
1813+
let partitioning = match partition_method {
1814+
protobuf::partitioning::PartitionMethod::RoundRobin(n) => {
1815+
Partitioning::RoundRobinBatch(*n as usize)
1816+
}
1817+
protobuf::partitioning::PartitionMethod::Hash(hash) => {
1818+
let exprs = hash
1819+
.hash_expr
1820+
.iter()
1821+
.map(|expr| ctx.decode_expr(expr, input_schema.as_ref()))
1822+
.collect::<Result<Vec<_>>>()?;
1823+
let partition_count =
1824+
usize::try_from(hash.partition_count).map_err(|_| {
1825+
datafusion_common::internal_datafusion_err!(
1826+
"Hash partition count {} exceeds usize::MAX",
1827+
hash.partition_count
1828+
)
1829+
})?;
1830+
Partitioning::Hash(exprs, partition_count)
1831+
}
1832+
protobuf::partitioning::PartitionMethod::Unknown(n) => {
1833+
Partitioning::UnknownPartitioning(*n as usize)
1834+
}
1835+
protobuf::partitioning::PartitionMethod::Range(range) => {
1836+
let sort_exprs = range
1837+
.sort_expr
1838+
.iter()
1839+
.map(|sort_expr| {
1840+
let expr = sort_expr.expr.as_ref().ok_or_else(|| {
1841+
datafusion_common::internal_datafusion_err!(
1842+
"Unexpected empty physical expression"
1843+
)
1844+
})?;
1845+
Ok(PhysicalSortExpr {
1846+
expr: ctx.decode_expr(expr, input_schema.as_ref())?,
1847+
options: SortOptions {
1848+
descending: !sort_expr.asc,
1849+
nulls_first: sort_expr.nulls_first,
1850+
},
1851+
})
1852+
})
1853+
.collect::<Result<Vec<_>>>()?;
1854+
let sort_expr_count = sort_exprs.len();
1855+
let ordering = LexOrdering::new(sort_exprs).ok_or_else(|| {
1856+
datafusion_common::internal_datafusion_err!(
1857+
"Range partitioning requires non-empty ordering"
1858+
)
1859+
})?;
1860+
if ordering.len() != sort_expr_count {
1861+
return datafusion_common::internal_err!(
1862+
"Range partitioning ordering must not contain duplicate expressions"
1863+
);
1864+
}
1865+
let split_points = range
1866+
.split_point
1867+
.iter()
1868+
.map(|split_point| {
1869+
let values = split_point
1870+
.value
1871+
.iter()
1872+
.map(|value| ScalarValue::try_from(value).map_err(Into::into))
1873+
.collect::<Result<Vec<_>>>()?;
1874+
Ok(SplitPoint::new(values))
1875+
})
1876+
.collect::<Result<Vec<_>>>()?;
1877+
Partitioning::Range(RangePartitioning::try_new(ordering, split_points)?)
1878+
}
1879+
};
1880+
1881+
let mut repart_exec = RepartitionExec::try_new(input, partitioning)?;
1882+
if repart.preserve_order {
1883+
repart_exec = repart_exec.with_preserve_order();
1884+
}
1885+
Ok(Arc::new(repart_exec))
1886+
}
17061887
}
17071888

17081889
impl RepartitionExec {

‎datafusion/proto/src/physical_plan/mod.rs‎

Lines changed: 24 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -105,8 +105,6 @@ use datafusion_physical_plan::{ExecutionPlan, InputOrderMode, PhysicalExpr, Wind
105105
use prost::Message;
106106
use prost::bytes::BufMut;
107107

108-
use self::from_proto::parse_protobuf_partitioning;
109-
use self::to_proto::serialize_partitioning;
110108
use crate::common::{byte_to_string, str_to_byte};
111109
use crate::convert::{FromProto, TryFromProto};
112110
use crate::convert_required;
@@ -786,8 +784,8 @@ pub trait PhysicalPlanNodeExt: Sized {
786784
PhysicalPlanType::Merge(_) => {
787785
CoalescePartitionsExec::try_from_proto(self.node(), &decode_ctx)
788786
}
789-
PhysicalPlanType::Repartition(repart) => {
790-
self.try_into_repartition_physical_plan(repart, ctx, proto_converter)
787+
PhysicalPlanType::Repartition(_) => {
788+
RepartitionExec::try_from_proto(self.node(), &decode_ctx)
791789
}
792790
PhysicalPlanType::GlobalLimit(_) => {
793791
GlobalLimitExec::try_from_proto(self.node(), &decode_ctx)
@@ -966,14 +964,6 @@ pub trait PhysicalPlanNodeExt: Sized {
966964
return Ok(node);
967965
}
968966

969-
if let Some(exec) = plan.downcast_ref::<RepartitionExec>() {
970-
return protobuf::PhysicalPlanNode::try_from_repartition_exec(
971-
exec,
972-
codec,
973-
proto_converter,
974-
);
975-
}
976-
977967
if let Some(union) = plan.downcast_ref::<UnionExec>() {
978968
return protobuf::PhysicalPlanNode::try_from_union_exec(
979969
union,
@@ -1432,25 +1422,27 @@ pub trait PhysicalPlanNodeExt: Sized {
14321422
CoalescePartitionsExec::try_from_proto(&node, &decode_ctx)
14331423
}
14341424

1425+
#[deprecated(
1426+
since = "55.0.0",
1427+
note = "unused by DataFusion; `RepartitionExec` deserializes itself via `RepartitionExec::try_from_proto`"
1428+
)]
14351429
fn try_into_repartition_physical_plan(
14361430
&self,
14371431
repart: &protobuf::RepartitionExecNode,
14381432
ctx: &PhysicalPlanDecodeContext<'_>,
14391433
proto_converter: &dyn PhysicalProtoConverterExtension,
14401434
) -> Result<Arc<dyn ExecutionPlan>> {
1441-
let input: Arc<dyn ExecutionPlan> =
1442-
into_physical_plan(&repart.input, ctx, proto_converter)?;
1443-
let partitioning = parse_protobuf_partitioning(
1444-
repart.partitioning.as_ref(),
1435+
let node = protobuf::PhysicalPlanNode {
1436+
physical_plan_type: Some(PhysicalPlanType::Repartition(Box::new(
1437+
repart.clone(),
1438+
))),
1439+
};
1440+
let decoder = ConverterPlanDecoder {
14451441
ctx,
1446-
input.schema().as_ref(),
14471442
proto_converter,
1448-
)?;
1449-
let mut repart_exec = RepartitionExec::try_new(input, partitioning.unwrap())?;
1450-
if repart.preserve_order {
1451-
repart_exec = repart_exec.with_preserve_order();
1452-
}
1453-
Ok(Arc::new(repart_exec))
1443+
};
1444+
let decode_ctx = ExecutionPlanDecodeCtx::new(&decoder);
1445+
RepartitionExec::try_from_proto(&node, &decode_ctx)
14541446
}
14551447

14561448
#[deprecated(
@@ -3362,28 +3354,22 @@ pub trait PhysicalPlanNodeExt: Sized {
33623354
})
33633355
}
33643356

3357+
#[deprecated(
3358+
since = "55.0.0",
3359+
note = "unused by DataFusion; `RepartitionExec` serializes itself via `ExecutionPlan::try_to_proto`"
3360+
)]
33653361
fn try_from_repartition_exec(
33663362
exec: &RepartitionExec,
33673363
codec: &dyn PhysicalExtensionCodec,
33683364
proto_converter: &dyn PhysicalProtoConverterExtension,
33693365
) -> Result<protobuf::PhysicalPlanNode> {
3370-
let input = protobuf::PhysicalPlanNode::try_from_physical_plan_with_converter(
3371-
exec.input().to_owned(),
3366+
let encoder = ConverterPlanEncoder {
33723367
codec,
33733368
proto_converter,
3374-
)?;
3375-
3376-
let pb_partitioning =
3377-
serialize_partitioning(exec.partitioning(), codec, proto_converter)?;
3378-
3379-
Ok(protobuf::PhysicalPlanNode {
3380-
physical_plan_type: Some(PhysicalPlanType::Repartition(Box::new(
3381-
protobuf::RepartitionExecNode {
3382-
input: Some(Box::new(input)),
3383-
partitioning: Some(pb_partitioning),
3384-
preserve_order: exec.preserve_order(),
3385-
},
3386-
))),
3369+
};
3370+
let encode_ctx = ExecutionPlanEncodeCtx::new(&encoder);
3371+
exec.try_to_proto(&encode_ctx)?.ok_or_else(|| {
3372+
internal_datafusion_err!("RepartitionExec is not serializable")
33873373
})
33883374
}
33893375

0 commit comments

Comments
 (0)