Skip to content

Commit 01f1c92

Browse files
committed
fix: exhaustive destructuring in proto hooks for sort_expr, partitioning, and repartition
Add `let Self { .. }` (encoder) and prost-node destructuring (decoder) to `PhysicalSortExpr`, `Partitioning`, and `RangeExpr` proto hooks so that adding a field to any of these types is a compile error in both directions rather than a silently-dropped round-trip gap. Follows the convention established by apache#24164 for the join-plan hooks. Also adds round-trip tests for `Partitioning::Range` and `RangeExpr`, the two hooks that had no existing colocated proto test. Closes apache#24619
1 parent 5e42247 commit 01f1c92

3 files changed

Lines changed: 181 additions & 36 deletions

File tree

‎datafusion/physical-expr-common/src/sort_expr.rs‎

Lines changed: 13 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -199,10 +199,11 @@ impl PhysicalSortExpr {
199199
&self,
200200
ctx: &crate::physical_expr::proto_encode::PhysicalExprEncodeCtx<'_>,
201201
) -> Result<datafusion_proto_models::protobuf::PhysicalSortExprNode> {
202+
let Self { expr, options } = self;
202203
Ok(datafusion_proto_models::protobuf::PhysicalSortExprNode {
203-
expr: Some(Box::new(ctx.encode_child(&self.expr)?)),
204-
asc: !self.options.descending,
205-
nulls_first: self.options.nulls_first,
204+
expr: Some(Box::new(ctx.encode_child(expr)?)),
205+
asc: !options.descending,
206+
nulls_first: options.nulls_first,
206207
})
207208
}
208209

@@ -211,16 +212,18 @@ impl PhysicalSortExpr {
211212
node: &datafusion_proto_models::protobuf::PhysicalSortExprNode,
212213
ctx: &crate::physical_expr::proto_decode::PhysicalExprDecodeCtx<'_>,
213214
) -> Result<Self> {
214-
let expr = ctx.decode_required_expression(
215-
node.expr.as_deref(),
216-
"PhysicalSortExpr",
217-
"expr",
218-
)?;
215+
let datafusion_proto_models::protobuf::PhysicalSortExprNode {
216+
expr,
217+
asc,
218+
nulls_first,
219+
} = node;
220+
let expr =
221+
ctx.decode_required_expression(expr.as_deref(), "PhysicalSortExpr", "expr")?;
219222
Ok(PhysicalSortExpr {
220223
expr,
221224
options: SortOptions {
222-
descending: !node.asc,
223-
nulls_first: node.nulls_first,
225+
descending: !asc,
226+
nulls_first: *nulls_first,
224227
},
225228
})
226229
}

‎datafusion/physical-expr/src/partitioning.rs‎

Lines changed: 74 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -629,9 +629,12 @@ impl Partitioning {
629629
)
630630
}
631631
Partitioning::Range(range) => {
632-
let sort_expr = sort_exprs_try_to_proto(range.ordering().iter(), ctx)?;
633-
let split_point = range
634-
.split_points()
632+
let RangePartitioning {
633+
ordering,
634+
split_points,
635+
} = range;
636+
let sort_expr = sort_exprs_try_to_proto(ordering.iter(), ctx)?;
637+
let split_point = split_points
635638
.iter()
636639
.map(|split_point| {
637640
let value = split_point
@@ -673,26 +676,34 @@ impl Partitioning {
673676

674677
let partition_count =
675678
|count: u64| usize_from_wire(count, "Partitioning", "partition_count");
676-
let Some(partition_method) = node.partition_method.as_ref() else {
679+
let protobuf::Partitioning { partition_method } = node;
680+
let Some(partition_method) = partition_method.as_ref() else {
677681
return Ok(None);
678682
};
679683
let partitioning = match partition_method {
680684
protobuf::partitioning::PartitionMethod::RoundRobin(n) => {
681685
Partitioning::RoundRobinBatch(partition_count(*n)?)
682686
}
683687
protobuf::partitioning::PartitionMethod::Hash(hash) => {
684-
let exprs = hash
685-
.hash_expr
688+
let protobuf::PhysicalHashRepartition {
689+
hash_expr,
690+
partition_count: hash_partition_count,
691+
} = hash;
692+
let exprs = hash_expr
686693
.iter()
687694
.map(|expr| ctx.decode(expr))
688695
.collect::<Result<Vec<_>>>()?;
689-
Partitioning::Hash(exprs, partition_count(hash.partition_count)?)
696+
Partitioning::Hash(exprs, partition_count(*hash_partition_count)?)
690697
}
691698
protobuf::partitioning::PartitionMethod::Unknown(n) => {
692699
Partitioning::UnknownPartitioning(partition_count(*n)?)
693700
}
694701
protobuf::partitioning::PartitionMethod::Range(range) => {
695-
let sort_exprs = sort_exprs_try_from_proto(&range.sort_expr, ctx)?;
702+
let protobuf::PhysicalRangePartitioning {
703+
sort_expr,
704+
split_point,
705+
} = range;
706+
let sort_exprs = sort_exprs_try_from_proto(sort_expr, ctx)?;
696707
let sort_expr_count = sort_exprs.len();
697708
let ordering = LexOrdering::new(sort_exprs).ok_or_else(|| {
698709
internal_datafusion_err!(
@@ -704,8 +715,7 @@ impl Partitioning {
704715
"Range partitioning ordering must not contain duplicate expressions"
705716
);
706717
}
707-
let split_points = range
708-
.split_point
718+
let split_points = split_point
709719
.iter()
710720
.map(|split_point| {
711721
let values = split_point
@@ -1575,13 +1585,16 @@ mod ordering_proto_tests {
15751585
mod partition_count_proto_tests {
15761586
use std::sync::Arc;
15771587

1588+
use arrow::compute::SortOptions;
15781589
use arrow::datatypes::{DataType, Field, Schema};
1590+
use datafusion_common::{ScalarValue, SplitPoint};
15791591
use datafusion_physical_expr_common::physical_expr::PhysicalExpr;
15801592
use datafusion_physical_expr_common::physical_expr::proto_decode::PhysicalExprDecodeCtx;
15811593
use datafusion_physical_expr_common::physical_expr::proto_encode::PhysicalExprEncodeCtx;
1594+
use datafusion_physical_expr_common::sort_expr::{LexOrdering, PhysicalSortExpr};
15821595
use datafusion_proto_models::protobuf;
15831596

1584-
use super::Partitioning;
1597+
use super::{Partitioning, RangePartitioning};
15851598
use crate::expressions::Column;
15861599
use crate::proto_test_util::{StubDecoder, StubEncoder, column_node};
15871600

@@ -1662,4 +1675,54 @@ mod partition_count_proto_tests {
16621675
// Every variant widens to the same wire value, with no truncation.
16631676
assert_eq!(encoded, vec![u64::try_from(usize::MAX).unwrap(); 3]);
16641677
}
1678+
1679+
#[test]
1680+
fn range_partitioning_round_trip_preserves_split_points_and_options() {
1681+
// Single-column ordering so the stub encoder (which emits an identical
1682+
// placeholder for every expression) cannot produce duplicates that the
1683+
// decoder's deduplication step would collapse.
1684+
let sort_options = SortOptions {
1685+
descending: false,
1686+
nulls_first: true,
1687+
};
1688+
let ordering = LexOrdering::from([PhysicalSortExpr::new(
1689+
Arc::new(Column::new("a", 0)),
1690+
sort_options,
1691+
)]);
1692+
let split_points = vec![
1693+
SplitPoint::new(vec![ScalarValue::Int32(Some(10))]),
1694+
SplitPoint::new(vec![ScalarValue::Int32(Some(30))]),
1695+
];
1696+
let range =
1697+
RangePartitioning::try_new(ordering.clone(), split_points.clone()).unwrap();
1698+
1699+
let encoder = StubEncoder::ok();
1700+
let encode_ctx = PhysicalExprEncodeCtx::new(&encoder);
1701+
let node = Partitioning::Range(range)
1702+
.try_to_proto(&encode_ctx)
1703+
.unwrap();
1704+
1705+
let schema = Schema::new(vec![Field::new("a", DataType::Int32, false)]);
1706+
let decoder = StubDecoder::ok();
1707+
let decode_ctx = PhysicalExprDecodeCtx::new(&schema, &decoder);
1708+
let decoded = Partitioning::try_from_proto(&node, &decode_ctx)
1709+
.unwrap()
1710+
.unwrap();
1711+
1712+
let Partitioning::Range(decoded_range) = decoded else {
1713+
panic!("expected Range partitioning, got {decoded:?}");
1714+
};
1715+
// Sort options survive the wire (the expression itself is a stub but
1716+
// the options are encoded directly in PhysicalSortExprNode).
1717+
assert_eq!(
1718+
decoded_range
1719+
.ordering()
1720+
.iter()
1721+
.map(|e| e.options)
1722+
.collect::<Vec<_>>(),
1723+
[sort_options],
1724+
);
1725+
// ScalarValue split points round-trip without loss.
1726+
assert_eq!(decoded_range.split_points(), split_points);
1727+
}
16651728
}

‎datafusion/physical-plan/src/repartition/mod.rs‎

Lines changed: 94 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -848,17 +848,16 @@ impl PhysicalExpr for RangeExpr {
848848
&self,
849849
ctx: &datafusion_physical_expr_common::physical_expr::proto_encode::PhysicalExprEncodeCtx<'_>,
850850
) -> Result<Option<protobuf::PhysicalExprNode>> {
851+
let Self { on_columns, router } = self;
851852
// Encode the raw ordered children: rebuilding a `LexOrdering` would
852853
// deduplicate equivalent children after dynamic-filter remapping.
853-
let sort_exprs = self
854-
.on_columns
854+
let sort_exprs = on_columns
855855
.iter()
856-
.zip(self.router.sort_options())
856+
.zip(router.sort_options())
857857
.map(|(expr, options)| PhysicalSortExpr::new(Arc::clone(expr), *options))
858858
.collect::<Vec<_>>();
859859
let sort_expr = sort_exprs_try_to_proto(&sort_exprs, ctx)?;
860-
let split_point = self
861-
.router
860+
let split_point = router
862861
.split_points()
863862
.iter()
864863
.map(|split_point| {
@@ -889,20 +888,27 @@ impl RangeExpr {
889888
node: &protobuf::PhysicalExprNode,
890889
ctx: &datafusion_physical_expr_common::physical_expr::proto_decode::PhysicalExprDecodeCtx<'_>,
891890
) -> Result<PhysicalExprRef> {
891+
let protobuf::PhysicalExprNode {
892+
expr_id: _,
893+
expr_type,
894+
} = node;
892895
// Decode the raw ordered children for the same reason as `try_to_proto`.
893896
let Some(protobuf::physical_expr_node::ExprType::RangeExpr(range_expr)) =
894-
&node.expr_type
897+
expr_type
895898
else {
896899
return internal_err!("PhysicalExprNode is not a RangeExpr");
897900
};
898-
let sort_exprs = sort_exprs_try_from_proto(&range_expr.sort_expr, ctx)?;
901+
let protobuf::PhysicalRangeExprNode {
902+
sort_expr,
903+
split_point,
904+
} = range_expr;
905+
let sort_exprs = sort_exprs_try_from_proto(sort_expr, ctx)?;
899906
let (on_columns, sort_options): (Vec<PhysicalExprRef>, Vec<SortOptions>) =
900907
sort_exprs
901908
.into_iter()
902909
.map(|sort_expr| (sort_expr.expr, sort_expr.options))
903910
.unzip();
904-
let split_points = range_expr
905-
.split_point
911+
let split_points = split_point
906912
.iter()
907913
.map(|split_point| {
908914
let values = split_point
@@ -2058,9 +2064,6 @@ impl ExecutionPlan for RepartitionExec {
20582064
&self,
20592065
ctx: &crate::proto::ExecutionPlanEncodeCtx<'_>,
20602066
) -> Result<Option<protobuf::PhysicalPlanNode>> {
2061-
// Destructure exhaustively (no `..`) so that adding a field to
2062-
// `RepartitionExec` is a compile error here until it is either
2063-
// serialized or explicitly documented as not needing to be.
20642067
let Self {
20652068
input,
20662069
// Execution-time channel state, created on `execute()`.
@@ -2104,9 +2107,6 @@ impl RepartitionExec {
21042107
protobuf::physical_plan_node::PhysicalPlanType::Repartition,
21052108
"RepartitionExec",
21062109
);
2107-
// Destructure exhaustively so that a new field on
2108-
// `RepartitionExecNode` is a compile error here rather than a silently
2109-
// dropped field.
21102110
let protobuf::RepartitionExecNode {
21112111
input,
21122112
partitioning,
@@ -5033,3 +5033,82 @@ mod test {
50335033
Ok(())
50345034
}
50355035
}
5036+
5037+
#[cfg(all(test, feature = "proto"))]
5038+
mod range_expr_proto_tests {
5039+
use std::sync::Arc;
5040+
5041+
use arrow::compute::SortOptions;
5042+
use arrow::datatypes::{DataType, Field, Schema};
5043+
use datafusion_common::{Result, ScalarValue, SplitPoint};
5044+
use datafusion_physical_expr::expressions::col;
5045+
use datafusion_physical_expr_common::sort_expr::PhysicalSortExpr;
5046+
5047+
use super::*;
5048+
use crate::proto::{ExecutionPlanDecodeCtx, ExecutionPlanEncodeCtx};
5049+
use crate::proto_test_util::{StubPlanDecoder, StubPlanEncoder};
5050+
5051+
fn schema() -> Schema {
5052+
Schema::new(vec![
5053+
Field::new("a", DataType::Int32, false),
5054+
Field::new("b", DataType::Int32, false),
5055+
])
5056+
}
5057+
5058+
fn split_points() -> Vec<SplitPoint> {
5059+
vec![SplitPoint::new(vec![
5060+
ScalarValue::Int32(Some(10)),
5061+
ScalarValue::Int32(Some(20)),
5062+
])]
5063+
}
5064+
5065+
fn sort_options() -> [SortOptions; 2] {
5066+
[
5067+
SortOptions {
5068+
descending: true,
5069+
nulls_first: false,
5070+
},
5071+
SortOptions {
5072+
descending: false,
5073+
nulls_first: true,
5074+
},
5075+
]
5076+
}
5077+
5078+
#[test]
5079+
fn range_expr_round_trip_preserves_sort_options_and_split_points() -> Result<()> {
5080+
let schema = schema();
5081+
let sort_opts = sort_options();
5082+
let split_pts = split_points();
5083+
let on_columns = vec![col("a", &schema)?, col("b", &schema)?];
5084+
let range_partitioning = RangePartitioning::try_new(
5085+
[
5086+
PhysicalSortExpr::new(Arc::clone(&on_columns[0]), sort_opts[0]),
5087+
PhysicalSortExpr::new(Arc::clone(&on_columns[1]), sort_opts[1]),
5088+
]
5089+
.into(),
5090+
split_pts.clone(),
5091+
)?;
5092+
let expr =
5093+
RangeExpr::try_new_with_schema(on_columns, &range_partitioning, &schema)?;
5094+
5095+
let encoder = StubPlanEncoder::ok();
5096+
let encode_ctx = ExecutionPlanEncodeCtx::new(&encoder);
5097+
let node = PhysicalExpr::try_to_proto(&expr, &encode_ctx.expr_ctx())
5098+
.unwrap()
5099+
.expect("RangeExpr should encode to Some(node)");
5100+
5101+
let decoder = StubPlanDecoder::ok();
5102+
let decode_ctx = ExecutionPlanDecodeCtx::new(&decoder);
5103+
let decoded = RangeExpr::try_from_proto(&node, &decode_ctx.expr_ctx(&schema))?;
5104+
let decoded = decoded
5105+
.downcast_ref::<RangeExpr>()
5106+
.expect("decoded expression should be a RangeExpr");
5107+
5108+
// Sort options survive the wire (expression columns are stubs but options are not).
5109+
assert_eq!(decoded.sort_options(), sort_opts);
5110+
// ScalarValue split points round-trip without loss.
5111+
assert_eq!(decoded.split_points(), split_pts);
5112+
Ok(())
5113+
}
5114+
}

0 commit comments

Comments
 (0)