Skip to content

Commit ecb0fea

Browse files
authored
feat: destructure proto hooks (apache#25204)
## Which issue does this PR close? <!-- We generally require a GitHub issue to be filed for all bug fixes and enhancements and this helps us generate change logs for our releases. You can link an issue to this PR using the GitHub syntax. For example `Closes #123` indicates that this PR will close issue #123. --> - Closes apache#24619. ## Rationale for this change Adding a field to `PhysicalSortExpr`, `Partitioning`, or `RangeExpr` would silently drop it on round-trip serialization with no compile error. <!-- Why are you proposing this change? If this is already explained clearly in the issue then this section is not needed. Explaining clearly why changes are proposed helps reviewers understand your changes and offer better suggestions for fixes. Please explain the problem you are trying to solve in terms of the user-visible behavior, rather than the implementation. For example, "The code in `foo.rs` doesn't handle nulls" is a symptom of the implementation. "COUNT(DISTINCT) returns wrong results when the column contains nulls" is the user-visible problem. --> ## What changes are included in this PR? Exhaustive destructuring (`let Self { .. }` / prost-node destructuring) in the `try_to_proto` and `try_from_proto` hooks for `PhysicalSortExpr`, `Partitioning`, and `RangeExpr`. <!-- There is no need to duplicate the description in the issue here, but it is sometimes worth providing a summary of the individual changes in this PR. --> ## What is the testing strategy for this PR? Added round-trip tests for `Partitioning::Range` and `RangeExpr` (the two hooks with no existing proto test). Existing tests cover `PhysicalSortExpr`. <!-- We typically require tests for all PRs in order to: 1. Prevent the code from being accidentally broken by subsequent changes 2. Serve as another way to document the expected behavior of the code Briefly describe how this PR is tested, and point to the specific tests you added. For example: 'This new feature is covered by the `sqllogictest` cases added in `foo.slt`'. If this PR does not add tests, explain why. For example, if the change is already covered by existing tests, please mention it. You should also check the `codecov` bot reply on this PR to confirm the changed code is exercised. --> ## Are there any user-facing changes? no <!-- If there are user-facing changes then we may require documentation to be updated before approving the PR. If there are any breaking changes to public APIs, please add the `api change` label. -->
1 parent 6ff484a commit ecb0fea

3 files changed

Lines changed: 180 additions & 40 deletions

File tree

‎datafusion/physical-expr-common/src/sort_expr.rs‎

Lines changed: 13 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -199,10 +199,11 @@ impl PhysicalSortExpr {
199199
&self,
200200
ctx: &crate::physical_expr::proto_encode::PhysicalExprEncodeCtx<'_>,
201201
) -> Result<datafusion_proto_models::protobuf::PhysicalSortExprNode> {
202+
let Self { expr, options } = self;
202203
Ok(datafusion_proto_models::protobuf::PhysicalSortExprNode {
203-
expr: Some(Box::new(ctx.encode_child(&self.expr)?)),
204-
asc: !self.options.descending,
205-
nulls_first: self.options.nulls_first,
204+
expr: Some(Box::new(ctx.encode_child(expr)?)),
205+
asc: !options.descending,
206+
nulls_first: options.nulls_first,
206207
})
207208
}
208209

@@ -211,16 +212,18 @@ impl PhysicalSortExpr {
211212
node: &datafusion_proto_models::protobuf::PhysicalSortExprNode,
212213
ctx: &crate::physical_expr::proto_decode::PhysicalExprDecodeCtx<'_>,
213214
) -> Result<Self> {
214-
let expr = ctx.decode_required_expression(
215-
node.expr.as_deref(),
216-
"PhysicalSortExpr",
217-
"expr",
218-
)?;
215+
let datafusion_proto_models::protobuf::PhysicalSortExprNode {
216+
expr,
217+
asc,
218+
nulls_first,
219+
} = node;
220+
let expr =
221+
ctx.decode_required_expression(expr.as_deref(), "PhysicalSortExpr", "expr")?;
219222
Ok(PhysicalSortExpr {
220223
expr,
221224
options: SortOptions {
222-
descending: !node.asc,
223-
nulls_first: node.nulls_first,
225+
descending: !asc,
226+
nulls_first: *nulls_first,
224227
},
225228
})
226229
}

‎datafusion/physical-expr/src/partitioning.rs‎

Lines changed: 73 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -629,9 +629,12 @@ impl Partitioning {
629629
)
630630
}
631631
Partitioning::Range(range) => {
632-
let sort_expr = sort_exprs_try_to_proto(range.ordering().iter(), ctx)?;
633-
let split_point = range
634-
.split_points()
632+
let RangePartitioning {
633+
ordering,
634+
split_points,
635+
} = range;
636+
let sort_expr = sort_exprs_try_to_proto(ordering.iter(), ctx)?;
637+
let split_point = split_points
635638
.iter()
636639
.map(|split_point| {
637640
let value = split_point
@@ -673,26 +676,34 @@ impl Partitioning {
673676

674677
let partition_count =
675678
|count: u64| usize_from_wire(count, "Partitioning", "partition_count");
676-
let Some(partition_method) = node.partition_method.as_ref() else {
679+
let protobuf::Partitioning { partition_method } = node;
680+
let Some(partition_method) = partition_method.as_ref() else {
677681
return Ok(None);
678682
};
679683
let partitioning = match partition_method {
680684
protobuf::partitioning::PartitionMethod::RoundRobin(n) => {
681685
Partitioning::RoundRobinBatch(partition_count(*n)?)
682686
}
683687
protobuf::partitioning::PartitionMethod::Hash(hash) => {
684-
let exprs = hash
685-
.hash_expr
688+
let protobuf::PhysicalHashRepartition {
689+
hash_expr,
690+
partition_count: hash_partition_count,
691+
} = hash;
692+
let exprs = hash_expr
686693
.iter()
687694
.map(|expr| ctx.decode(expr))
688695
.collect::<Result<Vec<_>>>()?;
689-
Partitioning::Hash(exprs, partition_count(hash.partition_count)?)
696+
Partitioning::Hash(exprs, partition_count(*hash_partition_count)?)
690697
}
691698
protobuf::partitioning::PartitionMethod::Unknown(n) => {
692699
Partitioning::UnknownPartitioning(partition_count(*n)?)
693700
}
694701
protobuf::partitioning::PartitionMethod::Range(range) => {
695-
let sort_exprs = sort_exprs_try_from_proto(&range.sort_expr, ctx)?;
702+
let protobuf::PhysicalRangePartitioning {
703+
sort_expr,
704+
split_point,
705+
} = range;
706+
let sort_exprs = sort_exprs_try_from_proto(sort_expr, ctx)?;
696707
let sort_expr_count = sort_exprs.len();
697708
let ordering = LexOrdering::new(sort_exprs).ok_or_else(|| {
698709
internal_datafusion_err!(
@@ -704,12 +715,11 @@ impl Partitioning {
704715
"Range partitioning ordering must not contain duplicate expressions"
705716
);
706717
}
707-
let split_points = range
708-
.split_point
718+
let split_points = split_point
709719
.iter()
710720
.map(|split_point| {
711-
let values = split_point
712-
.value
721+
let protobuf::PhysicalRangeSplitPoint { value } = split_point;
722+
let values = value
713723
.iter()
714724
.map(|value| ScalarValue::try_from(value).map_err(Into::into))
715725
.collect::<Result<Vec<_>>>()?;
@@ -1575,13 +1585,16 @@ mod ordering_proto_tests {
15751585
mod partition_count_proto_tests {
15761586
use std::sync::Arc;
15771587

1588+
use arrow::compute::SortOptions;
15781589
use arrow::datatypes::{DataType, Field, Schema};
1590+
use datafusion_common::{ScalarValue, SplitPoint};
15791591
use datafusion_physical_expr_common::physical_expr::PhysicalExpr;
15801592
use datafusion_physical_expr_common::physical_expr::proto_decode::PhysicalExprDecodeCtx;
15811593
use datafusion_physical_expr_common::physical_expr::proto_encode::PhysicalExprEncodeCtx;
1594+
use datafusion_physical_expr_common::sort_expr::{LexOrdering, PhysicalSortExpr};
15821595
use datafusion_proto_models::protobuf;
15831596

1584-
use super::Partitioning;
1597+
use super::{Partitioning, RangePartitioning};
15851598
use crate::expressions::Column;
15861599
use crate::proto_test_util::{StubDecoder, StubEncoder, column_node};
15871600

@@ -1662,4 +1675,51 @@ mod partition_count_proto_tests {
16621675
// Every variant widens to the same wire value, with no truncation.
16631676
assert_eq!(encoded, vec![u64::try_from(usize::MAX).unwrap(); 3]);
16641677
}
1678+
1679+
#[test]
1680+
fn range_partitioning_round_trip_preserves_split_points_and_options() {
1681+
// Single-column ordering so the stub encoder (which emits an identical
1682+
// placeholder for every expression) cannot produce duplicates that the
1683+
// decoder's deduplication step would collapse.
1684+
let sort_options = SortOptions {
1685+
descending: false,
1686+
nulls_first: true,
1687+
};
1688+
let ordering = LexOrdering::from([PhysicalSortExpr::new(
1689+
Arc::new(Column::new("a", 0)),
1690+
sort_options,
1691+
)]);
1692+
let split_points = vec![
1693+
SplitPoint::new(vec![ScalarValue::Int32(Some(10))]),
1694+
SplitPoint::new(vec![ScalarValue::Int32(Some(30))]),
1695+
];
1696+
let range =
1697+
RangePartitioning::try_new(ordering.clone(), split_points.clone()).unwrap();
1698+
1699+
let encoder = StubEncoder::ok();
1700+
let encode_ctx = PhysicalExprEncodeCtx::new(&encoder);
1701+
let node = Partitioning::Range(range)
1702+
.try_to_proto(&encode_ctx)
1703+
.unwrap();
1704+
1705+
let schema = Schema::new(vec![Field::new("a", DataType::Int32, false)]);
1706+
let decoder = StubDecoder::ok();
1707+
let decode_ctx = PhysicalExprDecodeCtx::new(&schema, &decoder);
1708+
let decoded = Partitioning::try_from_proto(&node, &decode_ctx)
1709+
.unwrap()
1710+
.unwrap();
1711+
1712+
let Partitioning::Range(decoded_range) = decoded else {
1713+
panic!("expected Range partitioning, got {decoded:?}");
1714+
};
1715+
assert_eq!(
1716+
decoded_range
1717+
.ordering()
1718+
.iter()
1719+
.map(|e| e.options)
1720+
.collect::<Vec<_>>(),
1721+
[sort_options],
1722+
);
1723+
assert_eq!(decoded_range.split_points(), split_points);
1724+
}
16651725
}

‎datafusion/physical-plan/src/repartition/mod.rs‎

Lines changed: 94 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -849,17 +849,16 @@ impl PhysicalExpr for RangeExpr {
849849
&self,
850850
ctx: &datafusion_physical_expr_common::physical_expr::proto_encode::PhysicalExprEncodeCtx<'_>,
851851
) -> Result<Option<protobuf::PhysicalExprNode>> {
852+
let Self { on_columns, router } = self;
852853
// Encode the raw ordered children: rebuilding a `LexOrdering` would
853854
// deduplicate equivalent children after dynamic-filter remapping.
854-
let sort_exprs = self
855-
.on_columns
855+
let sort_exprs = on_columns
856856
.iter()
857-
.zip(self.router.sort_options())
857+
.zip(router.sort_options())
858858
.map(|(expr, options)| PhysicalSortExpr::new(Arc::clone(expr), *options))
859859
.collect::<Vec<_>>();
860860
let sort_expr = sort_exprs_try_to_proto(&sort_exprs, ctx)?;
861-
let split_point = self
862-
.router
861+
let split_point = router
863862
.split_points()
864863
.iter()
865864
.map(|split_point| {
@@ -890,24 +889,31 @@ impl RangeExpr {
890889
node: &protobuf::PhysicalExprNode,
891890
ctx: &datafusion_physical_expr_common::physical_expr::proto_decode::PhysicalExprDecodeCtx<'_>,
892891
) -> Result<PhysicalExprRef> {
892+
let protobuf::PhysicalExprNode {
893+
expr_id: _,
894+
expr_type,
895+
} = node;
893896
// Decode the raw ordered children for the same reason as `try_to_proto`.
894897
let Some(protobuf::physical_expr_node::ExprType::RangeExpr(range_expr)) =
895-
&node.expr_type
898+
expr_type
896899
else {
897900
return internal_err!("PhysicalExprNode is not a RangeExpr");
898901
};
899-
let sort_exprs = sort_exprs_try_from_proto(&range_expr.sort_expr, ctx)?;
902+
let protobuf::PhysicalRangeExprNode {
903+
sort_expr,
904+
split_point,
905+
} = range_expr;
906+
let sort_exprs = sort_exprs_try_from_proto(sort_expr, ctx)?;
900907
let (on_columns, sort_options): (Vec<PhysicalExprRef>, Vec<SortOptions>) =
901908
sort_exprs
902909
.into_iter()
903910
.map(|sort_expr| (sort_expr.expr, sort_expr.options))
904911
.unzip();
905-
let split_points = range_expr
906-
.split_point
912+
let split_points = split_point
907913
.iter()
908914
.map(|split_point| {
909-
let values = split_point
910-
.value
915+
let protobuf::PhysicalRangeSplitPoint { value } = split_point;
916+
let values = value
911917
.iter()
912918
.map(|value| ScalarValue::try_from(value).map_err(Into::into))
913919
.collect::<Result<Vec<_>>>()?;
@@ -2067,9 +2073,6 @@ impl ExecutionPlan for RepartitionExec {
20672073
&self,
20682074
ctx: &crate::proto::ExecutionPlanEncodeCtx<'_>,
20692075
) -> Result<Option<protobuf::PhysicalPlanNode>> {
2070-
// Destructure exhaustively (no `..`) so that adding a field to
2071-
// `RepartitionExec` is a compile error here until it is either
2072-
// serialized or explicitly documented as not needing to be.
20732076
let Self {
20742077
input,
20752078
// Execution-time channel state, created on `execute()`.
@@ -2115,9 +2118,6 @@ impl RepartitionExec {
21152118
protobuf::physical_plan_node::PhysicalPlanType::Repartition,
21162119
"RepartitionExec",
21172120
);
2118-
// Destructure exhaustively so that a new field on
2119-
// `RepartitionExecNode` is a compile error here rather than a silently
2120-
// dropped field.
21212121
let protobuf::RepartitionExecNode {
21222122
input,
21232123
partitioning,
@@ -5084,3 +5084,80 @@ mod test {
50845084
Ok(())
50855085
}
50865086
}
5087+
5088+
#[cfg(all(test, feature = "proto"))]
5089+
mod range_expr_proto_tests {
5090+
use std::sync::Arc;
5091+
5092+
use arrow::compute::SortOptions;
5093+
use arrow::datatypes::{DataType, Field, Schema};
5094+
use datafusion_common::{Result, ScalarValue, SplitPoint};
5095+
use datafusion_physical_expr::expressions::col;
5096+
use datafusion_physical_expr_common::sort_expr::PhysicalSortExpr;
5097+
5098+
use super::*;
5099+
use crate::proto::{ExecutionPlanDecodeCtx, ExecutionPlanEncodeCtx};
5100+
use crate::proto_test_util::{StubPlanDecoder, StubPlanEncoder};
5101+
5102+
fn schema() -> Schema {
5103+
Schema::new(vec![
5104+
Field::new("a", DataType::Int32, false),
5105+
Field::new("b", DataType::Int32, false),
5106+
])
5107+
}
5108+
5109+
fn split_points() -> Vec<SplitPoint> {
5110+
vec![SplitPoint::new(vec![
5111+
ScalarValue::Int32(Some(10)),
5112+
ScalarValue::Int32(Some(20)),
5113+
])]
5114+
}
5115+
5116+
fn sort_options() -> [SortOptions; 2] {
5117+
[
5118+
SortOptions {
5119+
descending: true,
5120+
nulls_first: false,
5121+
},
5122+
SortOptions {
5123+
descending: false,
5124+
nulls_first: true,
5125+
},
5126+
]
5127+
}
5128+
5129+
#[test]
5130+
fn range_expr_round_trip_preserves_sort_options_and_split_points() -> Result<()> {
5131+
let schema = schema();
5132+
let sort_opts = sort_options();
5133+
let split_pts = split_points();
5134+
let on_columns = vec![col("a", &schema)?, col("b", &schema)?];
5135+
let range_partitioning = RangePartitioning::try_new(
5136+
[
5137+
PhysicalSortExpr::new(Arc::clone(&on_columns[0]), sort_opts[0]),
5138+
PhysicalSortExpr::new(Arc::clone(&on_columns[1]), sort_opts[1]),
5139+
]
5140+
.into(),
5141+
split_pts.clone(),
5142+
)?;
5143+
let expr =
5144+
RangeExpr::try_new_with_schema(on_columns, &range_partitioning, &schema)?;
5145+
5146+
let encoder = StubPlanEncoder::ok();
5147+
let encode_ctx = ExecutionPlanEncodeCtx::new(&encoder);
5148+
let node = PhysicalExpr::try_to_proto(&expr, &encode_ctx.expr_ctx())
5149+
.unwrap()
5150+
.expect("RangeExpr should encode to Some(node)");
5151+
5152+
let decoder = StubPlanDecoder::ok();
5153+
let decode_ctx = ExecutionPlanDecodeCtx::new(&decoder);
5154+
let decoded = RangeExpr::try_from_proto(&node, &decode_ctx.expr_ctx(&schema))?;
5155+
let decoded = decoded
5156+
.downcast_ref::<RangeExpr>()
5157+
.expect("decoded expression should be a RangeExpr");
5158+
5159+
assert_eq!(decoded.sort_options(), sort_opts);
5160+
assert_eq!(decoded.split_points(), split_pts);
5161+
Ok(())
5162+
}
5163+
}

0 commit comments

Comments
 (0)