Skip to content

Commit 22159ad

Browse files
author
Gene Bordegaray
committed
Add Distribution::HashPartitioned to Distribution::KeyPartitioned API bridge (apache#23259)
<!-- We generally require a GitHub issue to be filed for all bug fixes and enhancements and this helps us generate change logs for our releases. You can link to an issue to this PR using the GitHub syntax. For example `Closes #123` indicates that this PR will close issue #123. --> - Closes apache#23236. <!-- Why are you proposing this change? If this is already explained clearly in the issue then this section is not needed. Explaining clearly why changes are proposed helps reviewers understand your changes and offer better suggestions for fixes. --> `HashPartitioned` is historical naming for a key-partitioning requirement. This keeps the old variant as a deprecated compatibility bridge while moving DataFusion internals to `KeyPartitioned`. <!-- There is no need to duplicate the description in the issue here but it is sometimes worth providing a summary of the individual changes in this PR. --> Adds `KeyPartitioned`, deprecates `HashPartitioned`, and treats both equivalently during the transition to avoid breaking changes for downstream consumers. A blast radius report on this was done here apache#23241 <!-- We typically require tests for all PRs in order to: 1. Prevent the code from being accidentally broken by subsequent changes 2. Serve as another way to document the expected behavior of the code If tests are not included in your PR, please explain why (for example, are they covered by existing tests)? --> Yes. <!-- If there are user-facing changes then we may require documentation to be updated before approving the PR. --> Yes, `Distribution::HashPartitioned` is deprecated and trainsitioned to `Distribution::KeyPartitioned`. <!-- If there are any breaking changes to public APIs, please add the `api change` label. -->
1 parent 7a05648 commit 22159ad

12 files changed

Lines changed: 119 additions & 83 deletions

File tree

‎datafusion/core/tests/physical_optimizer/projection_pushdown.rs‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -724,7 +724,7 @@ fn test_output_req_after_projection() -> Result<()> {
724724
]
725725
.into(),
726726
)),
727-
Distribution::HashPartitioned(vec![
727+
Distribution::KeyPartitioned(vec![
728728
Arc::new(Column::new("a", 0)),
729729
Arc::new(Column::new("b", 1)),
730730
]),
@@ -746,7 +746,7 @@ fn test_output_req_after_projection() -> Result<()> {
746746
actual,
747747
@r"
748748
ProjectionExec: expr=[c@2 as c, a@0 as new_a, b@1 as b]
749-
OutputRequirementExec: order_by=[(b@1, asc), (c@2 + a@0, asc)], dist_by=HashPartitioned[[a@0, b@1]])
749+
OutputRequirementExec: order_by=[(b@1, asc), (c@2 + a@0, asc)], dist_by=KeyPartitioned[[a@0, b@1]])
750750
DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=csv, has_header=false
751751
"
752752
);
@@ -762,7 +762,7 @@ fn test_output_req_after_projection() -> Result<()> {
762762
assert_snapshot!(
763763
actual,
764764
@r"
765-
OutputRequirementExec: order_by=[(b@2, asc), (c@0 + new_a@1, asc)], dist_by=HashPartitioned[[new_a@1, b@2]])
765+
OutputRequirementExec: order_by=[(b@2, asc), (c@0 + new_a@1, asc)], dist_by=KeyPartitioned[[new_a@1, b@2]])
766766
DataSourceExec: file_groups={1 group: [[x]]}, projection=[c, a@0 as new_a, b], file_type=csv, has_header=false
767767
"
768768
);
@@ -797,7 +797,7 @@ fn test_output_req_after_projection() -> Result<()> {
797797
Arc::new(Column::new("new_a", 1)),
798798
Arc::new(Column::new("b", 2)),
799799
];
800-
if let Distribution::HashPartitioned(vec) = after_optimize
800+
if let Distribution::KeyPartitioned(vec) = after_optimize
801801
.downcast_ref::<OutputRequirementExec>()
802802
.unwrap()
803803
.required_input_distribution()[0]
@@ -809,7 +809,7 @@ fn test_output_req_after_projection() -> Result<()> {
809809
.all(|(actual, expected)| actual.eq(&expected))
810810
);
811811
} else {
812-
panic!("Expected HashPartitioned distribution!");
812+
panic!("Expected KeyPartitioned distribution!");
813813
};
814814

815815
Ok(())

‎datafusion/physical-expr/src/partitioning.rs‎

Lines changed: 84 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -518,6 +518,10 @@ impl Partitioning {
518518

519519
/// Returns how this [`Partitioning`] satisfies the partitioning scheme mandated
520520
/// by the `required` [`Distribution`].
521+
#[expect(
522+
deprecated,
523+
reason = "HashPartitioned is accepted during the KeyPartitioned migration"
524+
)]
521525
pub fn satisfaction(
522526
&self,
523527
required: &Distribution,
@@ -639,8 +643,9 @@ pub enum Distribution {
639643
UnspecifiedDistribution,
640644
/// A single partition is required
641645
SinglePartition,
642-
/// Requires children to be distributed in such a way that the same
643-
/// values of the keys end up in the same partition
646+
/// Deprecated historical name for [`Distribution::KeyPartitioned`].
647+
/// See <https://github.com/apache/datafusion/issues/23236> for details.
648+
#[deprecated(since = "55.0.0", note = "Use Distribution::KeyPartitioned")]
644649
HashPartitioned(Vec<Arc<dyn PhysicalExpr>>),
645650
/// Requires rows with equal values for the given keys to be colocated in
646651
/// the same partition, without requiring a specific partitioning algorithm.
@@ -657,6 +662,10 @@ pub enum Distribution {
657662
KeyPartitioned(Vec<Arc<dyn PhysicalExpr>>),
658663
}
659664

665+
#[expect(
666+
deprecated,
667+
reason = "HashPartitioned is accepted during the KeyPartitioned migration"
668+
)]
660669
impl Distribution {
661670
/// Returns key expressions for distribution variants that require
662671
/// co-locating equal key values.
@@ -681,6 +690,10 @@ impl Distribution {
681690
}
682691
}
683692

693+
#[expect(
694+
deprecated,
695+
reason = "HashPartitioned display is preserved during the KeyPartitioned migration"
696+
)]
684697
impl Display for Distribution {
685698
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
686699
match self {
@@ -760,11 +773,11 @@ mod tests {
760773
Partitioning::Hash(self.cols(indices), partition_count)
761774
}
762775

763-
fn hash_distribution(
776+
fn key_distribution(
764777
&self,
765778
indices: impl IntoIterator<Item = usize>,
766779
) -> Distribution {
767-
Distribution::HashPartitioned(self.cols(indices))
780+
Distribution::KeyPartitioned(self.cols(indices))
768781
}
769782

770783
fn key_partitioned_distribution(
@@ -824,6 +837,10 @@ mod tests {
824837
}
825838

826839
#[test]
840+
#[expect(
841+
deprecated,
842+
reason = "test intentionally covers deprecated HashPartitioned compatibility"
843+
)]
827844
fn partitioning_satisfy_distribution() -> Result<()> {
828845
let fixture = PartitioningTestFixture::new(vec![
829846
("column_1", DataType::Int64),
@@ -869,7 +886,7 @@ mod tests {
869886
Distribution::SinglePartition => {
870887
assert_eq!(result, (true, false, false, false, false))
871888
}
872-
Distribution::HashPartitioned(_) => {
889+
Distribution::HashPartitioned(_) | Distribution::KeyPartitioned(_) => {
873890
assert_eq!(result, (true, false, false, true, false))
874891
}
875892
Distribution::KeyPartitioned(_) => {
@@ -881,43 +898,66 @@ mod tests {
881898
Ok(())
882899
}
883900

901+
#[test]
902+
#[expect(
903+
deprecated,
904+
reason = "test intentionally covers deprecated HashPartitioned compatibility"
905+
)]
906+
fn deprecated_hash_partitioned_matches_key_partitioned() -> Result<()> {
907+
let fixture = PartitioningTestFixture::int64(&["a", "b"])?;
908+
let partitioning = fixture.hash_partitioning([0, 1], 4);
909+
let hash_distribution = Distribution::HashPartitioned(fixture.cols([0, 1]));
910+
let key_distribution = fixture.key_distribution([0, 1]);
911+
912+
assert_eq!(
913+
partitioning.satisfaction(&hash_distribution, &fixture.eq_properties, false),
914+
partitioning.satisfaction(&key_distribution, &fixture.eq_properties, false)
915+
);
916+
assert_eq!(
917+
hash_distribution.create_partitioning(4),
918+
key_distribution.create_partitioning(4)
919+
);
920+
921+
Ok(())
922+
}
923+
884924
#[test]
885925
fn test_partitioning_satisfy_by_subset() -> Result<()> {
886926
let fixture = PartitioningTestFixture::int64(&["a", "b", "c"])?;
887927

888928
let test_cases = vec![
889929
(
890-
"Hash([a]) vs Hash([a, b])",
930+
"KeyPartitioned([a, b]) satisfied by Hash([a])",
891931
fixture.hash_partitioning([0], 4),
892-
fixture.hash_distribution([0, 1]),
932+
fixture.key_distribution([0, 1]),
893933
PartitioningSatisfaction::Subset,
894934
PartitioningSatisfaction::NotSatisfied,
895935
),
896936
(
897-
"Hash([a]) vs Hash([a, b, c])",
937+
"KeyPartitioned([a, b, c]) satisfied by Hash([a])",
898938
fixture.hash_partitioning([0], 4),
899-
fixture.hash_distribution([0, 1, 2]),
939+
fixture.key_distribution([0, 1, 2]),
900940
PartitioningSatisfaction::Subset,
901941
PartitioningSatisfaction::NotSatisfied,
902942
),
903943
(
904-
"Hash([a, b]) vs Hash([a, b, c])",
944+
"KeyPartitioned([a, b, c]) satisfied by Hash([a, b])",
905945
fixture.hash_partitioning([0, 1], 4),
906-
fixture.hash_distribution([0, 1, 2]),
946+
fixture.key_distribution([0, 1, 2]),
907947
PartitioningSatisfaction::Subset,
908948
PartitioningSatisfaction::NotSatisfied,
909949
),
910950
(
911-
"Hash([b]) vs Hash([a, b, c])",
951+
"KeyPartitioned([a, b, c]) satisfied by Hash([b])",
912952
fixture.hash_partitioning([1], 4),
913-
fixture.hash_distribution([0, 1, 2]),
953+
fixture.key_distribution([0, 1, 2]),
914954
PartitioningSatisfaction::Subset,
915955
PartitioningSatisfaction::NotSatisfied,
916956
),
917957
(
918-
"Hash([b, a]) vs Hash([a, b, c])",
958+
"KeyPartitioned([a, b, c]) satisfied by Hash([b, a])",
919959
fixture.hash_partitioning([1, 0], 4),
920-
fixture.hash_distribution([0, 1, 2]),
960+
fixture.key_distribution([0, 1, 2]),
921961
PartitioningSatisfaction::Subset,
922962
PartitioningSatisfaction::NotSatisfied,
923963
),
@@ -948,23 +988,23 @@ mod tests {
948988

949989
let test_cases = vec![
950990
(
951-
"Hash([a, b]) vs Hash([a])",
991+
"KeyPartitioned([a]) satisfied by Hash([a, b])",
952992
fixture.hash_partitioning([0, 1], 4),
953-
fixture.hash_distribution([0]),
993+
fixture.key_distribution([0]),
954994
PartitioningSatisfaction::NotSatisfied,
955995
PartitioningSatisfaction::NotSatisfied,
956996
),
957997
(
958-
"Hash([a, b, c]) vs Hash([a])",
998+
"KeyPartitioned([a]) satisfied by Hash([a, b, c])",
959999
fixture.hash_partitioning([0, 1, 2], 4),
960-
fixture.hash_distribution([0]),
1000+
fixture.key_distribution([0]),
9611001
PartitioningSatisfaction::NotSatisfied,
9621002
PartitioningSatisfaction::NotSatisfied,
9631003
),
9641004
(
965-
"Hash([a, b, c]) vs Hash([a, b])",
1005+
"KeyPartitioned([a, b]) satisfied by Hash([a, b, c])",
9661006
fixture.hash_partitioning([0, 1, 2], 4),
967-
fixture.hash_distribution([0, 1]),
1007+
fixture.key_distribution([0, 1]),
9681008
PartitioningSatisfaction::NotSatisfied,
9691009
PartitioningSatisfaction::NotSatisfied,
9701010
),
@@ -994,9 +1034,9 @@ mod tests {
9941034
let fixture = PartitioningTestFixture::int64(&["a", "b", "c"])?;
9951035

9961036
let test_cases = vec![(
997-
"Partial overlap: Hash([a, c]) vs Hash([a, b])",
1037+
"Partial overlap: KeyPartitioned([a, b]) satisfied by Hash([a, c])",
9981038
fixture.hash_partitioning([0, 2], 4),
999-
fixture.hash_distribution([0, 1]),
1039+
fixture.key_distribution([0, 1]),
10001040
PartitioningSatisfaction::NotSatisfied,
10011041
PartitioningSatisfaction::NotSatisfied,
10021042
)];
@@ -1026,16 +1066,16 @@ mod tests {
10261066

10271067
let test_cases = vec![
10281068
(
1029-
"Hash([a]) vs Hash([b, c])",
1069+
"KeyPartitioned([b, c]) satisfied by Hash([a])",
10301070
fixture.hash_partitioning([0], 4),
1031-
fixture.hash_distribution([1, 2]),
1071+
fixture.key_distribution([1, 2]),
10321072
PartitioningSatisfaction::NotSatisfied,
10331073
PartitioningSatisfaction::NotSatisfied,
10341074
),
10351075
(
1036-
"Hash([a, b]) vs Hash([c])",
1076+
"KeyPartitioned([c]) satisfied by Hash([a, b])",
10371077
fixture.hash_partitioning([0, 1], 4),
1038-
fixture.hash_distribution([2]),
1078+
fixture.key_distribution([2]),
10391079
PartitioningSatisfaction::NotSatisfied,
10401080
PartitioningSatisfaction::NotSatisfied,
10411081
),
@@ -1066,16 +1106,16 @@ mod tests {
10661106

10671107
let test_cases = vec![
10681108
(
1069-
"Hash([a, b]) vs Hash([a, b])",
1109+
"KeyPartitioned([a, b]) satisfied by Hash([a, b])",
10701110
fixture.hash_partitioning([0, 1], 4),
1071-
fixture.hash_distribution([0, 1]),
1111+
fixture.key_distribution([0, 1]),
10721112
PartitioningSatisfaction::Exact,
10731113
PartitioningSatisfaction::Exact,
10741114
),
10751115
(
1076-
"Hash([a]) vs Hash([a])",
1116+
"KeyPartitioned([a]) satisfied by Hash([a])",
10771117
fixture.hash_partitioning([0], 4),
1078-
fixture.hash_distribution([0]),
1118+
fixture.key_distribution([0]),
10791119
PartitioningSatisfaction::Exact,
10801120
PartitioningSatisfaction::Exact,
10811121
),
@@ -1107,23 +1147,23 @@ mod tests {
11071147

11081148
let test_cases = vec![
11091149
(
1110-
"Hash([unknown]) vs Hash([a, b])",
1150+
"KeyPartitioned([a, b]) satisfied by Hash([unknown])",
11111151
Partitioning::Hash(vec![Arc::clone(&unknown)], 4),
1112-
fixture.hash_distribution([0, 1]),
1152+
fixture.key_distribution([0, 1]),
11131153
PartitioningSatisfaction::NotSatisfied,
11141154
PartitioningSatisfaction::NotSatisfied,
11151155
),
11161156
(
1117-
"Hash([a, b]) vs Hash([unknown])",
1157+
"KeyPartitioned([unknown]) satisfied by Hash([a, b])",
11181158
fixture.hash_partitioning([0, 1], 4),
1119-
Distribution::HashPartitioned(vec![Arc::clone(&unknown)]),
1159+
Distribution::KeyPartitioned(vec![Arc::clone(&unknown)]),
11201160
PartitioningSatisfaction::NotSatisfied,
11211161
PartitioningSatisfaction::NotSatisfied,
11221162
),
11231163
(
1124-
"Hash([unknown]) vs Hash([unknown])",
1164+
"KeyPartitioned([unknown]) satisfied by Hash([unknown])",
11251165
Partitioning::Hash(vec![Arc::clone(&unknown)], 4),
1126-
Distribution::HashPartitioned(vec![Arc::clone(&unknown)]),
1166+
Distribution::KeyPartitioned(vec![Arc::clone(&unknown)]),
11271167
PartitioningSatisfaction::NotSatisfied,
11281168
PartitioningSatisfaction::NotSatisfied,
11291169
),
@@ -1154,23 +1194,23 @@ mod tests {
11541194

11551195
let test_cases = vec![
11561196
(
1157-
"Hash([]) vs Hash([a])",
1197+
"KeyPartitioned([a]) satisfied by Hash([])",
11581198
Partitioning::Hash(vec![], 4),
1159-
fixture.hash_distribution([0]),
1199+
fixture.key_distribution([0]),
11601200
PartitioningSatisfaction::NotSatisfied,
11611201
PartitioningSatisfaction::NotSatisfied,
11621202
),
11631203
(
1164-
"Hash([a]) vs Hash([])",
1204+
"KeyPartitioned([]) satisfied by Hash([a])",
11651205
fixture.hash_partitioning([0], 4),
1166-
Distribution::HashPartitioned(vec![]),
1206+
Distribution::KeyPartitioned(vec![]),
11671207
PartitioningSatisfaction::NotSatisfied,
11681208
PartitioningSatisfaction::NotSatisfied,
11691209
),
11701210
(
1171-
"Hash([]) vs Hash([])",
1211+
"KeyPartitioned([]) satisfied by Hash([])",
11721212
Partitioning::Hash(vec![], 4),
1173-
Distribution::HashPartitioned(vec![]),
1213+
Distribution::KeyPartitioned(vec![]),
11741214
PartitioningSatisfaction::NotSatisfied,
11751215
PartitioningSatisfaction::NotSatisfied,
11761216
),
@@ -1573,7 +1613,7 @@ mod tests {
15731613
let fixture = PartitioningTestFixture::int64(&["a", "b"])?;
15741614
let range_partitioning =
15751615
fixture.range_partitioning([0, 1], vec![int_split_point([10, 100])]);
1576-
let required = fixture.hash_distribution([0, 1]);
1616+
let required = fixture.key_distribution([0, 1]);
15771617

15781618
assert_eq!(
15791619
range_partitioning.satisfaction(&required, &fixture.eq_properties, false),

‎datafusion/physical-optimizer/src/enforce_distribution.rs‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -888,7 +888,7 @@ fn add_hash_on_top(
888888
return Ok(input);
889889
}
890890

891-
let dist = Distribution::HashPartitioned(hash_exprs);
891+
let dist = Distribution::KeyPartitioned(hash_exprs);
892892
let current_partitions = input.plan.output_partitioning().partition_count();
893893
let satisfaction = input.plan.output_partitioning().satisfaction(
894894
&dist,
@@ -1326,6 +1326,10 @@ fn partitioned_join_distribution(
13261326
/// This function is intended to be used in a bottom up traversal, as it
13271327
/// can first repartition (or newly partition) at the datasources -- these
13281328
/// source partitions may be later repartitioned with additional data exchange operators.
1329+
#[expect(
1330+
deprecated,
1331+
reason = "HashPartitioned is accepted during the KeyPartitioned migration"
1332+
)]
13291333
pub fn ensure_distribution(
13301334
dist_context: DistributionContext,
13311335
config: &ConfigOptions,

0 commit comments

Comments
 (0)