Skip to content

Commit 075934f

Browse files
adriangbclaude
andcommitted
refactor: allow the hash join dynamic filter for any input partitioning
The routed filter checked a probe row only against the build keys of the partition that the router assigned the row to. It was correct only when the probe side was split the way the router assumed, so Partitioned joins needed co-partitioned Hash or Range inputs, and Hash inputs under `preserve_file_partitions` were rejected because file groups do not follow the hash router. The global filter holds the keys of all build partitions. It passes every probe row that any build partition can match, whatever the partitioning. This commit removes `has_partitioned_dynamic_filter_routing` and the `preserve_file_partitions` Hash guard, and replaces the two partitioning tests with one that checks the filter is allowed for every partitioning. This enables dynamic filters for `preserve_file_partitions` scans (apache#20195). Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent a57d1f4 commit 075934f

1 file changed

Lines changed: 18 additions & 83 deletions

File tree

  • datafusion/physical-plan/src/joins/hash_join

‎datafusion/physical-plan/src/joins/hash_join/exec.rs‎

Lines changed: 18 additions & 83 deletions
Original file line numberDiff line numberDiff line change
@@ -1055,57 +1055,12 @@ impl HashJoinExec {
10551055
return false;
10561056
}
10571057

1058-
// Under `preserve_file_partitions`, file groups can report Hash without following
1059-
// the dynamic filter hash router and look like a real hash repartition, so reject
1060-
// Hash. Range inputs with matching split points are safe.
1061-
// Follow-up work: enable dynamic filtering for preserve_file_partitioned scans (issue #20195).
1062-
// https://github.com/apache/datafusion/issues/20195
1063-
if config.optimizer.preserve_file_partitions > 0
1064-
&& self.mode == PartitionMode::Partitioned
1065-
&& matches!(
1066-
(
1067-
self.left.output_partitioning(),
1068-
self.right.output_partitioning()
1069-
),
1070-
(Partitioning::Hash(_, _), Partitioning::Hash(_, _))
1071-
)
1072-
{
1073-
return false;
1074-
}
1075-
1076-
if self.mode == PartitionMode::Partitioned
1077-
&& !self.has_partitioned_dynamic_filter_routing()
1078-
{
1079-
return false;
1080-
}
1081-
1058+
// The partitioning of the inputs does not matter. The filter holds the
1059+
// keys of all build partitions, so it passes every probe row that any
1060+
// build partition can match.
10821061
true
10831062
}
10841063

1085-
fn has_partitioned_dynamic_filter_routing(&self) -> bool {
1086-
match (
1087-
self.left.output_partitioning(),
1088-
self.right.output_partitioning(),
1089-
) {
1090-
(
1091-
Partitioning::Hash(_, left_partition_count),
1092-
Partitioning::Hash(_, right_partition_count),
1093-
) => left_partition_count == right_partition_count,
1094-
(Partitioning::Range(_), Partitioning::Range(_)) => {
1095-
let children = [self.left.as_ref(), self.right.as_ref()];
1096-
matches!(
1097-
self.input_distribution_requirements()
1098-
.unsatisfied_co_partitioned_children(self.name(), &children),
1099-
Ok(unsatisfied) if unsatisfied.is_empty()
1100-
)
1101-
}
1102-
(left_partitioning, right_partitioning) => {
1103-
left_partitioning.partition_count() == 1
1104-
&& right_partitioning.partition_count() == 1
1105-
}
1106-
}
1107-
}
1108-
11091064
/// left (build) side which gets hashed
11101065
pub fn left(&self) -> &Arc<dyn ExecutionPlan> {
11111066
&self.left
@@ -11398,52 +11353,32 @@ mod tests {
1139811353
.build()
1139911354
}
1140011355

11356+
/// The filter holds the keys of all build partitions, so it does not
11357+
/// depend on how the inputs are partitioned.
1140111358
#[test]
11402-
fn test_partitioned_dynamic_filter_pushdown_allows_supported_partitioning()
11403-
-> Result<()> {
11404-
let (range_join, on) = range_partitioned_dynamic_filter_test_join(10, 10)?;
11405-
let hash_join = with_hash_partitioned_children(&range_join, &on)?;
11406-
let mut session_config = SessionConfig::default();
11407-
session_config
11408-
.options_mut()
11409-
.optimizer
11410-
.enable_join_dynamic_filter_pushdown = true;
11411-
11412-
assert!(range_join.allow_join_dynamic_filter_pushdown(session_config.options()));
11413-
assert!(hash_join.allow_join_dynamic_filter_pushdown(session_config.options()));
11414-
11415-
session_config
11416-
.options_mut()
11417-
.optimizer
11418-
.preserve_file_partitions = 1;
11419-
assert!(range_join.allow_join_dynamic_filter_pushdown(session_config.options()));
11420-
11421-
Ok(())
11422-
}
11423-
11424-
#[test]
11425-
fn test_partitioned_dynamic_filter_pushdown_rejects_unsupported_partitioning()
11426-
-> Result<()> {
11359+
fn test_partitioned_dynamic_filter_pushdown_allows_any_partitioning() -> Result<()> {
1142711360
let (range_join, on) = range_partitioned_dynamic_filter_test_join(10, 10)?;
1142811361
let hash_join = with_hash_partitioned_children(&range_join, &on)?;
1142911362
let (mismatched_range_join, _) =
1143011363
range_partitioned_dynamic_filter_test_join(10, 11)?;
11364+
let joins = [&range_join, &hash_join, &mismatched_range_join];
1143111365
let mut session_config = SessionConfig::default();
1143211366
session_config
1143311367
.options_mut()
1143411368
.optimizer
1143511369
.enable_join_dynamic_filter_pushdown = true;
1143611370

11437-
assert!(
11438-
!mismatched_range_join
11439-
.allow_join_dynamic_filter_pushdown(session_config.options())
11440-
);
11441-
11442-
session_config
11443-
.options_mut()
11444-
.optimizer
11445-
.preserve_file_partitions = 1;
11446-
assert!(!hash_join.allow_join_dynamic_filter_pushdown(session_config.options()));
11371+
for preserve_file_partitions in [0, 1] {
11372+
session_config
11373+
.options_mut()
11374+
.optimizer
11375+
.preserve_file_partitions = preserve_file_partitions;
11376+
for join in joins {
11377+
assert!(
11378+
join.allow_join_dynamic_filter_pushdown(session_config.options())
11379+
);
11380+
}
11381+
}
1144711382

1144811383
Ok(())
1144911384
}

0 commit comments

Comments
 (0)