Skip to content

Commit 6a2eada

Browse files
adriangbclaude
andcommitted
refactor: allow the hash join dynamic filter for any input partitioning
The routed filter checked a probe row only against the build keys of the partition that the router assigned the row to. It was correct only when the probe side was split the way the router assumed, so Partitioned joins needed co-partitioned Hash or Range inputs, and Hash inputs under `preserve_file_partitions` were rejected because file groups do not follow the hash router. The global filter holds the keys of all build partitions. It passes every probe row that any build partition can match, whatever the partitioning. This commit removes `has_partitioned_dynamic_filter_routing` and the `preserve_file_partitions` Hash guard, and replaces the two partitioning tests with one that checks the filter is allowed for every partitioning. This enables dynamic filters for `preserve_file_partitions` scans (apache#20195). Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent dc548b9 commit 6a2eada

1 file changed

Lines changed: 18 additions & 83 deletions

File tree

  • datafusion/physical-plan/src/joins/hash_join

‎datafusion/physical-plan/src/joins/hash_join/exec.rs‎

Lines changed: 18 additions & 83 deletions
Original file line numberDiff line numberDiff line change
@@ -1113,57 +1113,12 @@ impl HashJoinExec {
11131113
return false;
11141114
}
11151115

1116-
// Under `preserve_file_partitions`, file groups can report Hash without following
1117-
// the dynamic filter hash router and look like a real hash repartition, so reject
1118-
// Hash. Range inputs with matching split points are safe.
1119-
// Follow-up work: enable dynamic filtering for preserve_file_partitioned scans (issue #20195).
1120-
// https://github.com/apache/datafusion/issues/20195
1121-
if config.optimizer.preserve_file_partitions > 0
1122-
&& self.mode == PartitionMode::Partitioned
1123-
&& matches!(
1124-
(
1125-
self.left.output_partitioning(),
1126-
self.right.output_partitioning()
1127-
),
1128-
(Partitioning::Hash(_, _), Partitioning::Hash(_, _))
1129-
)
1130-
{
1131-
return false;
1132-
}
1133-
1134-
if self.mode == PartitionMode::Partitioned
1135-
&& !self.has_partitioned_dynamic_filter_routing()
1136-
{
1137-
return false;
1138-
}
1139-
1116+
// The partitioning of the inputs does not matter. The filter holds the
1117+
// keys of all build partitions, so it passes every probe row that any
1118+
// build partition can match.
11401119
true
11411120
}
11421121

1143-
fn has_partitioned_dynamic_filter_routing(&self) -> bool {
1144-
match (
1145-
self.left.output_partitioning(),
1146-
self.right.output_partitioning(),
1147-
) {
1148-
(
1149-
Partitioning::Hash(_, left_partition_count),
1150-
Partitioning::Hash(_, right_partition_count),
1151-
) => left_partition_count == right_partition_count,
1152-
(Partitioning::Range(_), Partitioning::Range(_)) => {
1153-
let children = [self.left.as_ref(), self.right.as_ref()];
1154-
matches!(
1155-
self.input_distribution_requirements()
1156-
.unsatisfied_co_partitioned_children(self.name(), &children),
1157-
Ok(unsatisfied) if unsatisfied.is_empty()
1158-
)
1159-
}
1160-
(left_partitioning, right_partitioning) => {
1161-
left_partitioning.partition_count() == 1
1162-
&& right_partitioning.partition_count() == 1
1163-
}
1164-
}
1165-
}
1166-
11671122
/// left (build) side which gets hashed
11681123
pub fn left(&self) -> &Arc<dyn ExecutionPlan> {
11691124
&self.left
@@ -12246,52 +12201,32 @@ mod tests {
1224612201
.build()
1224712202
}
1224812203

12204+
/// The filter holds the keys of all build partitions, so it does not
12205+
/// depend on how the inputs are partitioned.
1224912206
#[test]
12250-
fn test_partitioned_dynamic_filter_pushdown_allows_supported_partitioning()
12251-
-> Result<()> {
12252-
let (range_join, on) = range_partitioned_dynamic_filter_test_join(10, 10)?;
12253-
let hash_join = with_hash_partitioned_children(&range_join, &on)?;
12254-
let mut session_config = SessionConfig::default();
12255-
session_config
12256-
.options_mut()
12257-
.optimizer
12258-
.enable_join_dynamic_filter_pushdown = true;
12259-
12260-
assert!(range_join.allow_join_dynamic_filter_pushdown(session_config.options()));
12261-
assert!(hash_join.allow_join_dynamic_filter_pushdown(session_config.options()));
12262-
12263-
session_config
12264-
.options_mut()
12265-
.optimizer
12266-
.preserve_file_partitions = 1;
12267-
assert!(range_join.allow_join_dynamic_filter_pushdown(session_config.options()));
12268-
12269-
Ok(())
12270-
}
12271-
12272-
#[test]
12273-
fn test_partitioned_dynamic_filter_pushdown_rejects_unsupported_partitioning()
12274-
-> Result<()> {
12207+
fn test_partitioned_dynamic_filter_pushdown_allows_any_partitioning() -> Result<()> {
1227512208
let (range_join, on) = range_partitioned_dynamic_filter_test_join(10, 10)?;
1227612209
let hash_join = with_hash_partitioned_children(&range_join, &on)?;
1227712210
let (mismatched_range_join, _) =
1227812211
range_partitioned_dynamic_filter_test_join(10, 11)?;
12212+
let joins = [&range_join, &hash_join, &mismatched_range_join];
1227912213
let mut session_config = SessionConfig::default();
1228012214
session_config
1228112215
.options_mut()
1228212216
.optimizer
1228312217
.enable_join_dynamic_filter_pushdown = true;
1228412218

12285-
assert!(
12286-
!mismatched_range_join
12287-
.allow_join_dynamic_filter_pushdown(session_config.options())
12288-
);
12289-
12290-
session_config
12291-
.options_mut()
12292-
.optimizer
12293-
.preserve_file_partitions = 1;
12294-
assert!(!hash_join.allow_join_dynamic_filter_pushdown(session_config.options()));
12219+
for preserve_file_partitions in [0, 1] {
12220+
session_config
12221+
.options_mut()
12222+
.optimizer
12223+
.preserve_file_partitions = preserve_file_partitions;
12224+
for join in joins {
12225+
assert!(
12226+
join.allow_join_dynamic_filter_pushdown(session_config.options())
12227+
);
12228+
}
12229+
}
1229512230

1229612231
Ok(())
1229712232
}

0 commit comments

Comments
 (0)