Skip to content

Commit 19d31c5

Browse files
authored
refactor: thread PhysicalOptimizerContext through join_selection stats helpers (apache#24716)
## Which issue does this PR close? - Closes apache#23671. ## Rationale for this change `JoinSelection`'s internal statistics helpers each took separate `config: &ConfigOptions` and `registry: Option<&StatisticsRegistry>` parameters. Any new session-scoped CBO input to these rules (e.g. apache#21120) would require touching every helper's signature. ## What changes are included in this PR? Replace the `config`/`registry` parameter pair on the internal statistics helpers with a single `context: &dyn PhysicalOptimizerContext`, reading `context.config_options()` and `context.statistics_registry()` internally. This is a pure internal signature change: all affected functions are private or `pub(crate)`, with no cross-crate callers. Behavior is unchanged. ## Are these changes tested? No new tests; this is a non-functional refactor covered by existing tests (`cargo test -p datafusion-physical-optimizer`, the `physical_optimizer::join_selection` integration tests, and `statistics_registry.slt`). ## Are there any user-facing changes? No. ---- Disclaimer: I used AI to assist in the code generation, I have manually reviewed the output and it matches my intention and understanding.
1 parent 17547e2 commit 19d31c5

2 files changed

Lines changed: 111 additions & 50 deletions

File tree

‎datafusion/core/tests/physical_optimizer/join_selection.rs‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,12 +37,16 @@ use datafusion_physical_expr::expressions::{BinaryExpr, Column, NegativeExpr};
3737
use datafusion_physical_expr::intervals::utils::check_support;
3838
use datafusion_physical_expr::{EquivalenceProperties, Partitioning, PhysicalExpr};
3939
use datafusion_physical_expr_common::sort_expr::PhysicalSortExpr;
40+
use datafusion_physical_optimizer::PhysicalOptimizerContext;
4041
use datafusion_physical_optimizer::PhysicalOptimizerRule;
4142
use datafusion_physical_optimizer::join_selection::JoinSelection;
4243
use datafusion_physical_plan::displayable;
4344
use datafusion_physical_plan::joins::utils::ColumnIndex;
4445
use datafusion_physical_plan::joins::utils::JoinFilter;
4546
use datafusion_physical_plan::joins::{HashJoinExec, NestedLoopJoinExec, PartitionMode};
47+
use datafusion_physical_plan::operator_statistics::{
48+
ClosureStatisticsProvider, StatisticsRegistry, StatisticsResult,
49+
};
4650
use datafusion_physical_plan::projection::ProjectionExec;
4751
use datafusion_physical_plan::sorts::sort_preserving_merge::SortPreservingMergeExec;
4852
use datafusion_physical_plan::{
@@ -640,6 +644,72 @@ async fn test_null_aware_left_anti_with_filter_does_not_swap() -> Result<()> {
640644
Ok(())
641645
}
642646

647+
#[tokio::test]
648+
async fn test_join_swap_uses_context_statistics_registry() -> Result<()> {
649+
let left: Arc<dyn ExecutionPlan> = Arc::new(StatisticsExec::new(
650+
big_statistics(),
651+
Schema::new(vec![Field::new("left_col", DataType::Int32, false)]),
652+
));
653+
let right: Arc<dyn ExecutionPlan> = Arc::new(StatisticsExec::new(
654+
big_statistics(),
655+
Schema::new(vec![Field::new("right_col", DataType::Int32, false)]),
656+
));
657+
let join = HashJoinExec::try_new(
658+
Arc::clone(&left),
659+
Arc::clone(&right),
660+
vec![(
661+
Arc::new(Column::new_with_schema("left_col", &left.schema())?),
662+
Arc::new(Column::new_with_schema("right_col", &right.schema())?),
663+
)],
664+
None,
665+
&JoinType::LeftAnti,
666+
None,
667+
PartitionMode::Auto,
668+
NullEquality::NullEqualsNothing,
669+
false,
670+
)?;
671+
672+
// Reports `right` as much smaller than its real stats: the swap below only
673+
// happens if `context`'s registry is actually threaded through.
674+
let mut registry = StatisticsRegistry::new();
675+
registry.register(Arc::new(ClosureStatisticsProvider::with_matches(
676+
|plan| plan.schema().field(0).name() == "right_col",
677+
|_plan, _child_stats| Ok(StatisticsResult::Computed(small_statistics().into())),
678+
)));
679+
680+
struct ContextWithRegistry {
681+
config: ConfigOptions,
682+
registry: StatisticsRegistry,
683+
}
684+
685+
impl PhysicalOptimizerContext for ContextWithRegistry {
686+
fn config_options(&self) -> &ConfigOptions {
687+
&self.config
688+
}
689+
690+
fn statistics_registry(&self) -> Option<&StatisticsRegistry> {
691+
Some(&self.registry)
692+
}
693+
}
694+
695+
let context = ContextWithRegistry {
696+
config: ConfigOptions::new(),
697+
registry,
698+
};
699+
700+
let optimized_join =
701+
JoinSelection::new().optimize_with_context(Arc::new(join), &context)?;
702+
let swapped_join = optimized_join
703+
.downcast_ref::<HashJoinExec>()
704+
.expect("optimize_with_context should return a HashJoinExec");
705+
706+
assert_eq!(*swapped_join.join_type(), JoinType::RightAnti);
707+
assert_eq!(swapped_join.left().schema().field(0).name(), "right_col");
708+
assert_eq!(swapped_join.right().schema().field(0).name(), "left_col");
709+
710+
Ok(())
711+
}
712+
643713
#[tokio::test]
644714
async fn test_join_with_swap_mark() {
645715
let join_types = [JoinType::LeftMark, JoinType::RightMark];

‎datafusion/physical-optimizer/src/join_selection.rs‎

Lines changed: 41 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,6 @@ use datafusion_physical_plan::joins::{
3939
CrossJoinExec, HashJoinExec, NestedLoopJoinExec, PartitionMode,
4040
StreamJoinPartitionMode, SymmetricHashJoinExec,
4141
};
42-
use datafusion_physical_plan::operator_statistics::StatisticsRegistry;
4342
use datafusion_physical_plan::statistics::{StatisticsArgs, StatisticsContext};
4443
use datafusion_physical_plan::{ExecutionPlan, ExecutionPlanProperties};
4544
use std::sync::Arc;
@@ -61,9 +60,9 @@ impl JoinSelection {
6160
/// is available.
6261
fn get_stats(
6362
plan: &dyn ExecutionPlan,
64-
registry: Option<&StatisticsRegistry>,
63+
context: &dyn PhysicalOptimizerContext,
6564
) -> Result<Arc<Statistics>> {
66-
let ctx = match registry {
65+
let ctx = match context.statistics_registry() {
6766
Some(reg) => StatisticsContext::new_with_registry(reg.clone()),
6867
None => StatisticsContext::new(),
6968
};
@@ -75,28 +74,28 @@ fn get_stats(
7574
/// Checks whether join inputs should be swapped using available statistics.
7675
///
7776
/// It follows these steps:
78-
/// 1. If a [`StatisticsRegistry`] is provided, use it for cross-operator estimates
77+
/// 1. If a [`datafusion_physical_plan::operator_statistics::StatisticsRegistry`] is
78+
/// provided, use it for cross-operator estimates
7979
/// (e.g., intermediate join outputs that would otherwise have `Absent` statistics).
8080
/// 2. Compare the in-memory sizes of both sides, and place the smaller side on
8181
/// the left (build) side.
8282
/// 3. If in-memory byte sizes are unavailable, fall back to row counts.
8383
/// 4. Do not reorder the join if neither statistic is available, or if
8484
/// `datafusion.optimizer.join_reordering` is disabled.
8585
///
86-
/// Used configurations inside arg `config`
87-
/// - `config.optimizer.join_reordering`: allows or forbids statistics-driven join swapping
86+
/// Used configurations
87+
/// - `optimizer.join_reordering`: allows or forbids statistics-driven join swapping
8888
pub(crate) fn should_swap_join_order(
8989
left: &dyn ExecutionPlan,
9090
right: &dyn ExecutionPlan,
91-
config: &ConfigOptions,
92-
registry: Option<&StatisticsRegistry>,
91+
context: &dyn PhysicalOptimizerContext,
9392
) -> Result<bool> {
94-
if !config.optimizer.join_reordering {
93+
if !context.config_options().optimizer.join_reordering {
9594
return Ok(false);
9695
}
9796

98-
let left_stats = get_stats(left, registry)?;
99-
let right_stats = get_stats(right, registry)?;
97+
let left_stats = get_stats(left, context)?;
98+
let right_stats = get_stats(right, context)?;
10099

101100
// First compare total_byte_size, then fall back to num_rows if byte
102101
// sizes are unavailable.
@@ -119,9 +118,9 @@ fn supports_collect_by_thresholds(
119118
plan: &dyn ExecutionPlan,
120119
threshold_byte_size: usize,
121120
threshold_num_rows: usize,
122-
registry: Option<&StatisticsRegistry>,
121+
context: &dyn PhysicalOptimizerContext,
123122
) -> bool {
124-
let Ok(stats) = get_stats(plan, registry) else {
123+
let Ok(stats) = get_stats(plan, context) else {
125124
return false;
126125
};
127126

@@ -152,19 +151,15 @@ impl PhysicalOptimizerRule for JoinSelection {
152151
plan: Arc<dyn ExecutionPlan>,
153152
context: &dyn PhysicalOptimizerContext,
154153
) -> Result<Arc<dyn ExecutionPlan>> {
155-
let config = context.config_options();
156-
let registry = context.statistics_registry();
157154
let subrules: Vec<Box<PipelineFixerSubrule>> = vec![
158155
Box::new(hash_join_convert_symmetric_subrule),
159156
Box::new(hash_join_swap_subrule),
160157
];
161158
let new_plan = plan
162-
.transform_up(|p| apply_subrules(p, &subrules, config))
159+
.transform_up(|p| apply_subrules(p, &subrules, context.config_options()))
163160
.data()?;
164161
new_plan
165-
.transform_up(|plan| {
166-
statistical_join_selection_subrule(plan, config, registry)
167-
})
162+
.transform_up(|plan| statistical_join_selection_subrule(plan, context))
168163
.data()
169164
}
170165

@@ -192,40 +187,39 @@ fn can_swap_hash_join(hash_join: &HashJoinExec) -> bool {
192187
/// When the `ignore_threshold` is false, this function will also check left
193188
/// and right sizes in bytes or rows.
194189
///
195-
/// Used configurations inside arg `config`
196-
/// - `config.optimizer.hash_join_single_partition_threshold`: byte threshold for `CollectLeft`
197-
/// - `config.optimizer.hash_join_single_partition_threshold_rows`: row threshold for `CollectLeft`
198-
/// - `config.optimizer.join_reordering`: allows or forbids input swapping
190+
/// Used configurations
191+
/// - `optimizer.hash_join_single_partition_threshold`: byte threshold for `CollectLeft`
192+
/// - `optimizer.hash_join_single_partition_threshold_rows`: row threshold for `CollectLeft`
193+
/// - `optimizer.join_reordering`: allows or forbids input swapping
199194
pub(crate) fn try_collect_left(
200195
hash_join: &HashJoinExec,
201196
ignore_threshold: bool,
202-
config: &ConfigOptions,
203-
registry: Option<&StatisticsRegistry>,
197+
context: &dyn PhysicalOptimizerContext,
204198
) -> Result<Option<Arc<dyn ExecutionPlan>>> {
205199
let left = hash_join.left();
206200
let right = hash_join.right();
207-
let optimizer_config = &config.optimizer;
201+
let optimizer_config = &context.config_options().optimizer;
208202

209203
let left_can_collect = ignore_threshold
210204
|| supports_collect_by_thresholds(
211205
&**left,
212206
optimizer_config.hash_join_single_partition_threshold,
213207
optimizer_config.hash_join_single_partition_threshold_rows,
214-
registry,
208+
context,
215209
);
216210
let right_can_collect = ignore_threshold
217211
|| supports_collect_by_thresholds(
218212
&**right,
219213
optimizer_config.hash_join_single_partition_threshold,
220214
optimizer_config.hash_join_single_partition_threshold_rows,
221-
registry,
215+
context,
222216
);
223217

224218
match (left_can_collect, right_can_collect) {
225219
(true, true) => {
226220
// For null-aware joins, we only swap `LeftAnti` joins where the left side is > right side
227221
if can_swap_hash_join(hash_join)
228-
&& should_swap_join_order(&**left, &**right, config, registry)?
222+
&& should_swap_join_order(&**left, &**right, context)?
229223
{
230224
Ok(Some(hash_join.swap_inputs(PartitionMode::CollectLeft)?))
231225
} else {
@@ -260,12 +254,11 @@ pub(crate) fn try_collect_left(
260254
/// If swapping is optimal and supported, creates a swapped partitioned hash join; otherwise,
261255
/// creates a standard partitioned hash join.
262256
///
263-
/// Used configurations inside arg `config`
264-
/// - `config.optimizer.join_reordering`: allows or forbids statistics-driven join swapping
257+
/// Used configurations
258+
/// - `optimizer.join_reordering`: allows or forbids statistics-driven join swapping
265259
pub(crate) fn partitioned_hash_join(
266260
hash_join: &HashJoinExec,
267-
config: &ConfigOptions,
268-
registry: Option<&StatisticsRegistry>,
261+
context: &dyn PhysicalOptimizerContext,
269262
) -> Result<Arc<dyn ExecutionPlan>> {
270263
let left = hash_join.left();
271264
let right = hash_join.right();
@@ -275,7 +268,7 @@ pub(crate) fn partitioned_hash_join(
275268
PartitionMode::Partitioned
276269
};
277270
if can_swap_hash_join(hash_join)
278-
&& should_swap_join_order(&**left, &**right, config, registry)?
271+
&& should_swap_join_order(&**left, &**right, context)?
279272
{
280273
hash_join.swap_inputs(partition_mode)
281274
} else {
@@ -297,33 +290,31 @@ pub(crate) fn partitioned_hash_join(
297290
/// optimize hash and cross joins in the plan according to available statistical
298291
/// information.
299292
///
300-
/// Used configurations inside arg `config`
301-
/// - `config.optimizer.hash_join_single_partition_threshold`: byte threshold for `CollectLeft`
302-
/// - `config.optimizer.hash_join_single_partition_threshold_rows`: row threshold for `CollectLeft`
303-
/// - `config.optimizer.join_reordering`: allows or forbids input swapping
293+
/// Used configurations
294+
/// - `optimizer.hash_join_single_partition_threshold`: byte threshold for `CollectLeft`
295+
/// - `optimizer.hash_join_single_partition_threshold_rows`: row threshold for `CollectLeft`
296+
/// - `optimizer.join_reordering`: allows or forbids input swapping
304297
fn statistical_join_selection_subrule(
305298
plan: Arc<dyn ExecutionPlan>,
306-
config: &ConfigOptions,
307-
registry: Option<&StatisticsRegistry>,
299+
context: &dyn PhysicalOptimizerContext,
308300
) -> Result<Transformed<Arc<dyn ExecutionPlan>>> {
309301
let transformed = if let Some(hash_join) = plan.downcast_ref::<HashJoinExec>() {
310302
match hash_join.partition_mode() {
311-
PartitionMode::Auto => try_collect_left(hash_join, false, config, registry)?
303+
PartitionMode::Auto => try_collect_left(hash_join, false, context)?
312304
.map_or_else(
313-
|| partitioned_hash_join(hash_join, config, registry).map(Some),
305+
|| partitioned_hash_join(hash_join, context).map(Some),
314306
|v| Ok(Some(v)),
315307
)?,
316-
PartitionMode::CollectLeft => {
317-
try_collect_left(hash_join, true, config, registry)?.map_or_else(
318-
|| partitioned_hash_join(hash_join, config, registry).map(Some),
308+
PartitionMode::CollectLeft => try_collect_left(hash_join, true, context)?
309+
.map_or_else(
310+
|| partitioned_hash_join(hash_join, context).map(Some),
319311
|v| Ok(Some(v)),
320-
)?
321-
}
312+
)?,
322313
PartitionMode::Partitioned => {
323314
let left = hash_join.left();
324315
let right = hash_join.right();
325316
if can_swap_hash_join(hash_join)
326-
&& should_swap_join_order(&**left, &**right, config, registry)?
317+
&& should_swap_join_order(&**left, &**right, context)?
327318
{
328319
// Null-aware RightAnti only supports CollectLeft
329320
let partition_mode = if hash_join.null_aware {
@@ -340,7 +331,7 @@ fn statistical_join_selection_subrule(
340331
} else if let Some(cross_join) = plan.downcast_ref::<CrossJoinExec>() {
341332
let left = cross_join.left();
342333
let right = cross_join.right();
343-
if should_swap_join_order(&**left, &**right, config, registry)? {
334+
if should_swap_join_order(&**left, &**right, context)? {
344335
cross_join.swap_inputs().map(Some)?
345336
} else {
346337
None
@@ -349,7 +340,7 @@ fn statistical_join_selection_subrule(
349340
let left = nl_join.left();
350341
let right = nl_join.right();
351342
if nl_join.join_type().supports_swap()
352-
&& should_swap_join_order(&**left, &**right, config, registry)?
343+
&& should_swap_join_order(&**left, &**right, context)?
353344
{
354345
nl_join.swap_inputs().map(Some)?
355346
} else {

0 commit comments

Comments
 (0)