Skip to content

Commit 227b2ac

Browse files
authored
Merge branch 'main' into neilc/perf-fd-convert-to-semi-join
2 parents 808a3a5 + 04ef3c7 commit 227b2ac

28 files changed

Lines changed: 3104 additions & 97 deletions

File tree

‎benchmarks/src/hj.rs‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -405,6 +405,27 @@ const HASH_QUERIES: &[HashJoinQuery] = &[
405405
build_size: "100K",
406406
probe_size: "60M_RightAnti",
407407
},
408+
// Q22: RightSemi, Medium build (100K rows), ~1% Hit rate, fanout ~100
409+
//
410+
// Build Side: supplier (100K rows) collapsed onto 1K distinct keys
411+
// Probe Side: lineitem (60M rows). Each matching probe row produces many
412+
// duplicate probe indices before RightSemi deduplication.
413+
HashJoinQuery {
414+
sql: r###"SELECT l.k
415+
FROM (
416+
SELECT CAST(((s_suppkey - 1) % 1000) + 1 AS INT) as k
417+
FROM supplier
418+
) s
419+
RIGHT SEMI JOIN (
420+
SELECT CAST(l_suppkey AS INT) as k
421+
FROM lineitem
422+
) l
423+
ON s.k = l.k"###,
424+
density: 1.0,
425+
prob_hit: 0.01,
426+
build_size: "100K_(fanout_100)",
427+
probe_size: "60M_RightSemi",
428+
},
408429
];
409430

410431
impl RunOpt {

‎datafusion/common/src/config.rs‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -596,6 +596,17 @@ config_namespace! {
596596
/// the new schema verification step.
597597
pub skip_physical_aggregate_schema_check: bool, default = false
598598

599+
/// Temporary switch for aggregate stream implementations that are being
600+
/// migrated from `GroupedHashAggregateStream`.
601+
///
602+
/// When set to true, DataFusion tries the migrated implementations when
603+
/// their preconditions are satisfied. When set to false, grouped
604+
/// aggregation falls back to `GroupedHashAggregateStream`. This option
605+
/// will be removed after the migration is finished.
606+
///
607+
/// See <https://github.com/apache/datafusion/issues/22710> for details.
608+
pub enable_migration_aggregate: bool, default = false
609+
599610
/// Sets the compression codec used when spilling data to disk.
600611
///
601612
/// Since datafusion writes spill files using the Arrow IPC Stream format,

‎datafusion/common/src/scalar/mod.rs‎

Lines changed: 44 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -148,6 +148,30 @@ pub fn date_to_timestamp_multiplier(
148148
}
149149
}
150150

151+
/// Returns the multiplier that converts the input timestamp representation into
152+
/// the desired timestamp unit, if the conversion requires a multiplication that
153+
/// can overflow an `i64`.
154+
pub fn timestamp_to_timestamp_multiplier(
155+
source_type: &DataType,
156+
target_type: &DataType,
157+
) -> Option<i64> {
158+
let (DataType::Timestamp(source_unit, _), DataType::Timestamp(target_unit, _)) =
159+
(source_type, target_type)
160+
else {
161+
return None;
162+
};
163+
164+
match (source_unit, target_unit) {
165+
(TimeUnit::Second, TimeUnit::Millisecond) => Some(1_000),
166+
(TimeUnit::Second, TimeUnit::Microsecond) => Some(1_000_000),
167+
(TimeUnit::Second, TimeUnit::Nanosecond) => Some(1_000_000_000),
168+
(TimeUnit::Millisecond, TimeUnit::Microsecond) => Some(1_000),
169+
(TimeUnit::Millisecond, TimeUnit::Nanosecond) => Some(1_000_000),
170+
(TimeUnit::Microsecond, TimeUnit::Nanosecond) => Some(1_000),
171+
_ => None,
172+
}
173+
}
174+
151175
/// Ensures the provided value can be represented as a timestamp with the given
152176
/// multiplier. Returns an [`DataFusionError::Execution`] when the converted
153177
/// value would overflow the timestamp range.
@@ -4265,7 +4289,8 @@ impl ScalarValue {
42654289
}
42664290

42674291
if let Some(multiplier) = date_to_timestamp_multiplier(&source_type, target_type)
4268-
&& let Some(value) = self.date_scalar_value_as_i64()
4292+
.or_else(|| timestamp_to_timestamp_multiplier(&source_type, target_type))
4293+
&& let Some(value) = self.temporal_scalar_value_as_i64()
42694294
{
42704295
ensure_timestamp_in_bounds(value, multiplier, &source_type, target_type)?;
42714296
}
@@ -4287,10 +4312,14 @@ impl ScalarValue {
42874312
ScalarValue::try_from_array(&cast_arr, 0)
42884313
}
42894314

4290-
fn date_scalar_value_as_i64(&self) -> Option<i64> {
4315+
fn temporal_scalar_value_as_i64(&self) -> Option<i64> {
42914316
match self {
42924317
ScalarValue::Date32(Some(value)) => Some(i64::from(*value)),
42934318
ScalarValue::Date64(Some(value)) => Some(*value),
4319+
ScalarValue::TimestampSecond(Some(value), _)
4320+
| ScalarValue::TimestampMillisecond(Some(value), _)
4321+
| ScalarValue::TimestampMicrosecond(Some(value), _)
4322+
| ScalarValue::TimestampNanosecond(Some(value), _) => Some(*value),
42944323
_ => None,
42954324
}
42964325
}
@@ -10161,6 +10190,19 @@ mod tests {
1016110190
);
1016210191
}
1016310192

10193+
#[test]
10194+
fn cast_timestamp_to_timestamp_overflow_returns_error() {
10195+
let scalar = ScalarValue::TimestampSecond(Some(i64::MAX), None);
10196+
let err = scalar
10197+
.cast_to(&DataType::Timestamp(TimeUnit::Nanosecond, None))
10198+
.expect_err("expected cast to fail");
10199+
assert!(
10200+
err.to_string()
10201+
.contains("converted value exceeds the representable i64 range"),
10202+
"unexpected error: {err}"
10203+
);
10204+
}
10205+
1016410206
#[test]
1016510207
fn null_dictionary_scalar_produces_null_dictionary_array() {
1016610208
let dictionary_scalar = ScalarValue::Dictionary(

‎datafusion/core/tests/fuzz_cases/aggregation_fuzzer/context_generator.rs‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@ use crate::fuzz_cases::aggregation_fuzzer::data_generator::Dataset;
4141
/// - `batch_size`
4242
/// - `target_partitions`
4343
/// - `skip_partial parameters`
44+
/// - `enable_migration_aggregate`
4445
/// - hint `sorted` or not
4546
/// - `spilling` or not (TODO, I think a special `MemoryPool` may be needed
4647
/// to support this)
@@ -96,11 +97,13 @@ impl SessionContextGenerator {
9697
let batch_size = self.max_batch_size;
9798
let target_partitions = 1;
9899
let skip_partial_params = SkipPartialParams::ensure_not_trigger();
100+
let enable_migration_aggregate = false;
99101

100102
let builder = GeneratedSessionContextBuilder {
101103
batch_size,
102104
target_partitions,
103105
skip_partial_params,
106+
enable_migration_aggregate,
104107
sort_hint: false,
105108
table_name: self.table_name.clone(),
106109
table_provider: Arc::new(provider),
@@ -120,6 +123,7 @@ impl SessionContextGenerator {
120123
// - `batch_size`, from range: [1, `total_rows_num`]
121124
// - `target_partitions`, from range: [1, cpu_num]
122125
// - `skip_partial`, trigger or not trigger currently for simplicity
126+
// - `enable_migration_aggregate`, true or false
123127
// - `sorted`, if found a sorted dataset, will or will not push down this information
124128
// - `spilling`(TODO)
125129
let batch_size = rng.random_range(1..=self.max_batch_size);
@@ -131,6 +135,8 @@ impl SessionContextGenerator {
131135
let skip_partial_params =
132136
self.candidate_skip_partial_params[skip_partial_params_idx];
133137

138+
let enable_migration_aggregate = rng.random_bool(0.5);
139+
134140
let (provider, sort_hint) =
135141
if rng.random_bool(0.5) && !self.dataset.sort_keys.is_empty() {
136142
// Sort keys exist and random to push down
@@ -150,6 +156,7 @@ impl SessionContextGenerator {
150156
target_partitions,
151157
sort_hint,
152158
skip_partial_params,
159+
enable_migration_aggregate,
153160
table_name: self.table_name.clone(),
154161
table_provider: Arc::new(provider),
155162
};
@@ -173,6 +180,7 @@ struct GeneratedSessionContextBuilder {
173180
target_partitions: usize,
174181
sort_hint: bool,
175182
skip_partial_params: SkipPartialParams,
183+
enable_migration_aggregate: bool,
176184
table_name: String,
177185
table_provider: Arc<dyn TableProvider>,
178186
}
@@ -197,6 +205,10 @@ impl GeneratedSessionContextBuilder {
197205
"datafusion.execution.skip_partial_aggregation_probe_ratio_threshold",
198206
&ScalarValue::Float64(Some(self.skip_partial_params.ratio_threshold)),
199207
);
208+
session_config = session_config.set_bool(
209+
"datafusion.execution.enable_migration_aggregate",
210+
self.enable_migration_aggregate,
211+
);
200212

201213
let ctx = SessionContext::new_with_config(session_config);
202214
ctx.register_table(self.table_name, self.table_provider)?;
@@ -206,6 +218,7 @@ impl GeneratedSessionContextBuilder {
206218
target_partitions: self.target_partitions,
207219
sort_hint: self.sort_hint,
208220
skip_partial_params: self.skip_partial_params,
221+
enable_migration_aggregate: self.enable_migration_aggregate,
209222
};
210223

211224
Ok(SessionContextWithParams { ctx, params })
@@ -220,6 +233,7 @@ pub struct SessionContextParams {
220233
target_partitions: usize,
221234
sort_hint: bool,
222235
skip_partial_params: SkipPartialParams,
236+
enable_migration_aggregate: bool,
223237
}
224238

225239
/// Partial skipping parameters

‎datafusion/core/tests/physical_optimizer/ensure_requirements.rs‎

Lines changed: 18 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,8 @@ use datafusion_physical_plan::projection::ProjectionExec;
5555
use datafusion_physical_plan::repartition::RepartitionExec;
5656
use datafusion_physical_plan::sorts::sort_preserving_merge::SortPreservingMergeExec;
5757

58+
const TEST_TARGET_PARTITIONS: usize = 8;
59+
5860
/// Mock ExecutionPlan with configurable partition count and output ordering.
5961
#[derive(Debug)]
6062
struct MockMultiPartitionExec {
@@ -124,11 +126,18 @@ impl ExecutionPlan for MockMultiPartitionExec {
124126
}
125127
}
126128

129+
fn test_config() -> ConfigOptions {
130+
let mut config = ConfigOptions::default();
131+
// Keep plan-shape tests deterministic across machines with different CPU counts.
132+
config.execution.target_partitions = TEST_TARGET_PARTITIONS;
133+
config
134+
}
135+
127136
/// Helper: run EnsureRequirements and verify SanityCheckPlan passes
128137
fn optimize_and_sanity_check(
129138
plan: Arc<dyn ExecutionPlan>,
130139
) -> Result<Arc<dyn ExecutionPlan>> {
131-
let config = ConfigOptions::default();
140+
let config = test_config();
132141
let optimized = EnsureRequirements::new().optimize(plan, &config)?;
133142
// SanityCheckPlan must pass
134143
SanityCheckPlan::new().optimize(Arc::clone(&optimized), &config)?;
@@ -137,7 +146,7 @@ fn optimize_and_sanity_check(
137146

138147
/// Helper: verify idempotency — running twice produces the same plan
139148
fn assert_idempotent(plan: Arc<dyn ExecutionPlan>) {
140-
let config = ConfigOptions::default();
149+
let config = test_config();
141150
let p1 = EnsureRequirements::new()
142151
.optimize(plan, &config)
143152
.expect("first optimize failed");
@@ -195,7 +204,7 @@ fn plan_string(plan: &Arc<dyn ExecutionPlan>) -> String {
195204
/// updating an intentional plan change is a single `cargo insta accept`.
196205
macro_rules! assert_ensure_requirements_plan {
197206
($plan:expr, @ $snapshot:literal $(,)?) => {{
198-
let config = ConfigOptions::default();
207+
let config = test_config();
199208
let p1 = EnsureRequirements::new()
200209
.optimize($plan, &config)
201210
.expect("EnsureRequirements::optimize failed (pass 1)");
@@ -655,7 +664,7 @@ fn test_issue_21973_idempotent_spm_sort_multi_partition() {
655664
// No-extra-SPM property: count SortPreservingMergeExec occurrences in
656665
// the first optimisation pass — must be ≤ 1 (the original SPM survives,
657666
// none are added).
658-
let config = ConfigOptions::default();
667+
let config = test_config();
659668
let optimized = EnsureRequirements::new()
660669
.optimize(Arc::clone(&limit), &config)
661670
.expect("optimize failed");
@@ -688,7 +697,7 @@ fn test_issue_21973_parallel_sort_survives_multiple_passes() {
688697
let sort: Arc<dyn ExecutionPlan> = Arc::new(SortExec::new(sort_expr, coalesce));
689698
let limit: Arc<dyn ExecutionPlan> = Arc::new(GlobalLimitExec::new(sort, 0, Some(10)));
690699

691-
let config = ConfigOptions::default();
700+
let config = test_config();
692701

693702
let p1 = EnsureRequirements::new()
694703
.optimize(Arc::clone(&limit), &config)
@@ -816,7 +825,7 @@ fn test_issue_14150_fetch_survives_multiple_passes() {
816825
let sort = Arc::new(SortExec::new(sort_expr, repartition as _).with_fetch(Some(5)));
817826
let limit: Arc<dyn ExecutionPlan> = Arc::new(GlobalLimitExec::new(sort, 0, Some(5)));
818827

819-
let config = ConfigOptions::default();
828+
let config = test_config();
820829

821830
// Pass 1
822831
let p1 = EnsureRequirements::new()
@@ -885,7 +894,7 @@ fn test_issue_14150_fetch_survives_with_input_spm() {
885894

886895
let limit: Arc<dyn ExecutionPlan> = Arc::new(GlobalLimitExec::new(spm, 0, Some(5)));
887896

888-
let config = ConfigOptions::default();
897+
let config = test_config();
889898

890899
let p1 = EnsureRequirements::new()
891900
.optimize(Arc::clone(&limit), &config)
@@ -1018,7 +1027,7 @@ fn test_idempotent_parallelize_sorts() {
10181027
let limit: Arc<dyn ExecutionPlan> = Arc::new(GlobalLimitExec::new(sort, 0, Some(10)));
10191028

10201029
// First pass should parallelize the sort (Sort+SPM replaces Coalesce+Sort)
1021-
let config = ConfigOptions::default();
1030+
let config = test_config();
10221031
let p1 = EnsureRequirements::new()
10231032
.optimize(Arc::clone(&limit), &config)
10241033
.expect("first optimize failed");
@@ -1194,7 +1203,7 @@ fn test_enforce_distribution_idempotent_hash_join() {
11941203
.expect("HashJoinExec creation failed"),
11951204
);
11961205

1197-
let config = ConfigOptions::default();
1206+
let config = test_config();
11981207
let p1 = EnsureRequirements::new()
11991208
.optimize(Arc::clone(&join), &config)
12001209
.expect("first EnsureRequirements pass failed");

0 commit comments

Comments
 (0)