Skip to content

Commit 6fdef65

Browse files
authored
refactor: Split hash aggregation logic into separated streams (apache#22729)
## Which issue does this PR close? <!-- We generally require a GitHub issue to be filed for all bug fixes and enhancements and this helps us generate change logs for our releases. You can link an issue to this PR using the GitHub syntax. For example `Closes apache#123` indicates that this PR will close issue apache#123. --> - part of apache#22710 - 1/N of apache#22712 ## Rationale for this change <!-- Why are you proposing this change? If this is already explained clearly in the issue then this section is not needed. Explaining clearly why changes are proposed helps reviewers understand your changes and offer better suggestions for fixes. --> See issues. This PR split out partial and final aggregate strem from `GroupsHashAggregateStream` To fully migrate hash aggregation, we have to - Port this optimization back apache#11627 - Support spilling I think they should be leave to follow up PRs Todo in this PR: - [x] Add a temporary configuration `enable_migration_aggregate` to turn off this path Since it should be a regression if the above features are not added, it also helps if to prevent potential regressions from the migration of other aggregate streams. ## What changes are included in this PR? <!-- There is no need to duplicate the description in the issue here but it is sometimes worth providing a summary of the individual changes in this PR. --> Split out the streams from `GroupsHashAggregateStream` 1. Partial stage of hash aggregation 2. Final stage of hash aggregation ## Are these changes tested? <!-- We typically require tests for all PRs in order to: 1. Prevent the code from being accidentally broken by subsequent changes 3. Serve as another way to document the expected behavior of the code If tests are not included in your PR, please explain why (for example, are they covered by existing tests)? --> ## Are there any user-facing changes? <!-- If there are user-facing changes then we may require documentation to be updated before approving the PR. --> <!-- If there are any breaking changes to public APIs, please add the `api change` label. -->
1 parent b23e6b6 commit 6fdef65

8 files changed

Lines changed: 1208 additions & 6 deletions

File tree

‎datafusion/common/src/config.rs‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -596,6 +596,17 @@ config_namespace! {
596596
/// the new schema verification step.
597597
pub skip_physical_aggregate_schema_check: bool, default = false
598598

599+
/// Temporary switch for aggregate stream implementations that are being
600+
/// migrated from `GroupedHashAggregateStream`.
601+
///
602+
/// When set to true, DataFusion tries the migrated implementations when
603+
/// their preconditions are satisfied. When set to false, grouped
604+
/// aggregation falls back to `GroupedHashAggregateStream`. This option
605+
/// will be removed after the migration is finished.
606+
///
607+
/// See <https://github.com/apache/datafusion/issues/22710> for details.
608+
pub enable_migration_aggregate: bool, default = false
609+
599610
/// Sets the compression codec used when spilling data to disk.
600611
///
601612
/// Since datafusion writes spill files using the Arrow IPC Stream format,

‎datafusion/core/tests/fuzz_cases/aggregation_fuzzer/context_generator.rs‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@ use crate::fuzz_cases::aggregation_fuzzer::data_generator::Dataset;
4141
/// - `batch_size`
4242
/// - `target_partitions`
4343
/// - `skip_partial parameters`
44+
/// - `enable_migration_aggregate`
4445
/// - hint `sorted` or not
4546
/// - `spilling` or not (TODO, I think a special `MemoryPool` may be needed
4647
/// to support this)
@@ -96,11 +97,13 @@ impl SessionContextGenerator {
9697
let batch_size = self.max_batch_size;
9798
let target_partitions = 1;
9899
let skip_partial_params = SkipPartialParams::ensure_not_trigger();
100+
let enable_migration_aggregate = false;
99101

100102
let builder = GeneratedSessionContextBuilder {
101103
batch_size,
102104
target_partitions,
103105
skip_partial_params,
106+
enable_migration_aggregate,
104107
sort_hint: false,
105108
table_name: self.table_name.clone(),
106109
table_provider: Arc::new(provider),
@@ -120,6 +123,7 @@ impl SessionContextGenerator {
120123
// - `batch_size`, from range: [1, `total_rows_num`]
121124
// - `target_partitions`, from range: [1, cpu_num]
122125
// - `skip_partial`, trigger or not trigger currently for simplicity
126+
// - `enable_migration_aggregate`, true or false
123127
// - `sorted`, if found a sorted dataset, will or will not push down this information
124128
// - `spilling`(TODO)
125129
let batch_size = rng.random_range(1..=self.max_batch_size);
@@ -131,6 +135,8 @@ impl SessionContextGenerator {
131135
let skip_partial_params =
132136
self.candidate_skip_partial_params[skip_partial_params_idx];
133137

138+
let enable_migration_aggregate = rng.random_bool(0.5);
139+
134140
let (provider, sort_hint) =
135141
if rng.random_bool(0.5) && !self.dataset.sort_keys.is_empty() {
136142
// Sort keys exist and random to push down
@@ -150,6 +156,7 @@ impl SessionContextGenerator {
150156
target_partitions,
151157
sort_hint,
152158
skip_partial_params,
159+
enable_migration_aggregate,
153160
table_name: self.table_name.clone(),
154161
table_provider: Arc::new(provider),
155162
};
@@ -173,6 +180,7 @@ struct GeneratedSessionContextBuilder {
173180
target_partitions: usize,
174181
sort_hint: bool,
175182
skip_partial_params: SkipPartialParams,
183+
enable_migration_aggregate: bool,
176184
table_name: String,
177185
table_provider: Arc<dyn TableProvider>,
178186
}
@@ -197,6 +205,10 @@ impl GeneratedSessionContextBuilder {
197205
"datafusion.execution.skip_partial_aggregation_probe_ratio_threshold",
198206
&ScalarValue::Float64(Some(self.skip_partial_params.ratio_threshold)),
199207
);
208+
session_config = session_config.set_bool(
209+
"datafusion.execution.enable_migration_aggregate",
210+
self.enable_migration_aggregate,
211+
);
200212

201213
let ctx = SessionContext::new_with_config(session_config);
202214
ctx.register_table(self.table_name, self.table_provider)?;
@@ -206,6 +218,7 @@ impl GeneratedSessionContextBuilder {
206218
target_partitions: self.target_partitions,
207219
sort_hint: self.sort_hint,
208220
skip_partial_params: self.skip_partial_params,
221+
enable_migration_aggregate: self.enable_migration_aggregate,
209222
};
210223

211224
Ok(SessionContextWithParams { ctx, params })
@@ -220,6 +233,7 @@ pub struct SessionContextParams {
220233
target_partitions: usize,
221234
sort_hint: bool,
222235
skip_partial_params: SkipPartialParams,
236+
enable_migration_aggregate: bool,
223237
}
224238

225239
/// Partial skipping parameters

‎datafusion/physical-plan/src/aggregates/group_values/metrics.rs‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,7 @@ mod tests {
5959
use arrow::record_batch::RecordBatch;
6060
use datafusion_common::Result;
6161
use datafusion_execution::TaskContext;
62+
use datafusion_execution::runtime_env::RuntimeEnvBuilder;
6263
use datafusion_functions_aggregate::count::count_udaf;
6364
use datafusion_functions_aggregate::sum::sum_udaf;
6465
use datafusion_physical_expr::aggregate::AggregateExprBuilder;
@@ -135,7 +136,13 @@ mod tests {
135136
schema,
136137
)?);
137138

138-
let task_ctx = Arc::new(TaskContext::default());
139+
// This test is for `GroupByMetrics`, which are maintained by
140+
// `GroupedHashAggregateStream`. Use a finite memory pool so the partial
141+
// aggregate does not take the initial-partial stream path.
142+
let runtime = RuntimeEnvBuilder::new()
143+
.with_memory_limit(10 * 1024 * 1024, 1.0)
144+
.build_arc()?;
145+
let task_ctx = Arc::new(TaskContext::default().with_runtime(runtime));
139146
let _result =
140147
collect(Arc::clone(&aggregate_exec) as _, Arc::clone(&task_ctx)).await?;
141148

0 commit comments

Comments
 (0)