Skip to content

Commit 56181e3

Browse files
committed
Use hidden subpartition column for hash aggregate repartition
1 parent 2b2e7f4 commit 56181e3

6 files changed

Lines changed: 240 additions & 683 deletions

File tree

‎datafusion/physical-plan/Cargo.toml‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -139,11 +139,6 @@ harness = false
139139
name = "hash_join_semi_anti"
140140
required-features = ["test_utils"]
141141

142-
[[bench]]
143-
harness = false
144-
name = "hash_aggregate_output_coalescer"
145-
required-features = ["test_utils"]
146-
147142
[[bench]]
148143
harness = false
149144
name = "hash_group_by_reuse"

‎datafusion/physical-plan/benches/hash_aggregate_output_coalescer.rs‎

Lines changed: 0 additions & 285 deletions
This file was deleted.

‎datafusion/physical-plan/src/aggregates/hash_aggregate.rs‎

Lines changed: 17 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@ use super::aggregate_hash_table::{
4242
AggregateHashTable, FinalMarker, PartialMarker, PartialSkipMarker,
4343
};
4444
use super::skip_partial::SkipAggregationProbe;
45-
use super::{AggregateExec, partition_runs};
45+
use super::{AggregateExec, partition_runs, reorder_by_subpartition};
4646
use crate::metrics::{
4747
BaselineMetrics, MetricBuilder, MetricCategory, RecordOutput, SpillMetrics,
4848
};
@@ -891,17 +891,22 @@ impl FinalHashAggregateStream {
891891
}
892892

893893
fn stage_partition_runs(&mut self, batch: &RecordBatch) -> Result<Option<usize>> {
894-
let Some(runs) = partition_runs(batch.schema_ref())? else {
895-
if self
896-
.partition_run_state
897-
.as_ref()
898-
.is_some_and(|state| !state.runs.is_empty())
899-
{
900-
return internal_err!(
901-
"missing partition run metadata for final partitioned aggregation after buffered runs have started"
902-
);
903-
}
904-
return Ok(None);
894+
let (batch, runs) = if let Some((batch, runs)) = reorder_by_subpartition(batch)? {
895+
(batch, runs)
896+
} else {
897+
let Some(runs) = partition_runs(batch.schema_ref())? else {
898+
if self
899+
.partition_run_state
900+
.as_ref()
901+
.is_some_and(|state| !state.runs.is_empty())
902+
{
903+
return internal_err!(
904+
"missing partition run metadata for final partitioned aggregation after buffered runs have started"
905+
);
906+
}
907+
return Ok(None);
908+
};
909+
(batch.clone(), runs)
905910
};
906911

907912
let mut offset = 0;

‎datafusion/physical-plan/src/aggregates/mod.rs‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -69,8 +69,11 @@ use datafusion_physical_expr_common::sort_expr::{
6969
use datafusion_expr::utils::AggregateOrderSensitivity;
7070
use datafusion_physical_expr_common::utils::evaluate_expressions_to_arrays;
7171
use itertools::Itertools;
72+
#[cfg(test)]
73+
pub(crate) use partition_runs::set_partition_runs_metadata;
7274
pub(crate) use partition_runs::{
73-
PartitionRun, partition_runs, set_partition_runs_metadata,
75+
PartitionRun, append_subpartition_column, partition_runs, reorder_by_subpartition,
76+
subpartition_schema,
7477
};
7578
use topk::hash_table::is_supported_hash_key_type;
7679
use topk::heap::is_supported_heap_type;

0 commit comments

Comments
 (0)