Skip to content
Merged
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
56 changes: 47 additions & 9 deletions datafusion/physical-plan/src/aggregates/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1036,7 +1036,7 @@ impl AggregateExec {
.execution
.enable_migration_aggregate
{
if self.should_use_ordered_partial_aggregate_stream() {
if self.should_use_ordered_partial_aggregate_stream(context) {
return Ok(StreamType::OrderedPartialAggregate(
OrderedPartialAggregateStream::new(self, context, partition)?,
));
Expand All @@ -1048,7 +1048,7 @@ impl AggregateExec {
)?));
}

if self.should_use_ordered_final_aggregate_stream() {
if self.should_use_ordered_final_aggregate_stream(context) {
return Ok(StreamType::OrderedFinalAggregate(
OrderedFinalAggregateStream::new(self, context, partition)?,
));
Expand Down Expand Up @@ -1080,7 +1080,12 @@ impl AggregateExec {
&& self.limit_options_supported_by_hash_stream()
}

fn should_use_ordered_partial_aggregate_stream(&self) -> bool {
fn should_use_ordered_partial_aggregate_stream(&self, context: &TaskContext) -> bool {
// TODO: implement memory-limited path and remove this limitation
if matches!(context.memory_pool().memory_limit(), MemoryLimit::Finite(_)) {
return false;
}

self.mode == AggregateMode::Partial
&& self.input_order_mode != InputOrderMode::Linear
&& !self.group_by.is_true_no_grouping()
Expand All @@ -1103,7 +1108,12 @@ impl AggregateExec {
&& self.group_by.is_single()
}

fn should_use_ordered_final_aggregate_stream(&self) -> bool {
fn should_use_ordered_final_aggregate_stream(&self, context: &TaskContext) -> bool {
// TODO: implement memory-limited path and remove this limitation
if matches!(context.memory_pool().memory_limit(), MemoryLimit::Finite(_)) {
return false;
}

matches!(
self.mode,
AggregateMode::Final | AggregateMode::FinalPartitioned

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

codex also notes that the exising code doesn't use the partial group by operator when memory limited, as the new tables don't seem to handle the OOM path (though maybe I missed that)

This is somewhat unobviously implemented by OutOfMemoryMode choice here

  • let group_ordering = GroupOrdering::try_new(&agg.input_order_mode)?;
    let oom_mode = match (agg.mode, &group_ordering) {
    // In partial aggregation mode, always prefer to emit incomplete results early.
    (AggregateMode::Partial, _) => OutOfMemoryMode::EmitEarly,
    // For non-partial aggregation modes, emitting incomplete results is not an option.
    // Instead, use disk spilling to store sorted, incomplete results, and merge them
    // afterwards.
    (_, GroupOrdering::None | GroupOrdering::Partial(_))
    if context.runtime_env().disk_manager.tmp_files_enabled() =>
    {
    OutOfMemoryMode::Spill
    }
    // For `GroupOrdering::Full`, the incoming stream is already sorted. This ensures the
    // number of incomplete groups can be kept small at all times. If we still hit
    // an out-of-memory condition, spilling to disk would not be beneficial since the same
    // situation is likely to reoccur when reading back the spilled data.
    // Therefore, we fall back to simply reporting the error immediately.
    // This mode will also be used if the `DiskManager` is not configured to allow spilling
    // to disk.
    _ => OutOfMemoryMode::ReportError,

And a proposed PR to fix

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, the existing behavior is, if there is memory limit, ordered aggregation case fallback to regular hash aggregation path (no ordering optimization) for the spilling.

This is intended for partially ordered case, since it might OOM, and we haven't implement spilling for that.

For fully ordered cases, this is not necessary, since it ensures always use bounded memory, but there is no existing testing coverage for that, probably in a follow-up PR, we can enable ordered fast path even there is a memory limit.

Patch applied in c2ae7dc

Expand Down Expand Up @@ -2615,16 +2625,34 @@ mod tests {
Arc::new(task_ctx)
}

fn migrated_hash_session_config(batch_size: usize) -> SessionConfig {
SessionConfig::new()
.with_batch_size(batch_size)
.set_bool("datafusion.execution.enable_migration_aggregate", true)
}

fn new_migrated_hash_ctx(batch_size: usize) -> Arc<TaskContext> {
Arc::new(
TaskContext::default().with_session_config(
SessionConfig::new()
.with_batch_size(batch_size)
.set_bool("datafusion.execution.enable_migration_aggregate", true),
),
TaskContext::default()
.with_session_config(migrated_hash_session_config(batch_size)),
)
}

fn new_finite_memory_migrated_hash_ctx(
batch_size: usize,
max_memory: usize,
) -> Result<Arc<TaskContext>> {
let runtime = RuntimeEnvBuilder::default()
.with_memory_limit(max_memory, 1.0)
.build_arc()?;

Ok(Arc::new(
TaskContext::default()
.with_runtime(runtime)
.with_session_config(migrated_hash_session_config(batch_size)),
))
}

async fn check_grouping_sets(
input: Arc<dyn ExecutionPlan>,
spill: bool,
Expand Down Expand Up @@ -3455,6 +3483,11 @@ mod tests {
+----------+-----------+-------------------------+
");

// Ordered streams don't implement memory limits yet.
let finite_memory_task_ctx = new_finite_memory_migrated_hash_ctx(2, 1024 * 1024)?;
let stream = aggregate.execute_typed(0, &finite_memory_task_ctx)?;
assert!(matches!(stream, StreamType::GroupedHash(_)));

Ok(())
}

Expand Down Expand Up @@ -3527,6 +3560,11 @@ mod tests {
+-----+--------------+
");

// Ordered streams don't implement memory limits yet.
let finite_memory_task_ctx = new_finite_memory_migrated_hash_ctx(2, 1024 * 1024)?;
let stream = final_aggregate.execute_typed(0, &finite_memory_task_ctx)?;
assert!(matches!(stream, StreamType::GroupedHash(_)));

Ok(())
}

Expand Down
Loading