Skip to content

Commit 89dc47d

Browse files
Rachel IntRachel Int
authored andcommitted
perf: emit partial threshold flushes as whole batches
1 parent 90335ae commit 89dc47d

1 file changed

Lines changed: 18 additions & 6 deletions

File tree

‎datafusion/physical-plan/src/aggregates/hash_stream.rs‎

Lines changed: 18 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -400,12 +400,24 @@ impl PartialHashAggregateStream {
400400
self.early_emit_count.add(1);
401401
}
402402
timer.done();
403-
self.emit_on_memory_pressure(
404-
materialized_group_states,
405-
&mut emitter,
406-
hash_table.memory_size(),
407-
)
408-
.await?;
403+
if last_state == HandleInputResult::FlushThresholdReached {
404+
self.reduction_factor
405+
.add_part(materialized_group_states.num_rows());
406+
self.reservation.try_resize(hash_table.memory_size())?;
407+
emitter
408+
.emit(
409+
materialized_group_states
410+
.record_output(&self.baseline_metrics),
411+
)
412+
.await;
413+
} else {
414+
self.emit_on_memory_pressure(
415+
materialized_group_states,
416+
&mut emitter,
417+
hash_table.memory_size(),
418+
)
419+
.await?;
420+
}
409421
}
410422
}
411423
}

0 commit comments

Comments
 (0)