Skip to content
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion datafusion/execution/src/spill_file.rs
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ pub trait SpillFile: Send + Sync {

/// Writer for spill file backends.
pub trait SpillWriter: std::io::Write + Send {
/// Intended for close/sync/commit operations.
/// Intended for close/sync/commit operations.
fn finish(&mut self) -> Result<()>;
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -297,25 +297,25 @@ pub(super) enum AggregateHashTableState {
Building(AggregateHashTableBuffer),
/// Emitting results directly from group keys and aggregate state.
Outputting(AggregateHashTableBuffer),
/// Materialize all output rows, then incrementally output slices from the
/// `OutputtingMaterialized` state.
/// Materialize all the output results, and then incrementally output in the `OutputtingMaterialized` state.
///
/// Note this is a temporary solution until the `GroupValues` issue is solved:
/// Issue: <https://github.com/apache/datafusion/issues/23178>
OutputtingMaterialized(MaterializedOutput),
OutputtingMaterialized(MaterializedAggregateOutput),
Done,
}

/// Fully materialized aggregate output and the next row offset to emit.
/// Fully evaluated aggregate output and the next row offset to emit.
///
/// Some output paths consume accumulator state when emitting. Materialize those
/// rows once, then slice them to honor `batch_size` across output polls.
pub(super) struct MaterializedOutput {
/// Final aggregate evaluation consumes accumulator state, and partial terminal
/// output should not repeatedly renumber group values with `EmitTo::First`.
/// Materialize once and then slice to honor `batch_size` across output polls.
pub(super) struct MaterializedAggregateOutput {
batch: RecordBatch,
offset: usize,
}

impl MaterializedOutput {
impl MaterializedAggregateOutput {
pub(super) fn new(batch: RecordBatch) -> Self {
Self { batch, offset: 0 }
}
Expand Down Expand Up @@ -490,7 +490,7 @@ mod tests {
use super::*;

#[test]
fn materialized_output_slices_batches_until_exhausted() -> Result<()> {
fn materialized_aggregate_output_slices_batches_until_exhausted() -> Result<()> {
let schema = Arc::new(Schema::new(vec![Field::new(
"group_col",
DataType::Int32,
Expand All @@ -500,7 +500,7 @@ mod tests {
schema,
vec![Arc::new(Int32Array::from(vec![1, 2, 3, 4, 5]))],
)?;
let mut output = MaterializedOutput::new(batch);
let mut output = MaterializedAggregateOutput::new(batch);

assert_eq!(int32_values(&output.next_batch(2).unwrap(), 0), vec![1, 2]);
assert_eq!(int32_values(&output.next_batch(2).unwrap(), 0), vec![3, 4]);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ use crate::aggregates::AggregateExec;

use super::common::{
AggregateHashTable, AggregateHashTableBuffer, AggregateHashTableState, FinalMarker,
MaterializedOutput,
MaterializedAggregateOutput,
};

/// Methods specific to the aggregate hash table used in the final aggregation stage.
Expand Down Expand Up @@ -82,7 +82,7 @@ impl AggregateHashTable<FinalMarker> {
&self,
mut state: AggregateHashTableBuffer,
output_schema: SchemaRef,
) -> Result<MaterializedOutput> {
) -> Result<MaterializedAggregateOutput> {
// Final aggregate evaluation consumes accumulator state. Evaluate all
// groups once, then slice the materialized batch on subsequent polls.
let emit_to = EmitTo::All;
Expand All @@ -96,12 +96,12 @@ impl AggregateHashTable<FinalMarker> {

let batch = RecordBatch::try_new(output_schema, output)?;
debug_assert!(batch.num_rows() > 0);
Ok(MaterializedOutput::new(batch))
Ok(MaterializedAggregateOutput::new(batch))
}

fn emit_next_materialized_batch(
&mut self,
mut output: MaterializedOutput,
mut output: MaterializedAggregateOutput,
batch_size: usize,
) -> Option<RecordBatch> {
let batch = output.next_batch(batch_size);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ use crate::aggregates::AggregateExec;

use super::common::{
AggregateHashTable, AggregateHashTableBuffer, AggregateHashTableState,
MaterializedOutput, PartialReduceMarker,
MaterializedAggregateOutput, PartialReduceMarker,
};

/// Methods specific to the aggregate hash table used in the partial-reduce stage.
Expand Down Expand Up @@ -83,7 +83,7 @@ impl AggregateHashTable<PartialReduceMarker> {
&self,
mut state: AggregateHashTableBuffer,
output_schema: SchemaRef,
) -> Result<MaterializedOutput> {
) -> Result<MaterializedAggregateOutput> {
// `state(EmitTo::All)` consumes accumulator state. Emit all groups once,
// then slice the materialized batch on subsequent polls.
let emit_to_all = EmitTo::All;
Expand All @@ -97,12 +97,12 @@ impl AggregateHashTable<PartialReduceMarker> {

let batch = RecordBatch::try_new(output_schema, output)?;
debug_assert!(batch.num_rows() > 0);
Ok(MaterializedOutput::new(batch))
Ok(MaterializedAggregateOutput::new(batch))
}

fn emit_next_materialized_batch(
&mut self,
mut output: MaterializedOutput,
mut output: MaterializedAggregateOutput,
batch_size: usize,
) -> Option<RecordBatch> {
let batch = output.next_batch(batch_size);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -81,9 +81,6 @@ impl AggregateHashTable<PartialMarker> {
AggregateHashTableState::Building(_) => {
internal_err!("next_output_batch must be called in the outputting state")
}
AggregateHashTableState::OutputtingMaterialized(_) => {
internal_err!("partial aggregate output should not materialize output")
}
}
}

Expand Down