Skip to content

Commit 4a2b907

Browse files
committed
add config to enable migration aggregate path
1 parent 3184e00 commit 4a2b907

3 files changed

Lines changed: 49 additions & 16 deletions

File tree

‎datafusion/common/src/config.rs‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -587,6 +587,17 @@ config_namespace! {
587587
/// the new schema verification step.
588588
pub skip_physical_aggregate_schema_check: bool, default = false
589589

590+
/// Temporary switch for aggregate stream implementations that are being
591+
/// migrated from `GroupedHashAggregateStream`.
592+
///
593+
/// When set to true, DataFusion tries the migrated implementations when
594+
/// their preconditions are satisfied. When set to false, grouped
595+
/// aggregation falls back to `GroupedHashAggregateStream`. This option
596+
/// will be removed after the migration is finished.
597+
///
598+
/// See <https://github.com/apache/datafusion/issues/22710> for details.
599+
pub enable_migration_aggregate: bool, default = true
600+
590601
/// Sets the compression codec used when spilling data to disk.
591602
///
592603
/// Since datafusion writes spill files using the Arrow IPC Stream format,

‎datafusion/physical-plan/src/aggregates/mod.rs‎

Lines changed: 36 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -524,7 +524,7 @@ enum StreamType {
524524
/// Final stage of the hash aggregation
525525
/// Input output scheme: partial state -> final result
526526
PartialFinalHash(PartialFinalHashAggregateStream),
527-
/// Hash aggregation resused for multiple stages
527+
/// Hash aggregation reused for multiple stages
528528
///
529529
/// Note this is being incrementally migrated to dedicated streams like
530530
/// [`StreamType::InitialPartialHash`] and [`StreamType::PartialFinalHash`]
@@ -1002,16 +1002,23 @@ impl AggregateExec {
10021002
));
10031003
}
10041004

1005-
if self.should_use_initial_partial_hash_stream(context) {
1006-
return Ok(StreamType::InitialPartialHash(
1007-
InitialPartialHashAggregateStream::new(self, context, partition)?,
1008-
));
1009-
}
1005+
if context
1006+
.session_config()
1007+
.options()
1008+
.execution
1009+
.enable_migration_aggregate
1010+
{
1011+
if self.should_use_initial_partial_hash_stream(context) {
1012+
return Ok(StreamType::InitialPartialHash(
1013+
InitialPartialHashAggregateStream::new(self, context, partition)?,
1014+
));
1015+
}
10101016

1011-
if self.should_use_partial_final_hash_stream(context) {
1012-
return Ok(StreamType::PartialFinalHash(
1013-
PartialFinalHashAggregateStream::new(self, context, partition)?,
1014-
));
1017+
if self.should_use_partial_final_hash_stream(context) {
1018+
return Ok(StreamType::PartialFinalHash(
1019+
PartialFinalHashAggregateStream::new(self, context, partition)?,
1020+
));
1021+
}
10151022
}
10161023

10171024
// grouping by something else and we need to just materialize all results
@@ -3089,10 +3096,20 @@ mod tests {
30893096
.with_session_config(SessionConfig::new().with_batch_size(2)),
30903097
);
30913098

3092-
let stream = partial_aggregate.execute_typed(0, &task_ctx)?;
3093-
assert!(matches!(stream, StreamType::InitialPartialHash(_)));
3099+
let partial_stream = partial_aggregate.execute_typed(0, &task_ctx)?;
3100+
assert!(matches!(partial_stream, StreamType::InitialPartialHash(_)));
30943101

3095-
let stream: SendableRecordBatchStream = stream.into();
3102+
let fallback_task_ctx = Arc::new(
3103+
TaskContext::default().with_session_config(
3104+
SessionConfig::new()
3105+
.with_batch_size(2)
3106+
.set_bool("datafusion.execution.enable_migration_aggregate", false),
3107+
),
3108+
);
3109+
let stream = partial_aggregate.execute_typed(0, &fallback_task_ctx)?;
3110+
assert!(matches!(stream, StreamType::GroupedHash(_)));
3111+
3112+
let stream: SendableRecordBatchStream = partial_stream.into();
30963113
let batches = collect(stream).await?;
30973114
assert_eq!(
30983115
batches
@@ -3113,10 +3130,13 @@ mod tests {
31133130
Arc::clone(&schema),
31143131
)?;
31153132

3116-
let stream = final_aggregate.execute_typed(0, &task_ctx)?;
3117-
assert!(matches!(stream, StreamType::PartialFinalHash(_)));
3133+
let final_stream = final_aggregate.execute_typed(0, &task_ctx)?;
3134+
assert!(matches!(final_stream, StreamType::PartialFinalHash(_)));
3135+
3136+
let stream = final_aggregate.execute_typed(0, &fallback_task_ctx)?;
3137+
assert!(matches!(stream, StreamType::GroupedHash(_)));
31183138

3119-
let stream: SendableRecordBatchStream = stream.into();
3139+
let stream: SendableRecordBatchStream = final_stream.into();
31203140
let batches = collect(stream).await?;
31213141
assert_eq!(
31223142
batches

‎datafusion/sqllogictest/test_files/information_schema.slt‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -218,6 +218,7 @@ datafusion.execution.batch_size 8192
218218
datafusion.execution.coalesce_batches true
219219
datafusion.execution.collect_statistics true
220220
datafusion.execution.enable_ansi_mode false
221+
datafusion.execution.enable_migration_aggregate true
221222
datafusion.execution.enable_recursive_ctes true
222223
datafusion.execution.enforce_batch_size_in_joins false
223224
datafusion.execution.hash_join_buffering_capacity 0
@@ -370,6 +371,7 @@ datafusion.execution.batch_size 8192 Default batch size while creating new batch
370371
datafusion.execution.coalesce_batches true When set to true, record batches will be examined between each operator and small batches will be coalesced into larger batches. This is helpful when there are highly selective filters or joins that could produce tiny output batches. The target batch size is determined by the configuration setting
371372
datafusion.execution.collect_statistics true Should DataFusion collect statistics when first creating a table. Has no effect after the table is created. Applies to the default `ListingTableProvider` in DataFusion. Defaults to true.
372373
datafusion.execution.enable_ansi_mode false Whether to enable ANSI SQL mode. The flag is experimental and relevant only for DataFusion Spark built-in functions When `enable_ansi_mode` is set to `true`, the query engine follows ANSI SQL semantics for expressions, casting, and error handling. This means: - **Strict type coercion rules:** implicit casts between incompatible types are disallowed. - **Standard SQL arithmetic behavior:** operations such as division by zero, numeric overflow, or invalid casts raise runtime errors rather than returning `NULL` or adjusted values. - **Consistent ANSI behavior** for string concatenation, comparisons, and `NULL` handling. When `enable_ansi_mode` is `false` (the default), the engine uses a more permissive, non-ANSI mode designed for user convenience and backward compatibility. In this mode: - Implicit casts between types are allowed (e.g., string to integer when possible). - Arithmetic operations are more lenient — for example, `abs()` on the minimum representable integer value returns the input value instead of raising overflow. - Division by zero or invalid casts may return `NULL` instead of failing. # Default `false` — ANSI SQL mode is disabled by default.
374+
datafusion.execution.enable_migration_aggregate true Temporary switch for aggregate stream implementations that are being migrated from `GroupedHashAggregateStream`. When set to true, DataFusion tries the migrated implementations when their preconditions are satisfied. When set to false, grouped aggregation falls back to `GroupedHashAggregateStream`. This option will be removed after the migration is finished. See <https://github.com/apache/datafusion/issues/22710> for details.
373375
datafusion.execution.enable_recursive_ctes true Should DataFusion support recursive CTEs
374376
datafusion.execution.enforce_batch_size_in_joins false Should DataFusion enforce batch size in joins or not. By default, DataFusion will not enforce batch size in joins. Enforcing batch size in joins can reduce memory usage when joining large tables with a highly-selective join filter, but is also slightly slower.
375377
datafusion.execution.hash_join_buffering_capacity 0 How many bytes to buffer in the probe side of hash joins while the build side is concurrently being built. Without this, hash joins will wait until the full materialization of the build side before polling the probe side. This is useful in scenarios where the query is not completely CPU bounded, allowing to do some early work concurrently and reducing the latency of the query. Note that when hash join buffering is enabled, the probe side will start eagerly polling data, not giving time for the producer side of dynamic filters to produce any meaningful predicate. Queries with dynamic filters might see performance degradation. Disabled by default, set to a number greater than 0 for enabling it.

0 commit comments

Comments
 (0)