Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
review: move old test to new impl + additional test
  • Loading branch information
2010YOUY01 committed Jun 11, 2026
commit f0b2d79c4dd0095f89bd75927c79dc650019a900
89 changes: 88 additions & 1 deletion datafusion/physical-plan/src/aggregates/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4104,7 +4104,94 @@ mod tests {
}

Comment thread
2010YOUY01 marked this conversation as resolved.
Resolved
#[tokio::test]
async fn test_migrated_hash_stream_skip_aggregation_after_threshold() -> Result<()> {
async fn test_partial_hash_stream_skip_aggregation_after_first_batch() -> Result<()> {
let schema = Arc::new(Schema::new(vec![
Field::new("key", DataType::Int32, true),
Field::new("val", DataType::Int32, true),
]));

let group_by =
PhysicalGroupBy::new_single(vec![(col("key", &schema)?, "key".to_string())]);

let aggr_expr = vec![
AggregateExprBuilder::new(count_udaf(), vec![col("val", &schema)?])
.schema(Arc::clone(&schema))
.alias(String::from("COUNT(val)"))
.build()
.map(Arc::new)?,
];

let input_data = vec![
RecordBatch::try_new(
Arc::clone(&schema),
vec![
Arc::new(Int32Array::from(vec![1, 2, 3])),
Arc::new(Int32Array::from(vec![0, 0, 0])),
],
)
.unwrap(),
RecordBatch::try_new(
Arc::clone(&schema),
vec![
Arc::new(Int32Array::from(vec![2, 3, 4])),
Arc::new(Int32Array::from(vec![0, 0, 0])),
],
)
.unwrap(),
];

let input =
TestMemoryExec::try_new_exec(&[input_data], Arc::clone(&schema), None)?;
let aggregate_exec = Arc::new(AggregateExec::try_new(
AggregateMode::Partial,
group_by,
aggr_expr,
vec![None],
Arc::clone(&input) as Arc<dyn ExecutionPlan>,
schema,
)?);

let session_config = SessionConfig::default()
.set_bool("datafusion.execution.enable_migration_aggregate", true)
.set(
"datafusion.execution.skip_partial_aggregation_probe_rows_threshold",
&ScalarValue::Int64(Some(2)),
)
.set(
"datafusion.execution.skip_partial_aggregation_probe_ratio_threshold",
&ScalarValue::Float64(Some(0.1)),
);

let ctx = Arc::new(TaskContext::default().with_session_config(session_config));
let output = collect(aggregate_exec.execute(0, Arc::clone(&ctx))?).await?;

allow_duplicates! {
assert_snapshot!(batches_to_sort_string(&output), @r"
+-----+-------------------+
| key | COUNT(val)[count] |
+-----+-------------------+
| 1 | 1 |
| 2 | 1 |
| 2 | 1 |
| 3 | 1 |
| 3 | 1 |
| 4 | 1 |
+-----+-------------------+
");
}

let metrics = aggregate_exec.metrics().unwrap();
let skipped_rows = metrics
.sum_by_name("skipped_aggregation_rows")
.map(|m| m.as_usize())
.unwrap_or(0);
assert_eq!(skipped_rows, 3);

Ok(())
}

#[tokio::test]
async fn test_partial_hash_stream_skip_aggregation_after_threshold() -> Result<()> {
let schema = Arc::new(Schema::new(vec![
Field::new("key", DataType::Int32, true),
Field::new("val", DataType::Int32, true),
Expand Down
168 changes: 0 additions & 168 deletions datafusion/physical-plan/src/aggregates/row_hash.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1386,7 +1386,6 @@ impl GroupedHashAggregateStream {
mod tests {
use super::*;
use crate::InputOrderMode;
use crate::execution_plan::ExecutionPlan;
use crate::test::TestMemoryExec;
use arrow::array::{Int32Array, Int64Array};
use arrow::datatypes::{DataType, Field, Schema};
Expand Down Expand Up @@ -1501,152 +1500,6 @@ mod tests {
Ok(())
}

#[tokio::test]
async fn test_skip_aggregation_probe_not_locked_until_skip() -> Result<()> {
// Test that the probe is not locked until we actually decide to skip.
// This allows us to continue evaluating the skip condition across multiple batches.
//
// Scenario:
// - Batch 1: Hits rows threshold but NOT ratio threshold (low cardinality) -> don't skip
// - Batch 2: Now hits ratio threshold (high cardinality) -> skip
//
// Without the fix, the probe would be locked after batch 1, preventing the skip
// decision from being made on batch 2.

let schema = Arc::new(Schema::new(vec![
Field::new("group_col", DataType::Int32, false),
Field::new("value_col", DataType::Int32, false),
]));

// Configure thresholds:
// - probe_rows_threshold: 100 rows
// - probe_ratio_threshold: 0.8 (80%)
let probe_rows_threshold = 100;
let probe_ratio_threshold = 0.8;

// Batch 1: 100 rows with only 10 unique groups
// Ratio: 10/100 = 0.1 (10%) < 0.8 -> should NOT skip
// This will hit the rows threshold but not the ratio threshold
let batch1_rows = 100;
let batch1_groups = 10;
let mut group_ids_batch1 = Vec::new();
for i in 0..batch1_rows {
group_ids_batch1.push((i % batch1_groups) as i32);
}
let values_batch1: Vec<i32> = vec![1; batch1_rows];

let batch1 = RecordBatch::try_new(
Arc::clone(&schema),
vec![
Arc::new(Int32Array::from(group_ids_batch1)),
Arc::new(Int32Array::from(values_batch1)),
],
)?;

// Batch 2: 360 rows with 360 unique NEW groups (starting from group 10)
// After batch 2, total: 460 rows, 370 groups
// Ratio: 370/460 ≈ 0.804 (80.4%) > 0.8 -> SHOULD decide to skip
let batch2_rows = 360;
let batch2_groups = 360;
let group_ids_batch2: Vec<i32> = (batch1_groups..(batch1_groups + batch2_groups))
.map(|x| x as i32)
.collect();
let values_batch2: Vec<i32> = vec![1; batch2_rows];

let batch2 = RecordBatch::try_new(
Arc::clone(&schema),
vec![
Arc::new(Int32Array::from(group_ids_batch2)),
Arc::new(Int32Array::from(values_batch2)),
],
)?;

// Batch 3: This batch should be skipped since we decided to skip after batch 2
// 100 rows with 100 unique groups (continuing from where batch 2 left off)
let batch3_rows = 100;
let batch3_groups = 100;
let batch3_start_group = batch1_groups + batch2_groups;
let group_ids_batch3: Vec<i32> = (batch3_start_group
..(batch3_start_group + batch3_groups))
.map(|x| x as i32)
.collect();
let values_batch3: Vec<i32> = vec![1; batch3_rows];

let batch3 = RecordBatch::try_new(
Arc::clone(&schema),
vec![
Arc::new(Int32Array::from(group_ids_batch3)),
Arc::new(Int32Array::from(values_batch3)),
],
)?;

let input_partitions = vec![vec![batch1, batch2, batch3]];

let runtime = RuntimeEnvBuilder::default().build_arc()?;
let mut task_ctx = TaskContext::default().with_runtime(runtime);

// Configure skip aggregation settings
let mut session_config = task_ctx.session_config().clone();
session_config = session_config.set(
"datafusion.execution.skip_partial_aggregation_probe_rows_threshold",
&datafusion_common::ScalarValue::UInt64(Some(probe_rows_threshold)),
);
session_config = session_config.set(
"datafusion.execution.skip_partial_aggregation_probe_ratio_threshold",
&datafusion_common::ScalarValue::Float64(Some(probe_ratio_threshold)),
);
task_ctx = task_ctx.with_session_config(session_config);
let task_ctx = Arc::new(task_ctx);

// Create aggregate: COUNT(*) GROUP BY group_col
let group_expr = vec![(col("group_col", &schema)?, "group_col".to_string())];
let aggr_expr = vec![Arc::new(
AggregateExprBuilder::new(count_udaf(), vec![col("value_col", &schema)?])
.schema(Arc::clone(&schema))
.alias("count_value")
.build()?,
)];

let exec = TestMemoryExec::try_new(&input_partitions, Arc::clone(&schema), None)?;
let exec = Arc::new(TestMemoryExec::update_cache(&Arc::new(exec)));

// Use Partial mode
let aggregate_exec = AggregateExec::try_new(
AggregateMode::Partial,
PhysicalGroupBy::new_single(group_expr),
aggr_expr,
vec![None],
exec,
Arc::clone(&schema),
)?;

// Execute and collect results
let mut stream =
GroupedHashAggregateStream::new(&aggregate_exec, &Arc::clone(&task_ctx), 0)?;
let mut results = Vec::new();

while let Some(result) = stream.next().await {
let batch = result?;
results.push(batch);
}

// Check that skip aggregation actually happened
// The key metric is skipped_aggregation_rows
let metrics = aggregate_exec.metrics().unwrap();
let skipped_rows = metrics
.sum_by_name("skipped_aggregation_rows")
.map(|m| m.as_usize())
.unwrap_or(0);

// We expect batch 3's rows to be skipped (100 rows)
assert_eq!(
skipped_rows, batch3_rows,
"Expected batch 3's rows ({batch3_rows}) to be skipped",
);

Ok(())
}

#[tokio::test]
async fn test_emit_early_with_partially_sorted() -> Result<()> {
// Reproducer for #20445: EmitEarly with PartiallySorted panics in
Expand Down Expand Up @@ -1730,25 +1583,4 @@ mod tests {

Ok(())
}

#[test]
fn test_skip_aggregation_probe_equality_does_not_skip() {
// When num_groups / input_rows == probe_ratio_threshold, the `>` boundary
// means we must NOT skip — equality is not sufficient to trigger skip.
let threshold_ratio = 0.5_f64;
let threshold_rows = 10_usize;
let mut probe = SkipAggregationProbe::new(
threshold_rows,
threshold_ratio,
metrics::Count::new(),
);

// 10 rows, 5 groups → ratio = 5/10 = 0.5 exactly equals threshold
probe.update_state(10, 5);

assert!(
!probe.should_skip(),
"ratio == threshold should not trigger skip (boundary is exclusive)"
);
}
}
Loading
Loading