Skip to content

Commit 107713f

Browse files
authored
fix: make skip_partial_aggregation_probe_ratio_threshold match the docs (#22752)
## Which issue does this PR close? - Closes #. ## Rationale for this change The config `skip_partial_aggregation_probe_ratio_threshold` was documented as triggering skip when the ratio is **greater than** the threshold, but the code used `>=`. This meant setting the threshold to `1.0` (to disable the feature) still skipped rows when cardinality was exactly 100%. ## What changes are included in this PR? - Changed `>=` to `>` in the ratio comparison to match the docs. - Return `None` for `SkipAggregationProbe` when `probe_ratio_threshold >= 1.0`, effectively disabling the feature since the ratio can never exceed `1.0`. ## Are these changes tested? Yes. Added `test_skip_aggregation_disabled_at_threshold_one` which sets threshold to `1.0` with 100% cardinality input and asserts that no rows are skipped. ## Are there any user-facing changes? Yes. Setting `skip_partial_aggregation_probe_ratio_threshold = 1.0` now reliably disables skip aggregation, matching the documented behavior.
1 parent 51673d0 commit 107713f

2 files changed

Lines changed: 125 additions & 14 deletions

File tree

‎datafusion/physical-plan/src/aggregates/mod.rs‎

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3791,6 +3791,90 @@ mod tests {
37913791
Ok(())
37923792
}
37933793

3794+
/// When `skip_partial_aggregation_probe_ratio_threshold` is set to 1.0,
3795+
/// the feature must be effectively disabled: even with 100% cardinality
3796+
/// (every row is a unique group), no rows should be skipped.
3797+
#[tokio::test]
3798+
async fn test_skip_aggregation_disabled_at_threshold_one() -> Result<()> {
3799+
let schema = Arc::new(Schema::new(vec![
3800+
Field::new("key", DataType::Int32, true),
3801+
Field::new("val", DataType::Int32, true),
3802+
]));
3803+
3804+
let group_by =
3805+
PhysicalGroupBy::new_single(vec![(col("key", &schema)?, "key".to_string())]);
3806+
3807+
let aggr_expr = vec![
3808+
AggregateExprBuilder::new(count_udaf(), vec![col("val", &schema)?])
3809+
.schema(Arc::clone(&schema))
3810+
.alias(String::from("COUNT(val)"))
3811+
.build()
3812+
.map(Arc::new)?,
3813+
];
3814+
3815+
// Two batches are required: batch 1 triggers the probe threshold so the
3816+
// skip decision is evaluated; batch 2 is what would be skipped on main
3817+
// (where >= caused threshold=1.0 to still skip at 100% cardinality).
3818+
// All rows have unique keys => ratio = 1.0 (100% cardinality).
3819+
let input_data = vec![
3820+
// Batch 1: fires the probe check (ratio = 5/5 = 1.0)
3821+
RecordBatch::try_new(
3822+
Arc::clone(&schema),
3823+
vec![
3824+
Arc::new(Int32Array::from(vec![1, 2, 3, 4, 5])),
3825+
Arc::new(Int32Array::from(vec![0, 0, 0, 0, 0])),
3826+
],
3827+
)
3828+
.unwrap(),
3829+
// Batch 2: would be skipped if threshold=1.0 did not disable the feature
3830+
RecordBatch::try_new(
3831+
Arc::clone(&schema),
3832+
vec![
3833+
Arc::new(Int32Array::from(vec![6, 7, 8, 9, 10])),
3834+
Arc::new(Int32Array::from(vec![0, 0, 0, 0, 0])),
3835+
],
3836+
)
3837+
.unwrap(),
3838+
];
3839+
3840+
let input =
3841+
TestMemoryExec::try_new_exec(&[input_data], Arc::clone(&schema), None)?;
3842+
let aggregate_exec = Arc::new(AggregateExec::try_new(
3843+
AggregateMode::Partial,
3844+
group_by,
3845+
aggr_expr,
3846+
vec![None],
3847+
Arc::clone(&input) as Arc<dyn ExecutionPlan>,
3848+
schema,
3849+
)?);
3850+
3851+
let session_config = SessionConfig::default()
3852+
.set(
3853+
"datafusion.execution.skip_partial_aggregation_probe_rows_threshold",
3854+
&ScalarValue::Int64(Some(1)),
3855+
)
3856+
.set(
3857+
"datafusion.execution.skip_partial_aggregation_probe_ratio_threshold",
3858+
&ScalarValue::Float64(Some(1.0)),
3859+
);
3860+
3861+
let ctx = TaskContext::default().with_session_config(session_config);
3862+
collect(aggregate_exec.execute(0, Arc::new(ctx))?).await?;
3863+
3864+
let metrics = aggregate_exec.metrics().unwrap();
3865+
let skipped_rows = metrics
3866+
.sum_by_name("skipped_aggregation_rows")
3867+
.map(|m| m.as_usize())
3868+
.unwrap_or(0);
3869+
3870+
assert_eq!(
3871+
skipped_rows, 0,
3872+
"threshold=1.0 should disable skip aggregation, but {skipped_rows} rows were skipped"
3873+
);
3874+
3875+
Ok(())
3876+
}
3877+
37943878
#[test]
37953879
fn group_exprs_nullable() -> Result<()> {
37963880
let input_schema = Arc::new(Schema::new(vec![

‎datafusion/physical-plan/src/aggregates/row_hash.rs‎

Lines changed: 41 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -195,7 +195,7 @@ impl SkipAggregationProbe {
195195
self.num_groups = num_groups;
196196
if self.input_rows >= self.probe_rows_threshold {
197197
self.should_skip = self.num_groups as f64 / self.input_rows as f64
198-
>= self.probe_ratio_threshold;
198+
> self.probe_ratio_threshold;
199199
// Set is_locked to true only if we have decided to skip, otherwise we can try to skip
200200
// during processing the next record_batch.
201201
self.is_locked = self.should_skip;
@@ -644,14 +644,20 @@ impl GroupedHashAggregateStream {
644644
options.skip_partial_aggregation_probe_rows_threshold;
645645
let probe_ratio_threshold =
646646
options.skip_partial_aggregation_probe_ratio_threshold;
647-
let skipped_aggregation_rows = MetricBuilder::new(&agg.metrics)
648-
.with_category(MetricCategory::Rows)
649-
.counter("skipped_aggregation_rows", partition);
650-
Some(SkipAggregationProbe::new(
651-
probe_rows_threshold,
652-
probe_ratio_threshold,
653-
skipped_aggregation_rows,
654-
))
647+
// A threshold >= 1.0 means the ratio (num_groups / input_rows) can
648+
// never exceed it, so the feature is effectively disabled.
649+
if probe_ratio_threshold >= 1.0 {
650+
None
651+
} else {
652+
let skipped_aggregation_rows = MetricBuilder::new(&agg.metrics)
653+
.with_category(MetricCategory::Rows)
654+
.counter("skipped_aggregation_rows", partition);
655+
Some(SkipAggregationProbe::new(
656+
probe_rows_threshold,
657+
probe_ratio_threshold,
658+
skipped_aggregation_rows,
659+
))
660+
}
655661
} else {
656662
None
657663
};
@@ -1630,11 +1636,11 @@ mod tests {
16301636
],
16311637
)?;
16321638

1633-
// Batch 2: 350 rows with 350 unique NEW groups (starting from group 10)
1634-
// After batch 2, total: 450 rows, 360 groups
1635-
// Ratio: 360/450 = 0.8 (80%) >= 0.8 -> SHOULD decide to skip
1636-
let batch2_rows = 350;
1637-
let batch2_groups = 350;
1639+
// Batch 2: 360 rows with 360 unique NEW groups (starting from group 10)
1640+
// After batch 2, total: 460 rows, 370 groups
1641+
// Ratio: 370/460 ≈ 0.804 (80.4%) > 0.8 -> SHOULD decide to skip
1642+
let batch2_rows = 360;
1643+
let batch2_groups = 360;
16381644
let group_ids_batch2: Vec<i32> = (batch1_groups..(batch1_groups + batch2_groups))
16391645
.map(|x| x as i32)
16401646
.collect();
@@ -1817,4 +1823,25 @@ mod tests {
18171823

18181824
Ok(())
18191825
}
1826+
1827+
#[test]
1828+
fn test_skip_aggregation_probe_equality_does_not_skip() {
1829+
// When num_groups / input_rows == probe_ratio_threshold, the `>` boundary
1830+
// means we must NOT skip — equality is not sufficient to trigger skip.
1831+
let threshold_ratio = 0.5_f64;
1832+
let threshold_rows = 10_usize;
1833+
let mut probe = SkipAggregationProbe::new(
1834+
threshold_rows,
1835+
threshold_ratio,
1836+
metrics::Count::new(),
1837+
);
1838+
1839+
// 10 rows, 5 groups → ratio = 5/10 = 0.5 exactly equals threshold
1840+
probe.update_state(10, 5);
1841+
1842+
assert!(
1843+
!probe.should_skip(),
1844+
"ratio == threshold should not trigger skip (boundary is exclusive)"
1845+
);
1846+
}
18201847
}

0 commit comments

Comments
 (0)