@@ -134,6 +134,15 @@ struct SkipAggregationProbe {
134134 /// (from `SessionConfig`). If the ratio exceeds this value, aggregation
135135 /// is skipped and input rows are directly converted to output
136136 probe_ratio_threshold : f64 ,
137+ /// (experimental) Cost-aware skip rule: skip when measured per-row wall
138+ /// time is high even at ratios below `probe_ratio_threshold`. Disabled
139+ /// by default; preserves the current bare-ratio behaviour.
140+ use_cost_model : bool ,
141+ /// Per-row wall-time threshold (ns) for cost-aware skip rule.
142+ cost_ns_per_row_threshold : u64 ,
143+ /// Minimum ratio for cost-aware skip rule to fire. Below this ratio
144+ /// partial aggregation is kept regardless of per-row cost.
145+ cost_min_ratio : f64 ,
137146
138147 // ========================================================================
139148 // STATES:
@@ -162,43 +171,98 @@ struct SkipAggregationProbe {
162171 /// * if greater than zero, the number of rows which were output directly
163172 /// without aggregation
164173 skipped_aggregation_rows : metrics:: Count ,
174+ /// Operator-wide `elapsed_compute` metric. Cloned from
175+ /// `BaselineMetrics` so the probe can read the accumulated nanoseconds
176+ /// at decision time and infer the wall time spent on partial
177+ /// aggregation so far (used by the cost-aware rule).
178+ elapsed_compute : metrics:: Time ,
179+ /// `elapsed_compute.value()` snapshot at probe construction. Diff
180+ /// against the live value gives wall time elapsed since the stream
181+ /// started — a close approximation of partial aggregation cost,
182+ /// since `elapsed_compute` covers exactly the timed work blocks in
183+ /// the partial-mode loop.
184+ elapsed_compute_at_probe_start : usize ,
165185}
166186
167187impl SkipAggregationProbe {
168188 fn new (
169189 probe_rows_threshold : usize ,
170190 probe_ratio_threshold : f64 ,
191+ use_cost_model : bool ,
192+ cost_ns_per_row_threshold : u64 ,
193+ cost_min_ratio : f64 ,
171194 skipped_aggregation_rows : metrics:: Count ,
195+ elapsed_compute : metrics:: Time ,
172196 ) -> Self {
197+ let elapsed_compute_at_probe_start = elapsed_compute. value ( ) ;
173198 Self {
174199 input_rows : 0 ,
175200 num_groups : 0 ,
176201 probe_rows_threshold,
177202 probe_ratio_threshold,
203+ use_cost_model,
204+ cost_ns_per_row_threshold,
205+ cost_min_ratio,
178206 should_skip : false ,
179207 is_locked : false ,
180208 skipped_aggregation_rows,
209+ elapsed_compute,
210+ elapsed_compute_at_probe_start,
181211 }
182212 }
183213
184214 /// Updates `SkipAggregationProbe` state:
185215 /// - increments the number of input rows
186216 /// - replaces the number of groups with the new value
187- /// - on `probe_rows_threshold` exceeded calculates
188- /// aggregation ratio and sets `should_skip` flag
217+ /// - on `probe_rows_threshold` exceeded applies the skip-decision rules
218+ /// (fixed ratio threshold, plus the optional cost-aware rule)
189219 /// - if `should_skip` is set, locks further state updates
220+ ///
221+ /// Two skip rules are evaluated, in order:
222+ ///
223+ /// 1. **Fixed ratio**: `num_groups / input_rows >= probe_ratio_threshold`
224+ /// — the original behaviour, preserved for backwards compatibility.
225+ /// 2. **Cost-aware** (opt-in via `use_cost_model`): when the ratio is
226+ /// above `cost_min_ratio` *and* the measured per-row wall time of
227+ /// the operator exceeds `cost_ns_per_row_threshold`, skip even if
228+ /// the ratio is below `probe_ratio_threshold`. This catches the
229+ /// case where partial aggregation is net-negative due to high
230+ /// per-row cost (heavy variable-length keys, expensive aggregates)
231+ /// rather than poor reduction alone.
190232 fn update_state ( & mut self , input_rows : usize , num_groups : usize ) {
191233 if self . is_locked {
192234 return ;
193235 }
194236 self . input_rows += input_rows;
195237 self . num_groups = num_groups;
196238 if self . input_rows >= self . probe_rows_threshold {
197- self . should_skip = self . num_groups as f64 / self . input_rows as f64
198- >= self . probe_ratio_threshold ;
199- // Set is_locked to true only if we have decided to skip, otherwise we can try to skip
200- // during processing the next record_batch.
201- self . is_locked = self . should_skip ;
239+ let ratio = self . num_groups as f64 / self . input_rows as f64 ;
240+
241+ // Rule 1: fixed-ratio skip (existing behaviour).
242+ if ratio >= self . probe_ratio_threshold {
243+ self . should_skip = true ;
244+ self . is_locked = true ;
245+ return ;
246+ }
247+
248+ // Rule 2 (opt-in): cost-aware skip for the medium-ratio,
249+ // high-per-row-cost band.
250+ if self . use_cost_model && ratio >= self . cost_min_ratio {
251+ let elapsed_ns = self
252+ . elapsed_compute
253+ . value ( )
254+ . saturating_sub ( self . elapsed_compute_at_probe_start ) ;
255+ let ns_per_row = ( elapsed_ns as u64 )
256+ . checked_div ( self . input_rows as u64 )
257+ . unwrap_or ( 0 ) ;
258+ if ns_per_row > self . cost_ns_per_row_threshold {
259+ self . should_skip = true ;
260+ self . is_locked = true ;
261+ }
262+ }
263+
264+ // No rule fired — leave `is_locked` false so the next batch
265+ // can re-evaluate.
202266 }
203267 }
204268
@@ -644,13 +708,21 @@ impl GroupedHashAggregateStream {
644708 options. skip_partial_aggregation_probe_rows_threshold ;
645709 let probe_ratio_threshold =
646710 options. skip_partial_aggregation_probe_ratio_threshold ;
711+ let use_cost_model = options. skip_partial_aggregation_use_cost_model ;
712+ let cost_ns_per_row_threshold =
713+ options. skip_partial_aggregation_cost_ns_per_row ;
714+ let cost_min_ratio = options. skip_partial_aggregation_cost_min_ratio ;
647715 let skipped_aggregation_rows = MetricBuilder :: new ( & agg. metrics )
648716 . with_category ( MetricCategory :: Rows )
649717 . counter ( "skipped_aggregation_rows" , partition) ;
650718 Some ( SkipAggregationProbe :: new (
651719 probe_rows_threshold,
652720 probe_ratio_threshold,
721+ use_cost_model,
722+ cost_ns_per_row_threshold,
723+ cost_min_ratio,
653724 skipped_aggregation_rows,
725+ baseline_metrics. elapsed_compute ( ) . clone ( ) ,
654726 ) )
655727 } else {
656728 None
@@ -1481,6 +1553,7 @@ mod tests {
14811553 use datafusion_functions_aggregate:: count:: count_udaf;
14821554 use datafusion_physical_expr:: aggregate:: AggregateExprBuilder ;
14831555 use datafusion_physical_expr:: expressions:: col;
1556+ use std:: time:: Duration ;
14841557
14851558 #[ tokio:: test]
14861559 async fn test_double_emission_race_condition_bug ( ) -> Result < ( ) > {
@@ -1817,4 +1890,106 @@ mod tests {
18171890
18181891 Ok ( ( ) )
18191892 }
1893+
1894+ // ---------------- SkipAggregationProbe unit tests ----------------
1895+
1896+ /// Build a probe with the given configuration. Skip-count metric is
1897+ /// disconnected (a fresh `Count`) and `elapsed_compute` is a fresh
1898+ /// `Time` that the test can drive with `add_duration`.
1899+ fn make_probe (
1900+ probe_rows_threshold : usize ,
1901+ probe_ratio_threshold : f64 ,
1902+ use_cost_model : bool ,
1903+ cost_ns_per_row_threshold : u64 ,
1904+ cost_min_ratio : f64 ,
1905+ ) -> ( SkipAggregationProbe , metrics:: Time ) {
1906+ let elapsed_compute = metrics:: Time :: new ( ) ;
1907+ let probe = SkipAggregationProbe :: new (
1908+ probe_rows_threshold,
1909+ probe_ratio_threshold,
1910+ use_cost_model,
1911+ cost_ns_per_row_threshold,
1912+ cost_min_ratio,
1913+ metrics:: Count :: new ( ) ,
1914+ elapsed_compute. clone ( ) ,
1915+ ) ;
1916+ ( probe, elapsed_compute)
1917+ }
1918+
1919+ /// With the cost model off, the behaviour is exactly the legacy bare
1920+ /// ratio check.
1921+ #[ test]
1922+ fn skip_probe_cost_model_off_matches_legacy_ratio_check ( ) {
1923+ let ( mut probe, _t) = make_probe ( 100 , 0.8 , false , 1000 , 0.3 ) ;
1924+
1925+ // 100 rows / 50 groups → ratio 0.5, below 0.8 → don't skip
1926+ probe. update_state ( 100 , 50 ) ;
1927+ assert ! ( !probe. should_skip( ) ) ;
1928+ assert ! ( !probe. is_locked) ;
1929+
1930+ // Next window: another 100 rows, total 200 / 170 groups → ratio
1931+ // 0.85, above 0.8 → skip.
1932+ probe. update_state ( 100 , 170 ) ;
1933+ assert ! ( probe. should_skip( ) ) ;
1934+ assert ! ( probe. is_locked) ;
1935+ }
1936+
1937+ /// With the cost model on, a high-cost slow query in the medium-ratio
1938+ /// band (0.3 ≤ ratio < 0.8) gets skipped — the ClickBench Q18 case.
1939+ #[ test]
1940+ fn skip_probe_cost_model_fires_in_medium_ratio_high_cost_band ( ) {
1941+ let ( mut probe, elapsed) = make_probe ( 100 , 0.8 , true , 1000 , 0.3 ) ;
1942+
1943+ // 200_000 ns over 100 rows → 2_000 ns/row → above the 1_000
1944+ // threshold. Ratio 0.5 sits in the medium band.
1945+ elapsed. add_duration ( Duration :: from_nanos ( 200_000 ) ) ;
1946+ probe. update_state ( 100 , 50 ) ;
1947+
1948+ assert ! ( probe. should_skip( ) ) ;
1949+ assert ! ( probe. is_locked) ;
1950+ }
1951+
1952+ /// Cost model on, ratio below `cost_min_ratio`: the partial agg is
1953+ /// genuinely reducing a lot (small / cheap output), so we keep it
1954+ /// regardless of per-row cost.
1955+ #[ test]
1956+ fn skip_probe_cost_model_does_not_fire_below_min_ratio ( ) {
1957+ let ( mut probe, elapsed) = make_probe ( 100 , 0.8 , true , 1000 , 0.3 ) ;
1958+
1959+ // Very expensive — 5_000 ns/row — but ratio is 0.2 (huge
1960+ // reduction), so partial agg is still net-positive.
1961+ elapsed. add_duration ( Duration :: from_nanos ( 500_000 ) ) ;
1962+ probe. update_state ( 100 , 20 ) ;
1963+
1964+ assert ! ( !probe. should_skip( ) ) ;
1965+ assert ! ( !probe. is_locked) ;
1966+ }
1967+
1968+ /// Cost model on but per-row cost is below threshold: ratio alone
1969+ /// doesn't justify skipping → keep aggregating.
1970+ #[ test]
1971+ fn skip_probe_cost_model_does_not_fire_when_cheap_per_row ( ) {
1972+ let ( mut probe, elapsed) = make_probe ( 100 , 0.8 , true , 1000 , 0.3 ) ;
1973+
1974+ // 50_000 ns over 100 rows → 500 ns/row → below the 1_000 ns
1975+ // threshold. Ratio 0.5 alone is below the fixed 0.8 cut-off.
1976+ elapsed. add_duration ( Duration :: from_nanos ( 50_000 ) ) ;
1977+ probe. update_state ( 100 , 50 ) ;
1978+
1979+ assert ! ( !probe. should_skip( ) ) ;
1980+ assert ! ( !probe. is_locked) ;
1981+ }
1982+
1983+ /// Cost model on still honours the original high-ratio rule: ratio
1984+ /// >= `probe_ratio_threshold` skips regardless of cost.
1985+ #[ test]
1986+ fn skip_probe_cost_model_still_honours_high_ratio_rule ( ) {
1987+ let ( mut probe, _t) = make_probe ( 100 , 0.8 , true , 1000 , 0.3 ) ;
1988+
1989+ // Ratio 0.9, no measured cost → skip via the fixed rule.
1990+ probe. update_state ( 100 , 90 ) ;
1991+
1992+ assert ! ( probe. should_skip( ) ) ;
1993+ assert ! ( probe. is_locked) ;
1994+ }
18201995}
0 commit comments