@@ -54,10 +54,10 @@ use datafusion_expr::{
5454/// GROUP BY a
5555/// ```
5656///
57- /// A non-distinct `count` is also allowed alongside the distinct aggregate. It
58- /// is the one supported function whose outer phase is a * different* function:
59- /// the inner group-by counts rows per `(group, distinct value)` pair and the
60- /// outer phase adds those partial counts up with `sum` .
57+ /// A non-distinct `count` may also appear alongside the distinct aggregate. It
58+ /// is the one supported function whose outer phase is a different function: the
59+ /// inner group-by counts rows per `(group, distinct value)` pair, and the outer
60+ /// phase sums those partial counts.
6161///
6262/// ```text
6363/// Before:
@@ -77,16 +77,10 @@ use datafusion_expr::{
7777/// GROUP BY a
7878/// ```
7979///
80- /// The `CASE` covers the one input on which the two phases disagree: over an
81- /// empty input the inner group by produces no rows at all, and a `sum` of no
82- /// rows is NULL where `count` is 0.
80+ /// The `CASE` restores the 0 that `count` reports over an empty input, where a
81+ /// `sum` of no rows is NULL.
8382///
84- /// That `count` is allowed only when the distinct aggregate reports that it has
85- /// no specialized `GroupsAccumulator` for its argument types, so that the
86- /// rewrite is taking the distinct aggregate off `GroupsAccumulatorAdapter`
87- /// rather than only adding an inner group by. A function that does not report
88- /// on this at all keeps the previous behaviour and is left alone. See
89- /// `rewrite_pays_for_count` for the measurements behind that.
83+ /// See `rewrite_pays_for_count` for when the `count` is allowed.
9084#[ derive( Default , Debug ) ]
9185pub struct SingleDistinctToGroupBy { }
9286
@@ -184,30 +178,22 @@ fn is_single_distinct_agg(
184178///
185179/// The rewrite is not free: every other aggregate moves down to the inner group
186180/// by, which has a row per `(group, distinct value)` pair rather than per group,
187- /// and each one keeps its state at that finer grain. What pays for it is taking
188- /// the distinct aggregate off `GroupsAccumulatorAdapter`, whose one boxed
189- /// accumulator per group is the expensive shape. A distinct aggregate that
190- /// already has a specialized `GroupsAccumulator` never went near the adapter, so
191- /// there is nothing to buy and only the inner group by to pay for: ClickBench
192- /// Q22, whose `count(DISTINCT "UserID")` is over an `Int64`, measured a 132%
193- /// increase in peak memory pool reservation when the rewrite applied to it.
181+ /// and keeps its state at that finer grain. What pays for that is taking the
182+ /// distinct aggregate off `GroupsAccumulatorAdapter` and its one boxed
183+ /// accumulator per group. An aggregate that already has a specialized
184+ /// `GroupsAccumulator` never went near the adapter, so there is nothing to buy
185+ /// and only the inner group by to pay for.
194186///
195- /// `Some(false)` is the only answer that buys anything. `Some(true)` says the
196- /// call never reaches the adapter. `None` says the function does not answer the
197- /// question from the argument types, which is the default and so the answer for
198- /// almost every function. Silence is not evidence, and reading it as `Some(false)`
199- /// would open this path to every such function: `sum(DISTINCT int_col)` beside a
200- /// `count(*)` measured 3.15x the peak memory once rewritten, over 4,000,000 rows
201- /// in 2,000 groups.
187+ /// `Some(false)` is therefore the only answer that permits the rewrite. `None`,
188+ /// the default for almost every function, is not evidence that it pays.
202189///
203- /// The predicate is a proxy, not the true discriminator. What decides the
204- /// outcome is the cost per distinct value on each side, which this rule cannot
205- /// see. The proxy is deliberately conservative in the direction that leaves a
206- /// plan alone.
190+ /// This is a proxy: what actually decides the outcome is the cost per distinct
191+ /// value on each side, which this rule cannot see. It is deliberately
192+ /// conservative in the direction that leaves a plan alone. The pre-existing
193+ /// tolerance of a non-distinct `sum`, `min` or `max` is not gated, since
194+ /// narrowing it would change plans that have always been rewritten.
207195///
208- /// The existing tolerance of a non-distinct `sum`, `min` or `max` predates this
209- /// and is left alone: narrowing it would change plans that have always been
210- /// rewritten, which no measurement here calls for.
196+ /// Measured in <https://github.com/apache/datafusion/pull/24859>.
211197fn rewrite_pays_for_count (
212198 distinct_aggs : & [ ( & Arc < AggregateUDF > , & [ Expr ] ) ] ,
213199 input_schema : & DFSchema ,
@@ -354,11 +340,8 @@ impl OptimizerRule for SingleDistinctToGroupBy {
354340 } else {
355341 index += 1 ;
356342 let alias_str = format ! ( "alias{index}" ) ;
357- // `count` is the one function whose two phases
358- // use different aggregates: the inner group-by
359- // counts the rows of each `(group, distinct
360- // value)` partition and the outer phase adds
361- // those partial counts up.
343+ // `count`'s outer phase is `sum`, over the
344+ // partial counts the inner group-by produced.
362345 let rollup = count_rollup
363346 . as_ref ( )
364347 . filter ( |rollup| rollup. is_count ( & func) ) ;
@@ -389,11 +372,9 @@ impl OptimizerRule for SingleDistinctToGroupBy {
389372 if rollup. is_none ( ) {
390373 return Ok ( ( outer. clone ( ) , outer) ) ;
391374 }
392- // The inner group-by produces no rows at all
393- // for an empty input, and `sum` reports that as
394- // NULL where `count` reports 0. Restore the 0,
395- // which also keeps the column non-nullable as
396- // `count` had it.
375+ // `sum` of no rows is NULL where `count` is 0.
376+ // Restoring the 0 also keeps the column
377+ // non-nullable, as `count` had it.
397378 let proj =
398379 when ( outer. clone ( ) . is_not_null ( ) , outer. clone ( ) )
399380 . otherwise ( lit ( 0_i64 ) ) ?;
@@ -921,8 +902,7 @@ mod tests {
921902 . build ( ) ?;
922903
923904 // Should not work: `sum` reports nothing about
924- // `GroupsAccumulatorAdapter` either, and measurement says this shape
925- // costs 3.15x the peak memory once rewritten
905+ // `GroupsAccumulatorAdapter` either, so the rewrite is not known to pay
926906 assert_optimized_plan_equal ! (
927907 plan,
928908 @r"
0 commit comments