Skip to content

Commit 526ed6c

Browse files
authored
Prevent empty grouping sets from being eliminated on empty input (apache#22039)
## Which issue does this PR close? - Closes apache#22040 ## Rationale for this change `GROUPING SETS (())` should return one row even when the input is empty, because the empty grouping set represents a global aggregate. In the current behavior, queries such as `WHERE false GROUP BY GROUPING SETS(())` can incorrectly be optimized into an empty result, returning 0 rows instead of the expected 1 row. This change fixes that SQL semantics bug. ## What changes are included in this PR? This PR makes two related fixes: 1. It updates the empty-relation propagation rule so aggregates with an empty grouping set are not eliminated. 2. It updates the grouped hash aggregate execution path so empty grouping sets are initialized correctly and still produce output on empty input. It also adds regression coverage in `grouping.slt` for: - `GROUPING SETS (())` on empty input - `COUNT(*)` on empty input - mixed grouping sets where only the empty grouping set should produce a row ## Are these changes tested? Yes. I ran: - `cargo test -p datafusion-sqllogictest -- grouping` - cargo test -p datafusion-optimizer -- propagate_empty - `cargo clippy --tests --all-features -p datafusion-physical-plan -p datafusion-optimizer -- -D warnings` - `cargo fmt --check` ## Are there any user-facing changes? Yes. Queries using `GROUPING SETS (())`, and other grouping-set forms that include the empty set such as `ROLLUP` and `CUBE`, will now return the correct row on empty input instead of incorrectly returning no rows.
1 parent 0305098 commit 526ed6c

4 files changed

Lines changed: 169 additions & 7 deletions

File tree

‎datafusion/optimizer/src/propagate_empty_relation.rs‎

Lines changed: 31 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ use datafusion_common::JoinType;
2323
use datafusion_common::tree_node::Transformed;
2424
use datafusion_common::{Column, DFSchemaRef, Result, ScalarValue, plan_err};
2525
use datafusion_expr::logical_plan::LogicalPlan;
26-
use datafusion_expr::{EmptyRelation, Expr, Projection, Union, cast, lit};
26+
use datafusion_expr::{EmptyRelation, Expr, GroupingSet, Projection, Union, cast, lit};
2727

2828
use crate::optimizer::ApplyOrder;
2929
use crate::{OptimizerConfig, OptimizerRule};
@@ -174,7 +174,13 @@ impl OptimizerRule for PropagateEmptyRelation {
174174
}
175175
}
176176
LogicalPlan::Aggregate(ref agg) => {
177+
// An aggregate over an empty input can be eliminated only when
178+
// there is no empty grouping set. An empty grouping set `()`
179+
// (from `GROUPING SETS(())`, `ROLLUP(...)`, or `CUBE(...)`)
180+
// always produces exactly one row even on empty input, so it
181+
// must not be replaced by an empty relation.
177182
if !agg.group_expr.is_empty()
183+
&& !has_empty_grouping_set(&agg.group_expr)
178184
&& let Some(empty_plan) = empty_child(&plan)?
179185
{
180186
return Ok(Transformed::yes(empty_plan));
@@ -315,6 +321,30 @@ fn build_null_padded_projection(
315321
)?))
316322
}
317323

324+
/// Returns `true` if any grouping set in the list of GROUP BY expressions is
325+
/// the empty set `()`.
326+
///
327+
/// An empty grouping set acts as a "grand total" group: the aggregate must
328+
/// always produce **exactly one row** for it, even when the input is empty.
329+
/// This means an aggregate with an empty grouping set cannot be replaced by
330+
/// an empty relation.
331+
///
332+
/// The three forms that can contain an empty grouping set:
333+
/// - `GROUPING SETS (…, (), …)` — explicitly listed.
334+
/// - `ROLLUP(exprs)` — always expands to include `()`.
335+
/// - `CUBE(exprs)` — always expands to include `()`.
336+
fn has_empty_grouping_set(group_expr: &[Expr]) -> bool {
337+
match group_expr.first() {
338+
Some(Expr::GroupingSet(GroupingSet::GroupingSets(groups))) => {
339+
groups.iter().any(|g| g.is_empty())
340+
}
341+
// Both ROLLUP and CUBE always include the empty grouping set ().
342+
Some(Expr::GroupingSet(GroupingSet::Rollup(_)))
343+
| Some(Expr::GroupingSet(GroupingSet::Cube(_))) => true,
344+
_ => false,
345+
}
346+
}
347+
318348
#[cfg(test)]
319349
mod tests {
320350

‎datafusion/physical-plan/src/aggregates/mod.rs‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2063,11 +2063,11 @@ fn evaluate_optional(
20632063
/// The integer type is chosen to be the smallest `UInt8 / UInt16 / UInt32 /
20642064
/// UInt64` that can represent both parts. It matches the type returned by
20652065
/// [`Aggregate::grouping_id_type`].
2066-
fn group_id_array(
2066+
pub(crate) fn group_id_array(
20672067
group: &[bool],
20682068
ordinal: usize,
20692069
max_ordinal: usize,
2070-
batch: &RecordBatch,
2070+
num_rows: usize,
20712071
) -> Result<ArrayRef> {
20722072
let n = group.len();
20732073
if n > 64 {
@@ -2087,7 +2087,6 @@ fn group_id_array(
20872087
(acc << 1) | if is_null { 1 } else { 0 }
20882088
});
20892089
let full_id = semantic_id | ((ordinal as u64) << n);
2090-
let num_rows = batch.num_rows();
20912090
if total_bits <= 8 {
20922091
Ok(Arc::new(UInt8Array::from(vec![full_id as u8; num_rows])))
20932092
} else if total_bits <= 16 {
@@ -2106,7 +2105,7 @@ fn group_id_array(
21062105
/// ordinal 0, the second gets 1, and so on. If the same `Vec<bool>` appears
21072106
/// three times the ordinals are 0, 1, 2 and this function returns 2.
21082107
/// Returns 0 when no grouping set is duplicated.
2109-
fn max_duplicate_ordinal(groups: &[Vec<bool>]) -> usize {
2108+
pub(crate) fn max_duplicate_ordinal(groups: &[Vec<bool>]) -> usize {
21102109
let mut counts: HashMap<&[bool], usize> = HashMap::new();
21112110
for group in groups {
21122111
*counts.entry(group).or_insert(0) += 1;
@@ -2160,7 +2159,7 @@ pub fn evaluate_group_by(
21602159
group,
21612160
current_ordinal,
21622161
max_ordinal,
2163-
batch,
2162+
batch.num_rows(),
21642163
)?);
21652164
}
21662165
Ok(group_values)

‎datafusion/physical-plan/src/aggregates/row_hash.rs‎

Lines changed: 103 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,8 @@ use crate::aggregates::group_values::{GroupByMetrics, GroupValues, new_group_val
2727
use crate::aggregates::order::GroupOrderingFull;
2828
use crate::aggregates::{
2929
AggregateInputMode, AggregateMode, AggregateOutputMode, PhysicalGroupBy,
30-
create_schema, evaluate_group_by, evaluate_many, evaluate_optional,
30+
create_schema, evaluate_group_by, evaluate_many, evaluate_optional, group_id_array,
31+
max_duplicate_ordinal,
3132
};
3233
use crate::metrics::{BaselineMetrics, MetricBuilder, MetricCategory, RecordOutput};
3334
use crate::sorts::streaming_merge::{SortedSpillFile, StreamingMergeBuilder};
@@ -360,6 +361,7 @@ pub(crate) struct GroupedHashAggregateStream {
360361
// the execution.
361362
// ========================================================================
362363
schema: SchemaRef,
364+
input_schema: SchemaRef,
363365
input: SendableRecordBatchStream,
364366
mode: AggregateMode,
365367

@@ -661,6 +663,7 @@ impl GroupedHashAggregateStream {
661663

662664
Ok(GroupedHashAggregateStream {
663665
schema: agg_schema,
666+
input_schema: agg.input().schema(),
664667
input,
665668
mode: agg.mode,
666669
accumulators,
@@ -1125,6 +1128,104 @@ impl GroupedHashAggregateStream {
11251128
Ok(Some(batch))
11261129
}
11271130

1131+
/// Registers groups for empty grouping sets when no input rows were seen.
1132+
///
1133+
/// `GROUP BY GROUPING SETS (())` must always produce one row even when there
1134+
/// are no input rows (standard SQL semantics for a "grand total" group).
1135+
/// Mixed grouping sets like `GROUPING SETS (a, ())` also produce one row for
1136+
/// the empty set `()` on empty input.
1137+
///
1138+
/// This method interns the group keys and primes the accumulators so they
1139+
/// produce their zero-row aggregate values (e.g. `NULL` for `SUM`,
1140+
/// `0` for `COUNT`).
1141+
fn init_empty_grouping_sets(&mut self) -> Result<()> {
1142+
if !self.group_by.has_grouping_set() || !self.group_values.is_empty() {
1143+
return Ok(());
1144+
}
1145+
1146+
let max_ordinal = max_duplicate_ordinal(self.group_by.groups());
1147+
let mut ordinals: std::collections::HashMap<&[bool], usize> =
1148+
std::collections::HashMap::new();
1149+
let group_schema = self.group_by.group_schema(&self.input_schema)?;
1150+
let n_expr = self.group_by.expr().len();
1151+
let mut any_interned = false;
1152+
1153+
for group in self.group_by.groups() {
1154+
let ordinal = {
1155+
let entry = ordinals.entry(group.as_slice()).or_insert(0);
1156+
let o = *entry;
1157+
*entry += 1;
1158+
o
1159+
};
1160+
1161+
if !group.iter().all(|&is_null| is_null) {
1162+
continue;
1163+
}
1164+
1165+
// Build the group key: one NULL per group-by expression, then the grouping_id.
1166+
let mut cols: Vec<ArrayRef> = group_schema
1167+
.fields()
1168+
.iter()
1169+
.take(n_expr)
1170+
.map(|f| new_null_array(f.data_type(), 1))
1171+
.collect();
1172+
cols.push(group_id_array(group, ordinal, max_ordinal, 1)?);
1173+
1174+
let starting_groups = self.group_values.len();
1175+
self.group_values
1176+
.intern(&cols, &mut self.current_group_indices)?;
1177+
let total_groups = self.group_values.len();
1178+
if total_groups > starting_groups {
1179+
self.group_ordering.new_groups(
1180+
&cols,
1181+
&self.current_group_indices,
1182+
total_groups,
1183+
)?;
1184+
}
1185+
any_interned = true;
1186+
}
1187+
1188+
if any_interned {
1189+
// Prime each accumulator for the registered group count with no data.
1190+
//
1191+
// We build 1-row null arrays for each aggregate argument and pass them
1192+
// with an all-false filter. The filter ensures no row is accumulated
1193+
// into any group, which keeps every group in its "zero" initial state
1194+
// (NULL for SUM/AVG/MIN/MAX, 0 for COUNT).
1195+
//
1196+
// Using a 1-row batch rather than 0 rows is required to avoid a fast
1197+
// path in `NullState::accumulate` that treats "0 nulls in a 0-row
1198+
// array" as "all groups have been seen", which would cause SUM to
1199+
// return 0 instead of NULL.
1200+
//
1201+
// Argument types are inferred directly from the expression metadata so
1202+
// we never need to construct a full `RecordBatch`.
1203+
let total_groups = self.group_values.len();
1204+
let null_args: Vec<Vec<ArrayRef>> = self
1205+
.aggregate_arguments
1206+
.iter()
1207+
.map(|args| {
1208+
args.iter()
1209+
.map(|expr| {
1210+
let dt = expr.data_type(&self.input_schema)?;
1211+
Ok(new_null_array(&dt, 1))
1212+
})
1213+
.collect::<Result<Vec<_>>>()
1214+
})
1215+
.collect::<Result<Vec<_>>>()?;
1216+
let false_filter = BooleanArray::from(vec![false]);
1217+
for (acc, args) in self.accumulators.iter_mut().zip(null_args.iter()) {
1218+
if self.mode.input_mode() == AggregateInputMode::Raw {
1219+
acc.update_batch(args, &[0], Some(&false_filter), total_groups)?;
1220+
} else {
1221+
acc.merge_batch(args, &[0], Some(&false_filter), total_groups)?;
1222+
}
1223+
}
1224+
}
1225+
1226+
Ok(())
1227+
}
1228+
11281229
/// Emit all intermediate aggregation states, sort them, and store them on disk.
11291230
/// This process helps in reducing memory pressure by allowing the data to be
11301231
/// read back with streaming merge.
@@ -1223,6 +1324,7 @@ impl GroupedHashAggregateStream {
12231324
let timer = elapsed_compute.timer();
12241325
self.exec_state = if self.spill_state.spills.is_empty() {
12251326
// Input has been entirely processed without spilling to disk.
1327+
self.init_empty_grouping_sets()?;
12261328

12271329
// Flush any remaining group values.
12281330
let batch = self.emit(EmitTo::All, false)?;

‎datafusion/sqllogictest/test_files/grouping.slt‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -224,3 +224,34 @@ query I
224224
SELECT SUM(v1) FROM generate_series(10) AS t1(v1) GROUP BY GROUPING SETS(())
225225
----
226226
55
227+
228+
# grouping_sets_empty_input: GROUPING SETS (()) must produce one NULL row on empty input
229+
# (standard SQL: the empty grouping set always defines exactly one group)
230+
query I
231+
SELECT SUM(v1) FROM generate_series(10) AS t1(v1) WHERE false GROUP BY GROUPING SETS(())
232+
----
233+
NULL
234+
235+
# grouping_sets_empty_input_count: COUNT returns 0 for the empty group, not a missing row
236+
query I
237+
SELECT COUNT(*) FROM generate_series(10) AS t1(v1) WHERE false GROUP BY GROUPING SETS(())
238+
----
239+
0
240+
241+
# rollup_empty_input: ROLLUP includes the empty grouping set and must produce one row
242+
query II
243+
SELECT SUM(v1), COUNT(*) FROM generate_series(10) AS t1(v1) WHERE false GROUP BY ROLLUP(v1)
244+
----
245+
NULL 0
246+
247+
# cube_empty_input: CUBE includes the empty grouping set and must produce one row
248+
query II
249+
SELECT SUM(v1), COUNT(*) FROM generate_series(10) AS t1(v1) WHERE false GROUP BY CUBE(v1)
250+
----
251+
NULL 0
252+
253+
# grouping_sets_mixed_empty_and_non_empty: only the empty set (()) produces a row on empty input
254+
query II
255+
SELECT SUM(v1), COUNT(*) FROM generate_series(10) AS t1(v1) WHERE false GROUP BY GROUPING SETS((), (v1))
256+
----
257+
NULL 0

0 commit comments

Comments
 (0)