Skip to content

Commit 6ffc45d

Browse files
authored
fix(substrait): preserve grouping set output order (apache#23468)
## Which issue does this PR close? - Closes apache#17910. ## Rationale for this change For multiple grouping sets, Substrait orders aggregate output as grouping expressions, measures, then grouping-set ID. DataFusion puts its internal `__grouping_id` before measures. The consumer applied `RelCommon.emit` to DataFusion's order, so a mapping such as `[0, 1, 2]` returned `__grouping_id` where the plan requested its first measure. See the [Substrait AggregateRel output mapping specification](https://substrait.io/relations/logical_relations/#aggregate-operation). ## What changes are included in this PR? - Reorder consumer output to Substrait's direct order before applying emit mappings. - Emit a producer mapping back to DataFusion's aggregate order. - Deduplicate producer grouping expressions used by grouping references. - Test both issue reproducer and two-measure roundtrip. ## Are these changes tested? - `cargo test -p datafusion-substrait --features protoc` - `cargo clippy --all-targets --all-features -- -D warnings` ## Are there any user-facing changes? Substrait plans with multiple grouping sets now return emitted columns in specified order. No public API change.
1 parent 7a1f468 commit 6ffc45d

5 files changed

Lines changed: 289 additions & 10 deletions

File tree

‎datafusion/substrait/src/logical_plan/consumer/rel/aggregate_rel.rs‎

Lines changed: 45 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,11 @@
1717

1818
use crate::logical_plan::consumer::{NameTracker, SubstraitConsumer};
1919
use crate::logical_plan::consumer::{from_substrait_agg_func, from_substrait_sorts};
20-
use datafusion::common::{DFSchemaRef, not_impl_err};
21-
use datafusion::logical_expr::{Expr, GroupingSet, LogicalPlan, LogicalPlanBuilder};
20+
use datafusion::common::{Column, DFSchemaRef, internal_err, not_impl_err};
21+
use datafusion::logical_expr::builder::project;
22+
use datafusion::logical_expr::{
23+
Aggregate, Expr, GroupingSet, LogicalPlan, LogicalPlanBuilder,
24+
};
2225
use substrait::proto::AggregateRel;
2326
use substrait::proto::aggregate_function::AggregationInvocation;
2427
use substrait::proto::aggregate_rel::Grouping;
@@ -122,12 +125,51 @@ pub async fn from_aggregate_rel(
122125
.map(|e| name_tracker.get_uniquely_named_expr(e))
123126
.collect::<Result<Vec<Expr>, _>>()?;
124127

125-
input.aggregate(group_exprs, aggr_exprs)?.build()
128+
let plan = input.aggregate(group_exprs, aggr_exprs)?.build()?;
129+
if agg.groupings.len() > 1 {
130+
reorder_grouping_set_output(plan, agg.measures.len())
131+
} else {
132+
Ok(plan)
133+
}
126134
} else {
127135
not_impl_err!("Aggregate without an input is not valid")
128136
}
129137
}
130138

139+
/// Reorders DataFusion's `[groups, grouping_id, measures]` aggregate schema to
140+
/// Substrait's direct output order of `[groups, measures, grouping_id]`.
141+
fn reorder_grouping_set_output(
142+
plan: LogicalPlan,
143+
measure_count: usize,
144+
) -> datafusion::common::Result<LogicalPlan> {
145+
let exprs: Vec<Expr> = {
146+
let schema = plan.schema();
147+
let Some(grouping_id_index) =
148+
schema.index_of_column_by_name(None, Aggregate::INTERNAL_GROUPING_ID)
149+
else {
150+
return internal_err!(
151+
"Grouping set aggregate schema is missing {}",
152+
Aggregate::INTERNAL_GROUPING_ID
153+
);
154+
};
155+
if grouping_id_index + measure_count + 1 != schema.fields().len() {
156+
return internal_err!(
157+
"Grouping set aggregate schema has {} fields after {}, expected {} measures",
158+
schema.fields().len() - grouping_id_index - 1,
159+
Aggregate::INTERNAL_GROUPING_ID,
160+
measure_count
161+
);
162+
}
163+
164+
(0..grouping_id_index)
165+
.chain(grouping_id_index + 1..schema.fields().len())
166+
.chain(std::iter::once(grouping_id_index))
167+
.map(|index| Expr::Column(Column::from(schema.qualified_field(index))))
168+
.collect()
169+
};
170+
project(plan, exprs)
171+
}
172+
131173
#[expect(deprecated)]
132174
async fn from_substrait_grouping(
133175
consumer: &impl SubstraitConsumer,

‎datafusion/substrait/src/logical_plan/producer/rel/aggregate_rel.rs‎

Lines changed: 27 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,8 @@ use datafusion::logical_expr::utils::powerset;
2424
use datafusion::logical_expr::{Aggregate, Distinct, Expr, GroupingSet};
2525
use substrait::proto::aggregate_rel::{Grouping, Measure};
2626
use substrait::proto::rel::RelType;
27-
use substrait::proto::{AggregateRel, Expression, Rel};
27+
use substrait::proto::rel_common::EmitKind;
28+
use substrait::proto::{AggregateRel, Expression, Rel, RelCommon, rel_common};
2829

2930
pub fn from_aggregate(
3031
producer: &mut impl SubstraitProducer,
@@ -38,10 +39,12 @@ pub fn from_aggregate(
3839
.iter()
3940
.map(|e| to_substrait_agg_measure(producer, e, agg.input.schema()))
4041
.collect::<datafusion::common::Result<Vec<_>>>()?;
42+
let common = (groupings.len() > 1)
43+
.then(|| grouping_set_output_mapping(grouping_expressions.len(), measures.len()));
4144

4245
Ok(Box::new(Rel {
4346
rel_type: Some(RelType::Aggregate(Box::new(AggregateRel {
44-
common: None,
47+
common,
4548
input: Some(input),
4649
grouping_expressions,
4750
groupings,
@@ -51,6 +54,22 @@ pub fn from_aggregate(
5154
}))
5255
}
5356

57+
/// Maps Substrait's `[groups, measures, grouping_id]` direct output to
58+
/// DataFusion's `[groups, grouping_id, measures]` aggregate schema.
59+
fn grouping_set_output_mapping(grouping_count: usize, measure_count: usize) -> RelCommon {
60+
let grouping_id_index = grouping_count + measure_count;
61+
let output_mapping = (0..grouping_count)
62+
.chain(std::iter::once(grouping_id_index))
63+
.chain(grouping_count..grouping_id_index)
64+
.map(|index| index as i32)
65+
.collect();
66+
RelCommon {
67+
emit_kind: Some(EmitKind::Emit(rel_common::Emit { output_mapping })),
68+
hint: None,
69+
advanced_extension: None,
70+
}
71+
}
72+
5473
pub fn from_distinct(
5574
producer: &mut impl SubstraitProducer,
5675
distinct: &Distinct,
@@ -165,8 +184,12 @@ pub fn parse_flat_grouping_exprs(
165184
for e in exprs {
166185
let rex = producer.handle_expr(e, schema)?;
167186
grouping_expressions.push(rex.clone());
168-
ref_group_exprs.push(rex);
169-
expression_references.push((ref_group_exprs.len() - 1) as u32);
187+
let reference = ref_group_exprs.iter().position(|existing| existing == &rex);
188+
let reference = reference.unwrap_or_else(|| {
189+
ref_group_exprs.push(rex);
190+
ref_group_exprs.len() - 1
191+
});
192+
expression_references.push(reference as u32);
170193
}
171194
#[expect(deprecated)]
172195
Ok(Grouping {

‎datafusion/substrait/tests/cases/aggregation_tests.rs‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,4 +68,42 @@ mod tests {
6868

6969
Ok(())
7070
}
71+
72+
#[tokio::test]
73+
async fn multiple_grouping_sets_follow_substrait_output_order() -> Result<()> {
74+
let proto_plan = read_json(
75+
"tests/testdata/test_plans/aggregate_groupings/multiple_groupings.json",
76+
);
77+
let ctx = add_plan_schemas_to_ctx(SessionContext::new(), &proto_plan)?;
78+
let plan = from_substrait_plan(&ctx.state(), &proto_plan).await?;
79+
80+
assert_snapshot!(
81+
plan,
82+
@r"
83+
Projection: c0, c1, sum(c0) AS summation
84+
Aggregate: groupBy=[[GROUPING SETS ((c0), (c1), (c0, c1))]], aggr=[[sum(c0)]]
85+
Values: (Int64(1), Int64(10)), (Int64(1), Int64(20)), (Int64(2), Int64(10))
86+
"
87+
);
88+
89+
let results = DataFrame::new(ctx.state(), plan).collect().await?;
90+
datafusion::assert_batches_sorted_eq!(
91+
[
92+
"+----+----+-----------+",
93+
"| c0 | c1 | summation |",
94+
"+----+----+-----------+",
95+
"| | 10 | 3 |",
96+
"| | 20 | 1 |",
97+
"| 1 | | 2 |",
98+
"| 1 | 10 | 1 |",
99+
"| 1 | 20 | 1 |",
100+
"| 2 | | 2 |",
101+
"| 2 | 10 | 2 |",
102+
"+----+----+-----------+",
103+
],
104+
&results
105+
);
106+
107+
Ok(())
108+
}
71109
}

‎datafusion/substrait/tests/cases/roundtrip_logical_plan.rs‎

Lines changed: 52 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -344,10 +344,59 @@ async fn aggregate_multiple_keys() -> Result<()> {
344344

345345
#[tokio::test]
346346
async fn aggregate_grouping_sets() -> Result<()> {
347-
roundtrip(
348-
"SELECT a, c, d, avg(b) FROM data GROUP BY GROUPING SETS ((a, c), (a), (d), ())",
347+
let ctx = create_context().await?;
348+
let proto = roundtrip_with_ctx(
349+
"SELECT a, c, d, avg(b), sum(e) FROM data GROUP BY GROUPING SETS ((a, c), (a), (d), ())",
350+
ctx.clone(),
349351
)
350-
.await
352+
.await?;
353+
354+
let Some(plan_rel::RelType::Root(root)) = &proto.relations[0].rel_type else {
355+
panic!("expected root relation");
356+
};
357+
let Some(RelType::Project(project)) = &root.input.as_ref().unwrap().rel_type else {
358+
panic!("expected project relation");
359+
};
360+
let Some(RelType::Aggregate(aggregate)) = &project.input.as_ref().unwrap().rel_type
361+
else {
362+
panic!("expected aggregate relation");
363+
};
364+
365+
assert_eq!(aggregate.grouping_expressions.len(), 3);
366+
assert_eq!(aggregate.groupings[0].expression_references, [0, 1]);
367+
assert_eq!(aggregate.groupings[1].expression_references, [0]);
368+
assert_eq!(aggregate.groupings[2].expression_references, [2]);
369+
assert!(aggregate.groupings[3].expression_references.is_empty());
370+
let output_mapping = match aggregate
371+
.common
372+
.as_ref()
373+
.and_then(|common| common.emit_kind.as_ref())
374+
{
375+
Some(substrait::proto::rel_common::EmitKind::Emit(emit)) => &emit.output_mapping,
376+
_ => panic!("expected aggregate output mapping"),
377+
};
378+
assert_eq!(output_mapping, &[0, 1, 2, 5, 3, 4]);
379+
380+
let plan = from_substrait_plan(&ctx.state(), &proto).await?;
381+
let results = DataFrame::new(ctx.state(), plan).collect().await?;
382+
datafusion::assert_batches_sorted_eq!(
383+
[
384+
"+---+------------+-------+-------------+-------------+",
385+
"| a | c | d | avg(data.b) | sum(data.e) |",
386+
"+---+------------+-------+-------------+-------------+",
387+
"| | | | 3.250000 | 6442450943 |",
388+
"| | | false | 2.000000 | 4294967295 |",
389+
"| | | true | 4.500000 | 2147483648 |",
390+
"| 1 | | | 2.000000 | 4294967295 |",
391+
"| 1 | 2020-01-01 | | 2.000000 | 4294967295 |",
392+
"| 3 | | | 4.500000 | 2147483648 |",
393+
"| 3 | 2020-01-01 | | 4.500000 | 2147483648 |",
394+
"+---+------------+-------+-------------+-------------+",
395+
],
396+
&results
397+
);
398+
399+
Ok(())
351400
}
352401

353402
#[tokio::test]
Lines changed: 127 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,127 @@
1+
{
2+
"extensionUris": [
3+
{
4+
"extensionUriAnchor": 1,
5+
"uri": "https://github.com/substrait-io/substrait/blob/main/extensions/functions_arithmetic.yaml"
6+
}
7+
],
8+
"extensions": [
9+
{
10+
"extensionFunction": {
11+
"extensionUriReference": 1,
12+
"functionAnchor": 1,
13+
"name": "sum:i64"
14+
}
15+
}
16+
],
17+
"relations": [
18+
{
19+
"root": {
20+
"input": {
21+
"aggregate": {
22+
"common": {
23+
"emit": {
24+
"outputMapping": [0, 1, 2]
25+
}
26+
},
27+
"input": {
28+
"read": {
29+
"baseSchema": {
30+
"names": ["c0", "c1"],
31+
"struct": {
32+
"nullability": "NULLABILITY_REQUIRED",
33+
"types": [
34+
{
35+
"i64": {
36+
"nullability": "NULLABILITY_REQUIRED"
37+
}
38+
},
39+
{
40+
"i64": {
41+
"nullability": "NULLABILITY_REQUIRED"
42+
}
43+
}
44+
]
45+
}
46+
},
47+
"common": {
48+
"direct": {}
49+
},
50+
"virtualTable": {
51+
"expressions": [
52+
{
53+
"fields": [
54+
{ "literal": { "i64": "1", "nullable": false } },
55+
{ "literal": { "i64": "10", "nullable": false } }
56+
]
57+
},
58+
{
59+
"fields": [
60+
{ "literal": { "i64": "1", "nullable": false } },
61+
{ "literal": { "i64": "20", "nullable": false } }
62+
]
63+
},
64+
{
65+
"fields": [
66+
{ "literal": { "i64": "2", "nullable": false } },
67+
{ "literal": { "i64": "10", "nullable": false } }
68+
]
69+
}
70+
]
71+
}
72+
}
73+
},
74+
"groupingExpressions": [
75+
{
76+
"selection": {
77+
"directReference": { "structField": {} },
78+
"rootReference": {}
79+
}
80+
},
81+
{
82+
"selection": {
83+
"directReference": { "structField": { "field": 1 } },
84+
"rootReference": {}
85+
}
86+
}
87+
],
88+
"groupings": [
89+
{ "expressionReferences": [0] },
90+
{ "expressionReferences": [1] },
91+
{ "expressionReferences": [0, 1] }
92+
],
93+
"measures": [
94+
{
95+
"measure": {
96+
"arguments": [
97+
{
98+
"value": {
99+
"selection": {
100+
"directReference": { "structField": {} },
101+
"rootReference": {}
102+
}
103+
}
104+
}
105+
],
106+
"functionReference": 1,
107+
"invocation": "AGGREGATION_INVOCATION_ALL",
108+
"outputType": {
109+
"i64": {
110+
"nullability": "NULLABILITY_NULLABLE"
111+
}
112+
},
113+
"phase": "AGGREGATION_PHASE_INITIAL_TO_RESULT"
114+
}
115+
}
116+
]
117+
}
118+
},
119+
"names": ["c0", "c1", "summation"]
120+
}
121+
}
122+
],
123+
"version": {
124+
"minorNumber": 29,
125+
"producer": "substrait-go v4.2.0"
126+
}
127+
}

0 commit comments

Comments
 (0)