Skip to content

Commit b388b2c

Browse files
committed
fix(optimizer): dedup-aware group count + aliased peers in coalesce_first_last
Correctness fixes to the coalesce rule (folds into #23682 on squash-merge): - use `Aggregate::group_expr_len()` (deduplicated count, matching how `Aggregate::try_new` builds the schema) instead of `group_expr.len()`, which over-counts duplicate group exprs (e.g. `GROUP BY p, p`) and misaligned the group/aggregate split of the output columns, producing an invalid projection that referenced a non-existent column - classify `Alias`-wrapped `first_value` / `last_value` so aliased peers (e.g. from the DataFrame API) coalesce, per the documented `aggr_expr` shape - note that name-based bucketing is safe because a session resolves one canonical UDF per name Tests: duplicate GROUP BY (regression), aliased peers, a 3-member bucket, and negative cases that must NOT coalesce (different ORDER BY / FILTER / IGNORE NULLS).
1 parent f9e2eb2 commit b388b2c

1 file changed

Lines changed: 146 additions & 5 deletions

File tree

‎datafusion/optimizer/src/coalesce_first_last.rs‎

Lines changed: 146 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,9 @@ impl CoalesceFirstLast {
7474

7575
/// `(function name, ORDER BY key, null treatment)` — peers may be coalesced only
7676
/// when all three match.
77+
// Peers are bucketed by (function name, ORDER BY, null treatment). Using the
78+
// function *name* is safe because a session resolves one canonical UDF per
79+
// name, so same-keyed members share an implementation.
7780
type BucketKey = (String, Vec<Sort>, Option<NullTreatment>);
7881

7982
struct Coalesceable {
@@ -83,7 +86,14 @@ struct Coalesceable {
8386
}
8487

8588
fn classify(expr: &Expr) -> Option<Coalesceable> {
86-
let Expr::AggregateFunction(AggregateFunction { func, params }) = expr else {
89+
// `Aggregate::aggr_expr` entries are documented to be either an
90+
// `AggregateFunction` or an `Alias` wrapping one (e.g. built via the
91+
// DataFrame API); unwrap the alias so aliased peers are coalesced too.
92+
let inner = match expr {
93+
Expr::Alias(alias) => alias.expr.as_ref(),
94+
other => other,
95+
};
96+
let Expr::AggregateFunction(AggregateFunction { func, params }) = inner else {
8797
return None;
8898
};
8999
let name = func.name();
@@ -139,9 +149,9 @@ impl OptimizerRule for CoalesceFirstLast {
139149
return Ok(Transformed::no(plan));
140150
};
141151

142-
// Grouping sets expand the group columns (plus an internal grouping id)
143-
// beyond `group_expr.len()`, which the schema slicing below assumes; skip
144-
// them (consistent with the other cases this rule declines to coalesce).
152+
// Grouping sets add an internal grouping-id column and distinct
153+
// group-by semantics; this rule declines to coalesce them (consistent
154+
// with the other cases it skips).
145155
if matches!(aggregate.group_expr.as_slice(), [Expr::GroupingSet(_)]) {
146156
return Ok(Transformed::no(LogicalPlan::Aggregate(aggregate)));
147157
}
@@ -171,7 +181,11 @@ impl OptimizerRule for CoalesceFirstLast {
171181
return Ok(Transformed::no(LogicalPlan::Aggregate(aggregate)));
172182
};
173183

174-
let group_len = aggregate.group_expr.len();
184+
// Use `group_expr_len()` (deduplicated count, matching how
185+
// `Aggregate::try_new` builds the schema) rather than `group_expr.len()`,
186+
// which over-counts duplicate group exprs (e.g. `GROUP BY p, p`) and would
187+
// misalign the group/aggregate split of `orig_columns` below.
188+
let group_len = aggregate.group_expr_len()?;
175189
let orig_columns = aggregate.schema.columns();
176190

177191
// Original aggregate index -> (struct alias, struct field position).
@@ -514,4 +528,131 @@ mod tests {
514528
TableScan: t
515529
")
516530
}
531+
532+
#[test]
533+
fn coalesces_duplicate_group_by() -> Result<()> {
534+
// Regression: `GROUP BY p, p` stores two group exprs, but the schema
535+
// deduplicates them to one group column. The rule must split group vs
536+
// aggregate columns by the deduplicated count (`group_expr_len`), not
537+
// `group_expr.len()`, or the projection references a non-existent column.
538+
let plan = LogicalPlanBuilder::from(scan()?)
539+
.aggregate(
540+
vec![col("p"), col("p")],
541+
vec![
542+
first_value(col("a"), order_by_o(), false),
543+
first_value(col("b"), order_by_o(), false),
544+
],
545+
)?
546+
.build()?;
547+
assert_coalesced!(TestConfig::new(), plan, @ r#"
548+
Projection: t.p, get_field(__coalesce_first_last_1, Utf8("c0")) AS first_value(t.a) ORDER BY [t.o DESC NULLS LAST], get_field(__coalesce_first_last_1, Utf8("c1")) AS first_value(t.b) ORDER BY [t.o DESC NULLS LAST]
549+
Aggregate: groupBy=[[t.p, t.p]], aggr=[[first_value(named_struct(Utf8("c0"), t.a, Utf8("c1"), t.b)) ORDER BY [t.o DESC NULLS LAST] AS __coalesce_first_last_1]]
550+
TableScan: t
551+
"#)
552+
}
553+
554+
#[test]
555+
fn coalesces_aliased_peers() -> Result<()> {
556+
// Aliased first_value peers (e.g. from the DataFrame API) still coalesce.
557+
let plan = LogicalPlanBuilder::from(scan()?)
558+
.aggregate(
559+
vec![col("p")],
560+
vec![
561+
first_value(col("a"), order_by_o(), false).alias("fa"),
562+
first_value(col("b"), order_by_o(), false).alias("fb"),
563+
],
564+
)?
565+
.build()?;
566+
assert_coalesced!(TestConfig::new(), plan, @ r#"
567+
Projection: t.p, get_field(__coalesce_first_last_1, Utf8("c0")) AS fa, get_field(__coalesce_first_last_1, Utf8("c1")) AS fb
568+
Aggregate: groupBy=[[t.p]], aggr=[[first_value(named_struct(Utf8("c0"), t.a, Utf8("c1"), t.b)) ORDER BY [t.o DESC NULLS LAST] AS __coalesce_first_last_1]]
569+
TableScan: t
570+
"#)
571+
}
572+
573+
#[test]
574+
fn coalesces_three_peers() -> Result<()> {
575+
// A bucket with three members packs into c0 / c1 / c2.
576+
let plan = LogicalPlanBuilder::from(scan()?)
577+
.aggregate(
578+
vec![col("p")],
579+
vec![
580+
first_value(col("a"), order_by_o(), false),
581+
first_value(col("b"), order_by_o(), false),
582+
first_value(col("o"), order_by_o(), false),
583+
],
584+
)?
585+
.build()?;
586+
assert_coalesced!(TestConfig::new(), plan, @ r#"
587+
Projection: t.p, get_field(__coalesce_first_last_1, Utf8("c0")) AS first_value(t.a) ORDER BY [t.o DESC NULLS LAST], get_field(__coalesce_first_last_1, Utf8("c1")) AS first_value(t.b) ORDER BY [t.o DESC NULLS LAST], get_field(__coalesce_first_last_1, Utf8("c2")) AS first_value(t.o) ORDER BY [t.o DESC NULLS LAST]
588+
Aggregate: groupBy=[[t.p]], aggr=[[first_value(named_struct(Utf8("c0"), t.a, Utf8("c1"), t.b, Utf8("c2"), t.o)) ORDER BY [t.o DESC NULLS LAST] AS __coalesce_first_last_1]]
589+
TableScan: t
590+
"#)
591+
}
592+
593+
#[test]
594+
fn no_op_different_order_by() -> Result<()> {
595+
// Different ORDER BY => not peers => must not coalesce.
596+
let plan = LogicalPlanBuilder::from(scan()?)
597+
.aggregate(
598+
vec![col("p")],
599+
vec![
600+
first_value(col("a"), order_by_o(), false),
601+
first_value(col("b"), vec![Sort::new(col("o"), true, true)], false),
602+
],
603+
)?
604+
.build()?;
605+
assert_coalesced!(TestConfig::new(), plan, @ "
606+
Aggregate: groupBy=[[t.p]], aggr=[[first_value(t.a) ORDER BY [t.o DESC NULLS LAST], first_value(t.b) ORDER BY [t.o ASC NULLS FIRST]]]
607+
TableScan: t
608+
")
609+
}
610+
611+
#[test]
612+
fn no_op_filter() -> Result<()> {
613+
// A FILTER on a peer can't share the single struct scan => not coalesced.
614+
let filtered = Expr::AggregateFunction(AggregateFunction::new_udf(
615+
first_value_udaf(),
616+
vec![col("a")],
617+
false,
618+
Some(Box::new(col("a").gt(lit(0u32)))),
619+
order_by_o(),
620+
None,
621+
));
622+
let plan = LogicalPlanBuilder::from(scan()?)
623+
.aggregate(
624+
vec![col("p")],
625+
vec![filtered, first_value(col("b"), order_by_o(), false)],
626+
)?
627+
.build()?;
628+
assert_coalesced!(TestConfig::new(), plan, @ "
629+
Aggregate: groupBy=[[t.p]], aggr=[[first_value(t.a) FILTER (WHERE t.a > UInt32(0)) ORDER BY [t.o DESC NULLS LAST], first_value(t.b) ORDER BY [t.o DESC NULLS LAST]]]
630+
TableScan: t
631+
")
632+
}
633+
634+
#[test]
635+
fn no_op_ignore_nulls() -> Result<()> {
636+
// IGNORE NULLS can't be reproduced through a single struct => not coalesced.
637+
let ignore_nulls = |arg: Expr| {
638+
Expr::AggregateFunction(AggregateFunction::new_udf(
639+
first_value_udaf(),
640+
vec![arg],
641+
false,
642+
None,
643+
order_by_o(),
644+
Some(NullTreatment::IgnoreNulls),
645+
))
646+
};
647+
let plan = LogicalPlanBuilder::from(scan()?)
648+
.aggregate(
649+
vec![col("p")],
650+
vec![ignore_nulls(col("a")), ignore_nulls(col("b"))],
651+
)?
652+
.build()?;
653+
assert_coalesced!(TestConfig::new(), plan, @ "
654+
Aggregate: groupBy=[[t.p]], aggr=[[first_value(t.a) IGNORE NULLS ORDER BY [t.o DESC NULLS LAST], first_value(t.b) IGNORE NULLS ORDER BY [t.o DESC NULLS LAST]]]
655+
TableScan: t
656+
")
657+
}
517658
}

0 commit comments

Comments
 (0)