You signed in with another tab or window. Reload to refresh your session.You signed out in another tab or window. Reload to refresh your session.You switched accounts on another tab or window. Reload to refresh your session.Dismiss alert
{{ message }}
Repository navigation
Commit b388b2c
Browse filesBrowse the repository at this point in the historyBrowse files
fix(optimizer): dedup-aware group count + aliased peers in coalesce_first_last
Correctness fixes to the coalesce rule (folds into #23682 on squash-merge):
- use `Aggregate::group_expr_len()` (deduplicated count, matching how
`Aggregate::try_new` builds the schema) instead of `group_expr.len()`, which
over-counts duplicate group exprs (e.g. `GROUP BY p, p`) and misaligned the
group/aggregate split of the output columns, producing an invalid projection
that referenced a non-existent column
- classify `Alias`-wrapped `first_value` / `last_value` so aliased peers (e.g.
from the DataFrame API) coalesce, per the documented `aggr_expr` shape
- note that name-based bucketing is safe because a session resolves one
canonical UDF per name
Tests: duplicate GROUP BY (regression), aliased peers, a 3-member bucket, and
negative cases that must NOT coalesce (different ORDER BY / FILTER / IGNORE NULLS).
// Use `group_expr_len()` (deduplicated count, matching how
185
+
// `Aggregate::try_new` builds the schema) rather than `group_expr.len()`,
186
+
// which over-counts duplicate group exprs (e.g. `GROUP BY p, p`) and would
187
+
// misalign the group/aggregate split of `orig_columns` below.
188
+
let group_len = aggregate.group_expr_len()?;
175
189
let orig_columns = aggregate.schema.columns();
176
190
177
191
// Original aggregate index -> (struct alias, struct field position).
@@ -514,4 +528,131 @@ mod tests {
514
528
TableScan: t
515
529
")
516
530
}
531
+
532
+
#[test]
533
+
fncoalesces_duplicate_group_by() -> Result<()>{
534
+
// Regression: `GROUP BY p, p` stores two group exprs, but the schema
535
+
// deduplicates them to one group column. The rule must split group vs
536
+
// aggregate columns by the deduplicated count (`group_expr_len`), not
537
+
// `group_expr.len()`, or the projection references a non-existent column.
538
+
let plan = LogicalPlanBuilder::from(scan()?)
539
+
.aggregate(
540
+
vec![col("p"), col("p")],
541
+
vec![
542
+
first_value(col("a"), order_by_o(),false),
543
+
first_value(col("b"), order_by_o(),false),
544
+
],
545
+
)?
546
+
.build()?;
547
+
assert_coalesced!(TestConfig::new(), plan, @ r#"
548
+
Projection: t.p, get_field(__coalesce_first_last_1, Utf8("c0")) AS first_value(t.a) ORDER BY [t.o DESC NULLS LAST], get_field(__coalesce_first_last_1, Utf8("c1")) AS first_value(t.b) ORDER BY [t.o DESC NULLS LAST]
549
+
Aggregate: groupBy=[[t.p, t.p]], aggr=[[first_value(named_struct(Utf8("c0"), t.a, Utf8("c1"), t.b)) ORDER BY [t.o DESC NULLS LAST] AS __coalesce_first_last_1]]
550
+
TableScan: t
551
+
"#)
552
+
}
553
+
554
+
#[test]
555
+
fncoalesces_aliased_peers() -> Result<()>{
556
+
// Aliased first_value peers (e.g. from the DataFrame API) still coalesce.
Projection: t.p, get_field(__coalesce_first_last_1, Utf8("c0")) AS fa, get_field(__coalesce_first_last_1, Utf8("c1")) AS fb
568
+
Aggregate: groupBy=[[t.p]], aggr=[[first_value(named_struct(Utf8("c0"), t.a, Utf8("c1"), t.b)) ORDER BY [t.o DESC NULLS LAST] AS __coalesce_first_last_1]]
569
+
TableScan: t
570
+
"#)
571
+
}
572
+
573
+
#[test]
574
+
fncoalesces_three_peers() -> Result<()>{
575
+
// A bucket with three members packs into c0 / c1 / c2.
576
+
let plan = LogicalPlanBuilder::from(scan()?)
577
+
.aggregate(
578
+
vec![col("p")],
579
+
vec![
580
+
first_value(col("a"), order_by_o(),false),
581
+
first_value(col("b"), order_by_o(),false),
582
+
first_value(col("o"), order_by_o(),false),
583
+
],
584
+
)?
585
+
.build()?;
586
+
assert_coalesced!(TestConfig::new(), plan, @ r#"
587
+
Projection: t.p, get_field(__coalesce_first_last_1, Utf8("c0")) AS first_value(t.a) ORDER BY [t.o DESC NULLS LAST], get_field(__coalesce_first_last_1, Utf8("c1")) AS first_value(t.b) ORDER BY [t.o DESC NULLS LAST], get_field(__coalesce_first_last_1, Utf8("c2")) AS first_value(t.o) ORDER BY [t.o DESC NULLS LAST]
588
+
Aggregate: groupBy=[[t.p]], aggr=[[first_value(named_struct(Utf8("c0"), t.a, Utf8("c1"), t.b, Utf8("c2"), t.o)) ORDER BY [t.o DESC NULLS LAST] AS __coalesce_first_last_1]]
589
+
TableScan: t
590
+
"#)
591
+
}
592
+
593
+
#[test]
594
+
fnno_op_different_order_by() -> Result<()>{
595
+
// Different ORDER BY => not peers => must not coalesce.
Aggregate: groupBy=[[t.p]], aggr=[[first_value(t.a) IGNORE NULLS ORDER BY [t.o DESC NULLS LAST], first_value(t.b) IGNORE NULLS ORDER BY [t.o DESC NULLS LAST]]]
0 commit comments