Skip to content

Commit e30bb65

Browse files
committed
fix(optimizer): preserve alias qualifier in coalesce_first_last
A qualified alias on a coalesced peer lost its qualifier and tripped the optimizer's schema-invariant check. Also normalizes null treatment in the bucket key, notes in the docs that ties may resolve differently with the rule on vs off, and adds result-level slt coverage for NULL order keys, explicit null placement, multi-key ORDER BY, struct payloads, and tied keys.
1 parent b388b2c commit e30bb65

2 files changed

Lines changed: 316 additions & 30 deletions

File tree

‎datafusion/optimizer/src/coalesce_first_last.rs‎

Lines changed: 138 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,11 @@ const GET_FIELD: &str = "get_field";
6161
/// The input is scanned once, not once per expression, and holds one per-group
6262
/// state slot instead of N.
6363
///
64+
/// Ties are not guaranteed to break the same way with the rule on and off. Each
65+
/// plan resolves them deterministically on its own, but the two do not guarantee
66+
/// the same choice as each other, so duplicate `ORDER BY` keys can yield
67+
/// different (equally valid) winners depending on the flag.
68+
///
6469
/// Off by default (`optimizer.enable_coalesce_first_last`); no-ops if
6570
/// `named_struct` / `get_field` are not registered.
6671
#[derive(Default, Debug)]
@@ -74,10 +79,9 @@ impl CoalesceFirstLast {
7479

7580
/// `(function name, ORDER BY key, null treatment)` — peers may be coalesced only
7681
/// when all three match.
77-
// Peers are bucketed by (function name, ORDER BY, null treatment). Using the
78-
// function *name* is safe because a session resolves one canonical UDF per
79-
// name, so same-keyed members share an implementation.
80-
type BucketKey = (String, Vec<Sort>, Option<NullTreatment>);
82+
// Using the function *name* is safe because a session resolves one canonical
83+
// UDF per name, so same-keyed members share an implementation.
84+
type BucketKey = (String, Vec<Sort>, NullTreatment);
8185

8286
struct Coalesceable {
8387
key: BucketKey,
@@ -86,9 +90,8 @@ struct Coalesceable {
8690
}
8791

8892
fn classify(expr: &Expr) -> Option<Coalesceable> {
89-
// `Aggregate::aggr_expr` entries are documented to be either an
90-
// `AggregateFunction` or an `Alias` wrapping one (e.g. built via the
91-
// DataFrame API); unwrap the alias so aliased peers are coalesced too.
93+
// `aggr_expr` entries are an `AggregateFunction` or an `Alias` wrapping one
94+
// (e.g. built via the DataFrame API), so unwrap to catch aliased peers.
9295
let inner = match expr {
9396
Expr::Alias(alias) => alias.expr.as_ref(),
9497
other => other,
@@ -108,9 +111,8 @@ fn classify(expr: &Expr) -> Option<Coalesceable> {
108111
null_treatment,
109112
} = params;
110113

111-
// DISTINCT / FILTER would break the shared scan, and IGNORE NULLS cannot be
112-
// reproduced through a single struct (it would skip rows where the whole
113-
// struct is null rather than where the individual value is null).
114+
// DISTINCT / FILTER break the shared scan, and IGNORE NULLS can't go through
115+
// one struct (it would skip whole-struct nulls, not per-value nulls).
114116
if *distinct
115117
|| filter.is_some()
116118
|| args.len() != 1
@@ -120,8 +122,14 @@ fn classify(expr: &Expr) -> Option<Coalesceable> {
120122
return None;
121123
}
122124

125+
// `None` and `Some(RespectNulls)` mean the same to the physical planner (and
126+
// `IgnoreNulls` bailed above), so normalize or those peers miss each other.
123127
Some(Coalesceable {
124-
key: (name.to_string(), order_by.clone(), *null_treatment),
128+
key: (
129+
name.to_string(),
130+
order_by.clone(),
131+
null_treatment.unwrap_or(NullTreatment::RespectNulls),
132+
),
125133
func: Arc::clone(func),
126134
value: args[0].clone(),
127135
})
@@ -149,9 +157,8 @@ impl OptimizerRule for CoalesceFirstLast {
149157
return Ok(Transformed::no(plan));
150158
};
151159

152-
// Grouping sets add an internal grouping-id column and distinct
153-
// group-by semantics; this rule declines to coalesce them (consistent
154-
// with the other cases it skips).
160+
// Grouping sets add an internal grouping-id column and their own
161+
// group-by semantics, so decline them like the other skipped cases.
155162
if matches!(aggregate.group_expr.as_slice(), [Expr::GroupingSet(_)]) {
156163
return Ok(Transformed::no(LogicalPlan::Aggregate(aggregate)));
157164
}
@@ -181,10 +188,8 @@ impl OptimizerRule for CoalesceFirstLast {
181188
return Ok(Transformed::no(LogicalPlan::Aggregate(aggregate)));
182189
};
183190

184-
// Use `group_expr_len()` (deduplicated count, matching how
185-
// `Aggregate::try_new` builds the schema) rather than `group_expr.len()`,
186-
// which over-counts duplicate group exprs (e.g. `GROUP BY p, p`) and would
187-
// misalign the group/aggregate split of `orig_columns` below.
191+
// Deduplicated count, matching how `Aggregate::try_new` builds the schema.
192+
// `group_expr.len()` over-counts `GROUP BY p, p` and misaligns the split.
188193
let group_len = aggregate.group_expr_len()?;
189194
let orig_columns = aggregate.schema.columns();
190195

@@ -204,14 +209,16 @@ impl OptimizerRule for CoalesceFirstLast {
204209
}
205210

206211
let first = classified[idxs[0]].as_ref().expect("non-empty bucket");
207-
let (_, order_by, null_treatment) = &first.key;
212+
let (_, order_by, _) = &first.key;
213+
// Only respect-nulls members reach here (`classify` bails on
214+
// IgnoreNulls), and that is the default, so leave it unspelled.
208215
let coalesced = Expr::AggregateFunction(AggregateFunction::new_udf(
209216
Arc::clone(&first.func),
210217
vec![named_struct.call(struct_args)],
211218
/* distinct */ false,
212219
/* filter */ None,
213220
order_by.clone(),
214-
*null_treatment,
221+
/* null_treatment */ None,
215222
))
216223
.alias(alias);
217224
coalesced_aggr_exprs.push(coalesced);
@@ -243,7 +250,10 @@ impl OptimizerRule for CoalesceFirstLast {
243250
if let Some((alias, pos)) = coalesced_at.get(&i) {
244251
let field =
245252
get_field.call(vec![col(alias.as_str()), lit(format!("c{pos}"))]);
246-
projection_exprs.push(field.alias(column.name().to_string()));
253+
// Carry the qualifier over too, or a qualified peer loses it and
254+
// the rule trips the optimizer's schema-invariant check.
255+
projection_exprs
256+
.push(field.alias_qualified(column.relation.clone(), column.name()));
247257
} else {
248258
projection_exprs.push(Expr::Column(column.clone()));
249259
}
@@ -274,8 +284,8 @@ mod tests {
274284
use datafusion_functions_aggregate::expr_fn::count;
275285
use datafusion_functions_aggregate::first_last::{first_value_udaf, last_value_udaf};
276286

277-
/// An [`OptimizerConfig`] that enables the rule and exposes a registry with
278-
/// `named_struct` / `get_field`, so the rewrite can be exercised in isolation.
287+
// An OptimizerConfig that enables the rule and exposes a registry with
288+
// named_struct / get_field, so the rewrite can be exercised in isolation.
279289
#[derive(Debug)]
280290
struct TestConfig {
281291
options: Arc<ConfigOptions>,
@@ -334,8 +344,7 @@ mod tests {
334344
}};
335345
}
336346

337-
/// Scan with a partition key `p`, two value columns `a` / `b`, and an
338-
/// ordering column `o`.
347+
// Scan with a partition key p, two value columns a / b, and an ordering column o.
339348
fn scan() -> Result<LogicalPlan> {
340349
let schema = Schema::new(vec![
341350
Field::new("p", DataType::UInt32, false),
@@ -361,6 +370,21 @@ mod tests {
361370
))
362371
}
363372

373+
fn first_value_with_null_treatment(
374+
arg: Expr,
375+
order_by: Vec<Sort>,
376+
null_treatment: Option<NullTreatment>,
377+
) -> Expr {
378+
Expr::AggregateFunction(AggregateFunction::new_udf(
379+
first_value_udaf(),
380+
vec![arg],
381+
false,
382+
None,
383+
order_by,
384+
null_treatment,
385+
))
386+
}
387+
364388
fn last_value(arg: Expr, order_by: Vec<Sort>) -> Expr {
365389
Expr::AggregateFunction(AggregateFunction::new_udf(
366390
last_value_udaf(),
@@ -531,10 +555,8 @@ mod tests {
531555

532556
#[test]
533557
fn coalesces_duplicate_group_by() -> Result<()> {
534-
// Regression: `GROUP BY p, p` stores two group exprs, but the schema
535-
// deduplicates them to one group column. The rule must split group vs
536-
// aggregate columns by the deduplicated count (`group_expr_len`), not
537-
// `group_expr.len()`, or the projection references a non-existent column.
558+
// Regression: `GROUP BY p, p` dedups to one schema column, so splitting
559+
// on `group_expr.len()` would point the projection at a missing column.
538560
let plan = LogicalPlanBuilder::from(scan()?)
539561
.aggregate(
540562
vec![col("p"), col("p")],
@@ -570,6 +592,93 @@ mod tests {
570592
"#)
571593
}
572594

595+
#[test]
596+
fn coalesces_qualified_aliased_peers() -> Result<()> {
597+
// Regression: the qualifier must survive the rewrite or the schema
598+
// invariant check rejects the plan (the alias prints bare either way).
599+
let plan = LogicalPlanBuilder::from(scan()?)
600+
.aggregate(
601+
vec![col("p")],
602+
vec![
603+
first_value(col("a"), order_by_o(), false)
604+
.alias_qualified(Some("t"), "fa"),
605+
first_value(col("b"), order_by_o(), false)
606+
.alias_qualified(Some("t"), "fb"),
607+
],
608+
)?
609+
.build()?;
610+
assert_coalesced!(TestConfig::new(), plan, @ r#"
611+
Projection: t.p, get_field(__coalesce_first_last_1, Utf8("c0")) AS fa, get_field(__coalesce_first_last_1, Utf8("c1")) AS fb
612+
Aggregate: groupBy=[[t.p]], aggr=[[first_value(named_struct(Utf8("c0"), t.a, Utf8("c1"), t.b)) ORDER BY [t.o DESC NULLS LAST] AS __coalesce_first_last_1]]
613+
TableScan: t
614+
"#)
615+
}
616+
617+
#[test]
618+
fn coalesces_global_aggregate() -> Result<()> {
619+
// No GROUP BY, so the group/aggregate split of the output columns is at 0.
620+
let plan = LogicalPlanBuilder::from(scan()?)
621+
.aggregate(
622+
Vec::<Expr>::new(),
623+
vec![
624+
first_value(col("a"), order_by_o(), false),
625+
first_value(col("b"), order_by_o(), false),
626+
],
627+
)?
628+
.build()?;
629+
assert_coalesced!(TestConfig::new(), plan, @ r#"
630+
Projection: get_field(__coalesce_first_last_1, Utf8("c0")) AS first_value(t.a) ORDER BY [t.o DESC NULLS LAST], get_field(__coalesce_first_last_1, Utf8("c1")) AS first_value(t.b) ORDER BY [t.o DESC NULLS LAST]
631+
Aggregate: groupBy=[[]], aggr=[[first_value(named_struct(Utf8("c0"), t.a, Utf8("c1"), t.b)) ORDER BY [t.o DESC NULLS LAST] AS __coalesce_first_last_1]]
632+
TableScan: t
633+
"#)
634+
}
635+
636+
#[test]
637+
fn coalesces_two_buckets() -> Result<()> {
638+
// Two ORDER BY keys => two buckets => one coalesced aggregate each.
639+
let by_b = vec![Sort::new(col("b"), true, true)];
640+
let plan = LogicalPlanBuilder::from(scan()?)
641+
.aggregate(
642+
vec![col("p")],
643+
vec![
644+
first_value(col("a"), order_by_o(), false),
645+
first_value(col("b"), order_by_o(), false),
646+
first_value(col("a"), by_b.clone(), false),
647+
first_value(col("o"), by_b, false),
648+
],
649+
)?
650+
.build()?;
651+
assert_coalesced!(TestConfig::new(), plan, @ r#"
652+
Projection: t.p, get_field(__coalesce_first_last_1, Utf8("c0")) AS first_value(t.a) ORDER BY [t.o DESC NULLS LAST], get_field(__coalesce_first_last_1, Utf8("c1")) AS first_value(t.b) ORDER BY [t.o DESC NULLS LAST], get_field(__coalesce_first_last_2, Utf8("c0")) AS first_value(t.a) ORDER BY [t.b ASC NULLS FIRST], get_field(__coalesce_first_last_2, Utf8("c1")) AS first_value(t.o) ORDER BY [t.b ASC NULLS FIRST]
653+
Aggregate: groupBy=[[t.p]], aggr=[[first_value(named_struct(Utf8("c0"), t.a, Utf8("c1"), t.b)) ORDER BY [t.o DESC NULLS LAST] AS __coalesce_first_last_1, first_value(named_struct(Utf8("c0"), t.a, Utf8("c1"), t.o)) ORDER BY [t.b ASC NULLS FIRST] AS __coalesce_first_last_2]]
654+
TableScan: t
655+
"#)
656+
}
657+
658+
#[test]
659+
fn coalesces_across_null_treatment_spellings() -> Result<()> {
660+
// An implicit null treatment and an explicit RESPECT NULLS mean the same
661+
// thing, so they must land in one bucket.
662+
let plan = LogicalPlanBuilder::from(scan()?)
663+
.aggregate(
664+
vec![col("p")],
665+
vec![
666+
first_value_with_null_treatment(col("a"), order_by_o(), None),
667+
first_value_with_null_treatment(
668+
col("b"),
669+
order_by_o(),
670+
Some(NullTreatment::RespectNulls),
671+
),
672+
],
673+
)?
674+
.build()?;
675+
assert_coalesced!(TestConfig::new(), plan, @ r#"
676+
Projection: t.p, get_field(__coalesce_first_last_1, Utf8("c0")) AS first_value(t.a) ORDER BY [t.o DESC NULLS LAST], get_field(__coalesce_first_last_1, Utf8("c1")) AS first_value(t.b) RESPECT NULLS ORDER BY [t.o DESC NULLS LAST]
677+
Aggregate: groupBy=[[t.p]], aggr=[[first_value(named_struct(Utf8("c0"), t.a, Utf8("c1"), t.b)) ORDER BY [t.o DESC NULLS LAST] AS __coalesce_first_last_1]]
678+
TableScan: t
679+
"#)
680+
}
681+
573682
#[test]
574683
fn coalesces_three_peers() -> Result<()> {
575684
// A bucket with three members packs into c0 / c1 / c2.

0 commit comments

Comments
 (0)