Skip to content

Commit 61ec845

Browse files
authored
fix: reject DISTINCT in corr, covar and regr_* instead of ignoring it (apache#26484)
## Which issue does this PR close? - Closes apache#26471. ## Rationale for this change `corr`, `covar_pop`, `covar_samp` and the `regr_*` functions return the same result with `DISTINCT` as without it, because their accumulators never read `is_distinct`. ## What changes are included in this PR? Their accumulators now return a not implemented error for `DISTINCT`, as `stddev` and `approx_median` already do. `corr(DISTINCT x, x)` still works, since `SingleDistinctToGroupBy` rewrites it before it reaches the accumulator. ## What is the testing strategy for this PR? New cases in `aggregate.slt`. The two `REGR_COUNT(DISTINCT ...)` results in `eliminate_join_distinct.slt` were the non-distinct counts, so they now expect the error. ## Are there any user-facing changes? These queries now fail with an error instead of returning a wrong result. --------- Co-authored-by: Alejandro Borrego <[email protected]>
1 parent e6a144f commit 61ec845

5 files changed

Lines changed: 67 additions & 32 deletions

File tree

‎datafusion/functions-aggregate/src/correlation.rs‎

Lines changed: 10 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ use log::debug;
3838

3939
use crate::covariance::CovarianceAccumulator;
4040
use crate::stddev::StddevAccumulator;
41-
use datafusion_common::{Result, ScalarValue};
41+
use datafusion_common::{Result, ScalarValue, not_impl_err};
4242
use datafusion_expr::{
4343
Accumulator, AggregateUDFImpl, Documentation, Signature, Volatility,
4444
function::{AccumulatorArgs, StateFieldsArgs},
@@ -108,7 +108,10 @@ impl AggregateUDFImpl for Correlation {
108108
Ok(DataType::Float64)
109109
}
110110

111-
fn accumulator(&self, _acc_args: AccumulatorArgs) -> Result<Box<dyn Accumulator>> {
111+
fn accumulator(&self, acc_args: AccumulatorArgs) -> Result<Box<dyn Accumulator>> {
112+
if acc_args.is_distinct {
113+
return not_impl_err!("CORR(DISTINCT) aggregations are not available");
114+
}
112115
Ok(Box::new(CorrelationAccumulator::try_new()?))
113116
}
114117

@@ -139,8 +142,8 @@ impl AggregateUDFImpl for Correlation {
139142
self.doc()
140143
}
141144

142-
fn groups_accumulator_supported(&self, _args: AccumulatorArgs) -> bool {
143-
true
145+
fn groups_accumulator_supported(&self, args: AccumulatorArgs) -> bool {
146+
!args.is_distinct
144147
}
145148

146149
fn create_groups_accumulator(
@@ -152,9 +155,9 @@ impl AggregateUDFImpl for Correlation {
152155
}
153156

154157
fn distinct_handling(&self) -> DistinctHandling {
155-
// Duplicate-sensitive, but the accumulator does not read
156-
// `is_distinct` and today silently returns the non-distinct answer.
157-
// The tag records the intent; enforcement is a follow-up change.
158+
// The accumulator rejects `DISTINCT` with `not_impl_err!`. It only
159+
// works when the planner removes it first (`SingleDistinctToGroupBy`
160+
// for a single distinct field, e.g. `corr(DISTINCT x, x)`).
158161
DistinctHandling::Unsupported
159162
}
160163
}

‎datafusion/functions-aggregate/src/covariance.rs‎

Lines changed: 15 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@
2020
use arrow::array::ArrayRef;
2121
use arrow::datatypes::{DataType, Field, FieldRef};
2222
use datafusion_common::cast::{as_float64_array, as_uint64_array};
23-
use datafusion_common::{Result, ScalarValue};
23+
use datafusion_common::{Result, ScalarValue, not_impl_err};
2424
use datafusion_expr::DistinctHandling;
2525
use datafusion_expr::{
2626
Accumulator, AggregateUDFImpl, Documentation, Signature, Volatility,
@@ -118,7 +118,10 @@ impl AggregateUDFImpl for CovarianceSample {
118118
.collect())
119119
}
120120

121-
fn accumulator(&self, _acc_args: AccumulatorArgs) -> Result<Box<dyn Accumulator>> {
121+
fn accumulator(&self, acc_args: AccumulatorArgs) -> Result<Box<dyn Accumulator>> {
122+
if acc_args.is_distinct {
123+
return not_impl_err!("COVAR_SAMP(DISTINCT) aggregations are not available");
124+
}
122125
Ok(Box::new(CovarianceAccumulator::try_new(StatsType::Sample)?))
123126
}
124127

@@ -131,9 +134,9 @@ impl AggregateUDFImpl for CovarianceSample {
131134
}
132135

133136
fn distinct_handling(&self) -> DistinctHandling {
134-
// Duplicate-sensitive, but the accumulator does not read
135-
// `is_distinct` and today silently returns the non-distinct answer.
136-
// The tag records the intent; enforcement is a follow-up change.
137+
// The accumulator rejects `DISTINCT` with `not_impl_err!`. It only
138+
// works when the planner removes it first (`SingleDistinctToGroupBy`
139+
// for a single distinct field, e.g. `covar_samp(DISTINCT x, x)`).
137140
DistinctHandling::Unsupported
138141
}
139142
}
@@ -205,7 +208,10 @@ impl AggregateUDFImpl for CovariancePopulation {
205208
.collect())
206209
}
207210

208-
fn accumulator(&self, _acc_args: AccumulatorArgs) -> Result<Box<dyn Accumulator>> {
211+
fn accumulator(&self, acc_args: AccumulatorArgs) -> Result<Box<dyn Accumulator>> {
212+
if acc_args.is_distinct {
213+
return not_impl_err!("COVAR_POP(DISTINCT) aggregations are not available");
214+
}
209215
Ok(Box::new(CovarianceAccumulator::try_new(
210216
StatsType::Population,
211217
)?))
@@ -216,9 +222,9 @@ impl AggregateUDFImpl for CovariancePopulation {
216222
}
217223

218224
fn distinct_handling(&self) -> DistinctHandling {
219-
// Duplicate-sensitive, but the accumulator does not read
220-
// `is_distinct` and today silently returns the non-distinct answer.
221-
// The tag records the intent; enforcement is a follow-up change.
225+
// The accumulator rejects `DISTINCT` with `not_impl_err!`. It only
226+
// works when the planner removes it first (`SingleDistinctToGroupBy`
227+
// for a single distinct field, e.g. `covar_pop(DISTINCT x, x)`).
222228
DistinctHandling::Unsupported
223229
}
224230
}

‎datafusion/functions-aggregate/src/regr.rs‎

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@
2020
use arrow::datatypes::FieldRef;
2121
use arrow::{array::ArrayRef, datatypes::DataType, datatypes::Field};
2222
use datafusion_common::cast::{as_float64_array, as_uint64_array};
23-
use datafusion_common::{HashMap, Result, ScalarValue};
23+
use datafusion_common::{HashMap, Result, ScalarValue, not_impl_err};
2424
use datafusion_doc::aggregate_doc_sections::DOC_SECTION_STATISTICAL;
2525
use datafusion_expr::DistinctHandling;
2626
use datafusion_expr::function::{AccumulatorArgs, StateFieldsArgs};
@@ -469,7 +469,13 @@ impl AggregateUDFImpl for Regr {
469469
self.regr_type != RegrType::Count
470470
}
471471

472-
fn accumulator(&self, _acc_args: AccumulatorArgs) -> Result<Box<dyn Accumulator>> {
472+
fn accumulator(&self, acc_args: AccumulatorArgs) -> Result<Box<dyn Accumulator>> {
473+
if acc_args.is_distinct {
474+
return not_impl_err!(
475+
"{}(DISTINCT) aggregations are not available",
476+
self.func_name.to_uppercase()
477+
);
478+
}
473479
Ok(Box::new(RegrAccumulator::try_new(&self.regr_type)?))
474480
}
475481

@@ -516,9 +522,9 @@ impl AggregateUDFImpl for Regr {
516522
}
517523

518524
fn distinct_handling(&self) -> DistinctHandling {
519-
// Duplicate-sensitive, but the accumulator does not read
520-
// `is_distinct` and today silently returns the non-distinct answer.
521-
// The tag records the intent; enforcement is a follow-up change.
525+
// The accumulator rejects `DISTINCT` with `not_impl_err!`. It only
526+
// works when the planner removes it first (`SingleDistinctToGroupBy`
527+
// for a single distinct field, e.g. `regr_slope(DISTINCT x, x)`).
522528
DistinctHandling::Unsupported
523529
}
524530
}

‎datafusion/sqllogictest/test_files/aggregate.slt‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -432,6 +432,31 @@ GROUP BY g
432432
3 NULL
433433
4 NULL
434434

435+
# With two different arguments there is no single field to group by, so
436+
# DISTINCT reaches the accumulator, which does not implement it.
437+
statement error DataFusion error: This feature is not implemented: CORR\(DISTINCT\) aggregations are not available
438+
SELECT corr(DISTINCT x, y) FROM (VALUES (1.0, 1.0), (1.0, 1.0), (2.0, 3.0)) t(x, y)
439+
440+
statement error DataFusion error: This feature is not implemented: CORR\(DISTINCT\) aggregations are not available
441+
SELECT g, corr(DISTINCT x, y) FROM (VALUES (1, 1.0, 1.0), (1, 1.0, 1.0), (1, 2.0, 3.0)) t(g, x, y) GROUP BY g
442+
443+
statement error DataFusion error: This feature is not implemented: COVAR_SAMP\(DISTINCT\) aggregations are not available
444+
SELECT covar_samp(DISTINCT x, y) FROM (VALUES (1.0, 1.0), (1.0, 1.0), (2.0, 3.0)) t(x, y)
445+
446+
statement error DataFusion error: This feature is not implemented: COVAR_POP\(DISTINCT\) aggregations are not available
447+
SELECT covar_pop(x, y), covar_pop(DISTINCT x, y) FROM (VALUES (1.0, 1.0), (1.0, 1.0), (2.0, 3.0)) t(x, y)
448+
449+
statement error DataFusion error: This feature is not implemented: REGR_COUNT\(DISTINCT\) aggregations are not available
450+
SELECT regr_count(x, y), regr_count(DISTINCT x, y) FROM (VALUES (1.0, 1.0), (1.0, 1.0), (2.0, 3.0)) t(x, y)
451+
452+
# Window functions build the accumulator through `create_sliding_accumulator`,
453+
# which must reject DISTINCT too.
454+
statement error DataFusion error: This feature is not implemented: CORR\(DISTINCT\) aggregations are not available
455+
SELECT corr(DISTINCT x, y) OVER () FROM (VALUES (1.0, 1.0), (1.0, 1.0), (2.0, 3.0)) t(x, y)
456+
457+
statement error DataFusion error: This feature is not implemented: REGR_SLOPE\(DISTINCT\) aggregations are not available
458+
SELECT regr_slope(DISTINCT x, y) OVER () FROM (VALUES (1.0, 1.0), (1.0, 1.0), (2.0, 3.0)) t(x, y)
459+
435460
# all_nulls_query_correlation
436461
query R
437462
with data as (

‎datafusion/sqllogictest/test_files/eliminate_join_distinct.slt‎

Lines changed: 6 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -187,11 +187,10 @@ SELECT COUNT(DISTINCT l.x), COUNT(l.g) FROM l JOIN r ON l.id = r.id;
187187
----
188188
2 7
189189

190-
# REGR_COUNT declares `DistinctHandling::Unsupported`: its accumulator ignores
191-
# `is_distinct` and counts every joined row, so the results below are the
192-
# non-distinct counts (4 and 6; the distinct counts would be 2 and 4). DISTINCT
193-
# therefore does not hide the join fanout: the inner join must stay an inner
194-
# join, and the left join must be kept.
190+
# REGR_COUNT declares `DistinctHandling::Unsupported`: its accumulator rejects
191+
# `is_distinct`, so the queries below fail instead of returning the non-distinct
192+
# counts. DISTINCT therefore does not hide the join fanout: the inner join must
193+
# stay an inner join, and the left join must be kept.
195194
query TT
196195
EXPLAIN SELECT REGR_COUNT(DISTINCT l.x, l.g) FROM l JOIN r ON l.id = r.id;
197196
----
@@ -202,15 +201,11 @@ logical_plan
202201
04)------TableScan: l projection=[id, g, x]
203202
05)------TableScan: r projection=[id]
204203

205-
query I
204+
statement error DataFusion error: This feature is not implemented: REGR_COUNT\(DISTINCT\) aggregations are not available
206205
SELECT REGR_COUNT(DISTINCT l.x, l.g) FROM l JOIN r ON l.id = r.id;
207-
----
208-
4
209206

210-
query I
207+
statement error DataFusion error: This feature is not implemented: REGR_COUNT\(DISTINCT\) aggregations are not available
211208
SELECT REGR_COUNT(DISTINCT l.x, l.g) FROM l LEFT JOIN r ON l.id = r.id;
212-
----
213-
6
214209

215210
# A limit below the aggregate still observes repeated join rows.
216211
query I

0 commit comments

Comments
 (0)