Skip to content

Commit 4935e8d

Browse files
committed
test: move the regr_* fallback check into a SQL file test
One query per function, each expecting the #6423 fallback reason, over the reproducer's data: x constant at 0.1 in two three-row files, which the native merge gets wrong. regr.sql still opts in and covers the native path.
1 parent 1f0ac9d commit 4935e8d

2 files changed

Lines changed: 57 additions & 33 deletions

File tree

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
-- Licensed to the Apache Software Foundation (ASF) under one
2+
-- or more contributor license agreements. See the NOTICE file
3+
-- distributed with this work for additional information
4+
-- regarding copyright ownership. The ASF licenses this file
5+
-- to you under the Apache License, Version 2.0 (the
6+
-- "License"); you may not use this file except in compliance
7+
-- with the License. You may obtain a copy of the License at
8+
--
9+
-- http://www.apache.org/licenses/LICENSE-2.0
10+
--
11+
-- Unless required by applicable law or agreed to in writing,
12+
-- software distributed under the License is distributed on an
13+
-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
-- KIND, either express or implied. See the License for the
15+
-- specific language governing permissions and limitations
16+
-- under the License.
17+
18+
-- regr_slope, regr_intercept, regr_r2, regr_sxx, regr_syy and regr_sxy fall back to Spark by
19+
-- default, because their native merge of partial aggregates orders its floating-point operations
20+
-- differently from Spark's (https://github.com/apache/datafusion-comet/issues/6423). Each query
21+
-- below holds a single regr function, so it checks that function's own fallback. regr.sql opts in
22+
-- and covers the native path.
23+
24+
-- The data from #6423: x is constant at 0.1, and each INSERT writes one file of three rows, so the
25+
-- rows are merged from two partial aggregates. The native merge leaves x with a tiny non-zero
26+
-- variance there, and returns wrong values where Spark returns NULL, 0.0 or 1.0.
27+
statement
28+
CREATE TABLE test_regr_fallback(y double, x double) USING parquet
29+
30+
statement
31+
INSERT INTO test_regr_fallback SELECT CAST(id AS DOUBLE), 0.1D FROM range(0, 3, 1, 1)
32+
33+
statement
34+
INSERT INTO test_regr_fallback SELECT CAST(id AS DOUBLE), 0.1D FROM range(3, 6, 1, 1)
35+
36+
query expect_fallback(issues/6423)
37+
SELECT regr_slope(y, x) FROM test_regr_fallback
38+
39+
query expect_fallback(issues/6423)
40+
SELECT regr_intercept(y, x) FROM test_regr_fallback
41+
42+
query expect_fallback(issues/6423)
43+
SELECT regr_r2(y, x) FROM test_regr_fallback
44+
45+
-- The constant as the dependent variable
46+
query expect_fallback(issues/6423)
47+
SELECT regr_r2(x, y) FROM test_regr_fallback
48+
49+
query expect_fallback(issues/6423)
50+
SELECT regr_sxx(y, x) FROM test_regr_fallback
51+
52+
query expect_fallback(issues/6423)
53+
SELECT regr_sxy(y, x) FROM test_regr_fallback
54+
55+
query expect_fallback(issues/6423)
56+
SELECT regr_syy(x, y) FROM test_regr_fallback

‎spark/src/test/scala/org/apache/comet/exec/CometAggregateSuite.scala‎

Lines changed: 1 addition & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ import org.apache.spark.{CometListenerBusUtils, SparkConf}
2828
import org.apache.spark.scheduler.{SparkListener, SparkListenerTaskEnd}
2929
import org.apache.spark.sql.{CometTestBase, DataFrame, Row}
3030
import org.apache.spark.sql.catalyst.expressions.Cast
31-
import org.apache.spark.sql.catalyst.expressions.aggregate.{Final, Partial, PartialMerge, RegrIntercept, RegrR2, RegrReplacement, RegrSlope, RegrSXY}
31+
import org.apache.spark.sql.catalyst.expressions.aggregate.{Final, Partial, PartialMerge}
3232
import org.apache.spark.sql.catalyst.optimizer.EliminateSorts
3333
import org.apache.spark.sql.catalyst.plans.physical.{HashPartitioning, RangePartitioning}
3434
import org.apache.spark.sql.comet.{CometFilterExec, CometHashAggregateExec, CometNativeExec, CometProjectExec}
@@ -1815,38 +1815,6 @@ class CometAggregateSuite extends CometTestBase with AdaptiveSparkPlanHelper {
18151815
}
18161816
}
18171817

1818-
test("regression aggregates fall back to Spark by default") {
1819-
// https://github.com/apache/datafusion-comet/issues/6423: the native merge of partial
1820-
// aggregates orders its floating-point operations differently from Spark. For a variable that
1821-
// is constant at 0.1 and merged from two partial aggregates, m2 ends up around 1e-34 instead
1822-
// of 0, Spark's degenerate-case checks never fire, and the native results are wrong. Until the
1823-
// merge matches Spark, every regr aggregate has to fall back unless the user opts in.
1824-
withTempPath { dir =>
1825-
val path = dir.getCanonicalPath
1826-
spark
1827-
.range(0, 6, 1, 2)
1828-
.selectExpr("CAST(id AS DOUBLE) AS y", "0.1D AS x")
1829-
.write
1830-
.parquet(path)
1831-
withParquetTable(path, "t") {
1832-
// The wrong results need rows merged from more than one partial aggregate, one per file.
1833-
assert(sql("SELECT * FROM t").rdd.getNumPartitions == 2)
1834-
val optInKeys = Seq[Class[_]](
1835-
classOf[RegrSlope],
1836-
classOf[RegrIntercept],
1837-
classOf[RegrR2],
1838-
classOf[RegrSXY],
1839-
// regr_sxx and regr_syy
1840-
classOf[RegrReplacement]).map(cls => CometConf.getExprAllowIncompatConfigKey(cls))
1841-
checkSparkAnswerAndFallbackReasons(
1842-
"SELECT regr_slope(y, x), regr_intercept(y, x), regr_r2(y, x), regr_sxx(y, x), " +
1843-
"regr_sxy(y, x), regr_r2(x, y), regr_syy(x, y) FROM t",
1844-
optInKeys.map(key => s"set $key=true").toSet +
1845-
"https://github.com/apache/datafusion-comet/issues/6423")
1846-
}
1847-
}
1848-
}
1849-
18501818
test("avg/sum overflow on decimal(38, _)") {
18511819
val table = "overflow_decimal_38"
18521820
withTable(table) {

0 commit comments

Comments
 (0)