Skip to content

Commit 3752f31

Browse files
committed
feat: run min and max over string and binary columns natively
Spark plans SortAggregateExec for min and max over strings and binary, because their buffers are not fixed-width. Comet runs that operator natively since #4565, but CometMin and CometMax still declined these types, so the whole aggregate fell back. Accept them in AggSerde.minMaxSupportLevel. Spark and DataFusion's MinMaxBytesAccumulator both compare raw bytes. A string with a non-default collation still falls back, because Spark orders it by its collation.
1 parent 42db91b commit 3752f31

6 files changed

Lines changed: 125 additions & 20 deletions

File tree

‎docs/source/user-guide/latest/compatibility/operators.md‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -113,9 +113,11 @@ incorrect result. When any single window expression in a `WindowExec` falls back
113113
(`stddev`, `stddev_pop`, `stddev_samp`, `var_pop`, `var_samp`, `corr`, `covar_pop`, `covar_samp`). These run
114114
natively as plain aggregations but not as window functions
115115
([#4766](https://github.com/apache/datafusion-comet/issues/4766)).
116-
- `min` / `max` on string, binary, timestamp-without-time-zone, interval, or nested (array / struct) input types,
117-
and `sum` / `avg` on year-month or day-time interval input types. Windowed aggregates inherit the same input-type
118-
support as the batch aggregates, so these fall back in both contexts.
116+
- `min` / `max` on timestamp-without-time-zone, interval, or nested (array / struct) input types, and `sum` /
117+
`avg` on year-month or day-time interval input types. Windowed aggregates inherit the same input-type support as
118+
the batch aggregates, so these fall back in both contexts.
119+
- `min` / `max` on string or binary input types. These run natively as plain aggregations but not as window
120+
functions.
119121
- `sum` or `avg` on `DECIMAL` with a sliding (non ever-expanding) frame, because the sliding path would wrap on
120122
overflow instead of returning Spark's `NULL`.
121123
- `RANGE` frame with an explicit offset when the `ORDER BY` column is `DATE` or `DECIMAL`

‎spark/src/main/scala/org/apache/comet/serde/aggregates.scala‎

Lines changed: 22 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ import org.apache.spark.sql.types.{ArrayType, BinaryType, BooleanType, ByteType,
3131
import org.apache.comet.CometConf.COMET_EXEC_STRICT_FLOATING_POINT
3232
import org.apache.comet.CometSparkSessionExtensions.{isSpark41Plus, isSpark42Plus, withFallbackReason}
3333
import org.apache.comet.expressions.CometEvalMode
34-
import org.apache.comet.serde.QueryPlanSerde.{evalModeToProto, exprToProto, serializeDataType}
34+
import org.apache.comet.serde.QueryPlanSerde.{evalModeToProto, exprToProto, isStringCollationType, serializeDataType}
3535
import org.apache.comet.shims.{CometCollectShim, CometEvalModeUtil, CometTypeShim}
3636

3737
object CometMin extends CometAggregateExpressionSerde[Min] {
@@ -41,6 +41,8 @@ object CometMin extends CometAggregateExpressionSerde[Min] {
4141
// Native MIN emits one typed null for empty/all-null input; Spark's least merge ignores it.
4242
override def supportsNativePartialToSparkFinal(fn: Min): Boolean = true
4343

44+
override def getUnsupportedReasons(): Seq[String] = AggSerde.minMaxUnsupportedReasons
45+
4446
override def getSupportLevel(expr: Min): SupportLevel =
4547
AggSerde.minMaxSupportLevel(expr.dataType)
4648

@@ -80,6 +82,8 @@ object CometMax extends CometAggregateExpressionSerde[Max] {
8082
// Native MAX emits one typed null for empty/all-null input; Spark's greatest merge ignores it.
8183
override def supportsNativePartialToSparkFinal(fn: Max): Boolean = true
8284

85+
override def getUnsupportedReasons(): Seq[String] = AggSerde.minMaxUnsupportedReasons
86+
8387
override def getSupportLevel(expr: Max): SupportLevel =
8488
AggSerde.minMaxSupportLevel(expr.dataType)
8589

@@ -1475,14 +1479,24 @@ object AggSerde {
14751479

14761480
/**
14771481
* Shared support level for `Min` / `Max` based on the result data type. Floats follow Spark's
1478-
* ordering natively, so strict floating-point mode does not apply to them.
1482+
* ordering natively, so strict floating-point mode does not apply to them. Spark and the native
1483+
* accumulators both compare strings and binary as unsigned bytes, except that Spark orders a
1484+
* string with a non-default collation by that collation.
14791485
*/
1480-
def minMaxSupportLevel(dt: DataType): SupportLevel = {
1481-
if (!minMaxDataTypeSupported(dt)) {
1482-
Unsupported(Some(s"Unsupported data type: $dt"))
1483-
} else {
1484-
Compatible()
1485-
}
1486+
def minMaxSupportLevel(dt: DataType): SupportLevel = dt match {
1487+
case _: StringType if isStringCollationType(dt) =>
1488+
Unsupported(Some(ComparisonUtils.nonDefaultCollationReason("min/max")))
1489+
// DataFusion's grouped hash aggregate under-reports the memory these values hold (#6936).
1490+
case _: StringType | BinaryType => Compatible()
1491+
case _ if minMaxDataTypeSupported(dt) => Compatible()
1492+
case _ => Unsupported(Some(s"Unsupported data type: $dt"))
14861493
}
14871494

1495+
/** The inputs that `minMaxSupportLevel` rejects, for the generated compatibility guide. */
1496+
val minMaxUnsupportedReasons: Seq[String] = Seq(
1497+
"Input of a type other than boolean, integral, floating-point, decimal, date, timestamp," +
1498+
" string or binary, such as timestamp without time zone, intervals, arrays and structs.",
1499+
"A string with a non-default collation (Spark 4.0+), which Spark orders by its collation" +
1500+
" while Comet compares raw bytes.")
1501+
14881502
}

‎spark/src/test/resources/sql-tests/expressions/aggregate/min_max.sql‎

Lines changed: 50 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -15,17 +15,63 @@
1515
-- specific language governing permissions and limitations
1616
-- under the License.
1717

18+
-- Run with and without Parquet dictionary encoding, which changes how string and binary columns
19+
-- are read.
20+
-- ConfigMatrix: parquet.enable.dictionary=false,true
21+
1822
statement
1923
CREATE TABLE test_min_max(i int, d double, s string, grp string) USING parquet
2024

2125
statement
2226
INSERT INTO test_min_max VALUES (1, 1.5, 'b', 'x'), (3, 3.5, 'a', 'x'), (2, 2.5, 'c', 'y'), (NULL, NULL, NULL, 'y'), (-1, -1.5, 'z', 'x')
2327

24-
-- min/max over StringType is not yet supported natively, so this falls back to Spark.
25-
-- (Spark plans this global aggregate as SortAggregateExec because the string min/max buffer
26-
-- is not hash-friendly; that operator is now supported, but the string aggregate itself is not.)
27-
query expect_fallback(Unsupported data type: StringType)
28+
-- Spark plans this as SortAggregateExec, because a string min/max buffer is not hash-friendly.
29+
query
2830
SELECT min(i), max(i), min(d), max(d), min(s), max(s) FROM test_min_max
2931

3032
query
3133
SELECT grp, min(i), max(i) FROM test_min_max GROUP BY grp ORDER BY grp
34+
35+
-- Spark compares strings and binary as unsigned bytes, and a value sorts before any longer value
36+
-- it is a prefix of. The groups cover prefixes, case, the empty value, characters whose UTF-8
37+
-- order differs from their UTF-16 order ('😀' sorts after 'ア' by bytes but before it by UTF-16
38+
-- code units), bytes that a signed comparison would order the other way (X'80' after X'7F'),
39+
-- and a group of only NULLs.
40+
statement
41+
CREATE TABLE test_min_max_bytes(grp string, s string, b binary) USING parquet
42+
43+
statement
44+
INSERT INTO test_min_max_bytes VALUES
45+
('prefix', 'ab', X'0102'), ('prefix', 'a', X'01'), ('prefix', NULL, NULL),
46+
('case', 'a', X'61'), ('case', 'B', X'42'), ('case', 'A', X'41'),
47+
('empty', ' ', X'20'), ('empty', '', X''), ('empty', NULL, NULL),
48+
('utf8', 'ア', X'7F'), ('utf8', '😀', X'80'), ('utf8', 'é', X'FF'), ('utf8', 'z', X'00'),
49+
('nulls', NULL, NULL), ('nulls', NULL, NULL)
50+
51+
query
52+
SELECT grp, min(s), max(s), min(b), max(b) FROM test_min_max_bytes GROUP BY grp ORDER BY grp
53+
54+
query
55+
SELECT min(s), max(s), min(b), max(b) FROM test_min_max_bytes
56+
57+
query
58+
SELECT min(s), max(s), min(b), max(b) FROM test_min_max_bytes WHERE grp = 'missing'
59+
60+
-- collect_set makes Spark plan ObjectHashAggregateExec instead, with min/max in the same operator.
61+
query
62+
SELECT grp, min(s), max(b), sort_array(collect_set(s))
63+
FROM test_min_max_bytes GROUP BY grp ORDER BY grp
64+
65+
-- count(DISTINCT) makes Spark plan four sort aggregates, so the string and binary buffers also go
66+
-- through a PartialMerge.
67+
query
68+
SELECT grp, count(DISTINCT s), min(s), max(b)
69+
FROM test_min_max_bytes GROUP BY grp ORDER BY grp
70+
71+
-- A VALUES input keeps the partial aggregate on Spark, while the final one merges Spark's string
72+
-- and binary buffers natively. The plan is not fully native, so only the answer is checked.
73+
query spark_answer_only
74+
SELECT k, min(s), max(s), min(b), max(b)
75+
FROM VALUES (1, 'ab', X'0102'), (1, 'a', X'01'), (2, NULL, NULL), (2, '😀', X'80'), (2, 'ア', X'7F')
76+
AS t(k, s, b)
77+
GROUP BY k ORDER BY k
Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
-- Licensed to the Apache Software Foundation (ASF) under one
2+
-- or more contributor license agreements. See the NOTICE file
3+
-- distributed with this work for additional information
4+
-- regarding copyright ownership. The ASF licenses this file
5+
-- to you under the Apache License, Version 2.0 (the
6+
-- "License"); you may not use this file except in compliance
7+
-- with the License. You may obtain a copy of the License at
8+
--
9+
-- http://www.apache.org/licenses/LICENSE-2.0
10+
--
11+
-- Unless required by applicable law or agreed to in writing,
12+
-- software distributed under the License is distributed on an
13+
-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
-- KIND, either express or implied. See the License for the
15+
-- specific language governing permissions and limitations
16+
-- under the License.
17+
18+
-- Spark orders a string with a non-default collation by that collation, while Comet's native
19+
-- min/max compare raw bytes, so min/max over a collated string must fall back to Spark.
20+
-- Collation syntax requires Spark 4.0+.
21+
22+
-- MinSparkVersion: 4.0
23+
24+
statement
25+
CREATE TABLE test_min_max_collation(grp string, s string) USING parquet
26+
27+
statement
28+
INSERT INTO test_min_max_collation VALUES ('x', 'a'), ('x', 'B'), ('x', NULL), ('y', 'A'), ('y', 'b')
29+
30+
-- Under UTF8_LCASE, group x has min 'a' and max 'B', the reverse of their byte order.
31+
query expect_fallback(min/max does not support non-UTF8_BINARY collated operands)
32+
SELECT grp, min(CAST(s AS STRING COLLATE UTF8_LCASE)), max(CAST(s AS STRING COLLATE UTF8_LCASE))
33+
FROM test_min_max_collation GROUP BY grp ORDER BY grp

‎spark/src/test/scala/org/apache/comet/exec/CometShuffleInputConversionSuite.scala‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -236,10 +236,11 @@ class CometShuffleInputConversionSuite extends CometTestBase {
236236
assert(convertedShuffles(split).length == 1, split)
237237
assert(collect(split) { case a: HashAggregateExec => a }.length == 1, split)
238238
assert(collect(split) { case a: CometHashAggregateExec => a }.length == 1, split)
239-
// The min of a string makes Spark plan sort aggregates, which stay on Spark, so the final
240-
// aggregate reads the buffers back as rows. The shuffle stays native: the final aggregate
241-
// reads it through a sort, and the revert in the next test only takes a shuffle that an
242-
// aggregate reads directly.
239+
// The min of a string makes Spark plan sort aggregates. Both stay on Spark: the partial one
240+
// reads Spark rows, and count and the decimal sum cannot run a native final over a Spark
241+
// partial. So the final aggregate reads the buffers back as rows. The shuffle stays native:
242+
// the final aggregate reads it through a sort, and the revert in the next test only takes a
243+
// shuffle that an aggregate reads directly.
243244
val (_, sorted) = checkSparkAnswer(
244245
rowsDf()
245246
.groupBy((col("k") % 7).as("g"))

‎spark/src/test/scala/org/apache/spark/sql/benchmark/CometAggregateExpressionBenchmark.scala‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,12 @@ object CometAggregateExpressionBenchmark extends CometBenchmarkBase {
4141
"SELECT COUNT(DISTINCT c_int) FROM parquetV1Table GROUP BY grp"),
4242
AggExprConfig("min_int", "SELECT MIN(c_int) FROM parquetV1Table GROUP BY grp"),
4343
AggExprConfig("max_int", "SELECT MAX(c_int) FROM parquetV1Table GROUP BY grp"),
44+
AggExprConfig(
45+
"min_string",
46+
"SELECT MIN(CAST(c_int AS STRING)) FROM parquetV1Table GROUP BY grp"),
47+
AggExprConfig(
48+
"max_string",
49+
"SELECT MAX(CAST(c_int AS STRING)) FROM parquetV1Table GROUP BY grp"),
4450
AggExprConfig("min_double", "SELECT MIN(c_double) FROM parquetV1Table GROUP BY grp"),
4551
AggExprConfig("max_double", "SELECT MAX(c_double) FROM parquetV1Table GROUP BY grp"),
4652
AggExprConfig("sum_int", "SELECT SUM(c_int) FROM parquetV1Table GROUP BY grp"),
@@ -101,7 +107,10 @@ object CometAggregateExpressionBenchmark extends CometBenchmarkBase {
101107
"SELECT SUM(c_int) FROM parquetV1Table GROUP BY high_card_grp"),
102108
AggExprConfig(
103109
"count_distinct_high_card",
104-
"SELECT COUNT(DISTINCT c_int) FROM parquetV1Table GROUP BY high_card_grp"))
110+
"SELECT COUNT(DISTINCT c_int) FROM parquetV1Table GROUP BY high_card_grp"),
111+
AggExprConfig(
112+
"min_string_high_card",
113+
"SELECT MIN(CAST(c_int AS STRING)) FROM parquetV1Table GROUP BY high_card_grp"))
105114

106115
// Exact percentile. Only the single-percentage, default-frequency, numeric-input form runs
107116
// natively through Comet's Spark-compatible percentile UDAF; other forms fall back to Spark.

0 commit comments

Comments
 (0)