Skip to content

Commit 7fd7568

Browse files
committed
test: run the collect_set fixtures through SortAggregateExec
collect_set.sql and collect_set_floating_fallback.sql gain a ConfigMatrix over useObjectHashAggregateExec, so their whole fixtures run through both ObjectHashAggregateExec and SortAggregateExec. sort_aggregate.sql keeps only the shapes those files do not cover (several grouping keys, an expression key, hashable aggregates riding along) and adds the decimal SUM fallback. Its floating-point fallback query, which expected a fallback that Spark 4.2 no longer has, moves to the matrix. A new fixture checks that SortAggregateExec stays in Spark, with the shuffle reason, when Comet shuffle is disabled. min_by.sql and max_by.sql drop the claim that Comet never converts SortAggregate and check the type fallback without the TypedImperativeAggregate too. The two native collect_set Scala tests become one test over both queries.
1 parent 1c16c7d commit 7fd7568

7 files changed

Lines changed: 77 additions & 172 deletions

File tree

‎spark/src/test/resources/sql-tests/expressions/aggregate/collect_set.sql‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,9 @@
1616
-- under the License.
1717

1818
-- ConfigMatrix: parquet.enable.dictionary=false,true
19+
-- Disabling ObjectHashAggregate makes Spark plan SortAggregateExec for collect_set instead, so the
20+
-- matrix runs the whole fixture through both aggregate operators.
21+
-- ConfigMatrix: spark.sql.execution.useObjectHashAggregateExec=true,false
1922

2023
-- ============================================================
2124
-- Setup: tables

‎spark/src/test/resources/sql-tests/expressions/aggregate/collect_set_floating_fallback.sql‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,9 @@
1717

1818
-- MaxSparkVersion: 4.1
1919
-- Config: spark.comet.exec.strictFloatingPoint=true
20+
-- Disabling ObjectHashAggregate makes Spark plan SortAggregateExec for collect_set instead, so the
21+
-- matrix checks that both aggregate operators fall back.
22+
-- ConfigMatrix: spark.sql.execution.useObjectHashAggregateExec=true,false
2023

2124
statement
2225
CREATE TABLE cs_fallback_float(v float, grp string) USING parquet

‎spark/src/test/resources/sql-tests/expressions/aggregate/max_by.sql‎

Lines changed: 13 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -17,9 +17,8 @@
1717

1818
-- max_by(x, y) returns the value of x associated with the maximum value of y.
1919
--
20-
-- The value (x) must be a fixed-length type: Spark only uses HashAggregate (the aggregate
21-
-- operator Comet accelerates) when the aggregation buffer is mutable, so variable-length
22-
-- value types such as string force SortAggregate and fall back to Spark.
20+
-- The value (x) must be a fixed-length type. A variable-length value type such as string makes
21+
-- Spark plan SortAggregate rather than HashAggregate, and Comet falls back for it.
2322
--
2423
-- Ordering values are kept unique within each group so results are deterministic (max_by is
2524
-- non-deterministic when several rows tie on the maximum ordering).
@@ -262,12 +261,11 @@ SELECT grp, max_by(v, ord) FROM mb_signed_zero GROUP BY grp ORDER BY grp
262261
-- ============================================================
263262
-- Variable-length value or ordering falls back to Spark
264263
--
265-
-- A plain max_by over a string is planned as SortAggregate, which Comet never converts, so the
266-
-- serde's type check is not what stops it. Pairing it with a TypedImperativeAggregate switches
267-
-- Spark to ObjectHashAggregate, which Comet does convert, and then getSupportLevel is the only
268-
-- thing that keeps the aggregate off the native path. That matters most for a string *ordering*:
269-
-- Arrow's row format compares raw UTF-8 bytes, while Spark compares collation sort keys, so this
270-
-- check is load-bearing for correctness and not just an optimisation.
264+
-- A plain max_by over a string is planned as SortAggregate, and pairing it with a
265+
-- TypedImperativeAggregate switches Spark to ObjectHashAggregate. Comet converts both, so
266+
-- getSupportLevel is the only thing that keeps the aggregate off the native path. That matters
267+
-- most for a string *ordering*: Arrow's row format compares raw UTF-8 bytes, while Spark compares
268+
-- collation sort keys, so this check is load-bearing for correctness and not just an optimisation.
271269
-- ============================================================
272270

273271
statement
@@ -282,3 +280,9 @@ SELECT grp, max_by(s, v), percentile(v, 0.5) FROM mb_varlen GROUP BY grp ORDER B
282280

283281
query expect_fallback(Unsupported ordering data type)
284282
SELECT grp, max_by(v, s), percentile(v, 0.5) FROM mb_varlen GROUP BY grp ORDER BY grp
283+
284+
query expect_fallback(Unsupported value data type)
285+
SELECT grp, max_by(s, v) FROM mb_varlen GROUP BY grp ORDER BY grp
286+
287+
query expect_fallback(Unsupported ordering data type)
288+
SELECT grp, max_by(v, s) FROM mb_varlen GROUP BY grp ORDER BY grp

‎spark/src/test/resources/sql-tests/expressions/aggregate/min_by.sql‎

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -17,9 +17,8 @@
1717

1818
-- min_by(x, y) returns the value of x associated with the minimum value of y.
1919
--
20-
-- The value (x) must be a fixed-length type: Spark only uses HashAggregate (the aggregate
21-
-- operator Comet accelerates) when the aggregation buffer is mutable, so variable-length
22-
-- value types such as string force SortAggregate and fall back to Spark.
20+
-- The value (x) must be a fixed-length type. A variable-length value type such as string makes
21+
-- Spark plan SortAggregate rather than HashAggregate, and Comet falls back for it.
2322
--
2423
-- Ordering values are kept unique within each group so results are deterministic (min_by is
2524
-- non-deterministic when several rows tie on the minimum ordering).
@@ -247,9 +246,8 @@ SELECT grp, min_by(v, ord) FROM mnb_signed_zero GROUP BY grp ORDER BY grp
247246
-- ============================================================
248247
-- Variable-length value or ordering falls back to Spark
249248
--
250-
-- See the equivalent section in max_by.sql for why this needs a TypedImperativeAggregate
251-
-- alongside it: without one, Spark plans SortAggregate and Comet never sees the aggregate at all,
252-
-- so the serde's type check would go untested.
249+
-- See the equivalent section in max_by.sql: with or without a TypedImperativeAggregate
250+
-- alongside it, the serde's type check is what keeps the aggregate off the native path.
253251
-- ============================================================
254252

255253
statement
@@ -264,3 +262,9 @@ SELECT grp, min_by(s, v), percentile(v, 0.5) FROM mnb_varlen GROUP BY grp ORDER
264262

265263
query expect_fallback(Unsupported ordering data type)
266264
SELECT grp, min_by(v, s), percentile(v, 0.5) FROM mnb_varlen GROUP BY grp ORDER BY grp
265+
266+
query expect_fallback(Unsupported value data type)
267+
SELECT grp, min_by(s, v) FROM mnb_varlen GROUP BY grp ORDER BY grp
268+
269+
query expect_fallback(Unsupported ordering data type)
270+
SELECT grp, min_by(v, s) FROM mnb_varlen GROUP BY grp ORDER BY grp

‎spark/src/test/resources/sql-tests/expressions/aggregate/sort_aggregate.sql‎

Lines changed: 10 additions & 145 deletions
Original file line numberDiff line numberDiff line change
@@ -16,18 +16,10 @@
1616
-- under the License.
1717

1818
-- Disabling ObjectHashAggregate forces Spark to plan SortAggregateExec for the
19-
-- TypedImperativeAggregate function collect_set, which is the operator this file exercises.
20-
-- (The plan-shape assertion that a SortAggregateExec is actually produced lives in
21-
-- CometAggregateSuite; here the default `query` mode asserts the whole plan, including the
22-
-- pre-aggregate sorts and the Comet shuffle, runs natively and matches Spark.)
23-
-- collect_set returns elements in nondeterministic order, so every result is wrapped in
24-
-- sort_array for a stable comparison against Spark.
19+
-- TypedImperativeAggregate function collect_set. collect_set.sql runs its whole fixture through
20+
-- SortAggregateExec as well, so this file keeps only the shapes it does not cover. collect_set
21+
-- returns elements in nondeterministic order, so every result is wrapped in sort_array.
2522
-- Config: spark.sql.execution.useObjectHashAggregateExec=false
26-
-- Config: spark.comet.exec.strictFloatingPoint=true
27-
28-
-- ============================================================
29-
-- Setup: tables
30-
-- ============================================================
3123

3224
statement
3325
CREATE TABLE sa_int(i int, g string) USING parquet
@@ -46,35 +38,6 @@ INSERT INTO sa_multikey VALUES
4638
(1, 'x', 10), (2, 'x', 10), (1, 'x', 10),
4739
(3, 'x', 20), (4, 'y', 10), (NULL, 'y', 10)
4840

49-
statement
50-
CREATE TABLE sa_nulls(v int, g string) USING parquet
51-
52-
statement
53-
INSERT INTO sa_nulls VALUES (NULL, 'a'), (NULL, 'a'), (NULL, 'b'), (1, 'b')
54-
55-
statement
56-
CREATE TABLE sa_empty(v int, g string) USING parquet
57-
58-
statement
59-
CREATE TABLE sa_single(v int) USING parquet
60-
61-
statement
62-
INSERT INTO sa_single VALUES (42)
63-
64-
-- ============================================================
65-
-- Global aggregate (no GROUP BY): single SortAggregate group
66-
-- ============================================================
67-
68-
query
69-
SELECT sort_array(collect_set(i)) FROM sa_int
70-
71-
-- ============================================================
72-
-- Single grouping key
73-
-- ============================================================
74-
75-
query
76-
SELECT g, sort_array(collect_set(i)) FROM sa_int GROUP BY g ORDER BY g
77-
7841
-- ============================================================
7942
-- Multiple grouping keys (sort ordering spans both keys)
8043
-- ============================================================
@@ -91,30 +54,6 @@ query
9154
SELECT upper(g) AS gg, sort_array(collect_set(i))
9255
FROM sa_int GROUP BY upper(g) ORDER BY gg
9356

94-
-- ============================================================
95-
-- All-NULL group returns an empty array
96-
-- ============================================================
97-
98-
query
99-
SELECT g, sort_array(collect_set(v)) FROM sa_nulls GROUP BY g ORDER BY g
100-
101-
-- ============================================================
102-
-- Empty table
103-
-- ============================================================
104-
105-
query
106-
SELECT sort_array(collect_set(v)) FROM sa_empty
107-
108-
query
109-
SELECT g, sort_array(collect_set(v)) FROM sa_empty GROUP BY g ORDER BY g
110-
111-
-- ============================================================
112-
-- Single row
113-
-- ============================================================
114-
115-
query
116-
SELECT sort_array(collect_set(v)) FROM sa_single
117-
11857
-- ============================================================
11958
-- collect_set mixed with hashable aggregates: collect_set forces the
12059
-- whole aggregate onto SortAggregate, so count/sum/min/max ride along
@@ -125,90 +64,16 @@ SELECT g, sort_array(collect_set(i)), count(*), count(i), sum(i), min(i), max(i)
12564
FROM sa_int GROUP BY g ORDER BY g
12665

12766
-- ============================================================
128-
-- Multiple collect_set in one query
129-
-- ============================================================
130-
131-
query
132-
SELECT k1, sort_array(collect_set(a)), sort_array(collect_set(k2))
133-
FROM sa_multikey GROUP BY k1 ORDER BY k1
134-
135-
-- ============================================================
136-
-- DISTINCT (distinct-aggregate planner path)
137-
-- ============================================================
138-
139-
query
140-
SELECT g, sort_array(collect_set(DISTINCT i)) FROM sa_int GROUP BY g ORDER BY g
141-
142-
-- ============================================================
143-
-- HAVING over a collect_set-derived predicate
144-
-- ============================================================
145-
146-
query
147-
SELECT g, sort_array(collect_set(i))
148-
FROM sa_int GROUP BY g HAVING size(collect_set(i)) > 1 ORDER BY g
149-
150-
-- ============================================================
151-
-- Representative data types through the SortAggregate path
152-
-- ============================================================
153-
154-
statement
155-
CREATE TABLE sa_string(v string, g string) USING parquet
156-
157-
statement
158-
INSERT INTO sa_string VALUES
159-
('hello', 'a'), ('world', 'a'), ('hello', 'a'), (NULL, 'a'),
160-
('', 'b'), ('x', 'b'), ('', 'b'), (NULL, 'b')
161-
162-
query
163-
SELECT g, sort_array(collect_set(v)) FROM sa_string GROUP BY g ORDER BY g
164-
165-
statement
166-
CREATE TABLE sa_bigint(v bigint, g string) USING parquet
167-
168-
statement
169-
INSERT INTO sa_bigint VALUES
170-
(1000000000000, 'a'), (2000000000000, 'a'), (1000000000000, 'a'), (NULL, 'a'),
171-
(3000000000000, 'b'), (NULL, 'b')
172-
173-
query
174-
SELECT g, sort_array(collect_set(v)) FROM sa_bigint GROUP BY g ORDER BY g
175-
176-
statement
177-
CREATE TABLE sa_decimal(v decimal(10,2), g string) USING parquet
178-
179-
statement
180-
INSERT INTO sa_decimal VALUES
181-
(1.50, 'a'), (2.50, 'a'), (1.50, 'a'), (NULL, 'a'),
182-
(0.00, 'b'), (99999999.99, 'b'), (NULL, 'b')
183-
184-
query
185-
SELECT g, sort_array(collect_set(v)) FROM sa_decimal GROUP BY g ORDER BY g
186-
187-
statement
188-
CREATE TABLE sa_date(v date, g string) USING parquet
189-
190-
statement
191-
INSERT INTO sa_date VALUES
192-
(DATE '2024-01-01', 'a'), (DATE '2024-06-15', 'a'), (DATE '2024-01-01', 'a'), (NULL, 'a'),
193-
(DATE '1970-01-01', 'b'), (NULL, 'b')
194-
195-
query
196-
SELECT g, sort_array(collect_set(v)) FROM sa_date GROUP BY g ORDER BY g
197-
198-
-- ============================================================
199-
-- Unsupported aggregate input under the SortAggregate path:
200-
-- with strictFloatingPoint=true, collect_set on floating-point types is
201-
-- incompatible, so the SortAggregateExec must fall back to Spark cleanly
202-
-- rather than crash.
67+
-- Decimal SUM at maximum precision falls back: depending on the buffer
68+
-- schema and codegen, Spark's sort aggregation keeps the running sum
69+
-- unbounded or nulls it on overflow, and Comet does not track which
20370
-- ============================================================
20471

20572
statement
206-
CREATE TABLE sa_double(v double, g string) USING parquet
73+
CREATE TABLE sa_dec38(d decimal(38,0), i int, g string) USING parquet
20774

20875
statement
209-
INSERT INTO sa_double VALUES
210-
(1.1, 'a'), (2.2, 'a'), (1.1, 'a'), (NULL, 'a'),
211-
(CAST('NaN' AS DOUBLE), 'b'), (CAST('NaN' AS DOUBLE), 'b'), (1.0, 'b')
76+
INSERT INTO sa_dec38 VALUES (1, 1, 'a'), (2, 2, 'a'), (3, 3, 'b'), (NULL, 4, 'b')
21277

213-
query expect_fallback(not fully compatible with Spark)
214-
SELECT g, sort_array(collect_set(v)) FROM sa_double GROUP BY g ORDER BY g
78+
query expect_fallback(Decimal SUM at maximum precision cannot match Spark's sort aggregation buffer)
79+
SELECT g, sum(d), sort_array(collect_set(i)) FROM sa_dec38 GROUP BY g ORDER BY g
Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
-- Licensed to the Apache Software Foundation (ASF) under one
2+
-- or more contributor license agreements. See the NOTICE file
3+
-- distributed with this work for additional information
4+
-- regarding copyright ownership. The ASF licenses this file
5+
-- to you under the Apache License, Version 2.0 (the
6+
-- "License"); you may not use this file except in compliance
7+
-- with the License. You may obtain a copy of the License at
8+
--
9+
-- http://www.apache.org/licenses/LICENSE-2.0
10+
--
11+
-- Unless required by applicable law or agreed to in writing,
12+
-- software distributed under the License is distributed on an
13+
-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
-- KIND, either express or implied. See the License for the
15+
-- specific language governing permissions and limitations
16+
-- under the License.
17+
18+
-- Without Comet shuffle, a sort aggregate's Partial and Final would be split across Comet and
19+
-- Spark, and Spark cannot read Comet's collect_set buffer, so SortAggregateExec stays in Spark.
20+
-- Config: spark.sql.execution.useObjectHashAggregateExec=false
21+
-- Config: spark.comet.shuffle.enabled=false
22+
23+
statement
24+
CREATE TABLE sa_no_shuffle(i int, g string) USING parquet
25+
26+
statement
27+
INSERT INTO sa_no_shuffle VALUES (1, 'a'), (2, 'a'), (1, 'a'), (3, 'b'), (NULL, 'b')
28+
29+
query expect_fallback(Comet shuffle is not enabled, so converting SortAggregate would split the aggregate across Comet and Spark)
30+
SELECT g, sort_array(collect_set(i)) FROM sa_no_shuffle GROUP BY g ORDER BY g

‎spark/src/test/scala/org/apache/comet/exec/CometAggregateSuite.scala‎

Lines changed: 8 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -3367,10 +3367,9 @@ class CometAggregateSuite extends CometTestBase with AdaptiveSparkPlanHelper {
33673367
// useObjectHashAggregateExec=false forces Spark to plan SortAggregateExec for
33683368
// TypedImperativeAggregate functions like collect_set. Comet converts those just like
33693369
// ObjectHashAggregateExec via the shared CometBaseAggregate path. Broader data-type and
3370-
// edge-case coverage lives in the SQL file test
3371-
// spark/src/test/resources/sql-tests/expressions/aggregate/sort_aggregate.sql; these Scala
3372-
// tests additionally assert that Spark actually planned a SortAggregateExec, which the SQL
3373-
// framework cannot check.
3370+
// edge-case coverage lives in the SQL file tests collect_set.sql (whose ConfigMatrix runs it
3371+
// through SortAggregateExec) and sort_aggregate.sql; this test additionally asserts that Spark
3372+
// actually planned a SortAggregateExec, which the SQL framework cannot check.
33743373
private def assertSortAggregateRunsNatively(query: String): Unit = {
33753374
withSQLConf(
33763375
SQLConf.USE_OBJECT_HASH_AGG.key -> "false",
@@ -3394,14 +3393,11 @@ class CometAggregateSuite extends CometTestBase with AdaptiveSparkPlanHelper {
33943393
}
33953394

33963395
test("SortAggregate with collect_set is converted to native") {
3397-
assertSortAggregateRunsNatively(
3398-
"SELECT g, sort_array(collect_set(v)) FROM tbl GROUP BY g ORDER BY g")
3399-
}
3400-
3401-
test("SortAggregate global collect_set (no grouping keys) is converted to native") {
3402-
// Empty grouping is a distinct plan shape: no pre-aggregate sort, empty output ordering,
3403-
// and adjustOutputForNativeState with zero grouping columns.
3404-
assertSortAggregateRunsNatively("SELECT sort_array(collect_set(v)) FROM tbl")
3396+
Seq(
3397+
"SELECT g, sort_array(collect_set(v)) FROM tbl GROUP BY g ORDER BY g",
3398+
// Empty grouping is a distinct plan shape: no pre-aggregate sort, empty output ordering,
3399+
// and adjustOutputForNativeState with zero grouping columns.
3400+
"SELECT sort_array(collect_set(v)) FROM tbl").foreach(assertSortAggregateRunsNatively)
34053401
}
34063402

34073403
// Spark puts a SortExec between each sort aggregate and the exchange below it, so the passes

0 commit comments

Comments
 (0)