1616-- under the License.
1717
1818-- Disabling ObjectHashAggregate forces Spark to plan SortAggregateExec for the
19- -- TypedImperativeAggregate function collect_set, which is the operator this file exercises.
20- -- (The plan-shape assertion that a SortAggregateExec is actually produced lives in
21- -- CometAggregateSuite; here the default `query` mode asserts the whole plan, including the
22- -- pre-aggregate sorts and the Comet shuffle, runs natively and matches Spark.)
23- -- collect_set returns elements in nondeterministic order, so every result is wrapped in
24- -- sort_array for a stable comparison against Spark.
19+ -- TypedImperativeAggregate function collect_set. collect_set.sql runs its whole fixture through
20+ -- SortAggregateExec as well, so this file keeps only the shapes it does not cover. collect_set
21+ -- returns elements in nondeterministic order, so every result is wrapped in sort_array.
2522-- Config: spark.sql.execution.useObjectHashAggregateExec=false
26- -- Config: spark.comet.exec.strictFloatingPoint=true
27-
28- -- ============================================================
29- -- Setup: tables
30- -- ============================================================
3123
3224statement
3325CREATE TABLE sa_int (i int , g string) USING parquet
@@ -46,35 +38,6 @@ INSERT INTO sa_multikey VALUES
4638 (1 , ' x' , 10 ), (2 , ' x' , 10 ), (1 , ' x' , 10 ),
4739 (3 , ' x' , 20 ), (4 , ' y' , 10 ), (NULL , ' y' , 10 )
4840
49- statement
50- CREATE TABLE sa_nulls (v int , g string) USING parquet
51-
52- statement
53- INSERT INTO sa_nulls VALUES (NULL , ' a' ), (NULL , ' a' ), (NULL , ' b' ), (1 , ' b' )
54-
55- statement
56- CREATE TABLE sa_empty (v int , g string) USING parquet
57-
58- statement
59- CREATE TABLE sa_single (v int ) USING parquet
60-
61- statement
62- INSERT INTO sa_single VALUES (42 )
63-
64- -- ============================================================
65- -- Global aggregate (no GROUP BY): single SortAggregate group
66- -- ============================================================
67-
68- query
69- SELECT sort_array(collect_set(i)) FROM sa_int
70-
71- -- ============================================================
72- -- Single grouping key
73- -- ============================================================
74-
75- query
76- SELECT g, sort_array(collect_set(i)) FROM sa_int GROUP BY g ORDER BY g
77-
7841-- ============================================================
7942-- Multiple grouping keys (sort ordering spans both keys)
8043-- ============================================================
9154SELECT upper (g) AS gg, sort_array(collect_set(i))
9255FROM sa_int GROUP BY upper (g) ORDER BY gg
9356
94- -- ============================================================
95- -- All-NULL group returns an empty array
96- -- ============================================================
97-
98- query
99- SELECT g, sort_array(collect_set(v)) FROM sa_nulls GROUP BY g ORDER BY g
100-
101- -- ============================================================
102- -- Empty table
103- -- ============================================================
104-
105- query
106- SELECT sort_array(collect_set(v)) FROM sa_empty
107-
108- query
109- SELECT g, sort_array(collect_set(v)) FROM sa_empty GROUP BY g ORDER BY g
110-
111- -- ============================================================
112- -- Single row
113- -- ============================================================
114-
115- query
116- SELECT sort_array(collect_set(v)) FROM sa_single
117-
11857-- ============================================================
11958-- collect_set mixed with hashable aggregates: collect_set forces the
12059-- whole aggregate onto SortAggregate, so count/sum/min/max ride along
@@ -125,90 +64,16 @@ SELECT g, sort_array(collect_set(i)), count(*), count(i), sum(i), min(i), max(i)
12564FROM sa_int GROUP BY g ORDER BY g
12665
12766-- ============================================================
128- -- Multiple collect_set in one query
129- -- ============================================================
130-
131- query
132- SELECT k1, sort_array(collect_set(a)), sort_array(collect_set(k2))
133- FROM sa_multikey GROUP BY k1 ORDER BY k1
134-
135- -- ============================================================
136- -- DISTINCT (distinct-aggregate planner path)
137- -- ============================================================
138-
139- query
140- SELECT g, sort_array(collect_set(DISTINCT i)) FROM sa_int GROUP BY g ORDER BY g
141-
142- -- ============================================================
143- -- HAVING over a collect_set-derived predicate
144- -- ============================================================
145-
146- query
147- SELECT g, sort_array(collect_set(i))
148- FROM sa_int GROUP BY g HAVING size(collect_set(i)) > 1 ORDER BY g
149-
150- -- ============================================================
151- -- Representative data types through the SortAggregate path
152- -- ============================================================
153-
154- statement
155- CREATE TABLE sa_string (v string, g string) USING parquet
156-
157- statement
158- INSERT INTO sa_string VALUES
159- (' hello' , ' a' ), (' world' , ' a' ), (' hello' , ' a' ), (NULL , ' a' ),
160- (' ' , ' b' ), (' x' , ' b' ), (' ' , ' b' ), (NULL , ' b' )
161-
162- query
163- SELECT g, sort_array(collect_set(v)) FROM sa_string GROUP BY g ORDER BY g
164-
165- statement
166- CREATE TABLE sa_bigint (v bigint , g string) USING parquet
167-
168- statement
169- INSERT INTO sa_bigint VALUES
170- (1000000000000 , ' a' ), (2000000000000 , ' a' ), (1000000000000 , ' a' ), (NULL , ' a' ),
171- (3000000000000 , ' b' ), (NULL , ' b' )
172-
173- query
174- SELECT g, sort_array(collect_set(v)) FROM sa_bigint GROUP BY g ORDER BY g
175-
176- statement
177- CREATE TABLE sa_decimal (v decimal (10 ,2 ), g string) USING parquet
178-
179- statement
180- INSERT INTO sa_decimal VALUES
181- (1 .50 , ' a' ), (2 .50 , ' a' ), (1 .50 , ' a' ), (NULL , ' a' ),
182- (0 .00 , ' b' ), (99999999 .99 , ' b' ), (NULL , ' b' )
183-
184- query
185- SELECT g, sort_array(collect_set(v)) FROM sa_decimal GROUP BY g ORDER BY g
186-
187- statement
188- CREATE TABLE sa_date (v date , g string) USING parquet
189-
190- statement
191- INSERT INTO sa_date VALUES
192- (DATE ' 2024-01-01' , ' a' ), (DATE ' 2024-06-15' , ' a' ), (DATE ' 2024-01-01' , ' a' ), (NULL , ' a' ),
193- (DATE ' 1970-01-01' , ' b' ), (NULL , ' b' )
194-
195- query
196- SELECT g, sort_array(collect_set(v)) FROM sa_date GROUP BY g ORDER BY g
197-
198- -- ============================================================
199- -- Unsupported aggregate input under the SortAggregate path:
200- -- with strictFloatingPoint=true, collect_set on floating-point types is
201- -- incompatible, so the SortAggregateExec must fall back to Spark cleanly
202- -- rather than crash.
67+ -- Decimal SUM at maximum precision falls back: depending on the buffer
68+ -- schema and codegen, Spark's sort aggregation keeps the running sum
69+ -- unbounded or nulls it on overflow, and Comet does not track which
20370-- ============================================================
20471
20572statement
206- CREATE TABLE sa_double (v double , g string) USING parquet
73+ CREATE TABLE sa_dec38 (d decimal ( 38 , 0 ), i int , g string) USING parquet
20774
20875statement
209- INSERT INTO sa_double VALUES
210- (1 .1 , ' a' ), (2 .2 , ' a' ), (1 .1 , ' a' ), (NULL , ' a' ),
211- (CAST(' NaN' AS DOUBLE), ' b' ), (CAST(' NaN' AS DOUBLE), ' b' ), (1 .0 , ' b' )
76+ INSERT INTO sa_dec38 VALUES (1 , 1 , ' a' ), (2 , 2 , ' a' ), (3 , 3 , ' b' ), (NULL , 4 , ' b' )
21277
213- query expect_fallback(not fully compatible with Spark)
214- SELECT g, sort_array(collect_set(v )) FROM sa_double GROUP BY g ORDER BY g
78+ query expect_fallback(Decimal SUM at maximum precision cannot match Spark' s sort aggregation buffer )
79+ SELECT g, sum(d), sort_array(collect_set(i )) FROM sa_dec38 GROUP BY g ORDER BY g
0 commit comments