Skip to content

Commit 2348d78

Browse files
committed
test: pin the float sweep's array set operations, mode and map lookups to the paths they take
Review feedback: queries that fall back in the default config compared Spark with Spark. The array set operations (distinct, union, intersect, except) move to array_set_operations_4_2.sql and mode to aggregates_mode.sql, which turn on allowIncompatible so that a plain query pins native execution. The map lookup keeps only element_at, pinned with expect_fallback, since Spark rewrites map(l, 1)[r] to a CASE. Add a FLOAT array case to the hash fixture, and document the strict-test route for a known mismatch.
1 parent 84acfd1 commit 2348d78

6 files changed

Lines changed: 203 additions & 66 deletions

File tree

‎docs/source/contributor-guide/floating_point.md‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -209,10 +209,13 @@ and only for a type that contains a `FLOAT` or `DOUBLE`.
209209
- Add the expression to the sweep in `float_semantics/`, which crosses the edge values with
210210
operator contexts and with every expression and aggregate that accepts floating-point input.
211211
A new expression goes in `expressions.sql`, or in `aggregates.sql` for an aggregate, with one
212-
query for `DOUBLE` and one for `FLOAT`. If one does not match Spark yet, mark its queries
213-
`ignore(<issue>)`. A SQL file test cannot expect a difference, so a gap that should fail once it
214-
is fixed goes in `CometFloatSemanticsSuite`, whose test for `collect_set` before Spark 4.2 fails
215-
when the answers start to match.
212+
query for `DOUBLE` and one for `FLOAT`. An expression that falls back in the default config,
213+
such as `mode` or the array set operations, goes in a file that turns on its
214+
`allowIncompatible` config and sets `MinSparkVersion` to the first release where it runs
215+
natively (`aggregates_mode.sql`, `array_set_operations_4_2.sql`), so that a plain `query` pins
216+
native execution. If one does not match Spark yet, add a strict test in
217+
`CometFloatSemanticsSuite` that links the issue and fails once the answers match, like its
218+
`collect_set` test before Spark 4.2. A SQL file test cannot expect a difference.
216219

217220
Fixtures to start from include `expressions/conditional/float_comparisons.sql`,
218221
`expressions/math/nan_divisor.sql`, `expressions/math/greatest_least_floating_point.sql`,

‎spark/src/test/resources/sql-tests/expressions/hash/hash.sql‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -114,6 +114,9 @@ SELECT hash(d), hash(-d), xxhash64(d), xxhash64(-d), hash(f), hash(-f), xxhash64
114114
query
115115
SELECT hash(array(-d)), xxhash64(array(-d)), hash(named_struct('a', -d, 'b', -f)), xxhash64(named_struct('a', -d, 'b', -f)) FROM test_nan
116116

117+
query
118+
SELECT hash(array(-f)), xxhash64(array(-f)), hash(array(-d, -d)), xxhash64(array(f, -f)) FROM test_nan
119+
117120
-- Spark rejects hashing a map unless the legacy config above is set. Map keys cannot be null,
118121
-- so the null row is filtered out.
119122
query

‎spark/src/test/resources/sql-tests/float_semantics/aggregates.sql‎

Lines changed: 1 addition & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -146,17 +146,7 @@ query
146146
WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float)
147147
SELECT id, approx_count_distinct(x) FROM (SELECT id, pos, x FROM p LATERAL VIEW posexplode(array(l, r)) t AS pos, x) GROUP BY id
148148

149-
-- mode, DOUBLE
150-
-- mode falls back to Spark unless Mode.allowIncompatible is set, so only the answer is checked
151-
-- here. expressions/aggregate/mode.sql runs it natively.
152-
query spark_answer_only
153-
WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double)
154-
SELECT id, mode(x) FROM (SELECT id, x FROM p LATERAL VIEW explode(array(l, r, r)) t AS x) GROUP BY id
155-
156-
-- mode, FLOAT
157-
query spark_answer_only
158-
WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float)
159-
SELECT id, mode(x) FROM (SELECT id, x FROM p LATERAL VIEW explode(array(l, r, r)) t AS x) GROUP BY id
149+
-- mode is in aggregates_mode.sql, which turns on Mode.allowIncompatible.
160150

161151
-- global max and min, DOUBLE
162152
query
Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
-- Licensed to the Apache Software Foundation (ASF) under one
2+
-- or more contributor license agreements. See the NOTICE file
3+
-- distributed with this work for additional information
4+
-- regarding copyright ownership. The ASF licenses this file
5+
-- to you under the Apache License, Version 2.0 (the
6+
-- "License"); you may not use this file except in compliance
7+
-- with the License. You may obtain a copy of the License at
8+
--
9+
-- http://www.apache.org/licenses/LICENSE-2.0
10+
--
11+
-- Unless required by applicable law or agreed to in writing,
12+
-- software distributed under the License is distributed on an
13+
-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
-- KIND, either express or implied. See the License for the
15+
-- specific language governing permissions and limitations
16+
17+
-- mode over float input. Each pair of edge values is expanded to (l, r, r), so the mode is unique
18+
-- under every Spark equality rule. mode falls back to Spark unless Mode.allowIncompatible is set,
19+
-- so the config is on to run it natively. #7115 makes mode of a float fall back before Spark 3.5.2,
20+
-- so this file starts at 3.5. expressions/aggregate/mode.sql has the general cases. The sweep
21+
-- tracks #6385.
22+
--
23+
-- MinSparkVersion: 3.5
24+
-- Config: spark.comet.expression.Mode.allowIncompatible=true
25+
26+
statement
27+
CREATE TABLE fs_edge_double(id INT, c DOUBLE, neg BOOLEAN) USING parquet
28+
29+
statement
30+
INSERT INTO fs_edge_double VALUES
31+
(1, 0.0D, false),
32+
(2, double('-0.0'), false),
33+
(3, double('NaN'), false),
34+
(4, double('NaN'), true),
35+
(5, 1.0D, false),
36+
(6, -1.0D, false),
37+
(7, double('Infinity'), false),
38+
(8, double('-Infinity'), false),
39+
(9, NULL, false)
40+
41+
statement
42+
CREATE TABLE fs_pair_double USING parquet AS
43+
SELECT a.id * 10 + b.id AS id, a.c AS l, b.c AS r, a.neg AS lneg, b.neg AS rneg
44+
FROM fs_edge_double a CROSS JOIN fs_edge_double b
45+
46+
statement
47+
CREATE TABLE fs_edge_float(id INT, c FLOAT, neg BOOLEAN) USING parquet
48+
49+
statement
50+
INSERT INTO fs_edge_float VALUES
51+
(1, float(0.0), false),
52+
(2, float('-0.0'), false),
53+
(3, float('NaN'), false),
54+
(4, float('NaN'), true),
55+
(5, float(1.0), false),
56+
(6, float(-1.0), false),
57+
(7, float('Infinity'), false),
58+
(8, float('-Infinity'), false),
59+
(9, NULL, false)
60+
61+
statement
62+
CREATE TABLE fs_pair_float USING parquet AS
63+
SELECT a.id * 10 + b.id AS id, a.c AS l, b.c AS r, a.neg AS lneg, b.neg AS rneg
64+
FROM fs_edge_float a CROSS JOIN fs_edge_float b
65+
66+
-- mode, DOUBLE
67+
query
68+
WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double)
69+
SELECT id, mode(x) FROM (SELECT id, x FROM p LATERAL VIEW explode(array(l, r, r)) t AS x) GROUP BY id
70+
71+
-- mode, FLOAT
72+
query
73+
WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float)
74+
SELECT id, mode(x) FROM (SELECT id, x FROM p LATERAL VIEW explode(array(l, r, r)) t AS x) GROUP BY id
Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,107 @@
1+
-- Licensed to the Apache Software Foundation (ASF) under one
2+
-- or more contributor license agreements. See the NOTICE file
3+
-- distributed with this work for additional information
4+
-- regarding copyright ownership. The ASF licenses this file
5+
-- to you under the Apache License, Version 2.0 (the
6+
-- "License"); you may not use this file except in compliance
7+
-- with the License. You may obtain a copy of the License at
8+
--
9+
-- http://www.apache.org/licenses/LICENSE-2.0
10+
--
11+
-- Unless required by applicable law or agreed to in writing,
12+
-- software distributed under the License is distributed on an
13+
-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
-- KIND, either express or implied. See the License for the
15+
-- specific language governing permissions and limitations
16+
17+
-- The set operations on arrays of floats, over the edge values (see expressions.sql). Every 4.2+
18+
-- release normalizes signed zeros in these functions (array_set_signed_zero.sql), so native
19+
-- distinct and union, and the except and intersect kernels with allowIncompatible, must run
20+
-- natively and agree with Spark. Before 4.2 the signed-zero routing differs by release
21+
-- (array_set_signed_zero_spark_3.sql, array_set_signed_zero_spark_4_0_4_1.sql), so the sweep does
22+
-- not cover those releases.
23+
-- The sweep tracks #6385.
24+
--
25+
-- MinSparkVersion: 4.2
26+
-- Config: spark.comet.expression.ArrayExcept.allowIncompatible=true
27+
-- Config: spark.comet.expression.ArrayIntersect.allowIncompatible=true
28+
29+
statement
30+
CREATE TABLE fs_edge_double(id INT, c DOUBLE, neg BOOLEAN) USING parquet
31+
32+
statement
33+
INSERT INTO fs_edge_double VALUES
34+
(1, 0.0D, false),
35+
(2, double('-0.0'), false),
36+
(3, double('NaN'), false),
37+
(4, double('NaN'), true),
38+
(5, 1.0D, false),
39+
(6, -1.0D, false),
40+
(7, double('Infinity'), false),
41+
(8, double('-Infinity'), false),
42+
(9, NULL, false)
43+
44+
statement
45+
CREATE TABLE fs_pair_double USING parquet AS
46+
SELECT a.id * 10 + b.id AS id, a.c AS l, b.c AS r, a.neg AS lneg, b.neg AS rneg
47+
FROM fs_edge_double a CROSS JOIN fs_edge_double b
48+
49+
statement
50+
CREATE TABLE fs_edge_float(id INT, c FLOAT, neg BOOLEAN) USING parquet
51+
52+
statement
53+
INSERT INTO fs_edge_float VALUES
54+
(1, float(0.0), false),
55+
(2, float('-0.0'), false),
56+
(3, float('NaN'), false),
57+
(4, float('NaN'), true),
58+
(5, float(1.0), false),
59+
(6, float(-1.0), false),
60+
(7, float('Infinity'), false),
61+
(8, float('-Infinity'), false),
62+
(9, NULL, false)
63+
64+
statement
65+
CREATE TABLE fs_pair_float USING parquet AS
66+
SELECT a.id * 10 + b.id AS id, a.c AS l, b.c AS r, a.neg AS lneg, b.neg AS rneg
67+
FROM fs_edge_float a CROSS JOIN fs_edge_float b
68+
69+
-- array_distinct, DOUBLE
70+
query
71+
WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double)
72+
SELECT id, array_distinct(array(l, r)) FROM p
73+
74+
-- array_distinct, FLOAT
75+
query
76+
WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float)
77+
SELECT id, array_distinct(array(l, r)) FROM p
78+
79+
-- array_union, DOUBLE
80+
query
81+
WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double)
82+
SELECT id, array_union(array(l), array(r)) FROM p
83+
84+
-- array_union, FLOAT
85+
query
86+
WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float)
87+
SELECT id, array_union(array(l), array(r)) FROM p
88+
89+
-- array_intersect, DOUBLE
90+
query
91+
WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double)
92+
SELECT id, array_intersect(array(l), array(r)) FROM p
93+
94+
-- array_intersect, FLOAT
95+
query
96+
WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float)
97+
SELECT id, array_intersect(array(l), array(r)) FROM p
98+
99+
-- array_except, DOUBLE
100+
query
101+
WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double)
102+
SELECT id, array_except(array(l), array(r)) FROM p
103+
104+
-- array_except, FLOAT
105+
query
106+
WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float)
107+
SELECT id, array_except(array(l), array(r)) FROM p

‎spark/src/test/resources/sql-tests/float_semantics/expressions.sql‎

Lines changed: 11 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -300,49 +300,8 @@ query
300300
WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float)
301301
SELECT id, arrays_overlap(array(l), array(r)) FROM p
302302

303-
-- array_distinct, DOUBLE
304-
-- Whether float elements run natively depends on the Spark release (array_distinct.sql), so only
305-
-- the answer is checked.
306-
query spark_answer_only
307-
WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double)
308-
SELECT id, array_distinct(array(l, r)) FROM p
309-
310-
-- array_distinct, FLOAT
311-
query spark_answer_only
312-
WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float)
313-
SELECT id, array_distinct(array(l, r)) FROM p
314-
315-
-- array_union, DOUBLE
316-
-- Whether float elements run natively depends on the Spark release (array_union.sql), so only the
317-
-- answer is checked.
318-
query spark_answer_only
319-
WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double)
320-
SELECT id, array_union(array(l), array(r)) FROM p
321-
322-
-- array_union, FLOAT
323-
query spark_answer_only
324-
WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float)
325-
SELECT id, array_union(array(l), array(r)) FROM p
326-
327-
-- array_intersect, DOUBLE
328-
query
329-
WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double)
330-
SELECT id, array_intersect(array(l), array(r)) FROM p
331-
332-
-- array_intersect, FLOAT
333-
query
334-
WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float)
335-
SELECT id, array_intersect(array(l), array(r)) FROM p
336-
337-
-- array_except, DOUBLE
338-
query
339-
WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double)
340-
SELECT id, array_except(array(l), array(r)) FROM p
341-
342-
-- array_except, FLOAT
343-
query
344-
WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float)
345-
SELECT id, array_except(array(l), array(r)) FROM p
303+
-- array_distinct, array_union, array_intersect and array_except are in array_set_operations_4_2.sql,
304+
-- where every query runs natively.
346305

347306
-- sort_array, DOUBLE
348307
query
@@ -394,17 +353,18 @@ query
394353
WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float)
395354
SELECT id, array_min(array(l, r)) FROM p
396355

397-
-- map lookup, DOUBLE
398-
-- A map with a float key falls back to Spark (map_builders_float_key_gate.sql), so only the answer
399-
-- is checked.
400-
query spark_answer_only
356+
-- element_at on a map, DOUBLE
357+
-- Spark looks a key up in its SQL ordering, so a map with a float key falls back to Spark
358+
-- (map_builders_float_key_gate.sql, expressions/map/element_at_map.sql). `map(l, 1)[r]` is not
359+
-- here: Spark's optimizer rewrites it to a CASE over the keys, which comparison.sql covers.
360+
query expect_fallback(Spark looks a key up in its SQL ordering)
401361
WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double)
402-
SELECT id, map(l, 1)[r], element_at(map(l, 1), r) FROM p WHERE l IS NOT NULL
362+
SELECT id, element_at(map(l, 1), r) FROM p WHERE l IS NOT NULL
403363

404-
-- map lookup, FLOAT
405-
query spark_answer_only
364+
-- element_at on a map, FLOAT
365+
query expect_fallback(Spark looks a key up in its SQL ordering)
406366
WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float)
407-
SELECT id, map(l, 1)[r], element_at(map(l, 1), r) FROM p WHERE l IS NOT NULL
367+
SELECT id, element_at(map(l, 1), r) FROM p WHERE l IS NOT NULL
408368

409369
-- map_contains_key, DOUBLE
410370
query

0 commit comments

Comments
 (0)