|
| 1 | +-- Licensed to the Apache Software Foundation (ASF) under one |
| 2 | +-- or more contributor license agreements. See the NOTICE file |
| 3 | +-- distributed with this work for additional information |
| 4 | +-- regarding copyright ownership. The ASF licenses this file |
| 5 | +-- to you under the Apache License, Version 2.0 (the |
| 6 | +-- "License"); you may not use this file except in compliance |
| 7 | +-- with the License. You may obtain a copy of the License at |
| 8 | +-- |
| 9 | +-- http://www.apache.org/licenses/LICENSE-2.0 |
| 10 | +-- |
| 11 | +-- Unless required by applicable law or agreed to in writing, |
| 12 | +-- software distributed under the License is distributed on an |
| 13 | +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| 14 | +-- KIND, either express or implied. See the License for the |
| 15 | +-- specific language governing permissions and limitations |
| 16 | +-- under the License. |
| 17 | +-- Under `spark.sql.mapKeyDedupPolicy=LAST_WIN` a repeated map key is not an error: the builder keeps |
| 18 | +-- the last value. `CometMapFromArrays` and `CometMapFromEntries` mix in `CodegenDispatchFallback`, |
| 19 | +-- so with the dispatcher on (the default) both run Spark's own `ArrayBasedMapBuilder` in the JVM |
| 20 | +-- codegen dispatcher. The dedup-policy fixtures turn the dispatcher off to cover the fallback, so |
| 21 | +-- these queries cover the dispatched path with a key that really repeats: `from` then goes through |
| 22 | +-- `build()`, and both the kept value and the stored key are compared with Spark's. |
| 23 | +-- |
| 24 | +-- A `FLOAT` or `DOUBLE` key is declined natively on every version (#6549), so those dispatch too. |
| 25 | +-- NaNs that differ only in the sign bit are one key on every version. `-0.0` and `0.0` are one key |
| 26 | +-- from Spark 4.0, unless `spark.sql.legacy.disableMapKeyNormalization` is set, and two keys on |
| 27 | +-- earlier releases; either way the dispatcher has to agree with Spark. |
| 28 | + |
| 29 | +-- Config: spark.sql.mapKeyDedupPolicy=LAST_WIN |
| 30 | +-- Config: spark.comet.exec.scalaUDF.codegen.enabled=true |
| 31 | + |
| 32 | +-- Parquet canonicalizes NaN, so negating a stored NaN gives one with the sign bit set. |
| 33 | +statement |
| 34 | +CREATE TABLE float_map_keys_last_win(id int, d double, f float, k array<string>, v array<int>) |
| 35 | +USING parquet |
| 36 | + |
| 37 | +statement |
| 38 | +INSERT INTO float_map_keys_last_win VALUES |
| 39 | + (1, 0.0, 0.0, array('a', 'a', 'b'), array(1, 2, 3)), |
| 40 | + (2, double('NaN'), float('NaN'), array('x', 'x'), array(10, 20)), |
| 41 | + (3, 1.5, 1.5, array('a', 'b', 'c'), array(1, 2, 3)) |
| 42 | + |
| 43 | +-- A repeated string key: the last value wins. |
| 44 | +query expect_dispatch(map_from_arrays) |
| 45 | +SELECT id, map_keys(map_from_arrays(k, v)), map_values(map_from_arrays(k, v)) |
| 46 | +FROM float_map_keys_last_win |
| 47 | + |
| 48 | +query expect_dispatch(map_from_entries) |
| 49 | +SELECT id, |
| 50 | + map_keys(map_from_entries(arrays_zip(k, v))), |
| 51 | + map_values(map_from_entries(arrays_zip(k, v))) |
| 52 | +FROM float_map_keys_last_win |
| 53 | + |
| 54 | +-- NaNs with different sign bits are one DOUBLE or FLOAT key: one entry, holding the last value. |
| 55 | +query expect_dispatch(map_from_arrays) |
| 56 | +SELECT map_keys(map_from_arrays(array(d, -d), array(1, 2))), |
| 57 | + map_values(map_from_arrays(array(d, -d), array(1, 2))), |
| 58 | + map_keys(map_from_arrays(array(f, -f), array(1, 2))), |
| 59 | + map_values(map_from_arrays(array(f, -f), array(1, 2))) |
| 60 | +FROM float_map_keys_last_win WHERE id = 2 |
| 61 | + |
| 62 | +query expect_dispatch(map_from_entries) |
| 63 | +SELECT map_keys(map_from_entries(array(named_struct('k', d, 'v', 1), named_struct('k', -d, 'v', 2)))), |
| 64 | + map_values(map_from_entries(array(named_struct('k', d, 'v', 1), named_struct('k', -d, 'v', 2)))), |
| 65 | + map_keys(map_from_entries(array(named_struct('k', f, 'v', 1), named_struct('k', -f, 'v', 2)))), |
| 66 | + map_values(map_from_entries(array(named_struct('k', f, 'v', 1), named_struct('k', -f, 'v', 2)))) |
| 67 | +FROM float_map_keys_last_win WHERE id = 2 |
| 68 | + |
| 69 | +-- 0.0 and -0.0: one key holding the last value from Spark 4.0, two keys before that. |
| 70 | +query expect_dispatch(map_from_arrays) |
| 71 | +SELECT map_keys(map_from_arrays(array(d, -d), array(1, 2))), |
| 72 | + map_values(map_from_arrays(array(d, -d), array(1, 2))), |
| 73 | + map_keys(map_from_arrays(array(f, -f), array(1, 2))), |
| 74 | + map_values(map_from_arrays(array(f, -f), array(1, 2))) |
| 75 | +FROM float_map_keys_last_win WHERE id = 1 |
| 76 | + |
| 77 | +query expect_dispatch(map_from_entries) |
| 78 | +SELECT map_keys(map_from_entries(array(named_struct('k', d, 'v', 1), named_struct('k', -d, 'v', 2)))), |
| 79 | + map_values(map_from_entries(array(named_struct('k', d, 'v', 1), named_struct('k', -d, 'v', 2)))), |
| 80 | + map_keys(map_from_entries(array(named_struct('k', f, 'v', 1), named_struct('k', -f, 'v', 2)))), |
| 81 | + map_values(map_from_entries(array(named_struct('k', f, 'v', 1), named_struct('k', -f, 'v', 2)))) |
| 82 | +FROM float_map_keys_last_win WHERE id = 1 |
| 83 | + |
| 84 | +-- The stored key, when -0.0 is the first key: 0.0 from Spark 4.0 (normalized), -0.0 before that. |
| 85 | +query expect_dispatch(map_from_arrays) |
| 86 | +SELECT map_keys(map_from_arrays(array(-d, d), array(1, 2))), |
| 87 | + map_values(map_from_arrays(array(-d, d), array(1, 2))) |
| 88 | +FROM float_map_keys_last_win WHERE id = 1 |
| 89 | + |
| 90 | +query expect_dispatch(map_from_entries) |
| 91 | +SELECT map_keys(map_from_entries(array(named_struct('k', -d, 'v', 1), named_struct('k', d, 'v', 2)))), |
| 92 | + map_values(map_from_entries(array(named_struct('k', -d, 'v', 1), named_struct('k', d, 'v', 2)))) |
| 93 | +FROM float_map_keys_last_win WHERE id = 1 |
0 commit comments