|
| 1 | +-- Licensed to the Apache Software Foundation (ASF) under one |
| 2 | +-- or more contributor license agreements. See the NOTICE file |
| 3 | +-- distributed with this work for additional information |
| 4 | +-- regarding copyright ownership. The ASF licenses this file |
| 5 | +-- to you under the Apache License, Version 2.0 (the |
| 6 | +-- "License"); you may not use this file except in compliance |
| 7 | +-- with the License. You may obtain a copy of the License at |
| 8 | +-- |
| 9 | +-- http://www.apache.org/licenses/LICENSE-2.0 |
| 10 | +-- |
| 11 | +-- Unless required by applicable law or agreed to in writing, |
| 12 | +-- software distributed under the License is distributed on an |
| 13 | +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| 14 | +-- KIND, either express or implied. See the License for the |
| 15 | +-- specific language governing permissions and limitations |
| 16 | +-- under the License. |
| 17 | + |
| 18 | +-- array_union with a float element type runs as spark_array_union, which normalizes both arrays |
| 19 | +-- before delegating to DataFusion. Like every other element type, it has to skip its right operand |
| 20 | +-- where the left array is NULL, as Spark's null-intolerant BinaryExpression does, so a right operand |
| 21 | +-- that raises cannot fail a row Spark skips. array_null_short_circuit.sql covers array<int>, which |
| 22 | +-- runs through DataFusion's array_union unchanged. Each query names array_union with expect_native, |
| 23 | +-- because the codegen dispatcher would pass by running Spark's code. |
| 24 | +-- https://github.com/apache/datafusion-comet/issues/6613 |
| 25 | + |
| 26 | +-- Float elements run natively by default only on Spark 4.0.5+, 4.1.4+ and 4.2+. The opt-in makes |
| 27 | +-- every version run them natively. The data has no signed zero, which earlier releases keep apart. |
| 28 | +-- Config: spark.sql.ansi.enabled=true |
| 29 | +-- Config: spark.comet.expression.ArrayUnion.allowIncompatible=true |
| 30 | + |
| 31 | +statement |
| 32 | +CREATE TABLE array_null_short_circuit_float(a array<double>, b array<double>, s string) USING parquet |
| 33 | + |
| 34 | +-- `s` is malformed only where `a` is NULL. COALESCE(1) writes a single file, so the NULL arrays |
| 35 | +-- share a batch with the others. |
| 36 | +statement |
| 37 | +INSERT INTO array_null_short_circuit_float SELECT /*+ COALESCE(1) */ * FROM VALUES |
| 38 | + (NULL, array(1.5D, 2.5D), 'bad'), |
| 39 | + (array(1.5D, 2.5D), array(2.5D, 3.5D), '2'), |
| 40 | + (array(4.5D), array(5.5D, 6.5D), '7'), |
| 41 | + (NULL, NULL, 'worse'), |
| 42 | + (array(4.5D, NULL, 5.5D), array(5.5D, 6.5D), '5') |
| 43 | + AS v(a, b, s) |
| 44 | + |
| 45 | +-- A start of 0 makes slice raise in any mode, and only rows whose array is NULL have one |
| 46 | +query expect_native(array_union) |
| 47 | +SELECT array_union(a, slice(b, IF(a IS NULL, 0, 1), 1)) FROM array_null_short_circuit_float |
| 48 | + |
| 49 | +-- The ANSI cast of a malformed string raises, and only rows whose array is NULL have one |
| 50 | +query expect_native(array_union) |
| 51 | +SELECT array_union(a, array(CAST(CAST(s AS INT) AS DOUBLE))) FROM array_null_short_circuit_float |
0 commit comments