|
| 1 | +-- Licensed to the Apache Software Foundation (ASF) under one |
| 2 | +-- or more contributor license agreements. See the NOTICE file |
| 3 | +-- distributed with this work for additional information |
| 4 | +-- regarding copyright ownership. The ASF licenses this file |
| 5 | +-- to you under the Apache License, Version 2.0 (the |
| 6 | +-- "License"); you may not use this file except in compliance |
| 7 | +-- with the License. You may obtain a copy of the License at |
| 8 | +-- |
| 9 | +-- http://www.apache.org/licenses/LICENSE-2.0 |
| 10 | +-- |
| 11 | +-- Unless required by applicable law or agreed to in writing, |
| 12 | +-- software distributed under the License is distributed on an |
| 13 | +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| 14 | +-- KIND, either express or implied. See the License for the |
| 15 | +-- specific language governing permissions and limitations |
| 16 | +-- under the License. |
| 17 | + |
| 18 | +-- to_csv runs through the codegen dispatcher by default so results match Spark exactly, including |
| 19 | +-- quoting and escaping. The native path is opt-in via |
| 20 | +-- spark.comet.expression.StructsToCsv.allowIncompatible. |
| 21 | + |
| 22 | +statement |
| 23 | +CREATE TABLE test_to_csv(a int, b string, c double) USING parquet |
| 24 | + |
| 25 | +statement |
| 26 | +INSERT INTO test_to_csv VALUES |
| 27 | + (1, 'x', 2.5), |
| 28 | + (-3, 'hello,world', 0.0), |
| 29 | + (0, 'has "quote"', -1.5), |
| 30 | + (NULL, NULL, NULL), |
| 31 | + (7, '', 3.0) |
| 32 | + |
| 33 | +-- column struct: values with delimiters and quotes exercise Spark's CSV quoting rules |
| 34 | +query |
| 35 | +SELECT to_csv(named_struct('a', a, 'b', b, 'c', c)) FROM test_to_csv |
| 36 | + |
| 37 | +-- literal struct (constant folding is disabled by the test suite) |
| 38 | +query |
| 39 | +SELECT |
| 40 | + to_csv(named_struct('a', 1, 'b', 'x', 'c', 2.5)), |
| 41 | + to_csv(named_struct('s', 'a,b', 'n', CAST(NULL AS INT))) |
| 42 | + |
| 43 | +-- Options are the main axis of behavior difference for this expression, and the dispatcher path |
| 44 | +-- had no options coverage anywhere (CometCsvExpressionSuite only varies them under |
| 45 | +-- allowIncompatible=true). `sep` also changes which values need quoting. |
| 46 | +query |
| 47 | +SELECT to_csv(named_struct('a', a, 'b', b, 'c', c), map('sep', ';')) FROM test_to_csv |
| 48 | + |
| 49 | +-- timestampFormat over date and timestamp fields: these are the #3232 types that were |
| 50 | +-- Incompatible before this change and now route through the dispatcher. |
| 51 | +-- BinaryType is the other #3232 type but is deliberately absent: on Spark 3.4 / 3.5 the CSV |
| 52 | +-- converter has no binary branch and renders it with Java's default Object.toString(), e.g. |
| 53 | +-- "[B@10bc15e4", an identity hash that differs between any two evaluations, so the value is not |
| 54 | +-- assertable by any engine including Spark itself. (Spark 4.0 added a real binary formatter, but |
| 55 | +-- this fixture runs on every supported version.) |
| 56 | +statement |
| 57 | +CREATE TABLE test_to_csv_temporal(d date, t timestamp) USING parquet |
| 58 | + |
| 59 | +statement |
| 60 | +INSERT INTO test_to_csv_temporal VALUES |
| 61 | + (DATE '2024-01-31', TIMESTAMP '2024-01-31 12:34:56.789'), |
| 62 | + (DATE '1970-01-01', TIMESTAMP '1970-01-01 00:00:00'), |
| 63 | + (NULL, NULL) |
| 64 | + |
| 65 | +query |
| 66 | +SELECT to_csv(named_struct('d', d, 't', t)) FROM test_to_csv_temporal |
| 67 | + |
| 68 | +query |
| 69 | +SELECT to_csv(named_struct('d', d, 't', t), map('timestampFormat', 'yyyy/MM/dd HH:mm', 'dateFormat', 'dd-MM-yyyy')) FROM test_to_csv_temporal |
| 70 | + |
| 71 | +-- Complex field types (arrays, maps, nested structs). `StructsToCsv.checkInputDataTypes` accepts |
| 72 | +-- them and `CometBatchKernelCodegen.isSupportedDataType` recurses into them, so they pass the |
| 73 | +-- plan-time gate and reach the runtime dispatch. That is the accepted-at-plan-time / |
| 74 | +-- rejected-at-runtime shape #5219 found for TIME types: `canHandle` greenlights the expression |
| 75 | +-- before the plan commits, so a runtime gap is an execute-time failure with no fallback left. |
| 76 | +-- |
| 77 | +-- Only non-nullness is asserted here, because the rendered value is not comparable on every |
| 78 | +-- supported Spark version. Spark 3.4 / 3.5's `UnivocityGenerator.makeConverter` has no branch for |
| 79 | +-- complex types and lands on `getter.get(ordinal, dataType).toString`, an identity string such as |
| 80 | +-- "org.apache.spark.sql.vectorized.ColumnarArray@1ada50f0" whose hash differs between any two |
| 81 | +-- evaluations and whose class differs between Spark's converter input (`ColumnarArray` / |
| 82 | +-- `UnsafeArrayData`) and the kernel's (`InputArray_*`). That generic `get` is exactly what |
| 83 | +-- `CometSpecializedGettersDispatch` implements for `CometInternalRow` / `CometArrayData`, so on |
| 84 | +-- those versions these queries are the coverage for it. Spark 4.0 added real array/map/struct |
| 85 | +-- converters, which render deterministically; `to_csv_nested.sql` asserts those values in full. |
| 86 | +statement |
| 87 | +CREATE TABLE test_to_csv_nested(s struct<i: int, arr: array<int>, m: map<string, int>, n: struct<x: int, y: string>>) USING parquet |
| 88 | + |
| 89 | +statement |
| 90 | +INSERT INTO test_to_csv_nested VALUES |
| 91 | + (named_struct('i', 1, 'arr', array(1, 2, 3), 'm', map('k', 10), 'n', named_struct('x', 5, 'y', 'z'))), |
| 92 | + (named_struct('i', NULL, 'arr', CAST(NULL AS array<int>), 'm', CAST(NULL AS map<string, int>), 'n', CAST(NULL AS struct<x: int, y: string>))), |
| 93 | + (CAST(NULL AS struct<i: int, arr: array<int>, m: map<string, int>, n: struct<x: int, y: string>>)) |
| 94 | + |
| 95 | +-- Struct column straight from the scan: the kernel reads it with getStruct, so the converter's |
| 96 | +-- per-field reads of the array / map / struct fields all land on `CometInternalRow`. The all-null |
| 97 | +-- and null-struct rows keep the assertion from collapsing to a constant true. |
| 98 | +query |
| 99 | +SELECT to_csv(s) IS NOT NULL FROM test_to_csv_nested |
| 100 | + |
| 101 | +-- named_struct over the complex fields: the kernel's array / map / struct readers produce the |
| 102 | +-- values that CreateNamedStruct stores into the row the converter then reads. |
| 103 | +query |
| 104 | +SELECT to_csv(named_struct('arr', s.arr, 'm', s.m, 'n', s.n)) IS NOT NULL FROM test_to_csv_nested |
| 105 | + |
| 106 | +-- Complex types nested inside complex types, so the converter recurses through the kernel's |
| 107 | +-- element getters rather than stopping at the first level. |
| 108 | +query |
| 109 | +SELECT to_csv(named_struct('aa', array(array(1, 2), array(3)), 'ms', map('k', named_struct('x', 1)))) IS NOT NULL FROM test_to_csv_nested |
0 commit comments