|
| 1 | +-- Licensed to the Apache Software Foundation (ASF) under one |
| 2 | +-- or more contributor license agreements. See the NOTICE file |
| 3 | +-- distributed with this work for additional information |
| 4 | +-- regarding copyright ownership. The ASF licenses this file |
| 5 | +-- to you under the Apache License, Version 2.0 (the |
| 6 | +-- "License"); you may not use this file except in compliance |
| 7 | +-- with the License. You may obtain a copy of the License at |
| 8 | +-- |
| 9 | +-- http://www.apache.org/licenses/LICENSE-2.0 |
| 10 | +-- |
| 11 | +-- Unless required by applicable law or agreed to in writing, |
| 12 | +-- software distributed under the License is distributed on an |
| 13 | +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| 14 | +-- KIND, either express or implied. See the License for the |
| 15 | +-- specific language governing permissions and limitations |
| 16 | +-- under the License. |
| 17 | + |
| 18 | +-- Iceberg's years, months, days, and hours system functions on timestamps just after a pre-1970 |
| 19 | +-- unit boundary (https://github.com/apache/datafusion-comet/issues/6426). Iceberg's DateTimeUtil |
| 20 | +-- places a pre-1970 timestamp whose microsecond of second is 999999 by the second before it, so |
| 21 | +-- right after a boundary it gets the unit before: 1969-01-01 00:00:00.999999 is in year -2, |
| 22 | +-- month -13, day 1968-12-31, and hour -8761, where a floor gives -1, -12, 1969-01-01, and -8760. |
| 23 | +-- Each query also runs with Comet off, so the expected values come from Iceberg's own functions. |
| 24 | +-- CometIcebergSystemFunctionSuite writes the same timestamps with the native Iceberg writer. |
| 25 | + |
| 26 | +-- No Iceberg spark-runtime is published for Spark 4.2 yet, and the 4.0 runtime the build reuses |
| 27 | +-- is binary-incompatible with it, which is also why CometIcebergTestBase reports Iceberg as |
| 28 | +-- unavailable there. See https://github.com/apache/datafusion-comet/issues/4969. |
| 29 | +-- MaxSparkVersion: 4.1 |
| 30 | + |
| 31 | +-- Config: spark.sql.catalog.test_cat=org.apache.iceberg.spark.SparkCatalog |
| 32 | +-- Config: spark.sql.catalog.test_cat.type=hadoop |
| 33 | +-- Config: spark.sql.catalog.test_cat.warehouse=/tmp/comet-iceberg-sql-test |
| 34 | +-- UTC, so that the TIMESTAMP values sit on the same boundaries as the TIMESTAMP_NTZ ones. |
| 35 | +-- Config: spark.sql.session.timeZone=UTC |
| 36 | +-- Config: spark.sql.parquet.outputTimestampType=TIMESTAMP_MICROS |
| 37 | + |
| 38 | +-- A parquet table rather than an Iceberg one, so that no scan absorbs a filter. Rows 1 to 4 lie |
| 39 | +-- 999999 microseconds after a year, a month, a day, and an hour boundary (a year boundary is also |
| 40 | +-- a month, day, and hour boundary, and so on). Row 5 sits inside the hour Iceberg gives row 4, |
| 41 | +-- and row 6 inside the day it gives row 1, both away from any boundary. Row 7 is after the epoch, |
| 42 | +-- where Iceberg floors. |
| 43 | +statement |
| 44 | +CREATE TABLE iceberg_pre_epoch (id INT, ts TIMESTAMP, ntz TIMESTAMP_NTZ) USING parquet |
| 45 | + |
| 46 | +statement |
| 47 | +INSERT INTO iceberg_pre_epoch VALUES |
| 48 | + (1, TIMESTAMP '1969-01-01 00:00:00.999999', TIMESTAMP_NTZ '1969-01-01 00:00:00.999999'), |
| 49 | + (2, TIMESTAMP '1969-12-01 00:00:00.999999', TIMESTAMP_NTZ '1969-12-01 00:00:00.999999'), |
| 50 | + (3, TIMESTAMP '1969-12-31 00:00:00.999999', TIMESTAMP_NTZ '1969-12-31 00:00:00.999999'), |
| 51 | + (4, TIMESTAMP '1969-12-31 23:00:00.999999', TIMESTAMP_NTZ '1969-12-31 23:00:00.999999'), |
| 52 | + (5, TIMESTAMP '1969-12-31 22:30:00', TIMESTAMP_NTZ '1969-12-31 22:30:00'), |
| 53 | + (6, TIMESTAMP '1968-12-31 12:00:00', TIMESTAMP_NTZ '1968-12-31 12:00:00'), |
| 54 | + (7, TIMESTAMP '1970-01-01 01:00:00.999999', TIMESTAMP_NTZ '1970-01-01 01:00:00.999999'), |
| 55 | + (8, NULL, NULL) |
| 56 | + |
| 57 | +-- Every query names staticinvoke, the node Spark plans an Iceberg system function call as, in |
| 58 | +-- expect_native. A call Comet did not lower to its own kernel would run Iceberg's Java code through |
| 59 | +-- the codegen dispatcher and match Spark whatever the kernel returns. |
| 60 | +query expect_native(staticinvoke) |
| 61 | +SELECT id, |
| 62 | + test_cat.system.years(ts), test_cat.system.months(ts), |
| 63 | + test_cat.system.days(ts), test_cat.system.hours(ts), |
| 64 | + test_cat.system.years(ntz), test_cat.system.months(ntz), |
| 65 | + test_cat.system.days(ntz), test_cat.system.hours(ntz) |
| 66 | +FROM iceberg_pre_epoch |
| 67 | + |
| 68 | +-- Rows 1 and 6. A floor would leave out row 1 in these three. |
| 69 | +query expect_native(staticinvoke) |
| 70 | +SELECT id FROM iceberg_pre_epoch WHERE test_cat.system.years(ts) = -2 |
| 71 | + |
| 72 | +query expect_native(staticinvoke) |
| 73 | +SELECT id FROM iceberg_pre_epoch WHERE test_cat.system.months(ts) = -13 |
| 74 | + |
| 75 | +query expect_native(staticinvoke) |
| 76 | +SELECT id FROM iceberg_pre_epoch WHERE test_cat.system.days(ts) = DATE '1968-12-31' |
| 77 | + |
| 78 | +-- Rows 4 and 5. A floor would leave out row 4. |
| 79 | +query expect_native(staticinvoke) |
| 80 | +SELECT id FROM iceberg_pre_epoch WHERE test_cat.system.hours(ts) = -2 |
0 commit comments