Skip to content

Commit c5d00f7

Browse files
committed
test: share the pre-epoch Iceberg fixture through CometIcebergTestBase
CometIcebergSystemFunctionExtensionsSuite repeated the catalog setup and its own copy of the pre_epoch table. The Hadoop catalog, the pre-1970 timestamps and the parquet table now live in CometIcebergTestBase, which both suites use. The trait takes a CometTestBase self-type for withSQLConf, withTable and sql, and every suite that mixes it in already extends CometTestBase. The catalog helper is called withHadoopCatalog so that it does not overload CometIcebergWriteActionSuite's private withIcebergCatalog. The extensions suite's four predicates now run on the shared rows. The table drops the TIMESTAMP_NTZ column, which only the projection test that moved to the SQL file read.
1 parent ccbe740 commit c5d00f7

3 files changed

Lines changed: 72 additions & 81 deletions

File tree

‎spark/src/test/scala/org/apache/comet/CometIcebergSystemFunctionExtensionsSuite.scala‎

Lines changed: 18 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,6 @@ import org.apache.spark.SparkConf
2323
import org.apache.spark.sql.CometTestBase
2424
import org.apache.spark.sql.catalyst.expressions.ApplyFunctionExpression
2525
import org.apache.spark.sql.catalyst.plans.logical.Filter
26-
import org.apache.spark.sql.internal.SQLConf
2726

2827
/**
2928
* Iceberg's system functions once Iceberg's SQL extensions have rewritten them.
@@ -43,37 +42,24 @@ class CometIcebergSystemFunctionExtensionsSuite extends CometTestBase with Comet
4342

4443
test("rewritten temporal filters match Iceberg on pre-1970 timestamps ending in .999999") {
4544
assume(icebergAvailable, "Iceberg not available in classpath")
46-
withTempIcebergDir { warehouseDir =>
47-
withSQLConf(
48-
"spark.sql.catalog.ice" -> "org.apache.iceberg.spark.SparkCatalog",
49-
"spark.sql.catalog.ice.type" -> "hadoop",
50-
"spark.sql.catalog.ice.warehouse" -> warehouseDir.getAbsolutePath,
51-
SQLConf.SESSION_LOCAL_TIMEZONE.key -> "UTC",
52-
SQLConf.PARQUET_OUTPUT_TIMESTAMP_TYPE.key -> "TIMESTAMP_MICROS") {
53-
withTable("pre_epoch") {
54-
// A parquet table, so that no scan absorbs the filter and Comet evaluates it. Iceberg
55-
// places a pre-1970 timestamp ending in .999999 by the second before it: the first row
56-
// is in hour -8761, day 1968-12-31, month -13, and year -2, and the second in hour -2.
57-
sql("CREATE TABLE pre_epoch (id INT, ts TIMESTAMP) USING parquet")
58-
sql("""INSERT INTO pre_epoch VALUES
59-
(1, TIMESTAMP '1969-01-01 00:00:00.999999'),
60-
(2, TIMESTAMP '1969-12-31 23:00:00.999999'),
61-
(3, TIMESTAMP '1969-12-31 22:30:00'),
62-
(4, TIMESTAMP '1968-12-31 12:00:00')""")
63-
Seq(
64-
"ice.system.hours(ts) = -2",
65-
"ice.system.days(ts) = DATE '1968-12-31'",
66-
"ice.system.months(ts) = -13",
67-
"ice.system.years(ts) = -2").foreach { predicate =>
68-
val df = sql(s"SELECT id FROM pre_epoch WHERE $predicate")
69-
val plan = df.queryExecution.optimizedPlan
70-
assert(
71-
plan
72-
.collect { case filter: Filter => filter.condition }
73-
.exists(_.find(_.isInstanceOf[ApplyFunctionExpression]).isDefined),
74-
s"expected Iceberg's extensions to rewrite $predicate:\n$plan")
75-
checkSparkAnswerAndOperator(df)
76-
}
45+
withHadoopCatalog("ice") {
46+
withPreEpochTable {
47+
// Iceberg places a pre-1970 timestamp ending in .999999 by the second before it: row 1 is
48+
// in hour -8761, day 1968-12-31, month -13, and year -2, and row 4 in hour -2. So each
49+
// predicate matches one of them besides a row away from any boundary (6, or 5 for hours).
50+
Seq(
51+
"ice.system.hours(ts) = -2",
52+
"ice.system.days(ts) = DATE '1968-12-31'",
53+
"ice.system.months(ts) = -13",
54+
"ice.system.years(ts) = -2").foreach { predicate =>
55+
val df = sql(s"SELECT id FROM pre_epoch WHERE $predicate")
56+
val plan = df.queryExecution.optimizedPlan
57+
assert(
58+
plan
59+
.collect { case filter: Filter => filter.condition }
60+
.exists(_.find(_.isInstanceOf[ApplyFunctionExpression]).isDefined),
61+
s"expected Iceberg's extensions to rewrite $predicate:\n$plan")
62+
checkSparkAnswerAndOperator(df)
7763
}
7864
}
7965
}

‎spark/src/test/scala/org/apache/comet/CometIcebergSystemFunctionSuite.scala‎

Lines changed: 4 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -267,8 +267,9 @@ class CometIcebergSystemFunctionSuite
267267
// The sort in front of the write runs the native kernels and the native writer computes each
268268
// row's partition value, so both have to follow Iceberg for the table to hold the partitions
269269
// iceberg-java would have written. A spec takes one time transform per source column, hence a
270-
// column per transform.
271-
withIcebergCatalog {
270+
// column per transform. The kernels' results in projections and filters are checked by
271+
// sql-tests/iceberg/temporal_functions_pre_epoch.sql.
272+
withHadoopCatalog(catalog) {
272273
withPreEpochTable {
273274
val table = s"$catalog.db.pre_epoch_partitions"
274275
sql(s"""
@@ -459,52 +460,9 @@ class CometIcebergSystemFunctionSuite
459460
}
460461
}
461462

462-
/** Runs `f` with the Iceberg catalog registered. */
463-
private def withIcebergCatalog(f: => Unit): Unit = withTempIcebergDir { warehouseDir =>
464-
withSQLConf(
465-
s"spark.sql.catalog.$catalog" -> "org.apache.iceberg.spark.SparkCatalog",
466-
s"spark.sql.catalog.$catalog.type" -> "hadoop",
467-
s"spark.sql.catalog.$catalog.warehouse" -> warehouseDir.getAbsolutePath)(f)
468-
}
469-
470463
/** Runs `f` with the Iceberg catalog registered and the source parquet table in scope. */
471464
private def withSourceTable(f: => Unit): Unit =
472-
withIcebergCatalog(withParquetTable(sourcePath, source)(f))
473-
474-
/**
475-
* Timestamps just after pre-1970 unit boundaries, where Iceberg does not floor. Its
476-
* `DateTimeUtil` places a pre-1970 timestamp whose microsecond of second is 999999 by the
477-
* second before it, so right after a boundary it gets the unit before: 1969-01-01
478-
* 00:00:00.999999 is in year -2, month -13, day 1968-12-31, and hour -8761, where a floor gives
479-
* -1, -12, 1969-01-01, and -8760.
480-
*/
481-
private val preEpochTimestamps = Seq(
482-
"1969-01-01 00:00:00.999999", // a year, month, day, and hour boundary
483-
"1969-12-01 00:00:00.999999", // a month, day, and hour boundary
484-
"1969-12-31 00:00:00.999999", // a day and hour boundary
485-
"1969-12-31 23:00:00.999999", // an hour boundary
486-
"1969-12-31 22:30:00",
487-
"1968-12-31 12:00:00",
488-
// After the epoch, where Iceberg floors.
489-
"1970-01-01 01:00:00.999999")
490-
491-
/**
492-
* Runs `f` with a parquet table `pre_epoch (id, ts, ntz)` holding `preEpochTimestamps` as both
493-
* timestamp types. The session timezone is UTC, so that the `TIMESTAMP` values sit on the same
494-
* boundaries as the `TIMESTAMP_NTZ` ones.
495-
*/
496-
private def withPreEpochTable(f: => Unit): Unit = withSQLConf(
497-
SQLConf.SESSION_LOCAL_TIMEZONE.key -> "UTC",
498-
SQLConf.PARQUET_OUTPUT_TIMESTAMP_TYPE.key -> "TIMESTAMP_MICROS") {
499-
withTable("pre_epoch") {
500-
sql("CREATE TABLE pre_epoch (id INT, ts TIMESTAMP, ntz TIMESTAMP_NTZ) USING parquet")
501-
val rows = preEpochTimestamps.zipWithIndex.map { case (timestamp, i) =>
502-
s"(${i + 1}, TIMESTAMP '$timestamp', TIMESTAMP_NTZ '$timestamp')"
503-
}
504-
sql(s"INSERT INTO pre_epoch VALUES ${rows.mkString(", ")}")
505-
f
506-
}
507-
}
465+
withHadoopCatalog(catalog)(withParquetTable(sourcePath, source)(f))
508466

509467
private val sourceSchema = StructType(
510468
Seq(

‎spark/src/test/scala/org/apache/comet/CometIcebergTestBase.scala‎

Lines changed: 50 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -25,18 +25,20 @@ import java.nio.file.Files
2525
import scala.collection.mutable
2626

2727
import org.apache.spark.CometListenerBusUtils
28-
import org.apache.spark.sql.SparkSession
28+
import org.apache.spark.sql.{CometTestBase, SparkSession}
2929
import org.apache.spark.sql.connector.catalog.{Identifier, TableCatalog}
3030
import org.apache.spark.sql.execution.{QueryExecution, SparkPlan}
31+
import org.apache.spark.sql.internal.SQLConf
3132
import org.apache.spark.sql.util.QueryExecutionListener
3233

3334
import org.apache.comet.CometSparkSessionExtensions.isSpark42Plus
3435
import org.apache.comet.iceberg.IcebergReflection
3536

3637
/**
37-
* Shared fixtures for Iceberg-backed test suites: classpath probe and per-test temp directory.
38+
* Shared fixtures for Iceberg-backed test suites: classpath probe, per-test temp directory, a
39+
* Hadoop catalog, and a table of pre-1970 timestamps. Mix in alongside `CometTestBase`.
3840
*/
39-
trait CometIcebergTestBase {
41+
trait CometIcebergTestBase { this: CometTestBase =>
4042

4143
// No Iceberg spark-runtime is published for Spark 4.2 yet, so the build reuses the 4.0 runtime.
4244
// That jar is binary-incompatible with Spark 4.2, whose `connector.catalog.View` is a class
@@ -137,6 +139,51 @@ trait CometIcebergTestBase {
137139
file.delete()
138140
}
139141

142+
/** Runs `f` with an Iceberg `hadoop` catalog registered as `catalog`, in a temp warehouse. */
143+
protected def withHadoopCatalog(catalog: String)(f: => Unit): Unit =
144+
withTempIcebergDir { warehouseDir =>
145+
withSQLConf(
146+
s"spark.sql.catalog.$catalog" -> "org.apache.iceberg.spark.SparkCatalog",
147+
s"spark.sql.catalog.$catalog.type" -> "hadoop",
148+
s"spark.sql.catalog.$catalog.warehouse" -> warehouseDir.getAbsolutePath)(f)
149+
}
150+
151+
/**
152+
* Timestamps just after pre-1970 unit boundaries, where Iceberg does not floor. Its
153+
* `DateTimeUtil` places a pre-1970 timestamp whose microsecond of second is 999999 by the
154+
* second before it, so right after a boundary it gets the unit before: 1969-01-01
155+
* 00:00:00.999999 is in year -2, month -13, day 1968-12-31, and hour -8761, where a floor gives
156+
* -1, -12, 1969-01-01, and -8760. `sql-tests/iceberg/temporal_functions_pre_epoch.sql` runs the
157+
* system functions over the same timestamps in projections and filters.
158+
*/
159+
protected val preEpochTimestamps: Seq[String] = Seq(
160+
"1969-01-01 00:00:00.999999", // a year, month, day, and hour boundary
161+
"1969-12-01 00:00:00.999999", // a month, day, and hour boundary
162+
"1969-12-31 00:00:00.999999", // a day and hour boundary
163+
"1969-12-31 23:00:00.999999", // an hour boundary
164+
"1969-12-31 22:30:00",
165+
"1968-12-31 12:00:00",
166+
// After the epoch, where Iceberg floors.
167+
"1970-01-01 01:00:00.999999")
168+
169+
/**
170+
* Runs `f` with a parquet table `pre_epoch (id, ts)` holding `preEpochTimestamps`, numbered
171+
* from 1. A parquet table, so that no scan absorbs a filter on `ts` and Comet evaluates it. The
172+
* session timezone is UTC, so that the values sit on the unit boundaries.
173+
*/
174+
protected def withPreEpochTable(f: => Unit): Unit = withSQLConf(
175+
SQLConf.SESSION_LOCAL_TIMEZONE.key -> "UTC",
176+
SQLConf.PARQUET_OUTPUT_TIMESTAMP_TYPE.key -> "TIMESTAMP_MICROS") {
177+
withTable("pre_epoch") {
178+
sql("CREATE TABLE pre_epoch (id INT, ts TIMESTAMP) USING parquet")
179+
val rows = preEpochTimestamps.zipWithIndex.map { case (timestamp, i) =>
180+
s"(${i + 1}, TIMESTAMP '$timestamp')"
181+
}
182+
sql(s"INSERT INTO pre_epoch VALUES ${rows.mkString(", ")}")
183+
f
184+
}
185+
}
186+
140187
/**
141188
* The executed plan of every query that ran while `action` ran. Queries that failed are
142189
* included only when `includeFailures` is set, which is what an action expected to abort needs.

0 commit comments

Comments
 (0)