Skip to content

Commit e0c7963

Browse files
committed
test: run the sketch SQL files instead of excluding them globally
`ignoreList` registers a ScalaTest `ignore` before execution, so listing `thetasketch.sql` and `tuplesketch.sql` there skipped both files even with `ENABLE_COMET=false`, dropping 364 `SELECT`s from the Spark baseline. The recorded rationale was also wrong. The sketch aggregates run on Spark's `ObjectHashAggregate` and are never converted; what fails is the native scan reading the `t_string_collation` fixture, which contains invalid UTF-8 (`CAST(X'C1' AS STRING)`, `CAST(X'80' AS STRING)`): Parquet error: Arrow: Parquet argument error: Parquet error: encountered non UTF-8 data That is apache#4121, and it has nothing to do with collation: the uncollated query at tuplesketch.sql:825 fails identically. Switch both files to the `--SET spark.comet.enabled = false` header already used for `hll.sql`, which fails on the same fixture for the same reason. Every query then runs and is verified against the golden file in both the baseline and the Comet job, rather than the whole file being skipped.
1 parent 4f23ea0 commit e0c7963

1 file changed

Lines changed: 30 additions & 12 deletions

File tree

‎dev/diffs/4.2.0.diff‎

Lines changed: 30 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -349,6 +349,32 @@ index 7fbb1c12924..a612d40813a 100644
349349
--SET spark.sql.autoBroadcastJoinThreshold=-1
350350

351351
-- Test sort operator with codegen on and off.
352+
diff --git a/sql/core/src/test/resources/sql-tests/inputs/thetasketch.sql b/sql/core/src/test/resources/sql-tests/inputs/thetasketch.sql
353+
index 4782d2017f2..3c7fe46dcb3 100644
354+
--- a/sql/core/src/test/resources/sql-tests/inputs/thetasketch.sql
355+
+++ b/sql/core/src/test/resources/sql-tests/inputs/thetasketch.sql
356+
@@ -1,3 +1,8 @@
357+
+-- TODO(https://github.com/apache/datafusion-comet/issues/4121)
358+
+-- Comet's native scan rejects invalid UTF-8 byte sequences inserted into the
359+
+-- string test table, which Spark allows.
360+
+--SET spark.comet.enabled = false
361+
+
362+
-- Positive test cases
363+
-- Create tables with two columns for each data type
364+
365+
diff --git a/sql/core/src/test/resources/sql-tests/inputs/tuplesketch.sql b/sql/core/src/test/resources/sql-tests/inputs/tuplesketch.sql
366+
index fa83a54c830..1a9862bb565 100644
367+
--- a/sql/core/src/test/resources/sql-tests/inputs/tuplesketch.sql
368+
+++ b/sql/core/src/test/resources/sql-tests/inputs/tuplesketch.sql
369+
@@ -1,3 +1,8 @@
370+
+-- TODO(https://github.com/apache/datafusion-comet/issues/4121)
371+
+-- Comet's native scan rejects invalid UTF-8 byte sequences inserted into the
372+
+-- string test table, which Spark allows.
373+
+--SET spark.comet.enabled = false
374+
+
375+
-- Positive test cases
376+
-- Create tables with key-value pairs for tuple sketches
377+
352378
diff --git a/sql/core/src/test/resources/sql-tests/inputs/view-schema-binding-config.sql b/sql/core/src/test/resources/sql-tests/inputs/view-schema-binding-config.sql
353379
index e803254ea64..74db78aee38 100644
354380
--- a/sql/core/src/test/resources/sql-tests/inputs/view-schema-binding-config.sql
@@ -1339,10 +1365,10 @@ index da6f6aca2ad..c02b7c99490 100644
13391365
verifyCallCount(
13401366
df.selectExpr("testUdf(a + 1) + testUdf(1 + a)", "testUdf(a + 1)"), Row(4, 2), 1)
13411367
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala
1342-
index 395cb67f441..22296439d2e 100644
1368+
index 395cb67f441..1e46737c2e2 100644
13431369
--- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala
13441370
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala
1345-
@@ -188,7 +188,20 @@ class SQLQueryTestSuite extends SharedSparkSession with SQLHelper
1371+
@@ -188,7 +188,12 @@ class SQLQueryTestSuite extends SharedSparkSession with SQLHelper
13461372
if (TestUtils.testCommandAvailable("/bin/bash")) Nil else Set("transform.sql")
13471373
/** List of test cases to ignore, in lower cases. */
13481374
protected def ignoreList: Set[String] = Set(
@@ -1352,19 +1378,11 @@ index 395cb67f441..22296439d2e 100644
13521378
+ // running with high parallelism. Tracked for restoration once Comet
13531379
+ // produces stable ordering for these queries.
13541380
+ "replacing-missing-expression-with-alias.sql",
1355-
+ "in-set-operations.sql",
1356-
+ // Comet: theta_sketch_estimate(theta_sketch_agg(...)) over collated
1357-
+ // strings reads sketch binary as UTF-8 and fails. Skip until Comet
1358-
+ // adds proper handling or falls back for theta-sketch on collation.
1359-
+ "thetasketch.sql",
1360-
+ // Comet: same collation problem as thetasketch.sql, for
1361-
+ // tuple_sketch_estimate_double(tuple_sketch_agg_double(...)) over a collated
1362-
+ // string column (Spark 4.2 added these functions).
1363-
+ "tuplesketch.sql"
1381+
+ "in-set-operations.sql"
13641382
) ++ otherIgnoreList
13651383
/** List of test cases that require TPCDS table schemas to be loaded. */
13661384
private def requireTPCDSCases: Seq[String] = Seq("pipe-operators.sql")
1367-
@@ -829,9 +842,24 @@ class SQLQueryTestSuite extends SharedSparkSession with SQLHelper
1385+
@@ -829,9 +834,24 @@ class SQLQueryTestSuite extends SharedSparkSession with SQLHelper
13681386
s"Schema did not match for query #$i\n${expected.sql}: $output") {
13691387
output.schema
13701388
}

0 commit comments

Comments
 (0)