Skip to content

Commit 2907558

Browse files
authored
test: skip the Spark RocksDB state-store suites when Comet is enabled (apache#5987)
The Spark SQL matrix runs `sql/testOnly *`, so every RocksDB state-store suite under sql/core/.../streaming/state/ runs with Comet enabled. None of them exercise a Comet code path: CometExecRule and CometScanRule bail out for any streaming plan, and the rest drive StateStore/RocksDB APIs directly without building a query plan. Mix IgnoreCometSuite into those suites across all four diffs. StateStoreSuite, the HDFS-backed twin of RocksDBStateStoreSuite, is already skipped for the same reason on 4.0 and 4.1 (apache#4221). The mixin has to be listed before AlsoTestWithRocksDBFeatures / AlsoTestWithChangelogCheckpointingEnabled. Those traits register most of their tests through `super.test`, which only reaches the override when the override sits after them in the linearization. With IgnoreCometSuite last -- how RocksDBStateStoreIntegrationSuite was already patched on 4.1 -- only the plain `test(...)` registrations were skipped, one out of that suite's nine. RocksDBStateStoreLockHardeningSuite, RocksDBStateEncoderSuite and the Spark 3.4 RocksDBSuite are left running: they are plain SparkFunSuites that never start a SparkSession, so Comet is never loaded. Closes apache#5986
1 parent 4d931e3 commit 2907558

4 files changed

Lines changed: 401 additions & 3 deletions

File tree

‎dev/diffs/3.4.3.diff‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2533,6 +2533,62 @@ index d083cac48ff..3c11bcde807 100644
25332533

25342534
import testImplicits._
25352535

2536+
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/RocksDBStateStoreIntegrationSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/RocksDBStateStoreIntegrationSuite.scala
2537+
index dc505963b4d..79184d73023 100644
2538+
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/RocksDBStateStoreIntegrationSuite.scala
2539+
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/RocksDBStateStoreIntegrationSuite.scala
2540+
@@ -23,12 +23,16 @@ import scala.collection.JavaConverters
2541+
2542+
import org.scalatest.time.{Minute, Span}
2543+
2544+
+import org.apache.spark.sql.IgnoreCometSuite
2545+
import org.apache.spark.sql.execution.streaming.{MemoryStream, StreamingQueryWrapper}
2546+
import org.apache.spark.sql.functions.count
2547+
import org.apache.spark.sql.internal.SQLConf
2548+
import org.apache.spark.sql.streaming._
2549+
2550+
-class RocksDBStateStoreIntegrationSuite extends StreamTest {
2551+
+// Comet: skip this suite under Comet. Comet falls back to Spark for every streaming plan
2552+
+// (see CometExecRule and CometScanRule), so these tests exercise no Comet code path.
2553+
+// See https://github.com/apache/datafusion-comet/issues/5986
2554+
+class RocksDBStateStoreIntegrationSuite extends StreamTest with IgnoreCometSuite {
2555+
import testImplicits._
2556+
2557+
test("RocksDBStateStore") {
2558+
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/RocksDBStateStoreSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/RocksDBStateStoreSuite.scala
2559+
index a5993fdfe0d..9a378af944d 100644
2560+
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/RocksDBStateStoreSuite.scala
2561+
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/RocksDBStateStoreSuite.scala
2562+
@@ -22,7 +22,8 @@ import java.util.UUID
2563+
import scala.util.Random
2564+
2565+
import org.apache.hadoop.conf.Configuration
2566+
-import org.scalatest.BeforeAndAfter
2567+
+import org.scalactic.source.Position
2568+
+import org.scalatest.{BeforeAndAfter, Tag}
2569+
2570+
import org.apache.spark.SparkConf
2571+
import org.apache.spark.sql.LocalSparkSession.withSparkSession
2572+
@@ -38,6 +39,19 @@ import org.apache.spark.util.Utils
2573+
class RocksDBStateStoreSuite extends StateStoreSuiteBase[RocksDBStateStoreProvider]
2574+
with BeforeAndAfter {
2575+
2576+
+ // Comet: skip this suite under Comet. The tests target RocksDB state-store internals
2577+
+ // (StateStore get/put/commit), not SQL execution paths. This suite does not extend
2578+
+ // SQLTestUtils on 3.4, so IgnoreCometSuite cannot be mixed in and the check is inline.
2579+
+ // See https://github.com/apache/datafusion-comet/issues/5986
2580+
+ override protected def test(testName: String, testTags: Tag*)(testFun: => Any)
2581+
+ (implicit pos: Position): Unit = {
2582+
+ if (SparkSession.isCometEnabled) {
2583+
+ ignore(testName + " (disabled when Comet is on)", testTags: _*)(testFun)
2584+
+ } else {
2585+
+ super.test(testName, testTags: _*)(testFun)
2586+
+ }
2587+
+ }
2588+
+
2589+
before {
2590+
StateStore.stop()
2591+
require(!StateStore.isMaintenanceRunning)
25362592
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/sources/BucketedReadSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/sources/BucketedReadSuite.scala
25372593
index 266bb343526..50408a27a0a 100644
25382594
--- a/sql/core/src/test/scala/org/apache/spark/sql/sources/BucketedReadSuite.scala

‎dev/diffs/3.5.9.diff‎

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2529,6 +2529,92 @@ index d083cac48ff..3c11bcde807 100644
25292529

25302530
import testImplicits._
25312531

2532+
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/RocksDBStateStoreIntegrationSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/RocksDBStateStoreIntegrationSuite.scala
2533+
index 339d00058fc..93671c9ddd4 100644
2534+
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/RocksDBStateStoreIntegrationSuite.scala
2535+
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/RocksDBStateStoreIntegrationSuite.scala
2536+
@@ -23,6 +23,7 @@ import scala.collection.JavaConverters
2537+
2538+
import org.scalatest.time.{Minute, Span}
2539+
2540+
+import org.apache.spark.sql.IgnoreCometSuite
2541+
import org.apache.spark.sql.execution.streaming.{MemoryStream, StreamingQueryWrapper}
2542+
import org.apache.spark.sql.functions.count
2543+
import org.apache.spark.sql.internal.SQLConf
2544+
@@ -30,8 +31,14 @@ import org.apache.spark.sql.streaming._
2545+
import org.apache.spark.sql.streaming.OutputMode.Update
2546+
import org.apache.spark.util.Utils
2547+
2548+
+// Comet: skip this suite under Comet. Comet falls back to Spark for every streaming plan
2549+
+// (see CometExecRule and CometScanRule), so these tests exercise no Comet code path.
2550+
+// IgnoreCometSuite is listed before AlsoTestWithChangelogCheckpointingEnabled on purpose:
2551+
+// that trait registers most tests through `super.test`, which only reaches the override
2552+
+// when the override sits after it in the linearization.
2553+
+// See https://github.com/apache/datafusion-comet/issues/5986
2554+
class RocksDBStateStoreIntegrationSuite extends StreamTest
2555+
- with AlsoTestWithChangelogCheckpointingEnabled {
2556+
+ with IgnoreCometSuite with AlsoTestWithChangelogCheckpointingEnabled {
2557+
import testImplicits._
2558+
2559+
test("RocksDBStateStore") {
2560+
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/RocksDBStateStoreSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/RocksDBStateStoreSuite.scala
2561+
index d113085fd1c..a689e10cead 100644
2562+
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/RocksDBStateStoreSuite.scala
2563+
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/RocksDBStateStoreSuite.scala
2564+
@@ -26,6 +26,7 @@ import org.scalatest.BeforeAndAfter
2565+
2566+
import org.apache.spark.SparkConf
2567+
import org.apache.spark.io.CompressionCodec
2568+
+import org.apache.spark.sql.IgnoreCometSuite
2569+
import org.apache.spark.sql.LocalSparkSession.withSparkSession
2570+
import org.apache.spark.sql.SparkSession
2571+
import org.apache.spark.sql.catalyst.util.quietly
2572+
@@ -37,8 +38,16 @@ import org.apache.spark.tags.ExtendedSQLTest
2573+
import org.apache.spark.unsafe.Platform
2574+
import org.apache.spark.util.Utils
2575+
2576+
+// Comet: skip this suite under Comet. The tests target RocksDB state-store internals
2577+
+// (StateStore get/put/commit), not SQL execution paths. StateStoreSuite, the HDFS-backed
2578+
+// twin of this suite, is skipped for the same reason on Spark 4.0 and 4.1.
2579+
+// IgnoreCometSuite is listed before AlsoTestWithChangelogCheckpointingEnabled on purpose:
2580+
+// that trait registers most tests through `super.test`, which only reaches the override
2581+
+// when the override sits after it in the linearization.
2582+
+// See https://github.com/apache/datafusion-comet/issues/5986
2583+
@ExtendedSQLTest
2584+
class RocksDBStateStoreSuite extends StateStoreSuiteBase[RocksDBStateStoreProvider]
2585+
+ with IgnoreCometSuite
2586+
with AlsoTestWithChangelogCheckpointingEnabled
2587+
with SharedSparkSession
2588+
with BeforeAndAfter {
2589+
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/RocksDBSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/RocksDBSuite.scala
2590+
index 973c1e0cb3b..d8e86384e4a 100644
2591+
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/RocksDBSuite.scala
2592+
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/RocksDBSuite.scala
2593+
@@ -32,6 +32,7 @@ import org.apache.hadoop.fs.Path
2594+
import org.scalactic.source.Position
2595+
import org.scalatest.Tag
2596+
2597+
+import org.apache.spark.sql.IgnoreCometSuite
2598+
import org.apache.spark.sql.catalyst.util.quietly
2599+
import org.apache.spark.sql.execution.streaming.{CreateAtomicTestManager, FileSystemBasedCheckpointFileManager}
2600+
import org.apache.spark.sql.execution.streaming.CheckpointFileManager.{CancellableFSDataOutputStream, RenameBasedFSDataOutputStream}
2601+
@@ -119,8 +120,15 @@ trait AlsoTestWithChangelogCheckpointingEnabled
2602+
}
2603+
}
2604+
2605+
+// Comet: skip this suite under Comet. The tests target RocksDB internals (open, checkpoint,
2606+
+// changelog replay), not SQL execution paths, so nothing here runs through Comet.
2607+
+// IgnoreCometSuite is listed before AlsoTestWithChangelogCheckpointingEnabled on purpose:
2608+
+// that trait registers most tests through `super.test`, which only reaches the override
2609+
+// when the override sits after it in the linearization.
2610+
+// See https://github.com/apache/datafusion-comet/issues/5986
2611+
@SlowSQLTest
2612+
-class RocksDBSuite extends AlsoTestWithChangelogCheckpointingEnabled with SharedSparkSession {
2613+
+class RocksDBSuite extends IgnoreCometSuite with AlsoTestWithChangelogCheckpointingEnabled
2614+
+ with SharedSparkSession {
2615+
2616+
sqlConf.setConf(SQLConf.STATE_STORE_PROVIDER_CLASS, classOf[RocksDBStateStoreProvider].getName)
2617+
25322618
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/sources/BucketedReadSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/sources/BucketedReadSuite.scala
25332619
index 746f289c393..f2e0d1c93e5 100644
25342620
--- a/sql/core/src/test/scala/org/apache/spark/sql/sources/BucketedReadSuite.scala

‎dev/diffs/4.0.4.diff‎

Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3187,6 +3187,122 @@ index 7838e62013d..8fa09652921 100644
31873187

31883188
import testImplicits._
31893189

3190+
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/RocksDBStateStoreCheckpointFormatV2Suite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/RocksDBStateStoreCheckpointFormatV2Suite.scala
3191+
index ac519e713c6..38363d031be 100644
3192+
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/RocksDBStateStoreCheckpointFormatV2Suite.scala
3193+
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/RocksDBStateStoreCheckpointFormatV2Suite.scala
3194+
@@ -25,7 +25,7 @@ import org.apache.hadoop.fs.Path
3195+
import org.scalatest.Tag
3196+
3197+
import org.apache.spark.{SparkContext, SparkException, TaskContext}
3198+
-import org.apache.spark.sql.{DataFrame, ForeachWriter}
3199+
+import org.apache.spark.sql.{DataFrame, ForeachWriter, IgnoreCometSuite}
3200+
import org.apache.spark.sql.catalyst.expressions.UnsafeRow
3201+
import org.apache.spark.sql.execution.streaming.{CommitLog, MemoryStream, StreamExecution}
3202+
import org.apache.spark.sql.execution.streaming.state.StateStoreTestsHelper
3203+
@@ -190,8 +190,14 @@ class CkptIdCollectingStateStoreProviderWrapper extends StateStoreProvider {
3204+
innerProvider.supportedCustomMetrics
3205+
}
3206+
3207+
+// Comet: skip this suite under Comet. Comet falls back to Spark for every streaming plan
3208+
+// (see CometExecRule and CometScanRule), so these tests exercise no Comet code path.
3209+
+// IgnoreCometSuite is listed before AlsoTestWithRocksDBFeatures on purpose: that trait
3210+
+// registers most tests through `super.test`, which only reaches the override when the
3211+
+// override sits after it in the linearization.
3212+
+// See https://github.com/apache/datafusion-comet/issues/5986
3213+
class RocksDBStateStoreCheckpointFormatV2Suite extends StreamTest
3214+
- with AlsoTestWithRocksDBFeatures {
3215+
+ with IgnoreCometSuite with AlsoTestWithRocksDBFeatures {
3216+
import testImplicits._
3217+
3218+
val providerClassName = classOf[CkptIdCollectingStateStoreProviderWrapper].getCanonicalName
3219+
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/RocksDBStateStoreIntegrationSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/RocksDBStateStoreIntegrationSuite.scala
3220+
index 1f4fd7f7957..1ebea83fa08 100644
3221+
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/RocksDBStateStoreIntegrationSuite.scala
3222+
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/RocksDBStateStoreIntegrationSuite.scala
3223+
@@ -23,6 +23,7 @@ import scala.jdk.CollectionConverters.SetHasAsScala
3224+
3225+
import org.scalatest.time.{Minute, Span}
3226+
3227+
+import org.apache.spark.sql.IgnoreCometSuite
3228+
import org.apache.spark.sql.execution.streaming.{MemoryStream, StreamingQueryWrapper}
3229+
import org.apache.spark.sql.functions.count
3230+
import org.apache.spark.sql.internal.SQLConf
3231+
@@ -30,8 +31,14 @@ import org.apache.spark.sql.streaming._
3232+
import org.apache.spark.sql.streaming.OutputMode.Update
3233+
import org.apache.spark.util.Utils
3234+
3235+
+// Comet: skip this suite under Comet. Comet falls back to Spark for every streaming plan
3236+
+// (see CometExecRule and CometScanRule), so these tests exercise no Comet code path.
3237+
+// IgnoreCometSuite is listed before AlsoTestWithRocksDBFeatures on purpose: that trait
3238+
+// registers most tests through `super.test`, which only reaches the override when the
3239+
+// override sits after it in the linearization.
3240+
+// See https://github.com/apache/datafusion-comet/issues/5986
3241+
class RocksDBStateStoreIntegrationSuite extends StreamTest
3242+
- with AlsoTestWithRocksDBFeatures {
3243+
+ with IgnoreCometSuite with AlsoTestWithRocksDBFeatures {
3244+
import testImplicits._
3245+
3246+
testWithColumnFamilies("RocksDBStateStore",
3247+
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/RocksDBStateStoreSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/RocksDBStateStoreSuite.scala
3248+
index d4a2f333574..d6c124a5094 100644
3249+
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/RocksDBStateStoreSuite.scala
3250+
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/RocksDBStateStoreSuite.scala
3251+
@@ -28,6 +28,7 @@ import org.scalatest.BeforeAndAfter
3252+
3253+
import org.apache.spark.{SparkConf, SparkUnsupportedOperationException}
3254+
import org.apache.spark.io.CompressionCodec
3255+
+import org.apache.spark.sql.IgnoreCometSuite
3256+
import org.apache.spark.sql.LocalSparkSession.withSparkSession
3257+
import org.apache.spark.sql.SparkSession
3258+
import org.apache.spark.sql.catalyst.InternalRow
3259+
@@ -42,8 +43,16 @@ import org.apache.spark.unsafe.Platform
3260+
import org.apache.spark.unsafe.types.UTF8String
3261+
import org.apache.spark.util.Utils
3262+
3263+
+// Comet: skip this suite under Comet. The tests target RocksDB state-store internals
3264+
+// (StateStore get/put/commit), not SQL execution paths. StateStoreSuite, the HDFS-backed
3265+
+// twin of this suite, is skipped for the same reason.
3266+
+// IgnoreCometSuite is listed before the AlsoTestWith* traits on purpose: those traits
3267+
+// register most tests through `super.test`, which only reaches the override when the
3268+
+// override sits after them in the linearization.
3269+
+// See https://github.com/apache/datafusion-comet/issues/5986
3270+
@ExtendedSQLTest
3271+
class RocksDBStateStoreSuite extends StateStoreSuiteBase[RocksDBStateStoreProvider]
3272+
+ with IgnoreCometSuite
3273+
with AlsoTestWithEncodingTypes
3274+
with AlsoTestWithRocksDBFeatures
3275+
with SharedSparkSession
3276+
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/RocksDBSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/RocksDBSuite.scala
3277+
index 10f14d5655f..3c687cdcef6 100644
3278+
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/RocksDBSuite.scala
3279+
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/RocksDBSuite.scala
3280+
@@ -39,6 +39,7 @@ import org.scalatest.Tag
3281+
import org.apache.spark.{SparkConf, SparkException, SparkFunSuite, TaskContext}
3282+
import org.apache.spark.internal.Logging
3283+
import org.apache.spark.io.CompressionCodec
3284+
+import org.apache.spark.sql.IgnoreCometSuite
3285+
import org.apache.spark.sql.catalyst.InternalRow
3286+
import org.apache.spark.sql.catalyst.expressions.UnsafeProjection
3287+
import org.apache.spark.sql.catalyst.util.quietly
3288+
@@ -705,9 +706,15 @@ class RocksDBStateEncoderSuite extends SparkFunSuite {
3289+
}
3290+
}
3291+
3292+
+// Comet: skip this suite under Comet. The tests target RocksDB internals (open, checkpoint,
3293+
+// changelog replay), not SQL execution paths, so nothing here runs through Comet.
3294+
+// IgnoreCometSuite is listed before AlsoTestWithRocksDBFeatures on purpose: that trait
3295+
+// registers most tests through `super.test`, which only reaches the override when the
3296+
+// override sits after it in the linearization.
3297+
+// See https://github.com/apache/datafusion-comet/issues/5986
3298+
@SlowSQLTest
3299+
-class RocksDBSuite extends AlsoTestWithRocksDBFeatures with SharedSparkSession
3300+
- with PrivateMethodTester {
3301+
+class RocksDBSuite extends IgnoreCometSuite with AlsoTestWithRocksDBFeatures
3302+
+ with SharedSparkSession with PrivateMethodTester {
3303+
3304+
override protected def sparkConf: SparkConf = {
3305+
super.sparkConf
31903306
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/StateStoreSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/StateStoreSuite.scala
31913307
index 89f22186f7e..425233f00b2 100644
31923308
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/StateStoreSuite.scala

0 commit comments

Comments
 (0)