Skip to content

Commit 3e4b6a0

Browse files
schenksjclaude
andcommitted
test: add Delta Lake regression suite mirroring the Iceberg pattern
Clones Delta at a matching tag, applies a version-specific diff that wires Comet into Delta's test SparkSession, and runs Delta's own test suite with Comet enabled. Complements Comet's existing CometDeltaNativeSuite by exercising Delta's own test coverage against Comet. Each diff patches: - build.sbt: adds Comet as a test dep, adds mavenLocal resolver at ThisBuild scope so SBT finds the locally-installed Comet JAR, and (for 3.3.2) adds --add-opens flags required to run Spark 3.5 on JDK 17+ - DeltaSQLCommandTest / DeltaHiveTest: injects Comet plugin, shuffle manager, and native Delta scan configs into sparkConf - CometSmokeTest.scala (new): asserts the Comet plugin is registered AND that Comet operators actually appear in a Delta query's physical plan — catches silent config drift where Comet is on the classpath but no longer applied The CI workflow runs the smoke test first as a fail-fast check before running the full suite. Matrix covers Delta 2.4.0 (Spark 3.4), 3.3.2 (Spark 3.5), and 4.0.0 (Spark 4.0) with Java 17. Also adds dev/run-delta-regression.sh for running end-to-end locally with a single command. Co-Authored-By: Claude Opus 4.6 (1M context) <[email protected]>
1 parent c97b60e commit 3e4b6a0

7 files changed

Lines changed: 781 additions & 0 deletions

File tree

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
name: Setup Delta Builder
19+
description: 'Setup Delta Lake to run Spark SQL regression tests with Comet'
20+
inputs:
21+
delta-version:
22+
description: 'The Delta Lake version (e.g., 3.3.2) to build'
23+
required: true
24+
runs:
25+
using: "composite"
26+
steps:
27+
- name: Clone Delta Lake repo
28+
uses: actions/checkout@v6
29+
with:
30+
repository: delta-io/delta
31+
path: delta-lake
32+
ref: v${{inputs.delta-version}}
33+
fetch-depth: 1
34+
35+
- name: Setup Delta Lake for Comet
36+
shell: bash
37+
run: |
38+
cd delta-lake
39+
git apply ../dev/diffs/delta/${{inputs.delta-version}}.diff
Lines changed: 149 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,149 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
name: Delta Lake Regression Tests
19+
20+
concurrency:
21+
group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}
22+
cancel-in-progress: true
23+
24+
on:
25+
push:
26+
branches:
27+
- main
28+
paths-ignore:
29+
- "benchmarks/**"
30+
- "doc/**"
31+
- "docs/**"
32+
- "**.md"
33+
- "native/core/benches/**"
34+
- "native/spark-expr/benches/**"
35+
- "spark/src/test/scala/org/apache/spark/sql/benchmark/**"
36+
pull_request:
37+
paths-ignore:
38+
- "benchmarks/**"
39+
- "doc/**"
40+
- "docs/**"
41+
- "**.md"
42+
- "native/core/benches/**"
43+
- "native/spark-expr/benches/**"
44+
- "spark/src/test/scala/org/apache/spark/sql/benchmark/**"
45+
# manual trigger
46+
workflow_dispatch:
47+
48+
env:
49+
RUST_VERSION: stable
50+
RUST_BACKTRACE: 1
51+
52+
jobs:
53+
# Build native library once and share with all test jobs
54+
build-native:
55+
name: Build Native Library
56+
runs-on: ubuntu-24.04
57+
container:
58+
image: amd64/rust
59+
steps:
60+
- uses: actions/checkout@v6
61+
62+
- name: Setup Rust & Java toolchain
63+
uses: ./.github/actions/setup-builder
64+
with:
65+
rust-version: ${{ env.RUST_VERSION }}
66+
jdk-version: 17
67+
68+
- name: Restore Cargo cache
69+
uses: actions/cache/restore@v5
70+
with:
71+
path: |
72+
~/.cargo/registry
73+
~/.cargo/git
74+
native/target
75+
key: ${{ runner.os }}-cargo-ci-${{ hashFiles('native/**/Cargo.lock', 'native/**/Cargo.toml') }}-${{ hashFiles('native/**/*.rs') }}
76+
restore-keys: |
77+
${{ runner.os }}-cargo-ci-${{ hashFiles('native/**/Cargo.lock', 'native/**/Cargo.toml') }}-
78+
79+
- name: Build native library
80+
# Use CI profile for faster builds (no LTO) and to share cache with pr_build_linux.yml.
81+
run: |
82+
cd native && cargo build --profile ci
83+
env:
84+
RUSTFLAGS: "-Ctarget-cpu=x86-64-v3"
85+
86+
- name: Save Cargo cache
87+
uses: actions/cache/save@v5
88+
if: github.ref == 'refs/heads/main'
89+
with:
90+
path: |
91+
~/.cargo/registry
92+
~/.cargo/git
93+
native/target
94+
key: ${{ runner.os }}-cargo-ci-${{ hashFiles('native/**/Cargo.lock', 'native/**/Cargo.toml') }}-${{ hashFiles('native/**/*.rs') }}
95+
96+
- name: Upload native library
97+
uses: actions/upload-artifact@v7
98+
with:
99+
name: native-lib-delta-regression
100+
path: native/target/ci/libcomet.so
101+
retention-days: 1
102+
103+
delta-spark:
104+
needs: build-native
105+
strategy:
106+
matrix:
107+
os: [ubuntu-24.04]
108+
java-version: [17]
109+
delta-version:
110+
- {full: '3.3.2', spark-short: '3.5', scala: '2.13', module: 'spark'}
111+
- {full: '4.0.0', spark-short: '4.0', scala: '2.13', module: 'spark'}
112+
- {full: '2.4.0', spark-short: '3.4', scala: '2.12', module: 'core'}
113+
fail-fast: false
114+
name: delta-regression/${{ matrix.os }}/delta-${{ matrix.delta-version.full }}/java-${{ matrix.java-version }}
115+
runs-on: ${{ matrix.os }}
116+
container:
117+
image: amd64/rust
118+
env:
119+
SPARK_LOCAL_IP: localhost
120+
steps:
121+
- uses: actions/checkout@v6
122+
- name: Setup Rust & Java toolchain
123+
uses: ./.github/actions/setup-builder
124+
with:
125+
rust-version: ${{ env.RUST_VERSION }}
126+
jdk-version: ${{ matrix.java-version }}
127+
- name: Download native library
128+
uses: actions/download-artifact@v8
129+
with:
130+
name: native-lib-delta-regression
131+
path: native/target/release/
132+
- name: Build Comet
133+
run: |
134+
./mvnw install -Prelease -DskipTests -Pspark-${{ matrix.delta-version.spark-short }}
135+
- name: Setup Delta Lake
136+
uses: ./.github/actions/setup-delta-builder
137+
with:
138+
delta-version: ${{ matrix.delta-version.full }}
139+
- name: Run Comet smoke test (fail fast)
140+
# Verify Comet is actually wired into Delta's test SparkSession before
141+
# running the full suite. Catches silent config drift where the plugin
142+
# is on the classpath but not applied to query plans.
143+
run: |
144+
cd delta-lake
145+
build/sbt "${{ matrix.delta-version.module }}/testOnly org.apache.spark.sql.delta.CometSmokeTest"
146+
- name: Run Delta Lake Spark tests
147+
run: |
148+
cd delta-lake
149+
build/sbt "${{ matrix.delta-version.module }}/test"

‎dev/diffs/delta/2.4.0.diff‎

Lines changed: 130 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,130 @@
1+
diff --git a/build.sbt b/build.sbt
2+
index 7082915..3773825 100644
3+
--- a/build.sbt
4+
+++ b/build.sbt
5+
@@ -18,6 +18,8 @@ import java.nio.file.Files
6+
import TestParallelization._
7+
8+
val sparkVersion = "3.4.0"
9+
+val cometVersion = "0.15.0-SNAPSHOT"
10+
+ThisBuild / resolvers += Resolver.mavenLocal
11+
val scala212 = "2.12.15"
12+
val scala213 = "2.13.5"
13+
val default_scala_version = scala212
14+
@@ -65,6 +67,8 @@ lazy val core = (project in file("core"))
15+
"org.apache.spark" %% "spark-core" % sparkVersion % "test" classifier "tests",
16+
"org.apache.spark" %% "spark-sql" % sparkVersion % "test" classifier "tests",
17+
"org.apache.spark" %% "spark-hive" % sparkVersion % "test" classifier "tests",
18+
+ // Comet regression testing
19+
+ "org.apache.datafusion" % s"comet-spark-spark3.4_${scalaBinaryVersion.value}" % cometVersion % "test",
20+
21+
// Compiler plugins
22+
// -- Bump up the genjavadoc version explicitly to 0.18 to work with Scala 2.12
23+
diff --git a/core/src/test/scala/org/apache/spark/sql/delta/CometSmokeTest.scala b/core/src/test/scala/org/apache/spark/sql/delta/CometSmokeTest.scala
24+
new file mode 100644
25+
index 0000000..35f9456
26+
--- /dev/null
27+
+++ b/core/src/test/scala/org/apache/spark/sql/delta/CometSmokeTest.scala
28+
@@ -0,0 +1,55 @@
29+
+/*
30+
+ * Copyright (2021) The Delta Lake Project Authors.
31+
+ *
32+
+ * Licensed under the Apache License, Version 2.0 (the "License");
33+
+ * you may not use this file except in compliance with the License.
34+
+ * You may obtain a copy of the License at
35+
+ *
36+
+ * http://www.apache.org/licenses/LICENSE-2.0
37+
+ *
38+
+ * Unless required by applicable law or agreed to in writing, software
39+
+ * distributed under the License is distributed on an "AS IS" BASIS,
40+
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
41+
+ * See the License for the specific language governing permissions and
42+
+ * limitations under the License.
43+
+ */
44+
+
45+
+package org.apache.spark.sql.delta
46+
+
47+
+import org.apache.spark.sql.{QueryTest, Row}
48+
+import org.apache.spark.sql.delta.test.DeltaSQLCommandTest
49+
+import org.apache.spark.sql.test.SharedSparkSession
50+
+
51+
+/**
52+
+ * Smoke test asserting that Comet is actually loaded and executing queries when the
53+
+ * Delta regression diff is applied. This catches silent configuration drift where
54+
+ * Comet is on the classpath but not wired into the physical plan (e.g. a typo in
55+
+ * `spark.plugins` that Spark silently ignores).
56+
+ *
57+
+ * This test is added by the Comet regression diff at
58+
+ * `dev/diffs/delta/<version>.diff` in the Apache DataFusion Comet repository.
59+
+ */
60+
+class CometSmokeTest extends QueryTest with SharedSparkSession with DeltaSQLCommandTest {
61+
+
62+
+ test("Comet plugin is registered in SparkConf") {
63+
+ val plugins = spark.conf.get("spark.plugins")
64+
+ assert(plugins.contains("CometPlugin"),
65+
+ s"Comet plugin not registered. spark.plugins=$plugins")
66+
+ }
67+
+
68+
+ test("Comet operators appear in Delta query physical plan") {
69+
+ withTempDir { dir =>
70+
+ val path = dir.getCanonicalPath
71+
+ spark.range(10).toDF("id")
72+
+ .write.format("delta").save(path)
73+
+
74+
+ val df = spark.read.format("delta").load(path).filter("id > 2")
75+
+ val planString = df.queryExecution.executedPlan.toString
76+
+
77+
+ assert(planString.contains("Comet"),
78+
+ s"No Comet operators found in physical plan. Is Comet actually wired in?\n$planString")
79+
+
80+
+ checkAnswer(df, (3 until 10).map(i => Row(i)))
81+
+ }
82+
+ }
83+
+}
84+
diff --git a/core/src/test/scala/org/apache/spark/sql/delta/test/DeltaHiveTest.scala b/core/src/test/scala/org/apache/spark/sql/delta/test/DeltaHiveTest.scala
85+
index 0b8c556..685d045 100644
86+
--- a/core/src/test/scala/org/apache/spark/sql/delta/test/DeltaHiveTest.scala
87+
+++ b/core/src/test/scala/org/apache/spark/sql/delta/test/DeltaHiveTest.scala
88+
@@ -43,6 +43,17 @@ trait DeltaHiveTest extends SparkFunSuite with BeforeAndAfterAll { self: SQLTest
89+
conf.set(SQLConf.V2_SESSION_CATALOG_IMPLEMENTATION.key, classOf[DeltaCatalog].getName)
90+
conf.set(StaticSQLConf.SPARK_SESSION_EXTENSIONS.key,
91+
classOf[DeltaSparkSessionExtension].getName)
92+
+ conf.set("spark.plugins", "org.apache.spark.CometPlugin")
93+
+ conf.set("spark.shuffle.manager",
94+
+ "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager")
95+
+ conf.set("spark.comet.enabled", "true")
96+
+ conf.set("spark.comet.exec.enabled", "true")
97+
+ conf.set("spark.comet.exec.shuffle.enabled", "true")
98+
+ conf.set("spark.comet.scan.enabled", "true")
99+
+ conf.set("spark.comet.scan.deltaNative.enabled", "true")
100+
+ conf.set("spark.comet.explainFallback.enabled", "true")
101+
+ conf.set("spark.memory.offHeap.enabled", "true")
102+
+ conf.set("spark.memory.offHeap.size", "10g")
103+
_sc = new SparkContext("local", this.getClass.getName, conf)
104+
_hiveContext = new TestHiveContext(_sc)
105+
_session = _hiveContext.sparkSession
106+
diff --git a/core/src/test/scala/org/apache/spark/sql/delta/test/DeltaSQLCommandTest.scala b/core/src/test/scala/org/apache/spark/sql/delta/test/DeltaSQLCommandTest.scala
107+
index b712972..97cbebc 100644
108+
--- a/core/src/test/scala/org/apache/spark/sql/delta/test/DeltaSQLCommandTest.scala
109+
+++ b/core/src/test/scala/org/apache/spark/sql/delta/test/DeltaSQLCommandTest.scala
110+
@@ -46,7 +46,19 @@ trait DeltaSQLCommandTest { self: SharedSparkSession =>
111+
112+
override protected def createSparkSession: TestSparkSession = {
113+
SparkSession.cleanupAnyExistingSession()
114+
- val session = new DeltaTestSparkSession(sparkConf)
115+
+ val cometConf = sparkConf
116+
+ .set("spark.plugins", "org.apache.spark.CometPlugin")
117+
+ .set("spark.shuffle.manager",
118+
+ "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager")
119+
+ .set("spark.comet.enabled", "true")
120+
+ .set("spark.comet.exec.enabled", "true")
121+
+ .set("spark.comet.exec.shuffle.enabled", "true")
122+
+ .set("spark.comet.scan.enabled", "true")
123+
+ .set("spark.comet.scan.deltaNative.enabled", "true")
124+
+ .set("spark.comet.explainFallback.enabled", "true")
125+
+ .set("spark.memory.offHeap.enabled", "true")
126+
+ .set("spark.memory.offHeap.size", "10g")
127+
+ val session = new DeltaTestSparkSession(cometConf)
128+
session.conf.set(SQLConf.V2_SESSION_CATALOG_IMPLEMENTATION.key, classOf[DeltaCatalog].getName)
129+
session
130+
}

0 commit comments

Comments
 (0)