Skip to content

Commit a3dd8df

Browse files
authored
build: Support CI pipelines for Spark 3.2, 3.3 and 3.4 (apache#153)
1 parent cad16d5 commit a3dd8df

6 files changed

Lines changed: 90 additions & 13 deletions

File tree

‎.github/actions/java-test/action.yaml‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,13 @@
1515
# specific language governing permissions and limitations
1616
# under the License.
1717

18+
name: "Java Test"
19+
description: "Run Java tests"
20+
inputs:
21+
MAVEN_OPTS:
22+
description: 'Maven options passed to the mvn command'
23+
required: false
24+
default: ''
1825
runs:
1926
using: "composite"
2027
steps:
@@ -37,9 +44,9 @@ runs:
3744
- name: Run Maven compile
3845
shell: bash
3946
run: |
40-
./mvnw -B compile test-compile scalafix:scalafix -Psemanticdb
47+
./mvnw -B compile test-compile scalafix:scalafix -Psemanticdb ${{ inputs.MAVEN_OPTS }}
4148
4249
- name: Run tests
4350
shell: bash
4451
run: |
45-
SPARK_HOME=`pwd` ./mvnw -B clean install
52+
SPARK_HOME=`pwd` ./mvnw -B clean install ${{ inputs.MAVEN_OPTS }}

‎.github/workflows/pr_build.yml‎

Lines changed: 71 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -44,13 +44,14 @@ jobs:
4444
os: [ubuntu-latest]
4545
java_version: [8, 11, 17]
4646
test-target: [rust, java]
47+
spark-version: ['3.4']
4748
is_push_event:
4849
- ${{ github.event_name == 'push' }}
4950
exclude: # exclude java 11 for pull_request event
5051
- java_version: 11
5152
is_push_event: false
5253
fail-fast: false
53-
name: ${{ matrix.test-target }} test on ${{ matrix.os }} with java ${{ matrix.java_version }}
54+
name: ${{ matrix.os }}/java ${{ matrix.java_version }}-spark-${{matrix.spark-version}}/${{ matrix.test-target }}
5455
runs-on: ${{ matrix.os }}
5556
container:
5657
image: amd64/rust
@@ -61,24 +62,54 @@ jobs:
6162
with:
6263
rust-version: ${{env.RUST_VERSION}}
6364
jdk-version: ${{ matrix.java_version }}
64-
65-
- uses: actions/checkout@v4
6665
- if: matrix.test-target == 'rust'
6766
name: Rust test steps
6867
uses: ./.github/actions/rust-test
6968
- if: matrix.test-target == 'java'
7069
name: Java test steps
7170
uses: ./.github/actions/java-test
71+
with:
72+
MAVEN_OPTS: -Pspark-${{ matrix.spark-version }}
73+
74+
linux-test-with-old-spark:
75+
strategy:
76+
matrix:
77+
os: [ubuntu-latest]
78+
java_version: [8, 11, 17]
79+
test-target: [java]
80+
spark-version: ['3.2', '3.3']
81+
exclude:
82+
- java_version: 17
83+
spark-version: '3.2'
84+
- java_version: 11
85+
spark-version: '3.2'
86+
fail-fast: false
87+
name: ${{ matrix.os }}/java ${{ matrix.java_version }}-spark-${{matrix.spark-version}}/${{ matrix.test-target }}
88+
runs-on: ${{ matrix.os }}
89+
container:
90+
image: amd64/rust
91+
steps:
92+
- uses: actions/checkout@v4
93+
- name: Setup Rust & Java toolchain
94+
uses: ./.github/actions/setup-builder
95+
with:
96+
rust-version: ${{env.RUST_VERSION}}
97+
jdk-version: ${{ matrix.java_version }}
98+
- name: Java test steps
99+
uses: ./.github/actions/java-test
100+
with:
101+
MAVEN_OPTS: -Pspark-${{ matrix.spark-version }}
72102

73103
macos-test:
74104
strategy:
75105
matrix:
76106
os: [macos-13]
77107
java_version: [8, 11, 17]
78108
test-target: [rust, java]
109+
spark-version: ['3.4']
79110
fail-fast: false
80111
if: github.event_name == 'push'
81-
name: ${{ matrix.test-target }} test on ${{ matrix.os }} with java ${{ matrix.java_version }}
112+
name: ${{ matrix.os }}/java ${{ matrix.java_version }}-spark-${{matrix.spark-version}}/${{ matrix.test-target }}
82113
runs-on: ${{ matrix.os }}
83114
steps:
84115
- uses: actions/checkout@v4
@@ -87,27 +118,28 @@ jobs:
87118
with:
88119
rust-version: ${{env.RUST_VERSION}}
89120
jdk-version: ${{ matrix.java_version }}
90-
91-
- uses: actions/checkout@v4
92121
- if: matrix.test-target == 'rust'
93122
name: Rust test steps
94123
uses: ./.github/actions/rust-test
95124
- if: matrix.test-target == 'java'
96125
name: Java test steps
97126
uses: ./.github/actions/java-test
127+
with:
128+
MAVEN_OPTS: -Pspark-${{ matrix.spark-version }}
98129

99130
macos-aarch64-test:
100131
strategy:
101132
matrix:
102133
java_version: [8, 11, 17]
103134
test-target: [rust, java]
135+
spark-version: ['3.4']
104136
is_push_event:
105137
- ${{ github.event_name == 'push' }}
106138
exclude: # exclude java 11 for pull_request event
107139
- java_version: 11
108140
is_push_event: false
109141
fail-fast: false
110-
name: ${{ matrix.test-target }} test on macos-aarch64 with java ${{ matrix.java_version }}
142+
name: macos-14(Silicon)/java ${{ matrix.java_version }}-spark-${{matrix.spark-version}}/${{ matrix.test-target }}
111143
runs-on: macos-14
112144
steps:
113145
- uses: actions/checkout@v4
@@ -118,14 +150,44 @@ jobs:
118150
jdk-version: ${{ matrix.java_version }}
119151
jdk-architecture: aarch64
120152
protoc-architecture: aarch_64
121-
122-
- uses: actions/checkout@v4
123153
- if: matrix.test-target == 'rust'
124154
name: Rust test steps
125155
uses: ./.github/actions/rust-test
126156
- if: matrix.test-target == 'java'
127157
name: Java test steps
128158
uses: ./.github/actions/java-test
159+
with:
160+
MAVEN_OPTS: -Pspark-${{ matrix.spark-version }}
161+
162+
macos-aarch64-test-with-old-spark:
163+
strategy:
164+
matrix:
165+
java_version: [8, 17]
166+
test-target: [java]
167+
spark-version: ['3.2', '3.3']
168+
exclude:
169+
- java_version: 17
170+
spark-version: '3.2'
171+
- java_version: 8
172+
spark-version: '3.3'
173+
fail-fast: false
174+
name: macos-14(Silicon)/java ${{ matrix.java_version }}-spark-${{matrix.spark-version}}/${{ matrix.test-target }}
175+
runs-on: macos-14
176+
steps:
177+
- uses: actions/checkout@v4
178+
- name: Setup Rust & Java toolchain
179+
uses: ./.github/actions/setup-macos-builder
180+
with:
181+
rust-version: ${{env.RUST_VERSION}}
182+
jdk-version: ${{ matrix.java_version }}
183+
jdk-architecture: aarch64
184+
protoc-architecture: aarch_64
185+
- if: matrix.test-target == 'java'
186+
name: Java test steps
187+
uses: ./.github/actions/java-test
188+
with:
189+
MAVEN_OPTS: -Pspark-${{ matrix.spark-version }}
190+
129191
check-pr-title:
130192
runs-on: ubuntu-latest
131193
steps:

‎spark/src/main/scala/org/apache/comet/CometSparkSessionExtensions.scala‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -602,6 +602,10 @@ object CometSparkSessionExtensions extends Logging {
602602
org.apache.spark.SPARK_VERSION.matches("3\\.2\\..*")
603603
}
604604

605+
def isSpark33Plus: Boolean = {
606+
org.apache.spark.SPARK_VERSION >= "3.3"
607+
}
608+
605609
/** Used for operations that are available in Spark 3.4+ */
606610
def isSpark34Plus: Boolean = {
607611
org.apache.spark.SPARK_VERSION >= "3.4"

‎spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ import org.apache.spark.sql.internal.SQLConf
2929
import org.apache.spark.sql.internal.SQLConf.SESSION_LOCAL_TIMEZONE
3030
import org.apache.spark.sql.types.{Decimal, DecimalType, StructType}
3131

32-
import org.apache.comet.CometSparkSessionExtensions.{isSpark32, isSpark34Plus}
32+
import org.apache.comet.CometSparkSessionExtensions.{isSpark32, isSpark33Plus, isSpark34Plus}
3333

3434
class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper {
3535
import testImplicits._
@@ -393,6 +393,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper {
393393
}
394394

395395
test("date_trunc with format array") {
396+
assume(isSpark33Plus, "TimestampNTZ is supported in Spark 3.3+, See SPARK-36182")
396397
val numRows = 1000
397398
Seq(true, false).foreach { dictionaryEnabled =>
398399
withTempDir { dir =>

‎spark/src/test/scala/org/apache/comet/exec/CometExecSuite.scala‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,7 @@ class CometExecSuite extends CometTestBase {
7272
}
7373

7474
test("CometBroadcastExchangeExec") {
75+
assume(isSpark34Plus, "ChunkedByteBuffer is not serializable before Spark 3.4+")
7576
withSQLConf(CometConf.COMET_EXEC_BROADCAST_ENABLED.key -> "true") {
7677
withParquetTable((0 until 5).map(i => (i, i + 1)), "tbl_a") {
7778
withParquetTable((0 until 5).map(i => (i, i + 1)), "tbl_b") {
@@ -97,6 +98,7 @@ class CometExecSuite extends CometTestBase {
9798
}
9899

99100
test("CometBroadcastExchangeExec: empty broadcast") {
101+
assume(isSpark34Plus, "ChunkedByteBuffer is not serializable before Spark 3.4+")
100102
withSQLConf(CometConf.COMET_EXEC_BROADCAST_ENABLED.key -> "true") {
101103
withParquetTable((0 until 5).map(i => (i, i + 1)), "tbl_a") {
102104
withParquetTable((0 until 5).map(i => (i, i + 1)), "tbl_b") {
@@ -116,6 +118,7 @@ class CometExecSuite extends CometTestBase {
116118
}
117119

118120
test("CometExec.executeColumnarCollectIterator can collect ColumnarBatch results") {
121+
assume(isSpark34Plus, "ChunkedByteBuffer is not serializable before Spark 3.4+")
119122
withSQLConf(
120123
CometConf.COMET_EXEC_ENABLED.key -> "true",
121124
CometConf.COMET_EXEC_ALL_OPERATOR_ENABLED.key -> "true") {

‎spark/src/test/scala/org/apache/spark/sql/CometTestBase.scala‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -183,7 +183,7 @@ abstract class CometTestBase
183183

184184
protected def checkPlanContains(plan: SparkPlan, includePlans: Class[_]*): Unit = {
185185
includePlans.foreach { case planClass =>
186-
if (!plan.exists(op => planClass.isAssignableFrom(op.getClass))) {
186+
if (plan.find(op => planClass.isAssignableFrom(op.getClass)).isEmpty) {
187187
assert(
188188
false,
189189
s"Expected plan to contain ${planClass.getSimpleName}, but not.\n" +

0 commit comments

Comments
 (0)