Keep Spark version in a single place only (BeamModulePlugin)

Moritz Mack · Moritz Mack · commit 7b6cc89db079 · 2022-10-12T16:49:21.000+02:00
diff --git a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
@@ -499,6 +499,10 @@ class BeamModulePlugin implements Plugin<Project> {
     def arrow_version = "5.0.0"
     def jmh_version = "1.34"
 
+    // Export Spark versions, so they are defined in a single place only
+    project.ext.spark2_version = spark2_version
+    project.ext.spark3_version = spark3_version
+
     // A map of maps containing common libraries used per language. To use:
     // dependencies {
     //   compile library.java.slf4j_api
diff --git a/runners/spark/2/build.gradle b/runners/spark/2/build.gradle
@@ -19,8 +19,8 @@
 def basePath = '..'
 /* All properties required for loading the Spark build script */
 project.ext {
-  // Set the version of all Spark-related dependencies here.
-  spark_version = '2.4.8'
+  // Spark 2 version as defined in BeamModulePlugin
+  spark_version = spark2_version
   spark_scala_version = '2.11'
   // Copy shared sources for Spark 2 to use Spark 3 as primary version in place
   copySourceBase = true
diff --git a/runners/spark/3/build.gradle b/runners/spark/3/build.gradle
@@ -19,8 +19,8 @@
 def basePath = '..'
 /* All properties required for loading the Spark build script */
 project.ext {
-  // Set the version of all Spark-related dependencies here.
-  spark_version = '3.1.2'
+  // Spark 3 version as defined in BeamModulePlugin
+  spark_version = spark3_version
   spark_scala_version = '2.12'
   copySourceBase = false // disabled to use Spark 3 as primary dev version
   archives_base_name = 'beam-runners-spark-3'
@@ -29,7 +29,7 @@ project.ext {
 // Load the main build script which contains all build logic.
 apply from: "$basePath/spark_runner.gradle"
 
-
+// Additional supported Spark versions (used in compatibility tests)
 def sparkVersions = [
     "330": "3.3.0",
     "321": "3.2.1"
