change install.spark

felixcheung · felixcheung · commit cd1394a8aab3 · 2017-02-12T16:11:18.000-08:00
diff --git a/R/pkg/R/install.R b/R/pkg/R/install.R
@@ -21,9 +21,9 @@
 #' Download and Install Apache Spark to a Local Directory
 #'
 #' \code{install.spark} downloads and installs Spark to a local directory if
-#' it is not found. The Spark version we use is the same as the SparkR version.
-#' Users can specify a desired Hadoop version, the remote mirror site, and
-#' the directory where the package is installed locally.
+#' it is not found. If SPARK_HOME is set in the environment, and that directory is found, that is
+#' returned. The Spark version we use is the same as the SparkR version. Users can specify a desired
+#' Hadoop version, the remote mirror site, and the directory where the package is installed locally.
 #'
 #' The full url of remote file is inferred from \code{mirrorUrl} and \code{hadoopVersion}.
 #' \code{mirrorUrl} specifies the remote path to a Spark folder. It is followed by a subfolder
@@ -68,6 +68,16 @@
 #'          \href{http://spark.apache.org/downloads.html}{Apache Spark}
 install.spark <- function(hadoopVersion = "2.7", mirrorUrl = NULL,
                           localDir = NULL, overwrite = FALSE) {
+  sparkHome <- Sys.getenv("SPARK_HOME")
+  if (isSparkRShell()) {
+    stopifnot(nchar(sparkHome) > 0)
+    message("Spark is already running in sparkR shell.")
+    return(invisible(sparkHome))
+  } else if (!is.na(file.info(sparkHome)$isdir)) {
+    message("Spark package found in SPARK_HOME: ", sparkHome)
+    return(invisible(sparkHome))
+  }
+
   version <- paste0("spark-", packageVersion("SparkR"))
   hadoopVersion <- tolower(hadoopVersion)
   hadoopVersionName <- hadoopVersionName(hadoopVersion)
diff --git a/R/pkg/R/sparkR.R b/R/pkg/R/sparkR.R
@@ -585,10 +585,7 @@ processSparkPackages <- function(packages) {
 # @param deployMode whether to deploy your driver on the worker nodes (cluster)
 #        or locally as an external client (client).
 # @return NULL if no need to update sparkHome, and new sparkHome otherwise.
-sparkCheckInstall <- function(
-  sparkHome = Sys.getenv("SPARK_HOME"),
-  master = "local",
-  deployMode = "") {
+sparkCheckInstall <- function(sparkHome, master, deployMode) {
   if (!isSparkRShell()) {
     if (!is.na(file.info(sparkHome)$isdir)) {
       message("Spark package found in SPARK_HOME: ", sparkHome)
diff --git a/R/pkg/inst/tests/testthat/test_Serde.R b/R/pkg/inst/tests/testthat/test_Serde.R
@@ -17,9 +17,6 @@
 
 context("SerDe functionality")
 
-# Ensure Spark is installed
-sparkCheckInstall()
-
 sparkSession <- sparkR.session(enableHiveSupport = FALSE)
 
 test_that("SerDe of primitive types", {
diff --git a/R/pkg/inst/tests/testthat/test_binaryFile.R b/R/pkg/inst/tests/testthat/test_binaryFile.R
@@ -17,9 +17,6 @@
 
 context("functions on binary files")
 
-# Ensure Spark is installed
-sparkCheckInstall()
-
 # JavaSparkContext handle
 sparkSession <- sparkR.session(enableHiveSupport = FALSE)
 sc <- callJStatic("org.apache.spark.sql.api.r.SQLUtils", "getJavaSparkContext", sparkSession)
diff --git a/R/pkg/inst/tests/testthat/test_binary_function.R b/R/pkg/inst/tests/testthat/test_binary_function.R
@@ -17,9 +17,6 @@
 
 context("binary functions")
 
-# Ensure Spark is installed
-sparkCheckInstall()
-
 # JavaSparkContext handle
 sparkSession <- sparkR.session(enableHiveSupport = FALSE)
 sc <- callJStatic("org.apache.spark.sql.api.r.SQLUtils", "getJavaSparkContext", sparkSession)
diff --git a/R/pkg/inst/tests/testthat/test_broadcast.R b/R/pkg/inst/tests/testthat/test_broadcast.R
@@ -17,9 +17,6 @@
 
 context("broadcast variables")
 
-# Ensure Spark is installed
-sparkCheckInstall()
-
 # JavaSparkContext handle
 sparkSession <- sparkR.session(enableHiveSupport = FALSE)
 sc <- callJStatic("org.apache.spark.sql.api.r.SQLUtils", "getJavaSparkContext", sparkSession)
diff --git a/R/pkg/inst/tests/testthat/test_context.R b/R/pkg/inst/tests/testthat/test_context.R
@@ -17,9 +17,6 @@
 
 context("test functions in sparkR.R")
 
-# Ensure Spark is installed
-sparkCheckInstall()
-
 test_that("Check masked functions", {
   # Check that we are not masking any new function from base, stats, testthat unexpectedly
   # NOTE: We should avoid adding entries to *namesOfMaskedCompletely* as masked functions make it
diff --git a/R/pkg/inst/tests/testthat/test_includePackage.R b/R/pkg/inst/tests/testthat/test_includePackage.R
@@ -17,9 +17,6 @@
 
 context("include R packages")
 
-# Ensure Spark is installed
-sparkCheckInstall()
-
 # JavaSparkContext handle
 sparkSession <- sparkR.session(enableHiveSupport = FALSE)
 sc <- callJStatic("org.apache.spark.sql.api.r.SQLUtils", "getJavaSparkContext", sparkSession)
diff --git a/R/pkg/inst/tests/testthat/test_jvm_api.R b/R/pkg/inst/tests/testthat/test_jvm_api.R
@@ -17,9 +17,6 @@
 
 context("JVM API")
 
-# Ensure Spark is installed
-sparkCheckInstall()
-
 sparkSession <- sparkR.session(enableHiveSupport = FALSE)
 
 test_that("Create and call methods on object", {
diff --git a/R/pkg/inst/tests/testthat/test_mllib_classification.R b/R/pkg/inst/tests/testthat/test_mllib_classification.R
@@ -19,9 +19,6 @@ library(testthat)
 
 context("MLlib classification algorithms, except for tree-based algorithms")
 
-# Ensure Spark is installed
-sparkCheckInstall()
-
 # Tests for MLlib classification algorithms in SparkR
 sparkSession <- sparkR.session(enableHiveSupport = FALSE)
 
diff --git a/R/pkg/inst/tests/testthat/test_mllib_clustering.R b/R/pkg/inst/tests/testthat/test_mllib_clustering.R
@@ -19,9 +19,6 @@ library(testthat)
 
 context("MLlib clustering algorithms")
 
-# Ensure Spark is installed
-sparkCheckInstall()
-
 # Tests for MLlib clustering algorithms in SparkR
 sparkSession <- sparkR.session(enableHiveSupport = FALSE)
 
diff --git a/R/pkg/inst/tests/testthat/test_mllib_recommendation.R b/R/pkg/inst/tests/testthat/test_mllib_recommendation.R
@@ -19,9 +19,6 @@ library(testthat)
 
 context("MLlib recommendation algorithms")
 
-# Ensure Spark is installed
-sparkCheckInstall()
-
 # Tests for MLlib recommendation algorithms in SparkR
 sparkSession <- sparkR.session(enableHiveSupport = FALSE)
 
diff --git a/R/pkg/inst/tests/testthat/test_mllib_regression.R b/R/pkg/inst/tests/testthat/test_mllib_regression.R
@@ -19,9 +19,6 @@ library(testthat)
 
 context("MLlib regression algorithms, except for tree-based algorithms")
 
-# Ensure Spark is installed
-sparkCheckInstall()
-
 # Tests for MLlib regression algorithms in SparkR
 sparkSession <- sparkR.session(enableHiveSupport = FALSE)
 
diff --git a/R/pkg/inst/tests/testthat/test_mllib_stat.R b/R/pkg/inst/tests/testthat/test_mllib_stat.R
@@ -19,9 +19,6 @@ library(testthat)
 
 context("MLlib statistics algorithms")
 
-# Ensure Spark is installed
-sparkCheckInstall()
-
 # Tests for MLlib statistics algorithms in SparkR
 sparkSession <- sparkR.session(enableHiveSupport = FALSE)
 
diff --git a/R/pkg/inst/tests/testthat/test_mllib_tree.R b/R/pkg/inst/tests/testthat/test_mllib_tree.R
@@ -19,9 +19,6 @@ library(testthat)
 
 context("MLlib tree-based algorithms")
 
-# Ensure Spark is installed
-sparkCheckInstall()
-
 # Tests for MLlib tree-based algorithms in SparkR
 sparkSession <- sparkR.session(enableHiveSupport = FALSE)
 
diff --git a/R/pkg/inst/tests/testthat/test_parallelize_collect.R b/R/pkg/inst/tests/testthat/test_parallelize_collect.R
@@ -17,9 +17,6 @@
 
 context("parallelize() and collect()")
 
-# Ensure Spark is installed
-sparkCheckInstall()
-
 # Mock data
 numVector <- c(-10:97)
 numList <- list(sqrt(1), sqrt(2), sqrt(3), 4 ** 10)
diff --git a/R/pkg/inst/tests/testthat/test_rdd.R b/R/pkg/inst/tests/testthat/test_rdd.R
@@ -17,9 +17,6 @@
 
 context("basic RDD functions")
 
-# Ensure Spark is installed
-sparkCheckInstall()
-
 # JavaSparkContext handle
 sparkSession <- sparkR.session(enableHiveSupport = FALSE)
 sc <- callJStatic("org.apache.spark.sql.api.r.SQLUtils", "getJavaSparkContext", sparkSession)
diff --git a/R/pkg/inst/tests/testthat/test_shuffle.R b/R/pkg/inst/tests/testthat/test_shuffle.R
@@ -17,9 +17,6 @@
 
 context("partitionBy, groupByKey, reduceByKey etc.")
 
-# Ensure Spark is installed
-sparkCheckInstall()
-
 # JavaSparkContext handle
 sparkSession <- sparkR.session(enableHiveSupport = FALSE)
 sc <- callJStatic("org.apache.spark.sql.api.r.SQLUtils", "getJavaSparkContext", sparkSession)
diff --git a/R/pkg/inst/tests/testthat/test_sparkSQL.R b/R/pkg/inst/tests/testthat/test_sparkSQL.R
@@ -19,9 +19,6 @@ library(testthat)
 
 context("SparkSQL functions")
 
-# Ensure Spark is installed
-sparkCheckInstall()
-
 # Utility function for easily checking the values of a StructField
 checkStructField <- function(actual, expectedName, expectedType, expectedNullable) {
   expect_equal(class(actual), "structField")
diff --git a/R/pkg/inst/tests/testthat/test_take.R b/R/pkg/inst/tests/testthat/test_take.R
@@ -17,9 +17,6 @@
 
 context("tests RDD function take()")
 
-# Ensure Spark is installed
-sparkCheckInstall()
-
 # Mock data
 numVector <- c(-10:97)
 numList <- list(sqrt(1), sqrt(2), sqrt(3), 4 ** 10)
diff --git a/R/pkg/inst/tests/testthat/test_textFile.R b/R/pkg/inst/tests/testthat/test_textFile.R
@@ -17,9 +17,6 @@
 
 context("the textFile() function")
 
-# Ensure Spark is installed
-sparkCheckInstall()
-
 # JavaSparkContext handle
 sparkSession <- sparkR.session(enableHiveSupport = FALSE)
 sc <- callJStatic("org.apache.spark.sql.api.r.SQLUtils", "getJavaSparkContext", sparkSession)
diff --git a/R/pkg/inst/tests/testthat/test_utils.R b/R/pkg/inst/tests/testthat/test_utils.R
@@ -17,9 +17,6 @@
 
 context("functions in utils.R")
 
-# Ensure Spark is installed
-sparkCheckInstall()
-
 # JavaSparkContext handle
 sparkSession <- sparkR.session(enableHiveSupport = FALSE)
 sc <- callJStatic("org.apache.spark.sql.api.r.SQLUtils", "getJavaSparkContext", sparkSession)
diff --git a/R/pkg/tests/run-all.R b/R/pkg/tests/run-all.R
@@ -21,4 +21,7 @@ library(SparkR)
 # Turn all warnings into errors
 options("warn" = 2)
 
+# Setup global test environment
+install.spark()
+
 test_package("SparkR")
diff --git a/R/pkg/vignettes/sparkr-vignettes.Rmd b/R/pkg/vignettes/sparkr-vignettes.Rmd
@@ -28,7 +28,7 @@ library(SparkR)
 We use default settings in which it runs in local mode. It auto downloads Spark package in the background if no previous installation is found. For more details about setup, see [Spark Session](#SetupSparkSession).
 
 ```{r, include=FALSE}
-SparkR:::sparkCheckInstall()
+install.spark()
 ```
 ```{r, message=FALSE, results="hide"}
 sparkR.session()
