[SPARK-25957][K8S] Add ability to skip building optional k8s docker images

ramaddepally · ramaddepally · commit ae1d74376b7d · 2018-11-15T16:44:28.000-08:00
bin/docker-image-tool.sh tries to build all docker images (JVM, PySpark
and SparkR) by default. But not all spark distributions are built with
SparkR and hence this script will fail on such distros.

With this change,
- We should be able to skip building optional docker
  images (PySpark and SparkR) by specifying -pskip or -Rskip flags.
- We autodetect if SparkR is not installed in the build and skip building
  SparkR docker image.
- We skip pushing docker images that are not available locally.

Tested following scenarios.
- On source code and distro with SparkR support
  - Run bin/docker-image-tool.sh -r &lt;repo&gt; -t &lt;tag&gt; build. Verify that JVM, PySpark and SparkR docker images are built.
  - Run bin/docker-image-tool.sh -r &lt;repo&gt; -t &lt;tag&gt; -Rskip -pskip build. Verify that only JVM docker image is built. Building PySpark and SparkR images is skipped.
- On source code and distro without SparkR support
  - Run bin/docker-image-tool.sh -r &lt;repo&gt; -t &lt;tag&gt; build. Verify that only JVM, PySpark docker images are built. Building SparkR image is skipped.
- On system with JVM, PySpark and SparkR images built,
  - Run bin/docker-image-tool.sh -r &lt;repo&gt; -t &lt;tag&gt; push. Verify that all images are pushed to docker registry.
- On system with only JVM and PySpark images built.
  - Run bin/docker-image-tool.sh -r &lt;repo&gt; -t &lt;tag&gt; push. Verify that only JVM and PySpark images are pushed. Pushing SparkR images is skipped.
diff --git a/bin/docker-image-tool.sh b/bin/docker-image-tool.sh
@@ -41,6 +41,18 @@ function image_ref {
   echo "$image"
 }
 
+function docker_push {
+  local image_name="$1"
+  if [ ! -z $(docker images -q "$(image_ref ${image_name})") ]; then
+    docker push "$(image_ref ${image_name})"
+    if [ $? -ne 0 ]; then
+      error "Failed to push $image_name Docker image."
+    fi
+  else
+    echo "$(image_ref ${image_name}) image not found. Skipping push for this image."
+  fi
+}
+
 function build {
   local BUILD_ARGS
   local IMG_PATH
@@ -102,33 +114,33 @@ function build {
     error "Failed to build Spark JVM Docker image, please refer to Docker build output for details."
   fi
 
-  docker build $NOCACHEARG "${BINDING_BUILD_ARGS[@]}" \
-    -t $(image_ref spark-py) \
-    -f "$PYDOCKERFILE" .
+  if [ "${PYDOCKERFILE}" != "skip" ]; then
+    docker build $NOCACHEARG "${BINDING_BUILD_ARGS[@]}" \
+      -t $(image_ref spark-py) \
+      -f "$PYDOCKERFILE" .
+      if [ $? -ne 0 ]; then
+        error "Failed to build PySpark Docker image, please refer to Docker build output for details."
+      fi
+  else
+    echo "Skipped building PySpark docker image."
+  fi
+
+  if [ "${RDOCKERFILE}" != "skip" ] && [ -d "${SPARK_HOME}/R/lib" ]; then
+    docker build $NOCACHEARG "${BINDING_BUILD_ARGS[@]}" \
+      -t $(image_ref spark-r) \
+      -f "$RDOCKERFILE" .
     if [ $? -ne 0 ]; then
-      error "Failed to build PySpark Docker image, please refer to Docker build output for details."
+      error "Failed to build SparkR Docker image, please refer to Docker build output for details."
     fi
-  docker build $NOCACHEARG "${BINDING_BUILD_ARGS[@]}" \
-    -t $(image_ref spark-r) \
-    -f "$RDOCKERFILE" .
-  if [ $? -ne 0 ]; then
-    error "Failed to build SparkR Docker image, please refer to Docker build output for details."
+  else
+    echo "Skipped building SparkR docker image."
   fi
 }
 
 function push {
-  docker push "$(image_ref spark)"
-  if [ $? -ne 0 ]; then
-    error "Failed to push Spark JVM Docker image."
-  fi
-  docker push "$(image_ref spark-py)"
-  if [ $? -ne 0 ]; then
-    error "Failed to push PySpark Docker image."
-  fi
-  docker push "$(image_ref spark-r)"
-  if [ $? -ne 0 ]; then
-    error "Failed to push SparkR Docker image."
-  fi
+  docker_push "spark"
+  docker_push "spark-py"
+  docker_push "spark-r"
 }
 
 function usage {
@@ -145,6 +157,8 @@ Options:
   -f file               Dockerfile to build for JVM based Jobs. By default builds the Dockerfile shipped with Spark.
   -p file               Dockerfile to build for PySpark Jobs. Builds Python dependencies and ships with Spark.
   -R file               Dockerfile to build for SparkR Jobs. Builds R dependencies and ships with Spark.
+  -pskip                Skip building PySpark docker image.
+  -Rskip                Skip building SparkR docker image.
   -r repo               Repository address.
   -t tag                Tag to apply to the built image, or to identify the image to be pushed.
   -m                    Use minikube's Docker daemon.
