test(storage): deflake integration tests (#6792)

coryan · web-flow · commit 9cd83ccbd8db · 2021-06-16T12:55:18.000-04:00
Use a different instance of the emulator to run the **emulator** tests
vs. the tests that **use** the emulator. The emulator tests
intentionally crash some of the emulator workers, which then causes
problems for the regular tests. Note that the emulator tests themselves
are still flaky, but they are very fast so using `--flake-test_attempts`
mostly avoids the problem.
diff --git a/ci/cloudbuild/builds/lib/integration.sh b/ci/cloudbuild/builds/lib/integration.sh
@@ -167,6 +167,10 @@ function integration::bazel_with_emulators() {
   "google/cloud/pubsub/ci/${EMULATOR_SCRIPT}" \
     bazel "${verb}" "${args[@]}"
 
+  io::log_h2 "Running Storage Emulator integration tests"
+  "google/cloud/storage/emulator/ci/run_integration_tests_bazel.sh" \
+    bazel "${args[@]}"
+
   io::log_h2 "Running Storage integration tests (with emulator)"
   "google/cloud/storage/ci/${EMULATOR_SCRIPT}" \
     bazel "${verb}" "${args[@]}"
diff --git a/google/cloud/storage/ci/run_integration_tests_emulator_bazel.sh b/google/cloud/storage/ci/run_integration_tests_emulator_bazel.sh
@@ -66,18 +66,6 @@ pushd "${HOME}" >/dev/null
 start_emulator 8585 8000
 popd >/dev/null
 
-# Run the unittests of the emulator before running integration tests.
-# TODO(#6641): Remove the --flaky_test_attempts flag once the flakiness is fixed.
-"${BAZEL_BIN}" test "${bazel_test_args[@]}" "//google/cloud/storage/emulator:test_utils" \
-  "//google/cloud/storage/emulator:test_gcs" \
-  "--flaky_test_attempts=5" \
-  "--test_env=CLOUD_STORAGE_EMULATOR_ENDPOINT=${CLOUD_STORAGE_EMULATOR_ENDPOINT}"
-exit_status=$?
-
-if [[ "$exit_status" -ne 0 ]]; then
-  exit "${exit_status}"
-fi
-
 excluded_targets+=(
   # This test does not work with Bazel, because it depends on dynamic loading
   # and some CMake magic. It is also skipped against production, so most Bazel
diff --git a/google/cloud/storage/emulator/ci/run_integration_tests_bazel.sh b/google/cloud/storage/emulator/ci/run_integration_tests_bazel.sh
@@ -0,0 +1,54 @@
+#!/usr/bin/env bash
+# Copyright 2021 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -eu
+
+source "$(dirname "$0")/../../../../../ci/lib/init.sh"
+source module /ci/etc/integration-tests-config.sh
+source module /google/cloud/storage/tools/run_emulator_utils.sh
+
+if [[ $# -lt 1 ]]; then
+  echo "Usage: $(basename "$0") <bazel-program> [bazel-test-args]"
+  exit 1
+fi
+
+# Configure run_emulators_utils.sh to find the instance admin emulator.
+BAZEL_BIN="$1"
+shift
+
+bazel_test_args=("$@")
+
+# `start_emulator` creates unsightly *.log files in the current directory
+# (which is ${PROJECT_ROOT}) and we cannot use a subshell because we want the
+# environment variables that it sets.
+pushd "${HOME}" >/dev/null
+# Start the emulator on a fixed port, otherwise the Bazel cache gets
+# invalidated on each run.
+start_emulator 8586 8587
+popd >/dev/null
+
+# Run the unittests of the emulator before running integration tests.
+# TODO(#6641): Remove the --flaky_test_attempts flag once the flakiness is fixed.
+"${BAZEL_BIN}" test "${bazel_test_args[@]}" \
+  "--flaky_test_attempts=5" \
+  "--test_env=CLOUD_STORAGE_EMULATOR_ENDPOINT=${CLOUD_STORAGE_EMULATOR_ENDPOINT}" \
+  -- "//google/cloud/storage/emulator:test_utils" "//google/cloud/storage/emulator:test_gcs"
+exit_status=$?
+
+if [[ "${exit_status}" -ne 0 ]]; then
+  cat "${HOME}/gcs_emulator.log"
+fi
+
+exit "${exit_status}"
diff --git a/google/cloud/storage/tools/run_emulator_utils.sh b/google/cloud/storage/tools/run_emulator_utils.sh
@@ -89,7 +89,7 @@ start_emulator() {
 
   delay=1
   connected=no
-  for attempt in $(seq 1 8); do
+  for _ in $(seq 1 8); do
     if curl "${HTTPBIN_ENDPOINT}/get" >/dev/null 2>&1; then
       connected=yes
       break
