kubernetes · roycaihw · May 22, 2018 · May 22, 2018 · May 22, 2018 · May 29, 2018
diff --git a/hack/lib/util.sh b/hack/lib/util.sh
@@ -43,6 +43,45 @@ kube::util::wait_for_url() {
   return 1
 }
 
+# wait_for_url_ssl implements the SSL (TLS) version of wait_for_url. This util
+# function drops the "curl -k" (insecure) flag to enable SSL. It takes arguments
+# from 6th position to the end as arguments passed to curl. You can specify TLS
+# related arguments (e.g. --cacert, --oauth2-bearer) there, example:
+#
+#     kube::util::wait_for_url_ssl "https://${API_HOST}:${API_PORT}/healthz" \
+#     "apiserver: " 1 30 1 --cacert $KUBE_APISERVER_CERT -H "$AUTH_TOKEN_HEADER"
+#
+# performs a normal wait_for_url with additional "--cacert" and "-H" flags
+# against an https secure port.
+#
+# We have this util function separate from the wait_for_url util function to
+# keep the current behavior of wait_for_url safely.
+kube::util::wait_for_url_ssl() {
+  local url=$1
+  local prefix=${2:-}
+  local wait=${3:-1}
+  local times=${4:-30}
+  local maxtime=${5:-1}
+  shift 5
+
+  which curl >/dev/null || {
+    kube::log::usage "curl must be installed"
+    exit 1
+  }
+
+  local i
+  for i in $(seq 1 "$times"); do
+    local out
+    if out=$(curl --max-time "$maxtime" -gfs "$url" "$@" 2>/dev/null); then
+      kube::log::status "On try ${i}, ${prefix}: ${out}"
+      return 0
+    fi
+    sleep "${wait}"
+  done
+  kube::log::error "Timed out waiting for ${prefix} to answer at ${url}; tried ${times} waiting ${wait} between each"
+  return 1
+}
+
 # Example:  kube::util::trap_add 'echo "in trap DEBUG"' DEBUG
 # See: http://stackoverflow.com/questions/3338030/multiple-bash-traps-for-the-same-signal
 kube::util::trap_add() {

diff --git a/hack/update-api-resource-list.sh b/hack/update-api-resource-list.sh
@@ -0,0 +1,149 @@
+#!/bin/bash
+
+# Copyright 2018 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script fetches list of latest api resources served by kube apiserver and
+# saves the list at test/e2e/testing-manifests/apiresource/resources_all.csv.
+# This script also computes the list of api resources used for testing
+#   (test/e2e/testing-manifests/apiresource/resources.csv)
+# by removing whitelisted api resources
+#   (test/e2e/testing-manifests/apiresource/resources_whitelist.csv) from the
+# list of all resources
+#   (test/e2e/testing-manifests/apiresource/resources_all.csv)
+#
+# The result list of api resources for testing
+#   (test/e2e/testing-manifests/apiresource/resources.csv)
+# is used by the api coverage e2e test:
+#   test/e2e/apimachinery/coverage.go
+# Ref test/e2e/testing-manifests/apiresource/README.md for more information.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+KUBE_ROOT=$(dirname "${BASH_SOURCE}")/..
+source "${KUBE_ROOT}/hack/lib/init.sh"
+
+kube::golang::setup_env
+
+make -C "${KUBE_ROOT}" WHAT=cmd/kube-apiserver
+
+function cleanup()
+{
+    [[ -n ${APISERVER_PID-} ]] && kill ${APISERVER_PID} 1>&2 2>/dev/null
+
+    kube::etcd::cleanup
+
+    kube::log::status "Clean up complete"
+}
+
+TMP_DIR=$(mktemp -d /tmp/update-api-resource-list.XXXX)
+KUBE_APISERVER_CERT="$TMP_DIR/certs/apiserver.crt"
+AUTH_TOKEN_HEADER="Authorization: Bearer dummy_token"
+
+# fetch_api_resource walks through apiserver's API discovery paths
+# $APISERVER_URL/api and $APISERVER_URL/apis for each group, version and
+# resource. It returns the list of all exposed API resources verbs in csv format
+# of:
+#   GROUP,VERSION,RESOURCE,NAMESPACED,VERB
+#
+# GROUP is empty string "" for core group. RESOURCE may contain "/" to indicate
+# a subresource. NAMESPACED is either "true" or "false".
+# The output is not sorted yet. We have following steps to sort the outputs.
+function fetch_api_resource()
+{
+for RESOURCE in $(curl -s --cacert $KUBE_APISERVER_CERT -H "$AUTH_TOKEN_HEADER" $APISERVER_URL/api/v1 | jq -c .resources[]); do
+  NAME=$(echo $RESOURCE | jq .name | tr -d '"')
+  NAMESPACED=$(echo $RESOURCE | jq .namespaced)
+  for VERB in $(echo $RESOURCE | jq '.verbs' | tr -d '"[],'); do
+    echo ,v1,$NAME,$NAMESPACED,$VERB
+  done
+done
+
+# List group versions
+for GROUP_VERSION in $(curl -s --cacert $KUBE_APISERVER_CERT -H "$AUTH_TOKEN_HEADER" $APISERVER_URL/apis | jq .groups[].versions[].groupVersion | tr -d '"'); do
+  # List resources
+  for RESOURCE in $(curl -s --cacert $KUBE_APISERVER_CERT -H "$AUTH_TOKEN_HEADER" $APISERVER_URL/apis/$GROUP_VERSION | jq -c '.resources[]'); do
+    NAME=$(echo $RESOURCE | jq '.name' | tr -d '"')
+    NAMESPACED=$(echo $RESOURCE | jq '.namespaced')
+    for VERB in $(echo $RESOURCE | jq '.verbs' | tr -d '"[],'); do
+      echo $(echo $GROUP_VERSION | tr '/' ','),$NAME,$NAMESPACED,$VERB
+    done
+  done
+done
+}
+
+kube::util::trap_add cleanup EXIT SIGINT
+
+kube::golang::setup_env
+
+apiserver=$(kube::util::find-binary "kube-apiserver")
+
+ETCD_HOST=${ETCD_HOST:-127.0.0.1}
+ETCD_PORT=${ETCD_PORT:-2379}
+API_PORT=${API_PORT:-8050}
+API_HOST=${API_HOST:-127.0.0.1}
+
+kube::etcd::start
+
+echo "dummy_token,admin,admin" > $TMP_DIR/tokenauth.csv
+
+# Start kube-apiserver with all apis enabled
+kube::log::status "Starting kube-apiserver"
+"${KUBE_OUTPUT_HOSTBIN}/kube-apiserver" \
+  --bind-address="${API_HOST}" \
+  --secure-port="${API_PORT}" \
+  --etcd-servers="http://${ETCD_HOST}:${ETCD_PORT}" \
+  --advertise-address="10.10.10.10" \
+  --cert-dir="${TMP_DIR}/certs" \
+  --runtime-config="api/all=true" \
+  --token-auth-file=$TMP_DIR/tokenauth.csv \
+  --logtostderr \
+  --v=2 \
+  --service-cluster-ip-range="10.0.0.0/24" >/tmp/openapi-api-server.log 2>&1 &
+APISERVER_PID=$!
+
+# wait for apiserver to come up
+kube::util::wait_for_url_ssl "https://${API_HOST}:${API_PORT}/healthz" "apiserver: " 1 30 1 --cacert $KUBE_APISERVER_CERT -H "$AUTH_TOKEN_HEADER"
+
+APISERVER_URL="https://${API_HOST}:${API_PORT}"
+
+KUBE_RESOURCE_FILE="${KUBE_ROOT}/test/e2e/testing-manifests/apiresource/resources_all.csv"
+KUBE_RESOURCE_WHITELIST_FILE="${KUBE_ROOT}/test/e2e/testing-manifests/apiresource/resources_whitelist.csv"
+KUBE_RESOURCE_TEST_FILE="${KUBE_ROOT}/test/e2e/testing-manifests/apiresource/resources.csv"
+
+# For the propose of easily comparing KUBE_RESOURCE_FILE and
+# KUBE_RESOURCE_WHITELIST_FILE, in order to remove whitelisted API resource
+# lines, we need to sort the files.
+#
+# The API coverage e2e test requires reading parent resource before reading
+# subresource, to correctly construct the resource map. For example we want
+#   ,v1,pods,true,VERB
+# to be ordered before
+#   ,v1,pods/status,true,VERB
+fetch_api_resource | sort -t',' -k1,1 -k2,2 -k3,3 -k4 -o ${KUBE_RESOURCE_FILE}
+sort -t',' -k1,1 -k2,2 -k3,3 -k4 ${KUBE_RESOURCE_WHITELIST_FILE} -o ${KUBE_RESOURCE_WHITELIST_FILE}
+# Remove whitelisted API resource lines from KUBE_RESOURCE_FILE, to generated
+# KUBE_RESOURCE_TEST_FILE.
+#
+# NOTE: GNU comm and BSD comm have different defaulting. We sort the inputs to satisfy
+# the defaulting first, then sort the output to more readable csv format
+comm -13 <(sort ${KUBE_RESOURCE_WHITELIST_FILE}) <(sort ${KUBE_RESOURCE_FILE}) | sort -t',' -k1,1 -k2,2 -k3,3 -k4 -o ${KUBE_RESOURCE_TEST_FILE}
+
+kube::log::status "WARNING: (please ignore if the file <test/e2e/testing-manifests/apiresource/resources.csv> is not changed)"
+kube::log::status "  If the API resource list (test/e2e/testing-manifests/apiresource/resources.csv) gets changed AND/OR if you are making API change that adds/updates some APIs in <GROUP>/<VERSION>/<KIND>, please update the corresponding yamlfiles in test/e2e/testing-manifests/apiresource/yamlfiles/<GROUP>/<VERSION>/<KIND>.yaml to properly pass the API coverage e2e test (test/e2e/apimachinery/coverage.go). For more information, please refer to test/e2e/testing-manifests/apiresource/README.md"
+kube::log::status "SUCCESS: API resource list updated"
+
+# ex: ts=2 sw=2 et filetype=sh
diff --git a/hack/verify-api-resource-list.sh b/hack/verify-api-resource-list.sh
@@ -0,0 +1,94 @@
+#!/bin/bash
+
+# Copyright 2018 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script verifies whether we have the correct list of latest api resources
+# served by kube apiserver, under
+#   test/e2e/testing-manifests/apiresource/resources_all.csv
+# the list of whitelisted api resources, under
+#   test/e2e/testing-manifests/apiresource/resources_whitelist.csv
+# and the list of api resources used by API coverage e2e test
+# (test/e2e/apimachinery/coverage.go), under
+#   test/e2e/testing-manifests/apiresource/resources.csv
+#
+# The logic used to generate these api resource lists are in
+#   hack/update-api-resource-list.sh
+# Ref test/e2e/testing-manifests/apiresource/README.md for more information.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+KUBE_ROOT=$(dirname "${BASH_SOURCE}")/..
+source "${KUBE_ROOT}/hack/lib/init.sh"
+
+kube::golang::setup_env
+kube::etcd::install
+export PATH="${KUBE_ROOT}/third_party/etcd:${PATH}"
+
+make -C "${KUBE_ROOT}" WHAT=cmd/kube-apiserver
+
+apiserver=$(kube::util::find-binary "kube-apiserver")
+
+LISTROOT="${KUBE_ROOT}/test/e2e/testing-manifests/apiresource"
+TMP_LISTROOT="${KUBE_ROOT}/_tmp/apiresource"
+_tmp="${KUBE_ROOT}/_tmp"
+
+mkdir -p "${_tmp}"
+cp -a "${LISTROOT}" "${TMP_LISTROOT}"
+kube::util::trap_add "cp -a ${TMP_LISTROOT} ${LISTROOT}/..; rm -rf ${_tmp}" EXIT SIGINT
+rm ${LISTROOT}/resources_all.csv ${LISTROOT}/resources.csv
+
+"${KUBE_ROOT}/hack/update-api-resource-list.sh"
+
+# For the propose of easily comparing KUBE_RESOURCE_FILE and
+# KUBE_RESOURCE_WHITELIST_FILE, in order to remove whitelisted API resource
+# lines, we need to sort the files.
+#
+# The API coverage e2e test requires reading parent resource before reading
+# subresource, to correctly construct the resource map. For example we want
+#   ,v1,pods,true,VERB
+# to be ordered before
+#   ,v1,pods/status,true,VERB
+sort -t',' -k1,1 -k2,2 -k3,3 -k4 "${LISTROOT}/resources_whitelist.csv" -o "${LISTROOT}/resources_whitelist.csv"
+# Remove whitelisted API resource lines from KUBE_RESOURCE_FILE, to generated
+# KUBE_RESOURCE_TEST_FILE.
+#
+# NOTE: GNU comm and BSD comm have different defaulting. We sort the inputs to satisfy
+# the defaulting first, then sort the output to more readable csv format
+comm -13 <(sort "${LISTROOT}/resources_whitelist.csv") <(sort "${LISTROOT}/resources_all.csv") | sort -t',' -k1,1 -k2,2 -k3,3 -k4 -o "${LISTROOT}/resources.csv"
+
+function assert_equal() {
+  echo "Diffing $1 against freshly generated api resource list"
+  ret=0
+  diff -Naupr $1 $2 || ret=$?
+  if [[ $ret -eq 0 ]]
+  then
+    echo "$1 up to date."
+  else
+    echo "ERROR: $1 is out of date. Please run hack/update-api-resource-list.sh"
+    echo "WARNING:"
+    echo "  If the API resource list (test/e2e/testing-manifests/apiresource/resources.csv) gets changed AND/OR if you are making API change that adds/updates some APIs in <GROUP>/<VERSION>/<KIND>, please update the corresponding yamlfiles in test/e2e/testing-manifests/apiresource/yamlfiles/<GROUP>/<VERSION>/<KIND>.yaml to properly pass the API coverage e2e test (test/e2e/apimachinery/coverage.go). For more information, please refer to test/e2e/testing-manifests/apiresource/README.md"
+    exit 1
+  fi
+}
+
+assert_equal "${LISTROOT}/resources_all.csv" "${TMP_LISTROOT}/resources_all.csv"
+assert_equal "${LISTROOT}/resources_whitelist.csv" "${TMP_LISTROOT}/resources_whitelist.csv"
+assert_equal "${LISTROOT}/resources.csv" "${TMP_LISTROOT}/resources.csv"
+
+kube::log::status "SUCCESS"
+
+# ex: ts=2 sw=2 et filetype=sh
diff --git a/test/e2e/apimachinery/BUILD b/test/e2e/apimachinery/BUILD
@@ -11,6 +11,7 @@ go_library(
         "aggregator.go",
         "certs.go",
         "chunking.go",
+        "coverage.go",
         "crd_watch.go",
         "custom_resource_definition.go",
         "etcd_failure.go",
@@ -32,8 +33,11 @@ go_library(
         "//test/e2e/apps:go_default_library",
         "//test/e2e/framework:go_default_library",
         "//test/e2e/framework/metrics:go_default_library",
+        "//test/e2e/generated:go_default_library",
         "//test/utils:go_default_library",
         "//test/utils/image:go_default_library",
+        "//vendor/github.com/evanphx/json-patch:go_default_library",
+        "//vendor/github.com/golang/glog:go_default_library",
         "//vendor/github.com/onsi/ginkgo:go_default_library",
         "//vendor/github.com/onsi/gomega:go_default_library",
         "//vendor/github.com/stretchr/testify/assert:go_default_library",
@@ -62,6 +66,7 @@ go_library(
         "//vendor/k8s.io/apimachinery/pkg/util/intstr:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/util/uuid:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/yaml:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/watch:go_default_library",
         "//vendor/k8s.io/apiserver/pkg/authentication/user:go_default_library",
         "//vendor/k8s.io/apiserver/pkg/storage/names:go_default_library",