Skip to content

Commit 4c3b865

Browse files
Random-Liuestesp
Random-Liu
authored and
estesp
committed
Improve gce bootstrapping in various ways.
Signed-off-by: Lantao Liu <[email protected]>
1 parent bae03ff commit 4c3b865

4 files changed

Lines changed: 131 additions & 53 deletions

File tree

‎contrib/gce/cloud-init/master.yaml‎

Lines changed: 0 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -24,25 +24,6 @@ write_files:
2424
[Install]
2525
WantedBy=containerd.target
2626
27-
# containerd on master uses the cni binary and config in the
28-
# release tarball.
29-
- path: /etc/containerd/config.toml
30-
permissions: 0644
31-
owner: root
32-
content: |
33-
[plugins.linux]
34-
shim = "/home/containerd/usr/local/bin/containerd-shim"
35-
runtime = "/home/containerd/usr/local/sbin/runc"
36-
37-
[plugins.cri]
38-
enable_tls_streaming = true
39-
[plugins.cri.cni]
40-
bin_dir = "/home/containerd/opt/cni/bin"
41-
conf_dir = "/etc/cni/net.d"
42-
conf_template = "/home/containerd/opt/containerd/cluster/gce/cni.template"
43-
[plugins.cri.registry.mirrors."docker.io"]
44-
endpoint = ["https://mirror.gcr.io","https://registry-1.docker.io"]
45-
4627
- path: /etc/systemd/system/containerd.service
4728
permissions: 0644
4829
owner: root

‎contrib/gce/cloud-init/node.yaml‎

Lines changed: 0 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -24,23 +24,6 @@ write_files:
2424
[Install]
2525
WantedBy=containerd.target
2626
27-
- path: /etc/containerd/config.toml
28-
permissions: 0644
29-
owner: root
30-
content: |
31-
[plugins.linux]
32-
shim = "/home/containerd/usr/local/bin/containerd-shim"
33-
runtime = "/home/containerd/usr/local/sbin/runc"
34-
35-
[plugins.cri]
36-
enable_tls_streaming = true
37-
[plugins.cri.cni]
38-
bin_dir = "/home/containerd/opt/cni/bin"
39-
conf_dir = "/etc/cni/net.d"
40-
conf_template = "/home/containerd/opt/containerd/cluster/gce/cni.template"
41-
[plugins.cri.registry.mirrors."docker.io"]
42-
endpoint = ["https://mirror.gcr.io","https://registry-1.docker.io"]
43-
4427
- path: /etc/systemd/system/containerd.service
4528
permissions: 0644
4629
owner: root

‎contrib/gce/configure.sh‎

Lines changed: 129 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,8 @@ set -o pipefail
2222
# CONTAINERD_HOME is the directory for containerd.
2323
CONTAINERD_HOME="/home/containerd"
2424
cd "${CONTAINERD_HOME}"
25+
# KUBE_HOME is the directory for kubernetes.
26+
KUBE_HOME="/home/kubernetes"
2527

2628
# fetch_metadata fetches metadata from GCE metadata server.
2729
# Var set:
@@ -36,32 +38,144 @@ fetch_metadata() {
3638
fi
3739
}
3840

39-
# DEPLOY_PATH is the gcs path where cri-containerd tarball is stored.
40-
DEPLOY_PATH=${DEPLOY_PATH:-"cri-containerd-release"}
41+
# fetch_env fetches environment variables from GCE metadata server
42+
# and generate a env file under ${CONTAINERD_HOME}. It assumes that
43+
# the environment variables in metadata are in yaml format.
44+
fetch_env() {
45+
local -r env_file_name=$1
46+
(
47+
umask 077;
48+
local -r tmp_env_file="/tmp/${env_file_name}.yaml"
49+
tmp_env_content=$(fetch_metadata "${env_file_name}")
50+
if [ -z "${tmp_env_content}" ]; then
51+
echo "No environment variable is specified in ${env_file_name}"
52+
return
53+
fi
54+
echo "${tmp_env_content}" > "${tmp_env_file}"
55+
# Convert the yaml format file into a shell-style file.
56+
eval $(python -c '''
57+
import pipes,sys,yaml
58+
for k,v in yaml.load(sys.stdin).iteritems():
59+
print("readonly {var}={value}".format(var = k, value = pipes.quote(str(v))))
60+
''' < "${tmp_env_file}" > "${CONTAINERD_HOME}/${env_file_name}")
61+
rm -f "${tmp_env_file}"
62+
)
63+
}
64+
65+
# is_preloaded checks whether a package has been preloaded in the image.
66+
is_preloaded() {
67+
local -r tar=$1
68+
local -r sha1=$2
69+
grep -qs "${tar},${sha1}" "${KUBE_HOME}/preload_info"
70+
}
71+
72+
# KUBE_ENV_METADATA is the metadata key for kubernetes envs.
73+
KUBE_ENV_METADATA="kube-env"
74+
fetch_env ${KUBE_ENV_METADATA}
75+
if [ -f "${CONTAINERD_HOME}/${KUBE_ENV_METADATA}" ]; then
76+
source "${CONTAINERD_HOME}/${KUBE_ENV_METADATA}"
77+
fi
4178

42-
# PKG_PREFIX is the prefix of the cri-containerd tarball name.
79+
# CONTAINERD_ENV_METADATA is the metadata key for containerd envs.
80+
CONTAINERD_ENV_METADATA="containerd-env"
81+
fetch_env ${CONTAINERD_ENV_METADATA}
82+
if [ -f "${CONTAINERD_HOME}/${CONTAINERD_ENV_METADATA}" ]; then
83+
source "${CONTAINERD_HOME}/${CONTAINERD_ENV_METADATA}"
84+
fi
85+
86+
# CONTAINERD_PKG_PREFIX is the prefix of the cri-containerd tarball name.
4387
# By default use the release tarball with cni built in.
44-
PKG_PREFIX=${PKG_PREFIX:-"cri-containerd-cni"}
45-
46-
# VERSION is the cri-containerd version to use.
47-
VERSION_METADATA="version"
48-
VERSION=$(fetch_metadata "${VERSION_METADATA}")
49-
if [ -z "${VERSION}" ]; then
50-
echo "Version is not set."
51-
exit 1
88+
pkg_prefix=${CONTAINERD_PKG_PREFIX:-"cri-containerd-cni"}
89+
# Behave differently for test and production.
90+
if [ "${CONTAINERD_TEST:-"false"}" != "true" ]; then
91+
# CONTAINERD_DEPLOY_PATH is the gcs path where cri-containerd tarball is stored.
92+
deploy_path=${CONTAINERD_DEPLOY_PATH:-"cri-containerd-release"}
93+
# CONTAINERD_VERSION is the cri-containerd version to use.
94+
version=${CONTAINERD_VERSION:-""}
95+
if [ -z "${version}" ]; then
96+
echo "CONTAINERD_VERSION is not set."
97+
exit 1
98+
fi
99+
else
100+
deploy_path=${CONTAINERD_DEPLOY_PATH:-"cri-containerd-staging"}
101+
102+
# PULL_REFS_METADATA is the metadata key of PULL_REFS from prow.
103+
PULL_REFS_METADATA="PULL_REFS"
104+
pull_refs=$(fetch_metadata "${PULL_REFS_METADATA}")
105+
if [ ! -z "${pull_refs}" ]; then
106+
deploy_dir=$(echo "${pull_refs}" | sha1sum | awk '{print $1}')
107+
deploy_path="${deploy_path}/${deploy_dir}"
108+
fi
109+
110+
# TODO(random-liu): Put version into the metadata instead of
111+
# deciding it in cloud init. This may cause issue to reboot test.
112+
version=$(curl -f --ipv4 --retry 6 --retry-delay 3 --silent --show-error \
113+
https://storage.googleapis.com/${deploy_path}/latest)
52114
fi
53115

116+
TARBALL_GCS_NAME="${pkg_prefix}-${version}.linux-amd64.tar.gz"
54117
# TARBALL_GCS_PATH is the path to download cri-containerd tarball for node e2e.
55-
TARBALL_GCS_PATH="https://storage.googleapis.com/${DEPLOY_PATH}/${PKG_PREFIX}-${VERSION}.linux-amd64.tar.gz"
118+
TARBALL_GCS_PATH="https://storage.googleapis.com/${deploy_path}/${TARBALL_GCS_NAME}"
56119
# TARBALL is the name of the tarball after being downloaded.
57120
TARBALL="cri-containerd.tar.gz"
58121

59-
# Download and untar the release tar ball.
60-
curl -f --ipv4 -Lo "${TARBALL}" --connect-timeout 20 --max-time 300 --retry 6 --retry-delay 10 "${TARBALL_GCS_PATH}"
61-
tar xvf "${TARBALL}"
122+
# CONTAINERD_TAR_SHA1 is the sha1sum of containerd tarball.
123+
if is_preloaded "${TARBALL_GCS_NAME}" "${CONTAINERD_TAR_SHA1:-""}"; then
124+
echo "${TARBALL_GCS_NAME} is preloaded"
125+
else
126+
# Download and untar the release tar ball.
127+
curl -f --ipv4 -Lo "${TARBALL}" --connect-timeout 20 --max-time 300 --retry 6 --retry-delay 10 "${TARBALL_GCS_PATH}"
128+
tar xvf "${TARBALL}"
129+
rm -f "${TARBALL}"
130+
fi
62131

132+
# Configure containerd.
63133
# Copy crictl config.
64134
cp "${CONTAINERD_HOME}/etc/crictl.yaml" /etc
65135

136+
# Generate containerd config
137+
config_path=${CONTAINERD_CONFIG_PATH:-"/etc/containerd/config.toml"}
138+
mkdir -p $(dirname ${config_path})
139+
cni_bin_dir="${CONTAINERD_HOME}/opt/cni/bin"
140+
cni_template_path="${CONTAINERD_HOME}/opt/containerd/cluster/gce/cni.template"
141+
# NETWORK_POLICY_PROVIDER is from kube-env.
142+
network_policy_provider="${NETWORK_POLICY_PROVIDER:-"none"}"
143+
if [ -n "${network_policy_provider}" ] && [ "${network_policy_provider}" != "none" ] && [ "${KUBERNETES_MASTER:-}" != "true" ]; then
144+
# Use Kubernetes cni daemonset on node if network policy provider is specified.
145+
cni_bin_dir="${KUBE_HOME}/bin"
146+
cni_template_path=""
147+
fi
148+
cat > ${config_path} <<EOF
149+
[plugins.linux]
150+
shim = "${CONTAINERD_HOME}/usr/local/bin/containerd-shim"
151+
runtime = "${CONTAINERD_HOME}/usr/local/sbin/runc"
152+
153+
[plugins.cri]
154+
enable_tls_streaming = true
155+
[plugins.cri.cni]
156+
bin_dir = "${cni_bin_dir}"
157+
conf_dir = "/etc/cni/net.d"
158+
conf_template = "${cni_template_path}"
159+
[plugins.cri.registry.mirrors."docker.io"]
160+
endpoint = ["https://mirror.gcr.io","https://registry-1.docker.io"]
161+
EOF
162+
chmod 644 "${config_path}"
163+
66164
echo "export PATH=${CONTAINERD_HOME}/usr/local/bin/:${CONTAINERD_HOME}/usr/local/sbin/:\$PATH" > \
67165
/etc/profile.d/containerd_env.sh
166+
167+
# Run extra init script for test.
168+
if [ "${CONTAINERD_TEST:-"false"}" == "true" ]; then
169+
# EXTRA_INIT_SCRIPT is the name of the extra init script after being downloaded.
170+
EXTRA_INIT_SCRIPT="containerd-extra-init.sh"
171+
# EXTRA_INIT_SCRIPT_METADATA is the metadata key of init script.
172+
EXTRA_INIT_SCRIPT_METADATA="containerd-extra-init-sh"
173+
extra_init=$(fetch_metadata "${EXTRA_INIT_SCRIPT_METADATA}")
174+
# Return if containerd-extra-init-sh is not set.
175+
if [ -z "${extra_init}" ]; then
176+
exit 0
177+
fi
178+
echo "${extra_init}" > "${EXTRA_INIT_SCRIPT}"
179+
chmod 544 "${EXTRA_INIT_SCRIPT}"
180+
./${EXTRA_INIT_SCRIPT}
181+
fi

‎contrib/gce/env‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,8 @@ if [ ! -f "${version_file}" ]; then
88
echo "version file does not exist"
99
exit 1
1010
fi
11-
export KUBE_MASTER_EXTRA_METADATA="user-data=${GCE_DIR}/cloud-init/master.yaml,containerd-configure-sh=${GCE_DIR}/configure.sh,version=${version_file}"
12-
export KUBE_NODE_EXTRA_METADATA="user-data=${GCE_DIR}/cloud-init/node.yaml,containerd-configure-sh=${GCE_DIR}/configure.sh,version=${version_file}"
11+
export KUBE_MASTER_EXTRA_METADATA="user-data=${GCE_DIR}/cloud-init/master.yaml,containerd-configure-sh=${GCE_DIR}/configure.sh,containerd-env=${version_file}"
12+
export KUBE_NODE_EXTRA_METADATA="user-data=${GCE_DIR}/cloud-init/node.yaml,containerd-configure-sh=${GCE_DIR}/configure.sh,containerd-env=${version_file}"
1313
export KUBE_CONTAINER_RUNTIME="remote"
1414
export KUBE_CONTAINER_RUNTIME_ENDPOINT="/run/containerd/containerd.sock"
1515
export KUBE_LOAD_IMAGE_COMMAND="/home/containerd/usr/local/bin/ctr cri load"

0 commit comments

Comments
 (0)