Skip to content

Commit 2401698

Browse files
Random-Liuestesp
Random-Liu
authored and
estesp
committed
Update GCE cluster bootstrapping and e2e test
Signed-off-by: Lantao Liu <[email protected]>
1 parent 1fe0385 commit 2401698

5 files changed

Lines changed: 81 additions & 134 deletions

File tree

‎contrib/gce/cloud-init/master.yaml‎

Lines changed: 35 additions & 59 deletions
Original file line numberDiff line numberDiff line change
@@ -1,29 +1,31 @@
11
#cloud-config
22

33
write_files:
4-
# Setup cri-containerd.
5-
- path: /etc/systemd/system/cri-containerd-installation.service
4+
# Setup containerd.
5+
- path: /etc/systemd/system/containerd-installation.service
66
permissions: 0644
77
owner: root
88
content: |
99
# installed by cloud-init
1010
[Unit]
11-
Description=Download and install cri-containerd binaries and configurations.
11+
Description=Download and install containerd binaries and configurations.
1212
After=network-online.target
1313
1414
[Service]
1515
Type=oneshot
1616
RemainAfterExit=yes
17-
ExecStartPre=/bin/mkdir -p /home/cri-containerd
18-
ExecStartPre=/bin/mount --bind /home/cri-containerd /home/cri-containerd
19-
ExecStartPre=/bin/mount -o remount,exec /home/cri-containerd
20-
ExecStartPre=/usr/bin/curl --fail --retry 5 --retry-delay 3 --silent --show-error -H "X-Google-Metadata-Request: True" -o /home/cri-containerd/configure.sh http://metadata.google.internal/computeMetadata/v1/instance/attributes/cri-containerd-configure-sh
21-
ExecStartPre=/bin/chmod 544 /home/cri-containerd/configure.sh
22-
ExecStart=/home/cri-containerd/configure.sh
17+
ExecStartPre=/bin/mkdir -p /home/containerd
18+
ExecStartPre=/bin/mount --bind /home/containerd /home/containerd
19+
ExecStartPre=/bin/mount -o remount,exec /home/containerd
20+
ExecStartPre=/usr/bin/curl --fail --retry 5 --retry-delay 3 --silent --show-error -H "X-Google-Metadata-Request: True" -o /home/containerd/configure.sh http://metadata.google.internal/computeMetadata/v1/instance/attributes/containerd-configure-sh
21+
ExecStartPre=/bin/chmod 544 /home/containerd/configure.sh
22+
ExecStart=/home/containerd/configure.sh
2323
2424
[Install]
25-
WantedBy=cri-containerd.target
25+
WantedBy=containerd.target
2626
27+
# containerd on master uses the cni binary and config in the
28+
# release tarball.
2729
- path: /etc/containerd/config.toml
2830
permissions: 0644
2931
owner: root
@@ -35,8 +37,14 @@ write_files:
3537
path = "/runtime"
3638
3739
[plugins.linux]
38-
shim = "/home/cri-containerd/usr/local/bin/containerd-shim"
39-
runtime = "/home/cri-containerd/usr/local/sbin/runc"
40+
shim = "/home/containerd/usr/local/bin/containerd-shim"
41+
runtime = "/home/containerd/usr/local/sbin/runc"
42+
43+
[plugins.cri.cni]
44+
bin_dir = "/home/containerd/opt/cni/bin"
45+
conf_dir = "/home/containerd/etc/cni/net.d"
46+
[plugins.cri.registry.mirrors."docker.io"]
47+
endpoint = ["https://mirror.gcr.io","https://registry-1.docker.io"]
4048
4149
- path: /etc/systemd/system/containerd.service
4250
permissions: 0644
@@ -46,7 +54,7 @@ write_files:
4654
[Unit]
4755
Description=containerd container runtime
4856
Documentation=https://containerd.io
49-
After=cri-containerd-installation.service
57+
After=containerd-installation.service
5058
5159
[Service]
5260
Restart=always
@@ -59,67 +67,36 @@ write_files:
5967
LimitNPROC=infinity
6068
LimitCORE=infinity
6169
ExecStartPre=/sbin/modprobe overlay
62-
ExecStart=/home/cri-containerd/usr/local/bin/containerd --log-level debug
63-
64-
[Install]
65-
WantedBy=cri-containerd.target
66-
67-
- path: /etc/systemd/system/cri-containerd.service
68-
permissions: 0644
69-
owner: root
70-
content: |
71-
# installed by cloud-init
72-
[Unit]
73-
Description=Kubernetes containerd CRI shim
74-
Requires=network-online.target
75-
After=cri-containerd-installation.service
76-
77-
[Service]
78-
Restart=always
79-
RestartSec=5
80-
LimitNOFILE=1048576
81-
# Having non-zero Limit*s causes performance problems due to accounting overhead
82-
# in the kernel. We recommend using cgroups to do container-local accounting.
83-
LimitNPROC=infinity
84-
LimitCORE=infinity
85-
# cri-containerd on master uses the cni binary and config in the
86-
# release tarball.
87-
ExecStart=/home/cri-containerd/usr/local/bin/cri-containerd \
88-
--log-level=debug \
89-
--network-bin-dir=/home/cri-containerd/opt/cni/bin \
90-
--network-conf-dir=/home/cri-containerd/etc/cni/net.d \
91-
--cgroup-path=/runtime \
92-
--registry=docker.io=https://mirror.gcr.io,https://registry-1.docker.io
70+
ExecStart=/home/containerd/usr/local/bin/containerd --log-level debug
9371
9472
[Install]
95-
WantedBy=cri-containerd.target
73+
WantedBy=containerd.target
9674
97-
- path: /etc/systemd/system/cri-containerd-monitor.service
75+
- path: /etc/systemd/system/containerd-monitor.service
9876
permissions: 0644
9977
owner: root
10078
content: |
10179
[Unit]
102-
Description=Kubernetes health monitoring for cri-containerd and containerd
103-
After=containerd.service cri-containerd.service
80+
Description=Kubernetes health monitoring for containerd
81+
After=containerd.service
10482
10583
[Service]
10684
Restart=always
10785
RestartSec=10
10886
RemainAfterExit=yes
109-
ExecStartPre=/bin/chmod 544 /home/cri-containerd/opt/cri-containerd/cluster/health-monitor.sh
110-
ExecStart=/bin/bash -c 'CRICTL=/home/cri-containerd/usr/local/bin/crictl \
111-
/home/cri-containerd/opt/cri-containerd/cluster/health-monitor.sh'
87+
ExecStartPre=/bin/chmod 544 /home/containerd/opt/containerd/cluster/health-monitor.sh
88+
ExecStart=/bin/bash -c 'CRICTL=/home/containerd/usr/local/bin/crictl \
89+
/home/containerd/opt/containerd/cluster/health-monitor.sh'
11290
11391
[Install]
114-
WantedBy=cri-containerd.target
92+
WantedBy=containerd.target
11593
116-
# TODO(random-liu): Guarantee order.
117-
- path: /etc/systemd/system/cri-containerd.target
94+
- path: /etc/systemd/system/containerd.target
11895
permissions: 0644
11996
owner: root
12097
content: |
12198
[Unit]
122-
Description=CRI Containerd
99+
Description=Containerd
123100
124101
[Install]
125102
WantedBy=kubernetes.target
@@ -221,11 +198,10 @@ write_files:
221198
222199
runcmd:
223200
- systemctl daemon-reload
201+
- systemctl enable containerd-installation.service
224202
- systemctl enable containerd.service
225-
- systemctl enable cri-containerd-installation.service
226-
- systemctl enable cri-containerd.service
227-
- systemctl enable cri-containerd-monitor.service
228-
- systemctl enable cri-containerd.target
203+
- systemctl enable containerd-monitor.service
204+
- systemctl enable containerd.target
229205
- systemctl enable kube-master-installation.service
230206
- systemctl enable kube-master-configuration.service
231207
- systemctl enable kubelet-monitor.service

‎contrib/gce/cloud-init/node.yaml‎

Lines changed: 33 additions & 58 deletions
Original file line numberDiff line numberDiff line change
@@ -1,28 +1,28 @@
11
#cloud-config
22

33
write_files:
4-
# Setup cri-containerd.
5-
- path: /etc/systemd/system/cri-containerd-installation.service
4+
# Setup containerd.
5+
- path: /etc/systemd/system/containerd-installation.service
66
permissions: 0644
77
owner: root
88
content: |
99
# installed by cloud-init
1010
[Unit]
11-
Description=Download and install cri-containerd binaries and configurations.
11+
Description=Download and install containerd binaries and configurations.
1212
After=network-online.target
1313
1414
[Service]
1515
Type=oneshot
1616
RemainAfterExit=yes
17-
ExecStartPre=/bin/mkdir -p /home/cri-containerd
18-
ExecStartPre=/bin/mount --bind /home/cri-containerd /home/cri-containerd
19-
ExecStartPre=/bin/mount -o remount,exec /home/cri-containerd
20-
ExecStartPre=/usr/bin/curl --fail --retry 5 --retry-delay 3 --silent --show-error -H "X-Google-Metadata-Request: True" -o /home/cri-containerd/configure.sh http://metadata.google.internal/computeMetadata/v1/instance/attributes/cri-containerd-configure-sh
21-
ExecStartPre=/bin/chmod 544 /home/cri-containerd/configure.sh
22-
ExecStart=/home/cri-containerd/configure.sh
17+
ExecStartPre=/bin/mkdir -p /home/containerd
18+
ExecStartPre=/bin/mount --bind /home/containerd /home/containerd
19+
ExecStartPre=/bin/mount -o remount,exec /home/containerd
20+
ExecStartPre=/usr/bin/curl --fail --retry 5 --retry-delay 3 --silent --show-error -H "X-Google-Metadata-Request: True" -o /home/containerd/configure.sh http://metadata.google.internal/computeMetadata/v1/instance/attributes/containerd-configure-sh
21+
ExecStartPre=/bin/chmod 544 /home/containerd/configure.sh
22+
ExecStart=/home/containerd/configure.sh
2323
2424
[Install]
25-
WantedBy=cri-containerd.target
25+
WantedBy=containerd.target
2626
2727
- path: /etc/containerd/config.toml
2828
permissions: 0644
@@ -35,8 +35,14 @@ write_files:
3535
path = "/runtime"
3636
3737
[plugins.linux]
38-
shim = "/home/cri-containerd/usr/local/bin/containerd-shim"
39-
runtime = "/home/cri-containerd/usr/local/sbin/runc"
38+
shim = "/home/containerd/usr/local/bin/containerd-shim"
39+
runtime = "/home/containerd/usr/local/sbin/runc"
40+
41+
[plugins.cri.cni]
42+
bin_dir = "/home/kubernetes/bin"
43+
conf_dir = "/etc/cni/net.d"
44+
[plugins.cri.registry.mirrors."docker.io"]
45+
endpoint = ["https://mirror.gcr.io","https://registry-1.docker.io"]
4046
4147
- path: /etc/systemd/system/containerd.service
4248
permissions: 0644
@@ -46,7 +52,7 @@ write_files:
4652
[Unit]
4753
Description=containerd container runtime
4854
Documentation=https://containerd.io
49-
After=cri-containerd-installation.service
55+
After=containerd-installation.service
5056
5157
[Service]
5258
Restart=always
@@ -59,66 +65,36 @@ write_files:
5965
LimitNPROC=infinity
6066
LimitCORE=infinity
6167
ExecStartPre=/sbin/modprobe overlay
62-
ExecStart=/home/cri-containerd/usr/local/bin/containerd --log-level debug
63-
64-
[Install]
65-
WantedBy=cri-containerd.target
66-
67-
- path: /etc/systemd/system/cri-containerd.service
68-
permissions: 0644
69-
owner: root
70-
content: |
71-
# installed by cloud-init
72-
[Unit]
73-
Description=Kubernetes containerd CRI shim
74-
Requires=network-online.target
75-
After=cri-containerd-installation.service
76-
77-
[Service]
78-
Restart=always
79-
RestartSec=5
80-
LimitNOFILE=1048576
81-
# Having non-zero Limit*s causes performance problems due to accounting overhead
82-
# in the kernel. We recommend using cgroups to do container-local accounting.
83-
LimitNPROC=infinity
84-
LimitCORE=infinity
85-
# Point to /home/kubernetes/bin where calico setup cni binary in kube-up.sh.
86-
# Point to /etc/cni/net.d where calico put cni config in kube-up.sh.
87-
ExecStart=/home/cri-containerd/usr/local/bin/cri-containerd \
88-
--log-level=debug \
89-
--network-bin-dir=/home/kubernetes/bin \
90-
--network-conf-dir=/etc/cni/net.d \
91-
--cgroup-path=/runtime \
92-
--registry=docker.io=https://mirror.gcr.io,https://registry-1.docker.io
68+
ExecStart=/home/containerd/usr/local/bin/containerd --log-level debug
9369
9470
[Install]
95-
WantedBy=cri-containerd.target
71+
WantedBy=containerd.target
9672
97-
- path: /etc/systemd/system/cri-containerd-monitor.service
73+
- path: /etc/systemd/system/containerd-monitor.service
9874
permissions: 0644
9975
owner: root
10076
content: |
10177
[Unit]
102-
Description=Kubernetes health monitoring for cri-containerd and containerd
103-
After=containerd.service cri-containerd.service
78+
Description=Kubernetes health monitoring for containerd
79+
After=containerd.service
10480
10581
[Service]
10682
Restart=always
10783
RestartSec=10
10884
RemainAfterExit=yes
109-
ExecStartPre=/bin/chmod 544 /home/cri-containerd/opt/cri-containerd/cluster/health-monitor.sh
110-
ExecStart=/bin/bash -c 'CRICTL=/home/cri-containerd/usr/local/bin/crictl \
111-
/home/cri-containerd/opt/cri-containerd/cluster/health-monitor.sh'
85+
ExecStartPre=/bin/chmod 544 /home/containerd/opt/containerd/cluster/health-monitor.sh
86+
ExecStart=/bin/bash -c 'CRICTL=/home/containerd/usr/local/bin/crictl \
87+
/home/containerd/opt/containerd/cluster/health-monitor.sh'
11288
11389
[Install]
114-
WantedBy=cri-containerd.target
90+
WantedBy=containerd.target
11591
116-
- path: /etc/systemd/system/cri-containerd.target
92+
- path: /etc/systemd/system/containerd.target
11793
permissions: 0644
11894
owner: root
11995
content: |
12096
[Unit]
121-
Description=CRI Containerd
97+
Description=Containerd
12298
12399
[Install]
124100
WantedBy=kubernetes.target
@@ -220,11 +196,10 @@ write_files:
220196
221197
runcmd:
222198
- systemctl daemon-reload
199+
- systemctl enable containerd-installation.service
223200
- systemctl enable containerd.service
224-
- systemctl enable cri-containerd-installation.service
225-
- systemctl enable cri-containerd.service
226-
- systemctl enable cri-containerd-monitor.service
227-
- systemctl enable cri-containerd.target
201+
- systemctl enable containerd-monitor.service
202+
- systemctl enable containerd.target
228203
- systemctl enable kube-node-installation.service
229204
- systemctl enable kube-node-configuration.service
230205
- systemctl enable kubelet-monitor.service

‎contrib/gce/configure.sh‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -19,9 +19,9 @@ set -o errexit
1919
set -o nounset
2020
set -o pipefail
2121

22-
# CRI_CONTAINERD_HOME is the directory for cri-containerd.
23-
CRI_CONTAINERD_HOME="/home/cri-containerd"
24-
cd "${CRI_CONTAINERD_HOME}"
22+
# CONTAINERD_HOME is the directory for containerd.
23+
CONTAINERD_HOME="/home/containerd"
24+
cd "${CONTAINERD_HOME}"
2525

2626
# fetch_metadata fetches metadata from GCE metadata server.
2727
# Var set:
@@ -63,5 +63,5 @@ tar xvf "${TARBALL}"
6363
# Copy crictl config.
6464
cp "${CRI_CONTAINERD_HOME}/etc/crictl.yaml" /etc
6565

66-
echo "export PATH=${CRI_CONTAINERD_HOME}/usr/local/bin/:${CRI_CONTAINERD_HOME}/usr/local/sbin/:\$PATH" > \
67-
/etc/profile.d/cri-containerd_env.sh
66+
echo "export PATH=${CONTAINERD_HOME}/usr/local/bin/:${CONTAINERD_HOME}/usr/local/sbin/:\$PATH" > \
67+
/etc/profile.d/containerd_env.sh

‎contrib/gce/env‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -9,11 +9,11 @@ if [ -z "${CRI_CONTAINERD_VERSION:-}" ]; then
99
fi
1010
version_file=$(mktemp /tmp/version.XXXX)
1111
echo "${CRI_CONTAINERD_VERSION}" > "$version_file"
12-
export KUBE_MASTER_EXTRA_METADATA="user-data=${GCE_DIR}/cloud-init/master.yaml,cri-containerd-configure-sh=${GCE_DIR}/configure.sh,version=${version_file}"
13-
export KUBE_NODE_EXTRA_METADATA="user-data=${GCE_DIR}/cloud-init/node.yaml,cri-containerd-configure-sh=${GCE_DIR}/configure.sh,version=${version_file}"
12+
export KUBE_MASTER_EXTRA_METADATA="user-data=${GCE_DIR}/cloud-init/master.yaml,containerd-configure-sh=${GCE_DIR}/configure.sh,version=${version_file}"
13+
export KUBE_NODE_EXTRA_METADATA="user-data=${GCE_DIR}/cloud-init/node.yaml,containerd-configure-sh=${GCE_DIR}/configure.sh,version=${version_file}"
1414
export KUBE_CONTAINER_RUNTIME="remote"
15-
export KUBE_CONTAINER_RUNTIME_ENDPOINT="/var/run/cri-containerd.sock"
16-
export KUBE_LOAD_IMAGE_COMMAND="/home/cri-containerd/usr/local/bin/ctrcri load"
15+
export KUBE_CONTAINER_RUNTIME_ENDPOINT="/run/containerd/containerd.sock"
16+
export KUBE_LOAD_IMAGE_COMMAND="/home/containerd/usr/local/bin/ctrcri load"
1717
export NETWORK_POLICY_PROVIDER="calico"
1818
export NON_MASQUERADE_CIDR="0.0.0.0/0"
1919
export KUBE_KUBELET_EXTRA_ARGS="--runtime-cgroups=/runtime"

‎contrib/health-monitor.sh‎

Lines changed: 4 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -20,16 +20,15 @@ set -o pipefail
2020
# CRICTL is the path of crictl
2121
CRICTL=${CRICTL:-"crictl"}
2222
# INITIAL_WAIT_ATTEMPTS is the number to attempt, before start
23-
# performing health check. The problem is that cri-containerd
24-
# and containerd are started around the same time with health
25-
# monitor, they may not be ready yet when health-monitor is started.
23+
# performing health check. The problem is that containerd is
24+
# started around the same time with health monitor, it may
25+
# not be ready yet when health-monitor is started.
2626
INITIAL_WAIT_ATTEMPTS=${INITIAL_WAIT_ATTEMPTS:-5}
2727
# COMMAND_TIMEOUT is the timeout for the health check command.
2828
COMMAND_TIMEOUT=${COMMAND_TIMEOUT:-60}
2929
# CHECK_PERIOD is the health check period.
3030
CHECK_PERIOD=${CHECK_PERIOD:-10}
31-
# SLEEP_SECONDS is the time to sleep after killing cri-containerd
32-
# and containerd.
31+
# SLEEP_SECONDS is the time to sleep after killing containerd.
3332
SLEEP_SECONDS=${SLEEP_SECONDS:-120}
3433

3534
attempt=1
@@ -41,11 +40,8 @@ done
4140

4241
echo "Start performing health check."
4342
while true; do
44-
# Use crictl pods because it requires both containerd and
45-
# cri-containerd to be working.
4643
if ! timeout ${COMMAND_TIMEOUT} ${CRICTL} pods > /dev/null; then
4744
echo "\"$CRICTL pods\" failed!"
48-
pkill -x cri-containerd
4945
pkill -x containerd
5046
# Wait for a while, as we don't want to kill it again before it is really up.
5147
sleep ${SLEEP_SECONDS}

0 commit comments

Comments
 (0)