Skip to content

Commit b0ce296

Browse files
fuweidqiutongs
fuweid
authored and
qiutongs
committed
integration: Add injected failpoint testing for RunPodSandbox
Signed-off-by: Wei Fu <[email protected]> (cherry picked from commit 3c5e80b) Signed-off-by: Qiutong Song <[email protected]>
1 parent a7f956d commit b0ce296

5 files changed

Lines changed: 220 additions & 2 deletions

File tree

‎.github/workflows/ci.yml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -373,6 +373,10 @@ jobs:
373373
sudo apt-get update
374374
sudo apt-get install -y criu
375375
376+
- name: Install failpoint binaries
377+
run: |
378+
script/setup/install-failpoint-binaries
379+
376380
- name: Install containerd
377381
env:
378382
CGO_ENABLED: 1

‎integration/main_test.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -174,7 +174,8 @@ func PodSandboxConfig(name, ns string, opts ...PodSandboxOpts) *runtime.PodSandb
174174
Uid: util.GenerateID(),
175175
Namespace: Randomize(ns),
176176
},
177-
Linux: &runtime.LinuxPodSandboxConfig{},
177+
Linux: &runtime.LinuxPodSandboxConfig{},
178+
Annotations: make(map[string]string),
178179
}
179180
for _, opt := range opts {
180181
opt(config)

‎integration/sandbox_run_rollback_test.go‎

Lines changed: 122 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,122 @@
1+
//go:build linux
2+
// +build linux
3+
4+
/*
5+
Copyright The containerd Authors.
6+
7+
Licensed under the Apache License, Version 2.0 (the "License");
8+
you may not use this file except in compliance with the License.
9+
You may obtain a copy of the License at
10+
11+
http://www.apache.org/licenses/LICENSE-2.0
12+
13+
Unless required by applicable law or agreed to in writing, software
14+
distributed under the License is distributed on an "AS IS" BASIS,
15+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16+
See the License for the specific language governing permissions and
17+
limitations under the License.
18+
*/
19+
20+
package integration
21+
22+
import (
23+
"encoding/json"
24+
"fmt"
25+
"os"
26+
"path/filepath"
27+
"runtime"
28+
"strings"
29+
"testing"
30+
31+
criapiv1 "k8s.io/cri-api/pkg/apis/runtime/v1"
32+
33+
"github.com/containerd/containerd/pkg/failpoint"
34+
"github.com/stretchr/testify/require"
35+
)
36+
37+
const (
38+
failpointRuntimeHandler = "runc-fp"
39+
40+
failpointShimPrefixKey = "io.containerd.runtime.v2.shim.failpoint."
41+
42+
failpointCNIStateDirKey = "cniFailpointControlStateDir"
43+
)
44+
45+
func TestRunPodSandboxWithSetupCNIFailure(t *testing.T) {
46+
if runtime.GOOS != "linux" {
47+
t.Skip()
48+
}
49+
50+
t.Logf("Inject CNI failpoint")
51+
conf := &failpointConf{
52+
Add: "1*error(you-shall-not-pass!)",
53+
}
54+
55+
sbConfig := PodSandboxConfig(t.Name(), "failpoint")
56+
injectCNIFailpoint(t, sbConfig, conf)
57+
58+
t.Logf("Create a sandbox")
59+
_, err := runtimeService.RunPodSandbox(sbConfig, failpointRuntimeHandler)
60+
require.Error(t, err)
61+
require.Equal(t, true, strings.Contains(err.Error(), "you-shall-not-pass!"))
62+
63+
t.Logf("Retry to create sandbox with same config")
64+
sb, err := runtimeService.RunPodSandbox(sbConfig, failpointRuntimeHandler)
65+
require.NoError(t, err)
66+
67+
err = runtimeService.StopPodSandbox(sb)
68+
require.NoError(t, err)
69+
70+
err = runtimeService.RemovePodSandbox(sb)
71+
require.NoError(t, err)
72+
}
73+
74+
func TestRunPodSandboxWithShimStartFailure(t *testing.T) {
75+
if runtime.GOOS != "linux" {
76+
t.Skip()
77+
}
78+
79+
t.Logf("Inject Shim failpoint")
80+
81+
sbConfig := PodSandboxConfig(t.Name(), "failpoint")
82+
injectShimFailpoint(t, sbConfig, map[string]string{
83+
"Start": "1*error(no hard feelings)",
84+
})
85+
86+
t.Logf("Create a sandbox")
87+
_, err := runtimeService.RunPodSandbox(sbConfig, failpointRuntimeHandler)
88+
require.Error(t, err)
89+
require.Equal(t, true, strings.Contains(err.Error(), "no hard feelings"))
90+
}
91+
92+
// failpointConf is used to describe cmdAdd/cmdDel/cmdCheck command's failpoint.
93+
type failpointConf struct {
94+
Add string `json:"cmdAdd"`
95+
Del string `json:"cmdDel"`
96+
Check string `json:"cmdCheck"`
97+
}
98+
99+
func injectCNIFailpoint(t *testing.T, sbConfig *criapiv1.PodSandboxConfig, conf *failpointConf) {
100+
stateDir := t.TempDir()
101+
102+
metadata := sbConfig.Metadata
103+
fpFilename := filepath.Join(stateDir,
104+
fmt.Sprintf("%s-%s.json", metadata.Namespace, metadata.Name))
105+
106+
data, err := json.Marshal(conf)
107+
require.NoError(t, err)
108+
109+
err = os.WriteFile(fpFilename, data, 0666)
110+
require.NoError(t, err)
111+
112+
sbConfig.Annotations[failpointCNIStateDirKey] = stateDir
113+
}
114+
115+
func injectShimFailpoint(t *testing.T, sbConfig *criapiv1.PodSandboxConfig, methodFps map[string]string) {
116+
for method, fp := range methodFps {
117+
_, err := failpoint.NewFailpoint(method, fp)
118+
require.NoError(t, err, "check failpoint %s for shim method %s", fp, method)
119+
120+
sbConfig.Annotations[failpointShimPrefixKey+method] = fp
121+
}
122+
}

‎script/setup/install-failpoint-binaries‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
#!/usr/bin/env bash
2+
3+
# Copyright The containerd Authors.
4+
5+
# Licensed under the Apache License, Version 2.0 (the "License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
# Build and install
18+
#
19+
# * cni-bridge-fp into /opt/cni/bin
20+
# * containerd-shim-runc-fp-v1 into /usr/local/bin
21+
#
22+
set -euo pipefail
23+
24+
base_dir="$(dirname "${BASH_SOURCE[0]}")"
25+
root_dir="$( cd "${base_dir}" && pwd )"/../..
26+
27+
cd "${root_dir}"
28+
29+
CNI_BIN_DIR=${CNI_BIN_DIR:-"/opt/cni/bin"}
30+
make bin/cni-bridge-fp
31+
sudo install bin/cni-bridge-fp "${CNI_BIN_DIR}"
32+
33+
SHIM_BIN_DIR=${SHIM_BIN_DIR:-"/usr/local/bin"}
34+
make bin/containerd-shim-runc-fp-v1
35+
sudo install bin/containerd-shim-runc-fp-v1 "${SHIM_BIN_DIR}"

‎script/test/utils.sh‎

Lines changed: 57 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,9 +35,10 @@ CONTAINERD_RUNTIME=${CONTAINERD_RUNTIME:-""}
3535
if [ -z "${CONTAINERD_CONFIG_FILE}" ]; then
3636
config_file="${CONTAINERD_CONFIG_DIR}/containerd-config-cri.toml"
3737
truncate --size 0 "${config_file}"
38+
echo "version=2" >> ${config_file}
39+
3840
if command -v sestatus >/dev/null 2>&1; then
3941
cat >>${config_file} <<EOF
40-
version=2
4142
[plugins."io.containerd.grpc.v1.cri"]
4243
enable_selinux = true
4344
EOF
@@ -51,6 +52,59 @@ EOF
5152
CONTAINERD_CONFIG_FILE="${config_file}"
5253
fi
5354

55+
if [ $IS_WINDOWS -eq 0 ]; then
56+
FAILPOINT_CONTAINERD_RUNTIME="runc-fp.v1"
57+
FAILPOINT_CNI_CONF_DIR=${FAILPOINT_CNI_CONF_DIR:-"/tmp/failpoint-cni-net.d"}
58+
mkdir -p "${FAILPOINT_CNI_CONF_DIR}"
59+
60+
# Add runtime with failpoint
61+
cat << EOF | tee -a "${CONTAINERD_CONFIG_FILE}"
62+
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc-fp]
63+
cni_conf_dir = "${FAILPOINT_CNI_CONF_DIR}"
64+
cni_max_conf_num = 1
65+
pod_annotations = ["io.containerd.runtime.v2.shim.failpoint.*"]
66+
runtime_type = "${FAILPOINT_CONTAINERD_RUNTIME}"
67+
EOF
68+
69+
cat << EOF | tee "${FAILPOINT_CNI_CONF_DIR}/10-containerd-net.conflist"
70+
{
71+
"cniVersion": "1.0.0",
72+
"name": "containerd-net-failpoint",
73+
"plugins": [
74+
{
75+
"type": "cni-bridge-fp",
76+
"bridge": "cni-fp",
77+
"isGateway": true,
78+
"ipMasq": true,
79+
"promiscMode": true,
80+
"ipam": {
81+
"type": "host-local",
82+
"ranges": [
83+
[{
84+
"subnet": "10.88.0.0/16"
85+
}],
86+
[{
87+
"subnet": "2001:4860:4860::/64"
88+
}]
89+
],
90+
"routes": [
91+
{ "dst": "0.0.0.0/0" },
92+
{ "dst": "::/0" }
93+
]
94+
},
95+
"capabilities": {
96+
"io.kubernetes.cri.pod-annotations": true
97+
}
98+
},
99+
{
100+
"type": "portmap",
101+
"capabilities": {"portMappings": true}
102+
}
103+
]
104+
}
105+
EOF
106+
fi
107+
54108
# CONTAINERD_TEST_SUFFIX is the suffix appended to the root/state directory used
55109
# by test containerd.
56110
CONTAINERD_TEST_SUFFIX=${CONTAINERD_TEST_SUFFIX:-"-test"}
@@ -183,6 +237,8 @@ test_setup() {
183237
fi
184238
readiness_check run_ctr
185239
readiness_check run_crictl
240+
# Show the config about cri plugin in log when it's ready
241+
run_crictl
186242
}
187243

188244
# test_teardown kills containerd.

0 commit comments

Comments
 (0)