Skip to content

Commit b9a35c6

Browse files
qiutongsqiutongs
committed
Add integration tests with failpoint
Signed-off-by: Qiutong Song <[email protected]> (cherry picked from commit 4196fad) Signed-off-by: Qiutong Song <[email protected]>
1 parent 1f29fac commit b9a35c6

2 files changed

Lines changed: 152 additions & 2 deletions

File tree

‎integration/main_test.go‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -164,6 +164,15 @@ func WithPodHostname(hostname string) PodSandboxOpts {
164164
}
165165
}
166166

167+
// Add pod labels.
168+
func WithPodLabels(kvs map[string]string) PodSandboxOpts {
169+
return func(p *runtime.PodSandboxConfig) {
170+
for k, v := range kvs {
171+
p.Labels[k] = v
172+
}
173+
}
174+
}
175+
167176
// PodSandboxConfig generates a pod sandbox config for test.
168177
func PodSandboxConfig(name, ns string, opts ...PodSandboxOpts) *runtime.PodSandboxConfig {
169178
config := &runtime.PodSandboxConfig{
@@ -176,6 +185,7 @@ func PodSandboxConfig(name, ns string, opts ...PodSandboxOpts) *runtime.PodSandb
176185
},
177186
Linux: &runtime.LinuxPodSandboxConfig{},
178187
Annotations: make(map[string]string),
188+
Labels: make(map[string]string),
179189
}
180190
for _, opt := range opts {
181191
opt(config)

‎integration/sandbox_run_rollback_test.go‎

Lines changed: 142 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,10 +28,10 @@ import (
2828
"strings"
2929
"testing"
3030

31+
"github.com/stretchr/testify/require"
3132
criapiv1 "k8s.io/cri-api/pkg/apis/runtime/v1"
3233

3334
"github.com/containerd/containerd/pkg/failpoint"
34-
"github.com/stretchr/testify/require"
3535
)
3636

3737
const (
@@ -89,6 +89,146 @@ func TestRunPodSandboxWithShimStartFailure(t *testing.T) {
8989
require.Equal(t, true, strings.Contains(err.Error(), "no hard feelings"))
9090
}
9191

92+
// TestRunPodSandboxWithShimDeleteFailure should keep the sandbox record if
93+
// failed to rollback shim by shim.Delete API.
94+
func TestRunPodSandboxWithShimDeleteFailure(t *testing.T) {
95+
if runtime.GOOS != "linux" {
96+
t.Skip()
97+
}
98+
if os.Getenv("ENABLE_CRI_SANDBOXES") != "" {
99+
t.Skip()
100+
}
101+
102+
testCase := func(restart bool) func(*testing.T) {
103+
return func(t *testing.T) {
104+
t.Log("Init PodSandboxConfig with specific label")
105+
labels := map[string]string{
106+
t.Name(): "true",
107+
}
108+
sbConfig := PodSandboxConfig(t.Name(), "failpoint", WithPodLabels(labels))
109+
110+
t.Log("Inject Shim failpoint")
111+
injectShimFailpoint(t, sbConfig, map[string]string{
112+
"Start": "1*error(failed to start shim)",
113+
"Delete": "1*error(please retry)", // inject failpoint during rollback shim
114+
})
115+
116+
t.Log("Create a sandbox")
117+
_, err := runtimeService.RunPodSandbox(sbConfig, failpointRuntimeHandler)
118+
require.Error(t, err)
119+
require.Contains(t, err.Error(), "failed to start shim")
120+
121+
t.Log("ListPodSandbox with the specific label")
122+
l, err := runtimeService.ListPodSandbox(&criapiv1.PodSandboxFilter{LabelSelector: labels})
123+
require.NoError(t, err)
124+
require.Len(t, l, 1)
125+
126+
sb := l[0]
127+
require.Equal(t, sb.State, criapiv1.PodSandboxState_SANDBOX_NOTREADY)
128+
require.Equal(t, sb.Metadata.Name, sbConfig.Metadata.Name)
129+
require.Equal(t, sb.Metadata.Namespace, sbConfig.Metadata.Namespace)
130+
require.Equal(t, sb.Metadata.Uid, sbConfig.Metadata.Uid)
131+
require.Equal(t, sb.Metadata.Attempt, sbConfig.Metadata.Attempt)
132+
133+
t.Log("Check PodSandboxStatus")
134+
sbStatus, err := runtimeService.PodSandboxStatus(sb.Id)
135+
require.NoError(t, err)
136+
require.Equal(t, sbStatus.State, criapiv1.PodSandboxState_SANDBOX_NOTREADY)
137+
require.Greater(t, len(sbStatus.Network.Ip), 0)
138+
139+
if restart {
140+
t.Log("Restart containerd")
141+
RestartContainerd(t)
142+
143+
t.Log("ListPodSandbox with the specific label")
144+
l, err = runtimeService.ListPodSandbox(&criapiv1.PodSandboxFilter{Id: sb.Id})
145+
require.NoError(t, err)
146+
require.Len(t, l, 1)
147+
require.Equal(t, l[0].State, criapiv1.PodSandboxState_SANDBOX_NOTREADY)
148+
149+
t.Log("Check PodSandboxStatus")
150+
sbStatus, err := runtimeService.PodSandboxStatus(sb.Id)
151+
require.NoError(t, err)
152+
t.Log(sbStatus.Network)
153+
require.Equal(t, sbStatus.State, criapiv1.PodSandboxState_SANDBOX_NOTREADY)
154+
}
155+
156+
t.Log("Cleanup leaky sandbox")
157+
err = runtimeService.RemovePodSandbox(sb.Id)
158+
require.NoError(t, err)
159+
}
160+
}
161+
162+
t.Run("CleanupAfterRestart", testCase(true))
163+
t.Run("JustCleanup", testCase(false))
164+
}
165+
166+
// TestRunPodSandboxWithShimStartAndTeardownCNIFailure should keep the sandbox
167+
// record if failed to rollback CNI API.
168+
func TestRunPodSandboxWithShimStartAndTeardownCNIFailure(t *testing.T) {
169+
if runtime.GOOS != "linux" {
170+
t.Skip()
171+
}
172+
if os.Getenv("ENABLE_CRI_SANDBOXES") != "" {
173+
t.Skip()
174+
}
175+
176+
testCase := func(restart bool) func(*testing.T) {
177+
return func(t *testing.T) {
178+
t.Log("Init PodSandboxConfig with specific key")
179+
labels := map[string]string{
180+
t.Name(): "true",
181+
}
182+
sbConfig := PodSandboxConfig(t.Name(), "failpoint", WithPodLabels(labels))
183+
184+
t.Log("Inject Shim failpoint")
185+
injectShimFailpoint(t, sbConfig, map[string]string{
186+
"Start": "1*error(failed to start shim)",
187+
})
188+
189+
t.Log("Inject CNI failpoint")
190+
conf := &failpointConf{
191+
Del: "1*error(please retry)",
192+
}
193+
injectCNIFailpoint(t, sbConfig, conf)
194+
195+
t.Log("Create a sandbox")
196+
_, err := runtimeService.RunPodSandbox(sbConfig, failpointRuntimeHandler)
197+
require.Error(t, err)
198+
require.Contains(t, err.Error(), "failed to start shim")
199+
200+
t.Log("ListPodSandbox with the specific label")
201+
l, err := runtimeService.ListPodSandbox(&criapiv1.PodSandboxFilter{LabelSelector: labels})
202+
require.NoError(t, err)
203+
require.Len(t, l, 1)
204+
205+
sb := l[0]
206+
require.Equal(t, sb.State, criapiv1.PodSandboxState_SANDBOX_NOTREADY)
207+
require.Equal(t, sb.Metadata.Name, sbConfig.Metadata.Name)
208+
require.Equal(t, sb.Metadata.Namespace, sbConfig.Metadata.Namespace)
209+
require.Equal(t, sb.Metadata.Uid, sbConfig.Metadata.Uid)
210+
require.Equal(t, sb.Metadata.Attempt, sbConfig.Metadata.Attempt)
211+
212+
if restart {
213+
t.Log("Restart containerd")
214+
RestartContainerd(t)
215+
216+
t.Log("ListPodSandbox with the specific label")
217+
l, err = runtimeService.ListPodSandbox(&criapiv1.PodSandboxFilter{Id: sb.Id})
218+
require.NoError(t, err)
219+
require.Len(t, l, 1)
220+
require.Equal(t, l[0].State, criapiv1.PodSandboxState_SANDBOX_NOTREADY)
221+
}
222+
223+
t.Log("Cleanup leaky sandbox")
224+
err = runtimeService.RemovePodSandbox(sb.Id)
225+
require.NoError(t, err)
226+
}
227+
}
228+
t.Run("CleanupAfterRestart", testCase(true))
229+
t.Run("JustCleanup", testCase(false))
230+
}
231+
92232
// failpointConf is used to describe cmdAdd/cmdDel/cmdCheck command's failpoint.
93233
type failpointConf struct {
94234
Add string `json:"cmdAdd"`
@@ -101,7 +241,7 @@ func injectCNIFailpoint(t *testing.T, sbConfig *criapiv1.PodSandboxConfig, conf
101241

102242
metadata := sbConfig.Metadata
103243
fpFilename := filepath.Join(stateDir,
104-
fmt.Sprintf("%s-%s.json", metadata.Namespace, metadata.Name))
244+
fmt.Sprintf("%s-%s.json", metadata.Namespace, strings.Replace(metadata.Name, "/", "-", -1)))
105245

106246
data, err := json.Marshal(conf)
107247
require.NoError(t, err)

0 commit comments

Comments
 (0)