Skip to content

Commit a7f956d

Browse files
fuweidqiutongs
fuweid
authored and
qiutongs
committed
integration: CNI bridge wrapper with failpoint
Introduce cni-bridge-fp as CNI bridge plugin wrapper binary for CRI testing. With CNI `io.kubernetes.cri.pod-annotations` capability enabled, the user can inject the failpoint setting by pod's annotation `cniFailpointControlStateDir`, which stores each pod's failpoint setting named by `${K8S_POD_NAMESPACE}-${K8S_POD_NAME}.json`. When the plugin is invoked, the plugin will check the CNI_ARGS to get the failpoint for the CNI_COMMAND from disk. For the testing, the user can prepare setting before RunPodSandbox. Signed-off-by: Wei Fu <[email protected]> (cherry picked from commit be91a21) Signed-off-by: Qiutong Song <[email protected]>
1 parent 07c4794 commit a7f956d

2 files changed

Lines changed: 215 additions & 0 deletions

File tree

‎Makefile‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -228,6 +228,11 @@ bin/containerd-shim-runc-fp-v1: integration/failpoint/cmd/containerd-shim-runc-f
228228
@echo "$(WHALE) $@"
229229
@CGO_ENABLED=${SHIM_CGO_ENABLED} $(GO) build ${GO_BUILD_FLAGS} -o $@ ${SHIM_GO_LDFLAGS} ${GO_TAGS} ./integration/failpoint/cmd/containerd-shim-runc-fp-v1
230230

231+
# build CNI bridge plugin wrapper with failpoint support, only used by integration test
232+
bin/cni-bridge-fp: integration/failpoint/cmd/cni-bridge-fp FORCE
233+
@echo "$(WHALE) $@"
234+
@$(GO) build ${GO_BUILD_FLAGS} -o $@ ./integration/failpoint/cmd/cni-bridge-fp
235+
231236
benchmark: ## run benchmarks tests
232237
@echo "$(WHALE) $@"
233238
@$(GO) test ${TESTFLAGS} -bench . -run Benchmark -test.root
@@ -374,6 +379,7 @@ clean-test: ## clean up debris from previously failed tests
374379
@rm -rf /run/containerd/fifo/*
375380
@rm -rf /run/containerd-test/*
376381
@rm -rf bin/cri-integration.test
382+
@rm -rf bin/cni-bridge-fp
377383
@rm -rf bin/containerd-shim-runc-fp-v1
378384

379385
install: ## install binaries

‎integration/failpoint/cmd/cni-bridge-fp/main.go‎

Lines changed: 209 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,209 @@
1+
//go:build linux
2+
// +build linux
3+
4+
/*
5+
Copyright The containerd Authors.
6+
7+
Licensed under the Apache License, Version 2.0 (the "License");
8+
you may not use this file except in compliance with the License.
9+
You may obtain a copy of the License at
10+
11+
http://www.apache.org/licenses/LICENSE-2.0
12+
13+
Unless required by applicable law or agreed to in writing, software
14+
distributed under the License is distributed on an "AS IS" BASIS,
15+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16+
See the License for the specific language governing permissions and
17+
limitations under the License.
18+
*/
19+
20+
package main
21+
22+
import (
23+
"bytes"
24+
"encoding/json"
25+
"fmt"
26+
"io/ioutil"
27+
"os"
28+
"os/exec"
29+
"path/filepath"
30+
"strings"
31+
"syscall"
32+
33+
"github.com/containerd/containerd/pkg/failpoint"
34+
"github.com/containerd/continuity"
35+
"github.com/sirupsen/logrus"
36+
)
37+
38+
type inheritedPodAnnotations struct {
39+
// CNIFailpointControlStateDir is used to specify the location of
40+
// failpoint control setting. In that such stateDir, the failpoint
41+
// setting is stored in the json file named by
42+
// `${K8S_POD_NAMESPACE}-${K8S_POD_NAME}.json`. The detail of json file
43+
// is described by FailpointConf.
44+
CNIFailpointControlStateDir string `json:"cniFailpointControlStateDir,omitempty"`
45+
}
46+
47+
// FailpointConf is used to describe cmdAdd/cmdDel/cmdCheck command's failpoint.
48+
type FailpointConf struct {
49+
Add string `json:"cmdAdd"`
50+
Del string `json:"cmdDel"`
51+
Check string `json:"cmdCheck"`
52+
}
53+
54+
type netConf struct {
55+
RuntimeConfig struct {
56+
PodAnnotations inheritedPodAnnotations `json:"io.kubernetes.cri.pod-annotations"`
57+
} `json:"runtimeConfig,omitempty"`
58+
}
59+
60+
func main() {
61+
stdinData, err := ioutil.ReadAll(os.Stdin)
62+
if err != nil {
63+
logrus.Fatalf("failed to read stdin: %v", err)
64+
}
65+
66+
var conf netConf
67+
if err := json.Unmarshal(stdinData, &conf); err != nil {
68+
logrus.Fatalf("failed to parse network configuration: %v", err)
69+
}
70+
71+
cniCmd, ok := os.LookupEnv("CNI_COMMAND")
72+
if !ok {
73+
logrus.Fatal("required env CNI_COMMAND")
74+
}
75+
76+
cniPath, ok := os.LookupEnv("CNI_PATH")
77+
if !ok {
78+
logrus.Fatal("required env CNI_PATH")
79+
}
80+
81+
evalFn, err := buildFailpointEval(conf.RuntimeConfig.PodAnnotations.CNIFailpointControlStateDir, cniCmd)
82+
if err != nil {
83+
logrus.Fatalf("failed to build failpoint evaluate function: %v", err)
84+
}
85+
86+
if err := evalFn(); err != nil {
87+
logrus.Fatalf("failpoint: %v", err)
88+
}
89+
90+
cmd := exec.Command(filepath.Join(cniPath, "bridge"))
91+
cmd.Stdin = bytes.NewReader(stdinData)
92+
cmd.Stdout = os.Stdout
93+
cmd.Stderr = os.Stderr
94+
95+
if err := cmd.Start(); err != nil {
96+
logrus.Fatalf("failed to start bridge cni plugin: %v", err)
97+
}
98+
99+
if err := cmd.Wait(); err != nil {
100+
logrus.Fatalf("failed to wait for bridge cni plugin: %v", err)
101+
}
102+
}
103+
104+
// buildFailpointEval will read and update the failpoint setting and then
105+
// return delegated failpoint evaluate function
106+
func buildFailpointEval(stateDir string, cniCmd string) (failpoint.EvalFn, error) {
107+
cniArgs, ok := os.LookupEnv("CNI_ARGS")
108+
if !ok {
109+
return nopEvalFn, nil
110+
}
111+
112+
target := buildPodFailpointFilepath(stateDir, cniArgs)
113+
if target == "" {
114+
return nopEvalFn, nil
115+
}
116+
117+
f, err := os.OpenFile(target, os.O_RDWR, 0666)
118+
if err != nil {
119+
if os.IsNotExist(err) {
120+
return nopEvalFn, nil
121+
}
122+
return nil, fmt.Errorf("failed to open file %s: %w", target, err)
123+
}
124+
defer f.Close()
125+
126+
if err := syscall.Flock(int(f.Fd()), syscall.LOCK_EX); err != nil {
127+
return nil, fmt.Errorf("failed to lock failpoint setting %s: %w", target, err)
128+
}
129+
defer syscall.Flock(int(f.Fd()), syscall.LOCK_UN)
130+
131+
data, err := ioutil.ReadAll(f)
132+
if err != nil {
133+
return nil, fmt.Errorf("failed to read failpoint setting %s: %w", target, err)
134+
}
135+
136+
var conf FailpointConf
137+
if err := json.Unmarshal(data, &conf); err != nil {
138+
return nil, fmt.Errorf("failed to unmarshal failpoint conf %s: %w", string(data), err)
139+
}
140+
141+
var fpStr *string
142+
switch cniCmd {
143+
case "ADD":
144+
fpStr = &conf.Add
145+
case "DEL":
146+
fpStr = &conf.Del
147+
case "CHECK":
148+
fpStr = &conf.Check
149+
}
150+
151+
if fpStr == nil || *fpStr == "" {
152+
return nopEvalFn, nil
153+
}
154+
155+
fp, err := failpoint.NewFailpoint(cniCmd, *fpStr)
156+
if err != nil {
157+
return nil, fmt.Errorf("failed to parse failpoint %s: %w", *fpStr, err)
158+
}
159+
160+
evalFn := fp.DelegatedEval()
161+
162+
*fpStr = fp.Marshal()
163+
164+
data, err = json.Marshal(conf)
165+
if err != nil {
166+
return nil, fmt.Errorf("failed to marshal failpoint conf: %w", err)
167+
}
168+
return evalFn, continuity.AtomicWriteFile(target, data, 0666)
169+
}
170+
171+
// buildPodFailpointFilepath returns the expected failpoint setting filepath
172+
// by Pod metadata.
173+
func buildPodFailpointFilepath(stateDir, cniArgs string) string {
174+
args := cniArgsIntoKeyValue(cniArgs)
175+
176+
res := make([]string, 0, 2)
177+
for _, key := range []string{"K8S_POD_NAMESPACE", "K8S_POD_NAME"} {
178+
v, ok := args[key]
179+
if !ok {
180+
break
181+
}
182+
res = append(res, v)
183+
}
184+
if len(res) != 2 {
185+
return ""
186+
}
187+
return filepath.Join(stateDir, strings.Join(res, "-")+".json")
188+
}
189+
190+
// cniArgsIntoKeyValue converts the CNI ARGS from `key1=value1;key2=value2...`
191+
// into key/value hashmap.
192+
func cniArgsIntoKeyValue(envStr string) map[string]string {
193+
parts := strings.Split(envStr, ";")
194+
res := make(map[string]string, len(parts))
195+
196+
for _, part := range parts {
197+
keyValue := strings.SplitN(part, "=", 2)
198+
if len(keyValue) != 2 {
199+
continue
200+
}
201+
202+
res[keyValue[0]] = keyValue[1]
203+
}
204+
return res
205+
}
206+
207+
func nopEvalFn() error {
208+
return nil
209+
}

0 commit comments

Comments
 (0)