Skip to content

Commit 71ee7de

Browse files
fuweidqiutongs
fuweid
authored and
qiutongs
committed
bin/ctr,integration: new runc-shim with failpoint
Added new runc shim binary in integration testing. The shim is named by io.containerd.runc-fp.v1, which allows us to use additional OCI annotation `io.containerd.runtime.v2.shim.failpoint.*` to setup shim task API's failpoint. Since the shim can be shared with multiple container, like what kubernetes pod does, the failpoint will be initialized during setup the shim server. So, the following the container's OCI failpoint's annotation will not work. This commit also updates the ctr tool that we can use `--annotation` to specify annotations when run container. For example: ```bash ➜ ctr run -d --runtime runc-fp.v1 \ --annotation "io.containerd.runtime.v2.shim.failpoint.Kill=1*error(sorry)" \ docker.io/library/alpine:latest testing sleep 1d ➜ ctr t ls TASK PID STATUS testing 147304 RUNNING ➜ ctr t kill -s SIGKILL testing ctr: sorry: unknown ➜ ctr t kill -s SIGKILL testing ➜ sudo ctr t ls TASK PID STATUS testing 147304 STOPPED ``` The runc-fp.v1 shim is based on core runc.v2. We can use it to inject failpoint during testing complicated or big transcation API, like kubernetes PodRunPodsandbox. Signed-off-by: Wei Fu <[email protected]> (cherry picked from commit 5f9b318) Signed-off-by: Qiutong Song <[email protected]>
1 parent 3e2e778 commit 71ee7de

5 files changed

Lines changed: 205 additions & 0 deletions

File tree

‎Makefile‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -223,6 +223,11 @@ cri-integration: binaries bin/cri-integration.test ## run cri integration tests
223223
@bash -x ./script/test/cri-integration.sh
224224
@rm -rf bin/cri-integration.test
225225

226+
# build runc shimv2 with failpoint control, only used by integration test
227+
bin/containerd-shim-runc-fp-v1: integration/failpoint/cmd/containerd-shim-runc-fp-v1 FORCE
228+
@echo "$(WHALE) $@"
229+
@CGO_ENABLED=${SHIM_CGO_ENABLED} $(GO) build ${GO_BUILD_FLAGS} -o $@ ${SHIM_GO_LDFLAGS} ${GO_TAGS} ./integration/failpoint/cmd/containerd-shim-runc-fp-v1
230+
226231
benchmark: ## run benchmarks tests
227232
@echo "$(WHALE) $@"
228233
@$(GO) test ${TESTFLAGS} -bench . -run Benchmark -test.root
@@ -369,6 +374,7 @@ clean-test: ## clean up debris from previously failed tests
369374
@rm -rf /run/containerd/fifo/*
370375
@rm -rf /run/containerd-test/*
371376
@rm -rf bin/cri-integration.test
377+
@rm -rf bin/containerd-shim-runc-fp-v1
372378

373379
install: ## install binaries
374380
@echo "$(WHALE) $@ $(BINARIES)"

‎cmd/ctr/commands/commands.go‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -116,6 +116,10 @@ var (
116116
Name: "label",
117117
Usage: "specify additional labels (e.g. foo=bar)",
118118
},
119+
cli.StringSliceFlag{
120+
Name: "annotation",
121+
Usage: "specify additional OCI annotations (e.g. foo=bar)",
122+
},
119123
cli.StringSliceFlag{
120124
Name: "mount",
121125
Usage: "specify additional container mount (e.g. type=bind,src=/tmp,dst=/host,options=rbind:ro)",
@@ -227,6 +231,19 @@ func LabelArgs(labelStrings []string) map[string]string {
227231
return labels
228232
}
229233

234+
// AnnotationArgs returns a map of annotation key,value pairs.
235+
func AnnotationArgs(annoStrings []string) (map[string]string, error) {
236+
annotations := make(map[string]string, len(annoStrings))
237+
for _, anno := range annoStrings {
238+
parts := strings.SplitN(anno, "=", 2)
239+
if len(parts) != 2 {
240+
return nil, fmt.Errorf("invalid key=value format annotation: %v", anno)
241+
}
242+
annotations[parts[0]] = parts[1]
243+
}
244+
return annotations, nil
245+
}
246+
230247
// PrintAsJSON prints input in JSON format
231248
func PrintAsJSON(x interface{}) {
232249
b, err := json.MarshalIndent(x, "", " ")

‎cmd/ctr/commands/run/run_unix.go‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -213,6 +213,13 @@ func NewContainer(ctx gocontext.Context, client *containerd.Client, context *cli
213213
oci.WithEnv([]string{fmt.Sprintf("HOSTNAME=%s", hostname)}),
214214
)
215215
}
216+
if annoStrings := context.StringSlice("annotation"); len(annoStrings) > 0 {
217+
annos, err := commands.AnnotationArgs(annoStrings)
218+
if err != nil {
219+
return nil, err
220+
}
221+
opts = append(opts, oci.WithAnnotations(annos))
222+
}
216223

217224
if caps := context.StringSlice("cap-add"); len(caps) > 0 {
218225
for _, cap := range caps {

‎integration/failpoint/cmd/containerd-shim-runc-fp-v1/main.go‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
//go:build linux
2+
// +build linux
3+
4+
/*
5+
Copyright The containerd Authors.
6+
7+
Licensed under the Apache License, Version 2.0 (the "License");
8+
you may not use this file except in compliance with the License.
9+
You may obtain a copy of the License at
10+
11+
http://www.apache.org/licenses/LICENSE-2.0
12+
13+
Unless required by applicable law or agreed to in writing, software
14+
distributed under the License is distributed on an "AS IS" BASIS,
15+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16+
See the License for the specific language governing permissions and
17+
limitations under the License.
18+
*/
19+
20+
package main
21+
22+
import (
23+
"context"
24+
25+
"github.com/containerd/containerd/runtime/v2/runc/manager"
26+
"github.com/containerd/containerd/runtime/v2/shim"
27+
)
28+
29+
func main() {
30+
shim.RunManager(context.Background(), manager.NewShimManager("io.containerd.runc-fp.v1"))
31+
}

‎integration/failpoint/cmd/containerd-shim-runc-fp-v1/plugin.go‎

Lines changed: 144 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,144 @@
1+
//go:build linux
2+
// +build linux
3+
4+
/*
5+
Copyright The containerd Authors.
6+
7+
Licensed under the Apache License, Version 2.0 (the "License");
8+
you may not use this file except in compliance with the License.
9+
You may obtain a copy of the License at
10+
11+
http://www.apache.org/licenses/LICENSE-2.0
12+
13+
Unless required by applicable law or agreed to in writing, software
14+
distributed under the License is distributed on an "AS IS" BASIS,
15+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16+
See the License for the specific language governing permissions and
17+
limitations under the License.
18+
*/
19+
20+
package main
21+
22+
import (
23+
"context"
24+
"encoding/json"
25+
"fmt"
26+
"os"
27+
"path/filepath"
28+
"strings"
29+
30+
"github.com/containerd/containerd/oci"
31+
"github.com/containerd/containerd/pkg/failpoint"
32+
"github.com/containerd/containerd/pkg/shutdown"
33+
"github.com/containerd/containerd/plugin"
34+
"github.com/containerd/containerd/runtime/v2/runc/task"
35+
"github.com/containerd/containerd/runtime/v2/shim"
36+
taskapi "github.com/containerd/containerd/runtime/v2/task"
37+
"github.com/containerd/ttrpc"
38+
)
39+
40+
const (
41+
ociConfigFilename = "config.json"
42+
43+
failpointPrefixKey = "io.containerd.runtime.v2.shim.failpoint."
44+
)
45+
46+
func init() {
47+
plugin.Register(&plugin.Registration{
48+
Type: plugin.TTRPCPlugin,
49+
ID: "task",
50+
Requires: []plugin.Type{
51+
plugin.EventPlugin,
52+
plugin.InternalPlugin,
53+
},
54+
InitFn: func(ic *plugin.InitContext) (interface{}, error) {
55+
pp, err := ic.GetByID(plugin.EventPlugin, "publisher")
56+
if err != nil {
57+
return nil, err
58+
}
59+
ss, err := ic.GetByID(plugin.InternalPlugin, "shutdown")
60+
if err != nil {
61+
return nil, err
62+
}
63+
fps, err := newFailpointFromOCIAnnotation()
64+
if err != nil {
65+
return nil, err
66+
}
67+
service, err := task.NewTaskService(ic.Context, pp.(shim.Publisher), ss.(shutdown.Service))
68+
if err != nil {
69+
return nil, err
70+
}
71+
72+
return &taskServiceWithFp{
73+
fps: fps,
74+
local: service,
75+
}, nil
76+
},
77+
})
78+
79+
}
80+
81+
type taskServiceWithFp struct {
82+
fps map[string]*failpoint.Failpoint
83+
local taskapi.TaskService
84+
}
85+
86+
func (s *taskServiceWithFp) RegisterTTRPC(server *ttrpc.Server) error {
87+
taskapi.RegisterTaskService(server, s.local)
88+
return nil
89+
}
90+
91+
func (s *taskServiceWithFp) UnaryInterceptor() ttrpc.UnaryServerInterceptor {
92+
return func(ctx context.Context, unmarshal ttrpc.Unmarshaler, info *ttrpc.UnaryServerInfo, method ttrpc.Method) (interface{}, error) {
93+
methodName := filepath.Base(info.FullMethod)
94+
if fp, ok := s.fps[methodName]; ok {
95+
if err := fp.Evaluate(); err != nil {
96+
return nil, err
97+
}
98+
}
99+
return method(ctx, unmarshal)
100+
}
101+
}
102+
103+
// newFailpointFromOCIAnnotation reloads and parses the annotation from
104+
// bundle-path/config.json.
105+
//
106+
// The annotation controlling task API's failpoint should be like:
107+
//
108+
// io.containerd.runtime.v2.shim.failpoint.Create = 1*off->1*error(please retry)
109+
//
110+
// The `Create` is the shim unary API and the value of annotation is the
111+
// failpoint control. The function will return a set of failpoint controllers.
112+
func newFailpointFromOCIAnnotation() (map[string]*failpoint.Failpoint, error) {
113+
// NOTE: shim's current working dir is in bundle dir.
114+
cwd, err := os.Getwd()
115+
if err != nil {
116+
return nil, fmt.Errorf("failed to get current working dir: %w", err)
117+
}
118+
119+
configPath := filepath.Join(cwd, ociConfigFilename)
120+
data, err := os.ReadFile(configPath)
121+
if err != nil {
122+
return nil, fmt.Errorf("failed to read %v: %w", configPath, err)
123+
}
124+
125+
var spec oci.Spec
126+
if err := json.Unmarshal(data, &spec); err != nil {
127+
return nil, fmt.Errorf("failed to parse oci.Spec(%v): %w", string(data), err)
128+
}
129+
130+
res := make(map[string]*failpoint.Failpoint)
131+
for k, v := range spec.Annotations {
132+
if !strings.HasPrefix(k, failpointPrefixKey) {
133+
continue
134+
}
135+
136+
methodName := strings.TrimPrefix(k, failpointPrefixKey)
137+
fp, err := failpoint.NewFailpoint(methodName, v)
138+
if err != nil {
139+
return nil, fmt.Errorf("failed to parse failpoint %v: %w", v, err)
140+
}
141+
res[methodName] = fp
142+
}
143+
return res, nil
144+
}

0 commit comments

Comments
 (0)