Skip to content

Commit 7b61862

Browse files
fuweidthaJeztah
fuweid
authored and
thaJeztah
committed
*: add runc-fp as runc wrapper to inject failpoint
Signed-off-by: Wei Fu <[email protected]> (cherry picked from commit 11a7751) Signed-off-by: Sebastiaan van Stijn <[email protected]>
1 parent 5238a64 commit 7b61862

7 files changed

Lines changed: 273 additions & 0 deletions

File tree

‎Makefile‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -233,6 +233,11 @@ bin/cni-bridge-fp: integration/failpoint/cmd/cni-bridge-fp FORCE
233233
@echo "$(WHALE) $@"
234234
@$(GO) build ${GO_BUILD_FLAGS} -o $@ ./integration/failpoint/cmd/cni-bridge-fp
235235

236+
# build runc-fp as runc wrapper to support failpoint, only used by integration test
237+
bin/runc-fp: integration/failpoint/cmd/runc-fp FORCE
238+
@echo "$(WHALE) $@"
239+
@$(GO) build ${GO_BUILD_FLAGS} -o $@ ./integration/failpoint/cmd/runc-fp
240+
236241
benchmark: ## run benchmarks tests
237242
@echo "$(WHALE) $@"
238243
@$(GO) test ${TESTFLAGS} -bench . -run Benchmark -test.root

‎integration/client/container_linux_test.go‎

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,9 @@ import (
4141
"github.com/containerd/containerd/runtime/linux/runctypes"
4242
"github.com/containerd/containerd/runtime/v2/runc/options"
4343
"github.com/containerd/containerd/sys"
44+
4445
"github.com/opencontainers/runtime-spec/specs-go"
46+
"github.com/stretchr/testify/require"
4547
exec "golang.org/x/sys/execabs"
4648
"golang.org/x/sys/unix"
4749
)
@@ -1494,3 +1496,83 @@ func TestShimOOMScore(t *testing.T) {
14941496
case <-statusC:
14951497
}
14961498
}
1499+
1500+
// TestIssue9103 is used as regression case for issue 9103.
1501+
//
1502+
// The runc-fp will kill the init process so that the shim should return stopped
1503+
// status after container.NewTask. It's used to simulate that the runc-init
1504+
// might be killed by oom-kill.
1505+
func TestIssue9103(t *testing.T) {
1506+
if os.Getenv("RUNC_FLAVOR") == "crun" {
1507+
t.Skip("skip it when using crun")
1508+
}
1509+
if getRuntimeVersion() == "v1" {
1510+
t.Skip("skip it when using shim v1")
1511+
}
1512+
1513+
client, err := newClient(t, address)
1514+
require.NoError(t, err)
1515+
defer client.Close()
1516+
1517+
var (
1518+
image Image
1519+
ctx, cancel = testContext(t)
1520+
id = t.Name()
1521+
)
1522+
defer cancel()
1523+
1524+
image, err = client.GetImage(ctx, testImage)
1525+
require.NoError(t, err)
1526+
1527+
for idx, tc := range []struct {
1528+
desc string
1529+
cntrOpts []NewContainerOpts
1530+
expectedStatus ProcessStatus
1531+
}{
1532+
{
1533+
desc: "should be created status",
1534+
cntrOpts: []NewContainerOpts{
1535+
WithNewSpec(oci.WithImageConfig(image),
1536+
withProcessArgs("sleep", "30"),
1537+
),
1538+
},
1539+
expectedStatus: Created,
1540+
},
1541+
{
1542+
desc: "should be stopped status if init has been killed",
1543+
cntrOpts: []NewContainerOpts{
1544+
WithNewSpec(oci.WithImageConfig(image),
1545+
withProcessArgs("sleep", "30"),
1546+
oci.WithAnnotations(map[string]string{
1547+
"oci.runc.failpoint.profile": "issue9103",
1548+
}),
1549+
),
1550+
WithRuntime(client.Runtime(), &options.Options{
1551+
BinaryName: "runc-fp",
1552+
}),
1553+
},
1554+
expectedStatus: Stopped,
1555+
},
1556+
} {
1557+
tc := tc
1558+
tName := fmt.Sprintf("%s%d", id, idx)
1559+
t.Run(tc.desc, func(t *testing.T) {
1560+
container, err := client.NewContainer(ctx, tName,
1561+
append([]NewContainerOpts{WithNewSnapshot(tName, image)}, tc.cntrOpts...)...,
1562+
)
1563+
require.NoError(t, err)
1564+
defer container.Delete(ctx, WithSnapshotCleanup)
1565+
1566+
cctx, ccancel := context.WithTimeout(ctx, 30*time.Second)
1567+
task, err := container.NewTask(cctx, empty())
1568+
ccancel()
1569+
require.NoError(t, err)
1570+
1571+
defer task.Delete(ctx, WithProcessKill)
1572+
1573+
status, err := task.Status(ctx)
1574+
require.NoError(t, err)
1575+
require.Equal(t, status.Status, tc.expectedStatus)
1576+
})
1577+
}
1578+
}

‎integration/client/go.mod‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ require (
1717
github.com/opencontainers/image-spec v1.1.0-rc2.0.20221005185240-3a7f492d3f1b
1818
github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417
1919
github.com/sirupsen/logrus v1.9.3
20+
github.com/stretchr/testify v1.8.4
2021
golang.org/x/sys v0.7.0
2122
gotest.tools/v3 v3.5.0
2223
)
@@ -28,6 +29,7 @@ require (
2829
github.com/containerd/fifo v1.0.0 // indirect
2930
github.com/containerd/log v0.1.0 // indirect
3031
github.com/coreos/go-systemd/v22 v22.3.2 // indirect
32+
github.com/davecgh/go-spew v1.1.1 // indirect
3133
github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c // indirect
3234
github.com/docker/go-units v0.4.0 // indirect
3335
github.com/godbus/dbus/v5 v5.0.6 // indirect
@@ -45,13 +47,15 @@ require (
4547
github.com/opencontainers/selinux v1.10.1 // indirect
4648
github.com/pelletier/go-toml v1.9.5 // indirect
4749
github.com/pkg/errors v0.9.1 // indirect
50+
github.com/pmezard/go-difflib v1.0.0 // indirect
4851
go.opencensus.io v0.23.0 // indirect
4952
golang.org/x/net v0.8.0 // indirect
5053
golang.org/x/sync v0.1.0 // indirect
5154
golang.org/x/text v0.8.0 // indirect
5255
google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21 // indirect
5356
google.golang.org/grpc v1.50.1 // indirect
5457
google.golang.org/protobuf v1.28.1 // indirect
58+
gopkg.in/yaml.v3 v3.0.1 // indirect
5559
)
5660

5761
replace (

‎integration/client/go.sum‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1040,6 +1040,7 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8
10401040
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
10411041
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
10421042
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
1043+
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
10431044
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
10441045
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
10451046
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=

‎integration/failpoint/cmd/runc-fp/issue9103.go‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
//go:build linux
2+
3+
/*
4+
Copyright The containerd Authors.
5+
6+
Licensed under the Apache License, Version 2.0 (the "License");
7+
you may not use this file except in compliance with the License.
8+
You may obtain a copy of the License at
9+
10+
http://www.apache.org/licenses/LICENSE-2.0
11+
12+
Unless required by applicable law or agreed to in writing, software
13+
distributed under the License is distributed on an "AS IS" BASIS,
14+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
See the License for the specific language governing permissions and
16+
limitations under the License.
17+
*/
18+
19+
package main
20+
21+
import (
22+
"context"
23+
"fmt"
24+
"os"
25+
"strconv"
26+
"strings"
27+
"syscall"
28+
"time"
29+
)
30+
31+
// issue9103KillInitAfterCreate kills the runc.Init process after creating
32+
// command returns successfully.
33+
//
34+
// REF: https://github.com/containerd/containerd/issues/9103
35+
func issue9103KillInitAfterCreate(ctx context.Context, method invoker) error {
36+
isCreated := strings.Contains(strings.Join(os.Args, ","), ",create,")
37+
38+
if err := method(ctx); err != nil {
39+
return err
40+
}
41+
42+
if !isCreated {
43+
return nil
44+
}
45+
46+
initPidPath := "init.pid"
47+
data, err := os.ReadFile(initPidPath)
48+
if err != nil {
49+
return fmt.Errorf("failed to read %s: %w", initPidPath, err)
50+
}
51+
52+
pid, err := strconv.Atoi(string(data))
53+
if err != nil {
54+
return fmt.Errorf("failed to get init pid from string %s: %w", string(data), err)
55+
}
56+
57+
if pid <= 0 {
58+
return fmt.Errorf("unexpected init pid %v", pid)
59+
}
60+
61+
if err := syscall.Kill(pid, syscall.SIGKILL); err != nil {
62+
return fmt.Errorf("failed to kill the init pid %v: %w", pid, err)
63+
}
64+
65+
// Ensure that the containerd-shim has received the SIGCHLD and start
66+
// to cleanup
67+
time.Sleep(3 * time.Second)
68+
return nil
69+
}

‎integration/failpoint/cmd/runc-fp/main.go‎

Lines changed: 108 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,108 @@
1+
//go:build linux
2+
3+
/*
4+
Copyright The containerd Authors.
5+
6+
Licensed under the Apache License, Version 2.0 (the "License");
7+
you may not use this file except in compliance with the License.
8+
You may obtain a copy of the License at
9+
10+
http://www.apache.org/licenses/LICENSE-2.0
11+
12+
Unless required by applicable law or agreed to in writing, software
13+
distributed under the License is distributed on an "AS IS" BASIS,
14+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
See the License for the specific language governing permissions and
16+
limitations under the License.
17+
*/
18+
19+
package main
20+
21+
import (
22+
"context"
23+
"fmt"
24+
"os"
25+
"os/exec"
26+
"syscall"
27+
28+
"github.com/containerd/containerd/oci"
29+
"github.com/sirupsen/logrus"
30+
)
31+
32+
const (
33+
failpointProfileKey = "oci.runc.failpoint.profile"
34+
)
35+
36+
type invoker func(context.Context) error
37+
38+
type invokerInterceptor func(context.Context, invoker) error
39+
40+
var (
41+
failpointProfiles = map[string]invokerInterceptor{
42+
"issue9103": issue9103KillInitAfterCreate,
43+
}
44+
)
45+
46+
// setupLog setups messages into log file.
47+
func setupLog() {
48+
// containerd/go-runc always add --log option
49+
idx := 2
50+
for ; idx < len(os.Args); idx++ {
51+
if os.Args[idx] == "--log" {
52+
break
53+
}
54+
}
55+
56+
if idx >= len(os.Args)-1 || os.Args[idx] != "--log" {
57+
panic("option --log required")
58+
}
59+
60+
logFile := os.Args[idx+1]
61+
f, err := os.OpenFile(logFile, os.O_CREATE|os.O_WRONLY|os.O_APPEND|os.O_SYNC, 0o644)
62+
if err != nil {
63+
panic(fmt.Errorf("failed to open %s: %w", logFile, err))
64+
}
65+
66+
logrus.SetOutput(f)
67+
logrus.SetFormatter(new(logrus.JSONFormatter))
68+
}
69+
70+
func main() {
71+
setupLog()
72+
73+
fpProfile, err := failpointProfileFromOCIAnnotation()
74+
if err != nil {
75+
logrus.WithError(err).Fatal("failed to get failpoint profile")
76+
}
77+
78+
ctx := context.Background()
79+
if err := fpProfile(ctx, defaultRuncInvoker); err != nil {
80+
logrus.WithError(err).Fatal("failed to exec failpoint profile")
81+
}
82+
}
83+
84+
// defaultRuncInvoker is to call the runc command with same arguments.
85+
func defaultRuncInvoker(ctx context.Context) error {
86+
cmd := exec.CommandContext(ctx, "runc", os.Args[1:]...)
87+
cmd.SysProcAttr = &syscall.SysProcAttr{Pdeathsig: syscall.SIGKILL}
88+
return cmd.Run()
89+
}
90+
91+
// failpointProfileFromOCIAnnotation gets the profile from OCI annotations.
92+
func failpointProfileFromOCIAnnotation() (invokerInterceptor, error) {
93+
spec, err := oci.ReadSpec(oci.ConfigFilename)
94+
if err != nil {
95+
return nil, fmt.Errorf("failed to read %s: %w", oci.ConfigFilename, err)
96+
}
97+
98+
profileName, ok := spec.Annotations[failpointProfileKey]
99+
if !ok {
100+
return nil, fmt.Errorf("failpoint profile is required")
101+
}
102+
103+
fp, ok := failpointProfiles[profileName]
104+
if !ok {
105+
return nil, fmt.Errorf("no such failpoint profile %s", profileName)
106+
}
107+
return fp, nil
108+
}

‎script/setup/install-failpoint-binaries‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,3 +33,7 @@ sudo install bin/cni-bridge-fp "${CNI_BIN_DIR}"
3333
SHIM_BIN_DIR=${SHIM_BIN_DIR:-"/usr/local/bin"}
3434
make bin/containerd-shim-runc-fp-v1
3535
sudo install bin/containerd-shim-runc-fp-v1 "${SHIM_BIN_DIR}"
36+
37+
RUNCFP_BIN_DIR=${RUNCFP_BIN_DIR:-"/usr/local/bin"}
38+
make bin/runc-fp
39+
sudo install bin/runc-fp "${RUNCFP_BIN_DIR}"

0 commit comments

Comments
 (0)