Skip to content

Commit ed47d6b

Browse files
marquizmarquiz
committed
cri: implement RuntimeConfig rpc
The rpc only reports one field, i.e. the cgroup driver, to kubelet. Containerd determines the effective cgroup driver by looking at all runtime handlers, starting from the default runtime handler (the rest in alphabetical order), and returning the cgroup driver setting of the first runtime handler that supports one. If no runtime handler supports cgroup driver (i.e. has a config option for it) containerd falls back to auto-detection, returning systemd if systemd is running and cgroupfs otherwise. This patch implements the CRI server side of Kubernetes KEP-4033: https://github.com/kubernetes/enhancements/tree/master/keps/sig-node/4033-group-driver-detection-over-cri Signed-off-by: Markus Lehtonen <[email protected]>
1 parent 850b2e1 commit ed47d6b

80 files changed

Lines changed: 9669 additions & 0 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎go.sum‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -887,6 +887,7 @@ github.com/sclevine/spec v1.2.0/go.mod h1:W4J29eT/Kzv7/b9IWLB055Z+qvVC9vt0Arko24
887887
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
888888
github.com/seccomp/libseccomp-golang v0.9.1/go.mod h1:GbW5+tmTXfcxTToHLXlScSlAvWlF4P2Ca7zGrPiEpWo=
889889
github.com/seccomp/libseccomp-golang v0.9.2-0.20210429002308-3879420cc921/go.mod h1:JA8cRccbGaA1s33RQf7Y1+q9gHmZX1yB/z9WDN1C6fg=
890+
github.com/seccomp/libseccomp-golang v0.9.2-0.20220502022130-f33da4d89646 h1:RpforrEYXWkmGwJHIGnLZ3tTWStkjVVstwzNGqxX2Ds=
890891
github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
891892
github.com/sirupsen/logrus v1.0.4-0.20170822132746-89742aefa4b2/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc=
892893
github.com/sirupsen/logrus v1.0.6/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc=

‎pkg/cri/instrument/instrumented_service.go‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -621,3 +621,19 @@ func (in *instrumentedService) ListPodSandboxMetrics(ctx context.Context, r *run
621621
res, err = in.c.ListPodSandboxMetrics(ctx, r)
622622
return res, errdefs.ToGRPC(err)
623623
}
624+
625+
func (in *instrumentedService) RuntimeConfig(ctx context.Context, r *runtime.RuntimeConfigRequest) (res *runtime.RuntimeConfigResponse, err error) {
626+
if err := in.checkInitialized(); err != nil {
627+
return nil, err
628+
}
629+
log.G(ctx).Tracef("RuntimeConfig")
630+
defer func() {
631+
if err != nil {
632+
log.G(ctx).WithError(err).Error("RuntimeConfig failed")
633+
} else {
634+
log.G(ctx).Tracef("RuntimeConfig returns config %+v", res)
635+
}
636+
}()
637+
res, err = in.c.RuntimeConfig(ctx, r)
638+
return res, errdefs.ToGRPC(err)
639+
}

‎pkg/cri/sbserver/runtime_config.go‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
/*
2+
Copyright The containerd Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package sbserver
18+
19+
import (
20+
"context"
21+
22+
runtime "k8s.io/cri-api/pkg/apis/runtime/v1"
23+
)
24+
25+
// RuntimeConfig returns configuration information of the runtime.
26+
func (c *criService) RuntimeConfig(ctx context.Context, r *runtime.RuntimeConfigRequest) (*runtime.RuntimeConfigResponse, error) {
27+
resp := &runtime.RuntimeConfigResponse{
28+
Linux: c.getLinuxRuntimeConfig(ctx),
29+
}
30+
return resp, nil
31+
}

‎pkg/cri/sbserver/runtime_config_linux.go‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
/*
2+
Copyright The containerd Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package sbserver
18+
19+
import (
20+
"context"
21+
"sort"
22+
23+
"github.com/containerd/containerd/log"
24+
runcoptions "github.com/containerd/containerd/runtime/v2/runc/options"
25+
"github.com/opencontainers/runc/libcontainer/cgroups/systemd"
26+
runtime "k8s.io/cri-api/pkg/apis/runtime/v1"
27+
)
28+
29+
func (c *criService) getLinuxRuntimeConfig(ctx context.Context) *runtime.LinuxRuntimeConfiguration {
30+
return &runtime.LinuxRuntimeConfiguration{CgroupDriver: c.getCgroupDriver(ctx)}
31+
}
32+
33+
func (c *criService) getCgroupDriver(ctx context.Context) runtime.CgroupDriver {
34+
// Go through the runtime handlers in a predictable order, starting from the
35+
// default handler, others sorted in alphabetical order
36+
handlerNames := make([]string, 0, len(c.config.ContainerdConfig.Runtimes))
37+
for n := range c.config.ContainerdConfig.Runtimes {
38+
handlerNames = append(handlerNames, n)
39+
}
40+
sort.Slice(handlerNames, func(i, j int) bool {
41+
if handlerNames[i] == c.config.ContainerdConfig.DefaultRuntimeName {
42+
return true
43+
}
44+
if handlerNames[j] == c.config.ContainerdConfig.DefaultRuntimeName {
45+
return false
46+
}
47+
return handlerNames[i] < handlerNames[j]
48+
})
49+
50+
for _, handler := range handlerNames {
51+
opts, err := generateRuntimeOptions(c.config.ContainerdConfig.Runtimes[handler])
52+
if err != nil {
53+
log.G(ctx).Debugf("failed to parse runtime handler options for %q", handler)
54+
continue
55+
}
56+
if d, ok := getCgroupDriverFromRuntimeHandlerOpts(opts); ok {
57+
return d
58+
}
59+
log.G(ctx).Debugf("runtime handler %q does not provide cgroup driver information", handler)
60+
}
61+
62+
// If no runtime handlers have a setting, detect if systemd is running
63+
d := runtime.CgroupDriver_CGROUPFS
64+
if systemd.IsRunningSystemd() {
65+
d = runtime.CgroupDriver_SYSTEMD
66+
}
67+
log.G(ctx).Debugf("no runtime handler provided cgroup driver setting, using auto-detected %s", runtime.CgroupDriver_name[int32(d)])
68+
return d
69+
}
70+
71+
func getCgroupDriverFromRuntimeHandlerOpts(opts interface{}) (runtime.CgroupDriver, bool) {
72+
switch v := opts.(type) {
73+
case *runcoptions.Options:
74+
systemdCgroup := v.SystemdCgroup
75+
if systemdCgroup {
76+
return runtime.CgroupDriver_SYSTEMD, true
77+
}
78+
return runtime.CgroupDriver_CGROUPFS, true
79+
}
80+
return runtime.CgroupDriver_SYSTEMD, false
81+
}

‎pkg/cri/sbserver/runtime_config_linux_test.go‎

Lines changed: 105 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,105 @@
1+
/*
2+
Copyright The containerd Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package sbserver
18+
19+
import (
20+
"context"
21+
"testing"
22+
23+
criconfig "github.com/containerd/containerd/pkg/cri/config"
24+
"github.com/containerd/containerd/plugin"
25+
"github.com/opencontainers/runc/libcontainer/cgroups/systemd"
26+
"github.com/stretchr/testify/assert"
27+
runtime "k8s.io/cri-api/pkg/apis/runtime/v1"
28+
)
29+
30+
func newFakeRuntimeConfig(runcV2, systemdCgroup bool) criconfig.Runtime {
31+
r := criconfig.Runtime{Type: "default", Options: map[string]interface{}{}}
32+
if runcV2 {
33+
r.Type = plugin.RuntimeRuncV2
34+
if systemdCgroup {
35+
r.Options["SystemdCgroup"] = true
36+
}
37+
}
38+
return r
39+
}
40+
41+
func TestRuntimeConfig(t *testing.T) {
42+
autoDetected := runtime.CgroupDriver_CGROUPFS
43+
if systemd.IsRunningSystemd() {
44+
autoDetected = runtime.CgroupDriver_SYSTEMD
45+
}
46+
47+
for _, test := range []struct {
48+
desc string
49+
defaultRuntime string
50+
runtimes map[string]criconfig.Runtime
51+
expectedCgroupDriver runtime.CgroupDriver
52+
}{
53+
{
54+
desc: "no runtimes",
55+
expectedCgroupDriver: autoDetected,
56+
},
57+
{
58+
desc: "non-runc runtime",
59+
defaultRuntime: "non-runc",
60+
runtimes: map[string]criconfig.Runtime{"non-runc": newFakeRuntimeConfig(false, false)},
61+
expectedCgroupDriver: autoDetected,
62+
},
63+
{
64+
desc: "no default, pick first in alphabetical order",
65+
runtimes: map[string]criconfig.Runtime{
66+
"non-runc": newFakeRuntimeConfig(false, false),
67+
"runc-2": newFakeRuntimeConfig(true, true),
68+
"runc": newFakeRuntimeConfig(true, false),
69+
"non-runc-2": newFakeRuntimeConfig(false, false),
70+
},
71+
expectedCgroupDriver: runtime.CgroupDriver_CGROUPFS,
72+
},
73+
{
74+
desc: "pick default, cgroupfs",
75+
defaultRuntime: "runc-2",
76+
runtimes: map[string]criconfig.Runtime{
77+
"non-runc": newFakeRuntimeConfig(false, false),
78+
"runc": newFakeRuntimeConfig(true, true),
79+
"runc-2": newFakeRuntimeConfig(true, false),
80+
},
81+
expectedCgroupDriver: runtime.CgroupDriver_CGROUPFS,
82+
},
83+
{
84+
desc: "pick default, systemd",
85+
defaultRuntime: "runc-2",
86+
runtimes: map[string]criconfig.Runtime{
87+
"non-runc": newFakeRuntimeConfig(false, false),
88+
"runc": newFakeRuntimeConfig(true, false),
89+
"runc-2": newFakeRuntimeConfig(true, true),
90+
},
91+
expectedCgroupDriver: runtime.CgroupDriver_SYSTEMD,
92+
},
93+
} {
94+
test := test
95+
t.Run(test.desc, func(t *testing.T) {
96+
c := newTestCRIService()
97+
c.config.PluginConfig.ContainerdConfig.DefaultRuntimeName = test.defaultRuntime
98+
c.config.PluginConfig.ContainerdConfig.Runtimes = test.runtimes
99+
100+
resp, err := c.RuntimeConfig(context.TODO(), &runtime.RuntimeConfigRequest{})
101+
assert.NoError(t, err)
102+
assert.Equal(t, test.expectedCgroupDriver, resp.Linux.CgroupDriver, "got unexpected cgroup driver")
103+
})
104+
}
105+
}

‎pkg/cri/sbserver/runtime_config_other.go‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
//go:build !linux
2+
3+
/*
4+
Copyright The containerd Authors.
5+
6+
Licensed under the Apache License, Version 2.0 (the "License");
7+
you may not use this file except in compliance with the License.
8+
You may obtain a copy of the License at
9+
10+
http://www.apache.org/licenses/LICENSE-2.0
11+
12+
Unless required by applicable law or agreed to in writing, software
13+
distributed under the License is distributed on an "AS IS" BASIS,
14+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
See the License for the specific language governing permissions and
16+
limitations under the License.
17+
*/
18+
19+
package sbserver
20+
21+
import (
22+
"context"
23+
24+
runtime "k8s.io/cri-api/pkg/apis/runtime/v1"
25+
)
26+
27+
func (c *criService) getLinuxRuntimeConfig(ctx context.Context) *runtime.LinuxRuntimeConfiguration {
28+
return nil
29+
}

‎pkg/cri/server/runtime_config.go‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
/*
2+
Copyright The containerd Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package server
18+
19+
import (
20+
"context"
21+
22+
runtime "k8s.io/cri-api/pkg/apis/runtime/v1"
23+
)
24+
25+
// RuntimeConfig returns configuration information of the runtime.
26+
func (c *criService) RuntimeConfig(ctx context.Context, r *runtime.RuntimeConfigRequest) (*runtime.RuntimeConfigResponse, error) {
27+
resp := &runtime.RuntimeConfigResponse{
28+
Linux: c.getLinuxRuntimeConfig(ctx),
29+
}
30+
return resp, nil
31+
}

‎pkg/cri/server/runtime_config_linux.go‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
/*
2+
Copyright The containerd Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package server
18+
19+
import (
20+
"context"
21+
"sort"
22+
23+
"github.com/containerd/containerd/log"
24+
runcoptions "github.com/containerd/containerd/runtime/v2/runc/options"
25+
"github.com/opencontainers/runc/libcontainer/cgroups/systemd"
26+
runtime "k8s.io/cri-api/pkg/apis/runtime/v1"
27+
)
28+
29+
func (c *criService) getLinuxRuntimeConfig(ctx context.Context) *runtime.LinuxRuntimeConfiguration {
30+
return &runtime.LinuxRuntimeConfiguration{CgroupDriver: c.getCgroupDriver(ctx)}
31+
}
32+
33+
func (c *criService) getCgroupDriver(ctx context.Context) runtime.CgroupDriver {
34+
// Go through the runtime handlers in a predictable order, starting from the
35+
// default handler, others sorted in alphabetical order
36+
handlerNames := make([]string, 0, len(c.config.ContainerdConfig.Runtimes))
37+
for n := range c.config.ContainerdConfig.Runtimes {
38+
handlerNames = append(handlerNames, n)
39+
}
40+
sort.Slice(handlerNames, func(i, j int) bool {
41+
if handlerNames[i] == c.config.ContainerdConfig.DefaultRuntimeName {
42+
return true
43+
}
44+
if handlerNames[j] == c.config.ContainerdConfig.DefaultRuntimeName {
45+
return false
46+
}
47+
return handlerNames[i] < handlerNames[j]
48+
})
49+
50+
for _, handler := range handlerNames {
51+
opts, err := generateRuntimeOptions(c.config.ContainerdConfig.Runtimes[handler])
52+
if err != nil {
53+
log.G(ctx).Debugf("failed to parse runtime handler options for %q", handler)
54+
continue
55+
}
56+
if d, ok := getCgroupDriverFromRuntimeHandlerOpts(opts); ok {
57+
return d
58+
}
59+
log.G(ctx).Debugf("runtime handler %q does not provide cgroup driver information", handler)
60+
}
61+
62+
// If no runtime handlers have a setting, detect if systemd is running
63+
d := runtime.CgroupDriver_CGROUPFS
64+
if systemd.IsRunningSystemd() {
65+
d = runtime.CgroupDriver_SYSTEMD
66+
}
67+
log.G(ctx).Debugf("no runtime handler provided cgroup driver setting, using auto-detected %s", runtime.CgroupDriver_name[int32(d)])
68+
return d
69+
}
70+
71+
func getCgroupDriverFromRuntimeHandlerOpts(opts interface{}) (runtime.CgroupDriver, bool) {
72+
switch v := opts.(type) {
73+
case *runcoptions.Options:
74+
systemdCgroup := v.SystemdCgroup
75+
if systemdCgroup {
76+
return runtime.CgroupDriver_SYSTEMD, true
77+
}
78+
return runtime.CgroupDriver_CGROUPFS, true
79+
}
80+
return runtime.CgroupDriver_SYSTEMD, false
81+
}

0 commit comments

Comments
 (0)