Skip to content
This repository was archived by the owner on Mar 9, 2022. It is now read-only.

Commit 49d77dd

Browse files
Random-LiuRandom-Liu
committed
Support runtime specific configurations.
Signed-off-by: Lantao Liu <[email protected]>
1 parent a97094d commit 49d77dd

9 files changed

+258
-122
lines changed

‎docs/config.md

+73-17
Original file line numberDiff line numberDiff line change
@@ -21,17 +21,21 @@ The explanation and default value of each configuration item are as follows:
2121
# stats_collect_period is the period (in seconds) of snapshots stats collection.
2222
stats_collect_period = 10
2323

24-
# systemd_cgroup enables systemd cgroup support.
24+
# systemd_cgroup enables systemd cgroup support. This only works for runtime
25+
# type "io.containerd.runtime.v1.linux".
26+
# DEPRECATED: use Runtime.Options for runtime specific config for shim v2 runtimes.
27+
# For runtime "io.containerd.runc.v1", use the option `SystemdCgroup`.
2528
systemd_cgroup = false
2629

27-
# enable_tls_streaming enables the TLS streaming support.
30+
# enable_tls_streaming enables the TLS streaming support.
2831
# It generates a self-sign certificate unless the following x509_key_pair_streaming are both set.
2932
enable_tls_streaming = false
30-
33+
3134
# "plugins.cri.x509_key_pair_streaming" contains a x509 valid key pair to stream with tls.
3235
[plugins.cri.x509_key_pair_streaming]
3336
# tls_cert_file is the filepath to the certificate paired with the "tls_key_file"
3437
tls_cert_file = ""
38+
3539
# tls_key_file is the filepath to the private key paired with the "tls_cert_file"
3640
tls_key_file = ""
3741

@@ -46,7 +50,10 @@ The explanation and default value of each configuration item are as follows:
4650
# snapshotter is the snapshotter used by containerd.
4751
snapshotter = "overlayfs"
4852

49-
# no_pivot disables pivot-root (linux only), required when running a container in a RamDisk with runc
53+
# no_pivot disables pivot-root (linux only), required when running a container in a RamDisk with runc.
54+
# This only works for runtime type "io.containerd.runtime.v1.linux".
55+
# DEPRECATED: use Runtime.Options for runtime specific config for shim v2 runtimes.
56+
# For runtime "io.containerd.runc.v1", use the option `NoPivotRoot`.
5057
no_pivot = false
5158

5259
# "plugins.cri.containerd.default_runtime" is the runtime to use in containerd.
@@ -55,17 +62,41 @@ The explanation and default value of each configuration item are as follows:
5562
runtime_type = "io.containerd.runtime.v1.linux"
5663

5764
# runtime_engine is the name of the runtime engine used by containerd.
65+
# This only works for runtime type "io.containerd.runtime.v1.linux".
66+
# DEPRECATED: use Runtime.Options for runtime specific config for shim v2 runtimes.
67+
# For runtime "io.containerd.runc.v1", use the option `BinaryName`.
5868
runtime_engine = ""
5969

6070
# runtime_root is the directory used by containerd for runtime state.
71+
# This only works for runtime type "io.containerd.runtime.v1.linux".
72+
# DEPRECATED: use Runtime.Options for runtime specific config for shim v2 runtimes.
73+
# For runtime "io.containerd.runc.v1", use the option `Root`.
6174
runtime_root = ""
6275

63-
# "plugins.cri.containerd.untrusted_workload_runtime" is a runtime to run untrusted workloads on it.
76+
# "plugins.cri.containerd.default_runtime.options" is options specific to
77+
# the default runtime. The options type for "io.containerd.runtime.v1.linux" is:
78+
# https://github.com/containerd/containerd/blob/v1.2.0-rc.1/runtime/linux/runctypes/runc.pb.go#L40
79+
# NOTE: when `options` is specified, all related deprecated options will
80+
# be ignored, including `systemd_cgroup`, `no_pivot`, `runtime_engine`
81+
# and `runtime_root`.
82+
[plugins.cri.containerd.default_runtime.options]
83+
# Runtime is the binary name of the runtime.
84+
Runtime = ""
85+
86+
# RuntimeRoot is the root directory of the runtime.
87+
RuntimeRoot = ""
88+
89+
# CriuPath is the criu binary path.
90+
CriuPath = ""
6491

92+
# SystemdCgroup enables systemd cgroups.
93+
SystemdCgroup = false
94+
95+
# "plugins.cri.containerd.untrusted_workload_runtime" is a runtime to run untrusted workloads on it.
6596
# DEPRECATED: use plugins.cri.runtimes instead. If provided, this runtime is mapped to the
66-
# runtime handler named 'untrusted'. It is a configuration error to provide both the (now
67-
# deprecated) UntrustedWorkloadRuntime and a handler in the Runtimes handler map (below) for
68-
# 'untrusted' workloads at the same time. Please provide one or the other.
97+
# runtime handler named 'untrusted'. It is a configuration error to provide both the (now
98+
# deprecated) UntrustedWorkloadRuntime and a handler in the Runtimes handler map (below) for
99+
# 'untrusted' workloads at the same time. Please provide one or the other.
69100
[plugins.cri.containerd.untrusted_workload_runtime]
70101
# runtime_type is the runtime type to use in containerd e.g. io.containerd.runtime.v1.linux
71102
runtime_type = ""
@@ -76,18 +107,43 @@ The explanation and default value of each configuration item are as follows:
76107
# runtime_root is the directory used by containerd for runtime state.
77108
runtime_root = ""
78109

79-
# plugins.cri.containerd.runtimes is a map from CRI RuntimeHandler strings, which specify types
80-
# of runtime configurations, to the matching configurations. In this example,
81-
# 'runtime_handler_name' is the RuntimeHandler string to match.
82-
[plugins.cri.containerd.runtimes.runtime_handler_name]
110+
# plugins.cri.containerd.runtimes is a map from CRI RuntimeHandler strings, which specify types
111+
# of runtime configurations, to the matching configurations. In this example,
112+
# 'runc' is the RuntimeHandler string to match.
113+
[plugins.cri.containerd.runtimes.runc]
83114
# runtime_type is the runtime type to use in containerd e.g. io.containerd.runtime.v1.linux
84-
runtime_type = ""
115+
runtime_type = "io.containerd.runc.v1"
85116

86-
# runtime_engine is the name of the runtime engine used by containerd.
87-
runtime_engine = ""
117+
# "plugins.cri.containerd.runtimes.runc.options" is options specific to
118+
# "io.containerd.runc.v1". Its corresponding options type is:
119+
# https://github.com/containerd/containerd/blob/v1.2.0-rc.1/runtime/v2/runc/options/oci.pb.go#L39.
120+
[plugins.cri.containerd.runtimes.runc.options]
121+
# NoPivotRoot disables pivot root when creating a container.
122+
NoPivotRoot = false
88123

89-
# runtime_root is the directory used by containerd for runtime state.
90-
runtime_root = ""
124+
# NoNewKeyring disables new keyring for the container.
125+
NoNewKeyring = false
126+
127+
# ShimCgroup places the shim in a cgroup.
128+
ShimCgroup = ""
129+
130+
# IoUid sets the I/O's pipes uid.
131+
IoUid = 0
132+
133+
# IoGid sets the I/O's pipes gid.
134+
IoGid = 0
135+
136+
# BinaryName is the binary name of the runc binary.
137+
BinaryName = ""
138+
139+
# Root is the runc root directory.
140+
Root = ""
141+
142+
# CriuPath is the criu binary path.
143+
CriuPath = ""
144+
145+
# SystemdCgroup enables systemd cgroups.
146+
SystemdCgroup = false
91147

92148
# "plugins.cri.cni" contains config related to cni
93149
[plugins.cri.cni]

‎pkg/config/config.go

+15-1
Original file line numberDiff line numberDiff line change
@@ -16,17 +16,27 @@ limitations under the License.
1616

1717
package config
1818

19-
import "github.com/containerd/containerd"
19+
import (
20+
"github.com/BurntSushi/toml"
21+
"github.com/containerd/containerd"
22+
)
2023

2124
// Runtime struct to contain the type(ID), engine, and root variables for a default runtime
2225
// and a runtime for untrusted worload.
2326
type Runtime struct {
2427
// Type is the runtime type to use in containerd e.g. io.containerd.runtime.v1.linux
2528
Type string `toml:"runtime_type" json:"runtimeType"`
2629
// Engine is the name of the runtime engine used by containerd.
30+
// This only works for runtime type "io.containerd.runtime.v1.linux".
31+
// DEPRECATED: use Options instead. Remove when shim v1 is deprecated.
2732
Engine string `toml:"runtime_engine" json:"runtimeEngine"`
2833
// Root is the directory used by containerd for runtime state.
34+
// DEPRECATED: use Options instead. Remove when shim v1 is deprecated.
35+
// This only works for runtime type "io.containerd.runtime.v1.linux".
2936
Root string `toml:"runtime_root" json:"runtimeRoot"`
37+
// Options are config options for the runtime. If options is loaded
38+
// from toml config, it will be toml.Primitive.
39+
Options *toml.Primitive `toml:"options" json:"options"`
3040
}
3141

3242
// ContainerdConfig contains toml config related to containerd
@@ -46,6 +56,8 @@ type ContainerdConfig struct {
4656
// configurations, to the matching configurations.
4757
Runtimes map[string]Runtime `toml:"runtimes" json:"runtimes"`
4858
// NoPivot disables pivot-root (linux only), required when running a container in a RamDisk with runc
59+
// This only works for runtime type "io.containerd.runtime.v1.linux".
60+
// DEPRECATED: use Runtime.Options instead. Remove when shim v1 is deprecated.
4961
NoPivot bool `toml:"no_pivot" json:"noPivot"`
5062
}
5163

@@ -119,6 +131,8 @@ type PluginConfig struct {
119131
// StatsCollectPeriod is the period (in seconds) of snapshots stats collection.
120132
StatsCollectPeriod int `toml:"stats_collect_period" json:"statsCollectPeriod"`
121133
// SystemdCgroup enables systemd cgroup support.
134+
// This only works for runtime type "io.containerd.runtime.v1.linux".
135+
// DEPRECATED: config runc runtime handler instead. Remove when shim v1 is deprecated.
122136
SystemdCgroup bool `toml:"systemd_cgroup" json:"systemdCgroup"`
123137
// EnableTLSStreaming indicates to enable the TLS streaming support.
124138
EnableTLSStreaming bool `toml:"enable_tls_streaming" json:"enableTLSStreaming"`

‎pkg/server/container_create.go

+5-12
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,6 @@ import (
3030
"github.com/containerd/containerd/contrib/seccomp"
3131
"github.com/containerd/containerd/mount"
3232
"github.com/containerd/containerd/oci"
33-
"github.com/containerd/containerd/runtime/linux/runctypes"
3433
"github.com/containerd/typeurl"
3534
"github.com/davecgh/go-spew/spew"
3635
imagespec "github.com/opencontainers/image-spec/specs-go/v1"
@@ -125,11 +124,6 @@ func (c *criService) CreateContainer(ctx context.Context, r *runtime.CreateConta
125124
if err != nil {
126125
return nil, errors.Wrapf(err, "failed to get sandbox %q info", sandboxID)
127126
}
128-
ociRuntime, err := getRuntimeConfigFromContainerInfo(sandboxInfo)
129-
if err != nil {
130-
return nil, errors.Wrap(err, "failed to get OCI runtime")
131-
}
132-
logrus.Debugf("Use OCI %+v for container %q", ociRuntime, id)
133127

134128
// Create container root directory.
135129
containerRootDir := c.getContainerRootDir(id)
@@ -261,14 +255,13 @@ func (c *criService) CreateContainer(ctx context.Context, r *runtime.CreateConta
261255
}
262256
containerLabels := buildLabels(config.Labels, containerKindContainer)
263257

258+
runtimeOptions, err := getRuntimeOptions(sandboxInfo)
259+
if err != nil {
260+
return nil, errors.Wrap(err, "failed to get runtime options")
261+
}
264262
opts = append(opts,
265263
containerd.WithSpec(spec, specOpts...),
266-
containerd.WithRuntime(
267-
ociRuntime.Type,
268-
&runctypes.RuncOptions{
269-
Runtime: ociRuntime.Engine,
270-
RuntimeRoot: ociRuntime.Root,
271-
SystemdCgroup: c.config.SystemdCgroup}), // TODO (mikebrow): add CriuPath when we add support for pause
264+
containerd.WithRuntime(sandboxInfo.Runtime.Name, runtimeOptions),
272265
containerd.WithContainerLabels(containerLabels),
273266
containerd.WithContainerExtension(containerMetadataExtension, &meta))
274267
var cntr containerd.Container

‎pkg/server/container_start.go

+7-1
Original file line numberDiff line numberDiff line change
@@ -108,8 +108,14 @@ func (c *criService) startContainer(ctx context.Context,
108108
return cntr.IO, nil
109109
}
110110

111+
ctrInfo, err := container.Info(ctx)
112+
if err != nil {
113+
return errors.Wrap(err, "failed to get container info")
114+
}
115+
111116
var taskOpts []containerd.NewTaskOpts
112-
if c.config.NoPivot {
117+
// TODO(random-liu): Remove this after shim v1 is deprecated.
118+
if c.config.NoPivot && ctrInfo.Runtime.Name == linuxRuntime {
113119
taskOpts = append(taskOpts, containerd.WithNoPivotRoot)
114120
}
115121
task, err := container.NewTask(ctx, ioCreation, taskOpts...)

‎pkg/server/container_status.go

+13-12
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,6 @@ import (
2424
"golang.org/x/net/context"
2525
runtime "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2"
2626

27-
criconfig "github.com/containerd/cri/pkg/config"
2827
containerstore "github.com/containerd/cri/pkg/store/container"
2928
)
3029

@@ -102,14 +101,15 @@ func toCRIContainerStatus(container containerstore.Container, spec *runtime.Imag
102101

103102
type containerInfo struct {
104103
// TODO(random-liu): Add sandboxID in CRI container status.
105-
SandboxID string `json:"sandboxID"`
106-
Pid uint32 `json:"pid"`
107-
Removing bool `json:"removing"`
108-
SnapshotKey string `json:"snapshotKey"`
109-
Snapshotter string `json:"snapshotter"`
110-
Runtime *criconfig.Runtime `json:"runtime"`
111-
Config *runtime.ContainerConfig `json:"config"`
112-
RuntimeSpec *runtimespec.Spec `json:"runtimeSpec"`
104+
SandboxID string `json:"sandboxID"`
105+
Pid uint32 `json:"pid"`
106+
Removing bool `json:"removing"`
107+
SnapshotKey string `json:"snapshotKey"`
108+
Snapshotter string `json:"snapshotter"`
109+
RuntimeType string `json:"runtimeType"`
110+
RuntimeOptions interface{} `json:"runtimeOptions"`
111+
Config *runtime.ContainerConfig `json:"config"`
112+
RuntimeSpec *runtimespec.Spec `json:"runtimeSpec"`
113113
}
114114

115115
// toCRIContainerInfo converts internal container object information to CRI container status response info map.
@@ -142,11 +142,12 @@ func toCRIContainerInfo(ctx context.Context, container containerstore.Container,
142142
ci.SnapshotKey = ctrInfo.SnapshotKey
143143
ci.Snapshotter = ctrInfo.Snapshotter
144144

145-
ociRuntime, err := getRuntimeConfigFromContainerInfo(ctrInfo)
145+
runtimeOptions, err := getRuntimeOptions(ctrInfo)
146146
if err != nil {
147-
return nil, errors.Wrap(err, "failed to get container runtime config")
147+
return nil, errors.Wrap(err, "failed to get runtime options")
148148
}
149-
ci.Runtime = &ociRuntime
149+
ci.RuntimeType = ctrInfo.Runtime.Name
150+
ci.RuntimeOptions = runtimeOptions
150151

151152
infoBytes, err := json.Marshal(ci)
152153
if err != nil {

‎pkg/server/helpers.go

+52-20
Original file line numberDiff line numberDiff line change
@@ -25,8 +25,10 @@ import (
2525
"strconv"
2626
"strings"
2727

28+
"github.com/BurntSushi/toml"
2829
"github.com/containerd/containerd/containers"
2930
"github.com/containerd/containerd/runtime/linux/runctypes"
31+
runcoptions "github.com/containerd/containerd/runtime/v2/runc/options"
3032
"github.com/containerd/typeurl"
3133
"github.com/docker/distribution/reference"
3234
imagedigest "github.com/opencontainers/go-digest"
@@ -123,6 +125,14 @@ const (
123125
networkAttachCount = 2
124126
)
125127

128+
// Runtime type strings for various runtimes.
129+
const (
130+
// linuxRuntime is the legacy linux runtime for shim v1.
131+
linuxRuntime = "io.containerd.runtime.v1.linux"
132+
// runcRuntime is the runc runtime for shim v2.
133+
runcRuntime = "io.containerd.runc.v1"
134+
)
135+
126136
// makeSandboxName generates sandbox name from sandbox metadata. The name
127137
// generated is unique as long as sandbox metadata is unique.
128138
func makeSandboxName(s *runtime.PodSandboxMetadata) string {
@@ -390,26 +400,6 @@ func getPodCNILabels(id string, config *runtime.PodSandboxConfig) map[string]str
390400
}
391401
}
392402

393-
// getRuntimeConfigFromContainerInfo gets runtime configuration from containerd
394-
// container info.
395-
func getRuntimeConfigFromContainerInfo(c containers.Container) (criconfig.Runtime, error) {
396-
r := criconfig.Runtime{
397-
Type: c.Runtime.Name,
398-
}
399-
if c.Runtime.Options == nil {
400-
// CRI plugin makes sure that runtime option is always set.
401-
return criconfig.Runtime{}, errors.New("runtime options is nil")
402-
}
403-
data, err := typeurl.UnmarshalAny(c.Runtime.Options)
404-
if err != nil {
405-
return criconfig.Runtime{}, errors.Wrap(err, "failed to unmarshal runtime options")
406-
}
407-
runtimeOpts := data.(*runctypes.RuncOptions)
408-
r.Engine = runtimeOpts.Runtime
409-
r.Root = runtimeOpts.RuntimeRoot
410-
return r, nil
411-
}
412-
413403
// toRuntimeAuthConfig converts cri plugin auth config to runtime auth config.
414404
func toRuntimeAuthConfig(a criconfig.AuthConfig) *runtime.AuthConfig {
415405
return &runtime.AuthConfig{
@@ -464,3 +454,45 @@ func parseImageReferences(refs []string) ([]string, []string) {
464454
}
465455
return tags, digests
466456
}
457+
458+
// generateRuntimeOptions generates runtime options from cri plugin config.
459+
func generateRuntimeOptions(r criconfig.Runtime, c criconfig.Config) (interface{}, error) {
460+
if r.Options == nil {
461+
if r.Type != linuxRuntime {
462+
return nil, nil
463+
}
464+
// This is a legacy config, generate runctypes.RuncOptions.
465+
return &runctypes.RuncOptions{
466+
Runtime: r.Engine,
467+
RuntimeRoot: r.Root,
468+
SystemdCgroup: c.SystemdCgroup,
469+
}, nil
470+
}
471+
options := getRuntimeOptionsType(r.Type)
472+
if err := toml.PrimitiveDecode(*r.Options, options); err != nil {
473+
return nil, err
474+
}
475+
return options, nil
476+
}
477+
478+
// getRuntimeOptionsType gets empty runtime options by the runtime type name.
479+
func getRuntimeOptionsType(t string) interface{} {
480+
switch t {
481+
case runcRuntime:
482+
return &runcoptions.Options{}
483+
default:
484+
return &runctypes.RuncOptions{}
485+
}
486+
}
487+
488+
// getRuntimeOptions get runtime options from container metadata.
489+
func getRuntimeOptions(c containers.Container) (interface{}, error) {
490+
if c.Runtime.Options == nil {
491+
return nil, nil
492+
}
493+
opts, err := typeurl.UnmarshalAny(c.Runtime.Options)
494+
if err != nil {
495+
return nil, err
496+
}
497+
return opts, nil
498+
}

0 commit comments

Comments
 (0)