containerd · Random-Liu · Oct 9, 2018 · Oct 8, 2018 · mikebrow · Oct 8, 2018
diff --git a/docs/config.md b/docs/config.md
@@ -21,17 +21,21 @@ The explanation and default value of each configuration item are as follows:
   # stats_collect_period is the period (in seconds) of snapshots stats collection.
   stats_collect_period = 10
 
-  # systemd_cgroup enables systemd cgroup support.
+  # systemd_cgroup enables systemd cgroup support. This only works for runtime
+  # type "io.containerd.runtime.v1.linux".
+  # DEPRECATED: use Runtime.Options for runtime specific config for shim v2 runtimes.
+  #   For runtime "io.containerd.runc.v1", use the option `SystemdCgroup`.
   systemd_cgroup = false
 
-  # enable_tls_streaming enables the TLS streaming support. 
+  # enable_tls_streaming enables the TLS streaming support.
   # It generates a self-sign certificate unless the following x509_key_pair_streaming are both set.
   enable_tls_streaming = false
-  
+
   # "plugins.cri.x509_key_pair_streaming" contains a x509 valid key pair to stream with tls.
   [plugins.cri.x509_key_pair_streaming]
     # tls_cert_file is the filepath to the certificate paired with the "tls_key_file"
     tls_cert_file = ""
+
     # tls_key_file is the filepath to the private key paired with the "tls_cert_file"
     tls_key_file = ""
 
@@ -46,7 +50,10 @@ The explanation and default value of each configuration item are as follows:
     # snapshotter is the snapshotter used by containerd.
     snapshotter = "overlayfs"
 
-    # no_pivot disables pivot-root (linux only), required when running a container in a RamDisk with runc
+    # no_pivot disables pivot-root (linux only), required when running a container in a RamDisk with runc.
+    # This only works for runtime type "io.containerd.runtime.v1.linux".
+    # DEPRECATED: use Runtime.Options for runtime specific config for shim v2 runtimes.
+    #   For runtime "io.containerd.runc.v1", use the option `NoPivotRoot`.
     no_pivot = false
 
     # "plugins.cri.containerd.default_runtime" is the runtime to use in containerd.
@@ -55,17 +62,41 @@ The explanation and default value of each configuration item are as follows:
       runtime_type = "io.containerd.runtime.v1.linux"
 
       # runtime_engine is the name of the runtime engine used by containerd.
+      # This only works for runtime type "io.containerd.runtime.v1.linux".
+      # DEPRECATED: use Runtime.Options for runtime specific config for shim v2 runtimes.
+      #   For runtime "io.containerd.runc.v1", use the option `BinaryName`.
       runtime_engine = ""
 
       # runtime_root is the directory used by containerd for runtime state.
+      # This only works for runtime type "io.containerd.runtime.v1.linux".
+      # DEPRECATED: use Runtime.Options for runtime specific config for shim v2 runtimes.
+      #   For runtime "io.containerd.runc.v1", use the option `Root`.
       runtime_root = ""
 
-    # "plugins.cri.containerd.untrusted_workload_runtime" is a runtime to run untrusted workloads on it.
+      # "plugins.cri.containerd.default_runtime.options" is options specific to
+      # the default runtime. The options type for "io.containerd.runtime.v1.linux" is:
+      #   https://github.com/containerd/containerd/blob/v1.2.0-rc.1/runtime/linux/runctypes/runc.pb.go#L40
+      # NOTE: when `options` is specified, all related deprecated options will
+      #   be ignored, including `systemd_cgroup`, `no_pivot`, `runtime_engine`
+      #   and `runtime_root`.
+      [plugins.cri.containerd.default_runtime.options]
+        # Runtime is the binary name of the runtime.
+        Runtime = ""
+
+        # RuntimeRoot is the root directory of the runtime.
+        RuntimeRoot = ""
+
+        # CriuPath is the criu binary path.
+        CriuPath = ""
 
+        # SystemdCgroup enables systemd cgroups.
+        SystemdCgroup = false
+
+    # "plugins.cri.containerd.untrusted_workload_runtime" is a runtime to run untrusted workloads on it.
     # DEPRECATED: use plugins.cri.runtimes instead. If provided, this runtime is mapped to the
-    #     runtime handler named 'untrusted'. It is a configuration error to provide both the (now
-    #     deprecated) UntrustedWorkloadRuntime and a handler in the Runtimes handler map (below) for
-    #     'untrusted' workloads at the same time. Please provide one or the other.
+    #   runtime handler named 'untrusted'. It is a configuration error to provide both the (now
+    #   deprecated) UntrustedWorkloadRuntime and a handler in the Runtimes handler map (below) for
+    #   'untrusted' workloads at the same time. Please provide one or the other.
     [plugins.cri.containerd.untrusted_workload_runtime]
       # runtime_type is the runtime type to use in containerd e.g. io.containerd.runtime.v1.linux
       runtime_type = ""
@@ -78,16 +109,41 @@ The explanation and default value of each configuration item are as follows:
 
     # plugins.cri.containerd.runtimes is a map from CRI RuntimeHandler strings, which specify types
     # of runtime configurations, to the matching configurations. In this example,
-    # 'runtime_handler_name' is the RuntimeHandler string to match.
-    [plugins.cri.containerd.runtimes.runtime_handler_name]
+    # 'runc' is the RuntimeHandler string to match.
+    [plugins.cri.containerd.runtimes.runc]
       # runtime_type is the runtime type to use in containerd e.g. io.containerd.runtime.v1.linux
-      runtime_type = ""
+      runtime_type = "io.containerd.runc.v1"
 
-      # runtime_engine is the name of the runtime engine used by containerd.
-      runtime_engine = ""
+      # "plugins.cri.containerd.runtimes.runc.options" is options specific to
+      # "io.containerd.runc.v1". Its corresponding options type is:
+      #   https://github.com/containerd/containerd/blob/v1.2.0-rc.1/runtime/v2/runc/options/oci.pb.go#L39.
+      [plugins.cri.containerd.runtimes.runc.options]
+        # NoPivotRoot disables pivot root when creating a container.
+        NoPivotRoot = false
 
-      # runtime_root is the directory used by containerd for runtime state.
-      runtime_root = ""
+        # NoNewKeyring disables new keyring for the container.
+        NoNewKeyring = false
+
+        # ShimCgroup places the shim in a cgroup.
+        ShimCgroup = ""
+
+        # IoUid sets the I/O's pipes uid.
+        IoUid = 0
+
+        # IoGid sets the I/O's pipes gid.
+        IoGid = 0
+
+        # BinaryName is the binary name of the runc binary.
+        BinaryName = ""
+
+        # Root is the runc root directory.
+        Root = ""
+
+        # CriuPath is the criu binary path.
+        CriuPath = ""
+
+        # SystemdCgroup enables systemd cgroups.
+        SystemdCgroup = false
 
   # "plugins.cri.cni" contains config related to cni
   [plugins.cri.cni]

diff --git a/pkg/config/config.go b/pkg/config/config.go
@@ -16,17 +16,27 @@ limitations under the License.
 
 package config
 
-import "github.com/containerd/containerd"
+import (
+	"github.com/BurntSushi/toml"
+	"github.com/containerd/containerd"
+)
 
 // Runtime struct to contain the type(ID), engine, and root variables for a default runtime
 // and a runtime for untrusted worload.
 type Runtime struct {
 	// Type is the runtime type to use in containerd e.g. io.containerd.runtime.v1.linux
 	Type string `toml:"runtime_type" json:"runtimeType"`
 	// Engine is the name of the runtime engine used by containerd.
+	// This only works for runtime type "io.containerd.runtime.v1.linux".
+	// DEPRECATED: use Options instead. Remove when shim v1 is deprecated.
 	Engine string `toml:"runtime_engine" json:"runtimeEngine"`
 	// Root is the directory used by containerd for runtime state.
+	// DEPRECATED: use Options instead. Remove when shim v1 is deprecated.
+	// This only works for runtime type "io.containerd.runtime.v1.linux".
 	Root string `toml:"runtime_root" json:"runtimeRoot"`
+	// Options are config options for the runtime. If options is loaded
+	// from toml config, it will be toml.Primitive.
+	Options *toml.Primitive `toml:"options" json:"options"`
 }
 
 // ContainerdConfig contains toml config related to containerd
@@ -46,6 +56,8 @@ type ContainerdConfig struct {
 	// configurations, to the matching configurations.
 	Runtimes map[string]Runtime `toml:"runtimes" json:"runtimes"`
 	// NoPivot disables pivot-root (linux only), required when running a container in a RamDisk with runc
+	// This only works for runtime type "io.containerd.runtime.v1.linux".
+	// DEPRECATED: use Runtime.Options instead. Remove when shim v1 is deprecated.
 	NoPivot bool `toml:"no_pivot" json:"noPivot"`
 }
 
@@ -119,6 +131,8 @@ type PluginConfig struct {
 	// StatsCollectPeriod is the period (in seconds) of snapshots stats collection.
 	StatsCollectPeriod int `toml:"stats_collect_period" json:"statsCollectPeriod"`
 	// SystemdCgroup enables systemd cgroup support.
+	// This only works for runtime type "io.containerd.runtime.v1.linux".
+	// DEPRECATED: config runc runtime handler instead. Remove when shim v1 is deprecated.
 	SystemdCgroup bool `toml:"systemd_cgroup" json:"systemdCgroup"`
 	// EnableTLSStreaming indicates to enable the TLS streaming support.
 	EnableTLSStreaming bool `toml:"enable_tls_streaming" json:"enableTLSStreaming"`

diff --git a/pkg/server/container_create.go b/pkg/server/container_create.go
@@ -30,7 +30,6 @@ import (
 	"github.com/containerd/containerd/contrib/seccomp"
 	"github.com/containerd/containerd/mount"
 	"github.com/containerd/containerd/oci"
-	"github.com/containerd/containerd/runtime/linux/runctypes"
 	"github.com/containerd/typeurl"
 	"github.com/davecgh/go-spew/spew"
 	imagespec "github.com/opencontainers/image-spec/specs-go/v1"
@@ -125,11 +124,6 @@ func (c *criService) CreateContainer(ctx context.Context, r *runtime.CreateConta
 	if err != nil {
 		return nil, errors.Wrapf(err, "failed to get sandbox %q info", sandboxID)
 	}
-	ociRuntime, err := getRuntimeConfigFromContainerInfo(sandboxInfo)
-	if err != nil {
-		return nil, errors.Wrap(err, "failed to get OCI runtime")
-	}
-	logrus.Debugf("Use OCI %+v for container %q", ociRuntime, id)
 
 	// Create container root directory.
 	containerRootDir := c.getContainerRootDir(id)
@@ -261,14 +255,13 @@ func (c *criService) CreateContainer(ctx context.Context, r *runtime.CreateConta
 	}
 	containerLabels := buildLabels(config.Labels, containerKindContainer)
 
+	runtimeOptions, err := getRuntimeOptions(sandboxInfo)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to get runtime options")
+	}
 	opts = append(opts,
 		containerd.WithSpec(spec, specOpts...),
-		containerd.WithRuntime(
-			ociRuntime.Type,
-			&runctypes.RuncOptions{
-				Runtime:       ociRuntime.Engine,
-				RuntimeRoot:   ociRuntime.Root,
-				SystemdCgroup: c.config.SystemdCgroup}), // TODO (mikebrow): add CriuPath when we add support for pause
+		containerd.WithRuntime(sandboxInfo.Runtime.Name, runtimeOptions),
 		containerd.WithContainerLabels(containerLabels),
 		containerd.WithContainerExtension(containerMetadataExtension, &meta))
 	var cntr containerd.Container

diff --git a/pkg/server/container_start.go b/pkg/server/container_start.go
@@ -108,8 +108,14 @@ func (c *criService) startContainer(ctx context.Context,
 		return cntr.IO, nil
 	}
 
+	ctrInfo, err := container.Info(ctx)
+	if err != nil {
+		return errors.Wrap(err, "failed to get container info")
+	}
+
 	var taskOpts []containerd.NewTaskOpts
-	if c.config.NoPivot {
+	// TODO(random-liu): Remove this after shim v1 is deprecated.
+	if c.config.NoPivot && ctrInfo.Runtime.Name == linuxRuntime {
 		taskOpts = append(taskOpts, containerd.WithNoPivotRoot)
 	}
 	task, err := container.NewTask(ctx, ioCreation, taskOpts...)

diff --git a/pkg/server/container_status.go b/pkg/server/container_status.go
@@ -24,7 +24,6 @@ import (
 	"golang.org/x/net/context"
 	runtime "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2"
 
-	criconfig "github.com/containerd/cri/pkg/config"
 	containerstore "github.com/containerd/cri/pkg/store/container"
 )
 
@@ -102,14 +101,15 @@ func toCRIContainerStatus(container containerstore.Container, spec *runtime.Imag
 
 type containerInfo struct {
 	// TODO(random-liu): Add sandboxID in CRI container status.
-	SandboxID   string                   `json:"sandboxID"`
-	Pid         uint32                   `json:"pid"`
-	Removing    bool                     `json:"removing"`
-	SnapshotKey string                   `json:"snapshotKey"`
-	Snapshotter string                   `json:"snapshotter"`
-	Runtime     *criconfig.Runtime       `json:"runtime"`
-	Config      *runtime.ContainerConfig `json:"config"`
-	RuntimeSpec *runtimespec.Spec        `json:"runtimeSpec"`
+	SandboxID      string                   `json:"sandboxID"`
+	Pid            uint32                   `json:"pid"`
+	Removing       bool                     `json:"removing"`
+	SnapshotKey    string                   `json:"snapshotKey"`
+	Snapshotter    string                   `json:"snapshotter"`
+	RuntimeType    string                   `json:"runtimeType"`
+	RuntimeOptions interface{}              `json:"runtimeOptions"`
+	Config         *runtime.ContainerConfig `json:"config"`
+	RuntimeSpec    *runtimespec.Spec        `json:"runtimeSpec"`
 }
 
 // toCRIContainerInfo converts internal container object information to CRI container status response info map.
@@ -142,11 +142,12 @@ func toCRIContainerInfo(ctx context.Context, container containerstore.Container,
 	ci.SnapshotKey = ctrInfo.SnapshotKey
 	ci.Snapshotter = ctrInfo.Snapshotter
 
-	ociRuntime, err := getRuntimeConfigFromContainerInfo(ctrInfo)
+	runtimeOptions, err := getRuntimeOptions(ctrInfo)
 	if err != nil {
-		return nil, errors.Wrap(err, "failed to get container runtime config")
+		return nil, errors.Wrap(err, "failed to get runtime options")
 	}
-	ci.Runtime = &ociRuntime
+	ci.RuntimeType = ctrInfo.Runtime.Name
+	ci.RuntimeOptions = runtimeOptions
 
 	infoBytes, err := json.Marshal(ci)
 	if err != nil {

diff --git a/pkg/server/helpers.go b/pkg/server/helpers.go
@@ -25,8 +25,10 @@ import (
 	"strconv"
 	"strings"
 
+	"github.com/BurntSushi/toml"
 	"github.com/containerd/containerd/containers"
 	"github.com/containerd/containerd/runtime/linux/runctypes"
+	runcoptions "github.com/containerd/containerd/runtime/v2/runc/options"
 	"github.com/containerd/typeurl"
 	"github.com/docker/distribution/reference"
 	imagedigest "github.com/opencontainers/go-digest"
@@ -123,6 +125,14 @@ const (
 	networkAttachCount = 2
 )
 
+// Runtime type strings for various runtimes.
+const (
+	// linuxRuntime is the legacy linux runtime for shim v1.
+	linuxRuntime = "io.containerd.runtime.v1.linux"
+	// runcRuntime is the runc runtime for shim v2.
+	runcRuntime = "io.containerd.runc.v1"
+)
+
 // makeSandboxName generates sandbox name from sandbox metadata. The name
 // generated is unique as long as sandbox metadata is unique.
 func makeSandboxName(s *runtime.PodSandboxMetadata) string {
@@ -390,26 +400,6 @@ func getPodCNILabels(id string, config *runtime.PodSandboxConfig) map[string]str
 	}
 }
 
-// getRuntimeConfigFromContainerInfo gets runtime configuration from containerd
-// container info.
-func getRuntimeConfigFromContainerInfo(c containers.Container) (criconfig.Runtime, error) {
-	r := criconfig.Runtime{
-		Type: c.Runtime.Name,
-	}
-	if c.Runtime.Options == nil {
-		// CRI plugin makes sure that runtime option is always set.
-		return criconfig.Runtime{}, errors.New("runtime options is nil")
-	}
-	data, err := typeurl.UnmarshalAny(c.Runtime.Options)
-	if err != nil {
-		return criconfig.Runtime{}, errors.Wrap(err, "failed to unmarshal runtime options")
-	}
-	runtimeOpts := data.(*runctypes.RuncOptions)
-	r.Engine = runtimeOpts.Runtime
-	r.Root = runtimeOpts.RuntimeRoot
-	return r, nil
-}
-
 // toRuntimeAuthConfig converts cri plugin auth config to runtime auth config.
 func toRuntimeAuthConfig(a criconfig.AuthConfig) *runtime.AuthConfig {
 	return &runtime.AuthConfig{
@@ -464,3 +454,45 @@ func parseImageReferences(refs []string) ([]string, []string) {
 	}
 	return tags, digests
 }
+
+// generateRuntimeOptions generates runtime options from cri plugin config.
+func generateRuntimeOptions(r criconfig.Runtime, c criconfig.Config) (interface{}, error) {
+	if r.Options == nil {
+		if r.Type != linuxRuntime {
+			return nil, nil
+		}
+		// This is a legacy config, generate runctypes.RuncOptions.
+		return &runctypes.RuncOptions{
+			Runtime:       r.Engine,
+			RuntimeRoot:   r.Root,
+			SystemdCgroup: c.SystemdCgroup,
+		}, nil
+	}
+	options := getRuntimeOptionsType(r.Type)
+	if err := toml.PrimitiveDecode(*r.Options, options); err != nil {
+		return nil, err
+	}
+	return options, nil
+}
+
+// getRuntimeOptionsType gets empty runtime options by the runtime type name.
+func getRuntimeOptionsType(t string) interface{} {
+	switch t {
+	case runcRuntime:
+		return &runcoptions.Options{}
+	default:
+		return &runctypes.RuncOptions{}
+	}
+}
+
+// getRuntimeOptions get runtime options from container metadata.
+func getRuntimeOptions(c containers.Container) (interface{}, error) {
+	if c.Runtime.Options == nil {
+		return nil, nil
+	}
+	opts, err := typeurl.UnmarshalAny(c.Runtime.Options)
+	if err != nil {
+		return nil, err
+	}
+	return opts, nil
+}