containerd · mikebrow · Mar 24, 2025 · Jan 27, 2025 · Mar 7, 2025 · djdongjin
diff --git a/RELEASES.md b/RELEASES.md
@@ -469,6 +469,7 @@ The deprecated features are shown in the following table:
 | CRI `v1alpha2`                                                                   | containerd v1.7     | containerd v2.0 ✅                    | Use CRI `v1`                             |
 | Legacy CRI implementation of podsandbox support                                  | containerd v2.0     | containerd v2.0 ✅                    |                                          |
 | Go-Plugin library (`*.so`) as containerd runtime plugin                          | containerd v2.0     | containerd v2.1                       | Use external plugins (proxy or binary)   |
+| CNI `bin_dir` in CRI runtime config (`plugins.'io.containerd.cri.v1.runtime'.cni.bin_dir`) | containerd v2.1     | containerd v2.2                       | Change `bin_dir` to `bin_dirs` in the same section which supports a list of directories  |
 
 - Pulling Schema 1 images has been disabled in containerd v2.0, but it still can be enabled by setting an environment variable `CONTAINERD_ENABLE_DEPRECATED_PULL_SCHEMA_1_IMAGE=1`
   until containerd v2.1. `ctr` users have to specify `--local` too (e.g., `ctr images pull --local`). Users of CRI clients (such as Kubernetes and `crictl`) have to specify this environment variable on the containerd daemon (usually in the systemd unit).

diff --git a/docs/cri/config.md b/docs/cri/config.md
@@ -247,7 +247,9 @@ version = 3
             ShimCgroup = ''
 
     [plugins.'io.containerd.cri.v1.runtime'.cni]
-      bin_dir = '/opt/cni/bin'
+      # DEPRECATED, use `bin_dirs` instead (since containerd v2.1).
+      bin_dir = ''
+      bin_dirs = ['/opt/cni/bin']
       conf_dir = '/etc/cni/net.d'
       max_conf_num = 1
       setup_serially = false

diff --git a/internal/cri/config/config.go b/internal/cri/config/config.go
@@ -21,6 +21,7 @@ import (
 	"errors"
 	"fmt"
 	"net/url"
+	"slices"
 	"time"
 
 	"github.com/containerd/log"
@@ -152,7 +153,13 @@ type ContainerdConfig struct {
 // CniConfig contains toml config related to cni
 type CniConfig struct {
 	// NetworkPluginBinDir is the directory in which the binaries for the plugin is kept.
+	//
+	// DEPRECATED: use `NetworkPluginBinDirs` instead.`
 	NetworkPluginBinDir string `toml:"bin_dir" json:"binDir"`
+	// NetworkPluginBinDirs is the directories in which the binaries for the plugin is kept.
+	//
+	// Only use one of NetworkPluginBinDir and NetworkPluginBinDirs, not both.
+	NetworkPluginBinDirs []string `toml:"bin_dirs" json:"binDirs"`
 	// NetworkPluginConfDir is the directory in which the admin places a CNI conf.
 	NetworkPluginConfDir string `toml:"conf_dir" json:"confDir"`
 	// NetworkPluginMaxConfNum is the max number of plugin config files that will
@@ -528,6 +535,28 @@ func ValidateRuntimeConfig(ctx context.Context, c *RuntimeConfig) ([]deprecation
 		return warnings, fmt.Errorf("no corresponding runtime configured in `containerd.runtimes` for `containerd` `default_runtime_name = \"%s\"", c.ContainerdConfig.DefaultRuntimeName)
 	}
 
+	// Validation for CNI config
+	if len(c.CniConfig.NetworkPluginBinDir) != 0 {
+		warnings = append(warnings, deprecation.CRICNIBinDir)
+		log.G(ctx).Warning("`bin_dir` is deprecated, please use `bin_dirs` instead")
+
+		if slices.Equal(c.CniConfig.NetworkPluginBinDirs, defaultNetworkPluginBinDirs()) {
+			// if a user set `bin_dir` explicitly, we remove the default value of `bin_dirs`
+			// to avoid the unexpected conflict between the two since we don't allow setting both.
+			c.CniConfig.NetworkPluginBinDirs = nil
+		}
+		if len(c.CniConfig.NetworkPluginBinDirs) == 0 {
+			// Before `NetworkPluginBinDir` is deprecated and removed, we manually move it
+			// into `NetworkPluginBinDirs` (if `NetworkPluginBinDirs` is empty)
+			// so that we can use it in the rest of the code.
+			c.CniConfig.NetworkPluginBinDirs = []string{c.CniConfig.NetworkPluginBinDir}
+			c.CniConfig.NetworkPluginBinDir = ""
+		}
+	}
+	if len(c.CniConfig.NetworkPluginBinDirs) != 0 && len(c.CniConfig.NetworkPluginBinDir) != 0 {
+		return warnings, errors.New("`cni.bin_dir` and `cni.bin_dirs` cannot be set at the same time")
+	}
+
 	for k, r := range c.ContainerdConfig.Runtimes {
 		if r.CgroupWritable && !opts.IsCgroup2UnifiedMode() {
 			return warnings, fmt.Errorf("runtime %s: `cgroup_writable` is only supported on cgroup v2", k)

diff --git a/internal/cri/config/config_test.go b/internal/cri/config/config_test.go
@@ -52,6 +52,49 @@ func TestValidateConfig(t *testing.T) {
 			},
 			runtimeExpectedErr: "no corresponding runtime configured in `containerd.runtimes` for `containerd` `default_runtime_name = \"default\"",
 		},
+		"specify both cni.bin_dir and cni_bin_dirs": {
+			runtimeConfig: &RuntimeConfig{
+				ContainerdConfig: ContainerdConfig{
+					DefaultRuntimeName: RuntimeDefault,
+					Runtimes: map[string]Runtime{
+						RuntimeDefault: {},
+					},
+				},
+				CniConfig: CniConfig{
+					NetworkPluginBinDir:  "/opt/mycni/bin",
+					NetworkPluginBinDirs: []string{"/opt/mycni/bin"},
+				},
+			},
+			runtimeExpectedErr: "`cni.bin_dir` and `cni.bin_dirs` cannot be set at the same time",
+			warnings:           []deprecation.Warning{deprecation.CRICNIBinDir},
+		},
+		"cni.bin_dir, if used, is moved to cni.bin_dirs": {
+			runtimeConfig: &RuntimeConfig{
+				ContainerdConfig: ContainerdConfig{
+					DefaultRuntimeName: RuntimeDefault,
+					Runtimes: map[string]Runtime{
+						RuntimeDefault: {},
+					},
+				},
+				CniConfig: CniConfig{
+					NetworkPluginBinDir: "/opt/mycni/bin",
+				},
+			},
+			runtimeExpected: &RuntimeConfig{
+				ContainerdConfig: ContainerdConfig{
+					DefaultRuntimeName: RuntimeDefault,
+					Runtimes: map[string]Runtime{
+						RuntimeDefault: {
+							Sandboxer: string(ModePodSandbox),
+						},
+					},
+				},
+				CniConfig: CniConfig{
+					NetworkPluginBinDirs: []string{"/opt/mycni/bin"},
+				},
+			},
+			warnings: []deprecation.Warning{deprecation.CRICNIBinDir},
+		},
 
 		"deprecated auths": {
 			runtimeConfig: &RuntimeConfig{
@@ -259,6 +302,7 @@ func TestValidateConfig(t *testing.T) {
 			if test.runtimeConfig != nil {
 				w, err := ValidateRuntimeConfig(context.Background(), test.runtimeConfig)
 				if test.runtimeExpectedErr != "" {
+					assert.Error(t, err)
 					assert.Contains(t, err.Error(), test.runtimeExpectedErr)
 				} else {
 					assert.NoError(t, err)

diff --git a/internal/cri/config/config_unix.go b/internal/cri/config/config_unix.go
@@ -23,6 +23,10 @@ import (
 	"github.com/pelletier/go-toml/v2"
 )
 
+func defaultNetworkPluginBinDirs() []string {
+	return []string{"/opt/cni/bin"}
+}
+
 func DefaultImageConfig() ImageConfig {
 	return ImageConfig{
 		Snapshotter:                defaults.DefaultSnapshotter,
@@ -72,7 +76,7 @@ func DefaultRuntimeConfig() RuntimeConfig {
 
 	return RuntimeConfig{
 		CniConfig: CniConfig{
-			NetworkPluginBinDir:        "/opt/cni/bin",
+			NetworkPluginBinDirs:       defaultNetworkPluginBinDirs(),
 			NetworkPluginConfDir:       "/etc/cni/net.d",
 			NetworkPluginMaxConfNum:    1, // only one CNI plugin config file will be loaded
 			NetworkPluginSetupSerially: false,

diff --git a/internal/cri/config/config_windows.go b/internal/cri/config/config_windows.go
@@ -23,6 +23,10 @@ import (
 	"github.com/containerd/containerd/v2/defaults"
 )
 
+func defaultNetworkPluginBinDirs() []string {
+	return []string{filepath.Join(os.Getenv("ProgramFiles"), "containerd", "cni", "bin")}
+}
+
 func DefaultImageConfig() ImageConfig {
 	return ImageConfig{
 		Snapshotter:            defaults.DefaultSnapshotter,
@@ -42,7 +46,7 @@ func DefaultImageConfig() ImageConfig {
 func DefaultRuntimeConfig() RuntimeConfig {
 	return RuntimeConfig{
 		CniConfig: CniConfig{
-			NetworkPluginBinDir:        filepath.Join(os.Getenv("ProgramFiles"), "containerd", "cni", "bin"),
+			NetworkPluginBinDirs:       defaultNetworkPluginBinDirs(),
 			NetworkPluginConfDir:       filepath.Join(os.Getenv("ProgramFiles"), "containerd", "cni", "conf"),
 			NetworkPluginMaxConfNum:    1,
 			NetworkPluginSetupSerially: false,

diff --git a/internal/cri/server/service_linux.go b/internal/cri/server/service_linux.go
@@ -87,7 +87,7 @@ func (c *criService) initPlatform() (err error) {
 		i, err := cni.New(cni.WithMinNetworkCount(networkAttachCount),
 			cni.WithPluginConfDir(dir),
 			cni.WithPluginMaxConfNum(max),
-			cni.WithPluginDir([]string{c.config.NetworkPluginBinDir}))
+			cni.WithPluginDir(c.config.NetworkPluginBinDirs))
 		if err != nil {
 			return fmt.Errorf("failed to initialize cni: %w", err)
 		}

diff --git a/internal/cri/server/service_windows.go b/internal/cri/server/service_windows.go
@@ -53,7 +53,7 @@ func (c *criService) initPlatform() error {
 		i, err := cni.New(cni.WithMinNetworkCount(windowsNetworkAttachCount),
 			cni.WithPluginConfDir(dir),
 			cni.WithPluginMaxConfNum(max),
-			cni.WithPluginDir([]string{c.config.NetworkPluginBinDir}))
+			cni.WithPluginDir(c.config.NetworkPluginBinDirs))
 		if err != nil {
 			return fmt.Errorf("failed to initialize cni: %w", err)
 		}

diff --git a/pkg/deprecation/deprecation.go b/pkg/deprecation/deprecation.go
@@ -31,6 +31,8 @@ const (
 	CRIRegistryAuths Warning = Prefix + "cri-registry-auths"
 	// CRIRegistryConfigs is a warning for the use of the `configs` property
 	CRIRegistryConfigs Warning = Prefix + "cri-registry-configs"
+	// CRICNIBinDir is a warning for the use of the `bin_dir` property
+	CRICNIBinDir = Prefix + "cri-cni-bin-dir"
 	// OTLPTracingConfig is a warning for the use of the `otlp` property
 	TracingOTLPConfig Warning = Prefix + "tracing-processor-config"
 	// TracingServiceConfig is a warning for the use of the `tracing` property
@@ -52,6 +54,8 @@ var messages = map[Warning]string{
 		"Use `ImagePullSecrets` instead.",
 	CRIRegistryConfigs: "The `configs` property of `[plugins.\"io.containerd.grpc.v1.cri\".registry]` is deprecated since containerd v1.5 and will be removed in containerd v2.1." +
 		"Use `config_path` instead.",
+	CRICNIBinDir: "The `bin_dir` property of `[plugins.\"io.containerd.cri.v1.runtime\".cni`] is deprecated since containerd v2.1 and will be removed in containerd v2.2. " +
+		"Use `bin_dirs` in the same section instead.",
 
 	TracingOTLPConfig: "The `otlp` property of `[plugins.\"io.containerd.tracing.processor.v1\".otlp]` is deprecated since containerd v1.6 and will be removed in containerd v2.0." +
 		"Use OTLP environment variables instead: https://opentelemetry.io/docs/specs/otel/protocol/exporter/",

diff --git a/plugins/cri/runtime/plugin.go b/plugins/cri/runtime/plugin.go
@@ -206,6 +206,9 @@ func configMigration(ctx context.Context, configVersion int, pluginConfigs map[s
 func migrateConfig(dst, src map[string]interface{}) {
 	for k, v := range src {
 		switch k {
+		case "cni":
+			// skip (handled separately below)
+			continue
 		case "containerd":
 			// skip (handled separately below)
 			continue
@@ -235,6 +238,28 @@ func migrateConfig(dst, src map[string]interface{}) {
 		}
 	}
 
+	if cniConf, ok := src["cni"].(map[string]interface{}); ok {
+		newCniConf, ok := dst["cni"].(map[string]interface{})
+		if !ok {
+			newCniConf = map[string]interface{}{}
+		}
+		for k, v := range cniConf {
+			switch k {
+			case "bin_dir":
+				// migrate `bin_dir` to `bin_dirs` only if `bin_dirs`
+				// is not already set
+				if binDirs, ok := newCniConf["bin_dirs"].([]string); !ok || len(binDirs) == 0 {
+					newCniConf["bin_dirs"] = []string{v.(string)}
+				}
+			default:
+				if _, ok := newCniConf[k]; !ok {
+					newCniConf[k] = v
+				}
+			}
+		}
+		dst["cni"] = newCniConf
+	}
+
 	// migrate cri containerd configs
 	containerdConf, ok := src["containerd"].(map[string]interface{})
 	if !ok {

diff --git a/plugins/cri/runtime/plugin_test.go b/plugins/cri/runtime/plugin_test.go
@@ -28,6 +28,7 @@ import (
 
 func TestCRIRuntimePluginConfigMigration(t *testing.T) {
 	runcSandboxer := "podsandbox"
+	cniBinDir := "/opt/cni/bin"
 
 	grpcCri := map[string]interface{}{
 		"enable_selinux":              true,
@@ -41,6 +42,9 @@ func TestCRIRuntimePluginConfigMigration(t *testing.T) {
 				},
 			},
 		},
+		"cni": map[string]interface{}{
+			"bin_dir": cniBinDir,
+		},
 	}
 
 	pluginConfigs := map[string]interface{}{
@@ -66,4 +70,12 @@ func TestCRIRuntimePluginConfigMigration(t *testing.T) {
 	require.NotNil(t, runc)
 	assert.Equal(t, runcSandboxer, runc["sandboxer"])
 	assert.NotContains(t, runc, "sandbox_mode")
+
+	cni, ok := runtimeConf["cni"].(map[string]interface{})
+	require.True(t, ok)
+	require.NotNil(t, cni)
+	cniBinDirs, ok := cni["bin_dirs"].([]string)
+	require.True(t, ok)
+	require.Len(t, cniBinDirs, 1)
+	require.Equal(t, cniBinDir, cniBinDirs[0])
 }