blockio: add blockio_reconfigure option

askervin · askervin · commit d460cf1ecab1 · 2022-09-22T13:44:08.000+03:00
Reconfiguring blockio is needed in the following scenarios.
- Block devices (/dev/*) change, for example when a CSI driver adds a
  new block device for a pod. Reconfiguration finds current devices
  and enables applying appropriate blockio parameters for them.
- Blockio class configuration file has been updated in the
  system. Reconfiguration applies changes without restarting
  containerd.

Signed-off-by: Antti Kervinen &lt;antti.kervinen@intel.com&gt;
diff --git a/docs/man/containerd-config.toml.5.md b/docs/man/containerd-config.toml.5.md
@@ -105,6 +105,8 @@ documentation.
     (Default: **""**). Controls I/O scheduler priority and bandwidth throttling.
     See [blockio configuration](https://github.com/intel/goresctrl/blob/main/doc/blockio.md#configuration)
     for details of the file format.
+  - **blockio_reconfigure** (Linux only) specifies if the configuration file must be re-read
+    and block devices rescanned in the system when it can make a difference. (Default: **false**)
   - **rdt_config_file** (Linux only) specifies path to a configuration used for configuring
     RDT (Default: **""**). Enables support for Intel RDT, a technology
     for cache and memory bandwidth management.
@@ -197,6 +199,7 @@ imports = ["/etc/containerd/runtime_*.toml", "./debug.toml"]
     sched_core = true
   [plugins."io.containerd.service.v1.tasks-service"]
     blockio_config_file = ""
+    blockio_reconfigure = false
     rdt_config_file = ""
 ```
 
diff --git a/pkg/cri/server/blockio_linux.go b/pkg/cri/server/blockio_linux.go
@@ -50,5 +50,8 @@ func (c *criService) blockIOClassFromAnnotations(containerName string, container
 // blockIOToLinuxOci converts blockio class name into the LinuxBlockIO
 // structure in the OCI runtime spec.
 func blockIOToLinuxOci(className string) (*runtimespec.LinuxBlockIO, error) {
+	if err := tasks.BlockIOReconfigure(); err != nil {
+		return nil, err
+	}
 	return blockio.OciLinuxBlockIO(className)
 }
diff --git a/services/tasks/blockio_default.go b/services/tasks/blockio_default.go
@@ -21,4 +21,6 @@ package tasks
 
 func BlockIOEnabled() bool { return false }
 
-func initBlockIO(configFilePath string) error { return nil }
+func BlockIOReconfigure() error { return nil }
+
+func initBlockIO(configFilePath string, reconfigure bool) error { return nil }
diff --git a/services/tasks/blockio_linux.go b/services/tasks/blockio_linux.go
@@ -28,11 +28,26 @@ import (
 )
 
 var blockIOEnabled bool
+var blockIOConfigFilePath string
+var blockIOReconfigure bool
 
 func BlockIOEnabled() bool { return blockIOEnabled }
 
-func initBlockIO(configFilePath string) error {
+func BlockIOReconfigure() error {
+	if !blockIOEnabled || !blockIOReconfigure || blockIOConfigFilePath == "" {
+		return nil
+	}
+	err := blockio.SetConfigFromFile(blockIOConfigFilePath, true)
+	if err != nil {
+		log.L.Error("blockio reconfiguration error: %w", err)
+	}
+	return err
+}
+
+func initBlockIO(configFilePath string, reconfigure bool) error {
 	blockIOEnabled = false
+	blockIOConfigFilePath = ""
+	blockIOReconfigure = false
 
 	if configFilePath == "" {
 		log.L.Debug("No blockio config file specified, blockio not configured")
@@ -44,7 +59,8 @@ func initBlockIO(configFilePath string) error {
 	}
 
 	blockIOEnabled = true
+	blockIOConfigFilePath = configFilePath
+	blockIOReconfigure = reconfigure
 
 	return nil
-
 }
diff --git a/services/tasks/local.go b/services/tasks/local.go
@@ -69,6 +69,8 @@ const (
 type Config struct {
 	// BlockIOConfigFile specifies the path to blockio configuration file
 	BlockIOConfigFile string `toml:"blockio_config_file" json:"blockioConfigFile"`
+	// BlockIOReconfigure specifies if the configuration file and system's block devices are reconfigured
+	BlockIOReconfigure bool `toml:"blockio_reconfigure" json:"blockioReconfigure"`
 	// RdtConfigFile specifies the path to RDT configuration file
 	RdtConfigFile string `toml:"rdt_config_file" json:"rdtConfigFile"`
 }
@@ -141,7 +143,7 @@ func initFunc(ic *plugin.InitContext) (interface{}, error) {
 		l.monitor.Monitor(t, nil)
 	}
 
-	if err := initBlockIO(config.BlockIOConfigFile); err != nil {
+	if err := initBlockIO(config.BlockIOConfigFile, config.BlockIOReconfigure); err != nil {
 		log.G(ic.Context).WithError(err).Errorf("blockio initialization failed")
 	}
 	if err := initRdt(config.RdtConfigFile); err != nil {

Original file line number	Diff line number	Diff line change
`@@ -50,5 +50,8 @@ func (c *criService) blockIOClassFromAnnotations(containerName string, container`
`50`	`50`	`// blockIOToLinuxOci converts blockio class name into the LinuxBlockIO`
`51`	`51`	`// structure in the OCI runtime spec.`
`52`	`52`	`func blockIOToLinuxOci(className string) (*runtimespec.LinuxBlockIO, error) {`
	`53`	`+ if err := tasks.BlockIOReconfigure(); err != nil {`
	`54`	`+ return nil, err`
	`55`	`+ }`
`53`	`56`	`return blockio.OciLinuxBlockIO(className)`
`54`	`57`	`}`
Original file line number	Diff line number	Diff line change
`@@ -69,6 +69,8 @@ const (`
`69`	`69`	`type Config struct {`
`70`	`70`	`// BlockIOConfigFile specifies the path to blockio configuration file`
`71`	`71`	BlockIOConfigFile string `toml:"blockio_config_file" json:"blockioConfigFile"`
	`72`	`+ // BlockIOReconfigure specifies if the configuration file and system's block devices are reconfigured`
	`73`	+ BlockIOReconfigure bool `toml:"blockio_reconfigure" json:"blockioReconfigure"`
`72`	`74`	`// RdtConfigFile specifies the path to RDT configuration file`
`73`	`75`	RdtConfigFile string `toml:"rdt_config_file" json:"rdtConfigFile"`
`74`	`76`	`}`
`@@ -141,7 +143,7 @@ func initFunc(ic *plugin.InitContext) (interface{}, error) {`
`141`	`143`	`l.monitor.Monitor(t, nil)`
`142`	`144`	`}`
`143`	`145`
`144`		`- if err := initBlockIO(config.BlockIOConfigFile); err != nil {`
	`146`	`+ if err := initBlockIO(config.BlockIOConfigFile, config.BlockIOReconfigure); err != nil {`
`145`	`147`	`log.G(ic.Context).WithError(err).Errorf("blockio initialization failed")`
`146`	`148`	`}`
`147`	`149`	`if err := initRdt(config.RdtConfigFile); err != nil {`