Skip to content

Commit 8f936ae

Browse files
author
Tibor Vass
committed
Add DeviceRequests to HostConfig to support NVIDIA GPUs
This patch hard-codes support for NVIDIA GPUs. In a future patch it should move out into its own Device Plugin. Signed-off-by: Tibor Vass <[email protected]>
1 parent 36d2c8b commit 8f936ae

10 files changed

Lines changed: 529 additions & 8 deletions

File tree

‎api/swagger.yaml‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -210,6 +210,43 @@ definitions:
210210
PathInContainer: "/dev/deviceName"
211211
CgroupPermissions: "mrw"
212212

213+
DeviceRequest:
214+
type: "object"
215+
description: "A request for devices to be sent to device drivers"
216+
properties:
217+
Driver:
218+
type: "string"
219+
example: "nvidia"
220+
Count:
221+
type: "integer"
222+
example: -1
223+
DeviceIDs:
224+
type: "array"
225+
items:
226+
type: "string"
227+
example:
228+
- "0"
229+
- "1"
230+
- "GPU-fef8089b-4820-abfc-e83e-94318197576e"
231+
Capabilities:
232+
description: |
233+
A list of capabilities; an OR list of AND lists of capabilities.
234+
type: "array"
235+
items:
236+
type: "array"
237+
items:
238+
type: "string"
239+
example:
240+
# gpu AND nvidia AND compute
241+
- ["gpu", "nvidia", "compute"]
242+
Options:
243+
description: |
244+
Driver-specific options, specified as a key/value pairs. These options
245+
are passed directly to the driver.
246+
type: "object"
247+
additionalProperties:
248+
type: "string"
249+
213250
ThrottleDevice:
214251
type: "object"
215252
properties:
@@ -421,6 +458,11 @@ definitions:
421458
items:
422459
type: "string"
423460
example: "c 13:* rwm"
461+
DeviceRequests:
462+
description: "a list of requests for devices to be sent to device drivers"
463+
type: "array"
464+
items:
465+
$ref: "#/definitions/DeviceRequest"
424466
DiskQuota:
425467
description: "Disk limit (in bytes)."
426468
type: "integer"

‎api/types/container/host_config.go‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -244,6 +244,16 @@ func (n PidMode) Container() string {
244244
return ""
245245
}
246246

247+
// DeviceRequest represents a request for devices from a device driver.
248+
// Used by GPU device drivers.
249+
type DeviceRequest struct {
250+
Driver string // Name of device driver
251+
Count int // Number of devices to request (-1 = All)
252+
DeviceIDs []string // List of device IDs as recognizable by the device driver
253+
Capabilities [][]string // An OR list of AND lists of device capabilities (e.g. "gpu")
254+
Options map[string]string // Options to pass onto the device driver
255+
}
256+
247257
// DeviceMapping represents the device mapping between the host and the container.
248258
type DeviceMapping struct {
249259
PathOnHost string
@@ -327,6 +337,7 @@ type Resources struct {
327337
CpusetMems string // CpusetMems 0-2, 0,1
328338
Devices []DeviceMapping // List of devices to map inside the container
329339
DeviceCgroupRules []string // List of rule to be added to the device cgroup
340+
DeviceRequests []DeviceRequest // List of device requests for device drivers
330341
DiskQuota int64 // Disk limit (in bytes)
331342
KernelMemory int64 // Kernel memory limit (in bytes)
332343
KernelMemoryTCP int64 // Hard limit for kernel TCP buffer memory (in bytes)

‎daemon/devices_linux.go‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
package daemon // import "github.com/docker/docker/daemon"
2+
3+
import (
4+
"github.com/docker/docker/api/types/container"
5+
"github.com/docker/docker/pkg/capabilities"
6+
specs "github.com/opencontainers/runtime-spec/specs-go"
7+
)
8+
9+
var deviceDrivers = map[string]*deviceDriver{}
10+
11+
type deviceDriver struct {
12+
capset capabilities.Set
13+
updateSpec func(*specs.Spec, *deviceInstance) error
14+
}
15+
16+
type deviceInstance struct {
17+
req container.DeviceRequest
18+
selectedCaps []string
19+
}
20+
21+
func registerDeviceDriver(name string, d *deviceDriver) {
22+
deviceDrivers[name] = d
23+
}
24+
25+
func (daemon *Daemon) handleDevice(req container.DeviceRequest, spec *specs.Spec) error {
26+
if req.Driver == "" {
27+
for _, dd := range deviceDrivers {
28+
if selected := dd.capset.Match(req.Capabilities); selected != nil {
29+
return dd.updateSpec(spec, &deviceInstance{req: req, selectedCaps: selected})
30+
}
31+
}
32+
} else if dd := deviceDrivers[req.Driver]; dd != nil {
33+
if selected := dd.capset.Match(req.Capabilities); selected != nil {
34+
return dd.updateSpec(spec, &deviceInstance{req: req, selectedCaps: selected})
35+
}
36+
}
37+
return incompatibleDeviceRequest{req.Driver, req.Capabilities}
38+
}

‎daemon/errors.go‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,17 @@ func (e invalidIdentifier) Error() string {
8080

8181
func (invalidIdentifier) InvalidParameter() {}
8282

83+
type incompatibleDeviceRequest struct {
84+
driver string
85+
caps [][]string
86+
}
87+
88+
func (i incompatibleDeviceRequest) Error() string {
89+
return fmt.Sprintf("could not select device driver %q with capabilities: %v", i.driver, i.caps)
90+
}
91+
92+
func (incompatibleDeviceRequest) InvalidParameter() {}
93+
8394
type duplicateMountPointError string
8495

8596
func (e duplicateMountPointError) Error() string {

‎daemon/nvidia_linux.go‎

Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,107 @@
1+
package daemon
2+
3+
import (
4+
"os/exec"
5+
"strconv"
6+
7+
"github.com/containerd/containerd/contrib/nvidia"
8+
"github.com/docker/docker/pkg/capabilities"
9+
"github.com/opencontainers/runtime-spec/specs-go"
10+
"github.com/pkg/errors"
11+
)
12+
13+
// TODO: nvidia should not be hard-coded, and should be a device plugin instead on the daemon object.
14+
// TODO: add list of device capabilities in daemon/node info
15+
16+
var errConflictCountDeviceIDs = errors.New("cannot set both Count and DeviceIDs on device request")
17+
18+
// stolen from github.com/containerd/containerd/contrib/nvidia
19+
const nvidiaCLI = "nvidia-container-cli"
20+
21+
// These are NVIDIA-specific capabilities stolen from github.com/containerd/containerd/contrib/nvidia.allCaps
22+
var allNvidiaCaps = map[nvidia.Capability]struct{}{
23+
nvidia.Compute: {},
24+
nvidia.Compat32: {},
25+
nvidia.Graphics: {},
26+
nvidia.Utility: {},
27+
nvidia.Video: {},
28+
nvidia.Display: {},
29+
}
30+
31+
func init() {
32+
if _, err := exec.LookPath(nvidiaCLI); err != nil {
33+
// do not register Nvidia driver if helper binary is not present.
34+
return
35+
}
36+
capset := capabilities.Set{"gpu": struct{}{}, "nvidia": struct{}{}}
37+
nvidiaDriver := &deviceDriver{
38+
capset: capset,
39+
updateSpec: setNvidiaGPUs,
40+
}
41+
for c := range capset {
42+
nvidiaDriver.capset[c] = struct{}{}
43+
}
44+
registerDeviceDriver("nvidia", nvidiaDriver)
45+
}
46+
47+
func setNvidiaGPUs(s *specs.Spec, dev *deviceInstance) error {
48+
var opts []nvidia.Opts
49+
50+
req := dev.req
51+
if req.Count != 0 && len(req.DeviceIDs) > 0 {
52+
return errConflictCountDeviceIDs
53+
}
54+
55+
if len(req.DeviceIDs) > 0 {
56+
var ids []int
57+
var uuids []string
58+
for _, devID := range req.DeviceIDs {
59+
id, err := strconv.Atoi(devID)
60+
if err == nil {
61+
ids = append(ids, id)
62+
continue
63+
}
64+
// if not an integer, then assume UUID.
65+
uuids = append(uuids, devID)
66+
}
67+
if len(ids) > 0 {
68+
opts = append(opts, nvidia.WithDevices(ids...))
69+
}
70+
if len(uuids) > 0 {
71+
opts = append(opts, nvidia.WithDeviceUUIDs(uuids...))
72+
}
73+
}
74+
75+
if req.Count < 0 {
76+
opts = append(opts, nvidia.WithAllDevices)
77+
} else if req.Count > 0 {
78+
opts = append(opts, nvidia.WithDevices(countToDevices(req.Count)...))
79+
}
80+
81+
var nvidiaCaps []nvidia.Capability
82+
// req.Capabilities contains device capabilities, some but not all are NVIDIA driver capabilities.
83+
for _, c := range dev.selectedCaps {
84+
nvcap := nvidia.Capability(c)
85+
if _, isNvidiaCap := allNvidiaCaps[nvcap]; isNvidiaCap {
86+
nvidiaCaps = append(nvidiaCaps, nvcap)
87+
continue
88+
}
89+
// TODO: nvidia.WithRequiredCUDAVersion
90+
// for now we let the prestart hook verify cuda versions but errors are not pretty.
91+
}
92+
93+
if nvidiaCaps != nil {
94+
opts = append(opts, nvidia.WithCapabilities(nvidiaCaps...))
95+
}
96+
97+
return nvidia.WithGPUs(opts...)(nil, nil, nil, s)
98+
}
99+
100+
// countToDevices returns the list 0, 1, ... count-1 of deviceIDs.
101+
func countToDevices(count int) []int {
102+
devices := make([]int, count)
103+
for i := range devices {
104+
devices[i] = i
105+
}
106+
return devices
107+
}

‎daemon/oci_linux.go‎

Lines changed: 16 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -85,7 +85,7 @@ func setResources(s *specs.Spec, r containertypes.Resources) error {
8585
return nil
8686
}
8787

88-
func setDevices(s *specs.Spec, c *container.Container) error {
88+
func (daemon *Daemon) setDevices(s *specs.Spec, c *container.Container) error {
8989
// Build lists of devices allowed and created within the container.
9090
var devs []specs.LinuxDevice
9191
devPermissions := s.Linux.Resources.Devices
@@ -122,6 +122,13 @@ func setDevices(s *specs.Spec, c *container.Container) error {
122122

123123
s.Linux.Devices = append(s.Linux.Devices, devs...)
124124
s.Linux.Resources.Devices = devPermissions
125+
126+
for _, req := range c.HostConfig.DeviceRequests {
127+
if err := daemon.handleDevice(req, s); err != nil {
128+
return err
129+
}
130+
}
131+
125132
return nil
126133
}
127134

@@ -751,7 +758,7 @@ func (daemon *Daemon) createSpec(c *container.Container) (retSpec *specs.Spec, e
751758
if err := daemon.initCgroupsPath(parentPath); err != nil {
752759
return nil, fmt.Errorf("linux init cgroups path: %v", err)
753760
}
754-
if err := setDevices(&s, c); err != nil {
761+
if err := daemon.setDevices(&s, c); err != nil {
755762
return nil, fmt.Errorf("linux runtime spec devices: %v", err)
756763
}
757764
if err := daemon.setRlimits(&s, c); err != nil {
@@ -818,15 +825,16 @@ func (daemon *Daemon) createSpec(c *container.Container) (retSpec *specs.Spec, e
818825
return nil, fmt.Errorf("linux mounts: %v", err)
819826
}
820827

828+
if s.Hooks == nil {
829+
s.Hooks = &specs.Hooks{}
830+
}
821831
for _, ns := range s.Linux.Namespaces {
822832
if ns.Type == "network" && ns.Path == "" && !c.Config.NetworkDisabled {
823833
target := filepath.Join("/proc", strconv.Itoa(os.Getpid()), "exe")
824-
s.Hooks = &specs.Hooks{
825-
Prestart: []specs.Hook{{
826-
Path: target,
827-
Args: []string{"libnetwork-setkey", "-exec-root=" + daemon.configStore.GetExecRoot(), c.ID, daemon.netController.ID()},
828-
}},
829-
}
834+
s.Hooks.Prestart = append(s.Hooks.Prestart, specs.Hook{
835+
Path: target,
836+
Args: []string{"libnetwork-setkey", "-exec-root=" + daemon.configStore.GetExecRoot(), c.ID, daemon.netController.ID()},
837+
})
830838
}
831839
}
832840

‎docs/api/version-history.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,8 @@ keywords: "API, Docker, rcli, REST, documentation"
4949
* `GET /info` now returns information about `DataPathPort` that is currently used in swarm
5050
* `GET /info` now returns `PidsLimit` boolean to indicate if the host kernel has
5151
PID limit support enabled.
52+
* `POST /containers/create` now accepts `DeviceRequests` as part of `HostConfig`.
53+
Can be used to set Nvidia GPUs.
5254
* `GET /swarm` endpoint now returns DataPathPort info
5355
* `POST /containers/create` now takes `KernelMemoryTCP` field to set hard limit for kernel TCP buffer memory.
5456
* `GET /service` now returns `MaxReplicas` as part of the `Placement`.

‎pkg/capabilities/caps.go‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
// Package capabilities allows to generically handle capabilities.
2+
package capabilities
3+
4+
// Set represents a set of capabilities.
5+
type Set map[string]struct{}
6+
7+
// Match tries to match set with caps, which is an OR list of AND lists of capabilities.
8+
// The matched AND list of capabilities is returned; or nil if none are matched.
9+
func (set Set) Match(caps [][]string) []string {
10+
if set == nil {
11+
return nil
12+
}
13+
anyof:
14+
for _, andList := range caps {
15+
for _, cap := range andList {
16+
if _, ok := set[cap]; !ok {
17+
continue anyof
18+
}
19+
}
20+
return andList
21+
}
22+
return nil
23+
}

0 commit comments

Comments
 (0)