Skip to content

Commit cd16b31

Browse files
bart0shbart0sh
committed
Get CDI devices from CRI Config.CDIDevices field
Signed-off-by: Ed Bartosh <[email protected]>
1 parent d2b578f commit cd16b31

5 files changed

Lines changed: 439 additions & 19 deletions

File tree

‎pkg/cri/opts/spec_linux.go‎

Lines changed: 31 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -30,10 +30,12 @@ import (
3030
"github.com/containerd/cgroups/v3"
3131
"github.com/sirupsen/logrus"
3232
"golang.org/x/sys/unix"
33+
runtime "k8s.io/cri-api/pkg/apis/runtime/v1"
3334

3435
"github.com/containerd/containerd/containers"
3536
"github.com/containerd/containerd/log"
3637
"github.com/containerd/containerd/oci"
38+
ctrdutil "github.com/containerd/containerd/pkg/cri/util"
3739
)
3840

3941
// Linux dependent OCI spec opts.
@@ -139,18 +141,40 @@ func IsCgroup2UnifiedMode() bool {
139141
}
140142

141143
// WithCDI updates OCI spec with CDI content
142-
func WithCDI(annotations map[string]string) oci.SpecOpts {
144+
func WithCDI(annotations map[string]string, CDIDevices []*runtime.CDIDevice) oci.SpecOpts {
143145
return func(ctx context.Context, _ oci.Client, c *containers.Container, s *oci.Spec) error {
144-
// TODO: Once CRI is extended with native CDI support this will need to be updated...
145-
_, cdiDevices, err := cdi.ParseAnnotations(annotations)
146+
// Add devices from CDIDevices CRI field
147+
var devices []string
148+
var err error
149+
for _, device := range CDIDevices {
150+
devices = append(devices, device.Name)
151+
}
152+
log.G(ctx).Infof("Container %v: CDI devices from CRI Config.CDIDevices: %v", c.ID, devices)
153+
154+
// Add devices from CDI annotations
155+
_, devsFromAnnotations, err := cdi.ParseAnnotations(annotations)
146156
if err != nil {
147157
return fmt.Errorf("failed to parse CDI device annotations: %w", err)
148158
}
149-
if cdiDevices == nil {
150-
return nil
159+
160+
if devsFromAnnotations != nil {
161+
log.G(ctx).Infof("Container %v: CDI devices from annotations: %v", c.ID, devsFromAnnotations)
162+
for _, deviceName := range devsFromAnnotations {
163+
if ctrdutil.InStringSlice(devices, deviceName) {
164+
// TODO: change to Warning when passing CDI devices as annotations is deprecated
165+
log.G(ctx).Debugf("Skipping duplicated CDI device %s", deviceName)
166+
continue
167+
}
168+
devices = append(devices, deviceName)
169+
}
170+
// TODO: change to Warning when passing CDI devices as annotations is deprecated
171+
log.G(ctx).Debug("Passing CDI devices as annotations will be deprecated soon, please use CRI CDIDevices instead")
151172
}
152173

153-
log.G(ctx).Infof("container %v: CDI devices: %v", c.ID, cdiDevices)
174+
if len(devices) == 0 {
175+
// No devices found, skip device injection
176+
return nil
177+
}
154178

155179
registry := cdi.GetRegistry()
156180
if err = registry.Refresh(); err != nil {
@@ -162,7 +186,7 @@ func WithCDI(annotations map[string]string) oci.SpecOpts {
162186
log.G(ctx).Warnf("CDI registry refresh failed: %v", err)
163187
}
164188

165-
if _, err := registry.InjectDevices(s, cdiDevices...); err != nil {
189+
if _, err := registry.InjectDevices(s, devices...); err != nil {
166190
return fmt.Errorf("CDI device injection failed: %w", err)
167191
}
168192

‎pkg/cri/sbserver/container_create_linux.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -122,7 +122,7 @@ func (c *criService) containerSpecOpts(config *runtime.ContainerConfig, imageCon
122122
specOpts = append(specOpts, seccompSpecOpts)
123123
}
124124
if c.config.EnableCDI {
125-
specOpts = append(specOpts, customopts.WithCDI(config.Annotations))
125+
specOpts = append(specOpts, customopts.WithCDI(config.Annotations, config.CDIDevices))
126126
}
127127
return specOpts, nil
128128
}

‎pkg/cri/sbserver/container_create_linux_test.go‎

Lines changed: 203 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1634,13 +1634,20 @@ func TestCDIInjections(t *testing.T) {
16341634
for _, test := range []struct {
16351635
description string
16361636
cdiSpecFiles []string
1637+
cdiDevices []*runtime.CDIDevice
16371638
annotations map[string]string
16381639
expectError bool
16391640
expectDevices []runtimespec.LinuxDevice
16401641
expectEnv []string
16411642
}{
1642-
{description: "expect no CDI error for nil annotations"},
1643-
{description: "expect no CDI error for empty annotations",
1643+
{description: "expect no CDI error for nil annotations",
1644+
cdiDevices: []*runtime.CDIDevice{},
1645+
},
1646+
{description: "expect no CDI error for nil CDIDevices",
1647+
annotations: map[string]string{},
1648+
},
1649+
{description: "expect no CDI error for empty CDI devices and annotations",
1650+
cdiDevices: []*runtime.CDIDevice{},
16441651
annotations: map[string]string{},
16451652
},
16461653
{description: "expect CDI error for invalid CDI device reference in annotations",
@@ -1649,13 +1656,25 @@ func TestCDIInjections(t *testing.T) {
16491656
},
16501657
expectError: true,
16511658
},
1652-
{description: "expect CDI error for unresolvable devices",
1659+
{description: "expect CDI error for invalid CDI device reference in CDIDevices",
1660+
cdiDevices: []*runtime.CDIDevice{
1661+
{Name: "foobar"},
1662+
},
1663+
expectError: true,
1664+
},
1665+
{description: "expect CDI error for unresolvable devices in annotations",
16531666
annotations: map[string]string{
16541667
cdi.AnnotationPrefix + "vendor1_devices": "vendor1.com/device=no-such-dev",
16551668
},
16561669
expectError: true,
16571670
},
1658-
{description: "expect properly injected resolvable CDI devices",
1671+
{description: "expect CDI error for unresolvable devices in CDIDevices",
1672+
cdiDevices: []*runtime.CDIDevice{
1673+
{Name: "vendor1.com/device=no-such-dev"},
1674+
},
1675+
expectError: true,
1676+
},
1677+
{description: "expect properly injected resolvable CDI devices from annotations",
16591678
cdiSpecFiles: []string{
16601679
`
16611680
cdiVersion: "0.3.0"
@@ -1717,6 +1736,185 @@ containerEdits:
17171736
"VENDOR2=present",
17181737
},
17191738
},
1739+
{description: "expect properly injected resolvable CDI devices from CDIDevices",
1740+
cdiSpecFiles: []string{
1741+
`
1742+
cdiVersion: "0.3.0"
1743+
kind: "vendor1.com/device"
1744+
devices:
1745+
- name: foo
1746+
containerEdits:
1747+
deviceNodes:
1748+
- path: /dev/loop8
1749+
type: b
1750+
major: 7
1751+
minor: 8
1752+
env:
1753+
- FOO=injected
1754+
containerEdits:
1755+
env:
1756+
- "VENDOR1=present"
1757+
`,
1758+
`
1759+
cdiVersion: "0.3.0"
1760+
kind: "vendor2.com/device"
1761+
devices:
1762+
- name: bar
1763+
containerEdits:
1764+
deviceNodes:
1765+
- path: /dev/loop9
1766+
type: b
1767+
major: 7
1768+
minor: 9
1769+
env:
1770+
- BAR=injected
1771+
containerEdits:
1772+
env:
1773+
- "VENDOR2=present"
1774+
`,
1775+
},
1776+
cdiDevices: []*runtime.CDIDevice{
1777+
{Name: "vendor1.com/device=foo"},
1778+
{Name: "vendor2.com/device=bar"},
1779+
},
1780+
expectDevices: []runtimespec.LinuxDevice{
1781+
{
1782+
Path: "/dev/loop8",
1783+
Type: "b",
1784+
Major: 7,
1785+
Minor: 8,
1786+
},
1787+
{
1788+
Path: "/dev/loop9",
1789+
Type: "b",
1790+
Major: 7,
1791+
Minor: 9,
1792+
},
1793+
},
1794+
expectEnv: []string{
1795+
"FOO=injected",
1796+
"VENDOR1=present",
1797+
"BAR=injected",
1798+
"VENDOR2=present",
1799+
},
1800+
},
1801+
{description: "expect CDI devices from CDIDevices and annotations",
1802+
cdiSpecFiles: []string{
1803+
`
1804+
cdiVersion: "0.3.0"
1805+
kind: "vendor1.com/device"
1806+
devices:
1807+
- name: foo
1808+
containerEdits:
1809+
deviceNodes:
1810+
- path: /dev/loop8
1811+
type: b
1812+
major: 7
1813+
minor: 8
1814+
env:
1815+
- FOO=injected
1816+
containerEdits:
1817+
env:
1818+
- "VENDOR1=present"
1819+
`,
1820+
`
1821+
cdiVersion: "0.3.0"
1822+
kind: "vendor2.com/device"
1823+
devices:
1824+
- name: bar
1825+
containerEdits:
1826+
deviceNodes:
1827+
- path: /dev/loop9
1828+
type: b
1829+
major: 7
1830+
minor: 9
1831+
env:
1832+
- BAR=injected
1833+
containerEdits:
1834+
env:
1835+
- "VENDOR2=present"
1836+
`,
1837+
`
1838+
cdiVersion: "0.3.0"
1839+
kind: "vendor3.com/device"
1840+
devices:
1841+
- name: foo3
1842+
containerEdits:
1843+
deviceNodes:
1844+
- path: /dev/loop10
1845+
type: b
1846+
major: 7
1847+
minor: 10
1848+
env:
1849+
- FOO3=injected
1850+
containerEdits:
1851+
env:
1852+
- "VENDOR3=present"
1853+
`,
1854+
`
1855+
cdiVersion: "0.3.0"
1856+
kind: "vendor4.com/device"
1857+
devices:
1858+
- name: bar4
1859+
containerEdits:
1860+
deviceNodes:
1861+
- path: /dev/loop11
1862+
type: b
1863+
major: 7
1864+
minor: 11
1865+
env:
1866+
- BAR4=injected
1867+
containerEdits:
1868+
env:
1869+
- "VENDOR4=present"
1870+
`,
1871+
},
1872+
cdiDevices: []*runtime.CDIDevice{
1873+
{Name: "vendor1.com/device=foo"},
1874+
{Name: "vendor2.com/device=bar"},
1875+
{Name: "vendor3.com/device=foo3"},
1876+
},
1877+
annotations: map[string]string{
1878+
cdi.AnnotationPrefix + "vendor3_devices": "vendor3.com/device=foo3", // Duplicated device, should be ignored
1879+
cdi.AnnotationPrefix + "vendor4_devices": "vendor4.com/device=bar4",
1880+
},
1881+
expectDevices: []runtimespec.LinuxDevice{
1882+
{
1883+
Path: "/dev/loop8",
1884+
Type: "b",
1885+
Major: 7,
1886+
Minor: 8,
1887+
},
1888+
{
1889+
Path: "/dev/loop9",
1890+
Type: "b",
1891+
Major: 7,
1892+
Minor: 9,
1893+
},
1894+
{
1895+
Path: "/dev/loop10",
1896+
Type: "b",
1897+
Major: 7,
1898+
Minor: 10,
1899+
},
1900+
{
1901+
Path: "/dev/loop11",
1902+
Type: "b",
1903+
Major: 7,
1904+
Minor: 11,
1905+
},
1906+
},
1907+
expectEnv: []string{
1908+
"FOO=injected",
1909+
"VENDOR1=present",
1910+
"BAR=injected",
1911+
"VENDOR2=present",
1912+
"FOO3=injected",
1913+
"VENDOR3=present",
1914+
"BAR4=injected",
1915+
"VENDOR4=present",
1916+
},
1917+
},
17201918
} {
17211919
t.Run(test.description, func(t *testing.T) {
17221920
spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime)
@@ -1734,7 +1932,7 @@ containerEdits:
17341932
err = reg.Configure(cdi.WithSpecDirs(cdiDir))
17351933
require.NoError(t, err)
17361934

1737-
injectFun := customopts.WithCDI(test.annotations)
1935+
injectFun := customopts.WithCDI(test.annotations, test.cdiDevices)
17381936
err = injectFun(ctx, nil, testContainer, spec)
17391937
assert.Equal(t, test.expectError, err != nil)
17401938

‎pkg/cri/server/container_create_linux.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -415,7 +415,7 @@ func (c *criService) containerSpecOpts(config *runtime.ContainerConfig, imageCon
415415
specOpts = append(specOpts, seccompSpecOpts)
416416
}
417417
if c.config.EnableCDI {
418-
specOpts = append(specOpts, customopts.WithCDI(config.Annotations))
418+
specOpts = append(specOpts, customopts.WithCDI(config.Annotations, config.CDIDevices))
419419
}
420420
return specOpts, nil
421421
}

0 commit comments

Comments
 (0)