Skip to content

Commit 8898550

Browse files
crosbymichaelcrosbymichael
authored
Merge pull request #1498 from mxpv/base
Specify base OCI runtime spec
2 parents c7f25cb + 17c61e3 commit 8898550

11 files changed

Lines changed: 197 additions & 9 deletions

‎docs/config.md‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -135,6 +135,13 @@ version = 2
135135
# i.e pass host devices through to privileged containers.
136136
privileged_without_host_devices = false
137137

138+
# base_runtime_spec is a file path to a JSON file with the OCI spec that will be used as the base spec that all
139+
# container's are created from.
140+
# Use containerd's `ctr oci spec > /etc/containerd/cri-base.json` to output initial spec file.
141+
# Spec files are loaded at launch, so containerd daemon must be restared on any changes to refresh default specs.
142+
# Still running containers and restarted containers will still be using the original spec from which that container was created.
143+
base_runtime_spec = ""
144+
138145
# 'plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options' is options specific to
139146
# "io.containerd.runc.v1" and "io.containerd.runc.v2". Its corresponding options type is:
140147
# https://github.com/containerd/containerd/blob/v1.3.2/runtime/v2/runc/options/oci.pb.go#L26 .

‎pkg/config/config.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,8 @@ type Runtime struct {
5252
// PrivilegedWithoutHostDevices overloads the default behaviour for adding host devices to the
5353
// runtime spec when the container is privileged. Defaults to false.
5454
PrivilegedWithoutHostDevices bool `toml:"privileged_without_host_devices" json:"privileged_without_host_devices"`
55+
// BaseRuntimeSpec is a json file with OCI spec to use as base spec that all container's will be created from.
56+
BaseRuntimeSpec string `toml:"base_runtime_spec" json:"baseRuntimeSpec"`
5557
}
5658

5759
// ContainerdConfig contains toml config related to containerd

‎pkg/server/container_create.go‎

Lines changed: 27 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -297,12 +297,36 @@ func (c *criService) volumeMounts(containerRootDir string, criMounts []*runtime.
297297
}
298298

299299
// runtimeSpec returns a default runtime spec used in cri-containerd.
300-
func runtimeSpec(id string, opts ...oci.SpecOpts) (*runtimespec.Spec, error) {
300+
func (c *criService) runtimeSpec(id string, baseSpecFile string, opts ...oci.SpecOpts) (*runtimespec.Spec, error) {
301301
// GenerateSpec needs namespace.
302302
ctx := ctrdutil.NamespacedContext()
303-
spec, err := oci.GenerateSpec(ctx, nil, &containers.Container{ID: id}, opts...)
303+
container := &containers.Container{ID: id}
304+
305+
if baseSpecFile != "" {
306+
baseSpec, ok := c.baseOCISpecs[baseSpecFile]
307+
if !ok {
308+
return nil, errors.Errorf("can't find base OCI spec %q", baseSpecFile)
309+
}
310+
311+
spec := oci.Spec{}
312+
if err := util.DeepCopy(&spec, &baseSpec); err != nil {
313+
return nil, errors.Wrap(err, "failed to clone OCI spec")
314+
}
315+
316+
// Fix up cgroups path
317+
applyOpts := append([]oci.SpecOpts{oci.WithNamespacedCgroup()}, opts...)
318+
319+
if err := oci.ApplyOpts(ctx, nil, container, &spec, applyOpts...); err != nil {
320+
return nil, errors.Wrap(err, "failed to apply OCI options")
321+
}
322+
323+
return &spec, nil
324+
}
325+
326+
spec, err := oci.GenerateSpec(ctx, nil, container, opts...)
304327
if err != nil {
305-
return nil, err
328+
return nil, errors.Wrap(err, "failed to generate spec")
306329
}
330+
307331
return spec, nil
308332
}

‎pkg/server/container_create_test.go‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,13 +21,15 @@ import (
2121
"path/filepath"
2222
"testing"
2323

24+
"github.com/containerd/containerd/oci"
2425
imagespec "github.com/opencontainers/image-spec/specs-go/v1"
2526
runtimespec "github.com/opencontainers/runtime-spec/specs-go"
2627
"github.com/stretchr/testify/assert"
2728
"github.com/stretchr/testify/require"
2829
runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
2930

3031
"github.com/containerd/cri/pkg/config"
32+
"github.com/containerd/cri/pkg/constants"
3133
"github.com/containerd/cri/pkg/containerd/opts"
3234
)
3335

@@ -381,3 +383,25 @@ func TestContainerAnnotationPassthroughContainerSpec(t *testing.T) {
381383
})
382384
}
383385
}
386+
387+
func TestBaseRuntimeSpec(t *testing.T) {
388+
c := newTestCRIService()
389+
c.baseOCISpecs = map[string]*oci.Spec{
390+
"/etc/containerd/cri-base.json": {
391+
Version: "1.0.2",
392+
Hostname: "old",
393+
},
394+
}
395+
396+
out, err := c.runtimeSpec("id1", "/etc/containerd/cri-base.json", oci.WithHostname("new"))
397+
assert.NoError(t, err)
398+
399+
assert.Equal(t, "1.0.2", out.Version)
400+
assert.Equal(t, "new", out.Hostname)
401+
402+
// Make sure original base spec not changed
403+
assert.NotEqual(t, out, c.baseOCISpecs["/etc/containerd/cri-base.json"])
404+
assert.Equal(t, c.baseOCISpecs["/etc/containerd/cri-base.json"].Hostname, "old")
405+
406+
assert.Equal(t, filepath.Join("/", constants.K8sContainerdNamespace, "id1"), out.Linux.CgroupsPath)
407+
}

‎pkg/server/container_create_unix.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -262,7 +262,7 @@ func (c *criService) containerSpec(id string, sandboxID string, sandboxPid uint3
262262
Type: runtimespec.CgroupNamespace,
263263
}))
264264
}
265-
return runtimeSpec(id, specOpts...)
265+
return c.runtimeSpec(id, ociRuntime.BaseRuntimeSpec, specOpts...)
266266
}
267267

268268
func (c *criService) containerSpecOpts(config *runtime.ContainerConfig, imageConfig *imagespec.ImageConfig) ([]oci.SpecOpts, error) {

‎pkg/server/container_create_unix_test.go‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1203,3 +1203,45 @@ func TestPrivilegedDevices(t *testing.T) {
12031203
}
12041204
}
12051205
}
1206+
1207+
func TestBaseOCISpec(t *testing.T) {
1208+
c := newTestCRIService()
1209+
baseLimit := int64(100)
1210+
c.baseOCISpecs = map[string]*oci.Spec{
1211+
"/etc/containerd/cri-base.json": {
1212+
Process: &runtimespec.Process{
1213+
User: runtimespec.User{AdditionalGids: []uint32{9999}},
1214+
Capabilities: &runtimespec.LinuxCapabilities{
1215+
Permitted: []string{"CAP_SETUID"},
1216+
},
1217+
},
1218+
Linux: &runtimespec.Linux{
1219+
Resources: &runtimespec.LinuxResources{
1220+
Memory: &runtimespec.LinuxMemory{Limit: &baseLimit}, // Will be overwritten by `getCreateContainerTestData`
1221+
},
1222+
},
1223+
},
1224+
}
1225+
1226+
ociRuntime := config.Runtime{}
1227+
ociRuntime.BaseRuntimeSpec = "/etc/containerd/cri-base.json"
1228+
1229+
testID := "test-id"
1230+
testSandboxID := "sandbox-id"
1231+
testContainerName := "container-name"
1232+
testPid := uint32(1234)
1233+
containerConfig, sandboxConfig, imageConfig, specCheck := getCreateContainerTestData()
1234+
1235+
spec, err := c.containerSpec(testID, testSandboxID, testPid, "", testContainerName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime)
1236+
assert.NoError(t, err)
1237+
1238+
specCheck(t, testID, testSandboxID, testPid, spec)
1239+
1240+
assert.Contains(t, spec.Process.User.AdditionalGids, uint32(9999))
1241+
assert.Len(t, spec.Process.User.AdditionalGids, 3)
1242+
1243+
assert.Contains(t, spec.Process.Capabilities.Permitted, "CAP_SETUID")
1244+
assert.Len(t, spec.Process.Capabilities.Permitted, 1)
1245+
1246+
assert.Equal(t, *spec.Linux.Resources.Memory.Limit, containerConfig.Linux.Resources.MemoryLimitInBytes)
1247+
}

‎pkg/server/container_create_windows.go‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -91,8 +91,7 @@ func (c *criService) containerSpec(id string, sandboxID string, sandboxPid uint3
9191
customopts.WithAnnotation(annotations.SandboxID, sandboxID),
9292
customopts.WithAnnotation(annotations.ContainerName, containerName),
9393
)
94-
95-
return runtimeSpec(id, specOpts...)
94+
return c.runtimeSpec(id, ociRuntime.BaseRuntimeSpec, specOpts...)
9695
}
9796

9897
// No extra spec options needed for windows.

‎pkg/server/sandbox_run_unix.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -156,7 +156,7 @@ func (c *criService) sandboxContainerSpec(id string, config *runtime.PodSandboxC
156156
customopts.WithAnnotation(annotations.SandboxLogDir, config.GetLogDirectory()),
157157
)
158158

159-
return runtimeSpec(id, specOpts...)
159+
return c.runtimeSpec(id, "", specOpts...)
160160
}
161161

162162
// sandboxContainerSpecOpts generates OCI spec options for

‎pkg/server/sandbox_run_windows.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,7 @@ func (c *criService) sandboxContainerSpec(id string, config *runtime.PodSandboxC
6767
customopts.WithAnnotation(annotations.SandboxLogDir, config.GetLogDirectory()),
6868
)
6969

70-
return runtimeSpec(id, specOpts...)
70+
return c.runtimeSpec(id, "", specOpts...)
7171
}
7272

7373
// No sandbox container spec options for windows yet.

‎pkg/server/service.go‎

Lines changed: 51 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,22 +17,26 @@
1717
package server
1818

1919
import (
20+
"encoding/json"
2021
"fmt"
2122
"io"
2223
"net/http"
24+
"os"
2325
"path/filepath"
2426
"time"
2527

2628
"github.com/containerd/containerd"
29+
"github.com/containerd/containerd/oci"
2730
"github.com/containerd/containerd/plugin"
28-
"github.com/containerd/cri/pkg/store/label"
2931
cni "github.com/containerd/go-cni"
3032
"github.com/pkg/errors"
3133
"github.com/sirupsen/logrus"
3234
"google.golang.org/grpc"
3335
runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
3436
"k8s.io/kubernetes/pkg/kubelet/server/streaming"
3537

38+
"github.com/containerd/cri/pkg/store/label"
39+
3640
"github.com/containerd/cri/pkg/atomic"
3741
criconfig "github.com/containerd/cri/pkg/config"
3842
ctrdutil "github.com/containerd/cri/pkg/containerd/util"
@@ -95,6 +99,8 @@ type criService struct {
9599
// cniNetConfMonitor is used to reload cni network conf if there is
96100
// any valid fs change events from cni network conf dir.
97101
cniNetConfMonitor *cniNetConfSyncer
102+
// baseOCISpecs contains cached OCI specs loaded via `Runtime.BaseRuntimeSpec`
103+
baseOCISpecs map[string]*oci.Spec
98104
}
99105

100106
// NewCRIService returns a new instance of CRIService
@@ -138,6 +144,12 @@ func NewCRIService(config criconfig.Config, client *containerd.Client) (CRIServi
138144
return nil, errors.Wrap(err, "failed to create cni conf monitor")
139145
}
140146

147+
// Preload base OCI specs
148+
c.baseOCISpecs, err = loadBaseOCISpecs(&config)
149+
if err != nil {
150+
return nil, err
151+
}
152+
141153
return c, nil
142154
}
143155

@@ -273,3 +285,41 @@ func (c *criService) register(s *grpc.Server) error {
273285
func imageFSPath(rootDir, snapshotter string) string {
274286
return filepath.Join(rootDir, fmt.Sprintf("%s.%s", plugin.SnapshotPlugin, snapshotter))
275287
}
288+
289+
func loadOCISpec(filename string) (*oci.Spec, error) {
290+
file, err := os.Open(filename)
291+
if err != nil {
292+
return nil, errors.Wrapf(err, "failed to open base OCI spec: %s", filename)
293+
}
294+
defer file.Close()
295+
296+
spec := oci.Spec{}
297+
if err := json.NewDecoder(file).Decode(&spec); err != nil {
298+
return nil, errors.Wrap(err, "failed to parse base OCI spec file")
299+
}
300+
301+
return &spec, nil
302+
}
303+
304+
func loadBaseOCISpecs(config *criconfig.Config) (map[string]*oci.Spec, error) {
305+
specs := map[string]*oci.Spec{}
306+
for _, cfg := range config.Runtimes {
307+
if cfg.BaseRuntimeSpec == "" {
308+
continue
309+
}
310+
311+
// Don't load same file twice
312+
if _, ok := specs[cfg.BaseRuntimeSpec]; ok {
313+
continue
314+
}
315+
316+
spec, err := loadOCISpec(cfg.BaseRuntimeSpec)
317+
if err != nil {
318+
return nil, errors.Wrapf(err, "failed to load base OCI spec from file: %s", cfg.BaseRuntimeSpec)
319+
}
320+
321+
specs[cfg.BaseRuntimeSpec] = spec
322+
}
323+
324+
return specs, nil
325+
}

0 commit comments

Comments
 (0)