Skip to content

Commit 62c9126

Browse files
mikebrowmikebrow
authored
Merge pull request #1501 from dims/tolerate-missing-hugetlb-cgroups-controller
Tolerate missing HugeTLB cgroups controller
2 parents 8898550 + d7ce093 commit 62c9126

6 files changed

Lines changed: 100 additions & 13 deletions

File tree

‎pkg/config/config.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -230,6 +230,10 @@ type PluginConfig struct {
230230
// UnsetSeccompProfile is the profile containerd/cri will use If the provided seccomp profile is
231231
// unset (`""`) for a container (default is `unconfined`)
232232
UnsetSeccompProfile string `toml:"unset_seccomp_profile" json:"unsetSeccompProfile"`
233+
// TolerateMissingHugePagesCgroupController if set to false will error out on create/update
234+
// container requests with huge page limits if the cgroup controller for hugepages is not present.
235+
// This helps with supporting Kubernetes <=1.18 out of the box. (default is `true`)
236+
TolerateMissingHugePagesCgroupController bool `toml:"tolerate_missing_hugepages_controller" json:"tolerateMissingHugePagesCgroupController"`
233237
}
234238

235239
// X509KeyPairStreaming contains the x509 configuration for streaming

‎pkg/config/config_unix.go‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,8 @@ func DefaultConfig() PluginConfig {
6363
},
6464
},
6565
},
66-
MaxConcurrentDownloads: 3,
67-
DisableProcMount: false,
66+
MaxConcurrentDownloads: 3,
67+
DisableProcMount: false,
68+
TolerateMissingHugePagesCgroupController: true,
6869
}
6970
}

‎pkg/containerd/opts/spec_unix.go‎

Lines changed: 86 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,8 @@ import (
2727
"sort"
2828
"strconv"
2929
"strings"
30+
"sync"
31+
"syscall"
3032

3133
"github.com/containerd/containerd/containers"
3234
"github.com/containerd/containerd/log"
@@ -36,6 +38,7 @@ import (
3638
runtimespec "github.com/opencontainers/runtime-spec/specs-go"
3739
"github.com/opencontainers/selinux/go-selinux/label"
3840
"github.com/pkg/errors"
41+
"github.com/sirupsen/logrus"
3942
"golang.org/x/sys/unix"
4043
runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
4144

@@ -405,7 +408,7 @@ func WithSelinuxLabels(process, mount string) oci.SpecOpts {
405408
}
406409

407410
// WithResources sets the provided resource restrictions
408-
func WithResources(resources *runtime.LinuxContainerResources) oci.SpecOpts {
411+
func WithResources(resources *runtime.LinuxContainerResources, tolerateMissingHugePagesCgroupController bool) oci.SpecOpts {
409412
return func(ctx context.Context, client oci.Client, c *containers.Container, s *runtimespec.Spec) (err error) {
410413
if resources == nil {
411414
return nil
@@ -448,16 +451,93 @@ func WithResources(resources *runtime.LinuxContainerResources) oci.SpecOpts {
448451
if limit != 0 {
449452
s.Linux.Resources.Memory.Limit = &limit
450453
}
451-
for _, limit := range hugepages {
452-
s.Linux.Resources.HugepageLimits = append(s.Linux.Resources.HugepageLimits, runtimespec.LinuxHugepageLimit{
453-
Pagesize: limit.PageSize,
454-
Limit: limit.Limit,
455-
})
454+
if isHugePagesControllerPresent() {
455+
for _, limit := range hugepages {
456+
s.Linux.Resources.HugepageLimits = append(s.Linux.Resources.HugepageLimits, runtimespec.LinuxHugepageLimit{
457+
Pagesize: limit.PageSize,
458+
Limit: limit.Limit,
459+
})
460+
}
461+
} else {
462+
if !tolerateMissingHugePagesCgroupController {
463+
return errors.Errorf("huge pages limits are specified but hugetlb cgroup controller is missing. " +
464+
"Please set tolerate_missing_hugepages_controller to `true` to ignore this error")
465+
}
466+
logrus.Warn("hugetlb cgroup controller is absent. skipping huge pages limits")
456467
}
457468
return nil
458469
}
459470
}
460471

472+
var (
473+
supportsHugetlbOnce sync.Once
474+
supportsHugetlb bool
475+
)
476+
477+
func isHugePagesControllerPresent() bool {
478+
supportsHugetlbOnce.Do(func() {
479+
supportsHugetlb = false
480+
if IsCgroup2UnifiedMode() {
481+
supportsHugetlb, _ = cgroupv2HasHugetlb()
482+
} else {
483+
supportsHugetlb, _ = cgroupv1HasHugetlb()
484+
}
485+
})
486+
return supportsHugetlb
487+
}
488+
489+
var (
490+
_cgroupv1HasHugetlbOnce sync.Once
491+
_cgroupv1HasHugetlb bool
492+
_cgroupv1HasHugetlbErr error
493+
_cgroupv2HasHugetlbOnce sync.Once
494+
_cgroupv2HasHugetlb bool
495+
_cgroupv2HasHugetlbErr error
496+
isUnifiedOnce sync.Once
497+
isUnified bool
498+
)
499+
500+
// cgroupv1HasHugetlb returns whether the hugetlb controller is present on
501+
// cgroup v1.
502+
func cgroupv1HasHugetlb() (bool, error) {
503+
_cgroupv1HasHugetlbOnce.Do(func() {
504+
if _, err := ioutil.ReadDir("/sys/fs/cgroup/hugetlb"); err != nil {
505+
_cgroupv1HasHugetlbErr = errors.Wrap(err, "readdir /sys/fs/cgroup/hugetlb")
506+
_cgroupv1HasHugetlb = false
507+
} else {
508+
_cgroupv1HasHugetlbErr = nil
509+
_cgroupv1HasHugetlb = true
510+
}
511+
})
512+
return _cgroupv1HasHugetlb, _cgroupv1HasHugetlbErr
513+
}
514+
515+
// cgroupv2HasHugetlb returns whether the hugetlb controller is present on
516+
// cgroup v2.
517+
func cgroupv2HasHugetlb() (bool, error) {
518+
_cgroupv2HasHugetlbOnce.Do(func() {
519+
controllers, err := ioutil.ReadFile("/sys/fs/cgroup/cgroup.controllers")
520+
if err != nil {
521+
_cgroupv2HasHugetlbErr = errors.Wrap(err, "read /sys/fs/cgroup/cgroup.controllers")
522+
return
523+
}
524+
_cgroupv2HasHugetlb = strings.Contains(string(controllers), "hugetlb")
525+
})
526+
return _cgroupv2HasHugetlb, _cgroupv2HasHugetlbErr
527+
}
528+
529+
// IsCgroup2UnifiedMode returns whether we are running in cgroup v2 unified mode.
530+
func IsCgroup2UnifiedMode() bool {
531+
isUnifiedOnce.Do(func() {
532+
var st syscall.Statfs_t
533+
if err := syscall.Statfs("/sys/fs/cgroup", &st); err != nil {
534+
panic("cannot statfs cgroup root")
535+
}
536+
isUnified = st.Type == unix.CGROUP2_SUPER_MAGIC
537+
})
538+
return isUnified
539+
}
540+
461541
// WithOOMScoreAdj sets the oom score
462542
func WithOOMScoreAdj(config *runtime.ContainerConfig, restrict bool) oci.SpecOpts {
463543
return func(ctx context.Context, client oci.Client, c *containers.Container, s *runtimespec.Spec) error {

‎pkg/server/container_create_unix.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -225,7 +225,7 @@ func (c *criService) containerSpec(id string, sandboxID string, sandboxPid uint3
225225
if c.config.DisableCgroup {
226226
specOpts = append(specOpts, customopts.WithDisabledCgroups)
227227
} else {
228-
specOpts = append(specOpts, customopts.WithResources(config.GetLinux().GetResources()))
228+
specOpts = append(specOpts, customopts.WithResources(config.GetLinux().GetResources(), c.config.TolerateMissingHugePagesCgroupController))
229229
if sandboxConfig.GetLinux().GetCgroupParent() != "" {
230230
cgroupsPath := getCgroupsPath(sandboxConfig.GetLinux().GetCgroupParent(), id)
231231
specOpts = append(specOpts, oci.WithCgroup(cgroupsPath))

‎pkg/server/container_update_resources_unix.go‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -72,7 +72,8 @@ func (c *criService) updateContainerResources(ctx context.Context,
7272
if err != nil {
7373
return errors.Wrap(err, "failed to get container spec")
7474
}
75-
newSpec, err := updateOCILinuxResource(ctx, oldSpec, resources)
75+
newSpec, err := updateOCILinuxResource(ctx, oldSpec, resources,
76+
c.config.TolerateMissingHugePagesCgroupController)
7677
if err != nil {
7778
return errors.Wrap(err, "failed to update resource in spec")
7879
}
@@ -132,7 +133,8 @@ func updateContainerSpec(ctx context.Context, cntr containerd.Container, spec *r
132133
}
133134

134135
// updateOCILinuxResource updates container resource limit.
135-
func updateOCILinuxResource(ctx context.Context, spec *runtimespec.Spec, new *runtime.LinuxContainerResources) (*runtimespec.Spec, error) {
136+
func updateOCILinuxResource(ctx context.Context, spec *runtimespec.Spec, new *runtime.LinuxContainerResources,
137+
tolerateMissingHugePagesCgroupController bool) (*runtimespec.Spec, error) {
136138
// Copy to make sure old spec is not changed.
137139
var cloned runtimespec.Spec
138140
if err := util.DeepCopy(&cloned, spec); err != nil {
@@ -141,7 +143,7 @@ func updateOCILinuxResource(ctx context.Context, spec *runtimespec.Spec, new *ru
141143
if cloned.Linux == nil {
142144
cloned.Linux = &runtimespec.Linux{}
143145
}
144-
if err := opts.WithResources(new)(ctx, nil, nil, &cloned); err != nil {
146+
if err := opts.WithResources(new, tolerateMissingHugePagesCgroupController)(ctx, nil, nil, &cloned); err != nil {
145147
return nil, errors.Wrap(err, "unable to set linux container resources")
146148
}
147149
return &cloned, nil

‎pkg/server/container_update_resources_unix_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -153,7 +153,7 @@ func TestUpdateOCILinuxResource(t *testing.T) {
153153
},
154154
} {
155155
t.Logf("TestCase %q", desc)
156-
got, err := updateOCILinuxResource(context.Background(), test.spec, test.resources)
156+
got, err := updateOCILinuxResource(context.Background(), test.spec, test.resources, false)
157157
if test.expectErr {
158158
assert.Error(t, err)
159159
} else {

0 commit comments

Comments
 (0)