Skip to content

1.6.x: SElinux Relabelling broken for Kubernetes volume mounts #6767

Closed
Closed
1.6.x: SElinux Relabelling broken for Kubernetes volume mounts#6767
Labels
kind/bug
@log1cb0mb

Description

@log1cb0mb
log1cb0mb
opened on Apr 3, 2022

Description

The issue was originally reported here: coreos/fedora-coreos-tracker#1138 suspecting issue between FCOS v35.20220213.3.0 and 35.20220227.3.0 where containerd version upgraded from 1.5.8 to 1.6.0 in the latter. Most of the investigation details available on linked issue but in short:

containerd 1.6.0/1.6.2 not relabelling volume mounts with context labels passed by kubelet for containers. Example container coredns with configmap volume mount, system running containerd 1.5.8:

[root@dc1-worker1 kubelet]# ls -alZ pods/a1feb462-dfdd-40e2-942a-f92f3ac99917/volumes/kubernetes.io~configmap/config/
total 0
drwxrwxrwx. 3 root root system_u:object_r:container_file_t:s0:c584,c689 76 Apr  3 10:47 .
drwxr-xr-x. 3 root root system_u:object_r:var_lib_t:s0                  20 Apr  3 10:47 ..
drwxr-xr-x. 2 root root system_u:object_r:container_file_t:s0:c584,c689 22 Apr  3 10:47 ..2022_04_03_10_47_20.3644846533
lrwxrwxrwx. 1 root root system_u:object_r:container_file_t:s0:c584,c689 32 Apr  3 10:47 ..data -> ..2022_04_03_10_47_20.3644846533
lrwxrwxrwx. 1 root root system_u:object_r:container_file_t:s0:c584,c689 15 Apr  3 10:47 Corefile -> ..data/Corefile

Corefile is correctly labelled based on container's context label info provided by kubelet. However with 1.6.x:

[root@dc2-worker1 kubelet]# ls -alZ pods/4e118a5b-c54b-42fb-853b-088a6fd7e0dd/volumes/kubernetes.io~configmap/config/
total 0
drwxrwxrwx. 3 root root system_u:object_r:container_file_t:s0:c134,c755 76 Apr  3 10:47 .
drwxr-xr-x. 3 root root system_u:object_r:var_lib_t:s0                  20 Apr  3 10:47 ..
drwxr-xr-x. 2 root root system_u:object_r:container_file_t:s0:c134,c755 22 Apr  3 10:47 ..2022_04_03_10_47_20.1642956151
lrwxrwxrwx. 1 root root system_u:object_r:var_lib_t:s0                  32 Apr  3 10:47 ..data -> ..2022_04_03_10_47_20.1642956151
lrwxrwxrwx. 1 root root system_u:object_r:var_lib_t:s0                  15 Apr  3 10:47 Corefile -> ..data/Corefile

# audit log:
Apr 03 10:50:20 dc2-worker1.k8s.lab audit[16097]: AVC avc:  denied  { read } for  pid=16097 comm="coredns" name="Corefile" dev="vda4" ino=67109200 scontext=system_u:system_r:container_t:s0:c134,c755 tcontext=system_u:object_r:var_lib_t:s0 tclass=lnk_file permissive=0

Corefile is not relabelled at all which results in access deny as per audit log above.

Based on actual container inspection, selinux_relabel is actually set:

      "mounts": [
        {
          "container_path": "/etc/coredns",
          "host_path": "/var/lib/kubelet/pods/4e118a5b-c54b-42fb-853b-088a6fd7e0dd/volumes/kubernetes.io~configmap/config",
          "readonly": true,
          "selinux_relabel": true
        },

Also mountLabel is being passed by kubelet correctly as well:
"mountLabel": "system_u:object_r:container_file_t:s0:c134,c755"

Steps to reproduce the issue

  1. Kubernetes cluster with containerd runtime version 1.6.x
  2. Spin up a pod for example: coredns
  3. Pod failing with logs showing loading Caddyfile via flag: open /etc/coredns/Corefile: permission denied.
  4. The scenario assumes that there are no relabelling done by any other process or by administrator

Describe the results you received and expected

  • Kubernetes volume mounts for containers to be labelled correctly with respective (generated by kubelet or manually specified by administrator under spec.securityContext of a container) SElinux context labels. However SElinux context labels being ignored.

What version of containerd are you using?

containerd github.com/containerd/containerd v1.6.2 de8046a

Any other relevant information

# runc --version
runc version 1.1.0
commit: 9ac869a-dirty
spec: 1.0.2-dev
go: go1.16.13
libseccomp: 2.5.3

# crictl info
{
  "status": {
    "conditions": [
      {
        "type": "RuntimeReady",
        "status": true,
        "reason": "",
        "message": ""
      },
      {
        "type": "NetworkReady",
        "status": true,
        "reason": "",
        "message": ""
      }
    ]
  },
  "cniconfig": {
    "PluginDirs": [
      "/opt/cni/bin"
    ],
    "PluginConfDir": "/etc/cni/net.d",
    "PluginMaxConfNum": 1,
    "Prefix": "eth",
    "Networks": [
      {
        "Config": {
          "Name": "cni-loopback",
          "CNIVersion": "0.3.1",
          "Plugins": [
            {
              "Network": {
                "type": "loopback",
                "ipam": {},
                "dns": {}
              },
              "Source": "{\"type\":\"loopback\"}"
            }
          ],
          "Source": "{\n\"cniVersion\": \"0.3.1\",\n\"name\": \"cni-loopback\",\n\"plugins\": [{\n  \"type\": \"loopback\"\n}]\n}"
        },
        "IFName": "lo"
      },
      {
        "Config": {
          "Name": "k8s-pod-network",
          "CNIVersion": "0.3.1",
          "Plugins": [
            {
              "Network": {
                "type": "calico",
                "ipam": {
                  "type": "calico-ipam"
                },
                "dns": {}
              },
              "Source": "{\"datastore_type\":\"kubernetes\",\"ipam\":{\"type\":\"calico-ipam\"},\"kubernetes\":{\"kubeconfig\":\"/etc/cni/net.d/calico-kubeconfig\"},\"log_file_path\":\"/var/log/calico/cni/cni.log\",\"log_level\":\"info\",\"mtu\":0,\"nodename\":\"dc1-worker1.k8s.lab\",\"policy\":{\"type\":\"k8s\"},\"type\":\"calico\"}"
            },
            {
              "Network": {
                "type": "portmap",
                "capabilities": {
                  "portMappings": true
                },
                "ipam": {},
                "dns": {}
              },
              "Source": "{\"capabilities\":{\"portMappings\":true},\"snat\":true,\"type\":\"portmap\"}"
            },
            {
              "Network": {
                "type": "bandwidth",
                "capabilities": {
                  "bandwidth": true
                },
                "ipam": {},
                "dns": {}
              },
              "Source": "{\"capabilities\":{\"bandwidth\":true},\"type\":\"bandwidth\"}"
            }
          ],
          "Source": "{\n  \"name\": \"k8s-pod-network\",\n  \"cniVersion\": \"0.3.1\",\n  \"plugins\": [\n    {\n      \"type\": \"calico\",\n      \"log_level\": \"info\",\n      \"log_file_path\": \"/var/log/calico/cni/cni.log\",\n      \"datastore_type\": \"kubernetes\",\n      \"nodename\": \"dc1-worker1.k8s.lab\",\n      \"mtu\": 0,\n      \"ipam\": {\n        \"type\": \"calico-ipam\"\n      },\n      \"policy\": {\n        \"type\": \"k8s\"\n      },\n      \"kubernetes\": {\n        \"kubeconfig\": \"/etc/cni/net.d/calico-kubeconfig\"\n      }\n    },\n    {\n      \"type\": \"portmap\",\n      \"snat\": true,\n      \"capabilities\": {\"portMappings\": true}\n    },\n    {\n      \"type\": \"bandwidth\",\n      \"capabilities\": {\"bandwidth\": true}\n    }\n  ]\n}"
        },
        "IFName": "eth0"
      }
    ]
  },
  "config": {
    "containerd": {
      "snapshotter": "overlayfs",
      "defaultRuntimeName": "runc",
      "defaultRuntime": {
        "runtimeType": "",
        "runtimePath": "",
        "runtimeEngine": "",
        "PodAnnotations": null,
        "ContainerAnnotations": null,
        "runtimeRoot": "",
        "options": null,
        "privileged_without_host_devices": false,
        "baseRuntimeSpec": "",
        "cniConfDir": "",
        "cniMaxConfNum": 0
      },
      "untrustedWorkloadRuntime": {
        "runtimeType": "",
        "runtimePath": "",
        "runtimeEngine": "",
        "PodAnnotations": null,
        "ContainerAnnotations": null,
        "runtimeRoot": "",
        "options": null,
        "privileged_without_host_devices": false,
        "baseRuntimeSpec": "",
        "cniConfDir": "",
        "cniMaxConfNum": 0
      },
      "runtimes": {
        "runc": {
          "runtimeType": "io.containerd.runc.v2",
          "runtimePath": "",
          "runtimeEngine": "",
          "PodAnnotations": null,
          "ContainerAnnotations": null,
          "runtimeRoot": "",
          "options": {
            "SystemdCgroup": true
          },
          "privileged_without_host_devices": false,
          "baseRuntimeSpec": "",
          "cniConfDir": "",
          "cniMaxConfNum": 0
        }
      },
      "noPivot": false,
      "disableSnapshotAnnotations": true,
      "discardUnpackedLayers": false,
      "ignoreRdtNotEnabledErrors": false
    },
    "cni": {
      "binDir": "/opt/cni/bin",
      "confDir": "/etc/cni/net.d",
      "maxConfNum": 1,
      "confTemplate": "",
      "ipPref": ""
    },
    "registry": {
      "configPath": "",
      "mirrors": null,
      "configs": null,
      "auths": null,
      "headers": null
    },
    "imageDecryption": {
      "keyModel": "node"
    },
    "disableTCPService": true,
    "streamServerAddress": "127.0.0.1",
    "streamServerPort": "0",
    "streamIdleTimeout": "4h0m0s",
    "enableSelinux": true,
    "selinuxCategoryRange": 1024,
    "sandboxImage": "k8s.gcr.io/pause:3.6",
    "statsCollectPeriod": 10,
    "systemdCgroup": false,
    "enableTLSStreaming": false,
    "x509KeyPairStreaming": {
      "tlsCertFile": "",
      "tlsKeyFile": ""
    },
    "maxContainerLogSize": 16384,
    "disableCgroup": false,
    "disableApparmor": false,
    "restrictOOMScoreAdj": false,
    "maxConcurrentDownloads": 3,
    "disableProcMount": false,
    "unsetSeccompProfile": "",
    "tolerateMissingHugetlbController": true,
    "disableHugetlbController": true,
    "device_ownership_from_security_context": false,
    "ignoreImageDefinedVolumes": false,
    "netnsMountsUnderStateDir": false,
    "enableUnprivilegedPorts": false,
    "enableUnprivilegedICMP": false,
    "containerdRootDir": "/var/lib/containerd",
    "containerdEndpoint": "/run/containerd/containerd.sock",
    "rootDir": "/var/lib/containerd/io.containerd.grpc.v1.cri",
    "stateDir": "/run/containerd/io.containerd.grpc.v1.cri"
  },
  "golang": "go1.17.2",
  "lastCNILoadStatus": "OK",
  "lastCNILoadStatus.default": "OK"
}

# uname -a
Linux dc1-worker1.k8s.lab 5.16.13-200.fc35.x86_64 #1 SMP PREEMPT Tue Mar 8 22:50:58 UTC 2022 x86_64 x86_64 x86_64 GNU/Linux

Show configuration if it is related to CRI plugin.

# cat /etc/containerd/config.toml
version = 2
root = "/var/lib/containerd"
state = "/run/containerd"
subreaper = true
oom_score = -999
[grpc]
address = "/run/containerd/containerd.sock"
uid = 0
gid = 0
[plugins."io.containerd.grpc.v1.cri"]
enable_selinux = true
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
runtime_type = "io.containerd.runc.v2"
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
SystemdCgroup = true

Metadata

Metadata

Assignees

No one assigned

    Labels

    kind/bug

    Type

    No type

    Projects

    No projects

    Milestone

    No milestone

    Relationships

    None yet

    Development

    No branches or pull requests

    Issue actions