Skip to content

ctr: Can't use CNI on Windows: panic: invalid GUID length #6305

Closed
#6304
Closed
ctr: Can't use CNI on Windows: panic: invalid GUID length#6305
#6304
Labels
kind/bug
@lippertmarkus

Description

@lippertmarkus
lippertmarkus
opened on Dec 1, 2021

Description

Windows Containers started with ctr doesn't have networking by default:

./ctr.exe i pull mcr.microsoft.com/windows/nanoserver:1809
./ctr.exe run -rm  mcr.microsoft.com/windows/nanoserver:1809 test cmd /c ipconfig

# Windows IP Configuration

With crictl however they do:

.\crictl.exe pull mcr.microsoft.com/windows/nanoserver:1809
$POD_ID=(./crictl runp .\pod.json)
$CONTAINER_ID=(./crictl create $POD_ID .\container.json .\pod.json)
./crictl start $CONTAINER_ID
.\crictl.exe exec $CONTAINER_ID cmd /c ipconfig

# Windows IP Configuration
#
# Ethernet adapter vEthernet (3cceb90e9342cc0a23db648c9c4da002a329d285ba3733e8feb666a374e9f842_nat):
#
#   Connection-specific DNS Suffix  . : 
#   Link-local IPv6 Address . . . . . : fe80::1d7f:3f4c:efb5:457b%15
#   IPv4 Address. . . . . . . . . . . : 172.30.197.170
#   Subnet Mask . . . . . . . . . . . : 255.255.240.0

We found there is a --cni flag for Linux

containerd/cmd/ctr/commands/run/run_unix.go

Lines 82 to 85 in af1a090

cli.BoolFlag{
Name: "cni",
Usage: "enable cni networking for the container",
},

but not for Windows

containerd/cmd/ctr/commands/run/run_windows.go

Lines 33 to 38 in af1a090

var platformRunFlags = []cli.Flag{
cli.BoolFlag{
Name: "isolated",
Usage: "run the container with vm isolation",
},
}

I added the flag for Windows as well in #6304 but now while it seems to try the right thing it fails seemingly because of a panic with a wrong GUID:

PS C:\Program Files\containerd> .\myctr.exe run --rm --cni mcr.microsoft.com/windows/nanoserver:1809 test7 cmd /c ipconfig
ctr: plugin type="nat" name="nat" failed (add): netplugin failed: "{\"level\":\"debug\",\"msg\":\"[cni-net] Plugin wcn-net version .\",\"time\":\"2021-12-01T09:45:51Z\"}\n{\"level\":\"debug\",\"msg\":\"[net] Ne
twork interface: {Index:4 MTU:1500 Name:Ethernet HardwareAddr:00:15:5d:de:e3:37 Flags:up|broadcast|multicast} with IP addresses: [fe80::7521:b0fb:3b27:e219/64 10.1.0.149/8]\",\"time\":\"2021-12-01T09:45:51Z\"}\
n{\"level\":\"debug\",\"msg\":\"[net] Network interface: {Index:1 MTU:-1 Name:Loopback Pseudo-Interface 1 HardwareAddr: Flags:up|loopback|multicast} with IP addresses: [::1/128 127.0.0.1/8]\",\"time\":\"2021-12
-01T09:45:51Z\"}\n{\"level\":\"debug\",\"msg\":\"[net] Network interface: {Index:12 MTU:1500 Name:vEthernet (nat) HardwareAddr:00:15:5d:6e:5a:c1 Flags:up|broadcast|multicast} with IP addresses: [fe80::4d9f:83a4
:1a73:30c0/64 172.27.16.1/20]\",\"time\":\"2021-12-01T09:45:51Z\"}\n{\"level\":\"debug\",\"msg\":\"[cni-net] Plugin started.\",\"time\":\"2021-12-01T09:45:51Z\"}\n{\"level\":\"debug\",\"msg\":\"[cni-net] Proces
sing ADD command with args {ContainerID:default-test7 Netns:/proc/2604/ns/net IfName:eth0 Args: Path:/opt/cni/bin}.\",\"time\":\"2021-12-01T09:45:51Z\"}\n{\"level\":\"debug\",\"msg\":\"[cni-net] Read network co
nfiguration \\u0026{CniVersion:0.2.0 Name:nat Type:nat Ipam:{Type: Environment: AddrSpace: Subnet:172.27.16.0/20 Address: QueryInterval: Routes:[{Dst:{IP:\\u003cnil\\u003e Mask:\\u003cnil\\u003e} GW:\\u003cnil\
\u003e}]} DNS:{Nameservers:[] Domain: Search:[] Options:[]} OptionalFlags:{LocalRoutePortMapping:false AllowAclPortMapping:false} RuntimeConfig:{PortMappings:[] DNS:{Servers:[] Searches:[] Options:[]}} Addition
alArgs:[]}.\",\"time\":\"2021-12-01T09:45:51Z\"}\n{\"level\":\"debug\",\"msg\":\"Parsing port mappings from []\",\"time\":\"2021-12-01T09:45:51Z\"}\n{\"level\":\"debug\",\"msg\":\"[cni-net] Found network 7d46e0
f9-b777-42bc-9c62-5486ab3c0c44 with subnet [{{172.27.16.0 fffff000} 172.27.16.1 []}].\",\"time\":\"2021-12-01T09:45:51Z\"}\n{\"level\":\"debug\",\"msg\":\"[cni-net] Creating a new Endpoint\",\"time\":\"2021-12-
01T09:45:51Z\"}\n{\"level\":\"debug\",\"msg\":\"hcn::HostComputeEndpoint::Create id=\",\"time\":\"2021-12-01T09:45:51Z\"}\n{\"level\":\"debug\",\"msg\":\"hcn::HostComputeEndpoint::Create JSON: {\\\"Name\\\":\\\
"default-test7_nat\\\",\\\"HostComputeNetwork\\\":\\\"7d46e0f9-b777-42bc-9c62-5486ab3c0c44\\\",\\\"IpConfigurations\\\":[{}],\\\"Dns\\\":{},\\\"Routes\\\":[{\\\"NextHop\\\":\\\"172.27.16.1\\\",\\\"DestinationPr
efix\\\":\\\"0.0.0.0/0\\\"}],\\\"SchemaVersion\\\":{\\\"Major\\\":2,\\\"Minor\\\":0}}\",\"time\":\"2021-12-01T09:45:51Z\"}\n{\"level\":\"debug\",\"msg\":\"hcn::HostComputeEndpoint::AddNamespaceEndpoint id=f2960
3ad-4ae0-49a8-8491-c9b23853fa9a\",\"time\":\"2021-12-01T09:45:51Z\"}\n{\"level\":\"debug\",\"msg\":\"hcn::HostComputeNamespace::ModifyNamespaceSettings id=/proc/2604/ns/net\",\"time\":\"2021-12-01T09:45:51Z\"}\
npanic: invalid GUID length: 17\n\ngoroutine 1 [running]:\ngithub.com/Microsoft/windows-container-networking/vendor/github.com/Microsoft/hcsshim/internal/guid.FromString(0xc0000a1660, 0x11, 0x0, 0x0)\n\t/home/n
agiesek/repo/gopath/src/github.com/Microsoft/windows-container-networking/vendor/github.com/Microsoft/hcsshim/internal/guid/guid.go:32 +0x297\ngithub.com/Microsoft/windows-container-networking/vendor/github.com
/Microsoft/hcsshim/hcn.modifyNamespace(0xc0000a1660, 0x11, 0xc00009ed20, 0x70, 0xc00009ed20, 0x70, 0x0)\n\t/home/nagiesek/repo/gopath/src/github.com/Microsoft/windows-container-networking/vendor/github.com/Micr
osoft/hcsshim/hcn/hcnnamespace.go:168 +0x54\ngithub.com/Microsoft/windows-container-networking/vendor/github.com/Microsoft/hcsshim/hcn.ModifyNamespaceSettings(0xc0000a1660, 0x11, 0xc000032840, 0x35, 0x40)\n\t/h
ome/nagiesek/repo/gopath/src/github.com/Microsoft/windows-container-networking/vendor/github.com/Microsoft/hcsshim/hcn/hcnnamespace.go:383 +0x137\ngithub.com/Microsoft/windows-container-networking/vendor/github
.com/Microsoft/hcsshim/hcn.AddNamespaceEndpoint(0xc0000a1660, 0x11, 0xc0000a3590, 0x24, 0xc00007c000, 0x30)\n\t/home/nagiesek/repo/gopath/src/github.com/Microsoft/windows-container-networking/vendor/github.com/
Microsoft/hcsshim/hcn/hcnnamespace.go:405 +0x1f9\ngithub.com/Microsoft/windows-container-networking/network.(*networkManager).CreateEndpoint(0xc00007a060, 0xc0000a3410, 0x24, 0xc0000d2000, 0xc0000a1660, 0x11, 0
x0, 0x0, 0x0)\n\t/home/nagiesek/repo/gopath/src/github.com/Microsoft/windows-container-networking/network/manager.go:131 +0x23b\ngithub.com/Microsoft/windows-container-networking/common/core.(*netPlugin).Add(0x
c000004680, 0xc00007e310, 0x0, 0x0)\n\t/home/nagiesek/repo/gopath/src/github.com/Microsoft/windows-container-networking/common/core/network.go:172 +0x83c\ngithub.com/Microsoft/windows-container-networking/vendo
r/github.com/containernetworking/cni/pkg/skel.(*dispatcher).checkVersionAndCall(0xc000063e98, 0xc00007e310, 0x5cc600, 0xc00007a690, 0xc000063e80, 0x0, 0x28)\n\t/home/nagiesek/repo/gopath/src/github.com/Microsof
t/windows-container-networking/vendor/github.com/containernetworking/cni/pkg/skel/skel.go:185 +0x260\ngithub.com/Microsoft/windows-container-networking/vendor/github.com/containernetworking/cni/pkg/skel.(*dispa
tcher).pluginMain(0xc000063e98, 0xc000063e80, 0x0, 0xc000063e68, 0x5cc600, 0xc00007a690, 0x59d28d, 0x11, 0x53df1a)\n\t/home/nagiesek/repo/gopath/src/github.com/Microsoft/windows-container-networking/vendor/gith
ub.com/containernetworking/cni/pkg/skel/skel.go:221 +0x550\ngithub.com/Microsoft/windows-container-networking/vendor/github.com/containernetworking/cni/pkg/skel.PluginMainWithError(...)\n\t/home/nagiesek/repo/g
opath/src/github.com/Microsoft/windows-container-networking/vendor/github.com/containernetworking/cni/pkg/skel/skel.go:286\ngithub.com/Microsoft/windows-container-networking/cni.(*Plugin).Execute(0xc000006038,
0x5cc580, 0xc000004680, 0x0, 0xc000006030)\n\t/home/nagiesek/repo/gopath/src/github.com/Microsoft/windows-container-networking/cni/plugin.go:49 +0x225\ngithub.com/Microsoft/windows-container-networking/common/c
ore.Core()\n\t/home/nagiesek/repo/gopath/src/github.com/Microsoft/windows-container-networking/common/core/core.go:47 +0x29c\nmain.main()\n\t/home/nagiesek/repo/gopath/src/github.com/Microsoft/windows-container
-networking/plugins/nat/nat_windows.go:16 +0x27\n"

Steps to reproduce the issue

  1. Clone & build ctr from fix(ctr): enable networking for Windows containers #6304 which adds --cni flag to Windows run command
  2. ./ctr.exe i pull mcr.microsoft.com/windows/nanoserver:1809
  3. ./ctr.exe run --rm --cni mcr.microsoft.com/windows/nanoserver:1809 test cmd /c ipconfig

Describe the results you received and expected

Expected: Windows Container starts with networking
Received: Windows Container fails with a CNI error

What version of containerd are you using?

containerd github.com/containerd/containerd v1.6.0-beta.3 1a88cf5.m

Any other relevant information

.\crictl.exe info
{
  "status": {
    "conditions": [
      {
        "type": "RuntimeReady",
        "status": true,
        "reason": "",
        "message": ""
      },
      {
        "type": "NetworkReady",
        "status": true,
        "reason": "",
        "message": ""
      }
    ]
  },
  "cniconfig": {
    "PluginDirs": [
      "C:\\opt\\cni\\bin"
    ],
    "PluginConfDir": "C:\\etc\\cni\\net.d",
    "PluginMaxConfNum": 1,
    "Prefix": "eth",
    "Networks": [
      {
        "Config": {
          "Name": "nat",
          "CNIVersion": "0.2.0",
          "Plugins": [
            {
              "Network": {
                "cniVersion": "0.2.0",
                "name": "nat",
                "type": "nat",
                "capabilities": {
                  "dns": true,
                  "portMappings": true
                },
                "ipam": {},
                "dns": {}
              },
              "Source": "{\"capabilities\":{\"dns\":true,\"portMappings\":true},\"cniVersion\":\"0.2.0\",\"ipam\":{\"routes\":[{\"gateway\":\"172.27.16.1\"}],\"subnet\":\"172.27.16.0/20\"},\"master\":\"Ethernet\",\"name\":\"nat\",\"type\":\"nat\"}"
            }
          ],
          "Source": "{\"cniVersion\":\"0.2.0\",\"name\":\"nat\",\"plugins\":[{\"capabilities\":{\"dns\":true,\"portMappings\":true},\"cniVersion\":\"0.2.0\",\"ipam\":{\"routes\":[{\"gateway\":\"172.27.16.1\"}],\"subnet\":\"172.27.16.0/20\"},\"master\":\"Ethernet\",\"name\":\"nat\",\"type\":\"nat\"}]}"
        },
        "IFName": "eth0"
      }
    ]
  },
  "config": {
    "containerd": {
      "snapshotter": "windows",
      "defaultRuntimeName": "runhcs-wcow-process",
      "defaultRuntime": {
        "runtimeType": "",
        "runtimePath": "",
        "runtimeEngine": "",
        "PodAnnotations": [],
        "ContainerAnnotations": [],
        "runtimeRoot": "",
        "options": {},
        "privileged_without_host_devices": false,
        "baseRuntimeSpec": "",
        "cniConfDir": "",
        "cniMaxConfNum": 0
      },
      "untrustedWorkloadRuntime": {
        "runtimeType": "",
        "runtimePath": "",
        "runtimeEngine": "",
        "PodAnnotations": [],
        "ContainerAnnotations": [],
        "runtimeRoot": "",
        "options": {},
        "privileged_without_host_devices": false,
        "baseRuntimeSpec": "",
        "cniConfDir": "",
        "cniMaxConfNum": 0
      },
      "runtimes": {
        "runhcs-wcow-process": {
          "runtimeType": "io.containerd.runhcs.v1",
          "runtimePath": "",
          "runtimeEngine": "",
          "PodAnnotations": [],
          "ContainerAnnotations": [],
          "runtimeRoot": "",
          "options": {},
          "privileged_without_host_devices": false,
          "baseRuntimeSpec": "",
          "cniConfDir": "",
          "cniMaxConfNum": 0
        }
      },
      "noPivot": false,
      "disableSnapshotAnnotations": false,
      "discardUnpackedLayers": false
    },
    "cni": {
      "binDir": "C:\\opt\\cni\\bin",
      "confDir": "C:\\etc\\cni\\net.d",
      "maxConfNum": 1,
      "confTemplate": "",
      "ipPref": ""
    },
    "registry": {
      "configPath": "",
      "mirrors": {},
      "configs": {},
      "auths": {},
      "headers": {}
    },
    "imageDecryption": {
      "keyModel": "node"
    },
    "disableTCPService": true,
    "streamServerAddress": "127.0.0.1",
    "streamServerPort": "0",
    "streamIdleTimeout": "4h0m0s",
    "enableSelinux": false,
    "selinuxCategoryRange": 0,
    "sandboxImage": "k8s.gcr.io/pause:3.6",
    "statsCollectPeriod": 10,
    "systemdCgroup": false,
    "enableTLSStreaming": false,
    "x509KeyPairStreaming": {
      "tlsCertFile": "",
      "tlsKeyFile": ""
    },
    "maxContainerLogSize": 16384,
    "disableCgroup": false,
    "disableApparmor": false,
    "restrictOOMScoreAdj": false,
    "maxConcurrentDownloads": 3,
    "disableProcMount": false,
    "unsetSeccompProfile": "",
    "tolerateMissingHugetlbController": false,
    "disableHugetlbController": false,
    "device_ownership_from_security_context": false,
    "ignoreImageDefinedVolumes": false,
    "netnsMountsUnderStateDir": false,
    "enableUnprivilegedPorts": false,
    "enableUnprivilegedICMP": false,
    "containerdRootDir": "C:\\ProgramData\\containerd\\root",
    "containerdEndpoint": "\\\\.\\pipe\\containerd-containerd",
    "rootDir": "C:\\ProgramData\\containerd\\root\\io.containerd.grpc.v1.cri",
    "stateDir": "C:\\ProgramData\\containerd\\state\\io.containerd.grpc.v1.cri"
  },
  "golang": "go1.17.2",
  "lastCNILoadStatus": "OK",
  "lastCNILoadStatus.default": "OK"
}

Show configuration if it is related to CRI plugin.

config.toml

disabled_plugins = []
imports = []
oom_score = 0
plugin_dir = ""
required_plugins = []
root = "C:\\ProgramData\\containerd\\root"
state = "C:\\ProgramData\\containerd\\state"
temp = ""
version = 2

[cgroup]
  path = ""

[debug]
  address = ""
  format = ""
  gid = 0
  level = ""
  uid = 0

[grpc]
  address = "\\\\.\\pipe\\containerd-containerd"
  gid = 0
  max_recv_message_size = 16777216
  max_send_message_size = 16777216
  tcp_address = ""
  tcp_tls_ca = ""
  tcp_tls_cert = ""
  tcp_tls_key = ""
  uid = 0

[metrics]
  address = ""
  grpc_histogram = false

[plugins]

  [plugins."io.containerd.gc.v1.scheduler"]
    deletion_threshold = 0
    mutation_threshold = 100
    pause_threshold = 0.02
    schedule_delay = "0s"
    startup_delay = "100ms"

  [plugins."io.containerd.grpc.v1.cri"]
    device_ownership_from_security_context = false
    disable_apparmor = false
    disable_cgroup = false
    disable_hugetlb_controller = false
    disable_proc_mount = false
    disable_tcp_service = true
    enable_selinux = false
    enable_tls_streaming = false
    enable_unprivileged_icmp = false
    enable_unprivileged_ports = false
    ignore_image_defined_volumes = false
    max_concurrent_downloads = 3
    max_container_log_line_size = 16384
    netns_mounts_under_state_dir = false
    restrict_oom_score_adj = false
    sandbox_image = "k8s.gcr.io/pause:3.6"
    selinux_category_range = 0
    stats_collect_period = 10
    stream_idle_timeout = "4h0m0s"
    stream_server_address = "127.0.0.1"
    stream_server_port = "0"
    systemd_cgroup = false
    tolerate_missing_hugetlb_controller = false
    unset_seccomp_profile = ""

    [plugins."io.containerd.grpc.v1.cri".cni]
      bin_dir = "C:\\opt\\cni\\bin"
      conf_dir = "C:\\etc\\cni\\net.d"
      conf_template = ""
      ip_pref = ""
      max_conf_num = 1

    [plugins."io.containerd.grpc.v1.cri".containerd]
      default_runtime_name = "runhcs-wcow-process"
      disable_snapshot_annotations = false
      discard_unpacked_layers = false
      no_pivot = false
      snapshotter = "windows"

      [plugins."io.containerd.grpc.v1.cri".containerd.default_runtime]
        base_runtime_spec = ""
        cni_conf_dir = ""
        cni_max_conf_num = 0
        container_annotations = []
        pod_annotations = []
        privileged_without_host_devices = false
        runtime_engine = ""
        runtime_path = ""
        runtime_root = ""
        runtime_type = ""

        [plugins."io.containerd.grpc.v1.cri".containerd.default_runtime.options]

      [plugins."io.containerd.grpc.v1.cri".containerd.runtimes]

        [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runhcs-wcow-process]
          base_runtime_spec = ""
          cni_conf_dir = ""
          cni_max_conf_num = 0
          container_annotations = []
          pod_annotations = []
          privileged_without_host_devices = false
          runtime_engine = ""
          runtime_path = ""
          runtime_root = ""
          runtime_type = "io.containerd.runhcs.v1"

          [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runhcs-wcow-process.options]

      [plugins."io.containerd.grpc.v1.cri".containerd.untrusted_workload_runtime]
        base_runtime_spec = ""
        cni_conf_dir = ""
        cni_max_conf_num = 0
        container_annotations = []
        pod_annotations = []
        privileged_without_host_devices = false
        runtime_engine = ""
        runtime_path = ""
        runtime_root = ""
        runtime_type = ""

        [plugins."io.containerd.grpc.v1.cri".containerd.untrusted_workload_runtime.options]

    [plugins."io.containerd.grpc.v1.cri".image_decryption]
      key_model = "node"

    [plugins."io.containerd.grpc.v1.cri".registry]
      config_path = ""

      [plugins."io.containerd.grpc.v1.cri".registry.auths]

      [plugins."io.containerd.grpc.v1.cri".registry.configs]

      [plugins."io.containerd.grpc.v1.cri".registry.headers]

      [plugins."io.containerd.grpc.v1.cri".registry.mirrors]

    [plugins."io.containerd.grpc.v1.cri".x509_key_pair_streaming]
      tls_cert_file = ""
      tls_key_file = ""

  [plugins."io.containerd.internal.v1.opt"]
    path = "C:\\ProgramData\\containerd\\root\\opt"

  [plugins."io.containerd.internal.v1.restart"]
    interval = "10s"

  [plugins."io.containerd.internal.v1.tracing"]
    sampling_ratio = 0.0
    service_name = "containerd"

  [plugins."io.containerd.metadata.v1.bolt"]
    content_sharing_policy = "shared"

  [plugins."io.containerd.runtime-shim.v2.shim"]
    platforms = ["windows/amd64", "linux/amd64"]
    sched_core = false

  [plugins."io.containerd.service.v1.diff-service"]
    default = ["windows", "windows-lcow"]

  [plugins."io.containerd.tracing.processor.v1.otlp"]
    endpoint = ""
    insecure = false

[proxy_plugins]

[stream_processors]

  [stream_processors."io.containerd.ocicrypt.decoder.v1.tar"]
    accepts = ["application/vnd.oci.image.layer.v1.tar+encrypted"]
    args = ["--decryption-keys-path", "C:\\Program Files\\containerd\\ocicrypt\\keys"]
    env = ["OCICRYPT_KEYPROVIDER_CONFIG=C:\\Program Files\\containerd\\ocicrypt\\ocicrypt_keyprovider.conf"]
    path = "ctd-decoder"
    returns = "application/vnd.oci.image.layer.v1.tar"

  [stream_processors."io.containerd.ocicrypt.decoder.v1.tar.gzip"]
    accepts = ["application/vnd.oci.image.layer.v1.tar+gzip+encrypted"]
    args = ["--decryption-keys-path", "C:\\Program Files\\containerd\\ocicrypt\\keys"]
    env = ["OCICRYPT_KEYPROVIDER_CONFIG=C:\\Program Files\\containerd\\ocicrypt\\ocicrypt_keyprovider.conf"]
    path = "ctd-decoder"
    returns = "application/vnd.oci.image.layer.v1.tar+gzip"

[timeouts]
  "io.containerd.timeout.shim.cleanup" = "5s"
  "io.containerd.timeout.shim.load" = "5s"
  "io.containerd.timeout.shim.shutdown" = "3s"
  "io.containerd.timeout.task.state" = "2s"

[ttrpc]
  address = ""
  gid = 0
  uid = 0

Metadata

Metadata

Assignees

No one assigned

    Labels

    kind/bug

    Type

    No type

    Projects

    No projects

    Milestone

    No milestone

    Relationships

    None yet

    Development

    No branches or pull requests

    Issue actions