integration: Windows volume-copy-up images

claudiubelu · claudiubelu · commit 16deba098c90 · 2021-06-30T11:19:54.000Z
For Windows, the container image's OS version must closely match the host's OS version.
For this reason, we need to add the --os-version annotation in image manifest lists,
so the Windows nodes can pull the appropriate image from the list.

Previously, the docker manifest CLI did not have the capability to set the --os-version,
it, but it has been introduced in docker 20.10.0.

We're also adding busybox.exe in the image, so we can run Linux commands inside the
container, so the tests will be simpler.

When building Windows images, a docker buildx builder needs to be created and used. When
building Windows images with docker buildx, the flag --output=type=registry is required,
otherwise it cannot be referenced on a Linux node.

Signed-off-by: Claudiu Belu &lt;cbelu@cloudbasesolutions.com&gt;
diff --git a/integration/images/volume-copy-up/Dockerfile b/integration/images/volume-copy-up/Dockerfile
@@ -12,6 +12,8 @@
 #   See the License for the specific language governing permissions and
 #   limitations under the License.
 
-FROM busybox
+ARG BASE
+
+FROM $BASE
 RUN sh -c "mkdir /test_dir; echo test_content > /test_dir/test_file"
 VOLUME "/test_dir"
diff --git a/integration/images/volume-copy-up/Dockerfile_windows b/integration/images/volume-copy-up/Dockerfile_windows
@@ -0,0 +1,42 @@
+#   Copyright The containerd Authors.
+
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+
+#       http://www.apache.org/licenses/LICENSE-2.0
+
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+
+ARG BASE
+
+FROM --platform=linux/amd64 busybox as prep
+
+# Similar to: https://github.com/kubernetes/kubernetes/blob/7ad7c0757ac7fa37dfae9c7cdc628cf04e35e5cb/test/images/busybox/Dockerfile_windows
+
+# Available busybox functions retrieved by running busybox.exe --list
+ENV BUSYBOX_EXES="[ [[ ar arch ash awk base64 basename bash bunzip2 bzcat bzip2 cal cat chmod cksum clear cmp comm cp cpio cut date dc dd df diff dirname dos2unix dpkg-deb du echo ed egrep env expand expr factor false fgrep find fold fsync ftpget ftpput getopt grep groups gunzip gzip hd head hexdump id ipcalc kill killall less link ln logname ls lzcat lzma lzop lzopcat man md5sum mkdir mktemp mv nl od paste patch pgrep pidof pipe_progress pkill printenv printf ps pwd rev rm rmdir rpm rpm2cpio sed seq sh sha1sum sha256sum sha3sum sha512sum shred shuf sleep sort split ssl_client stat strings sum tac tail tar tee test timeout touch tr true truncate ttysize uname uncompress unexpand uniq unix2dos unlink unlzma unlzop unxz unzip usleep uudecode uuencode vi watch wc wget which whoami whois xargs xxd xz xzcat yes zcat"
+
+ADD https://github.com/kubernetes-sigs/windows-testing/raw/master/images/busybox/busybox.exe /busybox-dir/busybox.exe
+
+# NOTE(claudiub): We're creating symlinks for each of the busybox binaries and after that we're copying
+# # them over to Windows. Unfortunately, docker buildx has some issues copying over Windows symlinks.
+# # "Files/" is always prepended to the symlink target. The symlinks themselves are relative paths,
+# # so, in order to make use of them, we can simply add a busybox binary to Files\busybox.exe.
+RUN cd /busybox-dir/ && \
+    for busybox_binary in $BUSYBOX_EXES; do ln -s busybox.exe $busybox_binary.exe; done && \
+    mkdir Files && \
+    cp busybox.exe Files/busybox.exe
+
+RUN sh -c "mkdir /test_dir; echo test_content > /test_dir/test_file"
+
+FROM $BASE
+
+COPY --from=prep /busybox-dir /bin
+COPY --from=prep /test_dir /test_dir
+ENV PATH="C:\bin;C:\Windows\System32;C:\Windows;"
+VOLUME "/test_dir"
diff --git a/integration/images/volume-copy-up/Makefile b/integration/images/volume-copy-up/Makefile
@@ -15,20 +15,73 @@
 all: build
 
 PROJ=gcr.io/k8s-cri-containerd
-VERSION=2.0
+VERSION=2.1
 IMAGE=$(PROJ)/volume-copy-up:$(VERSION)
-PLATFORMS?=linux/amd64,linux/arm64
+
+# Operating systems supported: linux, windows
+OS ?= linux
+# Architectures supported: amd64, arm64
+ARCH ?= amd64
+# OS Version for the Windows images: 1809, 2004, 20H2
+OSVERSION ?= 1809
+
+# The output type could either be docker (local), or registry.
+# If it is registry, it will also allow us to push the Windows images.
+OUTPUT_TYPE ?= docker
+
+ALL_OS = linux windows
+ALL_ARCH.linux = amd64 arm64
+ALL_OS_ARCH.linux = $(foreach arch, ${ALL_ARCH.linux}, linux-$(arch))
+ALL_OSVERSIONS.windows := 1809 2004 20H2
+ALL_OS_ARCH.windows = $(foreach osversion, ${ALL_OSVERSIONS.windows}, windows-amd64-${osversion})
+ALL_OS_ARCH = $(foreach os, $(ALL_OS), ${ALL_OS_ARCH.${os}})
+
+BASE.linux.amd64 := busybox
+BASE.linux.arm64 := arm64v8/busybox
+BASE.linux := ${BASE.linux.${ARCH}}
+BASE.windows := mcr.microsoft.com/windows/nanoserver
+BASE := ${BASE.${OS}}
 
 configure-docker:
 	gcloud auth configure-docker
 
-build:
-	docker buildx build \
-	$(OUTPUT) \
-	--platform=${PLATFORMS} \
-	--tag $(IMAGE) .
+setup-buildx:
+	docker buildx use img-builder || docker buildx create --name img-builder --use
+
+build: setup-buildx build-local
+
+push: configure-docker setup-buildx build-registry push-manifest
+
+build-local: $(addprefix sub-container-docker-,$(ALL_OS_ARCH.linux))
+build-registry: $(addprefix sub-container-registry-,$(ALL_OS_ARCH))
+
+# split words on hyphen, access by 1-index
+word-hyphen = $(word $2,$(subst -, ,$1))
+sub-container-%:
+	$(MAKE) OUTPUT_TYPE=$(call word-hyphen,$*,1) OS=$(call word-hyphen,$*,2) ARCH=$(call word-hyphen,$*,3) OSVERSION=$(call word-hyphen,$*,4) container
+
+container: .container-${OS}-$(ARCH)
+
+.container-linux-$(ARCH):
+	docker buildx build --pull --output=type=${OUTPUT_TYPE} --platform ${OS}/${ARCH} \
+		-t $(IMAGE)-${OS}-${ARCH} --build-arg BASE=${BASE} .
+
+.container-windows-$(ARCH):
+	docker buildx build --pull --output=type=${OUTPUT_TYPE} --platform ${OS}/${ARCH} \
+		-t $(IMAGE)-${OS}-${ARCH}-${OSVERSION} --build-arg BASE=${BASE}:${OSVERSION} \
+		-f Dockerfile_windows .
 
-push: OUTPUT=--push
-push: configure-docker build
+# For Windows images, we also need to include the "os.version" in the manifest list images,
+# so the Windows node can pull the proper image it needs.
+push-manifest:
+	docker manifest create --amend $(IMAGE) $(shell echo $(ALL_OS_ARCH) | sed -e "s~[^ ]*~$(IMAGE)\-&~g")
+	set -x; for arch in $(ALL_ARCH.linux); do docker manifest annotate --os linux --arch $${arch} ${IMAGE} ${IMAGE}-linux-$${arch}; done
+	# we use awk to also trim the quotes around the OS version string.
+	set -x; \
+	for osversion in ${ALL_OSVERSIONS.windows}; do \
+		full_version=`docker manifest inspect ${BASE.windows}:$${osversion} | grep "os.version" | head -n 1 | awk -F\" '{print $$4}'` || true; \
+		docker manifest annotate --os windows --arch amd64 --os-version $${full_version} ${IMAGE} ${IMAGE}-windows-amd64-$${osversion}; \
+	done
+	docker manifest push --purge ${IMAGE}
 
-.PHONY: configure-docker build push
+.PHONY: configure-docker setup-buildx build push build-local build-registry container push-manifest
