Skip to content

Commit af1e2af

Browse files
cpuguy83cpuguy83
committed
ci: upload junit formatted test results
This allows us to dig more details out of test runs and maintain a better history. For this we can use `gotestsum`, which is a utility that wraps `go test` so that it outputs test2json (go's format) and output junit (a format more easily imported into other systems). The PR makes it possible to override the Makefile's use of `go test` to use any other command tto executet the test. For CI we'll use `gotestsum --`, where `gotestsum` expects everything after the `--` to be flags for `go test`. We then use environment variables to configure `gotestsum` (e.g. `GOTESTSUM_JUNITFILE` is an env var accepted by `gotestsum`). For cri tests, the test suite supports outputing test results to a directory, these are in junit format already. The file is not named properly just because the code that creates it (in ginkgo) is not configured well. We can fix that upstream to give us a better name... until then I'm keeping those results in a separate dir. A second workflow is also added so the test results can be summed up and a report added to the workflow run. The 2nd workflow is required for this since PR runs do not have access to do some of this due to safety reasons (https://securitylab.github.com/research/github-actions-preventing-pwn-requests/) Signed-off-by: Brian Goff <[email protected]>
1 parent 1c05317 commit af1e2af

5 files changed

Lines changed: 165 additions & 8 deletions

File tree

‎.github/workflows/ci.yml‎

Lines changed: 62 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -247,6 +247,8 @@ jobs:
247247
runs-on: windows-2019
248248
timeout-minutes: 30
249249
needs: [project, linters, protos, man]
250+
env:
251+
GOTEST: gotestsum --
250252

251253
defaults:
252254
run:
@@ -270,6 +272,7 @@ jobs:
270272
- name: Set env
271273
run: |
272274
echo "GOPATH=${{ github.workspace }}" >> $GITHUB_ENV
275+
echo "${{ github.workspace }}/bin" >> $GITHUB_PATH
273276
echo "${{ github.workspace }}/src/github.com/containerd/containerd/bin" >> $GITHUB_PATH
274277
275278
- run: script/setup/install-dev-tools
@@ -287,22 +290,32 @@ jobs:
287290
git checkout "${SHIM_COMMIT}"
288291
GO111MODULE=on go build -mod=vendor -o "${bindir}/integration/client/containerd-shim-runhcs-v1.exe" ./cmd/containerd-shim-runhcs-v1
289292
293+
- run: script/setup/install-gotestsum
290294
- name: Tests
291295
env:
292296
CGO_ENABLED: 1
297+
GOTESTSUM_JUNITFILE: ${{github.workspace}}/test-unit-root.xml
293298
run: mingw32-make.exe test root-test
294299

295300
- name: Integration 1
296301
env:
297302
CGO_ENABLED: 1
303+
GOTESTSUM_JUNITFILE: ${{github.workspace}}/test-integration-serial-junit.xml
298304
run: mingw32-make.exe integration
299305

300-
# Run the integration suite a second time. See discussion in github.com/containerd/containerd/pull/175
306+
# Run the integration suite a second time. See discussion in github.com/containerd/containerd/pull/1759
301307
- name: Integration 2
302308
env:
303309
TESTFLAGS_PARALLEL: 1
304310
CGO_ENABLED: 1
311+
GOTESTSUM_JUNITFILE: ${{github.workspace}}/test-integration-parallel-junit.xml
305312
run: mingw32-make.exe integration
313+
- uses: actions/upload-artifact@v2
314+
if: always()
315+
with:
316+
name: TestResults Windows
317+
path: |
318+
${{github.workspace}}/*-junit.xml
306319
307320
integration-linux:
308321
name: Linux Integration
@@ -321,6 +334,8 @@ jobs:
321334
- runtime: io.containerd.runtime.v1.linux
322335
runc: crun
323336

337+
env:
338+
GOTEST: gotestsum --
324339
steps:
325340
- uses: actions/setup-go@v2
326341
with:
@@ -370,9 +385,12 @@ jobs:
370385
sudo make install
371386
working-directory: src/github.com/containerd/containerd
372387

388+
- run: sudo -E PATH=$PATH script/setup/install-gotestsum
389+
working-directory: src/github.com/containerd/containerd
373390
- name: Tests
374391
env:
375392
GOPROXY: direct
393+
GOTESTSUM_JUNITFILE: ${{github.workspace}}/test-unit-root-junit.xml
376394
run: |
377395
make test
378396
sudo -E PATH=$PATH GOPATH=$GOPATH GOPROXY=$GOPROXY make root-test
@@ -383,6 +401,7 @@ jobs:
383401
GOPROXY: direct
384402
TEST_RUNTIME: ${{ matrix.runtime }}
385403
RUNC_FLAVOR: ${{ matrix.runc }}
404+
GOTESTSUM_JUNITFILE: ${{github.workspace}}/test-integration-serial-junit.xml
386405
run: |
387406
sudo GOPATH=$GOPATH GOPROXY=$GOPROXY TEST_RUNTIME=$TEST_RUNTIME RUNC_FLAVOR=$RUNC_FLAVOR make integration EXTRA_TESTFLAGS=-no-criu TESTFLAGS_RACE=-race
388407
working-directory: src/github.com/containerd/containerd
@@ -393,6 +412,7 @@ jobs:
393412
GOPROXY: direct
394413
TEST_RUNTIME: ${{ matrix.runtime }}
395414
RUNC_FLAVOR: ${{ matrix.runc }}
415+
GOTESTSUM_JUNITFILE: ${{github.workspace}}/test-integration-parallel-junit.xml
396416
run: |
397417
sudo GOPATH=$GOPATH GOPROXY=$GOPROXY TEST_RUNTIME=$TEST_RUNTIME RUNC_FLAVOR=$RUNC_FLAVOR TESTFLAGS_PARALLEL=1 make integration EXTRA_TESTFLAGS=-no-criu
398418
working-directory: src/github.com/containerd/containerd
@@ -404,6 +424,7 @@ jobs:
404424
GOPROXY: direct
405425
TEST_RUNTIME: ${{ matrix.runtime }}
406426
RUNC_FLAVOR: ${{ matrix.runc }}
427+
GOTESTSUM_JUNITFILE: ${{github.workspace}}/test-integration-criu-junit.xml
407428
# crun doesn't have "checkpoint" command.
408429
if: ${{ matrix.runc == 'runc' }}
409430
run: |
@@ -433,7 +454,7 @@ jobs:
433454
sudo ls /etc/cni/net.d
434455
sudo PATH=$PATH BDIR=$BDIR /usr/local/bin/containerd -a ${BDIR}/c.sock --config ${BDIR}/config.toml --root ${BDIR}/root --state ${BDIR}/state --log-level debug &> ${BDIR}/containerd-cri.log &
435456
sudo PATH=$PATH BDIR=$BDIR /usr/local/bin/ctr -a ${BDIR}/c.sock version
436-
sudo PATH=$PATH BDIR=$BDIR GOPATH=$GOPATH critest --runtime-endpoint=unix:///${BDIR}/c.sock --parallel=8
457+
sudo PATH=$PATH BDIR=$BDIR GOPATH=$GOPATH critest --report-dir "${{github.workspace}}/critestreport" --runtime-endpoint=unix:///${BDIR}/c.sock --parallel=8
437458
TEST_RC=$?
438459
test $TEST_RC -ne 0 && cat ${BDIR}/containerd-cri.log
439460
sudo pkill containerd
@@ -449,12 +470,21 @@ jobs:
449470
mount
450471
df
451472
losetup -l
473+
- uses: actions/upload-artifact@v2
474+
if: always()
475+
with:
476+
name: TestResults ${{ matrix.runtime }} ${{matrix.runc}}
477+
path: |
478+
*-junit.xml
479+
${{github.workspace}}/critestreport/*.xml
452480
453481
tests-mac-os:
454482
name: MacOS unit tests
455483
runs-on: macos-10.15
456484
timeout-minutes: 10
457485
needs: [project, linters, protos, man]
486+
env:
487+
GOTEST: gotestsum --
458488

459489
steps:
460490
- uses: actions/setup-go@v2
@@ -470,12 +500,21 @@ jobs:
470500
echo "GOPATH=${{ github.workspace }}" >> $GITHUB_ENV
471501
echo "${{ github.workspace }}/bin" >> $GITHUB_PATH
472502
503+
- run: sudo -E PATH=$PATH script/setup/install-gotestsum
504+
working-directory: src/github.com/containerd/containerd
473505
- name: Tests
474506
env:
475507
GOPROXY: direct
508+
GOTESTSUM_JUNITFILE: "${{ github.workspace }}/macos-test-junit.xml"
476509
run: |
477510
make test
478511
working-directory: src/github.com/containerd/containerd
512+
- uses: actions/upload-artifact@v2
513+
if: always()
514+
with:
515+
name: TestResults MacOS
516+
path: |
517+
*-junit.xml
479518
480519
cgroup2:
481520
name: CGroupsV2 and SELinux Integration
@@ -488,6 +527,8 @@ jobs:
488527
# Currently crun is disabled to decrease CI flakiness.
489528
# We can enable crun again when we get a better CI infra.
490529
runc: [runc]
530+
env:
531+
GOTEST: gotestsum --
491532
steps:
492533
- uses: actions/checkout@v2
493534

@@ -506,10 +547,27 @@ jobs:
506547
env:
507548
RUNC_FLAVOR: ${{ matrix.runc }}
508549
SELINUX: Enforcing
509-
run: vagrant up --provision-with=selinux,install-runc,test-integration
550+
GOTESTSUM_JUNITFILE: /tmp/test-integration-junit.xml
551+
run: vagrant up --provision-with=selinux,install-runc,install-gotestsum,test-integration
510552

511553
- name: CRI test
512554
env:
513555
RUNC_FLAVOR: ${{ matrix.runc }}
514556
SELINUX: Enforcing
515-
run: vagrant up --provision-with=selinux,install-runc,test-cri
557+
REPORT_DIR: /tmp/critestreport
558+
run: vagrant up --provision-with=selinux,install-runc,install-gotestsum,test-cri
559+
- name: Get test reports
560+
if: always()
561+
run: |
562+
set -e
563+
vagrant plugin install vagrant-vbguest
564+
vagrant plugin install vagrant-scp
565+
vagrant scp :/tmp/test-integration-junit.xml "${{ github.workspace }}/"
566+
vagrant scp :/tmp/critestreport "${{ github.workspace }}/critestreport"
567+
- uses: actions/upload-artifact@v2
568+
if: always()
569+
with:
570+
name: TestResults cgroup2 ${{ matrix.runtime }} ${{matrix.runc}}
571+
path: |
572+
${{github.workspace}}/*-junit.xml
573+
${{github.workspace}}/critestreport/*

‎.github/workflows/publish_test_results.yml‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
name: Publish Test Results
2+
3+
on:
4+
workflow_run:
5+
workflows: ["CI"]
6+
types:
7+
- completed
8+
9+
jobs:
10+
test-results:
11+
runs-on: ubuntu-latest
12+
name: Publish Test Results
13+
if: >
14+
github.event.workflow_run.conclusion != 'skipped' && (
15+
github.event.workflow_run.head_repository.full_name != github.repository
16+
)
17+
steps:
18+
- name: Download Artifacts
19+
uses: actions/[email protected]
20+
with:
21+
script: |
22+
var fs = require('fs');
23+
var path = require('path');
24+
var artifacts_path = path.join('${{github.workspace}}', 'artifacts')
25+
fs.mkdirSync(artifacts_path, { recursive: true })
26+
27+
var artifacts = await github.actions.listWorkflowRunArtifacts({
28+
owner: context.repo.owner,
29+
repo: context.repo.repo,
30+
run_id: ${{ github.event.workflow_run.id }},
31+
});
32+
33+
for (const artifact of artifacts.data.artifacts) {
34+
var download = await github.actions.downloadArtifact({
35+
owner: context.repo.owner,
36+
repo: context.repo.repo,
37+
artifact_id: artifact.id,
38+
archive_format: 'zip',
39+
});
40+
var artifact_path = path.join(artifacts_path, `${artifact.name}.zip`)
41+
fs.writeFileSync(artifact_path, Buffer.from(download.data));
42+
console.log(`Downloaded ${artifact_path}`);
43+
}
44+
- name: Extract Artifacts
45+
run: |
46+
for file in artifacts/*.zip
47+
do
48+
if [ -f "$file" ]
49+
then
50+
dir="${file/%.zip/}"
51+
mkdir -p "$dir"
52+
unzip -d "$dir" "$file"
53+
fi
54+
done
55+
- name: Publish Test Results
56+
uses: EnricoMi/publish-unit-test-result-action@v1
57+
with:
58+
files: artifacts/*/**/*.xml
59+
check_name: Test Results
60+
comment_on_pr: true
61+
commit: ${{ github.event.workflow_run.head_sha }}

‎Makefile‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -120,6 +120,9 @@ BINARIES=$(addprefix bin/,$(COMMANDS))
120120
TESTFLAGS ?= $(TESTFLAGS_RACE) $(EXTRA_TESTFLAGS)
121121
TESTFLAGS_PARALLEL ?= 8
122122

123+
# Use this to replace `go test` with, for instance, `gotestsum`
124+
GOTEST ?= go test
125+
123126
OUTPUTDIR = $(join $(ROOTDIR), _output)
124127
CRIDIR=$(OUTPUTDIR)/cri
125128

@@ -170,15 +173,15 @@ build: ## build the go packages
170173

171174
test: ## run tests, except integration tests and tests that require root
172175
@echo "$(WHALE) $@"
173-
@go test ${TESTFLAGS} ${PACKAGES}
176+
@$(GOTEST) ${TESTFLAGS} ${PACKAGES}
174177

175178
root-test: ## run tests, except integration tests
176179
@echo "$(WHALE) $@"
177-
@go test ${TESTFLAGS} ${TEST_REQUIRES_ROOT_PACKAGES} -test.root
180+
@$(GOTEST) ${TESTFLAGS} ${TEST_REQUIRES_ROOT_PACKAGES} -test.root
178181

179182
integration: ## run integration tests
180183
@echo "$(WHALE) $@"
181-
@cd "${ROOTDIR}/integration/client" && go mod download && go test -v ${TESTFLAGS} -test.root -parallel ${TESTFLAGS_PARALLEL} .
184+
@cd "${ROOTDIR}/integration/client" && go mod download && $(GOTEST) -v ${TESTFLAGS} -test.root -parallel ${TESTFLAGS_PARALLEL} .
182185

183186
# TODO integrate cri integration bucket with coverage
184187
bin/cri-integration.test:

‎Vagrantfile‎

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -173,6 +173,18 @@ EOF
173173
SHELL
174174
end
175175

176+
config.vm.provision "install-gotestsum", type: "shell", run: "once" do |sh|
177+
sh.upload_path = "/tmp/vagrant-install-gotestsum"
178+
sh.inline = <<~SHELL
179+
#!/usr/bin/env bash
180+
source /etc/environment
181+
source /etc/profile.d/sh.local
182+
set -eux -o pipefail
183+
${GOPATH}/src/github.com/containerd/containerd/script/setup/install-gotestsum
184+
sudo cp ${GOPATH}/bin/gotestsum /usr/local/bin/
185+
SHELL
186+
end
187+
176188
# SELinux is Enforcing by default.
177189
# To set SELinux as Disabled on a VM that has already been provisioned:
178190
# SELINUX=Disabled vagrant up --provision-with=selinux
@@ -196,6 +208,8 @@ EOF
196208
sh.upload_path = "/tmp/test-integration"
197209
sh.env = {
198210
'RUNC_FLAVOR': ENV['RUNC_FLAVOR'] || "runc",
211+
'GOTEST': ENV['GOTEST'] || "go test",
212+
'GOTESTSUM_JUNITFILE': ENV['GOTESTSUM_JUNITFILE'],
199213
}
200214
sh.inline = <<~SHELL
201215
#!/usr/bin/env bash
@@ -213,6 +227,10 @@ EOF
213227
#
214228
config.vm.provision "test-cri", type: "shell", run: "never" do |sh|
215229
sh.upload_path = "/tmp/test-cri"
230+
sh.env = {
231+
'GOTEST': ENV['GOTEST'] || "go test",
232+
'REPORT_DIR': ENV['REPORT_DIR'],
233+
}
216234
sh.inline = <<~SHELL
217235
#!/usr/bin/env bash
218236
source /etc/environment
@@ -235,7 +253,7 @@ EOF
235253
fi
236254
trap cleanup EXIT
237255
ctr version
238-
critest --parallel=$(nproc) --ginkgo.skip='HostIpc is true'
256+
critest --parallel=$(nproc) --report-dir="${REPORT_DIR}" --ginkgo.skip='HostIpc is true'
239257
SHELL
240258
end
241259

‎script/setup/install-gotestsum‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
#!/usr/bin/env bash
2+
3+
# Copyright The containerd Authors.
4+
5+
# Licensed under the Apache License, Version 2.0 (the "License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
GO111MODULE=on go install gotest.tools/[email protected]

0 commit comments

Comments
 (0)