Skip to content

Commit 6ef0d38

Browse files
estespestesp
authored
Merge pull request #5271 from cpuguy83/ci_output_junit
ci: upload junit formatted test results
2 parents cc393ea + af1e2af commit 6ef0d38

5 files changed

Lines changed: 165 additions & 8 deletions

File tree

‎.github/workflows/ci.yml‎

Lines changed: 62 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -247,6 +247,8 @@ jobs:
247247
runs-on: windows-2019
248248
timeout-minutes: 30
249249
needs: [project, linters, protos, man]
250+
env:
251+
GOTEST: gotestsum --
250252

251253
defaults:
252254
run:
@@ -270,6 +272,7 @@ jobs:
270272
- name: Set env
271273
run: |
272274
echo "GOPATH=${{ github.workspace }}" >> $GITHUB_ENV
275+
echo "${{ github.workspace }}/bin" >> $GITHUB_PATH
273276
echo "${{ github.workspace }}/src/github.com/containerd/containerd/bin" >> $GITHUB_PATH
274277
275278
- run: script/setup/install-dev-tools
@@ -287,22 +290,32 @@ jobs:
287290
git checkout "${SHIM_COMMIT}"
288291
GO111MODULE=on go build -mod=vendor -o "${bindir}/integration/client/containerd-shim-runhcs-v1.exe" ./cmd/containerd-shim-runhcs-v1
289292
293+
- run: script/setup/install-gotestsum
290294
- name: Tests
291295
env:
292296
CGO_ENABLED: 1
297+
GOTESTSUM_JUNITFILE: ${{github.workspace}}/test-unit-root.xml
293298
run: mingw32-make.exe test root-test
294299

295300
- name: Integration 1
296301
env:
297302
CGO_ENABLED: 1
303+
GOTESTSUM_JUNITFILE: ${{github.workspace}}/test-integration-serial-junit.xml
298304
run: mingw32-make.exe integration
299305

300-
# Run the integration suite a second time. See discussion in github.com/containerd/containerd/pull/175
306+
# Run the integration suite a second time. See discussion in github.com/containerd/containerd/pull/1759
301307
- name: Integration 2
302308
env:
303309
TESTFLAGS_PARALLEL: 1
304310
CGO_ENABLED: 1
311+
GOTESTSUM_JUNITFILE: ${{github.workspace}}/test-integration-parallel-junit.xml
305312
run: mingw32-make.exe integration
313+
- uses: actions/upload-artifact@v2
314+
if: always()
315+
with:
316+
name: TestResults Windows
317+
path: |
318+
${{github.workspace}}/*-junit.xml
306319
307320
integration-linux:
308321
name: Linux Integration
@@ -321,6 +334,8 @@ jobs:
321334
- runtime: io.containerd.runtime.v1.linux
322335
runc: crun
323336

337+
env:
338+
GOTEST: gotestsum --
324339
steps:
325340
- uses: actions/setup-go@v2
326341
with:
@@ -370,9 +385,12 @@ jobs:
370385
sudo make install
371386
working-directory: src/github.com/containerd/containerd
372387

388+
- run: sudo -E PATH=$PATH script/setup/install-gotestsum
389+
working-directory: src/github.com/containerd/containerd
373390
- name: Tests
374391
env:
375392
GOPROXY: direct
393+
GOTESTSUM_JUNITFILE: ${{github.workspace}}/test-unit-root-junit.xml
376394
run: |
377395
make test
378396
sudo -E PATH=$PATH GOPATH=$GOPATH GOPROXY=$GOPROXY make root-test
@@ -383,6 +401,7 @@ jobs:
383401
GOPROXY: direct
384402
TEST_RUNTIME: ${{ matrix.runtime }}
385403
RUNC_FLAVOR: ${{ matrix.runc }}
404+
GOTESTSUM_JUNITFILE: ${{github.workspace}}/test-integration-serial-junit.xml
386405
run: |
387406
sudo GOPATH=$GOPATH GOPROXY=$GOPROXY TEST_RUNTIME=$TEST_RUNTIME RUNC_FLAVOR=$RUNC_FLAVOR make integration EXTRA_TESTFLAGS=-no-criu TESTFLAGS_RACE=-race
388407
working-directory: src/github.com/containerd/containerd
@@ -393,6 +412,7 @@ jobs:
393412
GOPROXY: direct
394413
TEST_RUNTIME: ${{ matrix.runtime }}
395414
RUNC_FLAVOR: ${{ matrix.runc }}
415+
GOTESTSUM_JUNITFILE: ${{github.workspace}}/test-integration-parallel-junit.xml
396416
run: |
397417
sudo GOPATH=$GOPATH GOPROXY=$GOPROXY TEST_RUNTIME=$TEST_RUNTIME RUNC_FLAVOR=$RUNC_FLAVOR TESTFLAGS_PARALLEL=1 make integration EXTRA_TESTFLAGS=-no-criu
398418
working-directory: src/github.com/containerd/containerd
@@ -404,6 +424,7 @@ jobs:
404424
GOPROXY: direct
405425
TEST_RUNTIME: ${{ matrix.runtime }}
406426
RUNC_FLAVOR: ${{ matrix.runc }}
427+
GOTESTSUM_JUNITFILE: ${{github.workspace}}/test-integration-criu-junit.xml
407428
# crun doesn't have "checkpoint" command.
408429
if: ${{ matrix.runc == 'runc' }}
409430
run: |
@@ -434,7 +455,7 @@ jobs:
434455
sudo ls /etc/cni/net.d
435456
sudo PATH=$PATH BDIR=$BDIR /usr/local/bin/containerd -a ${BDIR}/c.sock --config ${BDIR}/config.toml --root ${BDIR}/root --state ${BDIR}/state --log-level debug &> ${BDIR}/containerd-cri.log &
436457
sudo PATH=$PATH BDIR=$BDIR /usr/local/bin/ctr -a ${BDIR}/c.sock version
437-
sudo PATH=$PATH BDIR=$BDIR GOPATH=$GOPATH critest --runtime-endpoint=unix:///${BDIR}/c.sock --parallel=8
458+
sudo PATH=$PATH BDIR=$BDIR GOPATH=$GOPATH critest --report-dir "${{github.workspace}}/critestreport" --runtime-endpoint=unix:///${BDIR}/c.sock --parallel=8
438459
TEST_RC=$?
439460
test $TEST_RC -ne 0 && cat ${BDIR}/containerd-cri.log
440461
sudo pkill containerd
@@ -450,12 +471,21 @@ jobs:
450471
mount
451472
df
452473
losetup -l
474+
- uses: actions/upload-artifact@v2
475+
if: always()
476+
with:
477+
name: TestResults ${{ matrix.runtime }} ${{matrix.runc}}
478+
path: |
479+
*-junit.xml
480+
${{github.workspace}}/critestreport/*.xml
453481
454482
tests-mac-os:
455483
name: MacOS unit tests
456484
runs-on: macos-10.15
457485
timeout-minutes: 10
458486
needs: [project, linters, protos, man]
487+
env:
488+
GOTEST: gotestsum --
459489

460490
steps:
461491
- uses: actions/setup-go@v2
@@ -471,12 +501,21 @@ jobs:
471501
echo "GOPATH=${{ github.workspace }}" >> $GITHUB_ENV
472502
echo "${{ github.workspace }}/bin" >> $GITHUB_PATH
473503
504+
- run: sudo -E PATH=$PATH script/setup/install-gotestsum
505+
working-directory: src/github.com/containerd/containerd
474506
- name: Tests
475507
env:
476508
GOPROXY: direct
509+
GOTESTSUM_JUNITFILE: "${{ github.workspace }}/macos-test-junit.xml"
477510
run: |
478511
make test
479512
working-directory: src/github.com/containerd/containerd
513+
- uses: actions/upload-artifact@v2
514+
if: always()
515+
with:
516+
name: TestResults MacOS
517+
path: |
518+
*-junit.xml
480519
481520
cgroup2:
482521
name: CGroupsV2 and SELinux Integration
@@ -489,6 +528,8 @@ jobs:
489528
# Currently crun is disabled to decrease CI flakiness.
490529
# We can enable crun again when we get a better CI infra.
491530
runc: [runc]
531+
env:
532+
GOTEST: gotestsum --
492533
steps:
493534
- uses: actions/checkout@v2
494535

@@ -507,10 +548,27 @@ jobs:
507548
env:
508549
RUNC_FLAVOR: ${{ matrix.runc }}
509550
SELINUX: Enforcing
510-
run: vagrant up --provision-with=selinux,install-runc,test-integration
551+
GOTESTSUM_JUNITFILE: /tmp/test-integration-junit.xml
552+
run: vagrant up --provision-with=selinux,install-runc,install-gotestsum,test-integration
511553

512554
- name: CRI test
513555
env:
514556
RUNC_FLAVOR: ${{ matrix.runc }}
515557
SELINUX: Enforcing
516-
run: vagrant up --provision-with=selinux,install-runc,test-cri
558+
REPORT_DIR: /tmp/critestreport
559+
run: vagrant up --provision-with=selinux,install-runc,install-gotestsum,test-cri
560+
- name: Get test reports
561+
if: always()
562+
run: |
563+
set -e
564+
vagrant plugin install vagrant-vbguest
565+
vagrant plugin install vagrant-scp
566+
vagrant scp :/tmp/test-integration-junit.xml "${{ github.workspace }}/"
567+
vagrant scp :/tmp/critestreport "${{ github.workspace }}/critestreport"
568+
- uses: actions/upload-artifact@v2
569+
if: always()
570+
with:
571+
name: TestResults cgroup2 ${{ matrix.runtime }} ${{matrix.runc}}
572+
path: |
573+
${{github.workspace}}/*-junit.xml
574+
${{github.workspace}}/critestreport/*

‎.github/workflows/publish_test_results.yml‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
name: Publish Test Results
2+
3+
on:
4+
workflow_run:
5+
workflows: ["CI"]
6+
types:
7+
- completed
8+
9+
jobs:
10+
test-results:
11+
runs-on: ubuntu-latest
12+
name: Publish Test Results
13+
if: >
14+
github.event.workflow_run.conclusion != 'skipped' && (
15+
github.event.workflow_run.head_repository.full_name != github.repository
16+
)
17+
steps:
18+
- name: Download Artifacts
19+
uses: actions/[email protected]
20+
with:
21+
script: |
22+
var fs = require('fs');
23+
var path = require('path');
24+
var artifacts_path = path.join('${{github.workspace}}', 'artifacts')
25+
fs.mkdirSync(artifacts_path, { recursive: true })
26+
27+
var artifacts = await github.actions.listWorkflowRunArtifacts({
28+
owner: context.repo.owner,
29+
repo: context.repo.repo,
30+
run_id: ${{ github.event.workflow_run.id }},
31+
});
32+
33+
for (const artifact of artifacts.data.artifacts) {
34+
var download = await github.actions.downloadArtifact({
35+
owner: context.repo.owner,
36+
repo: context.repo.repo,
37+
artifact_id: artifact.id,
38+
archive_format: 'zip',
39+
});
40+
var artifact_path = path.join(artifacts_path, `${artifact.name}.zip`)
41+
fs.writeFileSync(artifact_path, Buffer.from(download.data));
42+
console.log(`Downloaded ${artifact_path}`);
43+
}
44+
- name: Extract Artifacts
45+
run: |
46+
for file in artifacts/*.zip
47+
do
48+
if [ -f "$file" ]
49+
then
50+
dir="${file/%.zip/}"
51+
mkdir -p "$dir"
52+
unzip -d "$dir" "$file"
53+
fi
54+
done
55+
- name: Publish Test Results
56+
uses: EnricoMi/publish-unit-test-result-action@v1
57+
with:
58+
files: artifacts/*/**/*.xml
59+
check_name: Test Results
60+
comment_on_pr: true
61+
commit: ${{ github.event.workflow_run.head_sha }}

‎Makefile‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -120,6 +120,9 @@ BINARIES=$(addprefix bin/,$(COMMANDS))
120120
TESTFLAGS ?= $(TESTFLAGS_RACE) $(EXTRA_TESTFLAGS)
121121
TESTFLAGS_PARALLEL ?= 8
122122

123+
# Use this to replace `go test` with, for instance, `gotestsum`
124+
GOTEST ?= go test
125+
123126
OUTPUTDIR = $(join $(ROOTDIR), _output)
124127
CRIDIR=$(OUTPUTDIR)/cri
125128

@@ -170,15 +173,15 @@ build: ## build the go packages
170173

171174
test: ## run tests, except integration tests and tests that require root
172175
@echo "$(WHALE) $@"
173-
@go test ${TESTFLAGS} ${PACKAGES}
176+
@$(GOTEST) ${TESTFLAGS} ${PACKAGES}
174177

175178
root-test: ## run tests, except integration tests
176179
@echo "$(WHALE) $@"
177-
@go test ${TESTFLAGS} ${TEST_REQUIRES_ROOT_PACKAGES} -test.root
180+
@$(GOTEST) ${TESTFLAGS} ${TEST_REQUIRES_ROOT_PACKAGES} -test.root
178181

179182
integration: ## run integration tests
180183
@echo "$(WHALE) $@"
181-
@cd "${ROOTDIR}/integration/client" && go mod download && go test -v ${TESTFLAGS} -test.root -parallel ${TESTFLAGS_PARALLEL} .
184+
@cd "${ROOTDIR}/integration/client" && go mod download && $(GOTEST) -v ${TESTFLAGS} -test.root -parallel ${TESTFLAGS_PARALLEL} .
182185

183186
# TODO integrate cri integration bucket with coverage
184187
bin/cri-integration.test:

‎Vagrantfile‎

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -173,6 +173,18 @@ EOF
173173
SHELL
174174
end
175175

176+
config.vm.provision "install-gotestsum", type: "shell", run: "once" do |sh|
177+
sh.upload_path = "/tmp/vagrant-install-gotestsum"
178+
sh.inline = <<~SHELL
179+
#!/usr/bin/env bash
180+
source /etc/environment
181+
source /etc/profile.d/sh.local
182+
set -eux -o pipefail
183+
${GOPATH}/src/github.com/containerd/containerd/script/setup/install-gotestsum
184+
sudo cp ${GOPATH}/bin/gotestsum /usr/local/bin/
185+
SHELL
186+
end
187+
176188
# SELinux is Enforcing by default.
177189
# To set SELinux as Disabled on a VM that has already been provisioned:
178190
# SELINUX=Disabled vagrant up --provision-with=selinux
@@ -196,6 +208,8 @@ EOF
196208
sh.upload_path = "/tmp/test-integration"
197209
sh.env = {
198210
'RUNC_FLAVOR': ENV['RUNC_FLAVOR'] || "runc",
211+
'GOTEST': ENV['GOTEST'] || "go test",
212+
'GOTESTSUM_JUNITFILE': ENV['GOTESTSUM_JUNITFILE'],
199213
}
200214
sh.inline = <<~SHELL
201215
#!/usr/bin/env bash
@@ -213,6 +227,10 @@ EOF
213227
#
214228
config.vm.provision "test-cri", type: "shell", run: "never" do |sh|
215229
sh.upload_path = "/tmp/test-cri"
230+
sh.env = {
231+
'GOTEST': ENV['GOTEST'] || "go test",
232+
'REPORT_DIR': ENV['REPORT_DIR'],
233+
}
216234
sh.inline = <<~SHELL
217235
#!/usr/bin/env bash
218236
source /etc/environment
@@ -235,7 +253,7 @@ EOF
235253
fi
236254
trap cleanup EXIT
237255
ctr version
238-
critest --parallel=$(nproc) --ginkgo.skip='HostIpc is true'
256+
critest --parallel=$(nproc) --report-dir="${REPORT_DIR}" --ginkgo.skip='HostIpc is true'
239257
SHELL
240258
end
241259

‎script/setup/install-gotestsum‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
#!/usr/bin/env bash
2+
3+
# Copyright The containerd Authors.
4+
5+
# Licensed under the Apache License, Version 2.0 (the "License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
GO111MODULE=on go install gotest.tools/[email protected]

0 commit comments

Comments
 (0)