Skip to content

Commit 71a9432

Browse files
committed
build(make): keep running bats unit files after one fails
Each bats file is its own make target, so a plain `make bats-unit-tests` stopped scheduling files at the first red one and never ran the rest, while the output looked like a finished suite. CI passes -k and was not affected; a local run was. The target now runs the files in a --keep-going sub-make, which still exits non-zero when any file fails and shares the caller's job slots and output sync. Assisted-by: LLM Signed-off-by: Aleksei Sviridkin <[email protected]>
1 parent f023e2f commit 71a9432

5 files changed

Lines changed: 82 additions & 9 deletions

File tree

‎.github/workflows/pull-requests.yaml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -318,8 +318,8 @@ jobs:
318318
run: |
319319
helm plugin list 2>/dev/null | grep -q unittest \
320320
|| helm plugin install https://github.com/helm-unittest/helm-unittest
321-
# One make process owns all four runner CPUs. Each BATS file is a separate
322-
# prerequisite, so the long tail no longer runs as one nine-minute shell
321+
# The make processes share one four-slot jobserver. Each BATS file is a
322+
# separate target, so the long tail no longer runs as one nine-minute shell
323323
# loop; Helm and Go targets share the same bounded pool. Output stays
324324
# grouped by target instead of four test logs interleaving.
325325
#

‎Makefile‎

Lines changed: 13 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,10 @@
11
.PHONY: manifests assets prepare-env prepare-env-container unit-tests helm-unit-tests bats-unit-tests bats-unit-files-check rd-presets-check migrations-target-check test test-controllers preflight
22

3+
# Before any include, the last file make has read is this one: MAKEFILES and
4+
# -f files given before it are earlier in the list, -f files after it are not
5+
# read yet.
6+
ROOT_MAKEFILE := $(lastword $(MAKEFILE_LIST))
7+
38
include hack/common-envs.mk
49

510
build-deps:
@@ -196,17 +201,21 @@ BATS_UNIT_TARGETS := $(patsubst hack/%.bats,bats-unit-%,$(BATS_UNIT_FILES))
196201
# the live stream -- see the COZYTEST_TRACE comment in hack/cozytest.sh.
197202
COZYTEST_TRACE ?= 0
198203

199-
bats-unit-tests: bats-unit-files-check $(BATS_UNIT_TARGETS)
204+
# A sub-make with --keep-going, so one red file does not stop make from
205+
# scheduling the files after it; the sub-make still exits non-zero if any
206+
# file failed (hack/bats-unit-keep-going.bats covers both). It shares the
207+
# caller's -j slots and --output-sync through MAKEFLAGS.
208+
bats-unit-tests: bats-unit-files-check
209+
@$(MAKE) --file=$(ROOT_MAKEFILE) --no-print-directory --keep-going $(BATS_UNIT_TARGETS)
200210

201211
bats-unit-files-check:
202212
@if [ -z "$(BATS_UNIT_FILES)" ]; then \
203213
echo "ERROR: no hack/*.bats unit test files found"; \
204214
exit 1; \
205215
fi
206216

207-
# Each file is its own prerequisite so `make -jN` can schedule them, and the
208-
# trace switch main added for the serial loop rides along per target rather than
209-
# being lost with it.
217+
# Each file is its own target of the sub-make so `make -jN` can schedule them,
218+
# and the trace switch rides along per target.
210219
.PHONY: $(BATS_UNIT_TARGETS)
211220
$(BATS_UNIT_TARGETS): bats-unit-%: hack/%.bats
212221
@echo "--- running $< ---"

‎hack/bats-runner-coverage.bats‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -154,8 +154,11 @@ brc_audit() {
154154
printf '%s\n' "$makefile" | grep -qF 'BATS_UNIT_TARGETS := $(patsubst hack/%.bats,bats-unit-%,$(BATS_UNIT_FILES))' \
155155
|| { echo "FAIL: update the unit target model in hack/bats-runner-coverage.bats."; false; }
156156

157-
printf '%s\n' "$makefile" | grep -qF 'bats-unit-tests: bats-unit-files-check $(BATS_UNIT_TARGETS)' \
158-
|| { echo "FAIL: bats-unit-tests no longer depends on every discovered unit file."; false; }
157+
printf '%s\n' "$makefile" | grep -qF 'bats-unit-tests: bats-unit-files-check' \
158+
|| { echo "FAIL: bats-unit-tests no longer checks that unit files were discovered."; false; }
159+
160+
printf '%s\n' "$makefile" | grep -qF -- '--keep-going $(BATS_UNIT_TARGETS)' \
161+
|| { echo "FAIL: bats-unit-tests no longer runs every discovered unit file past a failing one."; false; }
159162

160163
printf '%s\n' "$makefile" | grep -qF '$(BATS_UNIT_TARGETS): bats-unit-%: hack/%.bats' \
161164
|| { echo "FAIL: each unit target must depend on its discovered bats file."; false; }

‎hack/bats-unit-keep-going.bats‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
#!/usr/bin/env bats
2+
# `make bats-unit-tests` must run every bats file even when an early one fails,
3+
# and still exit non-zero. Each file is its own make target, and without
4+
# --keep-going make stops scheduling targets at the first failure, so the files
5+
# sorting after it never run and the output reads like a complete suite.
6+
#
7+
# The tests drive the real root Makefile against a miniature tree: two bats
8+
# files and a stub hack/cozytest.sh that reports which file it was given.
9+
#
10+
# Requires: make.
11+
12+
REPO_ROOT="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")/.." && pwd)"
13+
14+
# kg_run <stub body line> [makefiles]: builds the fixture, runs the target with
15+
# MAKEFILES set to the second argument, and leaves the output in $fixture/out
16+
# and the exit status in $rc.
17+
kg_run() {
18+
fixture="$(mktemp -d)"
19+
mkdir "$fixture/hack"
20+
: >"$fixture/hack/common-envs.mk"
21+
: >"$fixture/hack/a.bats"
22+
: >"$fixture/hack/b.bats"
23+
printf '%s\n' '#!/bin/sh' 'echo "stub ran $1"' "$1" >"$fixture/hack/cozytest.sh"
24+
chmod +x "$fixture/hack/cozytest.sh"
25+
26+
# Under `make unit-tests -k` this suite inherits MAKEFLAGS carrying the
27+
# parent's --keep-going, which would let the old behaviour pass here.
28+
rc=0
29+
env -u MAKEFLAGS -u MFLAGS -u MAKELEVEL MAKEFILES="${2:-}" \
30+
make --file="$REPO_ROOT/Makefile" --directory="$fixture" bats-unit-tests >"$fixture/out" 2>&1 || rc=$?
31+
cat "$fixture/out"
32+
}
33+
34+
@test "bats-unit-tests runs every file after a failure and still fails" {
35+
# Fail whichever file runs first, whatever order make picks, so the exit
36+
# status cannot come from the last file alone.
37+
kg_run 'mkdir "$(dirname "$0")/.failed" 2>/dev/null && exit 1; exit 0'
38+
ran_a=$(grep -c '^stub ran hack/a.bats$' "$fixture/out" || true)
39+
ran_b=$(grep -c '^stub ran hack/b.bats$' "$fixture/out" || true)
40+
rm -rf "$fixture"
41+
42+
[ "$ran_a" = 1 ]
43+
[ "$ran_b" = 1 ]
44+
[ "$rc" -ne 0 ]
45+
}
46+
47+
@test "bats-unit-tests still finds the root Makefile when MAKEFILES reads another first" {
48+
extra="$(mktemp)"
49+
printf '%s\n' 'unrelated:' ' @true' >"$extra"
50+
kg_run 'exit 0' "$extra"
51+
ran_b=$(grep -c '^stub ran hack/b.bats$' "$fixture/out" || true)
52+
rm -rf "$fixture" "$extra"
53+
[ "$ran_b" = 1 ]
54+
[ "$rc" -eq 0 ]
55+
}
56+
57+
@test "bats-unit-tests exits zero when every file passes" {
58+
kg_run 'exit 0'
59+
rm -rf "$fixture"
60+
[ "$rc" -eq 0 ]
61+
}

‎hack/unit-test-parallelism.bats‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
#!/usr/bin/env bats
22
# Contract for the bounded parallel unit-test fan-out used by PR CI.
33

4-
@test "every non-E2E BATS file is an independent Make prerequisite" {
4+
@test "every non-E2E BATS file is an independent Make target" {
55
# --output-sync arrived in GNU Make 4.0, and macOS still ships 3.81, where
66
# this invocation dies on an unknown option rather than telling the reader
77
# why. CI is on 4.x, so the contract is still enforced where it gates a

0 commit comments

Comments
 (0)