Skip to content

Commit f924cb8

Browse files
test(tests): run unit suites with Bats and retain POSIX coverage
The shell translator rewrites test bodies and loses Bats reporting. The unit lane needs complete TAP results and JUnit reports without changing how live-cluster suites run. Sourced production helpers still need a dash pass, and unit test bodies need nounset without exporting shell options into subprocesses. Assisted-by: LLM Signed-off-by: Myasnikov Daniil <[email protected]>
1 parent 360f03b commit f924cb8

163 files changed

Lines changed: 1477 additions & 519 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/pre-commit.yml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,9 @@ jobs:
5757
curl -sSL https://github.com/cozystack/cozyvalues-gen/releases/download/v1.7.0/cozyvalues-gen-linux-amd64.tar.gz | tar -xzvf- -C /usr/local/bin/ cozyvalues-gen
5858
5959
- name: Run pre-commit hooks
60+
# The complete Bats pass is a manual local hook and is therefore outside
61+
# this changed-file lint run. pull-requests.yaml's checks job runs the
62+
# full unit path for code and the Bats contracts for docs-only changes.
6063
run: |
6164
git fetch origin main || git fetch origin master
6265
base_commit=$(git rev-parse --verify origin/main || git rev-parse --verify origin/master || echo "")

‎.github/workflows/pull-requests.yaml‎

Lines changed: 35 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -75,7 +75,6 @@ jobs:
7575
statuses: write
7676
outputs:
7777
code: ${{ steps.p.outputs.code }}
78-
trigger_map: ${{ steps.p.outputs.trigger_map }}
7978
matrix: ${{ steps.p.outputs.matrix }}
8079
any: ${{ steps.p.outputs.any }}
8180
touched: ${{ steps.p.outputs.touched }}
@@ -163,23 +162,6 @@ jobs:
163162
else
164163
echo "code=false" >> "$GITHUB_OUTPUT"
165164
fi
166-
# docs/agents/contributing.md carries the downstream trigger map, which
167-
# hack/downstream-trigger-map.bats asserts against: every path the map
168-
# cites still exists, and its repository list still matches the PR
169-
# template. Editing the map alone is a docs-only PR, which would skip the
170-
# unit tests and exempt the map from its own guard. This flag lets the
171-
# unit-test job opt back in, and only that job: it deliberately does not
172-
# feed `code`, so a docs PR still builds nothing.
173-
#
174-
# Detect the map by its old path too, which is why this reads the
175-
# rename-blind list above: a PR that moves the map would otherwise skip
176-
# the very test that pins the literal below to it, and break for
177-
# whoever pushes next.
178-
if grep -qxF 'docs/agents/contributing.md' /tmp/changed_norenames.txt; then
179-
echo "trigger_map=true" >> "$GITHUB_OUTPUT"
180-
else
181-
echo "trigger_map=false" >> "$GITHUB_OUTPUT"
182-
fi
183165
matrix="$(hack/build-matrix.sh /tmp/changed.txt)"
184166
echo "Build matrix: $matrix"
185167
echo "matrix=$matrix" >> "$GITHUB_OUTPUT"
@@ -301,12 +283,12 @@ jobs:
301283
permissions:
302284
contents: read
303285
needs: ["plan"]
304-
# trigger_map opts a docs-only PR back in when it edits the downstream trigger
305-
# map, so hack/downstream-trigger-map.bats still guards it. No other job reads
306-
# that flag: a docs PR runs the unit tests and nothing heavier.
286+
# Unit contracts read both code and documentation. Run this job for every
287+
# non-release PR: code changes take the complete unit path, while docs-only
288+
# changes run the Bats lane that owns those cross-tree contracts. Builds and
289+
# E2E remain gated on `code` below.
307290
if: |
308-
(needs.plan.outputs.code == 'true' || needs.plan.outputs.trigger_map == 'true')
309-
&& !contains(github.event.pull_request.labels.*.name, 'release')
291+
!contains(github.event.pull_request.labels.*.name, 'release')
310292
steps:
311293
- name: Checkout code
312294
uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
@@ -315,22 +297,45 @@ jobs:
315297
# Ephemeral runners don't carry the helm-unittest plugin the persistent
316298
# self-hosted runner accreted. Install if absent (idempotent).
317299
- name: Set up test toolchain
300+
env:
301+
CODE_CHANGED: ${{ needs.plan.outputs.code }}
302+
BATS_VERSION: "1.14.0"
303+
BATS_COMMIT: eb7f42f8d608ac693d7a4b67474f6714ea68cfc5
318304
run: |
319-
helm plugin list 2>/dev/null | grep -q unittest \
320-
|| helm plugin install https://github.com/helm-unittest/helm-unittest
321-
# The make processes share one four-slot jobserver. Each BATS file is a
322-
# separate target, so the long tail no longer runs as one nine-minute shell
323-
# loop; Helm and Go targets share the same bounded pool. Output stays
324-
# grouped by target instead of four test logs interleaving.
325-
#
305+
set -euo pipefail
306+
if [ "$CODE_CHANGED" = true ]; then
307+
helm plugin list 2>/dev/null | grep -q unittest \
308+
|| helm plugin install https://github.com/helm-unittest/helm-unittest
309+
fi
310+
# Source archive bytes can change; verify the tag's commit before install.
311+
if ! bats --version 2>/dev/null | grep -qx "Bats $BATS_VERSION"; then
312+
bats_parent=$(mktemp -d "$RUNNER_TEMP/bats-core.XXXXXX")
313+
bats_source="$bats_parent/source"
314+
git clone --quiet --depth=1 --branch "v$BATS_VERSION" \
315+
https://github.com/bats-core/bats-core.git "$bats_source"
316+
test "$(git -C "$bats_source" rev-parse HEAD)" = "$BATS_COMMIT"
317+
sudo "$bats_source/install.sh" /usr/local
318+
fi
319+
bats --version | grep -qx "Bats $BATS_VERSION"
326320
# -k is required, not cosmetic: unit-tests and test-controllers are two
327321
# independent goals on the same command line, and without -k make stops
328322
# handing out new work the moment any unit-tests prerequisite fails —
329323
# test-controllers then never starts and reports nothing, so a red BATS
330324
# file hides the entire Go controller suite. -k lets already-schedulable
331325
# goals keep running; the job still fails overall on any red target.
332326
- name: Run unit and controller tests
327+
if: needs.plan.outputs.code == 'true'
333328
run: make unit-tests test-controllers -j4 -k --output-sync=target
329+
- name: Run Bats unit tests for docs-only changes
330+
if: needs.plan.outputs.code == 'false'
331+
run: make bats-unit-tests
332+
- name: Upload unit test report
333+
if: always()
334+
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
335+
with:
336+
name: bats-unit-report
337+
path: _out/test-reports/
338+
if-no-files-found: ignore
334339

335340
# One image (package) per job, fanned out across the ephemeral pool. Each job
336341
# gets its own buildx (docker-container driver) so the shared-buildkit

‎.pre-commit-config.yaml‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,17 @@ repos:
2121
'
2222
language: system
2323
files: ^.*$
24+
# The complete hack/*.bats and POSIX-compatibility pass is intentionally a
25+
# manual local hook because it requires bats-core and has substantial wall
26+
# time even with GNU parallel. Run it before pushing with:
27+
# pre-commit run bats-unit-tests --hook-stage manual --all-files
28+
# Pull-request CI remains the unconditional gate.
29+
- id: bats-unit-tests
30+
name: Run hack/*.bats unit tests
31+
entry: make bats-unit-tests bats-posix-compat-tests
32+
language: system
33+
stages: [manual]
34+
pass_filenames: false
2435
# Lints changed GitHub Actions workflows for unpinned actions and excessive token
2536
# permissions (audit scope is set in .github/zizmor.yml). Runs offline so it needs
2637
# no GitHub token locally. The same gate runs tree-wide in .github/workflows/zizmor.yml.

‎Makefile‎

Lines changed: 72 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,4 @@
1-
.PHONY: manifests assets prepare-env prepare-env-container unit-tests helm-unit-tests bats-unit-tests bats-unit-files-check rd-presets-check migrations-target-check test test-controllers preflight
2-
3-
# Before any include, the last file make has read is this one: MAKEFILES and
4-
# -f files given before it are earlier in the list, -f files after it are not
5-
# read yet.
6-
ROOT_MAKEFILE := $(lastword $(MAKEFILE_LIST))
1+
.PHONY: manifests assets prepare-env prepare-env-container unit-tests helm-unit-tests bats-unit-tests bats-unit-files-check bats-posix-compat-tests print-bats-unit-files print-bats-posix-compat-files print-bats-jobs rd-presets-check migrations-target-check test test-controllers preflight
72

83
include hack/common-envs.mk
94

@@ -122,7 +117,7 @@ test:
122117
make -C packages/core/testing apply
123118
make -C packages/core/testing e2e
124119

125-
unit-tests: helm-unit-tests bats-unit-tests go-unit-tests go-module-tests rd-presets-check test-check-readiness migrations-target-check
120+
unit-tests: helm-unit-tests bats-unit-tests bats-posix-compat-tests go-unit-tests go-module-tests rd-presets-check test-check-readiness migrations-target-check
126121

127122
helm-unit-tests:
128123
hack/helm-unit-tests.sh
@@ -174,52 +169,91 @@ test-controllers:
174169
test-check-readiness:
175170
go test ./test/check-readiness/ -count=1
176171

177-
# Discover every hack/*.bats file that is NOT an e2e test and run it
178-
# through cozytest.sh. This glob is one level deep: live-cluster suites under
179-
# hack/e2e-* or in subdirectories need an invocation in packages/core/testing.
180-
# hack/bats-runner-coverage.bats reports files that neither runner selects.
181-
# Park a suite with the .bats.disabled suffix to exclude it from the audit.
172+
# Unit discovery is one level deep. Live-cluster files need an invocation in
173+
# packages/core/testing; hack/bats-runner-coverage.bats checks that boundary.
174+
# Bats supplies TAP and JUnit output; the live-cluster runner keeps its tracing.
182175
#
183176
# Caveat: $(wildcard ...) returns space-separated names, so a filename
184177
# containing a literal space would split into multiple tokens here. All
185178
# current bats files use hyphen-separated names; if the project ever
186179
# introduces whitespace-bearing filenames this recipe must be rewritten
187180
# (e.g. to use `find ... -print0 | xargs -0`).
188181
BATS_UNIT_FILES := $(filter-out hack/e2e-%.bats,$(wildcard hack/*.bats))
189-
BATS_UNIT_TARGETS := $(patsubst hack/%.bats,bats-unit-%,$(BATS_UNIT_FILES))
190-
191-
# Quiet by default: cozytest.sh streams every test's xtrace live, which over
192-
# this many suites buried the one failing assertion under ~240k lines of
193-
# trace in CI. COZYTEST_TRACE=0 keeps the per-test start/end lines and still dumps
194-
# the full trace of whichever test FAILS -- but not of one that is killed
195-
# instead, a hang reaped by the job timeout among them, since the dump runs
196-
# after the test returns. See the COZYTEST_TRACE comment in hack/cozytest.sh.
197-
# Override to watch a run line by
198-
# line: `COZYTEST_TRACE=1 make bats-unit-tests`, or invoke the runner
199-
# directly (`hack/cozytest.sh hack/foo.bats [pattern]`), which stays verbose.
200-
# The e2e call sites in packages/core/testing/Makefile are untouched and keep
201-
# the live stream -- see the COZYTEST_TRACE comment in hack/cozytest.sh.
202-
COZYTEST_TRACE ?= 0
203182

204-
# A sub-make with --keep-going, so one red file does not stop make from
205-
# scheduling the files after it; the sub-make still exits non-zero if any
206-
# file failed (hack/bats-unit-keep-going.bats covers both). It shares the
207-
# caller's -j slots and --output-sync through MAKEFLAGS.
208-
bats-unit-tests: bats-unit-files-check
209-
@$(MAKE) --file=$(ROOT_MAKEFILE) --no-print-directory --keep-going $(BATS_UNIT_TARGETS)
183+
# The same list, one file per line, for hack/bats-strict-setup.bats. Every unit
184+
# file has to load hack/test_helper.bash to get `set -u` back, and that audit is
185+
# only worth anything if the set it walks is the set that actually runs -- so it
186+
# asks here rather than keeping a second copy of the filter above.
187+
print-bats-unit-files:
188+
@printf '%s\n' $(BATS_UNIT_FILES)
189+
190+
# `bats -j` needs GNU parallel and exits non-zero rather than degrading when it
191+
# is missing. moreutils also ships a command named parallel, so identify the GNU
192+
# implementation from its version output before enabling concurrency. Resolve
193+
# the probe once per make process; `?=` would keep the `$(shell ...)` recursive
194+
# and rerun it at every expansion.
195+
ifndef BATS_JOBS
196+
BATS_JOBS := $(shell parallel --version 2>/dev/null | grep -q '^GNU parallel ' && nproc 2>/dev/null || echo 1)
197+
endif
198+
199+
print-bats-jobs:
200+
@printf '%s\n' "$(BATS_JOBS)"
201+
202+
# JUnit XML is preserved as a CI artifact for inspection. _out is gitignored.
203+
BATS_REPORT_DIR ?= _out/test-reports
210204

211205
bats-unit-files-check:
212206
@if [ -z "$(BATS_UNIT_FILES)" ]; then \
213207
echo "ERROR: no hack/*.bats unit test files found"; \
214208
exit 1; \
215209
fi
210+
bats-unit-tests: bats-unit-files-check
211+
@command -v bats >/dev/null 2>&1 || { \
212+
echo "ERROR: bats not found. Install bats-core >= 1.5 — https://bats-core.readthedocs.io"; \
213+
exit 1; \
214+
}
215+
@mkdir -p "$(BATS_REPORT_DIR)"
216+
bats -j $(BATS_JOBS) --report-formatter junit -o "$(BATS_REPORT_DIR)" $(BATS_UNIT_FILES)
217+
218+
# Real Bats is authoritative. Unit files that source production code whose
219+
# contract is POSIX sh retain a compatibility pass through cozytest.sh's
220+
# /bin/sh translator. Discover literal .sh dependencies wherever they live,
221+
# including hack/lib and package migration helpers, so a new non-Chainsaw
222+
# helper test cannot run only under Bash. The explicit entries retain reviewed
223+
# shell-facing tests, including files whose production path is held in a
224+
# variable and therefore cannot be identified from the source line alone.
225+
BATS_SOURCED_SH_FILES := $(shell grep -El '^[[:space:]]*(\.|source)[[:space:]]+.*\.sh' $(BATS_UNIT_FILES))
226+
BATS_POSIX_COMPAT_FILES := $(sort \
227+
$(BATS_SOURCED_SH_FILES) \
228+
hack/capture-dataplane.bats \
229+
hack/capture-previous-logs.bats \
230+
hack/cilium-leak-healer_test.bats \
231+
hack/container-lane-capacity_test.bats \
232+
hack/cozyreport-talos.bats \
233+
hack/cozyreport.bats \
234+
hack/cozystack-version-stamp.bats \
235+
hack/nightly-mirror_test.bats \
236+
hack/pod-label-census_test.bats \
237+
hack/promote-rewrite-tags_test.bats \
238+
hack/runner-identity.bats \
239+
hack/seaweedfs-naming-audit.bats \
240+
)
241+
242+
# CI's /bin/sh is dash. Keep the interpreter overridable so contributors can
243+
# run the same compatibility lane explicitly on hosts whose /bin/sh differs.
244+
BATS_POSIX_SHELL ?= /bin/sh
245+
COZYTEST_TRACE ?= 0
246+
247+
print-bats-posix-compat-files:
248+
@printf '%s\n' $(BATS_POSIX_COMPAT_FILES)
216249

217-
# Each file is its own target of the sub-make so `make -jN` can schedule them,
218-
# and the trace switch rides along per target.
219-
.PHONY: $(BATS_UNIT_TARGETS)
220-
$(BATS_UNIT_TARGETS): bats-unit-%: hack/%.bats
221-
@echo "--- running $< ---"
222-
@COZYTEST_TRACE=$(COZYTEST_TRACE) hack/cozytest.sh "$<"
250+
bats-posix-compat-tests:
251+
@status=0; \
252+
for f in $(BATS_POSIX_COMPAT_FILES); do \
253+
echo "--- running POSIX compatibility: $$f ---"; \
254+
COZYTEST_TRACE=$(COZYTEST_TRACE) "$(BATS_POSIX_SHELL)" hack/cozytest.sh "$$f" || status=1; \
255+
done; \
256+
exit $$status
223257

224258
# Operator-facing host preflight check. Warns about a standalone
225259
# containerd.service or docker.service running alongside the embedded

0 commit comments

Comments
 (0)