Skip to content

Commit 45019e8

Browse files
myasnikovdaniillexfrei
authored andcommitted
test(tests): run unit suites with Bats and retain POSIX coverage
The shell translator rewrites test bodies and loses Bats reporting. The unit lane needs complete TAP results and JUnit reports without changing how live-cluster suites run. Sourced production helpers still need a dash pass, and unit test bodies need nounset without exporting shell options into subprocesses. Assisted-by: LLM Signed-off-by: Myasnikov Daniil <[email protected]>
1 parent d2e3eb4 commit 45019e8

176 files changed

Lines changed: 1526 additions & 536 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/pre-commit.yml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,9 @@ jobs:
6060
curl -sSL https://github.com/cozystack/cozyvalues-gen/releases/download/v1.7.0/cozyvalues-gen-linux-amd64.tar.gz | tar -xzvf- -C /usr/local/bin/ cozyvalues-gen
6161
6262
- name: Run pre-commit hooks
63+
# The complete Bats pass is a manual local hook and is therefore outside
64+
# this changed-file lint run. pull-requests.yaml's checks job runs the
65+
# full unit path for code and the Bats contracts for docs-only changes.
6366
run: |
6467
git fetch origin main || git fetch origin master
6568
base_commit=$(git rev-parse --verify origin/main || git rev-parse --verify origin/master || echo "")

‎.github/workflows/pull-requests.yaml‎

Lines changed: 43 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -75,7 +75,6 @@ jobs:
7575
statuses: write
7676
outputs:
7777
code: ${{ steps.p.outputs.code }}
78-
trigger_map: ${{ steps.p.outputs.trigger_map }}
7978
matrix: ${{ steps.p.outputs.matrix }}
8079
any: ${{ steps.p.outputs.any }}
8180
touched: ${{ steps.p.outputs.touched }}
@@ -163,23 +162,6 @@ jobs:
163162
else
164163
echo "code=false" >> "$GITHUB_OUTPUT"
165164
fi
166-
# docs/agents/contributing.md carries the downstream trigger map, which
167-
# hack/downstream-trigger-map.bats asserts against: every path the map
168-
# cites still exists, and its repository list still matches the PR
169-
# template. Editing the map alone is a docs-only PR, which would skip the
170-
# unit tests and exempt the map from its own guard. This flag lets the
171-
# unit-test job opt back in, and only that job: it deliberately does not
172-
# feed `code`, so a docs PR still builds nothing.
173-
#
174-
# Detect the map by its old path too, which is why this reads the
175-
# rename-blind list above: a PR that moves the map would otherwise skip
176-
# the very test that pins the literal below to it, and break for
177-
# whoever pushes next.
178-
if grep -qxF 'docs/agents/contributing.md' /tmp/changed_norenames.txt; then
179-
echo "trigger_map=true" >> "$GITHUB_OUTPUT"
180-
else
181-
echo "trigger_map=false" >> "$GITHUB_OUTPUT"
182-
fi
183165
matrix="$(hack/build-matrix.sh /tmp/changed.txt)"
184166
echo "Build matrix: $matrix"
185167
echo "matrix=$matrix" >> "$GITHUB_OUTPUT"
@@ -300,59 +282,86 @@ jobs:
300282
permissions:
301283
contents: read
302284
needs: ["plan"]
303-
# trigger_map opts a docs-only PR back in when it edits the downstream trigger
304-
# map, so hack/downstream-trigger-map.bats still guards it. No other job reads
305-
# that flag: a docs PR runs the unit tests and nothing heavier.
285+
# Unit contracts read both code and documentation. Run this job for every
286+
# non-release PR: code changes take the complete unit path, while docs-only
287+
# changes run the Bats lane that owns those cross-tree contracts. Builds and
288+
# E2E remain gated on `code` below.
306289
if: |
307-
(needs.plan.outputs.code == 'true' || needs.plan.outputs.trigger_map == 'true')
308-
&& !contains(github.event.pull_request.labels.*.name, 'release')
290+
!contains(github.event.pull_request.labels.*.name, 'release')
309291
steps:
310292
- name: Checkout code
311293
uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
312294
with:
313295
fetch-depth: 0
314296
# The module cache is restored from the key build-main.yaml writes on
315297
# main, which every PR run can read, so a PR's first run starts warm. The
316-
# download fetches whatever the cache lacks, the whole set on a miss.
298+
# download fetches whatever the cache lacks, the whole set on a miss. The
299+
# Go steps are for code changes only: a docs-only run is the Bats lane alone.
317300
- name: Locate Go module cache
301+
if: needs.plan.outputs.code == 'true'
318302
id: gomodcache
319303
run: echo "dir=$(go env GOMODCACHE)" >> "$GITHUB_OUTPUT"
320304
- name: Cache Go modules
305+
if: needs.plan.outputs.code == 'true'
321306
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
322307
with:
323308
path: ${{ steps.gomodcache.outputs.dir }}
324309
key: go-mod-${{ runner.os }}-${{ hashFiles('**/go.sum') }}
325310
restore-keys: go-mod-${{ runner.os }}-
326311
- name: Download Go modules
312+
if: needs.plan.outputs.code == 'true'
327313
run: |
328314
for mod in $(git ls-files go.mod '*/go.mod' | xargs -n1 dirname); do
329315
(cd "$mod" && "$GITHUB_WORKSPACE/hack/go-mod-download.sh")
330316
done
331317
# Ephemeral runners don't carry the helm-unittest plugin the persistent
332318
# self-hosted runner accreted. The plugin embeds its own Helm, so its
333-
# version is pinned: a plugin release must not change test results.
319+
# version is pinned: a plugin release must not change test results. Bats is
320+
# installed on every run, docs-only ones included.
334321
- name: Set up test toolchain
335322
env:
323+
CODE_CHANGED: ${{ needs.plan.outputs.code }}
336324
HELM_UNITTEST_VERSION: "1.1.2"
325+
BATS_VERSION: "1.14.0"
326+
BATS_COMMIT: eb7f42f8d608ac693d7a4b67474f6714ea68cfc5
337327
run: |
338-
installed="$(helm plugin list 2>/dev/null | awk '$1 == "unittest" { print $2 }')"
339-
if [ "$installed" != "$HELM_UNITTEST_VERSION" ]; then
340-
[ -z "$installed" ] || helm plugin uninstall unittest
341-
helm plugin install https://github.com/helm-unittest/helm-unittest --version "v${HELM_UNITTEST_VERSION}"
328+
set -euo pipefail
329+
if [ "$CODE_CHANGED" = true ]; then
330+
installed="$(helm plugin list 2>/dev/null | awk '$1 == "unittest" { print $2 }' || true)"
331+
if [ "$installed" != "$HELM_UNITTEST_VERSION" ]; then
332+
[ -z "$installed" ] || helm plugin uninstall unittest
333+
helm plugin install https://github.com/helm-unittest/helm-unittest --version "v${HELM_UNITTEST_VERSION}"
334+
fi
342335
fi
343-
# The make processes share one four-slot jobserver. Each BATS file is a
344-
# separate target, so the long tail no longer runs as one nine-minute shell
345-
# loop; Helm and Go targets share the same bounded pool. Output stays
346-
# grouped by target instead of four test logs interleaving.
347-
#
336+
# Source archive bytes can change; verify the tag's commit before install.
337+
if ! bats --version 2>/dev/null | grep -qx "Bats $BATS_VERSION"; then
338+
bats_parent=$(mktemp -d "$RUNNER_TEMP/bats-core.XXXXXX")
339+
bats_source="$bats_parent/source"
340+
git clone --quiet --depth=1 --branch "v$BATS_VERSION" \
341+
https://github.com/bats-core/bats-core.git "$bats_source"
342+
test "$(git -C "$bats_source" rev-parse HEAD)" = "$BATS_COMMIT"
343+
sudo "$bats_source/install.sh" /usr/local
344+
fi
345+
bats --version | grep -qx "Bats $BATS_VERSION"
348346
# -k is required, not cosmetic: unit-tests and test-controllers are two
349347
# independent goals on the same command line, and without -k make stops
350348
# handing out new work the moment any unit-tests prerequisite fails —
351349
# test-controllers then never starts and reports nothing, so a red BATS
352350
# file hides the entire Go controller suite. -k lets already-schedulable
353351
# goals keep running; the job still fails overall on any red target.
354352
- name: Run unit and controller tests
353+
if: needs.plan.outputs.code == 'true'
355354
run: make unit-tests test-controllers -j4 -k --output-sync=target
355+
- name: Run Bats unit tests for docs-only changes
356+
if: needs.plan.outputs.code == 'false'
357+
run: make bats-unit-tests
358+
- name: Upload unit test report
359+
if: always()
360+
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
361+
with:
362+
name: bats-unit-report
363+
path: _out/test-reports/
364+
if-no-files-found: ignore
356365

357366
# One image (package) per job, fanned out across the ephemeral pool. Each job
358367
# gets its own buildx (docker-container driver) so the shared-buildkit

‎.pre-commit-config.yaml‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,17 @@ repos:
2121
'
2222
language: system
2323
files: ^.*$
24+
# The complete hack/*.bats and POSIX-compatibility pass is intentionally a
25+
# manual local hook because it requires bats-core and has substantial wall
26+
# time even with GNU parallel. Run it before pushing with:
27+
# pre-commit run bats-unit-tests --hook-stage manual --all-files
28+
# Pull-request CI remains the unconditional gate.
29+
- id: bats-unit-tests
30+
name: Run hack/*.bats unit tests
31+
entry: make bats-unit-tests bats-posix-compat-tests
32+
language: system
33+
stages: [manual]
34+
pass_filenames: false
2435
# Lints changed GitHub Actions workflows for unpinned actions and excessive token
2536
# permissions (audit scope is set in .github/zizmor.yml). Runs offline so it needs
2637
# no GitHub token locally. The same gate runs tree-wide in .github/workflows/zizmor.yml.

‎Makefile‎

Lines changed: 72 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,4 @@
1-
.PHONY: manifests assets prepare-env prepare-env-container unit-tests helm-unit-tests bats-unit-tests bats-unit-files-check rd-presets-check migrations-target-check test test-controllers preflight
2-
3-
# Before any include, the last file make has read is this one: MAKEFILES and
4-
# -f files given before it are earlier in the list, -f files after it are not
5-
# read yet.
6-
ROOT_MAKEFILE := $(lastword $(MAKEFILE_LIST))
1+
.PHONY: manifests assets prepare-env prepare-env-container unit-tests helm-unit-tests bats-unit-tests bats-unit-files-check bats-posix-compat-tests print-bats-unit-files print-bats-posix-compat-files print-bats-jobs rd-presets-check migrations-target-check test test-controllers preflight
72

83
include hack/common-envs.mk
94

@@ -123,7 +118,7 @@ test:
123118
make -C packages/core/testing apply
124119
make -C packages/core/testing e2e
125120

126-
unit-tests: helm-unit-tests bats-unit-tests go-unit-tests go-module-tests rd-presets-check test-check-readiness migrations-target-check
121+
unit-tests: helm-unit-tests bats-unit-tests bats-posix-compat-tests go-unit-tests go-module-tests rd-presets-check test-check-readiness migrations-target-check
127122

128123
helm-unit-tests:
129124
hack/helm-unit-tests.sh
@@ -175,52 +170,91 @@ test-controllers:
175170
test-check-readiness:
176171
go test ./test/check-readiness/ -count=1
177172

178-
# Discover every hack/*.bats file that is NOT an e2e test and run it
179-
# through cozytest.sh. This glob is one level deep: live-cluster suites under
180-
# hack/e2e-* or in subdirectories need an invocation in packages/core/testing.
181-
# hack/bats-runner-coverage.bats reports files that neither runner selects.
182-
# Park a suite with the .bats.disabled suffix to exclude it from the audit.
173+
# Unit discovery is one level deep. Live-cluster files need an invocation in
174+
# packages/core/testing; hack/bats-runner-coverage.bats checks that boundary.
175+
# Bats supplies TAP and JUnit output; the live-cluster runner keeps its tracing.
183176
#
184177
# Caveat: $(wildcard ...) returns space-separated names, so a filename
185178
# containing a literal space would split into multiple tokens here. All
186179
# current bats files use hyphen-separated names; if the project ever
187180
# introduces whitespace-bearing filenames this recipe must be rewritten
188181
# (e.g. to use `find ... -print0 | xargs -0`).
189182
BATS_UNIT_FILES := $(filter-out hack/e2e-%.bats,$(wildcard hack/*.bats))
190-
BATS_UNIT_TARGETS := $(patsubst hack/%.bats,bats-unit-%,$(BATS_UNIT_FILES))
191-
192-
# Quiet by default: cozytest.sh streams every test's xtrace live, which over
193-
# this many suites buried the one failing assertion under ~240k lines of
194-
# trace in CI. COZYTEST_TRACE=0 keeps the per-test start/end lines and still dumps
195-
# the full trace of whichever test FAILS -- but not of one that is killed
196-
# instead, a hang reaped by the job timeout among them, since the dump runs
197-
# after the test returns. See the COZYTEST_TRACE comment in hack/cozytest.sh.
198-
# Override to watch a run line by
199-
# line: `COZYTEST_TRACE=1 make bats-unit-tests`, or invoke the runner
200-
# directly (`hack/cozytest.sh hack/foo.bats [pattern]`), which stays verbose.
201-
# The e2e call sites in packages/core/testing/Makefile are untouched and keep
202-
# the live stream -- see the COZYTEST_TRACE comment in hack/cozytest.sh.
203-
COZYTEST_TRACE ?= 0
204183

205-
# A sub-make with --keep-going, so one red file does not stop make from
206-
# scheduling the files after it; the sub-make still exits non-zero if any
207-
# file failed (hack/bats-unit-keep-going.bats covers both). It shares the
208-
# caller's -j slots and --output-sync through MAKEFLAGS.
209-
bats-unit-tests: bats-unit-files-check
210-
@$(MAKE) --file=$(ROOT_MAKEFILE) --no-print-directory --keep-going $(BATS_UNIT_TARGETS)
184+
# The same list, one file per line, for hack/bats-strict-setup.bats. Every unit
185+
# file has to load hack/test_helper.bash to get `set -u` back, and that audit is
186+
# only worth anything if the set it walks is the set that actually runs -- so it
187+
# asks here rather than keeping a second copy of the filter above.
188+
print-bats-unit-files:
189+
@printf '%s\n' $(BATS_UNIT_FILES)
190+
191+
# `bats -j` needs GNU parallel and exits non-zero rather than degrading when it
192+
# is missing. moreutils also ships a command named parallel, so identify the GNU
193+
# implementation from its version output before enabling concurrency. Resolve
194+
# the probe once per make process; `?=` would keep the `$(shell ...)` recursive
195+
# and rerun it at every expansion.
196+
ifndef BATS_JOBS
197+
BATS_JOBS := $(shell parallel --version 2>/dev/null | grep -q '^GNU parallel ' && nproc 2>/dev/null || echo 1)
198+
endif
199+
200+
print-bats-jobs:
201+
@printf '%s\n' "$(BATS_JOBS)"
202+
203+
# JUnit XML is preserved as a CI artifact for inspection. _out is gitignored.
204+
BATS_REPORT_DIR ?= _out/test-reports
211205

212206
bats-unit-files-check:
213207
@if [ -z "$(BATS_UNIT_FILES)" ]; then \
214208
echo "ERROR: no hack/*.bats unit test files found"; \
215209
exit 1; \
216210
fi
211+
bats-unit-tests: bats-unit-files-check
212+
@command -v bats >/dev/null 2>&1 || { \
213+
echo "ERROR: bats not found. Install bats-core >= 1.5 — https://bats-core.readthedocs.io"; \
214+
exit 1; \
215+
}
216+
@mkdir -p "$(BATS_REPORT_DIR)"
217+
bats -j $(BATS_JOBS) --report-formatter junit -o "$(BATS_REPORT_DIR)" $(BATS_UNIT_FILES)
218+
219+
# Real Bats is authoritative. Unit files that source production code whose
220+
# contract is POSIX sh retain a compatibility pass through cozytest.sh's
221+
# /bin/sh translator. Discover literal .sh dependencies wherever they live,
222+
# including hack/lib and package migration helpers, so a new non-Chainsaw
223+
# helper test cannot run only under Bash. The explicit entries retain reviewed
224+
# shell-facing tests, including files whose production path is held in a
225+
# variable and therefore cannot be identified from the source line alone.
226+
BATS_SOURCED_SH_FILES := $(shell grep -El '^[[:space:]]*(\.|source)[[:space:]]+.*\.sh' $(BATS_UNIT_FILES))
227+
BATS_POSIX_COMPAT_FILES := $(sort \
228+
$(BATS_SOURCED_SH_FILES) \
229+
hack/capture-dataplane.bats \
230+
hack/capture-previous-logs.bats \
231+
hack/cilium-leak-healer_test.bats \
232+
hack/container-lane-capacity_test.bats \
233+
hack/cozyreport-talos.bats \
234+
hack/cozyreport.bats \
235+
hack/cozystack-version-stamp.bats \
236+
hack/nightly-mirror_test.bats \
237+
hack/pod-label-census_test.bats \
238+
hack/promote-rewrite-tags_test.bats \
239+
hack/runner-identity.bats \
240+
hack/seaweedfs-naming-audit.bats \
241+
)
242+
243+
# CI's /bin/sh is dash. Keep the interpreter overridable so contributors can
244+
# run the same compatibility lane explicitly on hosts whose /bin/sh differs.
245+
BATS_POSIX_SHELL ?= /bin/sh
246+
COZYTEST_TRACE ?= 0
247+
248+
print-bats-posix-compat-files:
249+
@printf '%s\n' $(BATS_POSIX_COMPAT_FILES)
217250

218-
# Each file is its own target of the sub-make so `make -jN` can schedule them,
219-
# and the trace switch rides along per target.
220-
.PHONY: $(BATS_UNIT_TARGETS)
221-
$(BATS_UNIT_TARGETS): bats-unit-%: hack/%.bats
222-
@echo "--- running $< ---"
223-
@COZYTEST_TRACE=$(COZYTEST_TRACE) hack/cozytest.sh "$<"
251+
bats-posix-compat-tests:
252+
@status=0; \
253+
for f in $(BATS_POSIX_COMPAT_FILES); do \
254+
echo "--- running POSIX compatibility: $$f ---"; \
255+
COZYTEST_TRACE=$(COZYTEST_TRACE) "$(BATS_POSIX_SHELL)" hack/cozytest.sh "$$f" || status=1; \
256+
done; \
257+
exit $$status
224258

225259
# Operator-facing host preflight check. Warns about a standalone
226260
# containerd.service or docker.service running alongside the embedded

0 commit comments

Comments
 (0)