|
| 1 | +#!/usr/bin/env bats |
| 2 | + |
| 3 | +# Every .bats file under hack/ must be run by something. |
| 4 | +# |
| 5 | +# Two files once sat in hack/e2e-apps/ that no runner invoked: #2826 deleted the |
| 6 | +# `test-apps-%:` pattern rule that used to run `hack/e2e-apps/$*.bats`, and #3176 |
| 7 | +# then added two new suites into the emptied directory. Git raises no conflict |
| 8 | +# when one branch deletes a directory's contents and another adds files to it, so |
| 9 | +# the only signal was a directory name that still looked live. Three commits of |
| 10 | +# review went into tests that never executed once. |
| 11 | +# |
| 12 | +# There are exactly two runners, and that is a closed list by design: |
| 13 | +# |
| 14 | +# A. Makefile's `bats-unit-tests` walks $(wildcard hack/*.bats) minus |
| 15 | +# hack/e2e-%.bats. The glob is ONE level deep, which is the whole bug: it |
| 16 | +# says nothing at all about hack/<subdir>/. |
| 17 | +# B. packages/core/testing/Makefile names its suites literally, one |
| 18 | +# `hack/cozytest.sh <path>` per recipe line, and runs them inside the e2e |
| 19 | +# sandbox. |
| 20 | +# |
| 21 | +# A file matching neither is dead code. A third runner appearing shows up here as |
| 22 | +# a false offender rather than as silence, which is correct: the runner inventory |
| 23 | +# is the thing this file is about, so a new runner registers here or it is not a |
| 24 | +# runner as far as this guard is concerned. |
| 25 | +# |
| 26 | +# No allowlist and no debt-declaration mechanism, deliberately. The project's |
| 27 | +# off-switch is the `.disabled` suffix (hack/select-e2e.sh:254-255 states the |
| 28 | +# convention for Chainsaw suites), and `find -name '*.bats'` stops seeing a |
| 29 | +# parked file on its own. An in-test list of exempt paths would be the same |
| 30 | +# artefact class as the runner that got deleted while its documentation stayed. |
| 31 | +# |
| 32 | +# Why the Makefiles are parsed rather than asked: hack/common-envs.mk is included |
| 33 | +# at Makefile:3, and when COZYSTACK_VERSION comes back empty it runs `git remote |
| 34 | +# add upstream` and `git fetch upstream --tags` in `$(shell ...)` AT PARSE TIME. |
| 35 | +# `make -p`, `make -n` and `$(info ...)` all pay that, so asking make would make a |
| 36 | +# unit test mutate git remotes and reach the network to answer a question about |
| 37 | +# filenames. Make also cannot answer for runner B at all -- those three suites are |
| 38 | +# literal recipe text, so `make -np` hands back the same text to grep, one |
| 39 | +# fragile evaluation later. The cost of grepping is that the model can drift out |
| 40 | +# of step with the Makefile it models; the second test below pins the modelled |
| 41 | +# lines verbatim so that drift fails loudly instead of quietly widening the set |
| 42 | +# of files this guard believes are covered. |
| 43 | +# |
| 44 | +# Scope, so the pin is not read as covering more than it does. No test in the |
| 45 | +# unit suite can catch deletion of `bats-unit-tests` itself, or of the `make |
| 46 | +# unit-tests` step in pull-requests.yaml -- this guard would simply stop running |
| 47 | +# with everything else. The `unit-tests:`-names-`bats-unit-tests` pin below covers |
| 48 | +# the dependency being dropped while the file still runs; past that the hole is |
| 49 | +# structural. Chainsaw suites are out of scope: `chainsaw test` does its own |
| 50 | +# discovery, and hack/select-e2e_test.bats already pins the suite-to-source |
| 51 | +# mapping in both directions. Reachability here is at file level; it is not a |
| 52 | +# proof that CI invokes every target. |
| 53 | +# |
| 54 | +# Note also what an `e2e-` prefix does and does not mean. It arms the cluster |
| 55 | +# captures in hack/cozytest.sh (pinned by hack/cozytest-capture-gate.bats) and it |
| 56 | +# excludes a top-level file from runner A. It has never been a claim that a runner |
| 57 | +# exists. Conflating the two is how the original bug stayed invisible. |
| 58 | + |
| 59 | +REPO_ROOT="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")/.." && pwd)" |
| 60 | +E2E_MAKEFILE_REL="packages/core/testing/Makefile" |
| 61 | + |
| 62 | +# A literal tab, for matching make recipe lines. Only tab-led lines are recipes; |
| 63 | +# a `#` inside one is shell, not a make comment, so comments are stripped first |
| 64 | +# and the recipe filter applied second. |
| 65 | +TAB="$(printf '\t')" |
| 66 | + |
| 67 | +# Drop make comment lines, so a commented-out runner can never satisfy the |
| 68 | +# contract. grep exits 1 when it selects nothing, which is legitimate here, and 2 |
| 69 | +# on a real error; the rc check turns only the latter into a failure. Under |
| 70 | +# hack/cozytest.sh the status is discarded anyway -- its translator appends |
| 71 | +# `return 0` to every line that is exactly `}`, file-level helpers included -- so |
| 72 | +# every test below asserts its extraction is non-empty before comparing, and a |
| 73 | +# swallowed error surfaces as empty input rather than as a pass. |
| 74 | +brc_code_lines() { |
| 75 | + local rc=0 |
| 76 | + grep -v '^[[:space:]]*#' || rc=$? |
| 77 | + [ "$rc" -le 1 ] |
| 78 | +} |
| 79 | + |
| 80 | +# Every .bats file under $1/hack, printed relative to $1. |
| 81 | +# |
| 82 | +# Relative rather than basename: the scan recurses, and hack/foo.bats and |
| 83 | +# hack/e2e-apps/foo.bats must not report under one name. Same reason as |
| 84 | +# bnet_audit in hack/bats-no-exit-trap.bats. |
| 85 | +brc_all_bats() { |
| 86 | + find "$1/hack" -name '*.bats' | sort | while IFS= read -r _f; do |
| 87 | + [ -e "$_f" ] || continue |
| 88 | + echo "${_f#"$1"/}" |
| 89 | + done |
| 90 | +} |
| 91 | + |
| 92 | +# Runner A's selection: $(filter-out hack/e2e-%.bats,$(wildcard hack/*.bats)). |
| 93 | +brc_unit_reachable() { |
| 94 | + find "$1/hack" -maxdepth 1 -name '*.bats' ! -name 'e2e-*' | sort | while IFS= read -r _f; do |
| 95 | + [ -e "$_f" ] || continue |
| 96 | + echo "${_f#"$1"/}" |
| 97 | + done |
| 98 | +} |
| 99 | + |
| 100 | +# Runner B's selection: paths named on a recipe line, each immediately after |
| 101 | +# `hack/cozytest.sh`. |
| 102 | +# |
| 103 | +# Requiring the token to FOLLOW cozytest.sh is what stops a stale mention in a |
| 104 | +# variable, a comment already dropped above, or a dead assignment from counting as |
| 105 | +# a runner. The trailing quote of the surrounding `sh -c '...'` falls outside the |
| 106 | +# match because the pattern ends at `.bats`; a quote leading the token, were a |
| 107 | +# path ever written quoted, is removed after extraction, since no path here |
| 108 | +# contains one. |
| 109 | +brc_named_suites() { |
| 110 | + brc_code_lines < "$1/$E2E_MAKEFILE_REL" \ |
| 111 | + | grep "^$TAB" \ |
| 112 | + | grep -oE "hack/cozytest\.sh[[:space:]]+[^[:space:]]+\.bats" \ |
| 113 | + | sed "s|.*cozytest\.sh[[:space:]]*||" \ |
| 114 | + | tr -d "\"'" \ |
| 115 | + | sort -u |
| 116 | +} |
| 117 | + |
| 118 | +# Named suites whose path is not a literal -- the shape of the deleted |
| 119 | +# `hack/e2e-apps/$*.bats` pattern rule. |
| 120 | +# |
| 121 | +# Reported rather than expanded. A pattern rule nobody invokes is exactly as dead |
| 122 | +# as no rule at all, so treating one as coverage would reintroduce this bug in a |
| 123 | +# new shape. If a pattern runner is ever wanted back, teach this guard the pattern |
| 124 | +# form on purpose. |
| 125 | +brc_unresolvable_suites() { |
| 126 | + brc_named_suites "$1" | grep '\$' || true |
| 127 | +} |
| 128 | + |
| 129 | +# Report every reachability offence under root $1, one line each, and print |
| 130 | +# nothing when there are none. |
| 131 | +# |
| 132 | +# Through stdout rather than an exit status: hack/cozytest.sh rewrites a |
| 133 | +# function's closing brace into `return 0`, so a status set in here would be |
| 134 | +# discarded before the caller could read it and the check would pass |
| 135 | +# unconditionally. Nothing is accumulated in a variable across a pipeline either, |
| 136 | +# because `find | while read` runs in a subshell under dash. |
| 137 | +brc_audit() { |
| 138 | + _root=$1 |
| 139 | + _reach=$( { brc_unit_reachable "$_root"; brc_named_suites "$_root" | grep -v '\$' || true; } | sort -u ) |
| 140 | + |
| 141 | + brc_all_bats "$_root" | while IFS= read -r _b; do |
| 142 | + if printf '%s\n' "$_reach" | grep -Fxq "$_b"; then |
| 143 | + continue |
| 144 | + fi |
| 145 | + echo "$_b: run by nothing. Runner A walks hack/*.bats one level deep and skips hack/e2e-*; $E2E_MAKEFILE_REL names no suite at this path. Add a runner there, port it to hack/e2e-chainsaw/, or park it as *.bats.disabled." |
| 146 | + done |
| 147 | + |
| 148 | + # The inverse: a suite the e2e Makefile still names after the file moved or was |
| 149 | + # renamed. Cheap here, and expensive anywhere else -- a stale name in |
| 150 | + # test-openapi surfaces ~40 minutes into the e2e job, and in prepare-cluster or |
| 151 | + # install-cozystack it wedges the whole bootstrap before a single test runs. |
| 152 | + brc_named_suites "$_root" | grep -v '\$' | while IFS= read -r _n; do |
| 153 | + if [ ! -f "$_root/$_n" ]; then |
| 154 | + echo "$E2E_MAKEFILE_REL runs $_n, which does not exist on disk." |
| 155 | + fi |
| 156 | + done |
| 157 | + |
| 158 | + brc_unresolvable_suites "$_root" | while IFS= read -r _u; do |
| 159 | + echo "$E2E_MAKEFILE_REL names $_u through a make variable; this guard resolves literal paths only. Name the suites literally, or teach the guard the pattern form on purpose." |
| 160 | + done |
| 161 | +} |
| 162 | + |
| 163 | +@test "every bats file under hack is run by a runner" { |
| 164 | + # The point of the file, run against the tree it governs. Everything below |
| 165 | + # exists to prove this test can fail. |
| 166 | + [ -f "$REPO_ROOT/Makefile" ] || { echo "FAIL: no root Makefile at $REPO_ROOT"; false; } |
| 167 | + [ -f "$REPO_ROOT/$E2E_MAKEFILE_REL" ] || { echo "FAIL: no $E2E_MAKEFILE_REL"; false; } |
| 168 | + |
| 169 | + # Counted with the enumeration brc_audit actually walks, not a second one: a |
| 170 | + # backstop that asks `ls` whether the `find` found anything answers about the |
| 171 | + # wrong list and would pass while the audit went vacuously green. |
| 172 | + files=$(brc_all_bats "$REPO_ROOT" | wc -l) |
| 173 | + [ "$files" -gt 0 ] || { echo "FAIL: found no .bats files to audit at all"; false; } |
| 174 | + |
| 175 | + # And the other premise: runner B naming nothing is precisely the #2826 shape, |
| 176 | + # so an empty extraction must fail rather than mark every e2e suite covered by |
| 177 | + # nothing or -- worse, once runner A is the only source -- reported as fine. |
| 178 | + named=$(brc_named_suites "$REPO_ROOT" | wc -l) |
| 179 | + [ "$named" -gt 0 ] || { echo "FAIL: $E2E_MAKEFILE_REL runs no bats suite at all -- either the e2e bootstrap was deleted or this extraction broke"; false; } |
| 180 | + |
| 181 | + offences=$(brc_audit "$REPO_ROOT") |
| 182 | + if [ -n "$offences" ]; then |
| 183 | + echo "FAIL: a bats file under hack/ is reachable by no runner." |
| 184 | + printf '%s\n' "$offences" |
| 185 | + echo "Runners: Makefile's bats-unit-tests (hack/*.bats, one level, non-e2e)" |
| 186 | + echo "and $E2E_MAKEFILE_REL (suites named literally)." |
| 187 | + echo "See docs/agents/e2e-testing.md." |
| 188 | + false |
| 189 | + fi |
| 190 | +} |
| 191 | + |
| 192 | +@test "the modelled runner selection still matches the Makefile" { |
| 193 | + # The one direction that would otherwise be silent. If someone narrows the |
| 194 | + # glob, brc_unit_reachable keeps calling files reachable that no longer are, |
| 195 | + # and the audit above stays green while coverage shrinks. Pinning the modelled |
| 196 | + # lines verbatim turns that into a failure naming the model rather than forty |
| 197 | + # phantom orphans. |
| 198 | + makefile=$(brc_code_lines < "$REPO_ROOT/Makefile") |
| 199 | + [ -n "$makefile" ] || { echo "FAIL: root Makefile extraction is empty"; false; } |
| 200 | + |
| 201 | + printf '%s\n' "$makefile" | grep -qF 'BATS_UNIT_FILES := $(filter-out hack/e2e-%.bats,$(wildcard hack/*.bats))' \ |
| 202 | + || { echo "FAIL: bats-unit-tests no longer selects files the way brc_unit_reachable models it (one level, hack/e2e-* excluded). Update the model in hack/bats-runner-coverage.bats in the same change."; false; } |
| 203 | + |
| 204 | + printf '%s\n' "$makefile" | grep -qF 'hack/cozytest.sh "$$f"' \ |
| 205 | + || { echo "FAIL: bats-unit-tests no longer runs its files through hack/cozytest.sh; the reachability model assumes it does."; false; } |
| 206 | + |
| 207 | + printf '%s\n' "$makefile" | grep -qE '^unit-tests:.*bats-unit-tests' \ |
| 208 | + || { echo "FAIL: unit-tests no longer depends on bats-unit-tests, so nothing in CI runs the bats sweep -- this guard included."; false; } |
| 209 | +} |
| 210 | + |
| 211 | +# A miniature root holding just what brc_audit reads: a hack/ tree and an e2e |
| 212 | +# Makefile. Fixtures are written with printf and never a heredoc, because |
| 213 | +# hack/cozytest.sh harvests any line starting with `@test "` into this suite -- |
| 214 | +# a heredoc of test source would become phantom tests here. |
| 215 | +brc_fixture() { |
| 216 | + _d=$(mktemp -d) |
| 217 | + mkdir -p "$_d/hack" "$_d/packages/core/testing" |
| 218 | + : > "$_d/$E2E_MAKEFILE_REL" |
| 219 | + echo "$_d" |
| 220 | +} |
| 221 | + |
| 222 | +brc_add_suite_file() { |
| 223 | + mkdir -p "$(dirname "$1/$2")" |
| 224 | + printf '#!/usr/bin/env bats\n' > "$1/$2" |
| 225 | +} |
| 226 | + |
| 227 | +brc_add_runner_line() { |
| 228 | + printf '%s:\n%sdocker exec "${SANDBOX_NAME}" sh -c '\''cd /workspace && hack/cozytest.sh %s'\''\n' \ |
| 229 | + "$(basename "$2" .bats)" "$TAB" "$2" >> "$1/$E2E_MAKEFILE_REL" |
| 230 | +} |
| 231 | + |
| 232 | +@test "a nested suite with no runner is reported" { |
| 233 | + d=$(brc_fixture) |
| 234 | + brc_add_suite_file "$d" hack/e2e-apps/orphan.bats |
| 235 | + brc_add_runner_line "$d" hack/e2e-install-cozystack.bats |
| 236 | + brc_add_suite_file "$d" hack/e2e-install-cozystack.bats |
| 237 | + |
| 238 | + out=$(brc_audit "$d") |
| 239 | + printf '%s\n' "$out" | grep -q 'hack/e2e-apps/orphan.bats: run by nothing' \ |
| 240 | + || { echo "FAIL: the audit did not report an unreachable nested suite. Got: $out"; rm -rf "$d"; false; } |
| 241 | + rm -rf "$d" |
| 242 | +} |
| 243 | + |
| 244 | +@test "a nested suite that IS named by a runner is not reported" { |
| 245 | + # The positive control. Without it every other assertion here is satisfied by |
| 246 | + # an audit that reports every file it sees. |
| 247 | + d=$(brc_fixture) |
| 248 | + brc_add_suite_file "$d" hack/e2e-apps/wired.bats |
| 249 | + brc_add_runner_line "$d" hack/e2e-apps/wired.bats |
| 250 | + |
| 251 | + out=$(brc_audit "$d") |
| 252 | + if printf '%s\n' "$out" | grep -q 'hack/e2e-apps/wired.bats'; then |
| 253 | + echo "FAIL: a nested suite with a runner line was reported anyway. Got: $out" |
| 254 | + rm -rf "$d" |
| 255 | + exit 1 |
| 256 | + fi |
| 257 | + rm -rf "$d" |
| 258 | +} |
| 259 | + |
| 260 | +@test "a top-level e2e- suite with no runner is reported" { |
| 261 | + # Runner A filters hack/e2e-* out, so a top-level e2e suite is only reachable |
| 262 | + # by being named. This class has no coverage at all today. |
| 263 | + d=$(brc_fixture) |
| 264 | + brc_add_suite_file "$d" hack/e2e-newthing.bats |
| 265 | + brc_add_runner_line "$d" hack/e2e-install-cozystack.bats |
| 266 | + brc_add_suite_file "$d" hack/e2e-install-cozystack.bats |
| 267 | + |
| 268 | + out=$(brc_audit "$d") |
| 269 | + printf '%s\n' "$out" | grep -q 'hack/e2e-newthing.bats: run by nothing' \ |
| 270 | + || { echo "FAIL: an unnamed top-level e2e- suite was not reported. Got: $out"; rm -rf "$d"; false; } |
| 271 | + rm -rf "$d" |
| 272 | +} |
| 273 | + |
| 274 | +@test "a plain top-level suite is reachable through the unit glob" { |
| 275 | + d=$(brc_fixture) |
| 276 | + brc_add_suite_file "$d" hack/plain.bats |
| 277 | + brc_add_runner_line "$d" hack/e2e-install-cozystack.bats |
| 278 | + brc_add_suite_file "$d" hack/e2e-install-cozystack.bats |
| 279 | + |
| 280 | + out=$(brc_audit "$d") |
| 281 | + if printf '%s\n' "$out" | grep -q 'hack/plain.bats'; then |
| 282 | + echo "FAIL: a plain top-level suite was reported as unreachable. Got: $out" |
| 283 | + rm -rf "$d" |
| 284 | + exit 1 |
| 285 | + fi |
| 286 | + rm -rf "$d" |
| 287 | +} |
| 288 | + |
| 289 | +@test "a parked .bats.disabled suite is invisible rather than an offence" { |
| 290 | + d=$(brc_fixture) |
| 291 | + brc_add_suite_file "$d" hack/e2e-apps/parked.bats |
| 292 | + mv "$d/hack/e2e-apps/parked.bats" "$d/hack/e2e-apps/parked.bats.disabled" |
| 293 | + brc_add_runner_line "$d" hack/e2e-install-cozystack.bats |
| 294 | + brc_add_suite_file "$d" hack/e2e-install-cozystack.bats |
| 295 | + |
| 296 | + out=$(brc_audit "$d") |
| 297 | + if printf '%s\n' "$out" | grep -q 'parked'; then |
| 298 | + echo "FAIL: a parked suite was reported. The .disabled suffix is the off-switch. Got: $out" |
| 299 | + rm -rf "$d" |
| 300 | + exit 1 |
| 301 | + fi |
| 302 | + rm -rf "$d" |
| 303 | +} |
| 304 | + |
| 305 | +@test "a commented-out runner line does not satisfy the contract" { |
| 306 | + d=$(brc_fixture) |
| 307 | + brc_add_suite_file "$d" hack/e2e-apps/orphan.bats |
| 308 | + printf '#%sdocker exec x sh -c "cd /workspace && hack/cozytest.sh hack/e2e-apps/orphan.bats"\n' "$TAB" \ |
| 309 | + >> "$d/$E2E_MAKEFILE_REL" |
| 310 | + brc_add_runner_line "$d" hack/e2e-install-cozystack.bats |
| 311 | + brc_add_suite_file "$d" hack/e2e-install-cozystack.bats |
| 312 | + |
| 313 | + out=$(brc_audit "$d") |
| 314 | + printf '%s\n' "$out" | grep -q 'hack/e2e-apps/orphan.bats: run by nothing' \ |
| 315 | + || { echo "FAIL: a commented-out runner line was accepted as coverage. Got: $out"; rm -rf "$d"; false; } |
| 316 | + rm -rf "$d" |
| 317 | +} |
| 318 | + |
| 319 | +@test "a named suite with no file on disk is reported" { |
| 320 | + d=$(brc_fixture) |
| 321 | + brc_add_runner_line "$d" hack/e2e-renamed-away.bats |
| 322 | + |
| 323 | + out=$(brc_audit "$d") |
| 324 | + printf '%s\n' "$out" | grep -q 'runs hack/e2e-renamed-away.bats, which does not exist on disk' \ |
| 325 | + || { echo "FAIL: a stale runner name was not reported. Got: $out"; rm -rf "$d"; false; } |
| 326 | + rm -rf "$d" |
| 327 | +} |
| 328 | + |
| 329 | +@test "a suite named through a make variable is reported as unresolvable" { |
| 330 | + d=$(brc_fixture) |
| 331 | + brc_add_suite_file "$d" hack/e2e-apps/orphan.bats |
| 332 | + printf 'test-apps-%%:\n%sdocker exec x sh -c "cd /workspace && hack/cozytest.sh hack/e2e-apps/$*.bats"\n' "$TAB" \ |
| 333 | + >> "$d/$E2E_MAKEFILE_REL" |
| 334 | + |
| 335 | + out=$(brc_audit "$d") |
| 336 | + printf '%s\n' "$out" | grep -q 'through a make variable' \ |
| 337 | + || { echo "FAIL: a pattern-rule runner was not reported as unresolvable. Got: $out"; rm -rf "$d"; false; } |
| 338 | + # And it must not be mistaken for coverage of the file it would have matched. |
| 339 | + printf '%s\n' "$out" | grep -q 'hack/e2e-apps/orphan.bats: run by nothing' \ |
| 340 | + || { echo "FAIL: a pattern-rule runner was treated as covering its would-be match. Got: $out"; rm -rf "$d"; false; } |
| 341 | + rm -rf "$d" |
| 342 | +} |
0 commit comments