|
| 1 | +#!/usr/bin/env bats |
| 2 | +# ----------------------------------------------------------------------------- |
| 3 | +# Every hack/*.bats unit file loads hack/test_helper.bash, and this is what |
| 4 | +# makes that true of the file someone adds next month. |
| 5 | +# |
| 6 | +# WHAT THE HELPER IS FOR. Bats enforces `set -e` on a test body but not `set -u`, |
| 7 | +# while hack/cozytest.sh -- the runner this suite used before #3453 -- ran every |
| 8 | +# body under `set -eu -x`. Without the helper a test that reads an unset variable |
| 9 | +# sees an empty string and passes, which is a test asserting nothing while |
| 10 | +# reporting `ok`. |
| 11 | +# |
| 12 | +# WHY IT IS A PER-FILE LINE. Because bats has no other hook. Its default |
| 13 | +# `setup()` is defined in bats-core's own test_functions.bash and sourced BEFORE |
| 14 | +# the test file, so nothing outside the file can replace it. `setup_suite` -- |
| 15 | +# whether from --setup-suite-file or an auto-discovered setup_suite.bash -- runs |
| 16 | +# once in bats-exec-suite's process and never in the process that runs a test. |
| 17 | +# And the two environment routes that do cross a process boundary, SHELLOPTS and |
| 18 | +# BASH_ENV, are exported by definition, so they reach the hack/*.sh scripts the |
| 19 | +# tests exercise as subprocesses and change the code under test rather than the |
| 20 | +# tests. `load` is the only mechanism there is. |
| 21 | +# |
| 22 | +# WHY IT IS AUDITED RATHER THAN REMEMBERED. The helper landed covering 32 of 32 |
| 23 | +# unit files and was found later covering 32 of 63 -- not because anyone removed |
| 24 | +# a line, but because 27 files were added after it and nothing anywhere held the |
| 25 | +# list of files that were supposed to have one. A requirement whose only record |
| 26 | +# is the diff that introduced it decays at exactly the rate the tree grows, and |
| 27 | +# it decays silently: an uncovered file still runs, still passes, and its tests |
| 28 | +# are simply weaker than they read. |
| 29 | +# |
| 30 | +# So the record is this file, and its input is the tree rather than a list. It |
| 31 | +# enumerates hack/*.bats minus hack/e2e-*.bats -- the same set the Makefile |
| 32 | +# hands the runner, and one of the tests below asserts that the two agree by |
| 33 | +# asking make for its own answer rather than trusting this file's copy of the |
| 34 | +# rule. A new unit file is in the audit's input from the moment it exists, so |
| 35 | +# the gap closes at the commit that opens it. There is nothing to update, which |
| 36 | +# is the property the first version lacked: it required an author to remember, |
| 37 | +# and the mechanism that requires memory is the mechanism that drifts. |
| 38 | +# |
| 39 | +# THE SECOND WAY THE STRICTNESS GOES MISSING is a file that loads the helper and |
| 40 | +# then defines its own `setup()`, which silently replaces the helper's. That is |
| 41 | +# checked too: a file-local `setup()` must call `strict_setup`. No file in the |
| 42 | +# tree defines one today, which is precisely when a guard is cheap to add. |
| 43 | +# |
| 44 | +# LIMITS, stated rather than discovered. The scan is lexical and anchored at |
| 45 | +# column zero. A `load` indented inside a function body would not run at source |
| 46 | +# time and is correctly not credited, but neither is a legitimate one hidden |
| 47 | +# behind a conditional -- the audit asks for the one spelling the suite uses, |
| 48 | +# and a file with a reason to differ has to change this guard rather than work |
| 49 | +# around it. In the other direction the line is credited wherever it appears at |
| 50 | +# column zero, including inside a heredoc that writes a fixture .bats, which |
| 51 | +# would let a file take credit for text it only generates; no file does that |
| 52 | +# today, and the fixtures below assemble the line from printf arguments so this |
| 53 | +# guard does not do it to itself. Finally, a green audit says the load is |
| 54 | +# present, not that `set -u` was in force for a given assertion -- for that, |
| 55 | +# hack/test_helper.bash's own effect is what the mutation check in #3453 |
| 56 | +# covered: remove the load and a test reading an unset variable stops aborting. |
| 57 | +# ----------------------------------------------------------------------------- |
| 58 | + |
| 59 | +load test_helper |
| 60 | + |
| 61 | +# The directory to audit. Under the bats binary that is this file's own |
| 62 | +# location; under cozytest.sh, which does not set BATS_TEST_FILENAME, `$0` is |
| 63 | +# the runner itself -- and the answer comes out the same only because the runner |
| 64 | +# lives in the directory it runs files from. The first test below refuses to |
| 65 | +# pass on an empty enumeration, so a wrong answer here fails rather than audits |
| 66 | +# nothing. |
| 67 | +BSS_DIR="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")" && pwd)" |
| 68 | +BSS_REPO_ROOT="$(cd "$BSS_DIR/.." && pwd)" |
| 69 | + |
| 70 | +# The unit files: hack/*.bats that are not hack/e2e-*.bats. |
| 71 | +# |
| 72 | +# Non-recursive, and the exclusion is on the basename, because that is what |
| 73 | +# `$(filter-out hack/e2e-%.bats,$(wildcard hack/*.bats))` in the Makefile means. |
| 74 | +# hack/e2e-apps/ holds live-cluster suites the Makefile's wildcard never reaches |
| 75 | +# and that packages/core/testing runs against a real cluster; they are outside |
| 76 | +# this suite and outside this audit. |
| 77 | +bss_units() { |
| 78 | + find "$1" -maxdepth 1 -name '*.bats' ! -name 'e2e-*' | sort |
| 79 | + return 0 |
| 80 | +} |
| 81 | + |
| 82 | +# Report every unit file whose strict-mode setup is missing or overridden, one |
| 83 | +# line each, and print nothing when they all comply. |
| 84 | +# |
| 85 | +# Through stdout rather than an exit status: cozytest.sh rewrites a function's |
| 86 | +# closing brace into `return 0`, so a status set in here would be discarded |
| 87 | +# before the caller could read it, and the check would pass unconditionally. |
| 88 | +bss_audit() { |
| 89 | + bss_units "$1" | while IFS= read -r _f; do |
| 90 | + [ -e "$_f" ] || continue |
| 91 | + _b=$(basename "$_f") |
| 92 | + if ! grep -q '^load test_helper$' "$_f"; then |
| 93 | + echo "$_b: does not load the strict-mode helper; add a column-zero \`load test_helper\` line under the file's leading comment block" |
| 94 | + continue |
| 95 | + fi |
| 96 | + # A file-local setup() replaces the helper's without a word from either |
| 97 | + # runner. Match the two spellings bash accepts for a definition at column |
| 98 | + # zero; one indented inside another function is not a definition bats will |
| 99 | + # call, and is not what this looks for. |
| 100 | + if grep -qE '^(function[[:space:]]+)?setup[[:space:]]*\(\)' "$_f" \ |
| 101 | + && ! grep -q 'strict_setup' "$_f"; then |
| 102 | + echo "$_b: defines its own setup() and never calls strict_setup, which drops the \`set -u\` the helper exists to restore" |
| 103 | + fi |
| 104 | + done |
| 105 | + return 0 |
| 106 | +} |
| 107 | + |
| 108 | +# Fixture writers. The load line is assembled from a printf argument rather than |
| 109 | +# written as a literal, because this file is audited by the guard it defines and |
| 110 | +# a column-zero literal here would be read as a second declaration of its own. |
| 111 | +bss_new_fixture() { |
| 112 | + printf '%s\n' '#!/usr/bin/env bats' > "$1/subject.bats" |
| 113 | + return 0 |
| 114 | +} |
| 115 | + |
| 116 | +bss_add_load() { |
| 117 | + printf '%s\n' 'load test_helper' >> "$1/subject.bats" |
| 118 | + return 0 |
| 119 | +} |
| 120 | + |
| 121 | +bss_add_indented_load() { |
| 122 | + printf ' %s\n' 'load test_helper' >> "$1/subject.bats" |
| 123 | + return 0 |
| 124 | +} |
| 125 | + |
| 126 | +bss_add_commented_load() { |
| 127 | + printf '# %s\n' 'load test_helper' >> "$1/subject.bats" |
| 128 | + return 0 |
| 129 | +} |
| 130 | + |
| 131 | +bss_add_own_setup() { |
| 132 | + printf '%s\n' 'setup() {' ' export FIXTURE=1' '}' >> "$1/subject.bats" |
| 133 | + return 0 |
| 134 | +} |
| 135 | + |
| 136 | +bss_add_strict_own_setup() { |
| 137 | + printf '%s\n' 'setup() {' ' strict_setup' ' export FIXTURE=1' '}' >> "$1/subject.bats" |
| 138 | + return 0 |
| 139 | +} |
| 140 | + |
| 141 | +bss_rename() { |
| 142 | + mv "$1/subject.bats" "$1/$2" |
| 143 | + return 0 |
| 144 | +} |
| 145 | + |
| 146 | +@test "the unit enumeration is not empty" { |
| 147 | + count=$(bss_units "$BSS_DIR" | wc -l) |
| 148 | + if [ "$count" -lt 2 ]; then |
| 149 | + echo "FAIL: enumerated $count unit file(s) under $BSS_DIR; the audit below would be vacuous" |
| 150 | + false |
| 151 | + fi |
| 152 | +} |
| 153 | + |
| 154 | +@test "the audited set is exactly the set the Makefile hands the runner" { |
| 155 | + # Ask make for its own answer instead of trusting this file's copy of the |
| 156 | + # rule. The whole point of the audit is that its input tracks the runner's, |
| 157 | + # so a divergence between the two is the failure it exists to prevent -- |
| 158 | + # narrow the Makefile's filter without narrowing this one and the audit goes |
| 159 | + # on reporting clean about files nobody runs, or vice versa. |
| 160 | + tmp=$(mktemp -d) |
| 161 | + bss_units "$BSS_DIR" | sed "s|^$BSS_REPO_ROOT/||" | sort > "$tmp/mine" |
| 162 | + # MAKEFLAGS cleared: this runs from inside `make bats-unit-tests`, and a |
| 163 | + # sub-make that inherits the parent's `-j` without its jobserver descriptors |
| 164 | + # warns about it. The warning goes to stderr and would not corrupt the |
| 165 | + # comparison, but it would land in the middle of a green suite's output. |
| 166 | + (cd "$BSS_REPO_ROOT" && MAKEFLAGS= MAKELEVEL= make --no-print-directory -s print-bats-unit-files) | sort > "$tmp/theirs" |
| 167 | + # Compared through files rather than process substitution: this file also runs |
| 168 | + # under hack/cozytest.sh, whose #!/bin/sh is dash on the CI runner. |
| 169 | + if ! diff -u "$tmp/theirs" "$tmp/mine" > "$tmp/delta"; then |
| 170 | + echo "FAIL: the audited set and \$(BATS_UNIT_FILES) disagree (-Makefile +audit):" |
| 171 | + cat "$tmp/delta" |
| 172 | + rm -rf "$tmp" |
| 173 | + false |
| 174 | + fi |
| 175 | + rm -rf "$tmp" |
| 176 | +} |
| 177 | + |
| 178 | +@test "every hack/*.bats unit file restores set -u through the shared helper" { |
| 179 | + report=$(bss_audit "$BSS_DIR") |
| 180 | + if [ -n "$report" ]; then |
| 181 | + echo "FAIL: strict-mode setup missing or overridden:" |
| 182 | + echo "$report" |
| 183 | + false |
| 184 | + fi |
| 185 | +} |
| 186 | + |
| 187 | +@test "a unit file with no load line is reported" { |
| 188 | + tmp=$(mktemp -d) |
| 189 | + bss_new_fixture "$tmp" |
| 190 | + report=$(bss_audit "$tmp") |
| 191 | + if ! echo "$report" | grep -q 'does not load the strict-mode helper'; then |
| 192 | + echo "FAIL: an uncovered file was not reported; got: $report" |
| 193 | + rm -rf "$tmp" |
| 194 | + false |
| 195 | + fi |
| 196 | + rm -rf "$tmp" |
| 197 | +} |
| 198 | + |
| 199 | +@test "a unit file with the load line is clean" { |
| 200 | + tmp=$(mktemp -d) |
| 201 | + bss_new_fixture "$tmp" |
| 202 | + bss_add_load "$tmp" |
| 203 | + report=$(bss_audit "$tmp") |
| 204 | + if [ -n "$report" ]; then |
| 205 | + echo "FAIL: a covered file was reported: $report" |
| 206 | + rm -rf "$tmp" |
| 207 | + false |
| 208 | + fi |
| 209 | + rm -rf "$tmp" |
| 210 | +} |
| 211 | + |
| 212 | +@test "an indented or commented-out load line does not count" { |
| 213 | + tmp=$(mktemp -d) |
| 214 | + bss_new_fixture "$tmp" |
| 215 | + bss_add_indented_load "$tmp" |
| 216 | + bss_add_commented_load "$tmp" |
| 217 | + report=$(bss_audit "$tmp") |
| 218 | + if ! echo "$report" | grep -q 'does not load the strict-mode helper'; then |
| 219 | + echo "FAIL: a load line that never runs was credited; got: $report" |
| 220 | + rm -rf "$tmp" |
| 221 | + false |
| 222 | + fi |
| 223 | + rm -rf "$tmp" |
| 224 | +} |
| 225 | + |
| 226 | +@test "a file-local setup that skips strict_setup is reported" { |
| 227 | + tmp=$(mktemp -d) |
| 228 | + bss_new_fixture "$tmp" |
| 229 | + bss_add_load "$tmp" |
| 230 | + bss_add_own_setup "$tmp" |
| 231 | + report=$(bss_audit "$tmp") |
| 232 | + if ! echo "$report" | grep -q 'never calls strict_setup'; then |
| 233 | + echo "FAIL: a setup() override was not reported; got: $report" |
| 234 | + rm -rf "$tmp" |
| 235 | + false |
| 236 | + fi |
| 237 | + rm -rf "$tmp" |
| 238 | +} |
| 239 | + |
| 240 | +@test "a file-local setup that calls strict_setup is clean" { |
| 241 | + tmp=$(mktemp -d) |
| 242 | + bss_new_fixture "$tmp" |
| 243 | + bss_add_load "$tmp" |
| 244 | + bss_add_strict_own_setup "$tmp" |
| 245 | + report=$(bss_audit "$tmp") |
| 246 | + if [ -n "$report" ]; then |
| 247 | + echo "FAIL: a compliant setup() override was reported: $report" |
| 248 | + rm -rf "$tmp" |
| 249 | + false |
| 250 | + fi |
| 251 | + rm -rf "$tmp" |
| 252 | +} |
| 253 | + |
| 254 | +@test "an e2e- file is outside the audit" { |
| 255 | + tmp=$(mktemp -d) |
| 256 | + bss_new_fixture "$tmp" |
| 257 | + bss_rename "$tmp" "e2e-subject.bats" |
| 258 | + report=$(bss_audit "$tmp") |
| 259 | + if [ -n "$report" ]; then |
| 260 | + echo "FAIL: a live-cluster suite was audited as a unit file: $report" |
| 261 | + rm -rf "$tmp" |
| 262 | + false |
| 263 | + fi |
| 264 | + rm -rf "$tmp" |
| 265 | +} |
0 commit comments