|
| 1 | +#!/usr/bin/env bats |
| 2 | +# ----------------------------------------------------------------------------- |
| 3 | +# Every file this repository hands to /bin/sh must parse under /bin/sh. |
| 4 | +# |
| 5 | +# Why this is not already covered. On a contributor's workstation /bin/sh is |
| 6 | +# very often a symlink to bash -- it is on Arch, and on any distribution where |
| 7 | +# someone repointed it -- while on the ubuntu-* runners this project builds on |
| 8 | +# it is dash. A bash-only construct in a file whose shebang says sh therefore |
| 9 | +# runs clean locally and is a syntax error in CI, and the local run that was |
| 10 | +# meant to catch it is not a weaker version of CI but a different test. |
| 11 | +# |
| 12 | +# The failure is worse than one red test, because of how the two runners load a |
| 13 | +# suite. hack/cozytest.sh awk-translates a .bats file into shell functions and |
| 14 | +# DOT-SOURCES the result into its own `#!/bin/sh` process, so the whole file is |
| 15 | +# parsed in one go before any test runs: a single `<(...)` anywhere in it is a |
| 16 | +# parse error that kills every test in the file, not the one that contains it. |
| 17 | +# And because bats-unit-tests stops at the first failing FILE, every suite |
| 18 | +# alphabetically after it never runs either. One process substitution has cost |
| 19 | +# this project a whole unit lane. |
| 20 | +# |
| 21 | +# What this guard therefore does is the cheapest possible version of that load: |
| 22 | +# it parses each file with the shell that will execute it, and nothing else. No |
| 23 | +# test runs, no cluster is touched, and the whole sweep is under two seconds -- |
| 24 | +# so the class lands here, in the unit lane, instead of inside a bats step that |
| 25 | +# has already given up or an e2e job three hours in. |
| 26 | +# |
| 27 | +# The two surfaces it covers: |
| 28 | +# |
| 29 | +# *.sh executed by sh an sh shebang, or NO shebang at all -- the latter |
| 30 | +# because every such file in this tree is a |
| 31 | +# hack/e2e-chainsaw/_lib/ helper that a Chainsaw step |
| 32 | +# dot-sources, which means it inherits that step's sh |
| 33 | +# and has no shebang of its own to honour. |
| 34 | +# *.bats translated exactly the way cozytest.sh translates it, |
| 35 | +# because the translated text is what dash actually |
| 36 | +# parses. Checking the raw file would be checking a |
| 37 | +# document no shell ever sees: `@test "x" {` is not a |
| 38 | +# shell construct. |
| 39 | +# |
| 40 | +# Two surfaces it deliberately does NOT cover, so that nobody reads a green run |
| 41 | +# as more than it is. Makefile recipes also run under /bin/sh -- make's default |
| 42 | +# SHELL, and this project sets none -- but a recipe line is full of `$(VAR)` |
| 43 | +# that a shell parser reads as command substitution, so a real parse there |
| 44 | +# yields confident nonsense. Chainsaw `script:` blocks run under `sh -c` and are |
| 45 | +# extractable, but only with a YAML parser this suite would then depend on. |
| 46 | +# Both are worth adding; neither is worth faking. |
| 47 | +# |
| 48 | +# The limit that matters most: this is a SYNTAX check, and a bashism dash parses |
| 49 | +# happily but runs differently -- `echo -e`, `source`, `set -o pipefail`, |
| 50 | +# `$HOSTNAME`, `$RANDOM` -- passes it. Catching those needs checkbashisms, which |
| 51 | +# is not installed on the runners and would mean a new dependency in this lane |
| 52 | +# for a class that has cost less than the syntax class has. `dash -n` is the |
| 53 | +# 90% that is free. |
| 54 | +# ----------------------------------------------------------------------------- |
| 55 | + |
| 56 | +# The directory to audit. Under the bats binary that is this file's own |
| 57 | +# location; under cozytest.sh, which sets no BATS_TEST_FILENAME, `$0` is the |
| 58 | +# runner -- and the answer is the same only because the runner lives in the |
| 59 | +# directory it runs files from. The repo root is one level up. The first test |
| 60 | +# below refuses to pass on an empty enumeration, so a wrong answer here fails |
| 61 | +# rather than audits nothing. |
| 62 | +SPP_HACK="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")" && pwd)" |
| 63 | +SPP_ROOT="$(cd "$SPP_HACK/.." && pwd)" |
| 64 | + |
| 65 | +# The parser to check with, and the honesty about which one it is. |
| 66 | +# |
| 67 | +# `dash` by name when it is installed, because that is what the runners execute |
| 68 | +# and naming it removes all doubt. Otherwise `sh`, which IS dash on every |
| 69 | +# ubuntu runner this project uses -- and is bash on a workstation, where the |
| 70 | +# check then degrades to catching only what bash also rejects. That degradation |
| 71 | +# is announced rather than hidden: a contributor whose sh is bash should not |
| 72 | +# read a green line here as a promise about CI. |
| 73 | +if command -v dash >/dev/null 2>&1; then |
| 74 | + SPP_SH=dash |
| 75 | +else |
| 76 | + SPP_SH=sh |
| 77 | +fi |
| 78 | + |
| 79 | +# Reproduce cozytest.sh's translation. Kept identical to the awk program there, |
| 80 | +# including its blunt rule that a line which is EXACTLY `}` closes a test -- a |
| 81 | +# nested brace at column zero is rewritten too. That is the runner's behaviour, |
| 82 | +# and a checker that quietly did something smarter would be vouching for text |
| 83 | +# CI never parses. |
| 84 | +spp_translate() { |
| 85 | + awk ' |
| 86 | + /^@test[[:space:]]+"/ { |
| 87 | + line = substr($0, index($0, "\"") + 1) |
| 88 | + title = substr(line, 1, index(line, "\"") - 1) |
| 89 | + fname = "test_" |
| 90 | + for (i = 1; i <= length(title); i++) { |
| 91 | + c = substr(title, i, 1) |
| 92 | + fname = fname (c ~ /[A-Za-z0-9]/ ? c : "_") |
| 93 | + } |
| 94 | + printf("%s() {\n", fname) |
| 95 | + print " set -e" |
| 96 | + next |
| 97 | + } |
| 98 | + /^}$/ { |
| 99 | + print " return 0" |
| 100 | + print "}" |
| 101 | + next |
| 102 | + } |
| 103 | + { print } |
| 104 | + ' "$1" |
| 105 | + return 0 |
| 106 | +} |
| 107 | + |
| 108 | +# Does this .sh file get executed by sh? An sh shebang, or none at all. A bash, |
| 109 | +# zsh or ksh shebang means the file is not this guard's business, and saying so |
| 110 | +# by shebang is the documented way to opt out. |
| 111 | +spp_is_sh_script() { |
| 112 | + _first=$(head -1 "$1") |
| 113 | + case "$_first" in |
| 114 | + '#!'*bash*|'#!'*zsh*|'#!'*ksh*) return 1 ;; |
| 115 | + '#!'*sh*) return 0 ;; |
| 116 | + '#!'*) return 1 ;; |
| 117 | + *) return 0 ;; |
| 118 | + esac |
| 119 | +} |
| 120 | + |
| 121 | +# Report every file under $1 that its own shell cannot parse, one line each, and |
| 122 | +# print nothing when they all parse. |
| 123 | +# |
| 124 | +# Through stdout rather than an exit status, for the reason hack/bats-no-exit-trap.bats |
| 125 | +# records: cozytest.sh rewrites a function's closing brace into `return 0`, so a |
| 126 | +# status set in here is discarded before the caller can read it and the check |
| 127 | +# would pass unconditionally. |
| 128 | +# |
| 129 | +# Vendored upstream charts are excluded. `make update` regenerates them verbatim |
| 130 | +# and this project does not own their prose or their scripts; a finding there is |
| 131 | +# not actionable in this tree. |
| 132 | +spp_audit() { |
| 133 | + _tmp=$(mktemp) || return 0 |
| 134 | + find "$1" -name charts -prune -o -name node_modules -prune -o -type f \ |
| 135 | + \( -name '*.sh' -o -name '*.bats' \) -print | sort | while IFS= read -r _f; do |
| 136 | + [ -e "$_f" ] || continue |
| 137 | + _rel=${_f#"$1"/} |
| 138 | + case "$_f" in |
| 139 | + *.bats) |
| 140 | + spp_translate "$_f" > "$_tmp" |
| 141 | + _out=$("$SPP_SH" -n "$_tmp" 2>&1) || \ |
| 142 | + echo "$_rel: $SPP_SH rejects the translated file: $_out" |
| 143 | + ;; |
| 144 | + *) |
| 145 | + spp_is_sh_script "$_f" || continue |
| 146 | + _out=$("$SPP_SH" -n "$_f" 2>&1) || \ |
| 147 | + echo "$_rel: $SPP_SH rejects it: $_out" |
| 148 | + ;; |
| 149 | + esac |
| 150 | + done |
| 151 | + rm -f "$_tmp" |
| 152 | + return 0 |
| 153 | +} |
| 154 | + |
| 155 | +@test "every file this repo hands to sh parses under sh" { |
| 156 | + # The whole point of the guard, run against the tree it governs. Everything |
| 157 | + # below exists to prove this test can fail. |
| 158 | + # |
| 159 | + # Counted with the enumeration spp_audit actually walks, not a second one: a |
| 160 | + # backstop that asks a different question can pass while the audit goes |
| 161 | + # vacuously green, which is the failure mode this file is about. |
| 162 | + files=$(find "$SPP_ROOT" -name charts -prune -o -name node_modules -prune \ |
| 163 | + -o -type f \( -name '*.sh' -o -name '*.bats' \) -print | wc -l) |
| 164 | + [ "$files" -gt 0 ] || { echo "FAIL: found no sh surfaces to audit at all"; false; } |
| 165 | + |
| 166 | + echo "parsing $files candidate file(s) with $SPP_SH" |
| 167 | + if [ "$SPP_SH" = sh ]; then |
| 168 | + # Announced, not hidden: on a workstation where /bin/sh is bash this check |
| 169 | + # verifies bash syntax and proves nothing about the runners. |
| 170 | + echo "note: checking with /bin/sh; where that is bash this is a DEGRADED check" |
| 171 | + fi |
| 172 | + |
| 173 | + offences=$(spp_audit "$SPP_ROOT") |
| 174 | + if [ -n "$offences" ]; then |
| 175 | + echo "FAIL: files executed by /bin/sh that /bin/sh cannot parse." |
| 176 | + printf '%s\n' "$offences" |
| 177 | + echo "On the runners /bin/sh is dash. Either write the construct in POSIX" |
| 178 | + echo "form, or -- if the script never has to run where bash is absent --" |
| 179 | + echo "give it a '#!/usr/bin/env bash' shebang and say so honestly." |
| 180 | + false |
| 181 | + fi |
| 182 | +} |
| 183 | + |
| 184 | +@test "a process substitution in a bats file is reported" { |
| 185 | + # The construct that has actually cost this project a unit lane. It is a |
| 186 | + # syntax error under dash, and because cozytest.sh sources the whole |
| 187 | + # translated file, it takes every test in that file down with it. |
| 188 | + tmp=$(mktemp -d) |
| 189 | + printf '%s\n' '@test "two streams" {' ' diff <(echo a) <(echo b)' '}' \ |
| 190 | + > "$tmp/subject.bats" |
| 191 | + |
| 192 | + out=$(spp_audit "$tmp") |
| 193 | + case "$out" in |
| 194 | + *"subject.bats: $SPP_SH rejects the translated file"*) ;; |
| 195 | + *) echo "FAIL: a process substitution in a bats file went unreported: $out"; false ;; |
| 196 | + esac |
| 197 | + rm -rf "$tmp" |
| 198 | +} |
| 199 | + |
| 200 | +@test "an unparseable sh script is reported" { |
| 201 | + tmp=$(mktemp -d) |
| 202 | + printf '%s\n' '#!/bin/sh' 'if [ 1 -eq 1 ]; then' 'echo unterminated' > "$tmp/subject.sh" |
| 203 | + |
| 204 | + out=$(spp_audit "$tmp") |
| 205 | + case "$out" in |
| 206 | + *"subject.sh: $SPP_SH rejects it"*) ;; |
| 207 | + *) echo "FAIL: an unterminated if went unreported: $out"; false ;; |
| 208 | + esac |
| 209 | + rm -rf "$tmp" |
| 210 | +} |
| 211 | + |
| 212 | +@test "a bash-only construct in a bash-shebanged script is not reported" { |
| 213 | + # Declaring bash is the documented way out, and a guard that refused it would |
| 214 | + # be demanding POSIX from files that never promised any. |
| 215 | + tmp=$(mktemp -d) |
| 216 | + printf '%s\n' '#!/usr/bin/env bash' 'if [[ -n "$1" ]]; then :; fi' > "$tmp/subject.sh" |
| 217 | + |
| 218 | + out=$(spp_audit "$tmp") |
| 219 | + [ -z "$out" ] || { echo "FAIL: a declared-bash script was audited anyway: $out"; false; } |
| 220 | + rm -rf "$tmp" |
| 221 | +} |
| 222 | + |
| 223 | +@test "a script with no shebang is audited as sh" { |
| 224 | + # Every no-shebang .sh in this tree is a hack/e2e-chainsaw/_lib/ helper that a |
| 225 | + # Chainsaw step dot-sources, so it inherits that step's sh. Skipping them |
| 226 | + # would leave the largest group of them unchecked. |
| 227 | + tmp=$(mktemp -d) |
| 228 | + printf '%s\n' 'diff <(echo a) <(echo b)' > "$tmp/subject.sh" |
| 229 | + |
| 230 | + out=$(spp_audit "$tmp") |
| 231 | + case "$out" in |
| 232 | + *"subject.sh: $SPP_SH rejects it"*) ;; |
| 233 | + *) echo "FAIL: a shebangless helper went unaudited: $out"; false ;; |
| 234 | + esac |
| 235 | + rm -rf "$tmp" |
| 236 | +} |
| 237 | + |
| 238 | +@test "a clean POSIX script and a clean bats file are accepted" { |
| 239 | + # The other direction: a guard that fires on correct code gets switched off, |
| 240 | + # and then it guards nothing. `local` in particular is fine -- dash has |
| 241 | + # supported it for decades, and several helpers in this tree rely on it. |
| 242 | + tmp=$(mktemp -d) |
| 243 | + printf '%s\n' '#!/bin/sh' 'set -eu' 'f() { local x="$1"; printf "%s\n" "$x"; }' 'f hi' \ |
| 244 | + > "$tmp/subject.sh" |
| 245 | + printf '%s\n' '@test "plain" {' ' [ 1 -eq 1 ]' '}' > "$tmp/subject.bats" |
| 246 | + |
| 247 | + out=$(spp_audit "$tmp") |
| 248 | + [ -z "$out" ] || { echo "FAIL: clean files were reported: $out"; false; } |
| 249 | + rm -rf "$tmp" |
| 250 | +} |
| 251 | + |
| 252 | +@test "a nested file is audited too and keeps its path" { |
| 253 | + # hack/e2e-apps/ and hack/e2e-chainsaw/_lib/ both hold surfaces a flat glob |
| 254 | + # would miss, and two files sharing a basename must not report under the same |
| 255 | + # name. |
| 256 | + tmp=$(mktemp -d) |
| 257 | + mkdir -p "$tmp/nested" |
| 258 | + printf '%s\n' '#!/bin/sh' 'case x in' > "$tmp/nested/deep.sh" |
| 259 | + |
| 260 | + out=$(spp_audit "$tmp") |
| 261 | + case "$out" in |
| 262 | + *"nested/deep.sh: $SPP_SH rejects it"*) ;; |
| 263 | + *) echo "FAIL: a nested file went unaudited or lost its path: $out"; false ;; |
| 264 | + esac |
| 265 | + rm -rf "$tmp" |
| 266 | +} |
| 267 | + |
| 268 | +@test "a vendored chart script is not audited" { |
| 269 | + # `make update` regenerates packages/*/*/charts/ verbatim, so a finding in |
| 270 | + # there is not actionable in this tree. |
| 271 | + tmp=$(mktemp -d) |
| 272 | + mkdir -p "$tmp/charts" |
| 273 | + printf '%s\n' '#!/bin/sh' 'case x in' > "$tmp/charts/vendored.sh" |
| 274 | + |
| 275 | + out=$(spp_audit "$tmp") |
| 276 | + [ -z "$out" ] || { echo "FAIL: a vendored chart script was audited: $out"; false; } |
| 277 | + rm -rf "$tmp" |
| 278 | +} |
| 279 | + |
| 280 | +@test "the translation matches the one cozytest.sh performs" { |
| 281 | + # This guard parses translated text, so it is only as good as its agreement |
| 282 | + # with the runner's own awk. Rather than compare the two programs, compare |
| 283 | + # their output: run the real runner's translation path over a fixture and |
| 284 | + # diff. If cozytest.sh's translation changes, this fails loudly instead of the |
| 285 | + # guard silently vouching for text CI no longer parses. |
| 286 | + tmp=$(mktemp -d) |
| 287 | + printf '%s\n' '@test "a title with spaces & punctuation" {' \ |
| 288 | + ' x=1' \ |
| 289 | + ' if [ "$x" = 1 ]; then' \ |
| 290 | + ' echo yes' \ |
| 291 | + ' fi' \ |
| 292 | + '}' > "$tmp/subject.bats" |
| 293 | + |
| 294 | + spp_translate "$tmp/subject.bats" > "$tmp/mine" |
| 295 | + |
| 296 | + # The runner's own translation, extracted from the runner rather than |
| 297 | + # reimplemented here: everything between its awk quotes. The closing line is |
| 298 | + # matched by its prefix and NOT anchored at the end -- in cozytest.sh it |
| 299 | + # continues `> "$TMP_SH"`, and an end-anchored pattern silently fails to close |
| 300 | + # the range, leaving a line of shell in the extracted awk program. |
| 301 | + sed -n "/^awk '\$/,/^' \"\\\$TEST_FILE\"/p" "$SPP_HACK/cozytest.sh" \ |
| 302 | + | sed '1d;$d' > "$tmp/prog.awk" |
| 303 | + [ -s "$tmp/prog.awk" ] || { echo "FAIL: could not extract cozytest.sh's translator"; false; } |
| 304 | + awk -f "$tmp/prog.awk" "$tmp/subject.bats" | grep -v '^### ' > "$tmp/theirs" |
| 305 | + |
| 306 | + # cozytest.sh emits a `### title` marker line this guard has no use for; drop |
| 307 | + # it from their side rather than emitting one here, since the marker is the |
| 308 | + # runner's own bookkeeping and not part of what dash parses. |
| 309 | + grep -v '^### ' "$tmp/mine" > "$tmp/mine.cmp" |
| 310 | + diff -u "$tmp/theirs" "$tmp/mine.cmp" || { |
| 311 | + echo "FAIL: this guard's translation has drifted from cozytest.sh's" |
| 312 | + false |
| 313 | + } |
| 314 | + rm -rf "$tmp" |
| 315 | +} |
0 commit comments