|
56 | 56 | rm -rf "$tmp" |
57 | 57 | } |
58 | 58 |
|
59 | | -# Every skipped chart has to carry a reason, so a skip cannot quietly become the |
60 | | -# way a chart avoids the check. |
61 | | -@test "every skip names a reason" { |
62 | | - output=$(hack/check-render-matrix.sh) |
63 | | - printf '%s\n' "$output" | grep '^SKIP' > /tmp/render-skips.$$ || true |
64 | | - while read -r line; do |
65 | | - [ -n "$line" ] || continue |
66 | | - case "$line" in |
67 | | - *"("*")"*) ;; |
68 | | - *) echo "skip without a reason: $line" >&2; rm -f /tmp/render-skips.$$; exit 1 ;; |
69 | | - esac |
70 | | - done < /tmp/render-skips.$$ |
71 | | - rm -f /tmp/render-skips.$$ |
| 59 | +# A skip that has stopped being necessary is worse than no skip: it silently |
| 60 | +# drops a chart from the sweep forever. This renders each skipped chart directly |
| 61 | +# and requires it to STILL fail, so a chart that becomes renderable shows up as a |
| 62 | +# red test rather than as permanent invisible exclusion. |
| 63 | +# |
| 64 | +# Replaces an earlier test that asserted every SKIP line contains parentheses. |
| 65 | +# The only line that emits one is `echo "SKIP $name ($reason)"` guarded on a |
| 66 | +# non-empty reason, so that assertion was unreachable by construction: it could |
| 67 | +# not fail for the property it named. |
| 68 | +@test "every skipped chart still genuinely fails to render" { |
| 69 | + # mktemp, not $BATS_TMPDIR: cozytest.sh is not real bats and defines none of |
| 70 | + # bats's variables, so under `set -u` that name aborts the suite. |
| 71 | + skips=$(mktemp) |
| 72 | + hack/check-render-matrix.sh | grep '^SKIP' | sed 's/^SKIP \([^ ]*\) .*/\1/' > "$skips" || true |
| 73 | + while read -r name; do |
| 74 | + [ -n "$name" ] || continue |
| 75 | + if helm template "$name-render-check" "packages/apps/$name" -n tenant-test \ |
| 76 | + -f hack/testdata/render-fixtures/fresh.yaml >/dev/null 2>&1; then |
| 77 | + echo "chart '$name' is on the skip list but renders fine now — remove the skip" >&2 |
| 78 | + rm -f "$skips" |
| 79 | + exit 1 |
| 80 | + fi |
| 81 | + done < "$skips" |
| 82 | + rm -f "$skips" |
72 | 83 | } |
73 | 84 |
|
74 | 85 | # The sweep must cover the charts that exist rather than a number frozen when |
|
92 | 103 | # `eq $x "true"`, so a real YAML bool makes helm fail on incompatible types. |
93 | 104 | @test "fixtures carry the platform's string-typed values" { |
94 | 105 | for f in hack/testdata/render-fixtures/*.yaml; do |
95 | | - python3 -c 'import sys,yaml; d=yaml.safe_load(open(sys.argv[1])) or {}; c=d.get("_cluster") or {}; bad=[k for k,v in c.items() if isinstance(v,bool)]; sys.exit(f"{sys.argv[1]}: _cluster keys are YAML booleans, the platform injects strings: {bad}") if bad else None; sys.exit(f"{sys.argv[1]}: no _cluster map") if not c else None' "$f" |
| 106 | + python3 -c 'import sys,yaml;d=yaml.safe_load(open(sys.argv[1])) or {};bad=[];[bad.append(f"{sec}.{k}") for sec in ("_cluster","_namespace") for k,v in (d.get(sec) or {}).items() if isinstance(v,(bool,int,float)) or v is None];sys.exit(f"{sys.argv[1]}: scalars must be strings, the platform quotes them: {bad}") if bad else None;sys.exit(f"{sys.argv[1]}: no _cluster map") if not (d.get("_cluster") or {}) else None' "$f" |
96 | 107 | done |
97 | 108 | } |
98 | 109 |
|
99 | | -# Each fixture is one cluster state and they must actually differ, or the sweep |
100 | | -# renders the same shape three times and reports triple the coverage it has. |
101 | | -@test "the fixtures represent different cluster states" { |
102 | | - count=$(find hack/testdata/render-fixtures -name '*.yaml' | wc -l | tr -d ' ') |
| 110 | +# Each fixture is one cluster state, and what matters is that they produce |
| 111 | +# DIFFERENT RENDERS -- otherwise the sweep runs helm three times over the same |
| 112 | +# shape and reports triple the coverage it has. |
| 113 | +# |
| 114 | +# An earlier version compared the fixture FILES on three key names. That passed |
| 115 | +# while the sweep was in fact rendering 19 of 21 charts identically across all |
| 116 | +# three states, which is the exact condition it claimed to prevent. Comparing |
| 117 | +# output is the only form of this test that can fail for the right reason. |
| 118 | +@test "the fixtures produce different renders for at least one chart" { |
| 119 | + count=$(find hack/testdata/render-fixtures -maxdepth 1 -name '*.yaml' | wc -l | tr -d ' ') |
103 | 120 | [ "$count" -ge 2 ] |
104 | | - distinct=$(for f in hack/testdata/render-fixtures/*.yaml; do |
105 | | - grep -E '^ (oidc-enabled|solver|wildcard-issue):' "$f" | tr -d ' ' | sort | tr '\n' ',' |
106 | | - echo |
107 | | - done | sort -u | wc -l | tr -d ' ') |
108 | | - if [ "$distinct" -lt "$count" ]; then |
109 | | - echo "fixtures do not differ on the keys charts branch on ($distinct distinct of $count files)" >&2 |
| 121 | + |
| 122 | + # packages/apps/kubernetes is the chart that branches most on these values |
| 123 | + # (the _namespace.<service> flags gate ~15 manifests). If a future fixture |
| 124 | + # change stops moving even this one, the states have collapsed. |
| 125 | + a=$(helm template kubernetes-render-check packages/apps/kubernetes -n tenant-test \ |
| 126 | + -f hack/testdata/render-fixtures/fresh.yaml 2>/dev/null | grep -c '^kind:') |
| 127 | + b=$(helm template kubernetes-render-check packages/apps/kubernetes -n tenant-test \ |
| 128 | + -f hack/testdata/render-fixtures/configured.yaml 2>/dev/null | grep -c '^kind:') |
| 129 | + if [ "$a" = "$b" ]; then |
| 130 | + echo "fresh and configured render the same number of documents ($a) for packages/apps/kubernetes — the fixtures no longer represent different cluster states" >&2 |
110 | 131 | exit 1 |
111 | 132 | fi |
112 | 133 | } |
0 commit comments