|
| 1 | +#!/usr/bin/env bats |
| 2 | +# ----------------------------------------------------------------------------- |
| 3 | +# Unit tests for the SeaweedFS db-split hand-over shared by platform migrations |
| 4 | +# 43 (original) and 45 (repair) — lib/seaweedfs-db-adopt.sh. |
| 5 | +# |
| 6 | +# The 1.5.0 split (PR #2601) moved Cluster/seaweedfs-db out of the <name>-system |
| 7 | +# release into a new <name>-db release. The hand-over must, before <name>-system |
| 8 | +# next renders, re-own the Cluster to <name>-db AND stamp |
| 9 | +# helm.sh/resource-policy: keep — otherwise the <name>-system upgrade prunes the |
| 10 | +# Cluster as a removed resource, CNPG takes its PVC with it, and the tenant's |
| 11 | +# filer metadata (all of its S3) is gone. |
| 12 | +# |
| 13 | +# Three properties are pinned here, each of which shipped broken: |
| 14 | +# |
| 15 | +# 1. INSTANCE NAME. Migration 43 compared the owner against the literal |
| 16 | +# "seaweedfs-system". `SeaweedFS` is user-creatable, so an instance named |
| 17 | +# `foo` is owned by `foo-system` and was silently skipped. Observed live: |
| 18 | +# four default-named tenants carry release-name=seaweedfs-db + keep and their |
| 19 | +# Clusters survived; the one tenant running `foo` has no Cluster at all. |
| 20 | +# |
| 21 | +# 2. OWNERSHIP IS NOT SAFETY. A Cluster already owned by <name>-db can still |
| 22 | +# need keep: where the hand-over was skipped, <name>-system prunes it and |
| 23 | +# <name>-db RECREATES it under its own ownership with no keep, while |
| 24 | +# <name>-system's prune baseline still lists it. Live proof that the shape is |
| 25 | +# real: tenant-l and tenant-root are <name>-db-owned and their <name>-system |
| 26 | +# deployed revision still contains the Cluster — only keep saves them. |
| 27 | +# |
| 28 | +# 3. FAIL CLOSED. Migrations never re-run, so a swallowed error permanently |
| 29 | +# leaves at-risk tenants exposed. `for ns in $(kubectl ...)` does not trip |
| 30 | +# errexit: on failure the loop runs zero times and the script stamps the |
| 31 | +# version anyway. Only "the resource type is not served" (no CNPG at all) and |
| 32 | +# "gone between scan and read" may be treated as empty. |
| 33 | +# |
| 34 | +# These drive the real migration scripts end-to-end against a fake kubectl |
| 35 | +# (hack/testdata/migration-seaweedfs-db/), mocking only the cluster boundary. |
| 36 | +# |
| 37 | +# SHELL. Production runs these under /bin/sh = busybox ash: the migrations image |
| 38 | +# is FROM alpine, and run-migrations.sh execs `/migrations/<n>` BY PATH, so the |
| 39 | +# kernel honours the `#!/bin/sh` shebang. `set -euo pipefail` and |
| 40 | +# errexit-in-function semantics differ from bash, and both are load-bearing here: |
| 41 | +# the fail-closed returns from adopt_seaweedfs_db_clusters have to abort the |
| 42 | +# script before it reaches its stamp. Asserting fail-closed in a shell that never |
| 43 | +# runs it would be asserting nothing. |
| 44 | +# |
| 45 | +# So these tests do not invoke the migrations through the runner's shell at all: |
| 46 | +# run_migration() executes them by path inside the image's own pinned base, read |
| 47 | +# from the migrations Dockerfile. Same base image, same interpreter, same |
| 48 | +# invocation form — production, rather than an approximation of it. This needs a |
| 49 | +# working docker; there is deliberately no host-shell fallback, because every |
| 50 | +# fallback available is a shell production never uses. |
| 51 | +# |
| 52 | +# What that replaced was `sh "$MIG_DIR/<n>"`, which is neither production nor |
| 53 | +# bash: on the GitHub ubuntu runner /bin/sh is dash, which has no `set -o |
| 54 | +# pipefail` and aborts on line 1 with "set: Illegal option -o pipefail" before |
| 55 | +# the script does any work, while on a developer box /bin/sh is often bash, which |
| 56 | +# runs green and proves nothing about ash. |
| 57 | +# |
| 58 | +# cozytest.sh's awk parser recognizes only @test blocks and a bare `}` on its |
| 59 | +# own line; there is no bats `run`/`$status`/`setup`. Assertions are direct |
| 60 | +# shell tests that exit non-zero on failure. |
| 61 | +# |
| 62 | +# Run with: hack/cozytest.sh hack/migration-seaweedfs-db-adopt.bats |
| 63 | +# ----------------------------------------------------------------------------- |
| 64 | + |
| 65 | +FAKEBIN="$PWD/hack/testdata/migration-seaweedfs-db" |
| 66 | +MIG_DIR="$PWD/packages/core/platform/images/migrations/migrations" |
| 67 | + |
| 68 | +# The production base image, read out of the migrations Dockerfile rather than |
| 69 | +# repeated here, so the interpreter under test cannot drift from the one the |
| 70 | +# migrations actually ship on when that pin is bumped. |
| 71 | +ALPINE=$(sed -n 's/^FROM \(alpine:[^ ]*\).*$/\1/p' \ |
| 72 | + "$PWD/packages/core/platform/images/migrations/Dockerfile" | head -1) |
| 73 | + |
| 74 | +# run_migration <n> -- run migrations/<n> the way run-migrations.sh does. |
| 75 | +# |
| 76 | +# By path, not `sh <file>`: that is what makes the shebang, and therefore the |
| 77 | +# interpreter, part of what is under test. The fake kubectl goes on PATH inside |
| 78 | +# the container and $WORK is bind-mounted, so $FAKE_CMDLOG is the same file the |
| 79 | +# assertions read back on the host. --network none because nothing here may |
| 80 | +# reach a real cluster; --user keeps $WORK removable by the test afterwards. |
| 81 | +# |
| 82 | +# The explicit `return` is load-bearing: cozytest.sh's awk generator rewrites |
| 83 | +# every bare `}` in column 0 into `return 0` + `}`, so a helper that falls off |
| 84 | +# its own end returns 0 no matter what it ran, and every fail-closed assertion |
| 85 | +# below would pass vacuously. Capture the status and return it by hand. |
| 86 | +run_migration() { |
| 87 | + _run_migration_rc=0 |
| 88 | + docker run --rm --network none \ |
| 89 | + --user "$(id -u):$(id -g)" \ |
| 90 | + -v "$MIG_DIR:/migrations:ro" \ |
| 91 | + -v "$FAKEBIN:/fakebin:ro" \ |
| 92 | + -v "$WORK:/work" \ |
| 93 | + -e PATH=/fakebin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \ |
| 94 | + -e FAKE_CMDLOG=/work/cmdlog \ |
| 95 | + -e NAMESPACE="${NAMESPACE-}" \ |
| 96 | + -e FAKE_CLUSTERS="${FAKE_CLUSTERS-}" \ |
| 97 | + -e FAKE_LIST_FAIL="${FAKE_LIST_FAIL-}" \ |
| 98 | + -e FAKE_GET_FAIL="${FAKE_GET_FAIL-}" \ |
| 99 | + -e FAKE_ANNOTATE_FAIL="${FAKE_ANNOTATE_FAIL-}" \ |
| 100 | + "$ALPINE" "/migrations/$1" || _run_migration_rc=$? |
| 101 | + return "$_run_migration_rc" |
| 102 | +} |
| 103 | + |
| 104 | +# prep resets env to a clean scenario. Tests set FAKE_* afterwards. |
| 105 | +prep() { |
| 106 | + # Fail here rather than at the first docker run, so the reason is legible. |
| 107 | + docker info >/dev/null 2>&1 || { |
| 108 | + echo "docker is required: these tests run the migrations inside $ALPINE," >&2 |
| 109 | + echo "the base image of the migrations image, so that they exercise busybox" >&2 |
| 110 | + echo "ash — the interpreter run-migrations.sh actually gives them." >&2 |
| 111 | + return 1 |
| 112 | + } |
| 113 | + chmod +x "$FAKEBIN/kubectl" |
| 114 | + WORK=$(mktemp -d) |
| 115 | + export FAKE_CMDLOG="$WORK/cmdlog" |
| 116 | + : > "$FAKE_CMDLOG" |
| 117 | + export NAMESPACE=cozy-system |
| 118 | + export FAKE_CLUSTERS="" |
| 119 | + unset FAKE_LIST_FAIL FAKE_GET_FAIL FAKE_ANNOTATE_FAIL || true |
| 120 | +} |
| 121 | + |
| 122 | +# --- 1. instance name ------------------------------------------------------- |
| 123 | + |
| 124 | +@test "hands over a default-named instance (seaweedfs-system -> seaweedfs-db)" { |
| 125 | + prep |
| 126 | + export FAKE_CLUSTERS="tenant-root seaweedfs-system -" |
| 127 | + rc=0 |
| 128 | + run_migration 43 >"$WORK/out" 2>&1 || rc=$? |
| 129 | + cat "$WORK/out"; cat "$FAKE_CMDLOG" |
| 130 | + [ "$rc" -eq 0 ] |
| 131 | + grep -qF -- "ANNOTATE tenant-root release-name=seaweedfs-db resource-policy=keep" "$FAKE_CMDLOG" |
| 132 | + # Migration 43 stamps 44 — asserting the number, not a bare "STAMP": a wrong |
| 133 | + # version would loop run-migrations.sh forever. |
| 134 | + grep -qF -- "STAMP 44" "$FAKE_CMDLOG" |
| 135 | + rm -rf "$WORK" |
| 136 | +} |
| 137 | + |
| 138 | +# THE original regression. Unfixed (owner compared against the literal |
| 139 | +# "seaweedfs-system") this namespace is skipped entirely: no ANNOTATE line, and |
| 140 | +# the Cluster is left with no keep for the foo-system upgrade to prune. |
| 141 | +@test "hands over a NON-default instance name (foo-system -> foo-db)" { |
| 142 | + prep |
| 143 | + export FAKE_CLUSTERS="tenant-named foo-system -" |
| 144 | + rc=0 |
| 145 | + run_migration 43 >"$WORK/out" 2>&1 || rc=$? |
| 146 | + cat "$WORK/out"; cat "$FAKE_CMDLOG" |
| 147 | + [ "$rc" -eq 0 ] |
| 148 | + grep -qF -- "ANNOTATE tenant-named release-name=foo-db resource-policy=keep" "$FAKE_CMDLOG" |
| 149 | + rm -rf "$WORK" |
| 150 | +} |
| 151 | + |
| 152 | +@test "migration 45 repairs a non-default instance the hardcoded 43 skipped, and stamps 46" { |
| 153 | + prep |
| 154 | + export FAKE_CLUSTERS="tenant-named foo-system -" |
| 155 | + rc=0 |
| 156 | + run_migration 45 >"$WORK/out" 2>&1 || rc=$? |
| 157 | + cat "$WORK/out"; cat "$FAKE_CMDLOG" |
| 158 | + [ "$rc" -eq 0 ] |
| 159 | + grep -qF -- "ANNOTATE tenant-named release-name=foo-db resource-policy=keep" "$FAKE_CMDLOG" |
| 160 | + grep -qF -- "STAMP 46" "$FAKE_CMDLOG" |
| 161 | + rm -rf "$WORK" |
| 162 | +} |
| 163 | + |
| 164 | +@test "handles a mixed fleet: every -system owner is handed over, in its own namespace" { |
| 165 | + prep |
| 166 | + # The shape of the upgrade stand: default-named tenants plus one `foo`. |
| 167 | + export FAKE_CLUSTERS="tenant-root seaweedfs-system - |
| 168 | +tenant-dsplit seaweedfs-system - |
| 169 | +tenant-l seaweedfs-system - |
| 170 | +tenant-named foo-system -" |
| 171 | + rc=0 |
| 172 | + run_migration 45 >"$WORK/out" 2>&1 || rc=$? |
| 173 | + cat "$WORK/out"; cat "$FAKE_CMDLOG" |
| 174 | + [ "$rc" -eq 0 ] |
| 175 | + grep -qF -- "ANNOTATE tenant-root release-name=seaweedfs-db resource-policy=keep" "$FAKE_CMDLOG" |
| 176 | + grep -qF -- "ANNOTATE tenant-dsplit release-name=seaweedfs-db resource-policy=keep" "$FAKE_CMDLOG" |
| 177 | + grep -qF -- "ANNOTATE tenant-l release-name=seaweedfs-db resource-policy=keep" "$FAKE_CMDLOG" |
| 178 | + grep -qF -- "ANNOTATE tenant-named release-name=foo-db resource-policy=keep" "$FAKE_CMDLOG" |
| 179 | + [ "$(grep -c 'ANNOTATE' "$FAKE_CMDLOG")" -eq 4 ] |
| 180 | + rm -rf "$WORK" |
| 181 | +} |
| 182 | + |
| 183 | +# --- 2. ownership is not safety -------------------------------------------- |
| 184 | + |
| 185 | +# A <name>-db-owned Cluster WITHOUT keep is exposed, not done: <name>-system's |
| 186 | +# prune baseline may still list the Cluster (live on the stand for tenant-l and |
| 187 | +# tenant-root), in which case its next reconcile deletes it. Skipping on |
| 188 | +# ownership alone — the shape the previous revision of this helper shipped — |
| 189 | +# leaves the database to be pruned. |
| 190 | +@test "protects a <name>-db-owned Cluster that is still missing keep" { |
| 191 | + prep |
| 192 | + export FAKE_CLUSTERS="tenant-named foo-db -" |
| 193 | + rc=0 |
| 194 | + run_migration 45 >"$WORK/out" 2>&1 || rc=$? |
| 195 | + cat "$WORK/out"; cat "$FAKE_CMDLOG" |
| 196 | + [ "$rc" -eq 0 ] |
| 197 | + grep -qF -- "ANNOTATE tenant-named release-name=<unset> resource-policy=keep" "$FAKE_CMDLOG" |
| 198 | + rm -rf "$WORK" |
| 199 | +} |
| 200 | + |
| 201 | +@test "protects a default-named <name>-db-owned Cluster that is still missing keep" { |
| 202 | + prep |
| 203 | + export FAKE_CLUSTERS="tenant-fresh seaweedfs-db -" |
| 204 | + rc=0 |
| 205 | + run_migration 45 >"$WORK/out" 2>&1 || rc=$? |
| 206 | + cat "$WORK/out"; cat "$FAKE_CMDLOG" |
| 207 | + [ "$rc" -eq 0 ] |
| 208 | + grep -qF -- "ANNOTATE tenant-fresh release-name=<unset> resource-policy=keep" "$FAKE_CMDLOG" |
| 209 | + rm -rf "$WORK" |
| 210 | +} |
| 211 | + |
| 212 | +@test "idempotent: a Cluster already owned by <name>-db AND carrying keep is left alone" { |
| 213 | + prep |
| 214 | + export FAKE_CLUSTERS="tenant-root seaweedfs-db keep |
| 215 | +tenant-named foo-db keep" |
| 216 | + rc=0 |
| 217 | + run_migration 45 >"$WORK/out" 2>&1 || rc=$? |
| 218 | + cat "$WORK/out"; cat "$FAKE_CMDLOG" |
| 219 | + [ "$rc" -eq 0 ] |
| 220 | + ! grep -q 'ANNOTATE' "$FAKE_CMDLOG" |
| 221 | + grep -qF -- "STAMP 46" "$FAKE_CMDLOG" |
| 222 | + rm -rf "$WORK" |
| 223 | +} |
| 224 | + |
| 225 | +@test "leaves a Cluster with no Helm owner annotation alone, but says so" { |
| 226 | + prep |
| 227 | + # Not Helm-managed. Guessing an owner would be worse than doing nothing, but |
| 228 | + # an unowned SeaweedFS database must not pass silently. |
| 229 | + export FAKE_CLUSTERS="tenant-manual - -" |
| 230 | + rc=0 |
| 231 | + run_migration 45 >"$WORK/out" 2>&1 || rc=$? |
| 232 | + cat "$WORK/out"; cat "$FAKE_CMDLOG" |
| 233 | + [ "$rc" -eq 0 ] |
| 234 | + ! grep -q 'ANNOTATE' "$FAKE_CMDLOG" |
| 235 | + grep -qF -- "carries no meta.helm.sh/release-name" "$WORK/out" |
| 236 | + rm -rf "$WORK" |
| 237 | +} |
| 238 | + |
| 239 | +@test "leaves a Cluster owned by an unrelated release alone" { |
| 240 | + prep |
| 241 | + export FAKE_CLUSTERS="tenant-x some-other-release -" |
| 242 | + rc=0 |
| 243 | + run_migration 45 >"$WORK/out" 2>&1 || rc=$? |
| 244 | + cat "$WORK/out"; cat "$FAKE_CMDLOG" |
| 245 | + [ "$rc" -eq 0 ] |
| 246 | + ! grep -q 'ANNOTATE' "$FAKE_CMDLOG" |
| 247 | + grep -qF -- "owned by unrelated release" "$WORK/out" |
| 248 | + rm -rf "$WORK" |
| 249 | +} |
| 250 | + |
| 251 | +@test "refuses a release literally named -system rather than annotating owner -db" { |
| 252 | + prep |
| 253 | + # "${current%-system}" would be empty, yielding release-name=-db, which no |
| 254 | + # release will ever claim: the Cluster would be orphaned by the very step |
| 255 | + # meant to protect it. |
| 256 | + export FAKE_CLUSTERS="tenant-weird -system -" |
| 257 | + rc=0 |
| 258 | + run_migration 45 >"$WORK/out" 2>&1 || rc=$? |
| 259 | + cat "$WORK/out"; cat "$FAKE_CMDLOG" |
| 260 | + [ "$rc" -eq 0 ] |
| 261 | + ! grep -q 'ANNOTATE' "$FAKE_CMDLOG" |
| 262 | + grep -qF -- "no instance name" "$WORK/out" |
| 263 | + rm -rf "$WORK" |
| 264 | +} |
| 265 | + |
| 266 | +# --- 3. fail closed --------------------------------------------------------- |
| 267 | + |
| 268 | +@test "a failing fleet scan aborts the migration instead of stamping past it" { |
| 269 | + prep |
| 270 | + export FAKE_CLUSTERS="tenant-named foo-system -" |
| 271 | + export FAKE_LIST_FAIL="Error from server (Timeout): the server was unable to return a response in the time allotted" |
| 272 | + rc=0 |
| 273 | + run_migration 45 >"$WORK/out" 2>&1 || rc=$? |
| 274 | + cat "$WORK/out"; cat "$FAKE_CMDLOG" |
| 275 | + # Must propagate: the Job retries rather than advancing the version. |
| 276 | + [ "$rc" -ne 0 ] |
| 277 | + grep -qF -- "refusing to stamp past an unverified fleet" "$WORK/out" |
| 278 | + ! grep -q 'STAMP' "$FAKE_CMDLOG" |
| 279 | + rm -rf "$WORK" |
| 280 | +} |
| 281 | + |
| 282 | +@test "an unreadable owner annotation aborts rather than being read as not-Helm-managed" { |
| 283 | + prep |
| 284 | + export FAKE_CLUSTERS="tenant-named foo-system -" |
| 285 | + export FAKE_GET_FAIL="Error from server (Forbidden): clusters.postgresql.cnpg.io \"seaweedfs-db\" is forbidden" |
| 286 | + rc=0 |
| 287 | + run_migration 45 >"$WORK/out" 2>&1 || rc=$? |
| 288 | + cat "$WORK/out"; cat "$FAKE_CMDLOG" |
| 289 | + [ "$rc" -ne 0 ] |
| 290 | + grep -qF -- "cannot read the Helm owner" "$WORK/out" |
| 291 | + ! grep -q 'STAMP' "$FAKE_CMDLOG" |
| 292 | + rm -rf "$WORK" |
| 293 | +} |
| 294 | + |
| 295 | +@test "a failed hand-over aborts rather than stamping a half-migrated fleet" { |
| 296 | + prep |
| 297 | + export FAKE_CLUSTERS="tenant-named foo-system -" |
| 298 | + export FAKE_ANNOTATE_FAIL="Error from server (Conflict): the object has been modified" |
| 299 | + rc=0 |
| 300 | + run_migration 45 >"$WORK/out" 2>&1 || rc=$? |
| 301 | + cat "$WORK/out"; cat "$FAKE_CMDLOG" |
| 302 | + [ "$rc" -ne 0 ] |
| 303 | + ! grep -q 'STAMP' "$FAKE_CMDLOG" |
| 304 | + rm -rf "$WORK" |
| 305 | +} |
| 306 | + |
| 307 | +# The fail-open that IS load-bearing: a cluster with no CNPG at all must not be |
| 308 | +# blocked from upgrading. "The server doesn't have a resource type" is the only |
| 309 | +# list failure allowed to mean "nothing to do". |
| 310 | +@test "a cluster with no CNPG resource type stamps cleanly without annotating" { |
| 311 | + prep |
| 312 | + export FAKE_LIST_FAIL="error: the server doesn't have a resource type \"cluster\" in group \"postgresql.cnpg.io\"" |
| 313 | + rc=0 |
| 314 | + run_migration 45 >"$WORK/out" 2>&1 || rc=$? |
| 315 | + cat "$WORK/out"; cat "$FAKE_CMDLOG" |
| 316 | + [ "$rc" -eq 0 ] |
| 317 | + grep -qF -- "resource type is not served" "$WORK/out" |
| 318 | + ! grep -q 'ANNOTATE' "$FAKE_CMDLOG" |
| 319 | + grep -qF -- "STAMP 46" "$FAKE_CMDLOG" |
| 320 | + rm -rf "$WORK" |
| 321 | +} |
| 322 | + |
| 323 | +@test "an empty fleet stamps without annotating" { |
| 324 | + prep |
| 325 | + export FAKE_CLUSTERS="" |
| 326 | + rc=0 |
| 327 | + run_migration 45 >"$WORK/out" 2>&1 || rc=$? |
| 328 | + cat "$WORK/out"; cat "$FAKE_CMDLOG" |
| 329 | + [ "$rc" -eq 0 ] |
| 330 | + ! grep -q 'ANNOTATE' "$FAKE_CMDLOG" |
| 331 | + grep -qF -- "STAMP 46" "$FAKE_CMDLOG" |
| 332 | + rm -rf "$WORK" |
| 333 | +} |
0 commit comments