Skip to content

Commit be9094c

Browse files
[Backport release-1.5] fix(seaweedfs): close the 4.31 rename fallout on the 1.5.x→1.6 upgrade path (#3370)
# Description Backport of #3339 to `release-1.5`.
2 parents f68b1d6 + 33611df commit be9094c

29 files changed

Lines changed: 3629 additions & 31 deletions

‎docs/operations/seaweedfs-431-rename-recovery.md‎

Lines changed: 384 additions & 0 deletions
Large diffs are not rendered by default.
Lines changed: 333 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,333 @@
1+
#!/usr/bin/env bats
2+
# -----------------------------------------------------------------------------
3+
# Unit tests for the SeaweedFS db-split hand-over shared by platform migrations
4+
# 43 (original) and 45 (repair) — lib/seaweedfs-db-adopt.sh.
5+
#
6+
# The 1.5.0 split (PR #2601) moved Cluster/seaweedfs-db out of the <name>-system
7+
# release into a new <name>-db release. The hand-over must, before <name>-system
8+
# next renders, re-own the Cluster to <name>-db AND stamp
9+
# helm.sh/resource-policy: keep — otherwise the <name>-system upgrade prunes the
10+
# Cluster as a removed resource, CNPG takes its PVC with it, and the tenant's
11+
# filer metadata (all of its S3) is gone.
12+
#
13+
# Three properties are pinned here, each of which shipped broken:
14+
#
15+
# 1. INSTANCE NAME. Migration 43 compared the owner against the literal
16+
# "seaweedfs-system". `SeaweedFS` is user-creatable, so an instance named
17+
# `foo` is owned by `foo-system` and was silently skipped. Observed live:
18+
# four default-named tenants carry release-name=seaweedfs-db + keep and their
19+
# Clusters survived; the one tenant running `foo` has no Cluster at all.
20+
#
21+
# 2. OWNERSHIP IS NOT SAFETY. A Cluster already owned by <name>-db can still
22+
# need keep: where the hand-over was skipped, <name>-system prunes it and
23+
# <name>-db RECREATES it under its own ownership with no keep, while
24+
# <name>-system's prune baseline still lists it. Live proof that the shape is
25+
# real: tenant-l and tenant-root are <name>-db-owned and their <name>-system
26+
# deployed revision still contains the Cluster — only keep saves them.
27+
#
28+
# 3. FAIL CLOSED. Migrations never re-run, so a swallowed error permanently
29+
# leaves at-risk tenants exposed. `for ns in $(kubectl ...)` does not trip
30+
# errexit: on failure the loop runs zero times and the script stamps the
31+
# version anyway. Only "the resource type is not served" (no CNPG at all) and
32+
# "gone between scan and read" may be treated as empty.
33+
#
34+
# These drive the real migration scripts end-to-end against a fake kubectl
35+
# (hack/testdata/migration-seaweedfs-db/), mocking only the cluster boundary.
36+
#
37+
# SHELL. Production runs these under /bin/sh = busybox ash: the migrations image
38+
# is FROM alpine, and run-migrations.sh execs `/migrations/<n>` BY PATH, so the
39+
# kernel honours the `#!/bin/sh` shebang. `set -euo pipefail` and
40+
# errexit-in-function semantics differ from bash, and both are load-bearing here:
41+
# the fail-closed returns from adopt_seaweedfs_db_clusters have to abort the
42+
# script before it reaches its stamp. Asserting fail-closed in a shell that never
43+
# runs it would be asserting nothing.
44+
#
45+
# So these tests do not invoke the migrations through the runner's shell at all:
46+
# run_migration() executes them by path inside the image's own pinned base, read
47+
# from the migrations Dockerfile. Same base image, same interpreter, same
48+
# invocation form — production, rather than an approximation of it. This needs a
49+
# working docker; there is deliberately no host-shell fallback, because every
50+
# fallback available is a shell production never uses.
51+
#
52+
# What that replaced was `sh "$MIG_DIR/<n>"`, which is neither production nor
53+
# bash: on the GitHub ubuntu runner /bin/sh is dash, which has no `set -o
54+
# pipefail` and aborts on line 1 with "set: Illegal option -o pipefail" before
55+
# the script does any work, while on a developer box /bin/sh is often bash, which
56+
# runs green and proves nothing about ash.
57+
#
58+
# cozytest.sh's awk parser recognizes only @test blocks and a bare `}` on its
59+
# own line; there is no bats `run`/`$status`/`setup`. Assertions are direct
60+
# shell tests that exit non-zero on failure.
61+
#
62+
# Run with: hack/cozytest.sh hack/migration-seaweedfs-db-adopt.bats
63+
# -----------------------------------------------------------------------------
64+
65+
FAKEBIN="$PWD/hack/testdata/migration-seaweedfs-db"
66+
MIG_DIR="$PWD/packages/core/platform/images/migrations/migrations"
67+
68+
# The production base image, read out of the migrations Dockerfile rather than
69+
# repeated here, so the interpreter under test cannot drift from the one the
70+
# migrations actually ship on when that pin is bumped.
71+
ALPINE=$(sed -n 's/^FROM \(alpine:[^ ]*\).*$/\1/p' \
72+
"$PWD/packages/core/platform/images/migrations/Dockerfile" | head -1)
73+
74+
# run_migration <n> -- run migrations/<n> the way run-migrations.sh does.
75+
#
76+
# By path, not `sh <file>`: that is what makes the shebang, and therefore the
77+
# interpreter, part of what is under test. The fake kubectl goes on PATH inside
78+
# the container and $WORK is bind-mounted, so $FAKE_CMDLOG is the same file the
79+
# assertions read back on the host. --network none because nothing here may
80+
# reach a real cluster; --user keeps $WORK removable by the test afterwards.
81+
#
82+
# The explicit `return` is load-bearing: cozytest.sh's awk generator rewrites
83+
# every bare `}` in column 0 into `return 0` + `}`, so a helper that falls off
84+
# its own end returns 0 no matter what it ran, and every fail-closed assertion
85+
# below would pass vacuously. Capture the status and return it by hand.
86+
run_migration() {
87+
_run_migration_rc=0
88+
docker run --rm --network none \
89+
--user "$(id -u):$(id -g)" \
90+
-v "$MIG_DIR:/migrations:ro" \
91+
-v "$FAKEBIN:/fakebin:ro" \
92+
-v "$WORK:/work" \
93+
-e PATH=/fakebin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \
94+
-e FAKE_CMDLOG=/work/cmdlog \
95+
-e NAMESPACE="${NAMESPACE-}" \
96+
-e FAKE_CLUSTERS="${FAKE_CLUSTERS-}" \
97+
-e FAKE_LIST_FAIL="${FAKE_LIST_FAIL-}" \
98+
-e FAKE_GET_FAIL="${FAKE_GET_FAIL-}" \
99+
-e FAKE_ANNOTATE_FAIL="${FAKE_ANNOTATE_FAIL-}" \
100+
"$ALPINE" "/migrations/$1" || _run_migration_rc=$?
101+
return "$_run_migration_rc"
102+
}
103+
104+
# prep resets env to a clean scenario. Tests set FAKE_* afterwards.
105+
prep() {
106+
# Fail here rather than at the first docker run, so the reason is legible.
107+
docker info >/dev/null 2>&1 || {
108+
echo "docker is required: these tests run the migrations inside $ALPINE," >&2
109+
echo "the base image of the migrations image, so that they exercise busybox" >&2
110+
echo "ash — the interpreter run-migrations.sh actually gives them." >&2
111+
return 1
112+
}
113+
chmod +x "$FAKEBIN/kubectl"
114+
WORK=$(mktemp -d)
115+
export FAKE_CMDLOG="$WORK/cmdlog"
116+
: > "$FAKE_CMDLOG"
117+
export NAMESPACE=cozy-system
118+
export FAKE_CLUSTERS=""
119+
unset FAKE_LIST_FAIL FAKE_GET_FAIL FAKE_ANNOTATE_FAIL || true
120+
}
121+
122+
# --- 1. instance name -------------------------------------------------------
123+
124+
@test "hands over a default-named instance (seaweedfs-system -> seaweedfs-db)" {
125+
prep
126+
export FAKE_CLUSTERS="tenant-root seaweedfs-system -"
127+
rc=0
128+
run_migration 43 >"$WORK/out" 2>&1 || rc=$?
129+
cat "$WORK/out"; cat "$FAKE_CMDLOG"
130+
[ "$rc" -eq 0 ]
131+
grep -qF -- "ANNOTATE tenant-root release-name=seaweedfs-db resource-policy=keep" "$FAKE_CMDLOG"
132+
# Migration 43 stamps 44 — asserting the number, not a bare "STAMP": a wrong
133+
# version would loop run-migrations.sh forever.
134+
grep -qF -- "STAMP 44" "$FAKE_CMDLOG"
135+
rm -rf "$WORK"
136+
}
137+
138+
# THE original regression. Unfixed (owner compared against the literal
139+
# "seaweedfs-system") this namespace is skipped entirely: no ANNOTATE line, and
140+
# the Cluster is left with no keep for the foo-system upgrade to prune.
141+
@test "hands over a NON-default instance name (foo-system -> foo-db)" {
142+
prep
143+
export FAKE_CLUSTERS="tenant-named foo-system -"
144+
rc=0
145+
run_migration 43 >"$WORK/out" 2>&1 || rc=$?
146+
cat "$WORK/out"; cat "$FAKE_CMDLOG"
147+
[ "$rc" -eq 0 ]
148+
grep -qF -- "ANNOTATE tenant-named release-name=foo-db resource-policy=keep" "$FAKE_CMDLOG"
149+
rm -rf "$WORK"
150+
}
151+
152+
@test "migration 45 repairs a non-default instance the hardcoded 43 skipped, and stamps 46" {
153+
prep
154+
export FAKE_CLUSTERS="tenant-named foo-system -"
155+
rc=0
156+
run_migration 45 >"$WORK/out" 2>&1 || rc=$?
157+
cat "$WORK/out"; cat "$FAKE_CMDLOG"
158+
[ "$rc" -eq 0 ]
159+
grep -qF -- "ANNOTATE tenant-named release-name=foo-db resource-policy=keep" "$FAKE_CMDLOG"
160+
grep -qF -- "STAMP 46" "$FAKE_CMDLOG"
161+
rm -rf "$WORK"
162+
}
163+
164+
@test "handles a mixed fleet: every -system owner is handed over, in its own namespace" {
165+
prep
166+
# The shape of the upgrade stand: default-named tenants plus one `foo`.
167+
export FAKE_CLUSTERS="tenant-root seaweedfs-system -
168+
tenant-dsplit seaweedfs-system -
169+
tenant-l seaweedfs-system -
170+
tenant-named foo-system -"
171+
rc=0
172+
run_migration 45 >"$WORK/out" 2>&1 || rc=$?
173+
cat "$WORK/out"; cat "$FAKE_CMDLOG"
174+
[ "$rc" -eq 0 ]
175+
grep -qF -- "ANNOTATE tenant-root release-name=seaweedfs-db resource-policy=keep" "$FAKE_CMDLOG"
176+
grep -qF -- "ANNOTATE tenant-dsplit release-name=seaweedfs-db resource-policy=keep" "$FAKE_CMDLOG"
177+
grep -qF -- "ANNOTATE tenant-l release-name=seaweedfs-db resource-policy=keep" "$FAKE_CMDLOG"
178+
grep -qF -- "ANNOTATE tenant-named release-name=foo-db resource-policy=keep" "$FAKE_CMDLOG"
179+
[ "$(grep -c 'ANNOTATE' "$FAKE_CMDLOG")" -eq 4 ]
180+
rm -rf "$WORK"
181+
}
182+
183+
# --- 2. ownership is not safety --------------------------------------------
184+
185+
# A <name>-db-owned Cluster WITHOUT keep is exposed, not done: <name>-system's
186+
# prune baseline may still list the Cluster (live on the stand for tenant-l and
187+
# tenant-root), in which case its next reconcile deletes it. Skipping on
188+
# ownership alone — the shape the previous revision of this helper shipped —
189+
# leaves the database to be pruned.
190+
@test "protects a <name>-db-owned Cluster that is still missing keep" {
191+
prep
192+
export FAKE_CLUSTERS="tenant-named foo-db -"
193+
rc=0
194+
run_migration 45 >"$WORK/out" 2>&1 || rc=$?
195+
cat "$WORK/out"; cat "$FAKE_CMDLOG"
196+
[ "$rc" -eq 0 ]
197+
grep -qF -- "ANNOTATE tenant-named release-name=<unset> resource-policy=keep" "$FAKE_CMDLOG"
198+
rm -rf "$WORK"
199+
}
200+
201+
@test "protects a default-named <name>-db-owned Cluster that is still missing keep" {
202+
prep
203+
export FAKE_CLUSTERS="tenant-fresh seaweedfs-db -"
204+
rc=0
205+
run_migration 45 >"$WORK/out" 2>&1 || rc=$?
206+
cat "$WORK/out"; cat "$FAKE_CMDLOG"
207+
[ "$rc" -eq 0 ]
208+
grep -qF -- "ANNOTATE tenant-fresh release-name=<unset> resource-policy=keep" "$FAKE_CMDLOG"
209+
rm -rf "$WORK"
210+
}
211+
212+
@test "idempotent: a Cluster already owned by <name>-db AND carrying keep is left alone" {
213+
prep
214+
export FAKE_CLUSTERS="tenant-root seaweedfs-db keep
215+
tenant-named foo-db keep"
216+
rc=0
217+
run_migration 45 >"$WORK/out" 2>&1 || rc=$?
218+
cat "$WORK/out"; cat "$FAKE_CMDLOG"
219+
[ "$rc" -eq 0 ]
220+
! grep -q 'ANNOTATE' "$FAKE_CMDLOG"
221+
grep -qF -- "STAMP 46" "$FAKE_CMDLOG"
222+
rm -rf "$WORK"
223+
}
224+
225+
@test "leaves a Cluster with no Helm owner annotation alone, but says so" {
226+
prep
227+
# Not Helm-managed. Guessing an owner would be worse than doing nothing, but
228+
# an unowned SeaweedFS database must not pass silently.
229+
export FAKE_CLUSTERS="tenant-manual - -"
230+
rc=0
231+
run_migration 45 >"$WORK/out" 2>&1 || rc=$?
232+
cat "$WORK/out"; cat "$FAKE_CMDLOG"
233+
[ "$rc" -eq 0 ]
234+
! grep -q 'ANNOTATE' "$FAKE_CMDLOG"
235+
grep -qF -- "carries no meta.helm.sh/release-name" "$WORK/out"
236+
rm -rf "$WORK"
237+
}
238+
239+
@test "leaves a Cluster owned by an unrelated release alone" {
240+
prep
241+
export FAKE_CLUSTERS="tenant-x some-other-release -"
242+
rc=0
243+
run_migration 45 >"$WORK/out" 2>&1 || rc=$?
244+
cat "$WORK/out"; cat "$FAKE_CMDLOG"
245+
[ "$rc" -eq 0 ]
246+
! grep -q 'ANNOTATE' "$FAKE_CMDLOG"
247+
grep -qF -- "owned by unrelated release" "$WORK/out"
248+
rm -rf "$WORK"
249+
}
250+
251+
@test "refuses a release literally named -system rather than annotating owner -db" {
252+
prep
253+
# "${current%-system}" would be empty, yielding release-name=-db, which no
254+
# release will ever claim: the Cluster would be orphaned by the very step
255+
# meant to protect it.
256+
export FAKE_CLUSTERS="tenant-weird -system -"
257+
rc=0
258+
run_migration 45 >"$WORK/out" 2>&1 || rc=$?
259+
cat "$WORK/out"; cat "$FAKE_CMDLOG"
260+
[ "$rc" -eq 0 ]
261+
! grep -q 'ANNOTATE' "$FAKE_CMDLOG"
262+
grep -qF -- "no instance name" "$WORK/out"
263+
rm -rf "$WORK"
264+
}
265+
266+
# --- 3. fail closed ---------------------------------------------------------
267+
268+
@test "a failing fleet scan aborts the migration instead of stamping past it" {
269+
prep
270+
export FAKE_CLUSTERS="tenant-named foo-system -"
271+
export FAKE_LIST_FAIL="Error from server (Timeout): the server was unable to return a response in the time allotted"
272+
rc=0
273+
run_migration 45 >"$WORK/out" 2>&1 || rc=$?
274+
cat "$WORK/out"; cat "$FAKE_CMDLOG"
275+
# Must propagate: the Job retries rather than advancing the version.
276+
[ "$rc" -ne 0 ]
277+
grep -qF -- "refusing to stamp past an unverified fleet" "$WORK/out"
278+
! grep -q 'STAMP' "$FAKE_CMDLOG"
279+
rm -rf "$WORK"
280+
}
281+
282+
@test "an unreadable owner annotation aborts rather than being read as not-Helm-managed" {
283+
prep
284+
export FAKE_CLUSTERS="tenant-named foo-system -"
285+
export FAKE_GET_FAIL="Error from server (Forbidden): clusters.postgresql.cnpg.io \"seaweedfs-db\" is forbidden"
286+
rc=0
287+
run_migration 45 >"$WORK/out" 2>&1 || rc=$?
288+
cat "$WORK/out"; cat "$FAKE_CMDLOG"
289+
[ "$rc" -ne 0 ]
290+
grep -qF -- "cannot read the Helm owner" "$WORK/out"
291+
! grep -q 'STAMP' "$FAKE_CMDLOG"
292+
rm -rf "$WORK"
293+
}
294+
295+
@test "a failed hand-over aborts rather than stamping a half-migrated fleet" {
296+
prep
297+
export FAKE_CLUSTERS="tenant-named foo-system -"
298+
export FAKE_ANNOTATE_FAIL="Error from server (Conflict): the object has been modified"
299+
rc=0
300+
run_migration 45 >"$WORK/out" 2>&1 || rc=$?
301+
cat "$WORK/out"; cat "$FAKE_CMDLOG"
302+
[ "$rc" -ne 0 ]
303+
! grep -q 'STAMP' "$FAKE_CMDLOG"
304+
rm -rf "$WORK"
305+
}
306+
307+
# The fail-open that IS load-bearing: a cluster with no CNPG at all must not be
308+
# blocked from upgrading. "The server doesn't have a resource type" is the only
309+
# list failure allowed to mean "nothing to do".
310+
@test "a cluster with no CNPG resource type stamps cleanly without annotating" {
311+
prep
312+
export FAKE_LIST_FAIL="error: the server doesn't have a resource type \"cluster\" in group \"postgresql.cnpg.io\""
313+
rc=0
314+
run_migration 45 >"$WORK/out" 2>&1 || rc=$?
315+
cat "$WORK/out"; cat "$FAKE_CMDLOG"
316+
[ "$rc" -eq 0 ]
317+
grep -qF -- "resource type is not served" "$WORK/out"
318+
! grep -q 'ANNOTATE' "$FAKE_CMDLOG"
319+
grep -qF -- "STAMP 46" "$FAKE_CMDLOG"
320+
rm -rf "$WORK"
321+
}
322+
323+
@test "an empty fleet stamps without annotating" {
324+
prep
325+
export FAKE_CLUSTERS=""
326+
rc=0
327+
run_migration 45 >"$WORK/out" 2>&1 || rc=$?
328+
cat "$WORK/out"; cat "$FAKE_CMDLOG"
329+
[ "$rc" -eq 0 ]
330+
! grep -q 'ANNOTATE' "$FAKE_CMDLOG"
331+
grep -qF -- "STAMP 46" "$FAKE_CMDLOG"
332+
rm -rf "$WORK"
333+
}

0 commit comments

Comments
 (0)