Skip to content

Commit c0f9f08

Browse files
committed
chore(platform): detach HTTPCache instances before the package is pruned
Pruning the http-cache PackageSource takes its chart artifacts with it, so without a migration every http-cache HelmRelease loses its chart and stays Ready=False for good, with HelmReleaseNotReady firing on every cluster that runs the naas bundle. Deleting the leftover Package by hand is worse: it takes the ApplicationDefinition with it while the pods of a surviving instance are still running. Migration 57 orphans each tenant release instead, the way migration 55 does, so the workloads stay in the tenant namespace as plain objects, and then removes the platform side. The Package goes before hr/http-cache-rd because the operator re-creates a release it owns for as long as that Package exists. Signed-off-by: Andrei Kvapil <[email protected]>
1 parent d7c0332 commit c0f9f08

4 files changed

Lines changed: 379 additions & 1 deletion

File tree

Lines changed: 195 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,195 @@
1+
#!/usr/bin/env bats
2+
# -----------------------------------------------------------------------------
3+
# Unit tests for platform migration 57 --> 58 (detach HTTPCache instances).
4+
#
5+
# HTTPCache stops being shipped. Its PackageSource is pruned on upgrade and takes
6+
# the chart artifacts with it, so the migration orphans every tenant
7+
# http-cache-<name> HelmRelease (suspend, drop the Flux finalizer, verify it is
8+
# gone, delete) and then removes the platform side: the Package, the
9+
# http-cache-rd release, the ApplicationDefinition and the release's Helm
10+
# storage.
11+
#
12+
# Pinned here:
13+
#
14+
# 1. SELECTION. A release is detached when it renders from the http-cache chart
15+
# or carries the HTTPCache application label, and no other release is touched.
16+
#
17+
# 2. ORDER. Each release goes suspend -> finalizer-drop -> delete. All tenant
18+
# releases are detached before anything on the platform side is deleted, and
19+
# the Package is deleted before hr/http-cache-rd: the operator owns that
20+
# release and re-creates it for as long as its Package exists.
21+
#
22+
# 3. FAIL CLOSED. A finalizer that survives its patch, a failed verify read, a
23+
# failed fleet scan or a failed platform delete aborts the migration before
24+
# it stamps the version.
25+
#
26+
# The harness (docker, jq baked onto the migrations base image, the explicit
27+
# `return` in run_migration) is the one hack/migration-55-fluxcd-orphan.bats
28+
# describes; see there for why each piece is load-bearing.
29+
#
30+
# Run with: hack/cozytest.sh hack/migration-57-http-cache-detach.bats
31+
# -----------------------------------------------------------------------------
32+
33+
FAKEBIN="$PWD/hack/testdata/migration-57-http-cache"
34+
MIG_DIR="$PWD/packages/core/platform/images/migrations/migrations"
35+
ALPINE=$(sed -n 's/^FROM \(alpine:[^ ]*\).*$/\1/p' \
36+
"$PWD/packages/core/platform/images/migrations/Dockerfile" | head -1)
37+
TESTIMG="cozystack-migration57-test:$(printf '%s' "$ALPINE" | sed 's/[^a-zA-Z0-9]/-/g')"
38+
CHART=cozystack-http-cache-application-default-http-cache
39+
40+
run_migration() {
41+
_run_migration_rc=0
42+
docker run --rm --network none \
43+
--user "$(id -u):$(id -g)" \
44+
-v "$MIG_DIR:/migrations:ro" \
45+
-v "$FAKEBIN:/fakebin:ro" \
46+
-v "$WORK:/work" \
47+
-e PATH=/fakebin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \
48+
-e FAKE_CMDLOG=/work/cmdlog \
49+
-e NAMESPACE="${NAMESPACE-}" \
50+
-e FAKE_HRS="${FAKE_HRS-}" \
51+
-e FAKE_LIST_FAIL="${FAKE_LIST_FAIL-}" \
52+
-e FAKE_FINALIZER_STICKS="${FAKE_FINALIZER_STICKS-}" \
53+
-e FAKE_VERIFY_READ_FAIL="${FAKE_VERIFY_READ_FAIL-}" \
54+
-e FAKE_PLATFORM_DELETE_FAIL="${FAKE_PLATFORM_DELETE_FAIL-}" \
55+
"$TESTIMG" "/migrations/$1" || _run_migration_rc=$?
56+
return "$_run_migration_rc"
57+
}
58+
59+
prep() {
60+
docker info >/dev/null 2>&1 || {
61+
echo "docker is required: these tests run migration 57 inside a jq-enabled" >&2
62+
echo "build of $ALPINE, the migrations image's base." >&2
63+
return 1
64+
}
65+
docker build -q -t "$TESTIMG" - >/dev/null <<DOCKERFILE
66+
FROM $ALPINE
67+
RUN apk add --no-cache jq
68+
DOCKERFILE
69+
chmod +x "$FAKEBIN/kubectl"
70+
WORK=$(mktemp -d)
71+
export FAKE_CMDLOG="$WORK/cmdlog"
72+
: > "$FAKE_CMDLOG"
73+
export NAMESPACE=cozy-system
74+
export FAKE_HRS=""
75+
unset FAKE_LIST_FAIL FAKE_FINALIZER_STICKS FAKE_VERIFY_READ_FAIL FAKE_PLATFORM_DELETE_FAIL || true
76+
return 0
77+
}
78+
79+
cmdlog_line() {
80+
grep -nE -- "$1" "$FAKE_CMDLOG" | head -1 | cut -d: -f1
81+
}
82+
83+
# Last matching line, for "every X happened before Y" checks.
84+
cmdlog_last() {
85+
grep -nE -- "$1" "$FAKE_CMDLOG" | tail -1 | cut -d: -f1
86+
}
87+
88+
assert_detached() {
89+
_s=$(cmdlog_line "^SUSPEND $1 $2$")
90+
_f=$(cmdlog_line "^FINALIZER-PATCH $1 $2$")
91+
_d=$(cmdlog_line "^DELETE-HR $1 $2$")
92+
[ -n "$_s" ] && [ -n "$_f" ] && [ -n "$_d" ] || { echo "missing step for $1/$2: suspend=$_s finalizer=$_f delete=$_d" >&2; return 1; }
93+
[ "$_s" -lt "$_f" ] && [ "$_f" -lt "$_d" ] || { echo "out-of-order for $1/$2: suspend=$_s finalizer=$_f delete=$_d" >&2; return 1; }
94+
return 0
95+
}
96+
97+
@test "detaches every HTTPCache release, then removes the platform side Package first, and stamps 58" {
98+
prep
99+
# tenant-a/http-cache-web matches on chart and label, tenant-b/http-cache-old
100+
# only on the label (chartRef never rewritten), tenant-a/redis-cache on neither.
101+
export FAKE_HRS="tenant-a http-cache-web $CHART HTTPCache
102+
tenant-b http-cache-old some-old-chart HTTPCache
103+
tenant-a redis-cache cozystack-redis-application-default-redis Redis
104+
cozy-system http-cache-rd ${CHART}-rd -"
105+
rc=0
106+
run_migration 57 >"$WORK/out" 2>&1 || rc=$?
107+
cat "$WORK/out"; cat "$FAKE_CMDLOG"
108+
[ "$rc" -eq 0 ]
109+
110+
assert_detached tenant-a http-cache-web
111+
assert_detached tenant-b http-cache-old
112+
[ "$(grep -cE '^SUSPEND ' "$FAKE_CMDLOG")" -eq 2 ]
113+
if grep -q 'redis-cache' "$FAKE_CMDLOG"; then echo "unrelated redis-cache was touched" >&2; return 1; fi
114+
# The rd release is removed by the platform step, never orphaned: orphaning it
115+
# would keep the ApplicationDefinition alive.
116+
if grep -qE '^(SUSPEND|FINALIZER-PATCH) cozy-system http-cache-rd$' "$FAKE_CMDLOG"; then echo "http-cache-rd was orphaned" >&2; return 1; fi
117+
118+
_last_tenant=$(cmdlog_last '^DELETE-HR tenant-')
119+
_pkg=$(cmdlog_line '^DELETE-PACKAGE cozystack.http-cache-application$')
120+
_rd=$(cmdlog_line '^DELETE-HR cozy-system http-cache-rd$')
121+
_ad=$(cmdlog_line '^DELETE-APPDEF http-cache$')
122+
_sec=$(cmdlog_line '^DELETE-SECRETS cozy-system owner=helm,name=http-cache-rd$')
123+
_stamp=$(cmdlog_line '^STAMP 58$')
124+
[ -n "$_pkg" ] && [ -n "$_rd" ] && [ -n "$_ad" ] && [ -n "$_sec" ] && [ -n "$_stamp" ] || { echo "missing platform step: pkg=$_pkg rd=$_rd appdef=$_ad secrets=$_sec stamp=$_stamp" >&2; return 1; }
125+
[ "$_last_tenant" -lt "$_pkg" ] || { echo "platform cleanup started before all tenants were detached" >&2; return 1; }
126+
[ "$_pkg" -lt "$_rd" ] || { echo "http-cache-rd deleted before its Package; the operator would re-create it" >&2; return 1; }
127+
[ "$_sec" -lt "$_stamp" ] && [ "$_ad" -lt "$_stamp" ]
128+
rm -rf "$WORK"
129+
}
130+
131+
@test "with no HTTPCache instances it still removes the platform side and stamps 58" {
132+
prep
133+
export FAKE_HRS="tenant-a redis-cache cozystack-redis-application-default-redis Redis"
134+
rc=0
135+
run_migration 57 >"$WORK/out" 2>&1 || rc=$?
136+
cat "$WORK/out"; cat "$FAKE_CMDLOG"
137+
[ "$rc" -eq 0 ]
138+
if grep -qE '^SUSPEND ' "$FAKE_CMDLOG"; then echo "unexpected SUSPEND with no instances" >&2; return 1; fi
139+
grep -qE '^DELETE-PACKAGE cozystack.http-cache-application$' "$FAKE_CMDLOG"
140+
grep -qE '^DELETE-APPDEF http-cache$' "$FAKE_CMDLOG"
141+
grep -qF 'STAMP 58' "$FAKE_CMDLOG"
142+
rm -rf "$WORK"
143+
}
144+
145+
@test "a finalizer surviving the patch aborts before any delete and before stamping" {
146+
prep
147+
export FAKE_HRS="tenant-a http-cache-web $CHART HTTPCache"
148+
export FAKE_FINALIZER_STICKS=1
149+
rc=0
150+
run_migration 57 >"$WORK/out" 2>&1 || rc=$?
151+
cat "$WORK/out"; cat "$FAKE_CMDLOG"
152+
[ "$rc" -ne 0 ]
153+
grep -qF 'refusing to delete' "$WORK/out"
154+
if grep -qE '^DELETE-' "$FAKE_CMDLOG"; then echo "unexpected delete after stuck finalizer" >&2; return 1; fi
155+
if grep -qF 'STAMP' "$FAKE_CMDLOG"; then echo "unexpected STAMP after stuck finalizer" >&2; return 1; fi
156+
rm -rf "$WORK"
157+
}
158+
159+
@test "a failing finalizer verify read aborts before any delete and before stamping" {
160+
prep
161+
export FAKE_HRS="tenant-a http-cache-web $CHART HTTPCache"
162+
export FAKE_VERIFY_READ_FAIL="Error from server (Forbidden): helmreleases.helm.toolkit.fluxcd.io \"http-cache-web\" is forbidden"
163+
rc=0
164+
run_migration 57 >"$WORK/out" 2>&1 || rc=$?
165+
cat "$WORK/out"; cat "$FAKE_CMDLOG"
166+
[ "$rc" -ne 0 ]
167+
if grep -qE '^DELETE-' "$FAKE_CMDLOG"; then echo "unexpected delete after failed verify read" >&2; return 1; fi
168+
if grep -qF 'STAMP' "$FAKE_CMDLOG"; then echo "unexpected STAMP after failed verify read" >&2; return 1; fi
169+
rm -rf "$WORK"
170+
}
171+
172+
@test "a failing fleet scan aborts before touching anything and before stamping" {
173+
prep
174+
export FAKE_HRS="tenant-a http-cache-web $CHART HTTPCache"
175+
export FAKE_LIST_FAIL="Error from server (Timeout): the server was unable to return a response in the time allotted"
176+
rc=0
177+
run_migration 57 >"$WORK/out" 2>&1 || rc=$?
178+
cat "$WORK/out"; cat "$FAKE_CMDLOG"
179+
[ "$rc" -ne 0 ]
180+
if grep -qE '^(SUSPEND|DELETE-)' "$FAKE_CMDLOG"; then echo "unexpected action after failed fleet scan" >&2; return 1; fi
181+
if grep -qF 'STAMP' "$FAKE_CMDLOG"; then echo "unexpected STAMP after failed fleet scan" >&2; return 1; fi
182+
rm -rf "$WORK"
183+
}
184+
185+
@test "a failing platform delete aborts before stamping" {
186+
prep
187+
export FAKE_HRS=""
188+
export FAKE_PLATFORM_DELETE_FAIL="Error from server (InternalError): etcdserver: request timed out"
189+
rc=0
190+
run_migration 57 >"$WORK/out" 2>&1 || rc=$?
191+
cat "$WORK/out"; cat "$FAKE_CMDLOG"
192+
[ "$rc" -ne 0 ]
193+
if grep -qF 'STAMP' "$FAKE_CMDLOG"; then echo "unexpected STAMP after failed platform delete" >&2; return 1; fi
194+
rm -rf "$WORK"
195+
}
Lines changed: 111 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,111 @@
1+
#!/bin/sh
2+
# Fake kubectl for hack/migration-57-http-cache-detach.bats. It serves only the
3+
# calls migration 57 (and lib/cozystack-version.sh) make; behaviour is driven by
4+
# FAKE_* env and every invocation is appended to $FAKE_CMDLOG so the test can
5+
# assert what was detached, what was deleted on the platform side, and in what
6+
# order. Modelled on hack/testdata/migration-55-fluxcd/kubectl.
7+
#
8+
# FAKE_CMDLOG file to append a one-line record of each call to
9+
# FAKE_HRS newline list of "<ns> <name> <chartRef.name> [<kind>]",
10+
# one per HelmRelease the fleet scan reports; <kind> is
11+
# the apps.cozystack.io/application.kind label, "-" or
12+
# absent for none. Selection is the migration's jq, so
13+
# non-matching rows must be listed and then ignored.
14+
# FAKE_LIST_FAIL non-empty => the fleet scan exits 1 with this stderr
15+
# FAKE_FINALIZER_STICKS non-empty => the verify read returns a finalizer
16+
# FAKE_VERIFY_READ_FAIL non-empty => the verify read exits 1 with this stderr
17+
# FAKE_PLATFORM_DELETE_FAIL non-empty => deleting the Package exits 1
18+
set -u
19+
20+
log() { [ -n "${FAKE_CMDLOG:-}" ] && printf '%s\n' "$*" >> "$FAKE_CMDLOG"; }
21+
log "KUBECTL $*"
22+
23+
args="$*"
24+
25+
arg_after() { k="$1"; shift; p=""; for a in "$@"; do [ "$p" = "$k" ] && { printf '%s' "$a"; return; }; p="$a"; done; }
26+
27+
case "$args" in
28+
*"apply --filename -"*)
29+
manifest=$(cat)
30+
v=$(printf '%s\n' "$manifest" | sed -n 's/^ version: "\{0,1\}\([0-9]\{1,\}\)"\{0,1\}$/\1/p' | head -1)
31+
log "STAMP ${v:-unknown}"
32+
exit 0
33+
;;
34+
35+
# Finalizer verify read. Shares the fleet scan's resource token, so it is
36+
# matched first and told apart by its jsonpath selector.
37+
*"get helmreleases.helm.toolkit.fluxcd.io"*"jsonpath"*)
38+
if [ -n "${FAKE_VERIFY_READ_FAIL:-}" ]; then
39+
printf '%s\n' "$FAKE_VERIFY_READ_FAIL" >&2
40+
exit 1
41+
fi
42+
if [ -n "${FAKE_FINALIZER_STICKS:-}" ]; then
43+
printf '%s' '["helm.toolkit.fluxcd.io/finalizer"]'
44+
fi
45+
exit 0
46+
;;
47+
48+
*"get helmreleases.helm.toolkit.fluxcd.io"*)
49+
if [ -n "${FAKE_LIST_FAIL:-}" ]; then
50+
printf '%s\n' "$FAKE_LIST_FAIL" >&2
51+
exit 1
52+
fi
53+
printf '{"items":['
54+
printf '%s\n' "${FAKE_HRS:-}" | {
55+
sep=""
56+
while read -r ns name chart kind; do
57+
[ -n "$ns" ] || continue
58+
if [ -n "${kind:-}" ] && [ "$kind" != "-" ]; then
59+
labels=$(printf '{"apps.cozystack.io/application.kind":"%s"}' "$kind")
60+
else
61+
labels=null
62+
fi
63+
printf '%s{"metadata":{"namespace":"%s","name":"%s","labels":%s},"spec":{"chartRef":{"name":"%s"}}}' \
64+
"$sep" "$ns" "$name" "$labels" "$chart"
65+
sep=","
66+
done
67+
}
68+
printf ']}\n'
69+
exit 0
70+
;;
71+
72+
*"patch helmreleases.helm.toolkit.fluxcd.io"*)
73+
ns=$(arg_after -n "$@")
74+
name=$(arg_after helmreleases.helm.toolkit.fluxcd.io "$@")
75+
case "$args" in
76+
*suspend*) log "SUSPEND $ns $name" ;;
77+
*finalizers*) log "FINALIZER-PATCH $ns $name" ;;
78+
esac
79+
exit 0
80+
;;
81+
82+
*"delete helmreleases.helm.toolkit.fluxcd.io"*)
83+
ns=$(arg_after -n "$@")
84+
name=$(arg_after helmreleases.helm.toolkit.fluxcd.io "$@")
85+
log "DELETE-HR $ns $name"
86+
exit 0
87+
;;
88+
89+
*"delete packages.cozystack.io"*)
90+
if [ -n "${FAKE_PLATFORM_DELETE_FAIL:-}" ]; then
91+
printf '%s\n' "$FAKE_PLATFORM_DELETE_FAIL" >&2
92+
exit 1
93+
fi
94+
log "DELETE-PACKAGE $(arg_after packages.cozystack.io "$@")"
95+
exit 0
96+
;;
97+
98+
*"delete applicationdefinitions.cozystack.io"*)
99+
log "DELETE-APPDEF $(arg_after applicationdefinitions.cozystack.io "$@")"
100+
exit 0
101+
;;
102+
103+
*"delete secret"*)
104+
log "DELETE-SECRETS $(arg_after -n "$@") $(arg_after -l "$@")"
105+
exit 0
106+
;;
107+
esac
108+
109+
log "UNHANDLED $args"
110+
printf 'fake kubectl: unmodelled call: %s\n' "$args" >&2
111+
exit 1
Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
#!/bin/sh
2+
# Migration 57 --> 58
3+
# Detach surviving HTTPCache instances from Cozystack and remove what the
4+
# application leaves behind on the platform side.
5+
#
6+
# HTTPCache is no longer shipped. Its PackageSource carries no keep annotation,
7+
# so Helm prunes it on upgrade, and the ArtifactGenerator it owns takes both
8+
# ExternalArtifacts with it. Left alone, every http-cache HelmRelease would lose
9+
# its chart and stay Ready=False for good, and HelmReleaseNotReady would fire on
10+
# every cluster with the naas bundle.
11+
#
12+
# So each tenant release is orphaned instead: suspend it, drop the Flux
13+
# finalizer so the delete does NOT run a Helm uninstall, verify the finalizer is
14+
# gone, then delete. The Deployments, Services and ConfigMaps it rendered stay
15+
# in the tenant namespace as plain objects, since nothing in them points back at
16+
# the HelmRelease. This is the orphan idiom of migrations 29 and 55.
17+
#
18+
# The platform side goes after, Package first. The operator owns
19+
# hr/http-cache-rd and re-creates it on delete for as long as its Package
20+
# exists, so deleting the release first would only race the operator; deleting
21+
# the Package garbage-collects the release through its controller
22+
# ownerReference. The ApplicationDefinition and the release's Helm storage are
23+
# deleted explicitly too, so the kind stops being served whether or not
24+
# helm-controller has run the uninstall by the time the upgrade goes on.
25+
#
26+
# Fail-closed for the same reason as migration 55: migrations never re-run, so a
27+
# swallowed error would stamp the version past a half-detached fleet. Every step
28+
# runs under `set -euo pipefail` with no `|| true`, and each tenant delete is
29+
# gated on a finalizer read that aborts on any API error.
30+
31+
set -euo pipefail
32+
33+
# shellcheck source=lib/cozystack-version.sh
34+
. "$(dirname "$0")/lib/cozystack-version.sh"
35+
36+
# Match on the chart the release renders from, and on the application label
37+
# migration 22 put on every tenant release, so a release whose chartRef was
38+
# never rewritten is still found.
39+
hrs=$(kubectl get helmreleases.helm.toolkit.fluxcd.io --all-namespaces -o json \
40+
| jq -r '.items[]
41+
| select(.spec.chartRef.name == "cozystack-http-cache-application-default-http-cache"
42+
or .metadata.labels["apps.cozystack.io/application.kind"] == "HTTPCache")
43+
| "\(.metadata.namespace) \(.metadata.name)"')
44+
45+
# Heredoc, not a pipe, so the loop runs in this shell and `exit 1` aborts the
46+
# migration rather than a subshell.
47+
if [ -n "$hrs" ]; then
48+
while read -r ns name; do
49+
[ -n "$ns" ] || continue
50+
echo "Detaching HTTPCache instance ${ns}/${name} from Cozystack"
51+
kubectl -n "$ns" patch helmreleases.helm.toolkit.fluxcd.io "$name" --type=merge \
52+
-p '{"spec":{"suspend":true}}'
53+
kubectl -n "$ns" patch helmreleases.helm.toolkit.fluxcd.io "$name" --type=merge \
54+
-p '{"metadata":{"finalizers":null}}'
55+
remaining=$(kubectl -n "$ns" get helmreleases.helm.toolkit.fluxcd.io "$name" \
56+
--ignore-not-found -o jsonpath='{.metadata.finalizers}')
57+
if [ -n "$remaining" ]; then
58+
echo "ERROR: finalizer still present on ${ns}/${name} after patch (${remaining}); refusing to delete" >&2
59+
exit 1
60+
fi
61+
kubectl -n "$ns" delete helmreleases.helm.toolkit.fluxcd.io "$name" --ignore-not-found --wait=false
62+
done <<EOF
63+
$hrs
64+
EOF
65+
fi
66+
67+
kubectl delete packages.cozystack.io cozystack.http-cache-application --ignore-not-found --wait=false
68+
kubectl -n cozy-system delete helmreleases.helm.toolkit.fluxcd.io http-cache-rd --ignore-not-found --wait=false
69+
kubectl delete applicationdefinitions.cozystack.io http-cache --ignore-not-found
70+
kubectl -n cozy-system delete secret -l owner=helm,name=http-cache-rd --ignore-not-found
71+
72+
stamp_cozystack_version 58

‎packages/core/platform/values.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ migrations:
1414
# refuses to advance past a migration file missing from the image (exit 1), so
1515
# a stale pin fails loudly rather than silently skipping the etcd adoption.
1616
image: ghcr.io/cozystack/cozystack/platform-migrations:v1.6.0@sha256:6777042e3e9c6bad76e7d96fb64baa115061975f39061cb3ede84e21d0e2213f
17-
targetVersion: 57
17+
targetVersion: 58
1818
# Adopt legacy etcd clusters onto the v1alpha2 operator WITHOUT the mandatory
1919
# pre-adoption safety snapshot (migration 50). Leave false unless migration 50
2020
# has actually refused to proceed and you accept the risk: adoption rewires

0 commit comments

Comments
 (0)