Skip to content

Commit 198d8ac

Browse files
feat(platform): upgrade snapshot-controller to v8.7.0
snapshot-controller v8.0.1 serves VolumeGroupSnapshots only as v1alpha1 and runs without the CSIVolumeGroupSnapshot gate, so no CSI driver on the platform can take a crash-consistent snapshot of several volumes at once. Velero 1.18 and piraeus-operator both look for groupsnapshot.storage.k8s.io/v1beta2, which v8.7.0 serves and which the upstream chart enables by default. The chart now renders its CRDs from a template instead of crds/. Helm refuses to take over objects that lack its ownership metadata, and the apiserver refuses to drop v1alpha1 from CRDs that list it in storedVersions, so migration 61 adopts the snapshot CRDs the release installed and deletes the v1alpha1 group snapshot CRDs when they are empty. keepCRDs is set because an uninstall would otherwise delete every VolumeSnapshot along with the CRDs. Assisted-by: LLM Signed-off-by: Mattia Eleuteri <[email protected]>
1 parent 970191d commit 198d8ac

38 files changed

Lines changed: 3848 additions & 2148 deletions
Lines changed: 194 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,194 @@
1+
#!/usr/bin/env bats
2+
# -----------------------------------------------------------------------------
3+
# Unit tests for platform migration 61 --> 62 (hand the snapshot CRDs over to
4+
# the snapshot-controller release and delete the empty v1alpha1
5+
# VolumeGroupSnapshot CRDs). See
6+
# packages/core/platform/images/migrations/migrations/61 for the mechanism.
7+
#
8+
# The fake keeps the CRDs in a state file and applies each annotate, label and
9+
# delete to it, so a second run reads what the first one wrote.
10+
#
11+
# The harness (docker, jq baked onto the migrations base image, the explicit
12+
# `return` in run_migration) is the one hack/migration-55-fluxcd-orphan.bats
13+
# describes; see there for why each piece is load-bearing.
14+
#
15+
# Run with: hack/cozytest.sh hack/migration-61-snapshot-crds.bats
16+
# -----------------------------------------------------------------------------
17+
18+
load test_helper
19+
20+
FIXTURES="$PWD/hack/testdata/migration-61-snapshot-crds"
21+
MIG_DIR="$PWD/packages/core/platform/images/migrations/migrations"
22+
ALPINE=$(sed -n 's/^FROM \(alpine:[^ ]*\).*$/\1/p' \
23+
"$PWD/packages/core/platform/images/migrations/Dockerfile" | head -1)
24+
TESTIMG="cozystack-migration61-test:$(printf '%s' "$ALPINE" | sed 's/[^a-zA-Z0-9]/-/g')"
25+
WORKROOT="${TMPDIR:-/tmp}/cozy-migration-61-$$"
26+
27+
cozy_cleanup() {
28+
rm -rf "$WORKROOT"
29+
return 0
30+
}
31+
32+
run_migration() {
33+
_run_migration_rc=0
34+
docker run --rm --network none \
35+
--user "$(id -u):$(id -g)" \
36+
-v "$MIG_DIR:/migrations:ro" \
37+
-v "$FIXTURES:/fakebin:ro" \
38+
-v "$WORK:/work" \
39+
-e PATH=/fakebin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \
40+
-e FAKE_CMDLOG=/work/cmdlog \
41+
-e FAKE_STATE=/work/state \
42+
-e NAMESPACE="${NAMESPACE-}" \
43+
-e FAKE_CRD_FAIL="${FAKE_CRD_FAIL-}" \
44+
-e FAKE_DELETE_FAIL="${FAKE_DELETE_FAIL-}" \
45+
"$TESTIMG" "/migrations/$1" || _run_migration_rc=$?
46+
return "$_run_migration_rc"
47+
}
48+
49+
prep() {
50+
docker info >/dev/null 2>&1 || {
51+
echo "docker is required: these tests run migration 61 inside a jq-enabled" >&2
52+
echo "build of $ALPINE, the migrations image's base." >&2
53+
return 1
54+
}
55+
docker build -q -t "$TESTIMG" - >/dev/null <<DOCKERFILE
56+
FROM $ALPINE
57+
RUN apk add --no-cache jq
58+
DOCKERFILE
59+
chmod +x "$FIXTURES/kubectl"
60+
mkdir -p "$WORKROOT"
61+
WORK=$(mktemp -d "$WORKROOT/XXXXXX")
62+
mkdir -p "$WORK/state"
63+
cp "$FIXTURES/crds.json" "$WORK/state/"
64+
echo '{}' > "$WORK/state/objects.json"
65+
FAKE_CMDLOG="$WORK/cmdlog"
66+
: > "$FAKE_CMDLOG"
67+
export NAMESPACE=cozy-system
68+
unset FAKE_CRD_FAIL FAKE_DELETE_FAIL || true
69+
return 0
70+
}
71+
72+
# edit_crds <jq filter> -- rewrite the CRD state before a run.
73+
edit_crds() {
74+
jq "$1" "$WORK/state/crds.json" > "$WORK/state/crds.json.new"
75+
mv "$WORK/state/crds.json.new" "$WORK/state/crds.json"
76+
return 0
77+
}
78+
79+
crd_field() {
80+
jq -r --arg n "$1" ".items[] | select(.metadata.name == \$n) | $2" "$WORK/state/crds.json"
81+
}
82+
83+
@test "adopts the snapshot CRDs and deletes the empty v1alpha1 group snapshot CRDs" {
84+
prep
85+
rc=0
86+
run_migration 61 >"$WORK/out" 2>&1 || rc=$?
87+
cat "$WORK/out"; cat "$FAKE_CMDLOG"
88+
[ "$rc" -eq 0 ]
89+
90+
for p in volumesnapshotclasses volumesnapshotcontents volumesnapshots; do
91+
crd="$p.snapshot.storage.k8s.io"
92+
[ "$(crd_field "$crd" '.metadata.annotations["meta.helm.sh/release-name"]')" = "snapshot-controller" ]
93+
[ "$(crd_field "$crd" '.metadata.annotations["meta.helm.sh/release-namespace"]')" = "cozy-snapshot-controller" ]
94+
[ "$(crd_field "$crd" '.metadata.annotations["helm.sh/resource-policy"]')" = "keep" ]
95+
[ "$(crd_field "$crd" '.metadata.labels["app.kubernetes.io/managed-by"]')" = "Helm" ]
96+
done
97+
98+
for p in volumegroupsnapshotclasses volumegroupsnapshotcontents volumegroupsnapshots; do
99+
grep -qxF "DELETE $p.groupsnapshot.storage.k8s.io" "$FAKE_CMDLOG"
100+
done
101+
if grep -qF "ANNOTATE volumegroupsnapshot" "$FAKE_CMDLOG"; then echo "a v1alpha1 CRD was adopted" >&2; return 1; fi
102+
if grep -qF "RECORD" "$FAKE_CMDLOG"; then echo "unexpected record" >&2; return 1; fi
103+
grep -qxF "STAMP 62" "$FAKE_CMDLOG"
104+
rm -rf "$WORK"
105+
}
106+
107+
@test "a second run finds nothing to change" {
108+
prep
109+
run_migration 61 >"$WORK/out" 2>&1
110+
: > "$FAKE_CMDLOG"
111+
rc=0
112+
run_migration 61 >"$WORK/out" 2>&1 || rc=$?
113+
cat "$WORK/out"; cat "$FAKE_CMDLOG"
114+
[ "$rc" -eq 0 ]
115+
if grep -qE "^(ANNOTATE|LABEL|DELETE)" "$FAKE_CMDLOG"; then echo "second run changed a CRD" >&2; return 1; fi
116+
grep -qxF "STAMP 62" "$FAKE_CMDLOG"
117+
rm -rf "$WORK"
118+
}
119+
120+
@test "a group snapshot CRD that holds objects is kept and recorded" {
121+
prep
122+
echo '{"volumegroupsnapshots.groupsnapshot.storage.k8s.io": 2}' > "$WORK/state/objects.json"
123+
rc=0
124+
run_migration 61 >"$WORK/out" 2>&1 || rc=$?
125+
cat "$WORK/out"; cat "$FAKE_CMDLOG"
126+
[ "$rc" -eq 0 ]
127+
if grep -qxF "DELETE volumegroupsnapshots.groupsnapshot.storage.k8s.io" "$FAKE_CMDLOG"; then
128+
echo "a CRD holding objects was deleted" >&2; return 1
129+
fi
130+
grep -qxF "DELETE volumegroupsnapshotcontents.groupsnapshot.storage.k8s.io" "$FAKE_CMDLOG"
131+
grep -qF "cozystack.io/migration-61-needs-attention=volumegroupsnapshots.groupsnapshot.storage.k8s.io=2" "$FAKE_CMDLOG"
132+
grep -qxF "STAMP 62" "$FAKE_CMDLOG"
133+
rm -rf "$WORK"
134+
}
135+
136+
@test "CRDs the release did not install are left alone" {
137+
prep
138+
edit_crds '.items |= map(.metadata.labels = {})'
139+
rc=0
140+
run_migration 61 >"$WORK/out" 2>&1 || rc=$?
141+
cat "$WORK/out"; cat "$FAKE_CMDLOG"
142+
[ "$rc" -eq 0 ]
143+
if grep -qE "^(ANNOTATE|LABEL|DELETE)" "$FAKE_CMDLOG"; then echo "a foreign CRD was changed" >&2; return 1; fi
144+
grep -qxF "STAMP 62" "$FAKE_CMDLOG"
145+
rm -rf "$WORK"
146+
}
147+
148+
@test "a group snapshot CRD that already serves v1beta2 is adopted, not deleted" {
149+
prep
150+
edit_crds '.items |= map(if .spec.group == "groupsnapshot.storage.k8s.io"
151+
then .spec.versions = [{name: "v1beta2", served: true, storage: true}] else . end)'
152+
rc=0
153+
run_migration 61 >"$WORK/out" 2>&1 || rc=$?
154+
cat "$WORK/out"; cat "$FAKE_CMDLOG"
155+
[ "$rc" -eq 0 ]
156+
if grep -qF "DELETE" "$FAKE_CMDLOG"; then echo "a v1beta2 CRD was deleted" >&2; return 1; fi
157+
[ "$(crd_field volumegroupsnapshots.groupsnapshot.storage.k8s.io '.metadata.annotations["meta.helm.sh/release-name"]')" = "snapshot-controller" ]
158+
grep -qxF "STAMP 62" "$FAKE_CMDLOG"
159+
rm -rf "$WORK"
160+
}
161+
162+
@test "a cluster without the snapshot CRDs only stamps the version" {
163+
prep
164+
edit_crds '.items = []'
165+
rc=0
166+
run_migration 61 >"$WORK/out" 2>&1 || rc=$?
167+
cat "$WORK/out"; cat "$FAKE_CMDLOG"
168+
[ "$rc" -eq 0 ]
169+
if grep -qE "^(ANNOTATE|LABEL|DELETE)" "$FAKE_CMDLOG"; then echo "unexpected change" >&2; return 1; fi
170+
grep -qxF "STAMP 62" "$FAKE_CMDLOG"
171+
rm -rf "$WORK"
172+
}
173+
174+
@test "a failing CRD read aborts instead of treating the CRD as absent" {
175+
prep
176+
export FAKE_CRD_FAIL="Error from server (InternalError): etcdserver: request timed out"
177+
rc=0
178+
run_migration 61 >"$WORK/out" 2>&1 || rc=$?
179+
cat "$WORK/out"; cat "$FAKE_CMDLOG"
180+
[ "$rc" -ne 0 ]
181+
if grep -qF "STAMP" "$FAKE_CMDLOG"; then echo "unexpected STAMP after failed CRD read" >&2; return 1; fi
182+
rm -rf "$WORK"
183+
}
184+
185+
@test "a failing delete aborts before stamping" {
186+
prep
187+
export FAKE_DELETE_FAIL="Error from server (InternalError): etcdserver: request timed out"
188+
rc=0
189+
run_migration 61 >"$WORK/out" 2>&1 || rc=$?
190+
cat "$WORK/out"; cat "$FAKE_CMDLOG"
191+
[ "$rc" -ne 0 ]
192+
if grep -qF "STAMP" "$FAKE_CMDLOG"; then echo "unexpected STAMP after failed delete" >&2; return 1; fi
193+
rm -rf "$WORK"
194+
}
Lines changed: 201 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,201 @@
1+
{
2+
"apiVersion": "v1",
3+
"kind": "List",
4+
"items": [
5+
{
6+
"apiVersion": "apiextensions.k8s.io/v1",
7+
"kind": "CustomResourceDefinition",
8+
"metadata": {
9+
"name": "volumesnapshotclasses.snapshot.storage.k8s.io",
10+
"labels": {
11+
"helm.toolkit.fluxcd.io/name": "snapshot-controller",
12+
"helm.toolkit.fluxcd.io/namespace": "cozy-snapshot-controller"
13+
},
14+
"annotations": {
15+
"api-approved.kubernetes.io": "https://github.com/kubernetes-csi/external-snapshotter/pull/814",
16+
"controller-gen.kubebuilder.io/version": "v0.15.0"
17+
}
18+
},
19+
"spec": {
20+
"group": "snapshot.storage.k8s.io",
21+
"versions": [
22+
{
23+
"name": "v1",
24+
"served": true,
25+
"storage": true
26+
},
27+
{
28+
"name": "v1beta1",
29+
"served": false,
30+
"storage": false
31+
}
32+
]
33+
},
34+
"status": {
35+
"storedVersions": [
36+
"v1"
37+
]
38+
}
39+
},
40+
{
41+
"apiVersion": "apiextensions.k8s.io/v1",
42+
"kind": "CustomResourceDefinition",
43+
"metadata": {
44+
"name": "volumesnapshotcontents.snapshot.storage.k8s.io",
45+
"labels": {
46+
"helm.toolkit.fluxcd.io/name": "snapshot-controller",
47+
"helm.toolkit.fluxcd.io/namespace": "cozy-snapshot-controller"
48+
},
49+
"annotations": {
50+
"api-approved.kubernetes.io": "https://github.com/kubernetes-csi/external-snapshotter/pull/814",
51+
"controller-gen.kubebuilder.io/version": "v0.15.0"
52+
}
53+
},
54+
"spec": {
55+
"group": "snapshot.storage.k8s.io",
56+
"versions": [
57+
{
58+
"name": "v1",
59+
"served": true,
60+
"storage": true
61+
},
62+
{
63+
"name": "v1beta1",
64+
"served": false,
65+
"storage": false
66+
}
67+
]
68+
},
69+
"status": {
70+
"storedVersions": [
71+
"v1"
72+
]
73+
}
74+
},
75+
{
76+
"apiVersion": "apiextensions.k8s.io/v1",
77+
"kind": "CustomResourceDefinition",
78+
"metadata": {
79+
"name": "volumesnapshots.snapshot.storage.k8s.io",
80+
"labels": {
81+
"helm.toolkit.fluxcd.io/name": "snapshot-controller",
82+
"helm.toolkit.fluxcd.io/namespace": "cozy-snapshot-controller"
83+
},
84+
"annotations": {
85+
"api-approved.kubernetes.io": "https://github.com/kubernetes-csi/external-snapshotter/pull/814",
86+
"controller-gen.kubebuilder.io/version": "v0.15.0"
87+
}
88+
},
89+
"spec": {
90+
"group": "snapshot.storage.k8s.io",
91+
"versions": [
92+
{
93+
"name": "v1",
94+
"served": true,
95+
"storage": true
96+
},
97+
{
98+
"name": "v1beta1",
99+
"served": false,
100+
"storage": false
101+
}
102+
]
103+
},
104+
"status": {
105+
"storedVersions": [
106+
"v1"
107+
]
108+
}
109+
},
110+
{
111+
"apiVersion": "apiextensions.k8s.io/v1",
112+
"kind": "CustomResourceDefinition",
113+
"metadata": {
114+
"name": "volumegroupsnapshotclasses.groupsnapshot.storage.k8s.io",
115+
"labels": {
116+
"helm.toolkit.fluxcd.io/name": "snapshot-controller",
117+
"helm.toolkit.fluxcd.io/namespace": "cozy-snapshot-controller"
118+
},
119+
"annotations": {
120+
"api-approved.kubernetes.io": "https://github.com/kubernetes-csi/external-snapshotter/pull/814",
121+
"controller-gen.kubebuilder.io/version": "v0.15.0"
122+
}
123+
},
124+
"spec": {
125+
"group": "groupsnapshot.storage.k8s.io",
126+
"versions": [
127+
{
128+
"name": "v1alpha1",
129+
"served": true,
130+
"storage": true
131+
}
132+
]
133+
},
134+
"status": {
135+
"storedVersions": [
136+
"v1alpha1"
137+
]
138+
}
139+
},
140+
{
141+
"apiVersion": "apiextensions.k8s.io/v1",
142+
"kind": "CustomResourceDefinition",
143+
"metadata": {
144+
"name": "volumegroupsnapshotcontents.groupsnapshot.storage.k8s.io",
145+
"labels": {
146+
"helm.toolkit.fluxcd.io/name": "snapshot-controller",
147+
"helm.toolkit.fluxcd.io/namespace": "cozy-snapshot-controller"
148+
},
149+
"annotations": {
150+
"api-approved.kubernetes.io": "https://github.com/kubernetes-csi/external-snapshotter/pull/814",
151+
"controller-gen.kubebuilder.io/version": "v0.15.0"
152+
}
153+
},
154+
"spec": {
155+
"group": "groupsnapshot.storage.k8s.io",
156+
"versions": [
157+
{
158+
"name": "v1alpha1",
159+
"served": true,
160+
"storage": true
161+
}
162+
]
163+
},
164+
"status": {
165+
"storedVersions": [
166+
"v1alpha1"
167+
]
168+
}
169+
},
170+
{
171+
"apiVersion": "apiextensions.k8s.io/v1",
172+
"kind": "CustomResourceDefinition",
173+
"metadata": {
174+
"name": "volumegroupsnapshots.groupsnapshot.storage.k8s.io",
175+
"labels": {
176+
"helm.toolkit.fluxcd.io/name": "snapshot-controller",
177+
"helm.toolkit.fluxcd.io/namespace": "cozy-snapshot-controller"
178+
},
179+
"annotations": {
180+
"api-approved.kubernetes.io": "https://github.com/kubernetes-csi/external-snapshotter/pull/814",
181+
"controller-gen.kubebuilder.io/version": "v0.15.0"
182+
}
183+
},
184+
"spec": {
185+
"group": "groupsnapshot.storage.k8s.io",
186+
"versions": [
187+
{
188+
"name": "v1alpha1",
189+
"served": true,
190+
"storage": true
191+
}
192+
]
193+
},
194+
"status": {
195+
"storedVersions": [
196+
"v1alpha1"
197+
]
198+
}
199+
}
200+
]
201+
}

0 commit comments

Comments
 (0)