Skip to content

Commit 9256c29

Browse files
feat(platform): create the v1beta2 group snapshot CRDs in migration 61
The linstor release renders its group snapshot gate and VolumeGroupSnapshotClass only when the v1beta2 API is served. Its dependency on snapshot-controller can be satisfied by the old Ready revision, so linstor could render before the new CRDs exist, and nothing would render it again once they did. Every HelmRelease waits for the platform's pre-upgrade hook, so migration 61 now creates the CRDs it deletes straight away, from v1beta2 copies the image build renders out of the snapshot-controller chart with that release's ownership metadata. Both releases then render against the v1beta2 API. Assisted-by: LLM Signed-off-by: Mattia Eleuteri <[email protected]>
1 parent 7215e98 commit 9256c29

9 files changed

Lines changed: 230 additions & 18 deletions

File tree

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -126,6 +126,7 @@ examples/backups/*/.sentinel.env
126126
examples/backups/*/.backup-name.env
127127
examples/backups/*/.source-dump.txt
128128
packages/core/platform/images/migrations/etcd-operator-crds/
129+
packages/core/platform/images/migrations/snapshot-controller-crds/
129130

130131
# CI scratch: promote-rc.yaml checks the dispatch ref here so release tooling
131132
# (validator, agent instructions) is current even when the checked-out rc tag

‎hack/migration-61-snapshot-crds.bats‎

Lines changed: 45 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,14 @@
11
#!/usr/bin/env bats
22
# -----------------------------------------------------------------------------
33
# Unit tests for platform migration 61 --> 62 (hand the snapshot CRDs over to
4-
# the snapshot-controller release and delete the empty v1alpha1
5-
# VolumeGroupSnapshot CRDs). See
4+
# the snapshot-controller release and replace the empty v1alpha1
5+
# VolumeGroupSnapshot CRDs with v1beta2 ones). See
66
# packages/core/platform/images/migrations/migrations/61 for the mechanism.
77
#
8-
# The fake keeps the CRDs in a state file and applies each annotate, label and
9-
# delete to it, so a second run reads what the first one wrote.
8+
# The fake keeps the CRDs in a state file and applies each annotate, label,
9+
# delete and create to it, so a second run reads what the first one wrote. The
10+
# v1beta2 CRDs the migration creates come from group-crds/, standing in for the
11+
# copies `make image-migrations` bakes into the image.
1012
#
1113
# The harness (docker, jq baked onto the migrations base image, the explicit
1214
# `return` in run_migration) is the one hack/migration-55-fluxcd-orphan.bats
@@ -43,6 +45,8 @@ run_migration() {
4345
-e FAKE_CRD_FAIL="${FAKE_CRD_FAIL-}" \
4446
-e FAKE_DELETE_FAIL="${FAKE_DELETE_FAIL-}" \
4547
-e FAKE_RECORD_FAIL="${FAKE_RECORD_FAIL-}" \
48+
-e FAKE_CREATE_FAIL="${FAKE_CREATE_FAIL-}" \
49+
-e GROUP_CRD_DIR=/fakebin/group-crds \
4650
"$TESTIMG" "/migrations/$1" || _run_migration_rc=$?
4751
return "$_run_migration_rc"
4852
}
@@ -66,7 +70,7 @@ DOCKERFILE
6670
FAKE_CMDLOG="$WORK/cmdlog"
6771
: > "$FAKE_CMDLOG"
6872
export NAMESPACE=cozy-system
69-
unset FAKE_CRD_FAIL FAKE_DELETE_FAIL FAKE_RECORD_FAIL || true
73+
unset FAKE_CRD_FAIL FAKE_DELETE_FAIL FAKE_RECORD_FAIL FAKE_CREATE_FAIL || true
7074
return 0
7175
}
7276

@@ -81,7 +85,7 @@ crd_field() {
8185
jq -r --arg n "$1" ".items[] | select(.metadata.name == \$n) | $2" "$WORK/state/crds.json"
8286
}
8387

84-
@test "adopts the snapshot CRDs and deletes the empty v1alpha1 group snapshot CRDs" {
88+
@test "adopts the snapshot CRDs and replaces the empty v1alpha1 group snapshot CRDs" {
8589
prep
8690
rc=0
8791
run_migration 61 >"$WORK/out" 2>&1 || rc=$?
@@ -97,8 +101,15 @@ crd_field() {
97101
done
98102

99103
for p in volumegroupsnapshotclasses volumegroupsnapshotcontents volumegroupsnapshots; do
100-
grep -qxF "DELETE $p.groupsnapshot.storage.k8s.io" "$FAKE_CMDLOG"
104+
crd="$p.groupsnapshot.storage.k8s.io"
105+
grep -qxF "DELETE $crd" "$FAKE_CMDLOG"
106+
grep -qxF "CREATE $crd" "$FAKE_CMDLOG"
107+
[ "$(crd_field "$crd" '[.spec.versions[] | select(.served) | .name] | index("v1beta2") != null')" = "true" ]
108+
[ "$(crd_field "$crd" '.metadata.annotations["meta.helm.sh/release-name"]')" = "snapshot-controller" ]
101109
done
110+
# Every delete comes before the first create, so a kept CRD is known before
111+
# anything is created.
112+
[ "$(grep -nE '^(DELETE|CREATE) ' "$FAKE_CMDLOG" | sed -n '3p' | cut -d' ' -f1 | cut -d: -f2)" = "DELETE" ]
102113
if grep -qF "ANNOTATE volumegroupsnapshot" "$FAKE_CMDLOG"; then echo "a v1alpha1 CRD was adopted" >&2; return 1; fi
103114
if grep -qF "RECORD" "$FAKE_CMDLOG"; then echo "unexpected record" >&2; return 1; fi
104115
grep -qxF "STAMP 62" "$FAKE_CMDLOG"
@@ -113,7 +124,7 @@ crd_field() {
113124
run_migration 61 >"$WORK/out" 2>&1 || rc=$?
114125
cat "$WORK/out"; cat "$FAKE_CMDLOG"
115126
[ "$rc" -eq 0 ]
116-
if grep -qE "^(ANNOTATE|LABEL|DELETE)" "$FAKE_CMDLOG"; then echo "second run changed a CRD" >&2; return 1; fi
127+
if grep -qE "^(ANNOTATE|LABEL|DELETE|CREATE)" "$FAKE_CMDLOG"; then echo "second run changed a CRD" >&2; return 1; fi
117128
grep -qxF "STAMP 62" "$FAKE_CMDLOG"
118129
rm -rf "$WORK"
119130
}
@@ -129,6 +140,7 @@ crd_field() {
129140
echo "a CRD holding objects was deleted" >&2; return 1
130141
fi
131142
grep -qxF "DELETE volumegroupsnapshotcontents.groupsnapshot.storage.k8s.io" "$FAKE_CMDLOG"
143+
if grep -qF "CREATE" "$FAKE_CMDLOG"; then echo "a v1beta2 CRD was created next to a kept v1alpha1 one" >&2; return 1; fi
132144
grep -qF "cozystack.io/migration-61-needs-attention=volumegroupsnapshots.groupsnapshot.storage.k8s.io=2" "$FAKE_CMDLOG"
133145
grep -qxF "STAMP 62" "$FAKE_CMDLOG"
134146
rm -rf "$WORK"
@@ -155,7 +167,7 @@ crd_field() {
155167
run_migration 61 >"$WORK/out" 2>&1 || rc=$?
156168
cat "$WORK/out"; cat "$FAKE_CMDLOG"
157169
[ "$rc" -eq 0 ]
158-
if grep -qE "^(ANNOTATE|LABEL|DELETE)" "$FAKE_CMDLOG"; then echo "a foreign CRD was changed" >&2; return 1; fi
170+
if grep -qE "^(ANNOTATE|LABEL|DELETE|CREATE)" "$FAKE_CMDLOG"; then echo "a foreign CRD was changed" >&2; return 1; fi
159171
grep -qxF "STAMP 62" "$FAKE_CMDLOG"
160172
rm -rf "$WORK"
161173
}
@@ -168,7 +180,7 @@ crd_field() {
168180
run_migration 61 >"$WORK/out" 2>&1 || rc=$?
169181
cat "$WORK/out"; cat "$FAKE_CMDLOG"
170182
[ "$rc" -eq 0 ]
171-
if grep -qF "DELETE" "$FAKE_CMDLOG"; then echo "a v1beta2 CRD was deleted" >&2; return 1; fi
183+
if grep -qE "^(DELETE|CREATE)" "$FAKE_CMDLOG"; then echo "a v1beta2 CRD was replaced" >&2; return 1; fi
172184
[ "$(crd_field volumegroupsnapshots.groupsnapshot.storage.k8s.io '.metadata.annotations["meta.helm.sh/release-name"]')" = "snapshot-controller" ]
173185
grep -qxF "STAMP 62" "$FAKE_CMDLOG"
174186
rm -rf "$WORK"
@@ -181,7 +193,7 @@ crd_field() {
181193
run_migration 61 >"$WORK/out" 2>&1 || rc=$?
182194
cat "$WORK/out"; cat "$FAKE_CMDLOG"
183195
[ "$rc" -eq 0 ]
184-
if grep -qE "^(ANNOTATE|LABEL|DELETE)" "$FAKE_CMDLOG"; then echo "unexpected change" >&2; return 1; fi
196+
if grep -qE "^(ANNOTATE|LABEL|DELETE|CREATE)" "$FAKE_CMDLOG"; then echo "unexpected change" >&2; return 1; fi
185197
grep -qxF "STAMP 62" "$FAKE_CMDLOG"
186198
rm -rf "$WORK"
187199
}
@@ -207,3 +219,25 @@ crd_field() {
207219
if grep -qF "STAMP" "$FAKE_CMDLOG"; then echo "unexpected STAMP after failed delete" >&2; return 1; fi
208220
rm -rf "$WORK"
209221
}
222+
223+
@test "a failing create aborts before stamping, and the retry creates the CRDs" {
224+
prep
225+
export FAKE_CREATE_FAIL="Error from server (InternalError): etcdserver: request timed out"
226+
rc=0
227+
run_migration 61 >"$WORK/out" 2>&1 || rc=$?
228+
cat "$WORK/out"; cat "$FAKE_CMDLOG"
229+
[ "$rc" -ne 0 ]
230+
if grep -qF "STAMP" "$FAKE_CMDLOG"; then echo "unexpected STAMP after failed create" >&2; return 1; fi
231+
232+
unset FAKE_CREATE_FAIL
233+
: > "$FAKE_CMDLOG"
234+
rc=0
235+
run_migration 61 >"$WORK/out" 2>&1 || rc=$?
236+
cat "$WORK/out"; cat "$FAKE_CMDLOG"
237+
[ "$rc" -eq 0 ]
238+
for p in volumegroupsnapshotclasses volumegroupsnapshotcontents volumegroupsnapshots; do
239+
grep -qxF "CREATE $p.groupsnapshot.storage.k8s.io" "$FAKE_CMDLOG"
240+
done
241+
grep -qxF "STAMP 62" "$FAKE_CMDLOG"
242+
rm -rf "$WORK"
243+
}
Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
{
2+
"apiVersion": "apiextensions.k8s.io/v1",
3+
"kind": "CustomResourceDefinition",
4+
"metadata": {
5+
"name": "volumegroupsnapshotclasses.groupsnapshot.storage.k8s.io",
6+
"labels": {
7+
"helm.toolkit.fluxcd.io/name": "snapshot-controller",
8+
"helm.toolkit.fluxcd.io/namespace": "cozy-snapshot-controller",
9+
"app.kubernetes.io/managed-by": "Helm"
10+
},
11+
"annotations": {
12+
"meta.helm.sh/release-name": "snapshot-controller",
13+
"meta.helm.sh/release-namespace": "cozy-snapshot-controller",
14+
"helm.sh/resource-policy": "keep"
15+
}
16+
},
17+
"spec": {
18+
"group": "groupsnapshot.storage.k8s.io",
19+
"versions": [
20+
{
21+
"name": "v1",
22+
"served": true,
23+
"storage": false
24+
},
25+
{
26+
"name": "v1beta1",
27+
"served": true,
28+
"storage": false
29+
},
30+
{
31+
"name": "v1beta2",
32+
"served": true,
33+
"storage": true
34+
}
35+
]
36+
}
37+
}
Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
{
2+
"apiVersion": "apiextensions.k8s.io/v1",
3+
"kind": "CustomResourceDefinition",
4+
"metadata": {
5+
"name": "volumegroupsnapshotcontents.groupsnapshot.storage.k8s.io",
6+
"labels": {
7+
"helm.toolkit.fluxcd.io/name": "snapshot-controller",
8+
"helm.toolkit.fluxcd.io/namespace": "cozy-snapshot-controller",
9+
"app.kubernetes.io/managed-by": "Helm"
10+
},
11+
"annotations": {
12+
"meta.helm.sh/release-name": "snapshot-controller",
13+
"meta.helm.sh/release-namespace": "cozy-snapshot-controller",
14+
"helm.sh/resource-policy": "keep"
15+
}
16+
},
17+
"spec": {
18+
"group": "groupsnapshot.storage.k8s.io",
19+
"versions": [
20+
{
21+
"name": "v1",
22+
"served": true,
23+
"storage": false
24+
},
25+
{
26+
"name": "v1beta1",
27+
"served": true,
28+
"storage": false
29+
},
30+
{
31+
"name": "v1beta2",
32+
"served": true,
33+
"storage": true
34+
}
35+
]
36+
}
37+
}
Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
{
2+
"apiVersion": "apiextensions.k8s.io/v1",
3+
"kind": "CustomResourceDefinition",
4+
"metadata": {
5+
"name": "volumegroupsnapshots.groupsnapshot.storage.k8s.io",
6+
"labels": {
7+
"helm.toolkit.fluxcd.io/name": "snapshot-controller",
8+
"helm.toolkit.fluxcd.io/namespace": "cozy-snapshot-controller",
9+
"app.kubernetes.io/managed-by": "Helm"
10+
},
11+
"annotations": {
12+
"meta.helm.sh/release-name": "snapshot-controller",
13+
"meta.helm.sh/release-namespace": "cozy-snapshot-controller",
14+
"helm.sh/resource-policy": "keep"
15+
}
16+
},
17+
"spec": {
18+
"group": "groupsnapshot.storage.k8s.io",
19+
"versions": [
20+
{
21+
"name": "v1",
22+
"served": true,
23+
"storage": false
24+
},
25+
{
26+
"name": "v1beta1",
27+
"served": true,
28+
"storage": false
29+
},
30+
{
31+
"name": "v1beta2",
32+
"served": true,
33+
"storage": true
34+
}
35+
]
36+
}
37+
}

‎hack/testdata/migration-61-snapshot-crds/kubectl‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,10 @@
1313
# FAKE_CRD_FAIL non-empty => reading a CRD exits 1 with this stderr
1414
# FAKE_DELETE_FAIL non-empty => deleting a CRD exits 1 with this stderr
1515
# FAKE_RECORD_FAIL non-empty => annotating cozystack-version exits 1 with this stderr
16+
# FAKE_CREATE_FAIL non-empty => creating a CRD exits 1 with this stderr
17+
#
18+
# The CRD files the migration creates from are JSON, which kubectl reads as
19+
# well as YAML, so the fake can load them with jq.
1620
set -u
1721

1822
log() { [ -n "${FAKE_CMDLOG:-}" ] && printf '%s\n' "$*" >> "$FAKE_CMDLOG"; }
@@ -75,6 +79,22 @@ case "$*" in
7579
exit 0
7680
;;
7781

82+
"create -f "*)
83+
if [ -n "${FAKE_CREATE_FAIL:-}" ]; then printf '%s\n' "$FAKE_CREATE_FAIL" >&2; exit 1; fi
84+
name=$(jq -r '.metadata.name' "$3") || exit 1
85+
if jq -e --arg n "$name" '.items[] | select(.metadata.name == $n)' "$CRDS" >/dev/null; then
86+
printf 'Error from server (AlreadyExists): %s already exists\n' "$name" >&2
87+
exit 1
88+
fi
89+
log "CREATE $name"
90+
jq --slurpfile crd "$3" '.items += $crd' "$CRDS" > "$CRDS.new" && mv "$CRDS.new" "$CRDS"
91+
exit 0
92+
;;
93+
94+
"wait --for=condition=Established "*)
95+
exit 0
96+
;;
97+
7898
"annotate configmap "*)
7999
if [ -n "${FAKE_RECORD_FAIL:-}" ]; then printf '%s\n' "$FAKE_RECORD_FAIL" >&2; exit 1; fi
80100
log "RECORD $*"

‎packages/core/platform/Makefile‎

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,13 @@ unprotect:
2727

2828
image: image-migrations
2929

30+
GROUP_SNAPSHOT_CRDS_EXPR := select(.spec.group == "groupsnapshot.storage.k8s.io") \
31+
| .metadata.annotations."meta.helm.sh/release-name" = "snapshot-controller" \
32+
| .metadata.annotations."meta.helm.sh/release-namespace" = "cozy-snapshot-controller" \
33+
| .metadata.labels."app.kubernetes.io/managed-by" = "Helm" \
34+
| .metadata.labels."helm.toolkit.fluxcd.io/name" = "snapshot-controller" \
35+
| .metadata.labels."helm.toolkit.fluxcd.io/namespace" = "cozy-snapshot-controller"
36+
3037
image-migrations:
3138
# Vendor the v1alpha2 CRDs into the build context so migration 50 can ensure
3239
# they exist before adoption (the etcd-operator-crds HelmRelease only installs
@@ -36,12 +43,22 @@ image-migrations:
3643
rm -rf images/migrations/etcd-operator-crds
3744
mkdir -p images/migrations/etcd-operator-crds
3845
cp ../../system/etcd-operator-crds/templates/*.yaml images/migrations/etcd-operator-crds/
46+
# The v1beta2 group snapshot CRDs, rendered from the snapshot-controller
47+
# chart with that release's ownership metadata, so migration 61 can create
48+
# them before any HelmRelease renders against the cluster.
49+
rm -rf images/migrations/snapshot-controller-crds
50+
mkdir -p images/migrations/snapshot-controller-crds
51+
helm template snapshot-controller ../../system/snapshot-controller \
52+
--namespace cozy-snapshot-controller \
53+
--show-only charts/snapshot-controller/templates/crds.yaml \
54+
| yq --split-exp '"images/migrations/snapshot-controller-crds/" + .metadata.name + ".yaml"' \
55+
'$(GROUP_SNAPSHOT_CRDS_EXPR)'
3956
docker buildx build images/migrations \
4057
$(call image-tags,platform-migrations,v$(COZYSTACK_VERSION)) \
4158
$(call cache-args,platform-migrations) \
4259
--metadata-file images/migrations.json \
4360
$(BUILDX_ARGS)
44-
rm -rf images/migrations/etcd-operator-crds
61+
rm -rf images/migrations/etcd-operator-crds images/migrations/snapshot-controller-crds
4562
IMAGE="$(REGISTRY)/platform-migrations:$(IMAGE_TAG)@$$(yq -e -o json -r '.["containerimage.digest"]' images/migrations.json)" \
4663
yq --inplace '.migrations.image = strenv(IMAGE)' values.yaml
4764
rm -f images/migrations.json

‎packages/core/platform/images/migrations/Dockerfile‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,9 @@ COPY migrations /migrations
3535
# the etcd-operator-crds HelmRelease (which only reconciles after the migration
3636
# pre-upgrade hook completes).
3737
COPY etcd-operator-crds /etcd-operator-crds
38+
# v1beta2 group snapshot CRDs, rendered by `make image-migrations`; migration 61
39+
# creates them before the snapshot-controller HelmRelease upgrades.
40+
COPY snapshot-controller-crds /snapshot-controller-crds
3841
COPY run-migrations.sh /usr/bin/run-migrations.sh
3942

4043
WORKDIR /migrations

0 commit comments

Comments
 (0)