Skip to content

Commit 0d2e0ca

Browse files
fix(platform): keep foreign snapshot CRDs out of the release
The vendored chart renders its CRDs as ordinary templates, so on a cluster whose distribution ships the snapshot CRDs, helm-controller would take them over on upgrade. Patch the CRD template to render a CRD only when it is absent or already carries this release's Helm ownership; migration 61 adopts the CRDs the release installed, which keeps them in the manifest. Migration 61 also took the release identity from the helm-controller labels on the CRDs. When only the group CRDs carried them, a failed create left them deleted, and the retry found no owned CRD and stamped the version without recreating them. Read the HelmRelease instead, and change nothing when it is absent. Assisted-by: LLM Signed-off-by: Mattia Eleuteri <[email protected]>
1 parent d94c983 commit 0d2e0ca

7 files changed

Lines changed: 196 additions & 17 deletions

File tree

‎hack/migration-61-snapshot-crds.bats‎

Lines changed: 54 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@ run_migration() {
4242
-e FAKE_CMDLOG=/work/cmdlog \
4343
-e FAKE_STATE=/work/state \
4444
-e NAMESPACE="${NAMESPACE-}" \
45+
-e FAKE_HR_FAIL="${FAKE_HR_FAIL-}" \
4546
-e FAKE_CRD_FAIL="${FAKE_CRD_FAIL-}" \
4647
-e FAKE_DELETE_FAIL="${FAKE_DELETE_FAIL-}" \
4748
-e FAKE_RECORD_FAIL="${FAKE_RECORD_FAIL-}" \
@@ -67,10 +68,11 @@ DOCKERFILE
6768
mkdir -p "$WORK/state"
6869
cp "$FIXTURES/crds.json" "$WORK/state/"
6970
echo '{}' > "$WORK/state/objects.json"
71+
: > "$WORK/state/helmrelease"
7072
FAKE_CMDLOG="$WORK/cmdlog"
7173
: > "$FAKE_CMDLOG"
7274
export NAMESPACE=cozy-system
73-
unset FAKE_CRD_FAIL FAKE_DELETE_FAIL FAKE_RECORD_FAIL FAKE_CREATE_FAIL || true
75+
unset FAKE_HR_FAIL FAKE_CRD_FAIL FAKE_DELETE_FAIL FAKE_RECORD_FAIL FAKE_CREATE_FAIL || true
7476
return 0
7577
}
7678

@@ -188,9 +190,9 @@ crd_field() {
188190
rm -rf "$WORK"
189191
}
190192

191-
@test "a cluster without the snapshot CRDs only stamps the version" {
193+
@test "a cluster without the snapshot-controller release only stamps the version" {
192194
prep
193-
edit_crds '.items = []'
195+
rm "$WORK/state/helmrelease"
194196
rc=0
195197
run_migration 61 >"$WORK/out" 2>&1 || rc=$?
196198
cat "$WORK/out"; cat "$FAKE_CMDLOG"
@@ -200,6 +202,31 @@ crd_field() {
200202
rm -rf "$WORK"
201203
}
202204

205+
@test "a cluster with the release but no group snapshot CRDs gets them created" {
206+
prep
207+
edit_crds '.items |= map(select(.spec.group != "groupsnapshot.storage.k8s.io"))'
208+
rc=0
209+
run_migration 61 >"$WORK/out" 2>&1 || rc=$?
210+
cat "$WORK/out"; cat "$FAKE_CMDLOG"
211+
[ "$rc" -eq 0 ]
212+
for p in volumegroupsnapshotclasses volumegroupsnapshotcontents volumegroupsnapshots; do
213+
grep -qxF "CREATE $p.groupsnapshot.storage.k8s.io" "$FAKE_CMDLOG"
214+
done
215+
grep -qxF "STAMP 62" "$FAKE_CMDLOG"
216+
rm -rf "$WORK"
217+
}
218+
219+
@test "a failing HelmRelease read aborts instead of treating the release as absent" {
220+
prep
221+
export FAKE_HR_FAIL="Error from server (InternalError): etcdserver: request timed out"
222+
rc=0
223+
run_migration 61 >"$WORK/out" 2>&1 || rc=$?
224+
cat "$WORK/out"; cat "$FAKE_CMDLOG"
225+
[ "$rc" -ne 0 ]
226+
if grep -qE "^(ANNOTATE|LABEL|DELETE|CREATE|STAMP)" "$FAKE_CMDLOG"; then echo "changed something after a failed HelmRelease read" >&2; return 1; fi
227+
rm -rf "$WORK"
228+
}
229+
203230
@test "a failing CRD read aborts instead of treating the CRD as absent" {
204231
prep
205232
export FAKE_CRD_FAIL="Error from server (InternalError): etcdserver: request timed out"
@@ -260,6 +287,30 @@ crd_field() {
260287
rm -rf "$WORK"
261288
}
262289

290+
@test "release labels on the group snapshot CRDs alone survive a failed create" {
291+
prep
292+
edit_crds '.items |= map(if .spec.group == "snapshot.storage.k8s.io" then .metadata.labels = {} else . end)'
293+
export FAKE_CREATE_FAIL="Error from server (InternalError): etcdserver: request timed out"
294+
rc=0
295+
run_migration 61 >"$WORK/out" 2>&1 || rc=$?
296+
cat "$WORK/out"; cat "$FAKE_CMDLOG"
297+
[ "$rc" -ne 0 ]
298+
if grep -qF "STAMP" "$FAKE_CMDLOG"; then echo "unexpected STAMP after failed create" >&2; return 1; fi
299+
300+
unset FAKE_CREATE_FAIL
301+
: > "$FAKE_CMDLOG"
302+
rc=0
303+
run_migration 61 >"$WORK/out" 2>&1 || rc=$?
304+
cat "$WORK/out"; cat "$FAKE_CMDLOG"
305+
[ "$rc" -eq 0 ]
306+
for p in volumegroupsnapshotclasses volumegroupsnapshotcontents volumegroupsnapshots; do
307+
grep -qxF "CREATE $p.groupsnapshot.storage.k8s.io" "$FAKE_CMDLOG"
308+
[ "$(crd_field "$p.groupsnapshot.storage.k8s.io" '[.spec.versions[] | select(.served) | .name] | index("v1beta2") != null')" = "true" ]
309+
done
310+
grep -qxF "STAMP 62" "$FAKE_CMDLOG"
311+
rm -rf "$WORK"
312+
}
313+
263314
@test "a v1beta2 group snapshot CRD that still stores v1alpha1 is recorded, not adopted" {
264315
prep
265316
edit_crds '.items |= map(if .metadata.name == "volumegroupsnapshotcontents.groupsnapshot.storage.k8s.io"

‎hack/testdata/migration-61-snapshot-crds/kubectl‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,10 +6,12 @@
66
# The CRDs live in $FAKE_STATE/crds.json and every annotate, label and delete
77
# is applied to that file, so a second run of the migration reads what the
88
# first one wrote. $FAKE_STATE/objects.json maps a CRD name to the number of
9-
# objects it holds.
9+
# objects it holds. The snapshot-controller HelmRelease exists while
10+
# $FAKE_STATE/helmrelease does.
1011
#
1112
# FAKE_CMDLOG file to append a one-line record of each call to
1213
# FAKE_STATE directory holding crds.json and objects.json
14+
# FAKE_HR_FAIL non-empty => reading the HelmRelease exits 1 with this stderr
1315
# FAKE_CRD_FAIL non-empty => reading a CRD exits 1 with this stderr
1416
# FAKE_DELETE_FAIL non-empty => deleting a CRD exits 1 with this stderr
1517
# FAKE_RECORD_FAIL non-empty => annotating cozystack-version exits 1 with this stderr
@@ -52,6 +54,12 @@ case "$*" in
5254
exit 1
5355
;;
5456

57+
"get helmreleases.helm.toolkit.fluxcd.io --namespace cozy-snapshot-controller snapshot-controller --ignore-not-found -o name")
58+
if [ -n "${FAKE_HR_FAIL:-}" ]; then printf '%s\n' "$FAKE_HR_FAIL" >&2; exit 1; fi
59+
[ -e "$FAKE_STATE/helmrelease" ] && echo "helmrelease.helm.toolkit.fluxcd.io/snapshot-controller"
60+
exit 0
61+
;;
62+
5563
"get customresourcedefinitions.apiextensions.k8s.io "*)
5664
if [ -n "${FAKE_CRD_FAIL:-}" ]; then printf '%s\n' "$FAKE_CRD_FAIL" >&2; exit 1; fi
5765
jq --arg n "$3" '.items[] | select(.metadata.name == $n)' "$CRDS"

‎packages/core/platform/images/migrations/migrations/61‎

Lines changed: 24 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -5,10 +5,12 @@
55
#
66
# The snapshot-controller chart used to ship its six CRDs in crds/, which Helm
77
# creates on install and never touches again, so the live CRDs carry no Helm
8-
# ownership metadata. The chart now renders them from a template. Helm refuses
9-
# to upgrade over an object that already exists without that metadata ("exists
10-
# and cannot be imported into the current release"), so the CRDs the release
11-
# installed are annotated for adoption here.
8+
# ownership metadata. The chart now renders them from a template, and renders a
9+
# CRD only when it is absent or already carries this release's ownership
10+
# metadata (patches/skip-foreign-crds.diff in the package). The CRDs the release
11+
# installed are annotated for adoption here, so they stay in its manifest; any
12+
# other CRD stays out of it, and the upgrade neither takes it over nor rewrites
13+
# it.
1214
#
1315
# The three groupsnapshot.storage.k8s.io CRDs cannot be adopted that way. The
1416
# old chart served them as v1alpha1 only, the new one serves v1beta1, v1beta2
@@ -34,7 +36,13 @@
3436
#
3537
# Only CRDs labelled by helm-controller as installed by
3638
# cozy-snapshot-controller/snapshot-controller are touched, so a cluster whose
37-
# distribution ships its own snapshot CRDs keeps them as they are.
39+
# distribution ships its own snapshot CRDs keeps them as they are, here and in
40+
# the release.
41+
#
42+
# Whether the release is installed is read from its HelmRelease, not from those
43+
# labels: a retry after a failed create finds the group CRDs it deleted gone,
44+
# and their labels with them. Without the HelmRelease nothing would render the
45+
# CRDs again, so nothing is changed.
3846
#
3947
# This runs as a pre-upgrade hook, before the release renders the new chart. The
4048
# old chart does not render the CRDs, so the adoption metadata and the deletion
@@ -58,9 +66,16 @@ GROUP_SNAPSHOT_CRDS="volumegroupsnapshotclasses volumegroupsnapshotcontents volu
5866
GROUP_CRD_DIR="${GROUP_CRD_DIR:-/snapshot-controller-crds}"
5967

6068
LEFTOVERS=""
61-
RELEASE_INSTALLED=0
6269
CREATE=""
6370

71+
# Assigned on its own line: inside a test, a failed read would not abort.
72+
hr=$(kubectl get helmreleases.helm.toolkit.fluxcd.io --namespace "$RELEASE_NS" "$RELEASE_NAME" --ignore-not-found -o name)
73+
if [ -z "$hr" ]; then
74+
echo "HelmRelease $RELEASE_NS/$RELEASE_NAME not found; leaving the snapshot CRDs alone"
75+
stamp_cozystack_version 62
76+
exit 0
77+
fi
78+
6479
# installed_by_release <crd json>
6580
installed_by_release() {
6681
printf '%s' "$1" | jq -e --arg name "$RELEASE_NAME" --arg ns "$RELEASE_NS" '
@@ -98,7 +113,6 @@ for plural in $SNAPSHOT_CRDS; do
98113
echo "CRD $crd was not installed by $RELEASE_NS/$RELEASE_NAME; leaving it alone"
99114
continue
100115
fi
101-
RELEASE_INSTALLED=1
102116
adopt "$crd" "$obj"
103117
done
104118

@@ -113,7 +127,6 @@ for plural in $GROUP_SNAPSHOT_CRDS; do
113127
echo "CRD $crd was not installed by $RELEASE_NS/$RELEASE_NAME; leaving it alone"
114128
continue
115129
fi
116-
RELEASE_INSTALLED=1
117130
if printf '%s' "$obj" | jq -e '[.spec.versions[].name] | index("v1beta2")' >/dev/null; then
118131
# The chart drops v1alpha1 from spec.versions, which the apiserver refuses
119132
# while v1alpha1 is still a stored version, so such a CRD is left to an
@@ -137,10 +150,9 @@ for plural in $GROUP_SNAPSHOT_CRDS; do
137150
CREATE="$CREATE $crd"
138151
done
139152

140-
# A CRD missing on a cluster without the release, or with a kept one, is left
141-
# to the release. A retry after a failed create finds the deleted CRDs missing,
142-
# so it creates them here too.
143-
if [ "$RELEASE_INSTALLED" = 1 ] && [ -z "$LEFTOVERS" ]; then
153+
# A CRD missing next to a kept one is left to the release. A retry after a
154+
# failed create finds the deleted CRDs missing, so it creates them here too.
155+
if [ -z "$LEFTOVERS" ]; then
144156
for crd in $CREATE; do
145157
echo "Creating CRD $crd at v1beta2 for $RELEASE_NS/$RELEASE_NAME"
146158
kubectl create -f "$GROUP_CRD_DIR/$crd.yaml" >/dev/null

‎packages/system/snapshot-controller/Makefile‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,8 @@ update:
88
helm repo add piraeus-charts https://piraeus.io/helm-charts/
99
helm repo update piraeus-charts
1010
helm pull piraeus-charts/snapshot-controller --untar --untardir charts
11+
# Leave CRDs owned outside this release out of the manifest.
12+
patch --no-backup-if-mismatch -p1 -d charts/snapshot-controller < patches/skip-foreign-crds.diff
1113

1214
test:
1315
helm unittest .

‎packages/system/snapshot-controller/charts/snapshot-controller/templates/crds.yaml‎

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -62,5 +62,22 @@ data:
6262
{{- if $ca }}
6363
{{- $clientConfig = set $clientConfig "caBundle" $ca }}
6464
{{- end }}
65-
{{ tpl (.Files.Get "crds.yaml") (dict "ClientConfig" $clientConfig "InsertCaFrom" $insertCaFrom "WebhookEnabled" .Values.webhook.enabled "KeepCRDs" .Values.keepCRDs) }}
65+
{{- $crds := tpl (.Files.Get "crds.yaml") (dict "ClientConfig" $clientConfig "InsertCaFrom" $insertCaFrom "WebhookEnabled" .Values.webhook.enabled "KeepCRDs" .Values.keepCRDs) }}
66+
{{- /* cozystack: a CRD is rendered only when it is absent or already belongs
67+
to this release. A CRD the distribution ships, or that another release
68+
owns, stays out of the manifest, so the upgrade neither takes it over nor
69+
rewrites it. */}}
70+
{{- range $crd := regexSplit "(?m)^---$" $crds -1 }}
71+
{{- $obj := fromYaml $crd }}
72+
{{- if hasKey $obj "Error" }}
73+
{{- fail (printf "snapshot-controller: cannot parse a CRD from crds.yaml: %s" $obj.Error) }}
74+
{{- end }}
75+
{{- if $obj.metadata }}
76+
{{- $live := lookup "apiextensions.k8s.io/v1" "CustomResourceDefinition" "" $obj.metadata.name }}
77+
{{- if or (not $live) (and (eq (dig "metadata" "annotations" "meta.helm.sh/release-name" "" $live) $.Release.Name) (eq (dig "metadata" "annotations" "meta.helm.sh/release-namespace" "" $live) $.Release.Namespace)) }}
78+
---
79+
{{ trim $crd }}
80+
{{- end }}
81+
{{- end }}
82+
{{- end }}
6683
{{- end }}
Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
diff --git a/templates/crds.yaml b/templates/crds.yaml
2+
index f868380e4..6add1b616 100644
3+
--- a/templates/crds.yaml
4+
+++ b/templates/crds.yaml
5+
@@ -62,5 +62,22 @@ data:
6+
{{- if $ca }}
7+
{{- $clientConfig = set $clientConfig "caBundle" $ca }}
8+
{{- end }}
9+
-{{ tpl (.Files.Get "crds.yaml") (dict "ClientConfig" $clientConfig "InsertCaFrom" $insertCaFrom "WebhookEnabled" .Values.webhook.enabled "KeepCRDs" .Values.keepCRDs) }}
10+
+{{- $crds := tpl (.Files.Get "crds.yaml") (dict "ClientConfig" $clientConfig "InsertCaFrom" $insertCaFrom "WebhookEnabled" .Values.webhook.enabled "KeepCRDs" .Values.keepCRDs) }}
11+
+{{- /* cozystack: a CRD is rendered only when it is absent or already belongs
12+
+ to this release. A CRD the distribution ships, or that another release
13+
+ owns, stays out of the manifest, so the upgrade neither takes it over nor
14+
+ rewrites it. */}}
15+
+{{- range $crd := regexSplit "(?m)^---$" $crds -1 }}
16+
+{{- $obj := fromYaml $crd }}
17+
+{{- if hasKey $obj "Error" }}
18+
+{{- fail (printf "snapshot-controller: cannot parse a CRD from crds.yaml: %s" $obj.Error) }}
19+
+{{- end }}
20+
+{{- if $obj.metadata }}
21+
+{{- $live := lookup "apiextensions.k8s.io/v1" "CustomResourceDefinition" "" $obj.metadata.name }}
22+
+{{- if or (not $live) (and (eq (dig "metadata" "annotations" "meta.helm.sh/release-name" "" $live) $.Release.Name) (eq (dig "metadata" "annotations" "meta.helm.sh/release-namespace" "" $live) $.Release.Namespace)) }}
23+
+---
24+
+{{ trim $crd }}
25+
+{{- end }}
26+
+{{- end }}
27+
+{{- end }}
28+
{{- end }}
Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
suite: snapshot CRDs owned outside the release
2+
3+
# A distribution can ship its own snapshot CRDs. The chart renders a CRD only
4+
# when it is absent or already carries this release's Helm ownership, so an
5+
# upgrade never takes over or rewrites one owned elsewhere. Migration 61 adopts
6+
# the CRDs the release installed before, which keeps them in the manifest.
7+
8+
templates:
9+
- charts/snapshot-controller/templates/crds.yaml
10+
11+
release:
12+
name: snapshot-controller
13+
namespace: cozy-snapshot-controller
14+
15+
kubernetesProvider:
16+
scheme:
17+
"apiextensions.k8s.io/v1/CustomResourceDefinition":
18+
gvr:
19+
group: apiextensions.k8s.io
20+
version: v1
21+
resource: customresourcedefinitions
22+
namespaced: false
23+
objects:
24+
- apiVersion: apiextensions.k8s.io/v1
25+
kind: CustomResourceDefinition
26+
metadata:
27+
name: volumesnapshots.snapshot.storage.k8s.io
28+
- apiVersion: apiextensions.k8s.io/v1
29+
kind: CustomResourceDefinition
30+
metadata:
31+
name: volumesnapshotcontents.snapshot.storage.k8s.io
32+
annotations:
33+
meta.helm.sh/release-name: other
34+
meta.helm.sh/release-namespace: cozy-snapshot-controller
35+
- apiVersion: apiextensions.k8s.io/v1
36+
kind: CustomResourceDefinition
37+
metadata:
38+
name: volumesnapshotclasses.snapshot.storage.k8s.io
39+
annotations:
40+
meta.helm.sh/release-name: snapshot-controller
41+
meta.helm.sh/release-namespace: cozy-snapshot-controller
42+
43+
tests:
44+
- it: renders only the absent CRDs and the ones this release owns
45+
asserts:
46+
- hasDocuments:
47+
count: 4
48+
- notEqual:
49+
path: metadata.name
50+
value: volumesnapshots.snapshot.storage.k8s.io
51+
- notEqual:
52+
path: metadata.name
53+
value: volumesnapshotcontents.snapshot.storage.k8s.io
54+
55+
- it: keeps rendering a CRD this release owns
56+
documentSelector:
57+
path: metadata.name
58+
value: volumesnapshotclasses.snapshot.storage.k8s.io
59+
asserts:
60+
- isKind:
61+
of: CustomResourceDefinition

0 commit comments

Comments
 (0)