Skip to content

Commit 9883e5c

Browse files
[kubernetes] Auto-propagate infra StorageClasses to tenant clusters
Replace the manual storageClass parameter with automatic discovery via Helm lookup. The kubevirt-csi-node driver now detects available StorageClasses from the infrastructure cluster and propagates them to tenant clusters automatically. Includes migration 33 to remove the deprecated storageClass key from existing Kubernetes app HelmRelease values Secrets. Signed-off-by: Mattia Music <[email protected]> Co-Authored-By: Claude Opus 4.6 <[email protected]> Signed-off-by: mattia-eleuteri <[email protected]>
1 parent 1fd1da4 commit 9883e5c

11 files changed

Lines changed: 188 additions & 31 deletions

File tree

‎hack/e2e-apps/run-kubernetes.sh‎

Lines changed: 36 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,6 @@ spec:
5959
minReplicas: 2
6060
roles:
6161
- ingress-nginx
62-
storageClass: replicated
6362
version: "${k8s_version}"
6463
EOF
6564
# Wait for the tenant-test namespace to be active
@@ -128,6 +127,41 @@ EOF
128127
exit 1
129128
fi
130129

130+
# Verify StorageClass propagation from infra to tenant cluster
131+
echo "Verifying StorageClass propagation..."
132+
133+
# Wait for at least one StorageClass to appear in the tenant cluster
134+
timeout 2m bash -c '
135+
until [ "$(kubectl --kubeconfig tenantkubeconfig-'"${test_name}"' get sc -o jsonpath="{.items[*].metadata.name}" 2>/dev/null | wc -w)" -ge 1 ]; do
136+
sleep 5
137+
done
138+
'
139+
140+
sc_names=$(kubectl --kubeconfig "tenantkubeconfig-${test_name}" get sc -o jsonpath='{.items[*].metadata.name}')
141+
if [ -z "$sc_names" ]; then
142+
echo "No StorageClasses found in tenant cluster" >&2
143+
exit 1
144+
fi
145+
echo "StorageClasses in tenant: ${sc_names}"
146+
147+
# Verify each propagated StorageClass uses the kubevirt CSI provisioner
148+
for sc in $sc_names; do
149+
provisioner=$(kubectl --kubeconfig "tenantkubeconfig-${test_name}" get sc "$sc" -o jsonpath='{.provisioner}')
150+
if [ "$provisioner" != "csi.kubevirt.io" ]; then
151+
echo "StorageClass $sc has unexpected provisioner: $provisioner (expected csi.kubevirt.io)" >&2
152+
exit 1
153+
fi
154+
done
155+
echo "All StorageClasses use csi.kubevirt.io provisioner"
156+
157+
# Verify exactly one default StorageClass is set
158+
default_count=$(kubectl --kubeconfig "tenantkubeconfig-${test_name}" get sc \
159+
-o jsonpath='{range .items[?(@.metadata.annotations.storageclass\.kubernetes\.io/is-default-class=="true")]}{.metadata.name}{"\n"}{end}' | grep -c .)
160+
if [ "$default_count" -ne 1 ]; then
161+
echo "Expected exactly 1 default StorageClass, found $default_count" >&2
162+
exit 1
163+
fi
164+
echo "Default StorageClass is correctly set"
131165

132166
kubectl --kubeconfig "tenantkubeconfig-${test_name}" apply -f - <<EOF
133167
apiVersion: v1
@@ -244,7 +278,7 @@ metadata:
244278
spec:
245279
accessModes:
246280
- ReadWriteMany
247-
storageClassName: kubevirt
281+
storageClassName: replicated
248282
resources:
249283
requests:
250284
storage: 1Gi

‎packages/apps/kubernetes/README.md‎

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -82,13 +82,6 @@ See the reference for components utilized in this service:
8282
8383
## Parameters
8484
85-
### Common Parameters
86-
87-
| Name | Description | Type | Value |
88-
| -------------- | ------------------------------------ | -------- | ------------ |
89-
| `storageClass` | StorageClass used to store the data. | `string` | `replicated` |
90-
91-
9285
### Application-specific Parameters
9386
9487
| Name | Description | Type | Value |
Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
apiVersion: v1
2+
kind: ConfigMap
3+
metadata:
4+
name: driver-config
5+
data:
6+
infraStorageClassEnforcement: |
7+
allowAll: false
8+
allowDefault: false
9+
allowList:
10+
{{- $storageClasses := (lookup "storage.k8s.io/v1" "StorageClass" "" "").items }}
11+
{{- if $storageClasses }}
12+
{{- range $storageClasses }}
13+
- {{ .metadata.name | quote }}
14+
{{- end }}
15+
{{- else }}
16+
- "replicated"
17+
{{- end }}

‎packages/apps/kubernetes/templates/helmreleases/csi.yaml‎

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -26,10 +26,20 @@ spec:
2626
upgrade:
2727
remediation:
2828
retries: -1
29-
{{- with .Values.storageClass }}
3029
values:
31-
storageClass: "{{ . }}"
32-
{{- end }}
30+
storageClasses:
31+
{{- $storageClasses := dict }}
32+
{{- range $sc := (lookup "storage.k8s.io/v1" "StorageClass" "" "").items }}
33+
{{- $isDefault := false }}
34+
{{- if $sc.metadata.annotations }}
35+
{{- $isDefault = eq (index $sc.metadata.annotations "storageclass.kubernetes.io/is-default-class" | default "false") "true" }}
36+
{{- end }}
37+
{{- $_ := set $storageClasses $sc.metadata.name (dict "infraStorageClass" $sc.metadata.name "default" $isDefault) }}
38+
{{- end }}
39+
{{- if not $storageClasses }}
40+
{{- $_ := set $storageClasses "replicated" (dict "infraStorageClass" "replicated" "default" true) }}
41+
{{- end }}
42+
{{- $storageClasses | toYaml | nindent 6 }}
3343
dependsOn:
3444
- name: {{ .Release.Name }}-vsnap-crd
3545
namespace: {{ .Release.Namespace }}

‎packages/apps/kubernetes/values.schema.json‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -613,11 +613,6 @@
613613
}
614614
}
615615
},
616-
"storageClass": {
617-
"description": "StorageClass used to store the data.",
618-
"type": "string",
619-
"default": "replicated"
620-
},
621616
"version": {
622617
"description": "Kubernetes major.minor version to deploy",
623618
"type": "string",

‎packages/apps/kubernetes/values.yaml‎

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,3 @@
1-
##
2-
## @section Common Parameters
3-
##
4-
5-
## @param {string} storageClass - StorageClass used to store the data.
6-
storageClass: replicated
7-
81
##
92
## @section Application-specific Parameters
103
##
Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
#!/bin/bash
2+
# Migration 33 --> 34
3+
# Remove deprecated 'storageClass' parameter from Kubernetes app instances.
4+
# StorageClasses are now auto-discovered from the infrastructure cluster.
5+
# Idempotent: skips if storageClass key is already absent.
6+
7+
set -euo pipefail
8+
9+
# ============================================================
10+
# STEP 1: Discover all Kubernetes HelmReleases
11+
# ============================================================
12+
echo "=== Discovering Kubernetes HelmReleases ==="
13+
INSTANCES=()
14+
while IFS=/ read -r ns name; do
15+
[ -z "$ns" ] && continue
16+
INSTANCES+=("${ns}/${name}")
17+
echo " Found: ${ns}/${name}"
18+
done < <(kubectl get hr -A -l "apps.cozystack.io/application.kind=Kubernetes" \
19+
-o jsonpath='{range .items[*]}{.metadata.namespace}/{.metadata.name}{"\n"}{end}' 2>/dev/null)
20+
21+
if [ ${#INSTANCES[@]} -eq 0 ]; then
22+
echo " No Kubernetes HelmReleases found. Nothing to migrate."
23+
kubectl create configmap -n cozy-system cozystack-version \
24+
--from-literal=version=34 --dry-run=client -o yaml | kubectl apply -f-
25+
exit 0
26+
fi
27+
echo " Total: ${#INSTANCES[@]} instance(s)"
28+
29+
# ============================================================
30+
# STEP 2: Migrate each instance
31+
# ============================================================
32+
for entry in "${INSTANCES[@]}"; do
33+
NAMESPACE="${entry%%/*}"
34+
HR_NAME="${entry#*/}"
35+
36+
echo ""
37+
echo "======================================================================"
38+
echo "=== Processing: ${HR_NAME} in ${NAMESPACE}"
39+
echo "======================================================================"
40+
41+
# --- Find values Secret ---
42+
VALUES_SECRET=$(kubectl -n "$NAMESPACE" get hr "$HR_NAME" -o json | \
43+
jq -r '.spec.valuesFrom // [] | map(select(.kind == "Secret" and (.name | test("cozystack-values") | not))) | .[0].name // ""')
44+
VALUES_KEY=$(kubectl -n "$NAMESPACE" get hr "$HR_NAME" -o json | \
45+
jq -r '.spec.valuesFrom // [] | map(select(.kind == "Secret" and (.name | test("cozystack-values") | not))) | .[0].valuesKey // "values.yaml"')
46+
47+
if [ -z "$VALUES_SECRET" ]; then
48+
echo " [SKIP] No values Secret found for hr/${HR_NAME}"
49+
continue
50+
fi
51+
52+
if ! kubectl -n "$NAMESPACE" get secret "$VALUES_SECRET" --no-headers 2>/dev/null | grep -q .; then
53+
echo " [SKIP] Secret ${VALUES_SECRET} not found"
54+
continue
55+
fi
56+
57+
echo " Reading values from secret: ${VALUES_SECRET} (key: ${VALUES_KEY})"
58+
59+
# --- Decode current values ---
60+
VALUES_YAML=$(kubectl -n "$NAMESPACE" get secret "$VALUES_SECRET" \
61+
-o jsonpath="{.data.${VALUES_KEY}}" 2>/dev/null | base64 -d 2>/dev/null || true)
62+
if [ -z "$VALUES_YAML" ]; then
63+
VALUES_YAML=$(kubectl -n "$NAMESPACE" get secret "$VALUES_SECRET" \
64+
-o jsonpath="{.stringData.${VALUES_KEY}}" 2>/dev/null || true)
65+
fi
66+
67+
if [ -z "$VALUES_YAML" ]; then
68+
echo " [SKIP] Could not read values from secret"
69+
continue
70+
fi
71+
72+
# --- Check if storageClass key exists ---
73+
STORAGE_CLASS=$(echo "$VALUES_YAML" | yq -r '.storageClass // ""')
74+
75+
if [ -z "$STORAGE_CLASS" ]; then
76+
echo " [SKIP] storageClass key is already absent or empty"
77+
continue
78+
fi
79+
80+
echo " [REMOVE] storageClass='${STORAGE_CLASS}', removing key"
81+
82+
# --- Remove storageClass key ---
83+
NEW_VALUES=$(echo "$VALUES_YAML" | yq 'del(.storageClass)')
84+
85+
# --- Patch the Secret ---
86+
NEW_VALUES_B64=$(echo "$NEW_VALUES" | base64)
87+
echo " [PATCH] Updating secret/${VALUES_SECRET}"
88+
kubectl -n "$NAMESPACE" get secret "$VALUES_SECRET" -o json | \
89+
jq --arg key "$VALUES_KEY" --arg val "$NEW_VALUES_B64" \
90+
'.data[$key] = $val' | \
91+
kubectl apply -f -
92+
93+
echo " [OK] Removed storageClass for ${HR_NAME}"
94+
done
95+
96+
echo ""
97+
echo "=== Migration complete (${#INSTANCES[@]} instance(s)) ==="
98+
99+
# Stamp version
100+
kubectl create configmap -n cozy-system cozystack-version \
101+
--from-literal=version=34 --dry-run=client -o yaml | kubectl apply -f-

‎packages/core/platform/values.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ sourceRef:
66
migrations:
77
enabled: false
88
image: ghcr.io/cozystack/cozystack/platform-migrations:v1.0.0-rc.1@sha256:21a09c9f8dfd0a0c9b8c14c70029a39bfce021c66f1d4cacad9764c35dce6e8f
9-
targetVersion: 33
9+
targetVersion: 34
1010
# Bundle deployment configuration
1111
bundles:
1212
system:

‎packages/system/kubernetes-rd/cozyrds/kubernetes.yaml‎

Lines changed: 2 additions & 2 deletions
Large diffs are not rendered by default.

‎packages/system/kubevirt-csi-node/templates/deploy.yaml‎

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -264,17 +264,28 @@ metadata:
264264
name: driver-config
265265
namespace: {{ .Release.Namespace }}
266266
data:
267-
infraClusterNamespace:
267+
infraClusterNamespace:
268268
infraClusterLabels: csi-driver/cluster=tenant
269+
infraStorageClassEnforcement: |
270+
allowAll: false
271+
allowDefault: false
272+
allowList:
273+
{{- range $name, $mapping := .Values.storageClasses }}
274+
- {{ $mapping.infraStorageClass | quote }}
275+
{{- end }}
276+
{{- range $name, $mapping := .Values.storageClasses }}
269277
---
270278
apiVersion: storage.k8s.io/v1
271279
kind: StorageClass
272280
metadata:
273-
name: kubevirt
281+
name: {{ $name }}
282+
{{- if $mapping.default }}
274283
annotations:
275284
storageclass.kubernetes.io/is-default-class: "true"
285+
{{- end }}
276286
provisioner: csi.kubevirt.io
277287
allowVolumeExpansion: true
278288
parameters:
279-
infraStorageClassName: {{ .Values.storageClass }}
289+
infraStorageClassName: {{ $mapping.infraStorageClass }}
280290
bus: scsi
291+
{{- end }}

0 commit comments

Comments
 (0)