Skip to content

Commit 416742f

Browse files
mattia-eleutericlaude
authored and
kvaps
committed
[kubernetes] Auto-propagate infra StorageClasses to tenant clusters
Replace the hardcoded 'kubevirt' StorageClass in tenant clusters with auto-discovered StorageClasses from the infrastructure cluster. Migration 35 handles the upgrade path: - STEP 1: Discover Kubernetes HelmReleases - STEP 2: Remove deprecated 'storageClass' from values Secrets - STEP 3: PVC rebinding in tenants (kubevirt → infra SC name) Co-Authored-By: Claude Opus 4.6 <[email protected]> Signed-off-by: mattia-eleuteri <[email protected]>
1 parent 9bb6625 commit 416742f

11 files changed

Lines changed: 310 additions & 31 deletions

File tree

‎hack/e2e-apps/run-kubernetes.sh‎

Lines changed: 36 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,6 @@ spec:
5959
minReplicas: 2
6060
roles:
6161
- ingress-nginx
62-
storageClass: replicated
6362
version: "${k8s_version}"
6463
EOF
6564
# Wait for the tenant-test namespace to be active
@@ -132,6 +131,41 @@ EOF
132131
exit 1
133132
fi
134133

134+
# Verify StorageClass propagation from infra to tenant cluster
135+
echo "Verifying StorageClass propagation..."
136+
137+
# Wait for at least one StorageClass to appear in the tenant cluster
138+
timeout 2m bash -c '
139+
until [ "$(kubectl --kubeconfig tenantkubeconfig-'"${test_name}"' get sc -o jsonpath="{.items[*].metadata.name}" 2>/dev/null | wc -w)" -ge 1 ]; do
140+
sleep 5
141+
done
142+
'
143+
144+
sc_names=$(kubectl --kubeconfig "tenantkubeconfig-${test_name}" get sc -o jsonpath='{.items[*].metadata.name}')
145+
if [ -z "$sc_names" ]; then
146+
echo "No StorageClasses found in tenant cluster" >&2
147+
exit 1
148+
fi
149+
echo "StorageClasses in tenant: ${sc_names}"
150+
151+
# Verify each propagated StorageClass uses the kubevirt CSI provisioner
152+
for sc in $sc_names; do
153+
provisioner=$(kubectl --kubeconfig "tenantkubeconfig-${test_name}" get sc "$sc" -o jsonpath='{.provisioner}')
154+
if [ "$provisioner" != "csi.kubevirt.io" ]; then
155+
echo "StorageClass $sc has unexpected provisioner: $provisioner (expected csi.kubevirt.io)" >&2
156+
exit 1
157+
fi
158+
done
159+
echo "All StorageClasses use csi.kubevirt.io provisioner"
160+
161+
# Verify exactly one default StorageClass is set
162+
default_count=$(kubectl --kubeconfig "tenantkubeconfig-${test_name}" get sc \
163+
-o jsonpath='{range .items[?(@.metadata.annotations.storageclass\.kubernetes\.io/is-default-class=="true")]}{.metadata.name}{"\n"}{end}' | grep -c .)
164+
if [ "$default_count" -ne 1 ]; then
165+
echo "Expected exactly 1 default StorageClass, found $default_count" >&2
166+
exit 1
167+
fi
168+
echo "Default StorageClass is correctly set"
135169

136170
kubectl --kubeconfig "tenantkubeconfig-${test_name}" apply -f - <<EOF
137171
apiVersion: v1
@@ -248,7 +282,7 @@ metadata:
248282
spec:
249283
accessModes:
250284
- ReadWriteMany
251-
storageClassName: kubevirt
285+
storageClassName: replicated
252286
resources:
253287
requests:
254288
storage: 1Gi

‎packages/apps/kubernetes/README.md‎

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -82,13 +82,6 @@ See the reference for components utilized in this service:
8282
8383
## Parameters
8484
85-
### Common Parameters
86-
87-
| Name | Description | Type | Value |
88-
| -------------- | ------------------------------------ | -------- | ------------ |
89-
| `storageClass` | StorageClass used to store the data. | `string` | `replicated` |
90-
91-
9285
### Application-specific Parameters
9386
9487
| Name | Description | Type | Value |

‎packages/apps/kubernetes/templates/csi/driver-config.yaml‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
apiVersion: v1
2+
kind: ConfigMap
3+
metadata:
4+
name: driver-config
5+
data:
6+
infraStorageClassEnforcement: |
7+
allowAll: false
8+
allowDefault: false
9+
allowList:
10+
{{- $storageClasses := (lookup "storage.k8s.io/v1" "StorageClass" "" "").items }}
11+
{{- if $storageClasses }}
12+
{{- range $storageClasses }}
13+
- {{ .metadata.name | quote }}
14+
{{- end }}
15+
{{- else }}
16+
- "replicated"
17+
{{- end }}

‎packages/apps/kubernetes/templates/helmreleases/csi.yaml‎

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -26,10 +26,20 @@ spec:
2626
upgrade:
2727
remediation:
2828
retries: -1
29-
{{- with .Values.storageClass }}
3029
values:
31-
storageClass: "{{ . }}"
32-
{{- end }}
30+
storageClasses:
31+
{{- $storageClasses := dict }}
32+
{{- range $sc := (lookup "storage.k8s.io/v1" "StorageClass" "" "").items }}
33+
{{- $isDefault := false }}
34+
{{- if $sc.metadata.annotations }}
35+
{{- $isDefault = eq (index $sc.metadata.annotations "storageclass.kubernetes.io/is-default-class" | default "false") "true" }}
36+
{{- end }}
37+
{{- $_ := set $storageClasses $sc.metadata.name (dict "infraStorageClass" $sc.metadata.name "default" $isDefault) }}
38+
{{- end }}
39+
{{- if not $storageClasses }}
40+
{{- $_ := set $storageClasses "replicated" (dict "infraStorageClass" "replicated" "default" true) }}
41+
{{- end }}
42+
{{- $storageClasses | toYaml | nindent 6 }}
3343
dependsOn:
3444
- name: {{ .Release.Name }}-vsnap-crd
3545
namespace: {{ .Release.Namespace }}

‎packages/apps/kubernetes/values.schema.json‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -613,11 +613,6 @@
613613
}
614614
}
615615
},
616-
"storageClass": {
617-
"description": "StorageClass used to store the data.",
618-
"type": "string",
619-
"default": "replicated"
620-
},
621616
"version": {
622617
"description": "Kubernetes major.minor version to deploy",
623618
"type": "string",

‎packages/apps/kubernetes/values.yaml‎

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,3 @@
1-
##
2-
## @section Common Parameters
3-
##
4-
5-
## @param {string} storageClass - StorageClass used to store the data.
6-
storageClass: replicated
7-
81
##
92
## @section Application-specific Parameters
103
##

‎packages/core/platform/images/migrations/migrations/35‎

Lines changed: 223 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,223 @@
1+
#!/bin/bash
2+
# Migration 35 --> 36
3+
# Remove deprecated 'storageClass' parameter from Kubernetes app instances.
4+
# StorageClasses are now auto-discovered from the infrastructure cluster.
5+
# Idempotent: skips if storageClass key is already absent.
6+
7+
set -euo pipefail
8+
9+
# ============================================================
10+
# STEP 1: Discover all Kubernetes HelmReleases
11+
# ============================================================
12+
echo "=== Discovering Kubernetes HelmReleases ==="
13+
INSTANCES=()
14+
while IFS=/ read -r ns name; do
15+
[ -z "$ns" ] && continue
16+
INSTANCES+=("${ns}/${name}")
17+
echo " Found: ${ns}/${name}"
18+
done < <(kubectl get hr -A -l "apps.cozystack.io/application.kind=Kubernetes" \
19+
-o jsonpath='{range .items[*]}{.metadata.namespace}/{.metadata.name}{"\n"}{end}' 2>/dev/null)
20+
21+
if [ ${#INSTANCES[@]} -eq 0 ]; then
22+
echo " No Kubernetes HelmReleases found. Nothing to migrate."
23+
kubectl create configmap -n cozy-system cozystack-version \
24+
--from-literal=version=36 --dry-run=client -o yaml | kubectl apply -f-
25+
exit 0
26+
fi
27+
echo " Total: ${#INSTANCES[@]} instance(s)"
28+
29+
# ============================================================
30+
# STEP 2: Migrate each instance
31+
# ============================================================
32+
for entry in "${INSTANCES[@]}"; do
33+
NAMESPACE="${entry%%/*}"
34+
HR_NAME="${entry#*/}"
35+
36+
echo ""
37+
echo "======================================================================"
38+
echo "=== Processing: ${HR_NAME} in ${NAMESPACE}"
39+
echo "======================================================================"
40+
41+
# --- Find values Secret ---
42+
VALUES_SECRET=$(kubectl -n "$NAMESPACE" get hr "$HR_NAME" -o json | \
43+
jq -r '.spec.valuesFrom // [] | map(select(.kind == "Secret" and (.name | test("cozystack-values") | not))) | .[0].name // ""')
44+
VALUES_KEY=$(kubectl -n "$NAMESPACE" get hr "$HR_NAME" -o json | \
45+
jq -r '.spec.valuesFrom // [] | map(select(.kind == "Secret" and (.name | test("cozystack-values") | not))) | .[0].valuesKey // "values.yaml"')
46+
47+
if [ -z "$VALUES_SECRET" ]; then
48+
echo " [SKIP] No values Secret found for hr/${HR_NAME}"
49+
continue
50+
fi
51+
52+
if ! kubectl -n "$NAMESPACE" get secret "$VALUES_SECRET" --no-headers 2>/dev/null | grep -q .; then
53+
echo " [SKIP] Secret ${VALUES_SECRET} not found"
54+
continue
55+
fi
56+
57+
echo " Reading values from secret: ${VALUES_SECRET} (key: ${VALUES_KEY})"
58+
59+
# --- Decode current values ---
60+
VALUES_YAML=$(kubectl -n "$NAMESPACE" get secret "$VALUES_SECRET" \
61+
-o jsonpath="{.data.${VALUES_KEY}}" 2>/dev/null | base64 -d 2>/dev/null || true)
62+
if [ -z "$VALUES_YAML" ]; then
63+
VALUES_YAML=$(kubectl -n "$NAMESPACE" get secret "$VALUES_SECRET" \
64+
-o jsonpath="{.stringData.${VALUES_KEY}}" 2>/dev/null || true)
65+
fi
66+
67+
if [ -z "$VALUES_YAML" ]; then
68+
echo " [SKIP] Could not read values from secret"
69+
continue
70+
fi
71+
72+
# --- Check if storageClass key exists ---
73+
STORAGE_CLASS=$(echo "$VALUES_YAML" | yq -r '.storageClass // ""')
74+
75+
if [ -z "$STORAGE_CLASS" ]; then
76+
echo " [SKIP] storageClass key is already absent or empty"
77+
continue
78+
fi
79+
80+
echo " [REMOVE] storageClass='${STORAGE_CLASS}', removing key"
81+
82+
# --- Remove storageClass key ---
83+
NEW_VALUES=$(echo "$VALUES_YAML" | yq 'del(.storageClass)')
84+
85+
# --- Patch the Secret ---
86+
NEW_VALUES_B64=$(echo "$NEW_VALUES" | base64)
87+
echo " [PATCH] Updating secret/${VALUES_SECRET}"
88+
kubectl -n "$NAMESPACE" get secret "$VALUES_SECRET" -o json | \
89+
jq --arg key "$VALUES_KEY" --arg val "$NEW_VALUES_B64" \
90+
'.data[$key] = $val' | \
91+
kubectl apply -f -
92+
93+
echo " [OK] Removed storageClass for ${HR_NAME}"
94+
done
95+
96+
# ============================================================
97+
# STEP 3: PVC rebinding in tenant clusters
98+
# Migrate PVCs with storageClassName=kubevirt to the new
99+
# auto-discovered StorageClass name.
100+
# ============================================================
101+
echo ""
102+
echo "=== STEP 3: Migrating PVCs inside tenant clusters ==="
103+
104+
for entry in "${INSTANCES[@]}"; do
105+
NAMESPACE="${entry%%/*}"
106+
HR_NAME="${entry#*/}"
107+
108+
echo ""
109+
echo "--- Tenant PVC migration: ${HR_NAME} in ${NAMESPACE} ---"
110+
111+
# 3a. Extract tenant kubeconfig
112+
KUBECONFIG_SECRET="${HR_NAME}-admin-kubeconfig"
113+
if ! kubectl -n "$NAMESPACE" get secret "$KUBECONFIG_SECRET" --no-headers 2>/dev/null | grep -q .; then
114+
echo " [SKIP] Kubeconfig secret ${KUBECONFIG_SECRET} not found"
115+
continue
116+
fi
117+
118+
TENANT_KUBECONFIG=$(mktemp)
119+
kubectl -n "$NAMESPACE" get secret "$KUBECONFIG_SECRET" \
120+
-o 'jsonpath={.data.super-admin\.svc}' | base64 -d > "$TENANT_KUBECONFIG"
121+
122+
# Verify connectivity
123+
if ! kubectl --kubeconfig="$TENANT_KUBECONFIG" get ns >/dev/null 2>&1; then
124+
echo " [SKIP] Cannot reach tenant cluster ${HR_NAME}"
125+
rm -f "$TENANT_KUBECONFIG"
126+
continue
127+
fi
128+
129+
# 3b. List all PVCs with storageClassName=kubevirt
130+
PVCS=$(kubectl --kubeconfig="$TENANT_KUBECONFIG" get pvc -A \
131+
-o jsonpath='{range .items[?(@.spec.storageClassName=="kubevirt")]}{.metadata.namespace}/{.metadata.name}{"\n"}{end}' 2>/dev/null || true)
132+
133+
if [ -z "$PVCS" ]; then
134+
echo " [SKIP] No PVCs with storageClassName=kubevirt found"
135+
rm -f "$TENANT_KUBECONFIG"
136+
continue
137+
fi
138+
139+
# 3c-3d. Rebind each PVC
140+
while IFS= read -r pvc_entry; do
141+
[ -z "$pvc_entry" ] && continue
142+
PVC_NS="${pvc_entry%%/*}"
143+
PVC_NAME="${pvc_entry#*/}"
144+
145+
echo " Processing PVC ${PVC_NS}/${PVC_NAME}..."
146+
147+
# Get the bound PV name
148+
PV_NAME=$(kubectl --kubeconfig="$TENANT_KUBECONFIG" -n "$PVC_NS" get pvc "$PVC_NAME" \
149+
-o jsonpath='{.spec.volumeName}' 2>/dev/null || true)
150+
151+
if [ -z "$PV_NAME" ]; then
152+
echo " [WARN] PVC ${PVC_NS}/${PVC_NAME} has no bound PV, skipping"
153+
continue
154+
fi
155+
156+
# Determine the new StorageClass name from PV's CSI volume attributes
157+
NEW_SC=$(kubectl --kubeconfig="$TENANT_KUBECONFIG" get pv "$PV_NAME" \
158+
-o jsonpath='{.spec.csi.volumeAttributes.infraStorageClassName}' 2>/dev/null || true)
159+
160+
# Fallback to tenant's default StorageClass
161+
if [ -z "$NEW_SC" ]; then
162+
NEW_SC=$(kubectl --kubeconfig="$TENANT_KUBECONFIG" get sc \
163+
-o jsonpath='{range .items[?(@.metadata.annotations.storageclass\.kubernetes\.io/is-default-class=="true")]}{.metadata.name}{end}' 2>/dev/null || true)
164+
fi
165+
166+
if [ -z "$NEW_SC" ]; then
167+
echo " [WARN] Cannot determine new StorageClass for PVC ${PVC_NS}/${PVC_NAME}, skipping"
168+
continue
169+
fi
170+
171+
echo " Rebinding PVC ${PVC_NS}/${PVC_NAME}: kubevirt → ${NEW_SC} (PV: ${PV_NAME})"
172+
173+
# 1. Save the PVC spec
174+
PVC_JSON=$(kubectl --kubeconfig="$TENANT_KUBECONFIG" -n "$PVC_NS" get pvc "$PVC_NAME" -o json)
175+
176+
# 2. Protect the PV with Retain policy
177+
OLD_RECLAIM=$(kubectl --kubeconfig="$TENANT_KUBECONFIG" get pv "$PV_NAME" \
178+
-o jsonpath='{.spec.persistentVolumeReclaimPolicy}')
179+
kubectl --kubeconfig="$TENANT_KUBECONFIG" patch pv "$PV_NAME" \
180+
-p '{"spec":{"persistentVolumeReclaimPolicy":"Retain"}}'
181+
182+
# 3. Delete the PVC
183+
kubectl --kubeconfig="$TENANT_KUBECONFIG" -n "$PVC_NS" delete pvc "$PVC_NAME" --wait=false
184+
kubectl --kubeconfig="$TENANT_KUBECONFIG" -n "$PVC_NS" wait --for=delete pvc/"$PVC_NAME" --timeout=60s 2>/dev/null || true
185+
186+
# 4. Patch the PV: update storageClassName and release claimRef
187+
kubectl --kubeconfig="$TENANT_KUBECONFIG" patch pv "$PV_NAME" --type=json \
188+
-p "[{\"op\":\"replace\",\"path\":\"/spec/storageClassName\",\"value\":\"${NEW_SC}\"},
189+
{\"op\":\"remove\",\"path\":\"/spec/claimRef/uid\"},
190+
{\"op\":\"remove\",\"path\":\"/spec/claimRef/resourceVersion\"}]"
191+
192+
# 5. Recreate PVC with new storageClassName
193+
echo "$PVC_JSON" | jq --arg sc "$NEW_SC" '
194+
.spec.storageClassName = $sc |
195+
del(.metadata.resourceVersion, .metadata.uid, .metadata.creationTimestamp,
196+
.metadata.annotations["pv.kubernetes.io/bind-completed"],
197+
.metadata.annotations["pv.kubernetes.io/bound-by-controller"],
198+
.status)
199+
' | kubectl --kubeconfig="$TENANT_KUBECONFIG" apply -f -
200+
201+
# 6. Wait for PVC to rebind
202+
if kubectl --kubeconfig="$TENANT_KUBECONFIG" -n "$PVC_NS" wait --for=jsonpath='{.status.phase}'=Bound \
203+
pvc/"$PVC_NAME" --timeout=60s 2>/dev/null; then
204+
# 7. Restore original reclaim policy
205+
kubectl --kubeconfig="$TENANT_KUBECONFIG" patch pv "$PV_NAME" \
206+
-p "{\"spec\":{\"persistentVolumeReclaimPolicy\":\"${OLD_RECLAIM}\"}}"
207+
echo " [OK] Migrated PVC ${PVC_NS}/${PVC_NAME}: kubevirt → ${NEW_SC}"
208+
else
209+
echo " [WARN] PVC ${PVC_NS}/${PVC_NAME} did not reach Bound state within timeout."
210+
echo " PV ${PV_NAME} is protected with Retain policy. Manual intervention may be needed."
211+
fi
212+
213+
done <<< "$PVCS"
214+
215+
rm -f "$TENANT_KUBECONFIG"
216+
done
217+
218+
echo ""
219+
echo "=== Migration complete (${#INSTANCES[@]} instance(s)) ==="
220+
221+
# Stamp version
222+
kubectl create configmap -n cozy-system cozystack-version \
223+
--from-literal=version=36 --dry-run=client -o yaml | kubectl apply -f-

‎packages/core/platform/values.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ sourceRef:
66
migrations:
77
enabled: false
88
image: ghcr.io/cozystack/cozystack/platform-migrations:v1.1.0@sha256:d7e8955c1ad8c8fbd4ce42b014c0f849d73d0c3faf0cedaac8e15d647fb2f663
9-
targetVersion: 35
9+
targetVersion: 36
1010
# Bundle deployment configuration
1111
bundles:
1212
system:

0 commit comments

Comments
 (0)