Skip to content

Commit 78891d1

Browse files
[kubernetes] Auto-propagate infra StorageClasses to tenant clusters
Replace the single static `storageClass` parameter with automatic discovery of all infrastructure StorageClasses using Helm `lookup`. Each infra StorageClass is propagated to tenant clusters with the same name, and an enforcement allowList restricts tenants to only use propagated classes. Closes: #1424 Signed-off-by: mattia-eleuteri <[email protected]>
1 parent 1fd1da4 commit 78891d1

9 files changed

Lines changed: 86 additions & 30 deletions

File tree

‎hack/e2e-apps/run-kubernetes.sh‎

Lines changed: 36 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,6 @@ spec:
5959
minReplicas: 2
6060
roles:
6161
- ingress-nginx
62-
storageClass: replicated
6362
version: "${k8s_version}"
6463
EOF
6564
# Wait for the tenant-test namespace to be active
@@ -128,6 +127,41 @@ EOF
128127
exit 1
129128
fi
130129

130+
# Verify StorageClass propagation from infra to tenant cluster
131+
echo "Verifying StorageClass propagation..."
132+
133+
# Wait for at least one StorageClass to appear in the tenant cluster
134+
timeout 2m bash -c '
135+
until [ "$(kubectl --kubeconfig tenantkubeconfig-'"${test_name}"' get sc -o jsonpath="{.items[*].metadata.name}" 2>/dev/null | wc -w)" -ge 1 ]; do
136+
sleep 5
137+
done
138+
'
139+
140+
sc_names=$(kubectl --kubeconfig "tenantkubeconfig-${test_name}" get sc -o jsonpath='{.items[*].metadata.name}')
141+
if [ -z "$sc_names" ]; then
142+
echo "No StorageClasses found in tenant cluster" >&2
143+
exit 1
144+
fi
145+
echo "StorageClasses in tenant: ${sc_names}"
146+
147+
# Verify each propagated StorageClass uses the kubevirt CSI provisioner
148+
for sc in $sc_names; do
149+
provisioner=$(kubectl --kubeconfig "tenantkubeconfig-${test_name}" get sc "$sc" -o jsonpath='{.provisioner}')
150+
if [ "$provisioner" != "csi.kubevirt.io" ]; then
151+
echo "StorageClass $sc has unexpected provisioner: $provisioner (expected csi.kubevirt.io)" >&2
152+
exit 1
153+
fi
154+
done
155+
echo "All StorageClasses use csi.kubevirt.io provisioner"
156+
157+
# Verify exactly one default StorageClass is set
158+
default_count=$(kubectl --kubeconfig "tenantkubeconfig-${test_name}" get sc \
159+
-o jsonpath='{range .items[?(@.metadata.annotations.storageclass\.kubernetes\.io/is-default-class=="true")]}{.metadata.name}{"\n"}{end}' | grep -c .)
160+
if [ "$default_count" -ne 1 ]; then
161+
echo "Expected exactly 1 default StorageClass, found $default_count" >&2
162+
exit 1
163+
fi
164+
echo "Default StorageClass is correctly set"
131165

132166
kubectl --kubeconfig "tenantkubeconfig-${test_name}" apply -f - <<EOF
133167
apiVersion: v1
@@ -244,7 +278,7 @@ metadata:
244278
spec:
245279
accessModes:
246280
- ReadWriteMany
247-
storageClassName: kubevirt
281+
storageClassName: replicated
248282
resources:
249283
requests:
250284
storage: 1Gi

‎packages/apps/kubernetes/README.md‎

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -82,13 +82,6 @@ See the reference for components utilized in this service:
8282
8383
## Parameters
8484
85-
### Common Parameters
86-
87-
| Name | Description | Type | Value |
88-
| -------------- | ------------------------------------ | -------- | ------------ |
89-
| `storageClass` | StorageClass used to store the data. | `string` | `replicated` |
90-
91-
9285
### Application-specific Parameters
9386
9487
| Name | Description | Type | Value |
Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
apiVersion: v1
2+
kind: ConfigMap
3+
metadata:
4+
name: driver-config
5+
data:
6+
infraStorageClassEnforcement: |
7+
allowAll: false
8+
allowDefault: false
9+
allowList:
10+
{{- $storageClasses := (lookup "storage.k8s.io/v1" "StorageClass" "" "").items }}
11+
{{- if $storageClasses }}
12+
{{- range $storageClasses }}
13+
- {{ .metadata.name | quote }}
14+
{{- end }}
15+
{{- else }}
16+
- "replicated"
17+
{{- end }}

‎packages/apps/kubernetes/templates/helmreleases/csi.yaml‎

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -26,10 +26,20 @@ spec:
2626
upgrade:
2727
remediation:
2828
retries: -1
29-
{{- with .Values.storageClass }}
3029
values:
31-
storageClass: "{{ . }}"
32-
{{- end }}
30+
storageClasses:
31+
{{- $storageClasses := dict }}
32+
{{- range $sc := (lookup "storage.k8s.io/v1" "StorageClass" "" "").items }}
33+
{{- $isDefault := false }}
34+
{{- if $sc.metadata.annotations }}
35+
{{- $isDefault = eq (index $sc.metadata.annotations "storageclass.kubernetes.io/is-default-class" | default "false") "true" }}
36+
{{- end }}
37+
{{- $_ := set $storageClasses $sc.metadata.name (dict "infraStorageClass" $sc.metadata.name "default" $isDefault) }}
38+
{{- end }}
39+
{{- if not $storageClasses }}
40+
{{- $_ := set $storageClasses "replicated" (dict "infraStorageClass" "replicated" "default" true) }}
41+
{{- end }}
42+
{{- $storageClasses | toYaml | nindent 6 }}
3343
dependsOn:
3444
- name: {{ .Release.Name }}-vsnap-crd
3545
namespace: {{ .Release.Namespace }}

‎packages/apps/kubernetes/values.schema.json‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -613,11 +613,6 @@
613613
}
614614
}
615615
},
616-
"storageClass": {
617-
"description": "StorageClass used to store the data.",
618-
"type": "string",
619-
"default": "replicated"
620-
},
621616
"version": {
622617
"description": "Kubernetes major.minor version to deploy",
623618
"type": "string",

‎packages/apps/kubernetes/values.yaml‎

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,3 @@
1-
##
2-
## @section Common Parameters
3-
##
4-
5-
## @param {string} storageClass - StorageClass used to store the data.
6-
storageClass: replicated
7-
81
##
92
## @section Application-specific Parameters
103
##

‎packages/system/kubernetes-rd/cozyrds/kubernetes.yaml‎

Lines changed: 2 additions & 2 deletions
Large diffs are not rendered by default.

‎packages/system/kubevirt-csi-node/templates/deploy.yaml‎

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -264,17 +264,28 @@ metadata:
264264
name: driver-config
265265
namespace: {{ .Release.Namespace }}
266266
data:
267-
infraClusterNamespace:
267+
infraClusterNamespace:
268268
infraClusterLabels: csi-driver/cluster=tenant
269+
infraStorageClassEnforcement: |
270+
allowAll: false
271+
allowDefault: false
272+
allowList:
273+
{{- range $name, $mapping := .Values.storageClasses }}
274+
- {{ $mapping.infraStorageClass | quote }}
275+
{{- end }}
276+
{{- range $name, $mapping := .Values.storageClasses }}
269277
---
270278
apiVersion: storage.k8s.io/v1
271279
kind: StorageClass
272280
metadata:
273-
name: kubevirt
281+
name: {{ $name }}
282+
{{- if $mapping.default }}
274283
annotations:
275284
storageclass.kubernetes.io/is-default-class: "true"
285+
{{- end }}
276286
provisioner: csi.kubevirt.io
277287
allowVolumeExpansion: true
278288
parameters:
279-
infraStorageClassName: {{ .Values.storageClass }}
289+
infraStorageClassName: {{ $mapping.infraStorageClass }}
280290
bus: scsi
291+
{{- end }}
Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,6 @@
1-
storageClass: replicated
1+
storageClasses:
2+
replicated:
3+
infraStorageClass: replicated
4+
default: true
25
csiDriver:
36
image: ghcr.io/cozystack/cozystack/kubevirt-csi-driver:0.0.0@sha256:604561e23df1b8eb25c24cf73fd93c7aaa6d1e7c56affbbda5c6f0f83424e4b1

0 commit comments

Comments
 (0)