Skip to content

Commit cbf862c

Browse files
committed
fix(seaweedfs): clean up orphaned data PVCs and cert/db secrets on tenant module delete (#3092)
When the seaweedfs tenant module is disabled, the StatefulSet volume PVCs and a set of certificate/credential secrets were left orphaned. Add a post-delete cleanup hook that deletes the volume PVCs by their release-scoped labels and the secrets by name (all templated from .Release.Name). CNPG PVCs are not targeted (CNPG cleans them). The cleanup Job is hardened (non-root, read-only root fs, dropped caps, resource limits); RBAC scoped via resourceNames; failures are logged, never blocking teardown. NOTE: disabling seaweedfs now permanently removes its volume PVCs and the data they hold. Signed-off-by: Алексей Артамонов <[email protected]>
1 parent 374cac1 commit cbf862c

3 files changed

Lines changed: 275 additions & 0 deletions

File tree

‎packages/extra/seaweedfs/Makefile‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ NAME=seaweedfs
22

33
include ../../../hack/package.mk
44

5+
.PHONY: test
56
test:
67
helm unittest .
78

Lines changed: 122 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,122 @@
1+
---
2+
apiVersion: v1
3+
kind: ServiceAccount
4+
metadata:
5+
name: {{ .Release.Name }}-cleanup
6+
labels:
7+
app.kubernetes.io/instance: {{ .Release.Name }}
8+
annotations:
9+
"helm.sh/hook": post-delete
10+
"helm.sh/hook-weight": "0"
11+
"helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
12+
---
13+
apiVersion: rbac.authorization.k8s.io/v1
14+
kind: Role
15+
metadata:
16+
name: {{ .Release.Name }}-cleanup
17+
labels:
18+
app.kubernetes.io/instance: {{ .Release.Name }}
19+
annotations:
20+
"helm.sh/hook": post-delete
21+
"helm.sh/hook-weight": "5"
22+
"helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
23+
rules:
24+
- apiGroups: [""]
25+
resources: ["persistentvolumeclaims"]
26+
verbs: ["get", "list", "delete"]
27+
- apiGroups: [""]
28+
resources: ["secrets"]
29+
verbs: ["get", "delete"]
30+
resourceNames:
31+
- {{ .Release.Name }}-s3-ingress-tls
32+
- {{ .Release.Name }}-system-admin-cert
33+
- {{ .Release.Name }}-system-ca-cert
34+
- {{ .Release.Name }}-system-client-cert
35+
- {{ .Release.Name }}-system-filer-cert
36+
- {{ .Release.Name }}-system-master-cert
37+
- {{ .Release.Name }}-system-volume-cert
38+
- {{ .Release.Name }}-system-worker-cert
39+
- {{ .Release.Name }}-system-db-secret
40+
---
41+
apiVersion: rbac.authorization.k8s.io/v1
42+
kind: RoleBinding
43+
metadata:
44+
name: {{ .Release.Name }}-cleanup
45+
labels:
46+
app.kubernetes.io/instance: {{ .Release.Name }}
47+
annotations:
48+
"helm.sh/hook": post-delete
49+
"helm.sh/hook-weight": "5"
50+
"helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
51+
roleRef:
52+
apiGroup: rbac.authorization.k8s.io
53+
kind: Role
54+
name: {{ .Release.Name }}-cleanup
55+
subjects:
56+
- kind: ServiceAccount
57+
name: {{ .Release.Name }}-cleanup
58+
namespace: {{ .Release.Namespace }}
59+
---
60+
apiVersion: batch/v1
61+
kind: Job
62+
metadata:
63+
name: {{ .Release.Name }}-cleanup
64+
labels:
65+
app.kubernetes.io/instance: {{ .Release.Name }}
66+
annotations:
67+
"helm.sh/hook": post-delete
68+
"helm.sh/hook-weight": "10"
69+
"helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
70+
spec:
71+
backoffLimit: 3
72+
template:
73+
metadata:
74+
labels:
75+
app.kubernetes.io/instance: {{ .Release.Name }}
76+
policy.cozystack.io/allow-to-apiserver: "true"
77+
spec:
78+
serviceAccountName: {{ .Release.Name }}-cleanup
79+
restartPolicy: Never
80+
securityContext:
81+
runAsNonRoot: true
82+
runAsUser: 65534
83+
runAsGroup: 65534
84+
fsGroup: 65534
85+
seccompProfile:
86+
type: RuntimeDefault
87+
volumes:
88+
- name: tmp
89+
emptyDir: {}
90+
containers:
91+
- name: cleanup
92+
image: docker.io/clastix/kubectl:v1.32
93+
env:
94+
- name: HOME
95+
value: /tmp
96+
securityContext:
97+
allowPrivilegeEscalation: false
98+
readOnlyRootFilesystem: true
99+
capabilities:
100+
drop:
101+
- ALL
102+
volumeMounts:
103+
- name: tmp
104+
mountPath: /tmp
105+
resources:
106+
requests:
107+
cpu: 10m
108+
memory: 64Mi
109+
limits:
110+
cpu: 200m
111+
memory: 128Mi
112+
command:
113+
- /bin/sh
114+
- -c
115+
- |
116+
echo "Deleting orphaned SeaweedFS volume PVCs for {{ .Release.Name }}..."
117+
kubectl delete pvc -n {{ .Release.Namespace }} -l app.kubernetes.io/name=seaweedfs,app.kubernetes.io/instance={{ .Release.Name }}-system,app.kubernetes.io/component=volume --ignore-not-found \
118+
|| echo "WARNING: SeaweedFS PVC cleanup failed for {{ .Release.Name }}; orphaned PVCs may remain" >&2
119+
echo "Deleting orphaned SeaweedFS cert/db secrets for {{ .Release.Name }}..."
120+
kubectl delete secret -n {{ .Release.Namespace }} {{ .Release.Name }}-s3-ingress-tls {{ .Release.Name }}-system-admin-cert {{ .Release.Name }}-system-ca-cert {{ .Release.Name }}-system-client-cert {{ .Release.Name }}-system-filer-cert {{ .Release.Name }}-system-master-cert {{ .Release.Name }}-system-volume-cert {{ .Release.Name }}-system-worker-cert {{ .Release.Name }}-system-db-secret --ignore-not-found \
121+
|| echo "WARNING: SeaweedFS secret cleanup failed for {{ .Release.Name }}; orphaned secrets may remain" >&2
122+
echo "SeaweedFS cleanup complete."
Lines changed: 152 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,152 @@
1+
suite: seaweedfs post-delete cleanup hook removes orphaned PVCs and secrets
2+
templates:
3+
- templates/hooks/cleanup.yaml
4+
5+
release:
6+
name: seaweedfs
7+
namespace: tenant-root
8+
9+
# After the seaweedfs module is disabled, the StatefulSet data PVCs
10+
# (data1-seaweedfs-system-volume-{0,1}) and the cert-manager TLS / db
11+
# secrets are left orphaned. The post-delete hook renders a Job (plus its
12+
# RBAC) that deletes the volume PVCs by label and the secrets by name.
13+
# All names are templated from .Release.Name: the leaked resources use the
14+
# {{ .Release.Name }}-system prefix because seaweedfs.yaml renders the
15+
# StatefulSet/cert HelmRelease as "{{ .Release.Name }}-system".
16+
17+
tests:
18+
- it: renders the cleanup ServiceAccount as a post-delete hook
19+
documentSelector:
20+
path: kind
21+
value: ServiceAccount
22+
asserts:
23+
- equal:
24+
path: metadata.name
25+
value: seaweedfs-cleanup
26+
- equal:
27+
path: metadata.annotations["helm.sh/hook"]
28+
value: post-delete
29+
- equal:
30+
path: metadata.annotations["helm.sh/hook-weight"]
31+
value: "0"
32+
- equal:
33+
path: metadata.annotations["helm.sh/hook-delete-policy"]
34+
value: before-hook-creation,hook-succeeded
35+
36+
- it: grants delete on PVCs and only the 9 named secrets
37+
documentSelector:
38+
path: kind
39+
value: Role
40+
asserts:
41+
- equal:
42+
path: metadata.annotations["helm.sh/hook-weight"]
43+
value: "5"
44+
- contains:
45+
path: rules
46+
content:
47+
apiGroups: [""]
48+
resources: ["persistentvolumeclaims"]
49+
verbs: ["get", "list", "delete"]
50+
- contains:
51+
path: rules
52+
content:
53+
apiGroups: [""]
54+
resources: ["secrets"]
55+
verbs: ["get", "delete"]
56+
resourceNames:
57+
- seaweedfs-s3-ingress-tls
58+
- seaweedfs-system-admin-cert
59+
- seaweedfs-system-ca-cert
60+
- seaweedfs-system-client-cert
61+
- seaweedfs-system-filer-cert
62+
- seaweedfs-system-master-cert
63+
- seaweedfs-system-volume-cert
64+
- seaweedfs-system-worker-cert
65+
- seaweedfs-system-db-secret
66+
67+
- it: binds the cleanup Role to the cleanup ServiceAccount
68+
documentSelector:
69+
path: kind
70+
value: RoleBinding
71+
asserts:
72+
- equal:
73+
path: metadata.annotations["helm.sh/hook-weight"]
74+
value: "5"
75+
- equal:
76+
path: roleRef.name
77+
value: seaweedfs-cleanup
78+
- equal:
79+
path: subjects[0].kind
80+
value: ServiceAccount
81+
- equal:
82+
path: subjects[0].name
83+
value: seaweedfs-cleanup
84+
85+
- it: renders a hardened cleanup Job with the right hook metadata
86+
documentSelector:
87+
path: kind
88+
value: Job
89+
asserts:
90+
- equal:
91+
path: metadata.name
92+
value: seaweedfs-cleanup
93+
- equal:
94+
path: metadata.annotations["helm.sh/hook"]
95+
value: post-delete
96+
- equal:
97+
path: metadata.annotations["helm.sh/hook-weight"]
98+
value: "10"
99+
- equal:
100+
path: metadata.annotations["helm.sh/hook-delete-policy"]
101+
value: before-hook-creation,hook-succeeded
102+
- equal:
103+
path: spec.template.metadata.labels["policy.cozystack.io/allow-to-apiserver"]
104+
value: "true"
105+
- equal:
106+
path: spec.template.spec.serviceAccountName
107+
value: seaweedfs-cleanup
108+
- equal:
109+
path: spec.template.spec.containers[0].image
110+
value: docker.io/clastix/kubectl:v1.32
111+
- equal:
112+
path: spec.template.spec.securityContext.runAsNonRoot
113+
value: true
114+
- equal:
115+
path: spec.template.spec.securityContext.runAsUser
116+
value: 65534
117+
- equal:
118+
path: spec.template.spec.securityContext.seccompProfile.type
119+
value: RuntimeDefault
120+
- equal:
121+
path: spec.template.spec.containers[0].securityContext.readOnlyRootFilesystem
122+
value: true
123+
- equal:
124+
path: spec.template.spec.containers[0].securityContext.allowPrivilegeEscalation
125+
value: false
126+
- contains:
127+
path: spec.template.spec.containers[0].securityContext.capabilities.drop
128+
content: ALL
129+
- equal:
130+
path: spec.template.spec.containers[0].resources.requests.cpu
131+
value: 10m
132+
- equal:
133+
path: spec.template.spec.containers[0].resources.requests.memory
134+
value: 64Mi
135+
- equal:
136+
path: spec.template.spec.containers[0].resources.limits.cpu
137+
value: 200m
138+
- equal:
139+
path: spec.template.spec.containers[0].resources.limits.memory
140+
value: 128Mi
141+
142+
- it: deletes volume PVCs by label and the named secrets in the cleanup command
143+
documentSelector:
144+
path: kind
145+
value: Job
146+
asserts:
147+
- matchRegex:
148+
path: spec.template.spec.containers[0].command[2]
149+
pattern: 'kubectl delete pvc -n tenant-root -l app\.kubernetes\.io/name=seaweedfs,app\.kubernetes\.io/instance=seaweedfs-system,app\.kubernetes\.io/component=volume --ignore-not-found'
150+
- matchRegex:
151+
path: spec.template.spec.containers[0].command[2]
152+
pattern: 'kubectl delete secret -n tenant-root seaweedfs-s3-ingress-tls seaweedfs-system-admin-cert seaweedfs-system-ca-cert seaweedfs-system-client-cert seaweedfs-system-filer-cert seaweedfs-system-master-cert seaweedfs-system-volume-cert seaweedfs-system-worker-cert seaweedfs-system-db-secret --ignore-not-found'

0 commit comments

Comments
 (0)