Skip to content

Commit e7ffc21

Browse files
lexfreiclaude
andcommitted
feat(harbor): switch registry storage to S3 via COSI BucketClaim
Replace PVC-based registry storage with S3 via COSI BucketClaim/BucketAccess. The system chart parses BucketInfo secret and creates a registry-s3 Secret with REGISTRY_STORAGE_S3_* env vars that override Harbor's ConfigMap values. - Add bucket-secret.yaml to system chart (BucketInfo parser) - Remove storageType/size from registry config (S3 is now the only option) - Use Harbor's existingSecret support for S3 credentials injection - Add objectstorage-controller to PackageSource dependencies - Update E2E test with COSI bucket provisioning waits and diagnostics Co-Authored-By: Claude <[email protected]> Signed-off-by: Aleksei Sviridkin <[email protected]>
1 parent efb9bc7 commit e7ffc21

10 files changed

Lines changed: 42 additions & 68 deletions

File tree

‎hack/e2e-apps/harbor.bats‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,8 +16,6 @@ spec:
1616
resources: {}
1717
resourcesPreset: "nano"
1818
registry:
19-
storageType: "pvc"
20-
size: 5Gi
2119
resources: {}
2220
resourcesPreset: "nano"
2321
jobservice:
@@ -37,6 +35,13 @@ spec:
3735
EOF
3836
sleep 5
3937
kubectl -n tenant-test wait hr $release --timeout=60s --for=condition=ready
38+
39+
# Wait for COSI to provision bucket
40+
kubectl -n tenant-test wait bucketclaims.objectstorage.k8s.io $release-registry \
41+
--timeout=300s --for=jsonpath='{.status.bucketReady}'=true
42+
kubectl -n tenant-test wait bucketaccesses.objectstorage.k8s.io $release-registry \
43+
--timeout=60s --for=jsonpath='{.status.accessGranted}'=true
44+
4045
kubectl -n tenant-test wait hr $release-system --timeout=600s --for=condition=ready || {
4146
echo "=== HelmRelease status ==="
4247
kubectl -n tenant-test get hr $release-system -o yaml 2>&1 || true
@@ -46,6 +51,12 @@ EOF
4651
kubectl -n tenant-test get events --sort-by='.lastTimestamp' 2>&1 | tail -30 || true
4752
echo "=== ExternalArtifact ==="
4853
kubectl -n cozy-system get externalartifact cozystack-harbor-application-default-harbor-system -o yaml 2>&1 || true
54+
echo "=== BucketClaim status ==="
55+
kubectl -n tenant-test get bucketclaims.objectstorage.k8s.io $release-registry -o yaml 2>&1 || true
56+
echo "=== BucketAccess status ==="
57+
kubectl -n tenant-test get bucketaccesses.objectstorage.k8s.io $release-registry -o yaml 2>&1 || true
58+
echo "=== BucketAccess Secret ==="
59+
kubectl -n tenant-test get secret $release-registry-bucket -o jsonpath='{.data.BucketInfo}' 2>&1 | base64 -d 2>&1 || true
4960
false
5061
}
5162
kubectl -n tenant-test wait deploy $release-core --timeout=120s --for=condition=available

‎packages/apps/harbor/README.md‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,8 +22,6 @@ Harbor is an open source trusted cloud native registry project that stores, sign
2222
| `core.resources.memory` | Amount of memory allocated. | `quantity` | `""` |
2323
| `core.resourcesPreset` | Default sizing preset used when `resources` is omitted. | `string` | `small` |
2424
| `registry` | Container image registry configuration. | `object` | `{}` |
25-
| `registry.storageType` | Storage backend type for images. | `string` | `pvc` |
26-
| `registry.size` | Persistent Volume size (only used when storageType is pvc). | `quantity` | `10Gi` |
2725
| `registry.resources` | Explicit CPU and memory configuration. When omitted, the preset defined in `resourcesPreset` is applied. | `object` | `{}` |
2826
| `registry.resources.cpu` | Number of CPU cores allocated. | `quantity` | `""` |
2927
| `registry.resources.memory` | Amount of memory allocated. | `quantity` | `""` |

‎packages/apps/harbor/templates/bucket.yaml‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,3 @@
1-
{{- if eq .Values.registry.storageType "s3" }}
21
{{- $seaweedfs := .Values._namespace.seaweedfs }}
32
apiVersion: objectstorage.k8s.io/v1alpha1
43
kind: BucketClaim
@@ -18,4 +17,3 @@ spec:
1817
bucketClaimName: {{ .Release.Name }}-registry
1918
credentialsSecretName: {{ .Release.Name }}-registry-bucket
2019
protocol: s3
21-
{{- end }}

‎packages/apps/harbor/templates/harbor.yaml‎

Lines changed: 4 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -66,21 +66,9 @@ spec:
6666
name: {{ .Release.Name }}-credentials
6767
valuesKey: redis-password
6868
targetPath: harbor.redis.external.password
69-
{{- if eq .Values.registry.storageType "s3" }}
70-
- kind: Secret
71-
name: {{ .Release.Name }}-registry-bucket
72-
valuesKey: accessKey
73-
targetPath: harbor.persistence.imageChartStorage.s3.accesskey
74-
- kind: Secret
75-
name: {{ .Release.Name }}-registry-bucket
76-
valuesKey: secretKey
77-
targetPath: harbor.persistence.imageChartStorage.s3.secretkey
78-
- kind: Secret
79-
name: {{ .Release.Name }}-registry-bucket
80-
valuesKey: endpoint
81-
targetPath: harbor.persistence.imageChartStorage.s3.regionendpoint
82-
{{- end }}
8369
values:
70+
bucket:
71+
secretName: {{ .Release.Name }}-registry-bucket
8472
db:
8573
replicas: {{ .Values.database.replicas }}
8674
size: {{ .Values.database.size }}
@@ -106,31 +94,21 @@ spec:
10694
persistence:
10795
enabled: true
10896
resourcePolicy: "keep"
109-
{{- if eq .Values.registry.storageType "s3" }}
11097
imageChartStorage:
11198
type: s3
11299
s3:
100+
existingSecret: {{ .Release.Name }}-registry-s3
113101
region: us-east-1
114102
bucket: {{ .Release.Name }}-registry
115103
secure: false
116104
v4auth: true
117-
{{- end }}
118-
{{- if or (eq .Values.registry.storageType "pvc") .Values.trivy.enabled }}
105+
{{- if .Values.trivy.enabled }}
119106
persistentVolumeClaim:
120-
{{- if eq .Values.registry.storageType "pvc" }}
121-
registry:
122-
size: {{ .Values.registry.size }}
123-
{{- with .Values.storageClass }}
124-
storageClass: {{ . }}
125-
{{- end }}
126-
{{- end }}
127-
{{- if .Values.trivy.enabled }}
128107
trivy:
129108
size: {{ .Values.trivy.size }}
130109
{{- with .Values.storageClass }}
131110
storageClass: {{ . }}
132111
{{- end }}
133-
{{- end }}
134112
{{- end }}
135113
portal:
136114
resources: {{- include "cozy-lib.resources.defaultingSanitize" (list "nano" (dict) $) | nindent 10 }}

‎packages/apps/harbor/values.schema.json‎

Lines changed: 0 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -179,9 +179,6 @@
179179
"description": "Container image registry configuration.",
180180
"type": "object",
181181
"default": {},
182-
"required": [
183-
"storageType"
184-
],
185182
"properties": {
186183
"resources": {
187184
"description": "Explicit CPU and memory configuration. When omitted, the preset defined in `resourcesPreset` is applied.",
@@ -229,29 +226,6 @@
229226
"xlarge",
230227
"2xlarge"
231228
]
232-
},
233-
"size": {
234-
"description": "Persistent Volume size (only used when storageType is pvc).",
235-
"default": "10Gi",
236-
"pattern": "^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$",
237-
"anyOf": [
238-
{
239-
"type": "integer"
240-
},
241-
{
242-
"type": "string"
243-
}
244-
],
245-
"x-kubernetes-int-or-string": true
246-
},
247-
"storageType": {
248-
"description": "Storage backend type for images.",
249-
"type": "string",
250-
"default": "pvc",
251-
"enum": [
252-
"s3",
253-
"pvc"
254-
]
255229
}
256230
}
257231
},

‎packages/apps/harbor/values.yaml‎

Lines changed: 0 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -34,20 +34,12 @@ core:
3434
resources: {}
3535
resourcesPreset: "small"
3636

37-
## @enum {string} StorageType - Storage backend for container image registry.
38-
## @value s3 - S3-compatible object storage via COSI BucketClaim (requires SeaweedFS in tenant).
39-
## @value pvc - Persistent Volume Claim.
40-
4137
## @typedef {struct} Registry - Container image registry configuration.
42-
## @field {StorageType} storageType - Storage backend type for images.
43-
## @field {quantity} [size] - Persistent Volume size (only used when storageType is pvc).
4438
## @field {Resources} [resources] - Explicit CPU and memory configuration. When omitted, the preset defined in `resourcesPreset` is applied.
4539
## @field {ResourcesPreset} [resourcesPreset] - Default sizing preset used when `resources` is omitted.
4640

4741
## @param {Registry} registry - Container image registry configuration.
4842
registry:
49-
storageType: "pvc"
50-
size: 10Gi
5143
resources: {}
5244
resourcesPreset: "small"
5345

‎packages/core/platform/sources/harbor-application.yaml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ spec:
1515
- cozystack.networking
1616
- cozystack.postgres-operator
1717
- cozystack.redis-operator
18+
- cozystack.objectstorage-controller
1819
libraries:
1920
- name: cozy-lib
2021
path: library/cozy-lib

‎packages/system/harbor-rd/cozyrds/harbor.yaml‎

Lines changed: 2 additions & 2 deletions
Large diffs are not rendered by default.
Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
{{- if .Values.bucket.secretName }}
2+
{{- $existingSecret := lookup "v1" "Secret" .Release.Namespace .Values.bucket.secretName }}
3+
{{- $bucketInfo := fromJson (b64dec (index $existingSecret.data "BucketInfo")) }}
4+
{{- $accessKeyID := index $bucketInfo.spec.secretS3 "accessKeyID" }}
5+
{{- $accessSecretKey := index $bucketInfo.spec.secretS3 "accessSecretKey" }}
6+
{{- $endpoint := index $bucketInfo.spec.secretS3 "endpoint" }}
7+
{{- $bucketName := $bucketInfo.spec.bucketName }}
8+
---
9+
apiVersion: v1
10+
kind: Secret
11+
metadata:
12+
name: {{ .Values.harbor.fullnameOverride }}-registry-s3
13+
type: Opaque
14+
stringData:
15+
REGISTRY_STORAGE_S3_ACCESSKEY: {{ $accessKeyID | quote }}
16+
REGISTRY_STORAGE_S3_SECRETKEY: {{ $accessSecretKey | quote }}
17+
REGISTRY_STORAGE_S3_REGIONENDPOINT: {{ $endpoint | quote }}
18+
REGISTRY_STORAGE_S3_BUCKET: {{ $bucketName | quote }}
19+
REGISTRY_STORAGE_S3_REGION: "us-east-1"
20+
{{- end }}

‎packages/system/harbor/values.yaml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,6 @@
11
harbor: {}
2+
bucket:
3+
secretName: ""
24
db:
35
replicas: 2
46
size: 5Gi

0 commit comments

Comments
 (0)