Skip to content

Commit c282d03

Browse files
lllamnyplllamnyp
authored
[platform] Update lineage labels at upgrade (#1452)
## What this PR does 1. Since the VictoriaMetrics operator aggressively manages the metadata on all owned components, the addition of labels by the lineage webhook causes non-stop updates sent to the k8s API server. We mitigate this by modifying the Monitoring Helm chart to set the `managedMetadata` field on all VictoriaMetrics custom resources, where applicable. 2. This patch adds a migration script, that adds an annotation to all resources that may be of interest, triggering an update event on the lineage webhook. This will analyze the ancestor tree of these resources and add labels to them, referencing their managing application. 3. This patch makes sure that migration #20 really uses the very latest chart versions by forcing a reconcile with cozypkg, instead of annotating the underlying HelmRelease. ### Release note ```release-note [monitoring] Explicitly set lineage labels on VictoriaMetrics' resources known not to play nice when something modifies their owned resources in flight. [platform] Add migration script to update pre-existing resources with lineage labels. [installer] Update cozypkg in installer and use it to bulletproof the 20th migration script by reconciling the HelmReleases with the --with-source flag. ``` Signed-off-by: Timofei Larkin <[email protected]>
2 parents bb653e5 + 0f8a9ac commit c282d03

10 files changed

Lines changed: 49 additions & 13 deletions

File tree

‎internal/lineagecontrollerwebhook/webhook.go‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -135,18 +135,18 @@ func (h *LineageControllerWebhook) computeLabels(ctx context.Context, o *unstruc
135135
}
136136
cfg := h.config.Load().(*runtimeConfig)
137137
crd := cfg.appCRDMap[appRef{gv.Group, obj.GetKind()}]
138-
if matchLabelsToExcludeInclude(o.GetLabels(), crd.Spec.Secrets.Exclude, crd.Spec.Secrets.Include) {
139-
labels["internal.cozystack.io/tenantsecret"] = ""
140-
}
138+
139+
// TODO: expand this to work with other resources than Secrets
140+
labels["apps.cozystack.io/tenantresource"] = func(b bool) string {
141+
if b {
142+
return "true"
143+
}
144+
return "false"
145+
}(matchLabelsToExcludeInclude(o.GetLabels(), crd.Spec.Secrets.Exclude, crd.Spec.Secrets.Include))
141146
return labels, err
142147
}
143148

144149
func (h *LineageControllerWebhook) applyLabels(o *unstructured.Unstructured, labels map[string]string) {
145-
if o.GetAPIVersion() == "operator.victoriametrics.com/v1beta1" && o.GetKind() == "VMCluster" {
146-
unstructured.SetNestedStringMap(o.Object, labels, "spec", "managedMetadata", "labels")
147-
return
148-
}
149-
150150
existing := o.GetLabels()
151151
if existing == nil {
152152
existing = make(map[string]string)

‎packages/core/installer/images/cozystack/Dockerfile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ RUN make repos
3232

3333
FROM alpine:3.22
3434

35-
RUN wget -O- https://github.com/cozystack/cozypkg/raw/refs/heads/main/hack/install.sh | sh -s -- -v 1.1.0
35+
RUN wget -O- https://github.com/cozystack/cozypkg/raw/refs/heads/main/hack/install.sh | sh -s -- -v 1.2.0
3636

3737
RUN apk add --no-cache make kubectl coreutils git jq
3838

‎packages/extra/monitoring/templates/alerta/alerta.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -232,6 +232,11 @@ kind: VMAlertmanager
232232
metadata:
233233
name: alertmanager
234234
spec:
235+
managedMetadata:
236+
labels:
237+
apps.cozystack.io/application.group: apps.cozystack.io
238+
apps.cozystack.io/application.kind: Monitoring
239+
apps.cozystack.io/application.name: {{ $.Release.Name }}
235240
replicaCount: 3
236241
configSecret: alertmanager
237242
podMetadata:

‎packages/extra/monitoring/templates/vlogs/vlogs.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,11 @@ kind: VLogs
44
metadata:
55
name: {{ .name }}
66
spec:
7+
managedMetadata:
8+
labels:
9+
apps.cozystack.io/application.group: apps.cozystack.io
10+
apps.cozystack.io/application.kind: Monitoring
11+
apps.cozystack.io/application.name: {{ $.Release.Name }}
712
image:
813
tag: v1.17.0-victorialogs
914
storage:

‎packages/extra/monitoring/templates/vm/vmalert.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,11 @@ kind: VMAlert
55
metadata:
66
name: vmalert-{{ .name }}
77
spec:
8+
managedMetadata:
9+
labels:
10+
apps.cozystack.io/application.group: apps.cozystack.io
11+
apps.cozystack.io/application.kind: Monitoring
12+
apps.cozystack.io/application.name: {{ $.Release.Name }}
813
datasource:
914
url: http://vmselect-{{ .name }}.{{ $.Release.Namespace }}.svc:8481/select/0/prometheus
1015
evaluationInterval: 15s

‎packages/extra/monitoring/templates/vm/vmcluster.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,11 @@ kind: VMCluster
55
metadata:
66
name: {{ .name }}
77
spec:
8+
managedMetadata:
9+
labels:
10+
apps.cozystack.io/application.group: apps.cozystack.io
11+
apps.cozystack.io/application.kind: Monitoring
12+
apps.cozystack.io/application.name: {{ $.Release.Name }}
813
replicationFactor: 2
914
retentionPeriod: {{ .retentionPeriod | quote }}
1015
vminsert:

‎packages/system/cozystack-controller/templates/mutatingwebhookconfiguration.yaml‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -20,14 +20,14 @@ webhooks:
2020
apiGroups: [""]
2121
apiVersions: ["v1"]
2222
resources: ["pods","secrets", "services", "persistentvolumeclaims"]
23-
- operations: ["CREATE", "UPDATE"]
24-
apiGroups: ["cozystack.io"]
25-
apiVersions: ["v1alpha1"]
26-
resources: ["workloadmonitors"]
2723
- operations: ["CREATE", "UPDATE"]
2824
apiGroups: ["networking.k8s.io"]
2925
apiVersions: ["v1"]
3026
resources: ["ingresses"]
27+
- operations: ["CREATE", "UPDATE"]
28+
apiGroups: ["cozystack.io"]
29+
apiVersions: ["v1alpha1"]
30+
resources: ["workloadmonitors"]
3131
failurePolicy: Fail
3232
namespaceSelector:
3333
matchExpressions:

‎scripts/migrations/10‎

100644100755
File mode changed.

‎scripts/migrations/15‎

100644100755
File mode changed.

‎scripts/migrations/20‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,24 @@
44
set -euo pipefail
55

66
kubectl delete ingresses.networking.k8s.io --all -n cozy-dashboard --ignore-not-found
7+
kubectl delete cozystackresourcedefinitions --all-namespaces --all --ignore-not-found
78
kubectl delete crd cozystackresourcedefinitions.cozystack.io --ignore-not-found
89

10+
timestamp=$(date --rfc-3339=ns)
11+
# Make sure webhook is upgraded first, then run migration
12+
cozypkg -C packages/system/cozystack-controller -n cozy-system reconcile cozystack-controller --force --with-source
13+
kubectl wait hr/cozystack-controller -n cozy-system --timeout=4m --for=condition=ready || exit 1
14+
cozypkg -C packages/system/cozystack-api -n cozy-system reconcile cozystack-api --force --with-source
15+
kubectl wait hr/cozystack-api -n cozy-system --timeout=4m --for=condition=ready || exit 1
16+
kubectl get namespace -o custom-columns=NAME:.metadata.name --no-headers |
17+
grep '^tenant-' |
18+
while read namespace ; do
19+
(set -x; \
20+
kubectl annotate \
21+
pods,services,pvc,secrets,ingresses.networking.k8s.io,workloadmonitors.cozystack.io \
22+
-n "$namespace" --all \
23+
migration.cozystack.io="$timestamp" --overwrite)
24+
done
925
# Stamp version
1026
kubectl create configmap -n cozy-system cozystack-version \
1127
--from-literal=version=21 --dry-run=client -o yaml | kubectl apply -f-

0 commit comments

Comments
 (0)