Skip to content

Commit ac2a1d3

Browse files
committed
fix(monitoring): spread cluster and Alertmanager replicas over nodes
vm-operator gives the VMCluster, VLCluster, VTCluster and VMAlertmanager pods exactly the affinity their resource carries, and the chart set none. The scheduler was free to start both replicas of a tier on one node. For vmstorage that leaves both copies replicationFactor 2 keeps of each series on one node, and on a node-local class such as local the volumes hold them there, so losing that node loses both copies. Every tier and the Alertmanager now prefer nodes that run no replica of the same component and resource. It stays a preference, as it does for etcd, because a storage pod bound to a node-local volume cannot leave its node. A required rule would keep it Pending on every install whose replicas already share a node, and on clusters with fewer nodes than replicas. The preference only shapes new placements. A replica bound to a node-local volume next to its peer keeps that node through the rolling restart the upgrade causes. Assisted-by: LLM Signed-off-by: Yan Bondarenko <[email protected]>
1 parent b47fed7 commit ac2a1d3

6 files changed

Lines changed: 196 additions & 0 deletions

File tree

‎packages/system/monitoring/templates/_helpers.tpl‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -336,3 +336,29 @@
336336
{{- define "monitoring.alertmanager.replicas" -}}
337337
3
338338
{{- end -}}
339+
340+
{{- /*
341+
Spreads the replicas of one component over nodes. The operator copies
342+
spec.affinity onto the pods as written and sets none of its own, so without
343+
this the scheduler may put both vmstorage replicas on one node, and with
344+
replicationFactor 2 both copies of each series are then lost with that node.
345+
It is a preference because the storage tiers keep a volume each, and on a
346+
node-local class the volume holds its pod on the node where it was first
347+
scheduled. A required rule would leave a replica Pending for good on an
348+
install whose replicas already share a node, and on any cluster with fewer
349+
nodes than replicas. For the same reason a storage pod already bound to a
350+
node-local volume stays on that node, so the preference spreads new volumes
351+
only. Takes the component's app.kubernetes.io/name and the name of its
352+
resource, which the operator stamps as app.kubernetes.io/instance.
353+
*/ -}}
354+
{{- define "monitoring.spreadOverNodes" -}}
355+
podAntiAffinity:
356+
preferredDuringSchedulingIgnoredDuringExecution:
357+
- weight: 100
358+
podAffinityTerm:
359+
topologyKey: kubernetes.io/hostname
360+
labelSelector:
361+
matchLabels:
362+
app.kubernetes.io/name: {{ index . 0 | quote }}
363+
app.kubernetes.io/instance: {{ index . 1 | quote }}
364+
{{- end -}}

‎packages/system/monitoring/templates/alerta/alerta.yaml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -420,6 +420,7 @@ spec:
420420
apps.cozystack.io/application.kind: Monitoring
421421
apps.cozystack.io/application.name: {{ $.Release.Name }}
422422
replicaCount: {{ include "monitoring.alertmanager.replicas" $ }}
423+
affinity: {{- include "monitoring.spreadOverNodes" (list "vmalertmanager" "alertmanager") | nindent 4 }}
423424
configSecret: alertmanager
424425
{{- if and $emailEnabled $email.authUsername }}
425426
secrets:

‎packages/system/monitoring/templates/vlogs/vlogs.yaml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,13 +23,16 @@ spec:
2323
vlinsert:
2424
replicaCount: 2
2525
resources: {}
26+
affinity: {{- include "monitoring.spreadOverNodes" (list "vlinsert" .name) | nindent 6 }}
2627
vlselect:
2728
replicaCount: 2
2829
resources: {}
30+
affinity: {{- include "monitoring.spreadOverNodes" (list "vlselect" .name) | nindent 6 }}
2931
vlstorage:
3032
retentionPeriod: {{ .retentionPeriod | quote }}
3133
replicaCount: 2
3234
resources: {}
35+
affinity: {{- include "monitoring.spreadOverNodes" (list "vlstorage" .name) | nindent 6 }}
3336
storage:
3437
volumeClaimTemplate:
3538
metadata:

‎packages/system/monitoring/templates/vm/vmcluster.yaml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,9 +19,11 @@ spec:
1919
maxLabelsPerTimeseries: "60"
2020
replicaCount: 2
2121
resources: {}
22+
affinity: {{- include "monitoring.spreadOverNodes" (list "vminsert" .name) | nindent 6 }}
2223
vmselect:
2324
replicaCount: 2
2425
resources: {}
26+
affinity: {{- include "monitoring.spreadOverNodes" (list "vmselect" .name) | nindent 6 }}
2527
extraArgs:
2628
search.maxUniqueTimeseries: "600000"
2729
vmalert.proxyURL: http://vmalert-{{ .name }}.{{ $.Release.Namespace }}.svc:8080
@@ -47,6 +49,7 @@ spec:
4749
vmstorage:
4850
replicaCount: 2
4951
resources: {}
52+
affinity: {{- include "monitoring.spreadOverNodes" (list "vmstorage" .name) | nindent 6 }}
5053
storage:
5154
volumeClaimTemplate:
5255
metadata:

‎packages/system/monitoring/templates/vtraces/vtraces.yaml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,9 +72,11 @@ spec:
7272
insert:
7373
replicaCount: 2
7474
resources: {}
75+
affinity: {{- include "monitoring.spreadOverNodes" (list "vtinsert" .name) | nindent 6 }}
7576
select:
7677
replicaCount: 2
7778
resources: {}
79+
affinity: {{- include "monitoring.spreadOverNodes" (list "vtselect" .name) | nindent 6 }}
7880
storage:
7981
retentionPeriod: {{ .retentionPeriod | quote }}
8082
{{- with .retentionDiskUsageBytes }}
@@ -85,6 +87,7 @@ spec:
8587
{{- end }}
8688
replicaCount: 2
8789
resources: {}
90+
affinity: {{- include "monitoring.spreadOverNodes" (list "vtstorage" .name) | nindent 6 }}
8891
storage:
8992
volumeClaimTemplate:
9093
metadata:
Lines changed: 160 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,160 @@
1+
suite: VM, VL and VT cluster replicas and Alertmanager spread over nodes
2+
3+
# vm-operator copies spec.affinity onto the pods as written and adds none of
4+
# its own, so every tier below would otherwise be free to start both of its
5+
# replicas on one node. The term has to select the pods of its own component
6+
# and resource, which the operator labels app.kubernetes.io/name=<vm|vl|vt><kind>
7+
# and app.kubernetes.io/instance=<resource name>. A selector that misses them
8+
# spreads nothing. It has to stay a preference, because a storage pod on a
9+
# node-local volume cannot leave its node, and a required term would keep it
10+
# Pending wherever its replicas already share one.
11+
12+
release:
13+
name: monitoring-system
14+
namespace: tenant-root
15+
16+
set:
17+
_namespace:
18+
host: example.org
19+
ingress: tenant-root
20+
_cluster:
21+
expose-ingress: ingress
22+
host: example.org
23+
24+
tests:
25+
- it: spreads every VMCluster tier over nodes with a preference
26+
templates:
27+
- templates/vm/vmcluster.yaml
28+
documentSelector:
29+
path: metadata.name
30+
value: shortterm
31+
asserts:
32+
- equal:
33+
path: spec.vmstorage.affinity
34+
value:
35+
podAntiAffinity:
36+
preferredDuringSchedulingIgnoredDuringExecution:
37+
- weight: 100
38+
podAffinityTerm:
39+
topologyKey: kubernetes.io/hostname
40+
labelSelector:
41+
matchLabels:
42+
app.kubernetes.io/name: vmstorage
43+
app.kubernetes.io/instance: shortterm
44+
- equal:
45+
path: spec.vmselect.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution[0].podAffinityTerm.labelSelector.matchLabels
46+
value:
47+
app.kubernetes.io/name: vmselect
48+
app.kubernetes.io/instance: shortterm
49+
- equal:
50+
path: spec.vminsert.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution[0].podAffinityTerm.labelSelector.matchLabels
51+
value:
52+
app.kubernetes.io/name: vminsert
53+
app.kubernetes.io/instance: shortterm
54+
- notExists:
55+
path: spec.vmstorage.affinity.podAntiAffinity.requiredDuringSchedulingIgnoredDuringExecution
56+
- notExists:
57+
path: spec.vmselect.affinity.podAntiAffinity.requiredDuringSchedulingIgnoredDuringExecution
58+
- notExists:
59+
path: spec.vminsert.affinity.podAntiAffinity.requiredDuringSchedulingIgnoredDuringExecution
60+
61+
# Each metrics storage is its own VMCluster, so its term must count only its
62+
# own pods. Pods of shortterm do not keep longterm replicas apart.
63+
- it: keys each VMCluster's spread to its own resource
64+
templates:
65+
- templates/vm/vmcluster.yaml
66+
documentSelector:
67+
path: metadata.name
68+
value: longterm
69+
asserts:
70+
- equal:
71+
path: spec.vmstorage.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution[0].podAffinityTerm.labelSelector.matchLabels
72+
value:
73+
app.kubernetes.io/name: vmstorage
74+
app.kubernetes.io/instance: longterm
75+
76+
- it: spreads every VLCluster tier over nodes with a preference
77+
templates:
78+
- templates/vlogs/vlogs.yaml
79+
documentSelector:
80+
path: metadata.name
81+
value: generic
82+
asserts:
83+
- equal:
84+
path: spec.vlstorage.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution[0]
85+
value:
86+
weight: 100
87+
podAffinityTerm:
88+
topologyKey: kubernetes.io/hostname
89+
labelSelector:
90+
matchLabels:
91+
app.kubernetes.io/name: vlstorage
92+
app.kubernetes.io/instance: generic
93+
- equal:
94+
path: spec.vlselect.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution[0].podAffinityTerm.labelSelector.matchLabels
95+
value:
96+
app.kubernetes.io/name: vlselect
97+
app.kubernetes.io/instance: generic
98+
- equal:
99+
path: spec.vlinsert.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution[0].podAffinityTerm.labelSelector.matchLabels
100+
value:
101+
app.kubernetes.io/name: vlinsert
102+
app.kubernetes.io/instance: generic
103+
- notExists:
104+
path: spec.vlstorage.affinity.podAntiAffinity.requiredDuringSchedulingIgnoredDuringExecution
105+
106+
- it: spreads every VTCluster tier over nodes with a preference
107+
templates:
108+
- templates/vtraces/vtraces.yaml
109+
set:
110+
tracingStorages:
111+
- name: generic
112+
mode: cluster
113+
retentionPeriod: "14d"
114+
storage: 10Gi
115+
documentSelector:
116+
path: metadata.name
117+
value: generic
118+
asserts:
119+
- equal:
120+
path: spec.storage.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution[0]
121+
value:
122+
weight: 100
123+
podAffinityTerm:
124+
topologyKey: kubernetes.io/hostname
125+
labelSelector:
126+
matchLabels:
127+
app.kubernetes.io/name: vtstorage
128+
app.kubernetes.io/instance: generic
129+
- equal:
130+
path: spec.select.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution[0].podAffinityTerm.labelSelector.matchLabels
131+
value:
132+
app.kubernetes.io/name: vtselect
133+
app.kubernetes.io/instance: generic
134+
- equal:
135+
path: spec.insert.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution[0].podAffinityTerm.labelSelector.matchLabels
136+
value:
137+
app.kubernetes.io/name: vtinsert
138+
app.kubernetes.io/instance: generic
139+
- notExists:
140+
path: spec.storage.affinity.podAntiAffinity.requiredDuringSchedulingIgnoredDuringExecution
141+
142+
- it: spreads the Alertmanager replicas over nodes with a preference
143+
templates:
144+
- templates/alerta/alerta.yaml
145+
documentSelector:
146+
path: kind
147+
value: VMAlertmanager
148+
asserts:
149+
- equal:
150+
path: spec.affinity
151+
value:
152+
podAntiAffinity:
153+
preferredDuringSchedulingIgnoredDuringExecution:
154+
- weight: 100
155+
podAffinityTerm:
156+
topologyKey: kubernetes.io/hostname
157+
labelSelector:
158+
matchLabels:
159+
app.kubernetes.io/name: vmalertmanager
160+
app.kubernetes.io/instance: alertmanager

0 commit comments

Comments
 (0)