Skip to content

Commit 2119b7a

Browse files
committed
fix(redis, valkey, harbor): add the selector to every platform spread constraint
A globalAppTopologySpreadConstraints policy can hold several constraints, one per zone and one per node for example. The selector was added under the last one only, and kube-scheduler counts no pods for a constraint without a selector, so the earlier constraints did not spread these pods at all. A last constraint with a selector of its own got a second labelSelector key, which a strict YAML parser rejects and Helm resolves to the chart's selector, dropping the policy's. Each constraint without a selector now gets the one of its component, and a constraint with its own keeps it, the rule the keycloak chart already applies to its database constraints. The policy is parsed up front, so one that does not parse or holds no non-empty topologySpreadConstraints list fails the render with a message. Before, such a policy failed only because the appended selector broke the YAML, and a map in place of the list rendered without an error. Assisted-by: LLM Signed-off-by: Yan Bondarenko <[email protected]>
1 parent 902051e commit 2119b7a

6 files changed

Lines changed: 430 additions & 39 deletions

File tree

‎packages/apps/redis/templates/redisfailover.yaml‎

Lines changed: 18 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,14 @@
11
{{- /* One image for the data pods and the Sentinels, resolved once so they cannot drift apart. */}}
22
{{- $redisImage := printf "redis:%s" (include "redis.versionMap" $) }}
33
{{- $tlsEnabled := (include "redis.tls.enabled" .) | eq "true" -}}
4-
{{- $spreadPolicy := dig "scheduling" "globalAppTopologySpreadConstraints" "" (.Values._cluster | default dict) -}}
4+
{{- $spreadConstraints := list -}}
5+
{{- with dig "scheduling" "globalAppTopologySpreadConstraints" "" (.Values._cluster | default dict) -}}
6+
{{- $policy := fromYaml . -}}
7+
{{- $spreadConstraints = dig "topologySpreadConstraints" (list) $policy -}}
8+
{{- if or (not (kindIs "slice" $spreadConstraints)) (not $spreadConstraints) -}}
9+
{{- fail (trim (printf "redis: _cluster.scheduling.globalAppTopologySpreadConstraints must be YAML with a non-empty topologySpreadConstraints list. %s" (get $policy "Error"))) -}}
10+
{{- end -}}
11+
{{- end -}}
512
{{- /*
613
A Sentinel holds only its configuration and failover state, so the Redis
714
sizing over-provisions it. An empty sentinel block still falls back to the
@@ -67,12 +74,11 @@ spec:
6774
# on every start, so a chart rollback to an older engine is safe here.
6875
image: {{ $redisImage | quote }}
6976
replicas: 3
70-
{{- if $spreadPolicy }}
71-
{{- $spreadPolicy | fromYaml | toYaml | nindent 4 }}
72-
labelSelector:
73-
matchLabels:
74-
app.kubernetes.io/name: {{ .Release.Name }}
75-
app.kubernetes.io/component: sentinel
77+
{{- if $spreadConstraints }}
78+
topologySpreadConstraints:
79+
{{- range $spreadConstraints }}
80+
- {{- toYaml (ternary . (merge (deepCopy .) (dict "labelSelector" (dict "matchLabels" (dict "app.kubernetes.io/name" $.Release.Name "app.kubernetes.io/component" "sentinel")))) (hasKey . "labelSelector")) | nindent 6 }}
81+
{{- end }}
7682
{{- else }}
7783
topologySpreadConstraints:
7884
- maxSkew: 1
@@ -91,12 +97,11 @@ spec:
9197
image: {{ $redisImage | quote }}
9298
resources: {{- include "cozy-lib.resources.defaultingSanitize" (list .Values.resourcesPreset .Values.resources $) | nindent 6 }}
9399
replicas: {{ .Values.replicas }}
94-
{{- if $spreadPolicy }}
95-
{{- $spreadPolicy | fromYaml | toYaml | nindent 4 }}
96-
labelSelector:
97-
matchLabels:
98-
app.kubernetes.io/name: {{ .Release.Name }}
99-
app.kubernetes.io/component: redis
100+
{{- if $spreadConstraints }}
101+
topologySpreadConstraints:
102+
{{- range $spreadConstraints }}
103+
- {{- toYaml (ternary . (merge (deepCopy .) (dict "labelSelector" (dict "matchLabels" (dict "app.kubernetes.io/name" $.Release.Name "app.kubernetes.io/component" "redis")))) (hasKey . "labelSelector")) | nindent 6 }}
104+
{{- end }}
100105
{{- else }}
101106
topologySpreadConstraints:
102107
- maxSkew: 1

‎packages/apps/redis/tests/topology_spread_test.yaml‎

Lines changed: 150 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -97,3 +97,153 @@ tests:
9797
matchLabels:
9898
app.kubernetes.io/name: my-redis
9999
app.kubernetes.io/component: redis
100+
101+
- it: gives every constraint of a cluster-wide policy the selector
102+
set:
103+
_cluster:
104+
scheduling:
105+
globalAppTopologySpreadConstraints: |
106+
topologySpreadConstraints:
107+
- maxSkew: 1
108+
topologyKey: topology.kubernetes.io/zone
109+
whenUnsatisfiable: DoNotSchedule
110+
- maxSkew: 1
111+
topologyKey: kubernetes.io/hostname
112+
whenUnsatisfiable: ScheduleAnyway
113+
documentSelector:
114+
path: kind
115+
value: RedisFailover
116+
asserts:
117+
- equal:
118+
path: spec.sentinel.topologySpreadConstraints
119+
value:
120+
- maxSkew: 1
121+
topologyKey: topology.kubernetes.io/zone
122+
whenUnsatisfiable: DoNotSchedule
123+
labelSelector:
124+
matchLabels:
125+
app.kubernetes.io/name: my-redis
126+
app.kubernetes.io/component: sentinel
127+
- maxSkew: 1
128+
topologyKey: kubernetes.io/hostname
129+
whenUnsatisfiable: ScheduleAnyway
130+
labelSelector:
131+
matchLabels:
132+
app.kubernetes.io/name: my-redis
133+
app.kubernetes.io/component: sentinel
134+
- equal:
135+
path: spec.redis.topologySpreadConstraints
136+
value:
137+
- maxSkew: 1
138+
topologyKey: topology.kubernetes.io/zone
139+
whenUnsatisfiable: DoNotSchedule
140+
labelSelector:
141+
matchLabels:
142+
app.kubernetes.io/name: my-redis
143+
app.kubernetes.io/component: redis
144+
- maxSkew: 1
145+
topologyKey: kubernetes.io/hostname
146+
whenUnsatisfiable: ScheduleAnyway
147+
labelSelector:
148+
matchLabels:
149+
app.kubernetes.io/name: my-redis
150+
app.kubernetes.io/component: redis
151+
152+
- it: keeps a selector the cluster-wide policy sets itself
153+
set:
154+
_cluster:
155+
scheduling:
156+
globalAppTopologySpreadConstraints: |
157+
topologySpreadConstraints:
158+
- maxSkew: 1
159+
topologyKey: kubernetes.io/hostname
160+
whenUnsatisfiable: ScheduleAnyway
161+
- maxSkew: 1
162+
topologyKey: topology.kubernetes.io/zone
163+
whenUnsatisfiable: ScheduleAnyway
164+
labelSelector:
165+
matchLabels:
166+
example.org/spread: all
167+
documentSelector:
168+
path: kind
169+
value: RedisFailover
170+
asserts:
171+
- equal:
172+
path: spec.sentinel.topologySpreadConstraints
173+
value:
174+
- maxSkew: 1
175+
topologyKey: kubernetes.io/hostname
176+
whenUnsatisfiable: ScheduleAnyway
177+
labelSelector:
178+
matchLabels:
179+
app.kubernetes.io/name: my-redis
180+
app.kubernetes.io/component: sentinel
181+
- maxSkew: 1
182+
topologyKey: topology.kubernetes.io/zone
183+
whenUnsatisfiable: ScheduleAnyway
184+
labelSelector:
185+
matchLabels:
186+
example.org/spread: all
187+
- equal:
188+
path: spec.redis.topologySpreadConstraints
189+
value:
190+
- maxSkew: 1
191+
topologyKey: kubernetes.io/hostname
192+
whenUnsatisfiable: ScheduleAnyway
193+
labelSelector:
194+
matchLabels:
195+
app.kubernetes.io/name: my-redis
196+
app.kubernetes.io/component: redis
197+
- maxSkew: 1
198+
topologyKey: topology.kubernetes.io/zone
199+
whenUnsatisfiable: ScheduleAnyway
200+
labelSelector:
201+
matchLabels:
202+
example.org/spread: all
203+
204+
- it: fails on a cluster-wide policy that does not parse
205+
set:
206+
_cluster:
207+
scheduling:
208+
globalAppTopologySpreadConstraints: |
209+
topologySpreadConstraints:
210+
- maxSkew: 1
211+
topologyKey: kubernetes.io/hostname
212+
asserts:
213+
- failedTemplate:
214+
errorPattern: "globalAppTopologySpreadConstraints must be YAML with a non-empty topologySpreadConstraints list\\. error converting YAML to JSON"
215+
216+
- it: fails on a cluster-wide policy without a topologySpreadConstraints key
217+
set:
218+
_cluster:
219+
scheduling:
220+
globalAppTopologySpreadConstraints: |
221+
- maxSkew: 1
222+
topologyKey: kubernetes.io/hostname
223+
whenUnsatisfiable: ScheduleAnyway
224+
asserts:
225+
- failedTemplate:
226+
errorPattern: "globalAppTopologySpreadConstraints must be YAML with a non-empty topologySpreadConstraints list"
227+
228+
- it: fails on a cluster-wide policy with an empty topologySpreadConstraints list
229+
set:
230+
_cluster:
231+
scheduling:
232+
globalAppTopologySpreadConstraints: |
233+
topologySpreadConstraints: []
234+
asserts:
235+
- failedTemplate:
236+
errorMessage: "redis: _cluster.scheduling.globalAppTopologySpreadConstraints must be YAML with a non-empty topologySpreadConstraints list."
237+
238+
- it: fails on a cluster-wide policy whose topologySpreadConstraints is not a list
239+
set:
240+
_cluster:
241+
scheduling:
242+
globalAppTopologySpreadConstraints: |
243+
topologySpreadConstraints:
244+
maxSkew: 1
245+
topologyKey: kubernetes.io/hostname
246+
whenUnsatisfiable: ScheduleAnyway
247+
asserts:
248+
- failedTemplate:
249+
errorMessage: "redis: _cluster.scheduling.globalAppTopologySpreadConstraints must be YAML with a non-empty topologySpreadConstraints list."

‎packages/apps/valkey/templates/valkeyfailover.yaml‎

Lines changed: 18 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,11 @@
1-
{{- $spreadPolicy := dig "scheduling" "globalAppTopologySpreadConstraints" "" (.Values._cluster | default dict) }}
1+
{{- $spreadConstraints := list }}
2+
{{- with dig "scheduling" "globalAppTopologySpreadConstraints" "" (.Values._cluster | default dict) }}
3+
{{- $policy := fromYaml . }}
4+
{{- $spreadConstraints = dig "topologySpreadConstraints" (list) $policy }}
5+
{{- if or (not (kindIs "slice" $spreadConstraints)) (not $spreadConstraints) }}
6+
{{- fail (trim (printf "valkey: _cluster.scheduling.globalAppTopologySpreadConstraints must be YAML with a non-empty topologySpreadConstraints list. %s" (get $policy "Error"))) }}
7+
{{- end }}
8+
{{- end }}
29
{{- if .Values.authEnabled }}
310
{{- $existingPassword := lookup "v1" "Secret" .Release.Namespace (printf "%s-auth" .Release.Name) }}
411
{{- $password := randAlphaNum 32 | b64enc }}
@@ -32,12 +39,11 @@ spec:
3239
engine: Valkey
3340
sentinel:
3441
replicas: 3
35-
{{- if $spreadPolicy }}
36-
{{- $spreadPolicy | fromYaml | toYaml | nindent 4 }}
37-
labelSelector:
38-
matchLabels:
39-
app.kubernetes.io/name: {{ .Release.Name }}
40-
app.kubernetes.io/component: sentinel
42+
{{- if $spreadConstraints }}
43+
topologySpreadConstraints:
44+
{{- range $spreadConstraints }}
45+
- {{- toYaml (ternary . (merge (deepCopy .) (dict "labelSelector" (dict "matchLabels" (dict "app.kubernetes.io/name" $.Release.Name "app.kubernetes.io/component" "sentinel")))) (hasKey . "labelSelector")) | nindent 6 }}
46+
{{- end }}
4147
{{- else }}
4248
topologySpreadConstraints:
4349
- maxSkew: 1
@@ -56,12 +62,11 @@ spec:
5662
image: "valkey/valkey:{{ include "valkey.versionMap" $ }}"
5763
resources: {{- include "cozy-lib.resources.defaultingSanitize" (list .Values.resourcesPreset .Values.resources $) | nindent 6 }}
5864
replicas: {{ .Values.replicas }}
59-
{{- if $spreadPolicy }}
60-
{{- $spreadPolicy | fromYaml | toYaml | nindent 4 }}
61-
labelSelector:
62-
matchLabels:
63-
app.kubernetes.io/name: {{ .Release.Name }}
64-
app.kubernetes.io/component: redis
65+
{{- if $spreadConstraints }}
66+
topologySpreadConstraints:
67+
{{- range $spreadConstraints }}
68+
- {{- toYaml (ternary . (merge (deepCopy .) (dict "labelSelector" (dict "matchLabels" (dict "app.kubernetes.io/name" $.Release.Name "app.kubernetes.io/component" "redis")))) (hasKey . "labelSelector")) | nindent 6 }}
69+
{{- end }}
6570
{{- else }}
6671
topologySpreadConstraints:
6772
- maxSkew: 1

‎packages/apps/valkey/tests/topology_spread_test.yaml‎

Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -78,3 +78,116 @@ tests:
7878
matchLabels:
7979
app.kubernetes.io/name: my-valkey
8080
app.kubernetes.io/component: redis
81+
82+
- it: gives every constraint of a cluster-wide policy the selector
83+
set:
84+
_cluster:
85+
scheduling:
86+
globalAppTopologySpreadConstraints: |
87+
topologySpreadConstraints:
88+
- maxSkew: 1
89+
topologyKey: topology.kubernetes.io/zone
90+
whenUnsatisfiable: DoNotSchedule
91+
- maxSkew: 1
92+
topologyKey: kubernetes.io/hostname
93+
whenUnsatisfiable: ScheduleAnyway
94+
documentSelector:
95+
path: kind
96+
value: RedisFailover
97+
asserts:
98+
- equal:
99+
path: spec.sentinel.topologySpreadConstraints
100+
value:
101+
- maxSkew: 1
102+
topologyKey: topology.kubernetes.io/zone
103+
whenUnsatisfiable: DoNotSchedule
104+
labelSelector:
105+
matchLabels:
106+
app.kubernetes.io/name: my-valkey
107+
app.kubernetes.io/component: sentinel
108+
- maxSkew: 1
109+
topologyKey: kubernetes.io/hostname
110+
whenUnsatisfiable: ScheduleAnyway
111+
labelSelector:
112+
matchLabels:
113+
app.kubernetes.io/name: my-valkey
114+
app.kubernetes.io/component: sentinel
115+
- equal:
116+
path: spec.redis.topologySpreadConstraints
117+
value:
118+
- maxSkew: 1
119+
topologyKey: topology.kubernetes.io/zone
120+
whenUnsatisfiable: DoNotSchedule
121+
labelSelector:
122+
matchLabels:
123+
app.kubernetes.io/name: my-valkey
124+
app.kubernetes.io/component: redis
125+
- maxSkew: 1
126+
topologyKey: kubernetes.io/hostname
127+
whenUnsatisfiable: ScheduleAnyway
128+
labelSelector:
129+
matchLabels:
130+
app.kubernetes.io/name: my-valkey
131+
app.kubernetes.io/component: redis
132+
133+
- it: keeps a selector the cluster-wide policy sets itself
134+
set:
135+
_cluster:
136+
scheduling:
137+
globalAppTopologySpreadConstraints: |
138+
topologySpreadConstraints:
139+
- maxSkew: 1
140+
topologyKey: kubernetes.io/hostname
141+
whenUnsatisfiable: ScheduleAnyway
142+
- maxSkew: 1
143+
topologyKey: topology.kubernetes.io/zone
144+
whenUnsatisfiable: ScheduleAnyway
145+
labelSelector:
146+
matchLabels:
147+
example.org/spread: all
148+
documentSelector:
149+
path: kind
150+
value: RedisFailover
151+
asserts:
152+
- equal:
153+
path: spec.sentinel.topologySpreadConstraints
154+
value:
155+
- maxSkew: 1
156+
topologyKey: kubernetes.io/hostname
157+
whenUnsatisfiable: ScheduleAnyway
158+
labelSelector:
159+
matchLabels:
160+
app.kubernetes.io/name: my-valkey
161+
app.kubernetes.io/component: sentinel
162+
- maxSkew: 1
163+
topologyKey: topology.kubernetes.io/zone
164+
whenUnsatisfiable: ScheduleAnyway
165+
labelSelector:
166+
matchLabels:
167+
example.org/spread: all
168+
- equal:
169+
path: spec.redis.topologySpreadConstraints
170+
value:
171+
- maxSkew: 1
172+
topologyKey: kubernetes.io/hostname
173+
whenUnsatisfiable: ScheduleAnyway
174+
labelSelector:
175+
matchLabels:
176+
app.kubernetes.io/name: my-valkey
177+
app.kubernetes.io/component: redis
178+
- maxSkew: 1
179+
topologyKey: topology.kubernetes.io/zone
180+
whenUnsatisfiable: ScheduleAnyway
181+
labelSelector:
182+
matchLabels:
183+
example.org/spread: all
184+
185+
- it: fails on a cluster-wide policy with an empty topologySpreadConstraints list
186+
set:
187+
_cluster:
188+
scheduling:
189+
globalAppTopologySpreadConstraints: |
190+
topologySpreadConstraints: []
191+
asserts:
192+
- failedTemplate:
193+
errorMessage: "valkey: _cluster.scheduling.globalAppTopologySpreadConstraints must be YAML with a non-empty topologySpreadConstraints list."

0 commit comments

Comments
 (0)