Skip to content

Commit 218384e

Browse files
committed
feat(keycloak): let Keycloak and its database proxy spread across zones
Neither the Keycloak StatefulSet nor the keycloak-kms-proxy Deployment exposed any placement, so the scheduler was free to put both replicas of each in one failure domain. For Keycloak that is worse than it looks: a StatefulSet does not replace a pod stuck on an unreachable node, so losing that domain takes every replica for as long as it stays lost. Expose affinity and topologySpreadConstraints for both, empty by default so nothing changes unless an installation opts in, and give the proxy a PodDisruptionBudget once it runs more than one replica. Assisted-By: LLM Signed-off-by: Kirill Ilin <[email protected]>
1 parent f6a615f commit 218384e

4 files changed

Lines changed: 181 additions & 0 deletions

File tree

‎packages/system/keycloak/templates/proxy.yaml‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -126,6 +126,14 @@ spec:
126126
app: keycloak-kms-proxy
127127
spec:
128128
serviceAccountName: keycloak-kms-proxy
129+
{{- with .Values.encryption.affinity }}
130+
affinity:
131+
{{- toYaml . | nindent 8 }}
132+
{{- end }}
133+
{{- with .Values.encryption.topologySpreadConstraints }}
134+
topologySpreadConstraints:
135+
{{- toYaml . | nindent 8 }}
136+
{{- end }}
129137
automountServiceAccountToken: {{ $k8sAuth }}
130138
securityContext:
131139
runAsNonRoot: true
@@ -303,6 +311,20 @@ spec:
303311
path: ca.crt
304312
{{- end }}
305313
{{- end }}
314+
{{- if gt $replicas 1 }}
315+
---
316+
apiVersion: policy/v1
317+
kind: PodDisruptionBudget
318+
metadata:
319+
name: keycloak-kms-proxy
320+
labels:
321+
app: keycloak-kms-proxy
322+
spec:
323+
maxUnavailable: 1
324+
selector:
325+
matchLabels:
326+
app: keycloak-kms-proxy
327+
{{- end }}
306328
---
307329
apiVersion: v1
308330
kind: Service

‎packages/system/keycloak/templates/sts.yaml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,14 @@ spec:
6262
imagePullSecrets:
6363
{{- toYaml . | nindent 8 }}
6464
{{- end }}
65+
{{- with .Values.affinity }}
66+
affinity:
67+
{{- toYaml . | nindent 8 }}
68+
{{- end }}
69+
{{- with .Values.topologySpreadConstraints }}
70+
topologySpreadConstraints:
71+
{{- toYaml . | nindent 8 }}
72+
{{- end }}
6573
securityContext:
6674
fsGroup: 1000
6775
{{- if $themes }}
Lines changed: 127 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,127 @@
1+
suite: keycloak and proxy placement across failure domains
2+
templates:
3+
- templates/sts.yaml
4+
- templates/proxy.yaml
5+
6+
release:
7+
name: keycloak
8+
namespace: cozy-keycloak
9+
10+
set:
11+
_cluster:
12+
root-host: example.org
13+
cluster-domain: cozy.local
14+
15+
tests:
16+
- it: leaves Keycloak placement to the scheduler by default
17+
template: templates/sts.yaml
18+
documentSelector:
19+
path: kind
20+
value: StatefulSet
21+
asserts:
22+
- notExists:
23+
path: spec.template.spec.affinity
24+
- notExists:
25+
path: spec.template.spec.topologySpreadConstraints
26+
27+
- it: renders Keycloak affinity and spread constraints when set
28+
template: templates/sts.yaml
29+
documentSelector:
30+
path: kind
31+
value: StatefulSet
32+
set:
33+
affinity:
34+
nodeAffinity:
35+
requiredDuringSchedulingIgnoredDuringExecution:
36+
nodeSelectorTerms:
37+
- matchExpressions:
38+
- key: topology.kubernetes.io/zone
39+
operator: In
40+
values: [zone-a, zone-b]
41+
topologySpreadConstraints:
42+
- maxSkew: 1
43+
topologyKey: topology.kubernetes.io/zone
44+
whenUnsatisfiable: DoNotSchedule
45+
nodeTaintsPolicy: Honor
46+
labelSelector:
47+
matchLabels:
48+
app: keycloak-ha
49+
asserts:
50+
- equal:
51+
path: spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions[0].values
52+
value: [zone-a, zone-b]
53+
- equal:
54+
path: spec.template.spec.topologySpreadConstraints[0].nodeTaintsPolicy
55+
value: Honor
56+
57+
- it: leaves proxy placement to the scheduler by default
58+
template: templates/proxy.yaml
59+
documentSelector:
60+
path: kind
61+
value: Deployment
62+
set:
63+
encryption.enabled: true
64+
asserts:
65+
- notExists:
66+
path: spec.template.spec.affinity
67+
- notExists:
68+
path: spec.template.spec.topologySpreadConstraints
69+
70+
- it: renders proxy affinity and spread constraints when set
71+
template: templates/proxy.yaml
72+
documentSelector:
73+
path: kind
74+
value: Deployment
75+
set:
76+
encryption.enabled: true
77+
encryption.affinity:
78+
nodeAffinity:
79+
requiredDuringSchedulingIgnoredDuringExecution:
80+
nodeSelectorTerms:
81+
- matchExpressions:
82+
- key: topology.kubernetes.io/zone
83+
operator: In
84+
values: [zone-a, zone-b]
85+
encryption.topologySpreadConstraints:
86+
- maxSkew: 1
87+
topologyKey: topology.kubernetes.io/zone
88+
whenUnsatisfiable: DoNotSchedule
89+
nodeTaintsPolicy: Honor
90+
labelSelector:
91+
matchLabels:
92+
app: keycloak-kms-proxy
93+
asserts:
94+
- equal:
95+
path: spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions[0].values
96+
value: [zone-a, zone-b]
97+
- equal:
98+
path: spec.template.spec.topologySpreadConstraints[0].whenUnsatisfiable
99+
value: DoNotSchedule
100+
101+
# A single replica has nothing to keep available during a drain, and a
102+
# PodDisruptionBudget on it would only block the drain.
103+
- it: renders no disruption budget for a single proxy replica
104+
template: templates/proxy.yaml
105+
set:
106+
encryption.enabled: true
107+
asserts:
108+
- hasDocuments:
109+
count: 4
110+
111+
- it: keeps one proxy replica through voluntary disruptions when replicated
112+
template: templates/proxy.yaml
113+
documentSelector:
114+
path: kind
115+
value: PodDisruptionBudget
116+
set:
117+
encryption.enabled: true
118+
encryption.replicas: 2
119+
encryption.deksetSecretName: keycloak-kms-proxy-dekset
120+
asserts:
121+
- equal:
122+
path: spec.maxUnavailable
123+
value: 1
124+
- equal:
125+
path: spec.selector.matchLabels
126+
value:
127+
app: keycloak-kms-proxy

‎packages/system/keycloak/values.yaml‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,25 @@ themes: []
6666
# arrives via _cluster.branding, so no per-package values plumbing is required.
6767

6868
imagePullSecrets: []
69+
70+
# Placement of the Keycloak pods. Empty leaves it to the scheduler, which is
71+
# free to put both replicas in one failure domain; a StatefulSet does not
72+
# replace a pod stuck on an unreachable node, so that domain's loss then takes
73+
# every replica with it. Example spreading one replica per zone:
74+
#
75+
# topologySpreadConstraints:
76+
# - maxSkew: 1
77+
# topologyKey: topology.kubernetes.io/zone
78+
# whenUnsatisfiable: DoNotSchedule
79+
# # Leave out nodes the pod cannot run on (an unreachable zone carries
80+
# # the unreachable NoSchedule taint), so the constraint does not count
81+
# # a lost zone as a domain the replacement pod must balance against.
82+
# nodeTaintsPolicy: Honor
83+
# labelSelector:
84+
# matchLabels:
85+
# app: keycloak-ha
86+
affinity: {}
87+
topologySpreadConstraints: []
6988
# - name: my-registry-secret
7089

7190
# Extra environment variables to inject into the Keycloak container.
@@ -123,6 +142,11 @@ encryption:
123142
# Number of proxy replicas. >1 requires deksetSecretName (a shared wrapped
124143
# DEK set) so every replica encrypts deterministic columns under one key.
125144
replicas: 1
145+
# Placement of the proxy pods, same shape as the top-level affinity and
146+
# topologySpreadConstraints (the label is app: keycloak-kms-proxy). With more
147+
# than one replica a PodDisruptionBudget keeps one of them up through drains.
148+
affinity: {}
149+
topologySpreadConstraints: []
126150
# Existing Secret holding the wrapped DEK set (dekset.json) minted by the
127151
# proxy's backfill tool (`cmd/backfill generate-dekset`). REQUIRED to migrate
128152
# an existing populated database — the proxy must reuse the DEK the backfill

0 commit comments

Comments
 (0)