Skip to content

Commit 02dd8e5

Browse files
fix(kubernetes): move the OIDC users out of the bootstrap Job's script
The bootstrap Job wrote a ClusterRoleBinding per oidc.users entry into its script, and the script reaches the container as one `sh -c` argument. Linux caps a single argument at 128 KiB, which about 240 users cross at roughly 530 bytes each. The container then cannot start, the post-install and post-upgrade hook fails, and the tenant cluster's HelmRelease stays unready. The values schema puts no bound on oidc.users, so a shared cluster reaches that size with ordinary per-person entries. The bindings and the list of their names now reach the Job through a ConfigMap it mounts, so the script is the same size for any list. Names, labels and roles of the bindings are unchanged, so the bindings already in a tenant cluster are updated in place and none is orphaned. Assisted-by: LLM Signed-off-by: Myasnikov Daniil <[email protected]> (cherry picked from commit 7b7444f) Signed-off-by: cozystack-ci[bot] <274107086+cozystack-ci[bot]@users.noreply.github.com>
1 parent e591ba6 commit 02dd8e5

4 files changed

Lines changed: 436 additions & 37 deletions

File tree

‎hack/kubernetes-oidc-bindings.bats‎

Lines changed: 187 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,187 @@
1+
#!/usr/bin/env bats
2+
# -----------------------------------------------------------------------------
3+
# packages/apps/kubernetes: the OIDC bootstrap Job must not carry the users[]
4+
# list inside its container command.
5+
#
6+
# The Job's script reaches the container as one `sh -c` argument, and Linux caps
7+
# one exec argument at 128 KiB (MAX_ARG_STRLEN). With a ClusterRoleBinding per
8+
# user written into that script, about 240 users take it past the cap: the
9+
# container cannot start, the post-install/post-upgrade hook fails, and the
10+
# tenant cluster's HelmRelease stays unready. The bindings reach the Job through
11+
# a mounted ConfigMap instead, so the script is the same size for any list.
12+
#
13+
# The last three tests execute the rendered script against a fake kubectl, since
14+
# no assertion on the script's text says what the prune step does with the list:
15+
# a condition forced either way passes every helm-unittest case.
16+
#
17+
# Requires: helm, yq (mikefarah v4+), jq, like hack/admin-kubeconfig-invariant.bats.
18+
# -----------------------------------------------------------------------------
19+
20+
load test_helper
21+
22+
USERS=300
23+
24+
# render_users <dir> <count> renders the chart in System mode with <count>
25+
# generated users and leaves one JSON document per object in <dir>/rendered.json.
26+
render_users() {
27+
jq -n --argjson n "$2" '
28+
{oidc: {users: [range($n) | {email: "user\(.)@example.org", role: (if . % 7 == 0 then "admin" else "view" end)}]}}' \
29+
> "$1/users.yaml"
30+
helm template bindings packages/apps/kubernetes \
31+
--namespace tenant-test \
32+
--values packages/apps/kubernetes/tests/values/oidc-system.yaml \
33+
--values "$1/users.yaml" \
34+
> "$1/rendered.yaml"
35+
yq --output-format=json eval-all '.' "$1/rendered.yaml" > "$1/rendered.json"
36+
}
37+
38+
# render_job <dir> <users JSON> renders the chart in CustomConfig mode and unpacks
39+
# what the bootstrap Job runs and mounts into <dir>: script.sh is its `sh -c`
40+
# script, bindings/ holds the two files of the mounted ConfigMap. -j keeps an
41+
# empty value an empty file, as the kubelet writes it.
42+
render_job() {
43+
jq -n --argjson users "$2" '
44+
{_cluster: {"oidc-enabled": "false"},
45+
oidc: {mode: "CustomConfig",
46+
customConfig: {config: "apiVersion: apiserver.config.k8s.io/v1beta1\nkind: AuthenticationConfiguration\njwt: []\n"},
47+
users: $users}}' > "$1/values.yaml"
48+
helm template bindings packages/apps/kubernetes \
49+
--namespace tenant-test \
50+
--values packages/apps/kubernetes/tests/values/oidc-system.yaml \
51+
--values "$1/values.yaml" \
52+
> "$1/rendered.yaml"
53+
yq --output-format=json eval-all '.' "$1/rendered.yaml" > "$1/rendered.json"
54+
jq -s -j '.[] | select(.kind == "Job" and .metadata.name == "bindings-oidc-bootstrap")
55+
| .spec.template.spec.containers[0].command[2]' "$1/rendered.json" > "$1/script.sh"
56+
mkdir -p "$1/bindings"
57+
jq -s -j '.[] | select(.kind == "ConfigMap" and .metadata.name == "bindings-oidc-bindings")
58+
| .data["crbs.yaml"]' "$1/rendered.json" > "$1/bindings/crbs.yaml"
59+
jq -s -j '.[] | select(.kind == "ConfigMap" and .metadata.name == "bindings-oidc-bindings")
60+
| .data["desired.txt"]' "$1/rendered.json" > "$1/bindings/desired.txt"
61+
}
62+
63+
# fake_kubectl <dir> <names> puts a kubectl first on PATH that logs every call and
64+
# answers the script's list of existing bindings with <names>.
65+
fake_kubectl() {
66+
mkdir -p "$1/bin"
67+
printf '%s' "$2" > "$1/existing"
68+
cat > "$1/bin/kubectl" <<'KEOF'
69+
#!/bin/sh
70+
echo "$*" >> "$KLOG"
71+
case "$*" in
72+
*"get clusterrolebindings"*) cat "$EXISTING" ;;
73+
esac
74+
KEOF
75+
chmod +x "$1/bin/kubectl"
76+
}
77+
78+
# run_job <dir> runs the unpacked script as the container does and leaves its
79+
# status in $rc and its output in <dir>/out. Run by bash, which has the pipefail
80+
# the script sets and which is what the unit lane runs under; the container's own
81+
# shell is not exercised.
82+
run_job() {
83+
: > "$1/kubectl.log"
84+
echo kubeconfig > "$1/kubeconfig"
85+
rc=0
86+
env PATH="$1/bin:$PATH" KLOG="$1/kubectl.log" EXISTING="$1/existing" HOME="$1" \
87+
KUBECONFIG_TENANT="$1/kubeconfig" RELEASE=bindings RELEASE_NAMESPACE=tenant-test \
88+
OIDC_MODE=CustomConfig OIDC_BINDINGS_DIR="$1/bindings" \
89+
bash "$1/script.sh" > "$1/out" 2>&1 || rc=$?
90+
}
91+
92+
@test "the OIDC bootstrap Job script stays under the exec argument cap and carries no user" {
93+
dir=$(mktemp -d)
94+
render_users "$dir" "$USERS"
95+
jq -s -r '.[] | select(.kind == "Job" and .metadata.name == "bindings-oidc-bootstrap")
96+
| .spec.template.spec.containers[0].command[2]' "$dir/rendered.json" > "$dir/script"
97+
size=$(wc -c < "$dir/script")
98+
[ "$size" -gt 0 ]
99+
[ "$size" -lt 131072 ]
100+
if grep -q -E 'user[0-9]+@example\.org' "$dir/script"; then
101+
echo "the Job script still carries users[] entries" >&2
102+
false
103+
fi
104+
rm -rf "$dir"
105+
}
106+
107+
@test "every user keeps one ClusterRoleBinding with its old name and role in the bindings ConfigMap" {
108+
dir=$(mktemp -d)
109+
render_users "$dir" "$USERS"
110+
jq -s -r '.[] | select(.kind == "ConfigMap" and .metadata.name == "bindings-oidc-bindings")
111+
| .data["crbs.yaml"]' "$dir/rendered.json" > "$dir/crbs.yaml"
112+
jq -s -r '.[] | select(.kind == "ConfigMap" and .metadata.name == "bindings-oidc-bindings")
113+
| .data["desired.txt"]' "$dir/rendered.json" > "$dir/desired.txt"
114+
[ -s "$dir/crbs.yaml" ]
115+
yq --output-format=json eval-all '.' "$dir/crbs.yaml" | jq -s '.' > "$dir/crbs.json"
116+
117+
[ "$(jq 'length' "$dir/crbs.json")" -eq "$USERS" ]
118+
[ "$(jq -r '[.[].subjects[0].name] | unique | length' "$dir/crbs.json")" -eq "$USERS" ]
119+
[ "$(jq -r '[.[] | select(.roleRef.name == "cluster-admin")] | length' "$dir/crbs.json")" -eq $(( (USERS + 6) / 7 )) ]
120+
121+
# The name is what the prune step and every binding already in a tenant cluster
122+
# go by, so it must not move: cozystack-oidc- plus the first 16 hex digits of
123+
# sha256("<release>-<email>").
124+
for i in 0 1 6 7; do
125+
email="user$i@example.org"
126+
name="cozystack-oidc-$(printf '%s' "bindings-$email" | sha256sum | cut -c1-16)"
127+
[ "$(jq -r --arg e "$email" '.[] | select(.subjects[0].name == $e) | .metadata.name' "$dir/crbs.json")" = "$name" ]
128+
grep -q -x -F "$name" "$dir/desired.txt"
129+
done
130+
131+
# The list the prune step compares against is exactly the applied names.
132+
[ "$(jq -r '.[].metadata.name' "$dir/crbs.json" | sort)" = "$(grep . "$dir/desired.txt" | sort)" ]
133+
rm -rf "$dir"
134+
}
135+
136+
@test "the Job applies the mounted bindings and deletes only the bindings no user has" {
137+
dir=$(mktemp -d)
138+
render_job "$dir" '[{"email":"[email protected]","role":"admin"},{"email":"[email protected]","role":"view"}]'
139+
alice="cozystack-oidc-$(printf '%s' '[email protected]' | sha256sum | cut -c1-16)"
140+
bob="cozystack-oidc-$(printf '%s' '[email protected]' | sha256sum | cut -c1-16)"
141+
fake_kubectl "$dir" "$alice cozystack-oidc-000000000000dead $bob cozystack-oidc-000000000000beef"
142+
run_job "$dir"
143+
if [ "$rc" -ne 0 ]; then
144+
cat "$dir/out"
145+
false
146+
fi
147+
grep -q -F -- "apply --filename $dir/bindings/crbs.yaml" "$dir/kubectl.log"
148+
grep -q -F -- "delete clusterrolebinding cozystack-oidc-000000000000dead --ignore-not-found" "$dir/kubectl.log"
149+
grep -q -F -- "delete clusterrolebinding cozystack-oidc-000000000000beef --ignore-not-found" "$dir/kubectl.log"
150+
if grep -F -e "delete clusterrolebinding $alice" -e "delete clusterrolebinding $bob" "$dir/kubectl.log"; then
151+
echo "the Job deleted a binding a user still has" >&2
152+
false
153+
fi
154+
[ "$(grep -c 'delete clusterrolebinding' "$dir/kubectl.log")" -eq 2 ]
155+
rm -rf "$dir"
156+
}
157+
158+
@test "a release with no users applies nothing and deletes every binding it left behind" {
159+
dir=$(mktemp -d)
160+
render_job "$dir" '[]'
161+
fake_kubectl "$dir" "cozystack-oidc-000000000000dead cozystack-oidc-000000000000beef"
162+
run_job "$dir"
163+
if [ "$rc" -ne 0 ]; then
164+
cat "$dir/out"
165+
false
166+
fi
167+
if grep -q -F -- ' apply ' "$dir/kubectl.log"; then
168+
echo "the Job ran kubectl apply with no bindings to apply" >&2
169+
false
170+
fi
171+
[ "$(grep -c 'delete clusterrolebinding' "$dir/kubectl.log")" -eq 2 ]
172+
rm -rf "$dir"
173+
}
174+
175+
@test "the Job stops before deleting anything when the desired list is not mounted" {
176+
dir=$(mktemp -d)
177+
render_job "$dir" '[{"email":"[email protected]","role":"admin"}]'
178+
fake_kubectl "$dir" "cozystack-oidc-000000000000dead cozystack-oidc-000000000000beef"
179+
rm "$dir/bindings/desired.txt"
180+
run_job "$dir"
181+
[ "$rc" -ne 0 ]
182+
if grep -q -E 'delete clusterrolebinding| apply ' "$dir/kubectl.log"; then
183+
cat "$dir/kubectl.log"
184+
false
185+
fi
186+
rm -rf "$dir"
187+
}
Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
{{- /*
2+
The ClusterRoleBindings the OIDC bootstrap Job (oidc-rbac-job.yaml) applies
3+
inside the tenant cluster, one per `oidc.users` entry, and the list of their
4+
names it prunes against. They travel in a ConfigMap the Job mounts and not as
5+
text inside its script: the script reaches the container as a single `sh -c`
6+
argument, Linux caps one argument at 128 KiB, and a binding per user crosses
7+
that cap at about 240 users, after which the container cannot start.
8+
9+
A release-owned object, not a hook resource, so it exists before the
10+
post-install and post-upgrade hooks run. It is rendered only where users are
11+
bound; at `mode: None` the Job takes its early branch and the volume is
12+
optional.
13+
*/}}
14+
{{- if ne .Values.oidc.mode "None" }}
15+
{{- $desired := "" }}
16+
{{- $bindings := "" }}
17+
{{- range .Values.oidc.users }}
18+
{{- $name := printf "cozystack-oidc-%s" (printf "%s-%s" $.Release.Name .email | sha256sum | trunc 16) }}
19+
{{- $role := "view" }}
20+
{{- if eq .role "admin" }}
21+
{{- $role = "cluster-admin" }}
22+
{{- end }}
23+
{{- $desired = printf "%s%s\n" $desired $name }}
24+
{{- $binding := dict
25+
"apiVersion" "rbac.authorization.k8s.io/v1"
26+
"kind" "ClusterRoleBinding"
27+
"metadata" (dict "name" $name "labels" (dict "app.kubernetes.io/managed-by" "cozystack-oidc" "app.kubernetes.io/instance" $.Release.Name))
28+
"roleRef" (dict "apiGroup" "rbac.authorization.k8s.io" "kind" "ClusterRole" "name" $role)
29+
"subjects" (list (dict "kind" "User" "apiGroup" "rbac.authorization.k8s.io" "name" .email)) }}
30+
{{- $bindings = printf "%s---\n%s\n" $bindings (toYaml $binding) }}
31+
{{- end }}
32+
---
33+
apiVersion: v1
34+
kind: ConfigMap
35+
metadata:
36+
name: {{ .Release.Name }}-oidc-bindings
37+
namespace: {{ .Release.Namespace }}
38+
labels:
39+
{{- include "kubernetes.labels" . | nindent 4 }}
40+
data:
41+
{{- dict "crbs.yaml" $bindings "desired.txt" $desired | toYaml | nindent 2 }}
42+
{{- end }}

‎packages/apps/kubernetes/templates/oidc-rbac-job.yaml‎

Lines changed: 27 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,9 @@
2525
- `mode: System | CustomConfig`:
2626
- Apply one ClusterRoleBinding per `users[]` entry inside the tenant
2727
cluster (`admin` → `cluster-admin`, `view` → `view`, subject = the
28-
literal email matched against the `email` claim).
28+
literal email matched against the `email` claim). The bindings are
29+
rendered by oidc-rbac-bindings.yaml into a ConfigMap this Job mounts,
30+
not into the script below.
2931
- Prune stale CRBs labelled by this release.
3032
- `mode: System`: patch the pre-rendered `<release>-oidc-kubeconfig`
3133
Secret with a real kubeconfig wired to `kubectl oidc-login`
@@ -168,6 +170,8 @@ spec:
168170
value: {{ $issuer | quote }}
169171
- name: OIDC_SERVER_URL
170172
value: {{ printf "https://%s:443" $externalHost | quote }}
173+
- name: OIDC_BINDINGS_DIR
174+
value: /etc/oidc-bindings
171175
resources:
172176
requests:
173177
cpu: 10m
@@ -254,46 +258,32 @@ spec:
254258
sleep 5
255259
done
256260
261+
# A missing list must stop the Job here: grep on a file that is not
262+
# there exits 2, `! grep` is then true, and the prune below would
263+
# delete every binding of the release.
264+
BINDINGS=$OIDC_BINDINGS_DIR
265+
if [ ! -f "$BINDINGS/desired.txt" ]; then
266+
echo "$BINDINGS/desired.txt is not mounted (ConfigMap $RELEASE-oidc-bindings); refusing to prune against a list that is not there" >&2
267+
exit 1
268+
fi
269+
257270
echo "Applying ${OIDC_MODE} ClusterRoleBindings to the tenant cluster..."
258-
DESIRED_FILE=$(mktemp)
259-
{{- range .Values.oidc.users }}
260-
{{- $hash := printf "%s-%s" $.Release.Name .email | sha256sum | trunc 16 }}
261-
{{- $crbName := printf "cozystack-oidc-%s" $hash }}
262-
{{- $role := "view" }}
263-
{{- if eq .role "admin" }}
264-
{{- $role = "cluster-admin" }}
265-
{{- end }}
266-
echo "{{ $crbName }}" >> "$DESIRED_FILE"
267-
cat <<'EOF' | kubectl --kubeconfig "$KUBECONFIG_TENANT" apply --filename -
268-
apiVersion: rbac.authorization.k8s.io/v1
269-
kind: ClusterRoleBinding
270-
metadata:
271-
name: {{ $crbName }}
272-
labels:
273-
app.kubernetes.io/managed-by: cozystack-oidc
274-
app.kubernetes.io/instance: {{ $.Release.Name }}
275-
roleRef:
276-
apiGroup: rbac.authorization.k8s.io
277-
kind: ClusterRole
278-
name: {{ $role }}
279-
subjects:
280-
- kind: User
281-
apiGroup: rbac.authorization.k8s.io
282-
name: {{ .email | quote }}
283-
EOF
284-
{{- end }}
271+
# kubectl apply fails on a file with no objects, and a release with
272+
# no users has none.
273+
if [ -s "$BINDINGS/crbs.yaml" ]; then
274+
kubectl --kubeconfig "$KUBECONFIG_TENANT" apply --filename "$BINDINGS/crbs.yaml"
275+
fi
285276
286277
echo "Pruning stale ClusterRoleBindings labelled by this release..."
287278
EXISTING=$(kubectl --kubeconfig "$KUBECONFIG_TENANT" get clusterrolebindings \
288279
--selector=app.kubernetes.io/managed-by=cozystack-oidc,app.kubernetes.io/instance=$RELEASE \
289280
--output=jsonpath='{.items[*].metadata.name}')
290281
for crb in $EXISTING; do
291-
if ! grep -x -q -F "$crb" "$DESIRED_FILE"; then
282+
if ! grep -x -q -F "$crb" "$BINDINGS/desired.txt"; then
292283
echo " removing $crb"
293284
kubectl --kubeconfig "$KUBECONFIG_TENANT" delete clusterrolebinding "$crb" --ignore-not-found
294285
fi
295286
done
296-
rm -f "$DESIRED_FILE"
297287
298288
if [ "$OIDC_MODE" = "System" ]; then
299289
echo "Patching the OIDC kubeconfig Secret on the management cluster..."
@@ -349,13 +339,20 @@ spec:
349339
- name: kubeconfig
350340
mountPath: /etc/kubernetes/kubeconfig
351341
readOnly: true
342+
- name: oidc-bindings
343+
mountPath: /etc/oidc-bindings
344+
readOnly: true
352345
- name: tmp
353346
mountPath: /tmp
354347
volumes:
355348
- name: kubeconfig
356349
secret:
357350
secretName: {{ .Release.Name }}-admin-kubeconfig
358351
optional: true
352+
- name: oidc-bindings
353+
configMap:
354+
name: {{ .Release.Name }}-oidc-bindings
355+
optional: true
359356
- name: tmp
360357
emptyDir: {}
361358
{{- /*

0 commit comments

Comments
 (0)