Skip to content

Commit b8c7c4f

Browse files
test(e2e): cover webhook admission after namespace churn
Regression for #3983, which reported fail-closed admission webhooks becoming unreachable from the API server while an unrelated namespace was deleted. The mechanism is not confirmed -- the errno in that report cannot come from a policy denial, and namespace deletion churns service backends as well as Cilium identities -- so the case asserts the property the issue is about, that both webhooks still admit across a namespace deletion, rather than claiming to reproduce a cause. Both admission paths are exercised immediately after the deletion with no retry, because a retry would paper over the window this exists to catch. The pod in the churned namespace is what makes it a churn test rather than a smoke test: k8s:io.kubernetes.pod.namespace is an identity-relevant label, so a pod in a brand-new namespace allocates a new Cilium identity and deleting the namespace releases it. An empty namespace drives no identity churn at all. Availability is asserted BEFORE the churn, as a precondition with its own message, because every apply in the case passes through both fail-closed webhooks -- so a cluster whose webhooks were already down fails here in a way that reads exactly like the outage under test. Coordinates come from the webhook registrations themselves, every registration of each, since one being down aborts the apply. Carried unchanged from the upstream change: both webhooks it names exist on this branch and register under the same names. Assisted-By: LLM Signed-off-by: Myasnikov Daniil <[email protected]>
1 parent b02692f commit b8c7c4f

1 file changed

Lines changed: 250 additions & 0 deletions

File tree

‎hack/e2e-chainsaw/gateway/chainsaw-test.yaml‎

Lines changed: 250 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1055,3 +1055,253 @@ spec:
10551055
try:
10561056
- apply:
10571057
file: ingress-external-allow.yaml
1058+
---
1059+
# Regression for #3983, which reported fail-closed admission webhooks becoming
1060+
# unreachable from the API server while an unrelated namespace was deleted. The
1061+
# mechanism is NOT confirmed: the errno in that report cannot come from a policy
1062+
# denial, and namespace deletion churns service backends as well as Cilium
1063+
# identities. So this case asserts the property the issue is about -- both
1064+
# webhooks still admit across a namespace deletion -- rather than claiming to
1065+
# reproduce a cause. Both affected paths are exercised immediately after the
1066+
# deletion, with no retry to hide a window.
1067+
#
1068+
# Every apply in this test passes through both fail-closed webhooks, so a
1069+
# cluster whose webhooks were ALREADY down fails here in a way that reads
1070+
# exactly like the outage under test. The first run of this case did precisely
1071+
# that: the ingress-nginx controller was crash-looping, the admission Service
1072+
# had no ready backend, and the setup apply died with "dial tcp ...: no route to
1073+
# host" before any churn happened. So availability is asserted before the churn
1074+
# as a precondition with its own message, and the churn verdict is only reached
1075+
# once both webhooks are known to have been up to begin with.
1076+
apiVersion: chainsaw.kyverno.io/v1alpha1
1077+
kind: Test
1078+
metadata:
1079+
name: admission-webhooks-survive-namespace-churn
1080+
spec:
1081+
catch:
1082+
- events: {}
1083+
steps:
1084+
- name: ingress-and-pod-admission-remain-available-after-deletion
1085+
try:
1086+
- script:
1087+
# Above the sum of the inner budgets, per the rule in
1088+
# hack/e2e-chainsaw/.chainsaw.yaml: 2x90s of webhook preconditions, 60s
1089+
# of pre-clean delete, 90s waiting for the churn pod, 180s for the
1090+
# namespace delete = 510s, and timeouts.exec would otherwise cap the
1091+
# whole script at 5m and SIGKILL it with no verdict.
1092+
timeout: 12m
1093+
content: |
1094+
set -eu
1095+
ns="$NAMESPACE"
1096+
churn_ns=tenant-admission-churn-probe
1097+
apex=$(kubectl get namespace "$ns" -o jsonpath='{.metadata.labels.namespace\.cozystack\.io/host}')
1098+
[ -n "$apex" ] || { echo "SETUP FAILURE: $ns lacks the namespace.cozystack.io/host label" >&2; exit 1; }
1099+
1100+
# Coordinates come from the webhook registrations themselves, which is
1101+
# what the API server dials, rather than from a guessed release name or
1102+
# namespace. A webhook that is not registered at all is a setup
1103+
# failure: nothing below could then prove anything about it.
1104+
webhook_services() {
1105+
kubectl get "$1" \
1106+
-o jsonpath='{range .items[*]}{range .webhooks[*]}{.name}{" "}{.clientConfig.service.namespace}{" "}{.clientConfig.service.name}{"\n"}{end}{end}'
1107+
}
1108+
1109+
# Ready endpoint addresses behind a Service. Counted from
1110+
# EndpointSlice .endpoints[].conditions.ready rather than from pod
1111+
# readiness, because that is the state the API server routes on, and
1112+
# because a `kubectl wait --for=condition=Ready` is satisfied
1113+
# transiently by a pod that is flapping.
1114+
ready_backends() {
1115+
n=$(kubectl -n "$1" get endpointslice -l "kubernetes.io/service-name=$2" \
1116+
-o jsonpath='{range .items[*]}{range .endpoints[*]}{.conditions.ready}{"\n"}{end}{end}' \
1117+
2>/dev/null | grep -c '^true$' || true)
1118+
[ -n "$n" ] || n=0
1119+
echo "$n"
1120+
}
1121+
1122+
# Distinct variable names from require_webhook_up below, deliberately:
1123+
# POSIX shell functions share one scope, so reusing `label` here
1124+
# appended each iteration's suffix to the caller's own copy and the
1125+
# second registration reported itself as "(1/2) (2/2)".
1126+
require_service_up() {
1127+
svclabel="$1"; svcns="$2"; svcname="$3"; svcbudget="$4"
1128+
svcdeadline=$(( $(date +%s) + svcbudget ))
1129+
while :; do
1130+
n=$(ready_backends "$svcns" "$svcname")
1131+
if [ "$n" -gt 0 ]; then
1132+
echo "PRECONDITION OK $svclabel ($svcns/$svcname) has $n ready backend(s)"
1133+
return 0
1134+
fi
1135+
[ "$(date +%s)" -lt "$svcdeadline" ] || break
1136+
sleep 5
1137+
done
1138+
echo "SETUP FAILURE: $svclabel ($svcns/$svcname) has no ready backend after ${svcbudget}s." >&2
1139+
echo " This test cannot tell a churn-induced outage from a webhook that was" >&2
1140+
echo " already down, and every apply below goes through it, so it stops here" >&2
1141+
echo " rather than reporting the wrong cause." >&2
1142+
kubectl -n "$svcns" get endpointslice -l "kubernetes.io/service-name=$svcname" -o yaml >&2 2>&1 || true
1143+
kubectl -n "$svcns" get pods -o wide >&2 2>&1 || true
1144+
return 1
1145+
}
1146+
1147+
# EVERY registration of the named webhook, not just the first. All of
1148+
# them fire on the applies below, so one being down aborts the apply
1149+
# and the failure would be reported as a churn-induced outage. The
1150+
# loop indexes with sed rather than piping into `while read`, because
1151+
# a pipeline's body runs in a subshell under dash and a `return` from
1152+
# it would not reach this function.
1153+
require_webhook_up() {
1154+
label="$1"; kind="$2"; hook="$3"; budget="$4"
1155+
coords=$(webhook_services "$kind" | awk -v h="$hook" '$1 == h { print $2, $3 }')
1156+
[ -n "$coords" ] || {
1157+
echo "SETUP FAILURE: no $kind registers webhook $hook, so its availability cannot be asserted" >&2
1158+
return 1
1159+
}
1160+
total=$(printf '%s\n' "$coords" | grep -c . || true)
1161+
[ -n "$total" ] || total=0
1162+
i=0
1163+
while [ "$i" -lt "$total" ]; do
1164+
i=$(( i + 1 ))
1165+
line=$(printf '%s\n' "$coords" | sed -n "${i}p")
1166+
require_service_up "$label ($i/$total)" "${line% *}" "${line#* }" "$budget" || return 1
1167+
done
1168+
return 0
1169+
}
1170+
1171+
require_webhook_up "ingress admission" validatingwebhookconfigurations \
1172+
validate.nginx.ingress.kubernetes.io 90
1173+
require_webhook_up "pod admission" mutatingwebhookconfigurations \
1174+
kube-ovn-webhook.cozystack.io 90
1175+
1176+
# Reproduce the incident shape: an ordinary tenant namespace carrying
1177+
# an admitted Ingress AND a running pod, deleted synchronously, with
1178+
# the fail-closed webhooks invoked while Cilium processes the removal.
1179+
#
1180+
# The pod is what makes this a churn test rather than a smoke test. A
1181+
# Cilium identity is the set of an endpoint's identity-relevant
1182+
# labels, and k8s:io.kubernetes.pod.namespace is one of them, so a pod
1183+
# in a brand-new namespace allocates a NEW identity, and deleting the
1184+
# namespace releases it. An empty namespace has no endpoint, allocates
1185+
# no identity, and its deletion drives no identity churn at all, so
1186+
# without the pod this case would exercise nothing it names.
1187+
kubectl delete namespace "$churn_ns" --ignore-not-found --wait=true --timeout=60s
1188+
kubectl apply -f - <<EOF
1189+
apiVersion: v1
1190+
kind: Namespace
1191+
metadata:
1192+
name: ${churn_ns}
1193+
labels:
1194+
namespace.cozystack.io/host: churn.${apex}
1195+
EOF
1196+
kubectl apply -n "$churn_ns" -f - <<EOF
1197+
apiVersion: networking.k8s.io/v1
1198+
kind: Ingress
1199+
metadata:
1200+
name: admission-churn-source
1201+
spec:
1202+
rules:
1203+
- host: probe.churn.${apex}
1204+
http:
1205+
paths:
1206+
- path: /
1207+
pathType: Prefix
1208+
backend:
1209+
service:
1210+
name: noop
1211+
port:
1212+
number: 80
1213+
EOF
1214+
# busybox:musl is staged onto every node by hack/e2e-prepull-images.sh,
1215+
# so this adds no image pull. The securityContext satisfies the
1216+
# restricted PodSecurity profile, so it runs whatever level the tenant
1217+
# namespace enforces.
1218+
kubectl apply -n "$churn_ns" -f - <<'EOF'
1219+
apiVersion: v1
1220+
kind: Pod
1221+
metadata:
1222+
name: admission-churn-identity
1223+
spec:
1224+
restartPolicy: Never
1225+
automountServiceAccountToken: false
1226+
securityContext:
1227+
runAsNonRoot: true
1228+
runAsUser: 65534
1229+
runAsGroup: 65534
1230+
seccompProfile:
1231+
type: RuntimeDefault
1232+
containers:
1233+
- name: hold
1234+
image: busybox:musl
1235+
command: ["sleep", "600"]
1236+
securityContext:
1237+
allowPrivilegeEscalation: false
1238+
capabilities:
1239+
drop: ["ALL"]
1240+
EOF
1241+
kubectl -n "$churn_ns" wait --for=condition=Ready pod/admission-churn-identity --timeout=90s || {
1242+
echo "SETUP FAILURE: the churn pod never became Ready, so deleting $churn_ns releases no Cilium identity and this test would assert nothing" >&2
1243+
kubectl -n "$churn_ns" describe pod admission-churn-identity >&2 2>&1 || true
1244+
exit 1
1245+
}
1246+
1247+
kubectl delete namespace "$churn_ns" --wait=true --timeout=180s
1248+
1249+
# No retry on either call: a retry would paper over exactly the window
1250+
# this test exists to catch.
1251+
#
1252+
# ingress-nginx validating admission
1253+
kubectl apply --dry-run=server -n "$ns" -o name -f - <<EOF
1254+
apiVersion: networking.k8s.io/v1
1255+
kind: Ingress
1256+
metadata:
1257+
name: admission-after-churn
1258+
spec:
1259+
rules:
1260+
- host: admission-after-churn.${apex}
1261+
http:
1262+
paths:
1263+
- path: /
1264+
pathType: Prefix
1265+
backend:
1266+
service:
1267+
name: noop
1268+
port:
1269+
number: 80
1270+
EOF
1271+
1272+
# kubeovn-webhook mutating admission. The securityContext is carried
1273+
# even though a server dry-run Pod is never scheduled: if the tenant
1274+
# namespace ever enforces restricted PodSecurity, a non-compliant Pod
1275+
# is rejected in the validating phase and `set -eu` would abort this
1276+
# check, which reads as the webhook being down.
1277+
kubectl apply --dry-run=server -n "$ns" -o name -f - <<'EOF'
1278+
apiVersion: v1
1279+
kind: Pod
1280+
metadata:
1281+
name: admission-after-churn
1282+
spec:
1283+
restartPolicy: Never
1284+
automountServiceAccountToken: false
1285+
securityContext:
1286+
runAsNonRoot: true
1287+
runAsUser: 65534
1288+
runAsGroup: 65534
1289+
seccompProfile:
1290+
type: RuntimeDefault
1291+
containers:
1292+
- name: pause
1293+
image: registry.k8s.io/pause:3.10
1294+
securityContext:
1295+
allowPrivilegeEscalation: false
1296+
capabilities:
1297+
drop: ["ALL"]
1298+
EOF
1299+
echo "OK both fail-closed webhooks admitted after $churn_ns was deleted"
1300+
check:
1301+
($error == null): true
1302+
finally:
1303+
- delete:
1304+
ref:
1305+
apiVersion: v1
1306+
kind: Namespace
1307+
name: tenant-admission-churn-probe

0 commit comments

Comments
 (0)