@@ -1055,3 +1055,253 @@ spec:
10551055 try :
10561056 - apply :
10571057 file : ingress-external-allow.yaml
1058+ ---
1059+ # Regression for #3983, which reported fail-closed admission webhooks becoming
1060+ # unreachable from the API server while an unrelated namespace was deleted. The
1061+ # mechanism is NOT confirmed: the errno in that report cannot come from a policy
1062+ # denial, and namespace deletion churns service backends as well as Cilium
1063+ # identities. So this case asserts the property the issue is about -- both
1064+ # webhooks still admit across a namespace deletion -- rather than claiming to
1065+ # reproduce a cause. Both affected paths are exercised immediately after the
1066+ # deletion, with no retry to hide a window.
1067+ #
1068+ # Every apply in this test passes through both fail-closed webhooks, so a
1069+ # cluster whose webhooks were ALREADY down fails here in a way that reads
1070+ # exactly like the outage under test. The first run of this case did precisely
1071+ # that: the ingress-nginx controller was crash-looping, the admission Service
1072+ # had no ready backend, and the setup apply died with "dial tcp ...: no route to
1073+ # host" before any churn happened. So availability is asserted before the churn
1074+ # as a precondition with its own message, and the churn verdict is only reached
1075+ # once both webhooks are known to have been up to begin with.
1076+ apiVersion : chainsaw.kyverno.io/v1alpha1
1077+ kind : Test
1078+ metadata :
1079+ name : admission-webhooks-survive-namespace-churn
1080+ spec :
1081+ catch :
1082+ - events : {}
1083+ steps :
1084+ - name : ingress-and-pod-admission-remain-available-after-deletion
1085+ try :
1086+ - script :
1087+ # Above the sum of the inner budgets, per the rule in
1088+ # hack/e2e-chainsaw/.chainsaw.yaml: 2x90s of webhook preconditions, 60s
1089+ # of pre-clean delete, 90s waiting for the churn pod, 180s for the
1090+ # namespace delete = 510s, and timeouts.exec would otherwise cap the
1091+ # whole script at 5m and SIGKILL it with no verdict.
1092+ timeout : 12m
1093+ content : |
1094+ set -eu
1095+ ns="$NAMESPACE"
1096+ churn_ns=tenant-admission-churn-probe
1097+ apex=$(kubectl get namespace "$ns" -o jsonpath='{.metadata.labels.namespace\.cozystack\.io/host}')
1098+ [ -n "$apex" ] || { echo "SETUP FAILURE: $ns lacks the namespace.cozystack.io/host label" >&2; exit 1; }
1099+
1100+ # Coordinates come from the webhook registrations themselves, which is
1101+ # what the API server dials, rather than from a guessed release name or
1102+ # namespace. A webhook that is not registered at all is a setup
1103+ # failure: nothing below could then prove anything about it.
1104+ webhook_services() {
1105+ kubectl get "$1" \
1106+ -o jsonpath='{range .items[*]}{range .webhooks[*]}{.name}{" "}{.clientConfig.service.namespace}{" "}{.clientConfig.service.name}{"\n"}{end}{end}'
1107+ }
1108+
1109+ # Ready endpoint addresses behind a Service. Counted from
1110+ # EndpointSlice .endpoints[].conditions.ready rather than from pod
1111+ # readiness, because that is the state the API server routes on, and
1112+ # because a `kubectl wait --for=condition=Ready` is satisfied
1113+ # transiently by a pod that is flapping.
1114+ ready_backends() {
1115+ n=$(kubectl -n "$1" get endpointslice -l "kubernetes.io/service-name=$2" \
1116+ -o jsonpath='{range .items[*]}{range .endpoints[*]}{.conditions.ready}{"\n"}{end}{end}' \
1117+ 2>/dev/null | grep -c '^true$' || true)
1118+ [ -n "$n" ] || n=0
1119+ echo "$n"
1120+ }
1121+
1122+ # Distinct variable names from require_webhook_up below, deliberately:
1123+ # POSIX shell functions share one scope, so reusing `label` here
1124+ # appended each iteration's suffix to the caller's own copy and the
1125+ # second registration reported itself as "(1/2) (2/2)".
1126+ require_service_up() {
1127+ svclabel="$1"; svcns="$2"; svcname="$3"; svcbudget="$4"
1128+ svcdeadline=$(( $(date +%s) + svcbudget ))
1129+ while :; do
1130+ n=$(ready_backends "$svcns" "$svcname")
1131+ if [ "$n" -gt 0 ]; then
1132+ echo "PRECONDITION OK $svclabel ($svcns/$svcname) has $n ready backend(s)"
1133+ return 0
1134+ fi
1135+ [ "$(date +%s)" -lt "$svcdeadline" ] || break
1136+ sleep 5
1137+ done
1138+ echo "SETUP FAILURE: $svclabel ($svcns/$svcname) has no ready backend after ${svcbudget}s." >&2
1139+ echo " This test cannot tell a churn-induced outage from a webhook that was" >&2
1140+ echo " already down, and every apply below goes through it, so it stops here" >&2
1141+ echo " rather than reporting the wrong cause." >&2
1142+ kubectl -n "$svcns" get endpointslice -l "kubernetes.io/service-name=$svcname" -o yaml >&2 2>&1 || true
1143+ kubectl -n "$svcns" get pods -o wide >&2 2>&1 || true
1144+ return 1
1145+ }
1146+
1147+ # EVERY registration of the named webhook, not just the first. All of
1148+ # them fire on the applies below, so one being down aborts the apply
1149+ # and the failure would be reported as a churn-induced outage. The
1150+ # loop indexes with sed rather than piping into `while read`, because
1151+ # a pipeline's body runs in a subshell under dash and a `return` from
1152+ # it would not reach this function.
1153+ require_webhook_up() {
1154+ label="$1"; kind="$2"; hook="$3"; budget="$4"
1155+ coords=$(webhook_services "$kind" | awk -v h="$hook" '$1 == h { print $2, $3 }')
1156+ [ -n "$coords" ] || {
1157+ echo "SETUP FAILURE: no $kind registers webhook $hook, so its availability cannot be asserted" >&2
1158+ return 1
1159+ }
1160+ total=$(printf '%s\n' "$coords" | grep -c . || true)
1161+ [ -n "$total" ] || total=0
1162+ i=0
1163+ while [ "$i" -lt "$total" ]; do
1164+ i=$(( i + 1 ))
1165+ line=$(printf '%s\n' "$coords" | sed -n "${i}p")
1166+ require_service_up "$label ($i/$total)" "${line% *}" "${line#* }" "$budget" || return 1
1167+ done
1168+ return 0
1169+ }
1170+
1171+ require_webhook_up "ingress admission" validatingwebhookconfigurations \
1172+ validate.nginx.ingress.kubernetes.io 90
1173+ require_webhook_up "pod admission" mutatingwebhookconfigurations \
1174+ kube-ovn-webhook.cozystack.io 90
1175+
1176+ # Reproduce the incident shape: an ordinary tenant namespace carrying
1177+ # an admitted Ingress AND a running pod, deleted synchronously, with
1178+ # the fail-closed webhooks invoked while Cilium processes the removal.
1179+ #
1180+ # The pod is what makes this a churn test rather than a smoke test. A
1181+ # Cilium identity is the set of an endpoint's identity-relevant
1182+ # labels, and k8s:io.kubernetes.pod.namespace is one of them, so a pod
1183+ # in a brand-new namespace allocates a NEW identity, and deleting the
1184+ # namespace releases it. An empty namespace has no endpoint, allocates
1185+ # no identity, and its deletion drives no identity churn at all, so
1186+ # without the pod this case would exercise nothing it names.
1187+ kubectl delete namespace "$churn_ns" --ignore-not-found --wait=true --timeout=60s
1188+ kubectl apply -f - <<EOF
1189+ apiVersion: v1
1190+ kind: Namespace
1191+ metadata:
1192+ name: ${churn_ns}
1193+ labels:
1194+ namespace.cozystack.io/host: churn.${apex}
1195+ EOF
1196+ kubectl apply -n "$churn_ns" -f - <<EOF
1197+ apiVersion: networking.k8s.io/v1
1198+ kind: Ingress
1199+ metadata:
1200+ name: admission-churn-source
1201+ spec:
1202+ rules:
1203+ - host: probe.churn.${apex}
1204+ http:
1205+ paths:
1206+ - path: /
1207+ pathType: Prefix
1208+ backend:
1209+ service:
1210+ name: noop
1211+ port:
1212+ number: 80
1213+ EOF
1214+ # busybox:musl is staged onto every node by hack/e2e-prepull-images.sh,
1215+ # so this adds no image pull. The securityContext satisfies the
1216+ # restricted PodSecurity profile, so it runs whatever level the tenant
1217+ # namespace enforces.
1218+ kubectl apply -n "$churn_ns" -f - <<'EOF'
1219+ apiVersion: v1
1220+ kind: Pod
1221+ metadata:
1222+ name: admission-churn-identity
1223+ spec:
1224+ restartPolicy: Never
1225+ automountServiceAccountToken: false
1226+ securityContext:
1227+ runAsNonRoot: true
1228+ runAsUser: 65534
1229+ runAsGroup: 65534
1230+ seccompProfile:
1231+ type: RuntimeDefault
1232+ containers:
1233+ - name: hold
1234+ image: busybox:musl
1235+ command: ["sleep", "600"]
1236+ securityContext:
1237+ allowPrivilegeEscalation: false
1238+ capabilities:
1239+ drop: ["ALL"]
1240+ EOF
1241+ kubectl -n "$churn_ns" wait --for=condition=Ready pod/admission-churn-identity --timeout=90s || {
1242+ echo "SETUP FAILURE: the churn pod never became Ready, so deleting $churn_ns releases no Cilium identity and this test would assert nothing" >&2
1243+ kubectl -n "$churn_ns" describe pod admission-churn-identity >&2 2>&1 || true
1244+ exit 1
1245+ }
1246+
1247+ kubectl delete namespace "$churn_ns" --wait=true --timeout=180s
1248+
1249+ # No retry on either call: a retry would paper over exactly the window
1250+ # this test exists to catch.
1251+ #
1252+ # ingress-nginx validating admission
1253+ kubectl apply --dry-run=server -n "$ns" -o name -f - <<EOF
1254+ apiVersion: networking.k8s.io/v1
1255+ kind: Ingress
1256+ metadata:
1257+ name: admission-after-churn
1258+ spec:
1259+ rules:
1260+ - host: admission-after-churn.${apex}
1261+ http:
1262+ paths:
1263+ - path: /
1264+ pathType: Prefix
1265+ backend:
1266+ service:
1267+ name: noop
1268+ port:
1269+ number: 80
1270+ EOF
1271+
1272+ # kubeovn-webhook mutating admission. The securityContext is carried
1273+ # even though a server dry-run Pod is never scheduled: if the tenant
1274+ # namespace ever enforces restricted PodSecurity, a non-compliant Pod
1275+ # is rejected in the validating phase and `set -eu` would abort this
1276+ # check, which reads as the webhook being down.
1277+ kubectl apply --dry-run=server -n "$ns" -o name -f - <<'EOF'
1278+ apiVersion: v1
1279+ kind: Pod
1280+ metadata:
1281+ name: admission-after-churn
1282+ spec:
1283+ restartPolicy: Never
1284+ automountServiceAccountToken: false
1285+ securityContext:
1286+ runAsNonRoot: true
1287+ runAsUser: 65534
1288+ runAsGroup: 65534
1289+ seccompProfile:
1290+ type: RuntimeDefault
1291+ containers:
1292+ - name: pause
1293+ image: registry.k8s.io/pause:3.10
1294+ securityContext:
1295+ allowPrivilegeEscalation: false
1296+ capabilities:
1297+ drop: ["ALL"]
1298+ EOF
1299+ echo "OK both fail-closed webhooks admitted after $churn_ns was deleted"
1300+ check :
1301+ ($error == null) : true
1302+ finally :
1303+ - delete :
1304+ ref :
1305+ apiVersion : v1
1306+ kind : Namespace
1307+ name : tenant-admission-churn-probe
0 commit comments