Skip to content

Commit c8f0597

Browse files
test(e2e): self-heal Cilium orphaned-endpoint leak during install
A churn-heavy platform install intermittently wedges a pod in ContainerCreating because the cilium-agent rejects its sandbox with '[PUT /endpoint/{id}][400] putEndpointIdInvalid "IP ipv4:<X> is already in use"', repeated until the agent restarts or the install times out. The disputed IP has no live owner: IPAM and the ipcache released it, but the agent's in-memory endpointManager still holds a stale Endpoint from a previously-deleted pod whose CNI DEL was skipped or raced (so removeReferencesLocked never ran). Neither the operator's CiliumEndpoint GC nor the agent's veth-based endpoint GC can reap it. This surfaced as ~6 of the post-#2558 e2e failures, each on a different victim pod (velero node-agent, fluent-bit, fdb-operator, cdi-operator, capi bootstrap), failing PRs unrelated to the change under test. Tracked upstream at cilium/cilium#38313 (closed stale); no released or pre-release Cilium fixes it and no flag disables it. hack/e2e-cilium-endpoint-leak-healer.sh runs as a file-scoped background watchdog (setup_file/teardown_file). It detects a wedged pod, confirms the IP is a true orphan -- an endpoint in the owning node's registry holds it, that endpoint backs no live Running pod, and a different pod is wedged requesting the same IP -- then evicts only that endpoint via 'cilium-dbg endpoint disconnect ipv4:<X>' (DELETE /endpoint/{id} -> unexpose -> removeReferencesLocked). Blast radius is one dead endpoint, not an agent restart. It is read-only until an orphan is positively confirmed and refuses (loudly) to touch an endpoint backing a live pod, so a genuine duplicate-IP bug stays visible rather than masked. This is an interim CI mitigation, not a product fix; remove the script and the bats hooks once a fixed Cilium ships. Assisted-By: Claude <[email protected]> Signed-off-by: Myasnikov Daniil <[email protected]>
1 parent 8c411d8 commit c8f0597

2 files changed

Lines changed: 141 additions & 0 deletions

File tree

Lines changed: 114 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,114 @@
1+
#!/usr/bin/env bash
2+
# Interim CI self-heal for the Cilium in-memory endpoint leak.
3+
#
4+
# Symptom: on a churn-heavy install a pod sticks in ContainerCreating/Init
5+
# because the cilium-agent rejects its sandbox with
6+
# [PUT /endpoint/{id}][400] putEndpointIdInvalid "IP ipv4:<X> is already in use"
7+
# repeated until the pod is deleted or the agent is restarted. The IP has no
8+
# live owner: IPAM and the ipcache released it, but the agent's in-memory
9+
# endpointManager still holds a stale Endpoint for it (the previous pod's CNI
10+
# DEL was skipped or raced, so unexpose()->removeReferencesLocked() never ran).
11+
# Neither the operator's CiliumEndpoint-CRD GC nor the agent's veth-based
12+
# endpoint GC reaps it, so it wedges the new pod for the whole install and
13+
# fails the run. Tracked upstream at cilium/cilium#38313 (closed stale); no
14+
# released or pre-release Cilium fixes the leak and no flag disables it.
15+
#
16+
# This watchdog applies the least-disruptive known remedy: instead of
17+
# restarting the agent (which drops the whole node's in-memory registry), it
18+
# evicts only the orphaned endpoint via
19+
# cilium-dbg endpoint disconnect ipv4:<X>
20+
# which maps to DELETE /endpoint/{id} -> RemoveEndpoint -> unexpose ->
21+
# removeReferencesLocked, clearing the stale IP-keyed entry. The wedged pod's
22+
# next CNI ADD retry then succeeds. Blast radius is one dead endpoint.
23+
#
24+
# It is READ-ONLY until it has positively confirmed an orphan: an endpoint in
25+
# the owning node's registry holds the disputed IP, that endpoint does not back
26+
# a live Running pod with that IP, and a different pod is currently wedged
27+
# requesting the same IP. If anything is ambiguous it logs and does nothing —
28+
# it never disconnects a live endpoint.
29+
#
30+
# REMOVE this script and the setup_file/teardown_file hooks in
31+
# hack/e2e-install-cozystack.bats once a fixed Cilium ships. This is a CI
32+
# band-aid, not a product fix.
33+
set -u
34+
35+
CILIUM_NS="${CILIUM_NS:-cozy-cilium}"
36+
POLL_INTERVAL="${CILIUM_HEALER_POLL_SEC:-15}"
37+
MAX_RUNTIME="${CILIUM_HEALER_MAX_SEC:-2700}" # 45m: covers install + tenant + app suite
38+
PIDFILE="${CILIUM_HEALER_PIDFILE:-/tmp/cilium-leak-healer.pid}"
39+
40+
echo $$ > "$PIDFILE" 2>/dev/null || true
41+
deadline=$(( $(date +%s) + MAX_RUNTIME ))
42+
43+
log() { echo "[cilium-leak-healer $(date -u +%H:%M:%S)] $*"; }
44+
45+
# cilium-agent pod running on a given node (DaemonSet selector k8s-app=cilium).
46+
agent_on_node() {
47+
kubectl get pod -n "$CILIUM_NS" -l k8s-app=cilium \
48+
--field-selector "spec.nodeName=$1" \
49+
-o jsonpath='{.items[0].metadata.name}' 2>/dev/null
50+
}
51+
52+
log "started (ns=$CILIUM_NS poll=${POLL_INTERVAL}s deadline=+${MAX_RUNTIME}s)"
53+
54+
while [ "$(date +%s)" -lt "$deadline" ]; do
55+
# Collect every "is already in use" rejection still attached to a pod sandbox.
56+
# The field-selector narrows to the relevant event reason; grep pins the exact
57+
# agent signature so unrelated sandbox failures are ignored.
58+
events=$(kubectl get events -A --field-selector reason=FailedCreatePodSandBox \
59+
-o jsonpath='{range .items[*]}{.involvedObject.namespace}{"|"}{.involvedObject.name}{"|"}{.message}{"\n"}{end}' 2>/dev/null \
60+
| grep -i "is already in use" | sort -u)
61+
62+
[ -n "$events" ] && while IFS='|' read -r ns pod msg; do
63+
[ -n "$ns" ] && [ -n "$pod" ] || continue
64+
ip=$(printf '%s' "$msg" | grep -oE 'ipv4:[0-9]{1,3}(\.[0-9]{1,3}){3}' | head -1 | cut -d: -f2)
65+
[ -n "$ip" ] || continue
66+
67+
# Only act on a pod that is still wedged (re-checked here = freshness gate).
68+
phase=$(kubectl get pod -n "$ns" "$pod" -o jsonpath='{.status.phase}' 2>/dev/null)
69+
case "$phase" in Running|Succeeded|"") continue ;; esac
70+
71+
node=$(kubectl get pod -n "$ns" "$pod" -o jsonpath='{.spec.nodeName}' 2>/dev/null)
72+
[ -n "$node" ] || continue
73+
agent=$(agent_on_node "$node")
74+
[ -n "$agent" ] || { log "no cilium-agent on node=$node for $ns/$pod ip=$ip"; continue; }
75+
76+
# Does an endpoint in this agent's registry hold the disputed IP at all?
77+
# (The wedged pod itself was never registered — CreateEndpoint rejected it
78+
# before exposing — so any endpoint holding the IP is the stale one.)
79+
ep_json=$(kubectl exec -n "$CILIUM_NS" "$agent" -c cilium-agent -- \
80+
cilium-dbg endpoint get "ipv4:$ip" -o json 2>/dev/null)
81+
[ -n "$ep_json" ] && [ "$ep_json" != "[]" ] || \
82+
{ log "no endpoint holds ip=$ip on node=$node (already cleared?) for $ns/$pod"; continue; }
83+
84+
epid=$(printf '%s' "$ep_json" | jq -r '.[0].id // empty' 2>/dev/null)
85+
epns=$(printf '%s' "$ep_json" | jq -r '.[0].status."external-identifiers"."k8s-namespace" // empty' 2>/dev/null)
86+
eppod=$(printf '%s' "$ep_json" | jq -r '.[0].status."external-identifiers"."k8s-pod-name" // empty' 2>/dev/null)
87+
88+
# If that endpoint claims a pod that is alive on the cluster with this IP,
89+
# it is a LIVE endpoint -> a genuine duplicate-IP, NOT the leak. Refuse and
90+
# shout so the real problem is visible instead of silently broken.
91+
if [ -n "$epns" ] && [ -n "$eppod" ]; then
92+
owner_ip=$(kubectl get pod -n "$epns" "$eppod" -o jsonpath='{.status.podIP}' 2>/dev/null)
93+
owner_phase=$(kubectl get pod -n "$epns" "$eppod" -o jsonpath='{.status.phase}' 2>/dev/null)
94+
if [ "$owner_ip" = "$ip" ] && [ "$owner_phase" = "Running" ]; then
95+
log "REFUSE ip=$ip on node=$node: held by LIVE pod $epns/$eppod (genuine duplicate IP, not the leak) — investigate"
96+
continue
97+
fi
98+
fi
99+
100+
# Confirmed orphan: endpoint $epid holds $ip, backs no live pod, and pod
101+
# $ns/$pod is wedged requesting the same IP. Evict only this endpoint.
102+
log "HEAL node=$node agent=$agent ep=$epid ip=$ip wedged=$ns/$pod -> endpoint disconnect ipv4:$ip"
103+
out=$(kubectl exec -n "$CILIUM_NS" "$agent" -c cilium-agent -- \
104+
cilium-dbg endpoint disconnect "ipv4:$ip" 2>&1)
105+
log " result: ${out:-<none>}"
106+
done <<EOF
107+
$events
108+
EOF
109+
110+
sleep "$POLL_INTERVAL"
111+
done
112+
113+
log "deadline reached, exiting"
114+
rm -f "$PIDFILE" 2>/dev/null || true

‎hack/e2e-install-cozystack.bats‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,32 @@
11
#!/usr/bin/env bats
22

3+
setup_file() {
4+
# Interim mitigation for the Cilium in-memory endpoint leak
5+
# (cilium/cilium#38313 class): a deleted pod's endpoint is orphaned in the
6+
# agent's registry, IPAM re-hands its IP to a new pod, and the agent then
7+
# rejects the new sandbox with "IP <X> is already in use" until an agent
8+
# restart — wedging the pod for the whole install and failing unrelated PRs.
9+
# The watchdog surgically evicts only the orphaned endpoint. Detached here so
10+
# it covers every test in this file (platform install + tenant + apps); it
11+
# self-bounds with its own deadline and writes a PID file we stop below.
12+
# Remove together with hack/e2e-cilium-endpoint-leak-healer.sh once a fixed
13+
# Cilium ships.
14+
hack/e2e-cilium-endpoint-leak-healer.sh > /tmp/cilium-leak-healer.log 2>&1 &
15+
}
16+
17+
teardown_file() {
18+
if [ -f /tmp/cilium-leak-healer.pid ]; then
19+
kill "$(cat /tmp/cilium-leak-healer.pid)" 2>/dev/null || true
20+
fi
21+
# Surface any action it took into the test trace; the run is otherwise green
22+
# and the heal would be invisible. REFUSE lines flag a genuine duplicate-IP
23+
# (a real bug the watchdog deliberately did not touch).
24+
if grep -qE "HEAL |REFUSE " /tmp/cilium-leak-healer.log 2>/dev/null; then
25+
echo "# cilium endpoint-leak watchdog activity:" >&3
26+
grep -E "HEAL |REFUSE " /tmp/cilium-leak-healer.log >&3 || true
27+
fi
28+
}
29+
330
@test "Required installer chart exists" {
431
if [ ! -f packages/core/installer/Chart.yaml ]; then
532
echo "Missing: packages/core/installer/Chart.yaml" >&2

0 commit comments

Comments
 (0)