|
| 1 | +#!/usr/bin/env bash |
| 2 | +# Interim CI self-heal for the Cilium in-memory endpoint leak. |
| 3 | +# |
| 4 | +# Symptom: on a churn-heavy install a pod sticks in ContainerCreating/Init |
| 5 | +# because the cilium-agent rejects its sandbox with |
| 6 | +# [PUT /endpoint/{id}][400] putEndpointIdInvalid "IP ipv4:<X> is already in use" |
| 7 | +# repeated until the pod is deleted or the agent is restarted. The IP has no |
| 8 | +# live owner: IPAM and the ipcache released it, but the agent's in-memory |
| 9 | +# endpointManager still holds a stale Endpoint for it (the previous pod's CNI |
| 10 | +# DEL was skipped or raced, so unexpose()->removeReferencesLocked() never ran). |
| 11 | +# Neither the operator's CiliumEndpoint-CRD GC nor the agent's veth-based |
| 12 | +# endpoint GC reaps it, so it wedges the new pod for the whole install and |
| 13 | +# fails the run. Tracked upstream at cilium/cilium#38313 (closed stale); no |
| 14 | +# released or pre-release Cilium fixes the leak and no flag disables it. |
| 15 | +# |
| 16 | +# This watchdog applies the least-disruptive known remedy: instead of |
| 17 | +# restarting the agent (which drops the whole node's in-memory registry), it |
| 18 | +# evicts only the orphaned endpoint via |
| 19 | +# cilium-dbg endpoint disconnect ipv4:<X> |
| 20 | +# which maps to DELETE /endpoint/{id} -> RemoveEndpoint -> unexpose -> |
| 21 | +# removeReferencesLocked, clearing the stale IP-keyed entry. The wedged pod's |
| 22 | +# next CNI ADD retry then succeeds. Blast radius is one dead endpoint. |
| 23 | +# |
| 24 | +# It is READ-ONLY until it has positively confirmed an orphan: an endpoint in |
| 25 | +# the owning node's registry holds the disputed IP, that endpoint does not back |
| 26 | +# a live Running pod with that IP, and a different pod is currently wedged |
| 27 | +# requesting the same IP. If anything is ambiguous it logs and does nothing — |
| 28 | +# it never disconnects a live endpoint. |
| 29 | +# |
| 30 | +# REMOVE this script and the setup_file/teardown_file hooks in |
| 31 | +# hack/e2e-install-cozystack.bats once a fixed Cilium ships. This is a CI |
| 32 | +# band-aid, not a product fix. |
| 33 | +set -u |
| 34 | + |
| 35 | +CILIUM_NS="${CILIUM_NS:-cozy-cilium}" |
| 36 | +POLL_INTERVAL="${CILIUM_HEALER_POLL_SEC:-15}" |
| 37 | +MAX_RUNTIME="${CILIUM_HEALER_MAX_SEC:-2700}" # 45m: covers install + tenant + app suite |
| 38 | +PIDFILE="${CILIUM_HEALER_PIDFILE:-/tmp/cilium-leak-healer.pid}" |
| 39 | + |
| 40 | +echo $$ > "$PIDFILE" 2>/dev/null || true |
| 41 | +deadline=$(( $(date +%s) + MAX_RUNTIME )) |
| 42 | + |
| 43 | +log() { echo "[cilium-leak-healer $(date -u +%H:%M:%S)] $*"; } |
| 44 | + |
| 45 | +# cilium-agent pod running on a given node (DaemonSet selector k8s-app=cilium). |
| 46 | +agent_on_node() { |
| 47 | + kubectl get pod -n "$CILIUM_NS" -l k8s-app=cilium \ |
| 48 | + --field-selector "spec.nodeName=$1" \ |
| 49 | + -o jsonpath='{.items[0].metadata.name}' 2>/dev/null |
| 50 | +} |
| 51 | + |
| 52 | +log "started (ns=$CILIUM_NS poll=${POLL_INTERVAL}s deadline=+${MAX_RUNTIME}s)" |
| 53 | + |
| 54 | +while [ "$(date +%s)" -lt "$deadline" ]; do |
| 55 | + # Collect every "is already in use" rejection still attached to a pod sandbox. |
| 56 | + # The field-selector narrows to the relevant event reason; grep pins the exact |
| 57 | + # agent signature so unrelated sandbox failures are ignored. |
| 58 | + events=$(kubectl get events -A --field-selector reason=FailedCreatePodSandBox \ |
| 59 | + -o jsonpath='{range .items[*]}{.involvedObject.namespace}{"|"}{.involvedObject.name}{"|"}{.message}{"\n"}{end}' 2>/dev/null \ |
| 60 | + | grep -i "is already in use" | sort -u) |
| 61 | + |
| 62 | + [ -n "$events" ] && while IFS='|' read -r ns pod msg; do |
| 63 | + [ -n "$ns" ] && [ -n "$pod" ] || continue |
| 64 | + ip=$(printf '%s' "$msg" | grep -oE 'ipv4:[0-9]{1,3}(\.[0-9]{1,3}){3}' | head -1 | cut -d: -f2) |
| 65 | + [ -n "$ip" ] || continue |
| 66 | + |
| 67 | + # Only act on a pod that is still wedged (re-checked here = freshness gate). |
| 68 | + phase=$(kubectl get pod -n "$ns" "$pod" -o jsonpath='{.status.phase}' 2>/dev/null) |
| 69 | + case "$phase" in Running|Succeeded|"") continue ;; esac |
| 70 | + |
| 71 | + node=$(kubectl get pod -n "$ns" "$pod" -o jsonpath='{.spec.nodeName}' 2>/dev/null) |
| 72 | + [ -n "$node" ] || continue |
| 73 | + agent=$(agent_on_node "$node") |
| 74 | + [ -n "$agent" ] || { log "no cilium-agent on node=$node for $ns/$pod ip=$ip"; continue; } |
| 75 | + |
| 76 | + # Does an endpoint in this agent's registry hold the disputed IP at all? |
| 77 | + # (The wedged pod itself was never registered — CreateEndpoint rejected it |
| 78 | + # before exposing — so any endpoint holding the IP is the stale one.) |
| 79 | + ep_json=$(kubectl exec -n "$CILIUM_NS" "$agent" -c cilium-agent -- \ |
| 80 | + cilium-dbg endpoint get "ipv4:$ip" -o json 2>/dev/null) |
| 81 | + [ -n "$ep_json" ] && [ "$ep_json" != "[]" ] || \ |
| 82 | + { log "no endpoint holds ip=$ip on node=$node (already cleared?) for $ns/$pod"; continue; } |
| 83 | + |
| 84 | + epid=$(printf '%s' "$ep_json" | jq -r '.[0].id // empty' 2>/dev/null) |
| 85 | + epns=$(printf '%s' "$ep_json" | jq -r '.[0].status."external-identifiers"."k8s-namespace" // empty' 2>/dev/null) |
| 86 | + eppod=$(printf '%s' "$ep_json" | jq -r '.[0].status."external-identifiers"."k8s-pod-name" // empty' 2>/dev/null) |
| 87 | + |
| 88 | + # If that endpoint claims a pod that is alive on the cluster with this IP, |
| 89 | + # it is a LIVE endpoint -> a genuine duplicate-IP, NOT the leak. Refuse and |
| 90 | + # shout so the real problem is visible instead of silently broken. |
| 91 | + if [ -n "$epns" ] && [ -n "$eppod" ]; then |
| 92 | + owner_ip=$(kubectl get pod -n "$epns" "$eppod" -o jsonpath='{.status.podIP}' 2>/dev/null) |
| 93 | + owner_phase=$(kubectl get pod -n "$epns" "$eppod" -o jsonpath='{.status.phase}' 2>/dev/null) |
| 94 | + if [ "$owner_ip" = "$ip" ] && [ "$owner_phase" = "Running" ]; then |
| 95 | + log "REFUSE ip=$ip on node=$node: held by LIVE pod $epns/$eppod (genuine duplicate IP, not the leak) — investigate" |
| 96 | + continue |
| 97 | + fi |
| 98 | + fi |
| 99 | + |
| 100 | + # Confirmed orphan: endpoint $epid holds $ip, backs no live pod, and pod |
| 101 | + # $ns/$pod is wedged requesting the same IP. Evict only this endpoint. |
| 102 | + log "HEAL node=$node agent=$agent ep=$epid ip=$ip wedged=$ns/$pod -> endpoint disconnect ipv4:$ip" |
| 103 | + out=$(kubectl exec -n "$CILIUM_NS" "$agent" -c cilium-agent -- \ |
| 104 | + cilium-dbg endpoint disconnect "ipv4:$ip" 2>&1) |
| 105 | + log " result: ${out:-<none>}" |
| 106 | + done <<EOF |
| 107 | +$events |
| 108 | +EOF |
| 109 | + |
| 110 | + sleep "$POLL_INTERVAL" |
| 111 | +done |
| 112 | + |
| 113 | +log "deadline reached, exiting" |
| 114 | +rm -f "$PIDFILE" 2>/dev/null || true |
0 commit comments