1818# root-caused after the fact. This collects exactly that state from the pod's
1919# node so the next recurrence is dispositive.
2020#
21- # What it captures, per affected pod (NotReady, scheduled, has a podIP):
21+ # What it captures, per affected pod (NotReady and scheduled; podIP may still
22+ # be empty while CNI endpoint allocation is the thing that is stuck):
2223# - cilium-agent on the node: cilium-dbg endpoint list; bpf ct entries for
2324# the podIP; a short bounded `cilium-dbg monitor --type drop`; hubble
2425# dropped-verdict observations (if the hubble CLI is present in the agent).
@@ -200,6 +201,16 @@ lb_budget_ok() {
200201 if [ " $1 " -lt " $2 " ]; then echo yes; else echo no; fi
201202}
202203
204+ # pod_filter_affected: stdin =
205+ # `namespace|name|podIP|nodeName|ready|phase` rows. Keep scheduled, non-terminal
206+ # pods whose Ready condition is not True. podIP is deliberately NOT a gate: the
207+ # cilium endpointManager leak this collector diagnoses can strand a pod before
208+ # an IP is assigned, and the node-global Cilium/OVN state plus pod events remain
209+ # useful in that state. IP-specific commands are gated later at their call sites.
210+ pod_filter_affected () {
211+ awk -F' |' ' $4 != "" && $5 != "True" && $6 != "Succeeded" && $6 != "Failed"'
212+ }
213+
203214# Sourcing guard: hack/capture-dataplane.bats sets E2E_CAPTURE_DATAPLANE_LIB and
204215# sources this file purely to reach the helpers above; return before touching $1
205216# or running any capture so the unit test never needs a cluster. The script's
@@ -240,40 +251,20 @@ mkdir -p "$OUT" 2>/dev/null || exit 0
240251
241252log () { echo " [capture-dataplane] $* " ; }
242253
243- # Affected = scheduled (has nodeName), has a podIP (so an endpoint exists to
244- # inspect), Ready!=True, and not already terminal. That is the superset of the
245- # "readiness/startup probe failing with connection refused" symptom -- a pod
246- # that is Running with an IP but whose Ready condition is False is exactly the
247- # transient's signature. Succeeded/Failed pods (completed Jobs, hook pods) are
248- # Ready!=True too but are not the symptom, so they are excluded to keep the
249- # MAX_PODS budget on Running/Pending pods that are actually wedged.
254+ # Affected = scheduled (has nodeName), Ready!=True, and not already terminal.
255+ # A podIP is intentionally optional: a CNI endpoint leak can strand the pod
256+ # before allocation. Running pods with an IP cover the host->pod probe failure;
257+ # Pending pods without one cover the endpoint-allocation failure. Succeeded/
258+ # Failed pods (completed Jobs, hook pods) are Ready!=True too but are not either
259+ # symptom, so they are excluded to keep the MAX_PODS budget on actual wedges.
250260# jsonpath keeps this dependency-free (no jq / go-template reassignment).
251261affected=$( kubectl get pods -A \
252262 -o jsonpath=' {range .items[*]}{.metadata.namespace}{"|"}{.metadata.name}{"|"}{.status.podIP}{"|"}{.spec.nodeName}{"|"}{.status.conditions[?(@.type=="Ready")].status}{"|"}{.status.phase}{"\n"}{end}' \
253- 2> /dev/null | awk -F' |' ' $3!="" && $4!="" && $5!="True" && $6!="Succeeded" && $6!="Failed"' )
254-
255- if [ -z " $affected " ]; then
256- log " no NotReady pods with a podIP+node -- nothing to capture"
257- exit 0
258- fi
263+ 2> /dev/null | pod_filter_affected)
259264
260- ncount=$( printf ' %s\n' " $affected " | wc -l | tr -d ' ' )
261- log " capturing host->pod data-plane for up to $MAX_PODS of $ncount affected pod(s) -> $OUT "
262-
263- # OVN southbound logical-flow dump is cluster-global, so capture it once rather
264- # than per pod. ovn-central is a Deployment (not per-node); any replica answers.
265- # --no-leader-only lets a read land on a raft follower instead of erroring.
266- central=$( kubectl get pod -n " $KUBEOVN_NS " -l app=ovn-central \
267- -o jsonpath=' {.items[0].metadata.name}' 2> /dev/null)
268- if [ -n " $central " ]; then
269- {
270- echo " === ovn-sbctl lflow-list (cluster-global, pod=$central ) ==="
271- timeout 30 kubectl exec -n " $KUBEOVN_NS " " $central " -c ovn-central -- \
272- ovn-sbctl --no-leader-only lflow-list 2>&1 || true
273- } > " $OUT /ovn-lflows.txt" 2>&1 || true
274- else
275- log " no ovn-central pod in $KUBEOVN_NS -- skipping OVN logical-flow dump"
276- fi
265+ # Set even when the pod path is empty; the per-pod capture references it, while
266+ # the independent LoadBalancer path below does not.
267+ central=" "
277268
278269# pod_on_node <ns> <label> <node> -> first matching pod name (empty if none).
279270pod_on_node () {
@@ -383,10 +374,32 @@ capture_node() {
383374 } >> " $nf " 2>&1 || true
384375}
385376
386- i=0
387- printf ' %s\n' " $affected " | {
377+ if [ -z " $affected " ]; then
378+ log " no scheduled NotReady pods -- skipping host->pod capture; checking LoadBalancers independently"
379+ else
380+ ncount=$( printf ' %s\n' " $affected " | wc -l | tr -d ' ' )
381+ log " capturing host->pod data-plane for up to $MAX_PODS of $ncount affected pod(s) -> $OUT "
382+
383+ # OVN southbound logical-flow dump is cluster-global, so capture it once
384+ # rather than per pod. ovn-central is a Deployment (not per-node); any replica
385+ # answers. --no-leader-only lets a read land on a raft follower instead of
386+ # erroring.
387+ central=$( kubectl get pod -n " $KUBEOVN_NS " -l app=ovn-central \
388+ -o jsonpath=' {.items[0].metadata.name}' 2> /dev/null)
389+ if [ -n " $central " ]; then
390+ {
391+ echo " === ovn-sbctl lflow-list (cluster-global, pod=$central ) ==="
392+ timeout 30 kubectl exec -n " $KUBEOVN_NS " " $central " -c ovn-central -- \
393+ ovn-sbctl --no-leader-only lflow-list 2>&1 || true
394+ } > " $OUT /ovn-lflows.txt" 2>&1 || true
395+ else
396+ log " no ovn-central pod in $KUBEOVN_NS -- skipping OVN logical-flow dump"
397+ fi
398+
399+ i=0
400+ printf ' %s\n' " $affected " | {
388401 while IFS=' |' read -r ns pod podip node _ready _phase; do
389- [ -n " $ns " ] && [ -n " $pod " ] && [ -n " $podip " ] && [ -n " $ node" ] || continue
402+ [ -n " $ns " ] && [ -n " $pod " ] && [ -n " $node " ] || continue
390403 i=$(( i + 1 ))
391404 if [ " $i " -gt " $MAX_PODS " ]; then
392405 log " reached MAX_PODS=$MAX_PODS cap; $(( ncount - MAX_PODS)) more affected pod(s) NOT captured"
@@ -403,7 +416,7 @@ printf '%s\n' "$affected" | {
403416 ovs=$( pod_on_node " $KUBEOVN_NS " app=ovs " $node " )
404417 {
405418 echo " ################################################################"
406- echo " # POD $ns /$pod podIP=$podip node=$node (Ready=$_ready )"
419+ echo " # POD $ns /$pod podIP=${ podip:- <none>} node=$node (Ready=$_ready )"
407420 echo " # node-global captures are in node-$node .txt"
408421 echo " ################################################################"
409422
@@ -414,15 +427,21 @@ printf '%s\n' "$affected" | {
414427 kubectl get events -n " $ns " --field-selector " involvedObject.name=$pod " \
415428 -o jsonpath=' {range .items[*]}{.lastTimestamp}{" "}{.reason}{": "}{.message}{"\n"}{end}' 2>&1 || true
416429
417- if [ -n " $agent " ]; then
430+ if [ -z " $podip " ]; then
431+ echo
432+ echo " === pod has no podIP; IP-specific route/conntrack capture skipped ==="
433+ echo " Node-global Cilium/OVN state and pod allocation events remain captured."
434+ fi
435+
436+ if [ -n " $podip " ] && [ -n " $agent " ]; then
418437 echo
419438 echo " === cilium-dbg bpf ct list global | grep $podip (node=$node agent=$agent ) ==="
420439 timeout 25 kubectl exec -n " $CILIUM_NS " " $agent " -c cilium-agent -- \
421440 sh -c " cilium-dbg bpf ct list global 2>/dev/null | grep -F '$podip ' || echo 'no CT entries for $podip '" 2>&1 || true
422441 fi
423442
424443 cni=$( pod_on_node " $KUBEOVN_NS " app=kube-ovn-cni " $node " )
425- if [ -n " $cni " ]; then
444+ if [ -n " $podip " ] && [ -n " $ cni" ]; then
426445 echo
427446 echo " === host netns: ip route get $podip (via kube-ovn cni-server) ==="
428447 timeout 15 kubectl exec -n " $KUBEOVN_NS " " $cni " -c cni-server -- \
@@ -471,7 +490,8 @@ printf '%s\n' "$affected" | {
471490 } > " $pf " 2>&1 || true
472491 done
473492 log " host->pod data-plane capture complete"
474- }
493+ }
494+ fi
475495
476496# ============================================================================ #
477497# LoadBalancer-datapath capture (see the header block for the full rationale). #
0 commit comments