Skip to content

Commit dc5928c

Browse files
committed
test(e2e): capture host-netns address attribution and a healthy-pod reference
The dataplane collector already characterises the host->local-pod route for an affected pod, but only through the ovn0 interface, so when a pod IP turns out to be a local address of the host netns instead, nothing records which interface actually holds it. Capture the whole host netns (every interface, plus the local routing table) so the owner can be named on the next occurrence, instead of only ruling out ovn0. The collector also only ever inspects NotReady pods, so a run has no same-window baseline for what a normal route and conntrack state look like on the same node. Extract the per-pod capture into a reusable function and run it a second time against one Ready, Running pod on the same node as an in-run healthy-pod control, recording the absence explicitly when no such pod exists on the node instead of leaving the gap silent. Signed-off-by: Aleksei Sviridkin <[email protected]>
1 parent 605030b commit dc5928c

2 files changed

Lines changed: 225 additions & 77 deletions

File tree

‎hack/capture-dataplane.bats‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
# - lb_announcer_node -- the speaker node of the most recent announce;
1212
# - lb_capture_decision -- capture only when every probe failed.
1313
# - pod_filter_affected -- retain scheduled NotReady pods even without IPs.
14+
# - pod_first_ready -- pick the in-run healthy-pod reference for a node.
1415
#
1516
# The kubectl exec / tcpdump capture itself is not unit-testable (it needs a
1617
# live cluster); these tests pin the derivations that decide WHICH node to
@@ -63,6 +64,55 @@ E2E_CAPTURE_DATAPLANE_LIB=1
6364
[ -z "$(printf '%s\n' "$rows" | pod_filter_affected)" ]
6465
}
6566

67+
@test "pod_first_ready picks the first Ready, Running pod and skips the rest" {
68+
rows="$(printf '%s\n' \
69+
'tenant|not-ready|192.0.2.90|False|Running' \
70+
'tenant|pending|192.0.2.91|True|Pending' \
71+
'tenant|healthy-1|192.0.2.92|True|Running' \
72+
'tenant|healthy-2|192.0.2.93|True|Running')"
73+
74+
out="$(printf '%s\n' "$rows" | pod_first_ready)"
75+
76+
[ "$out" = "tenant|healthy-1|192.0.2.92" ]
77+
}
78+
79+
@test "pod_first_ready emits nothing when no pod on the node is Ready and Running" {
80+
rows="$(printf '%s\n' \
81+
'tenant|not-ready|192.0.2.94|False|Running' \
82+
'tenant|pending|192.0.2.95|True|Pending')"
83+
84+
[ -z "$(printf '%s\n' "$rows" | pod_first_ready)" ]
85+
}
86+
87+
@test "pod_first_ready emits nothing on empty input" {
88+
[ -z "$(printf '' | pod_first_ready)" ]
89+
}
90+
91+
@test "pod_first_ready skips a hostNetwork pod even when it is Ready, Running, and first" {
92+
# cilium-agent / kube-ovn-cni / ovs-ovn run hostNetwork on every node, and a
93+
# hostNetwork pod's podIP IS the node's own address -- picking one as the
94+
# healthy-pod reference would show the same local-route fingerprint as the
95+
# bug #3615 exists to characterise, for an unrelated reason, defeating the
96+
# whole point of the control.
97+
rows="$(printf '%s\n' \
98+
'cozy-cilium|cilium-agent-abcde|192.0.2.10|True|Running|true' \
99+
'tenant|workload|192.0.2.96|True|Running|')"
100+
101+
out="$(printf '%s\n' "$rows" | pod_first_ready)"
102+
103+
[ "$out" = "tenant|workload|192.0.2.96" ]
104+
}
105+
106+
@test "pod_first_ready treats a blank hostNetwork column as NOT hostNetwork" {
107+
# The API omits spec.hostNetwork (omitempty) when it is false/unset, so
108+
# kubectl's jsonpath emits an empty string, not the literal "false".
109+
rows="$(printf '%s\n' 'tenant|workload|192.0.2.97|True|Running|')"
110+
111+
out="$(printf '%s\n' "$rows" | pod_first_ready)"
112+
113+
[ "$out" = "tenant|workload|192.0.2.97" ]
114+
}
115+
66116
@test "runtime checks LoadBalancers when there are no affected pods" {
67117
tmp=$(mktemp -d)
68118
trap 'rm -rf "$tmp"' EXIT

‎hack/e2e-capture-dataplane.sh‎

Lines changed: 175 additions & 77 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,16 @@
3535
# kernel-path misforward / missing route / wrong rule / unresolved neighbor
3636
# -- the actual mechanism behind host->local-pod "connection refused" on the
3737
# legacy-routing path (the crux of the transient).
38+
# - Address attribution for a podIP that turns out to be a LOCAL address of
39+
# the host netns instead of reachable via ovn0 (a route to nowhere-in-
40+
# particular is not enough to name the owner): the same host netns above
41+
# also gets `ip -o addr show` (every interface) and `ip route show table
42+
# local`, and one Ready, non-hostNetwork pod on the same node gets the
43+
# identical per-pod capture below as an in-run healthy-pod control, since
44+
# this collector otherwise inspects NotReady pods only and had no
45+
# same-window baseline. hostNetwork is excluded because its podIP IS the
46+
# node's own address, so it would show the same local-route fingerprint
47+
# as the bug for an unrelated reason.
3848
# - OVS/OVN on the node: `ovs-ofctl dump-flows br-int`, and the OVN
3949
# Port_Binding / Logical_Switch_Port for the pod (LSP name = <pod>.<ns> in
4050
# kube-ovn) plus a bounded `ovn-sbctl lflow-list`.
@@ -211,6 +221,24 @@ pod_filter_affected() {
211221
awk -F'|' '$4 != "" && $5 != "True" && $6 != "Succeeded" && $6 != "Failed"'
212222
}
213223

224+
# pod_first_ready: stdin = `namespace|name|podIP|ready|phase|hostNetwork` rows,
225+
# already scoped to one node by the caller's kubectl field-selector and with
226+
# the affected pod itself excluded. Emits `namespace|name|podIP` for the first
227+
# pod that is Ready=True, Running, and NOT hostNetwork -- the in-run
228+
# healthy-pod control for that node (see the address-attribution addition in
229+
# the header block) -- then stops. hostNetwork pods are excluded because their
230+
# podIP IS the node's own address: cilium-agent / kube-ovn-cni / ovs-ovn run
231+
# hostNetwork on every node, so without this exclusion the "healthy" reference
232+
# could legitimately resolve `ip route get <ip>` to `local <ip> dev lo table
233+
# local` -- the exact fingerprint #3615 is trying to distinguish from the
234+
# actual bug -- and silently defeat the whole point of the control. A blank
235+
# hostNetwork column counts as NOT hostNetwork (the API omits the field when
236+
# false); only the literal string "true" excludes. Empty output means no
237+
# eligible Ready pod exists on the node.
238+
pod_first_ready() {
239+
awk -F'|' '$4 == "True" && $5 == "Running" && $6 != "true" { print $1 "|" $2 "|" $3; exit }'
240+
}
241+
214242
# Sourcing guard: hack/capture-dataplane.bats sets E2E_CAPTURE_DATAPLANE_LIB and
215243
# sources this file purely to reach the helpers above; return before touching $1
216244
# or running any capture so the unit test never needs a cluster. The script's
@@ -367,13 +395,157 @@ capture_node() {
367395
echo "=== host netns: ip addr show ovn0 ==="
368396
timeout 15 kubectl exec -n "$KUBEOVN_NS" "$cni" -c cni-server -- \
369397
ip addr show ovn0 2>&1 || true
398+
399+
echo
400+
echo "=== host netns: ip -o addr show, all interfaces (address attribution) ==="
401+
# ovn0 alone cannot name the owner when a podIP turns out to be a LOCAL
402+
# address instead: the interface actually holding it might be anything on
403+
# the host (see the header block). This is the same info scoped to every
404+
# interface, not just ovn0.
405+
timeout 15 kubectl exec -n "$KUBEOVN_NS" "$cni" -c cni-server -- \
406+
ip -o addr show 2>&1 || true
407+
408+
echo
409+
echo "=== host netns: ip route show table local (address attribution) ==="
410+
# A podIP resolving to `local <ip> dev lo table local` here is the
411+
# fingerprint from the header block: it wins over the ovn0 route in
412+
# `main` because table local is consulted first (see `ip rule` above).
413+
timeout 15 kubectl exec -n "$KUBEOVN_NS" "$cni" -c cni-server -- \
414+
ip route show table local 2>&1 || true
370415
else
371416
echo
372417
echo "(no kube-ovn-cni pod found on node $node -- host netns capture skipped)"
373418
fi
374419
} >> "$nf" 2>&1 || true
375420
}
376421

422+
# capture_pod_dataplane <ns> <pod> <podip> <node> <label> -- the pod-specific
423+
# captures (Ready conditions/events, cilium CT, host route, kernel conntrack,
424+
# OVN port binding, OVS ovn-installed) for ONE pod on <node>. <label> is
425+
# stamped in the section header: "NotReady" for an affected pod, "REFERENCE
426+
# (Ready)" for the in-run healthy-pod control added for the address-
427+
# attribution gap in the header block. Writes to stdout; the caller redirects
428+
# into the pod's output file. $central (the ovn-central pod, or empty) is read
429+
# from the caller's scope rather than threaded as a parameter, matching how
430+
# the rest of this script shares node-global lookups.
431+
capture_pod_dataplane() {
432+
_cpd_ns=$1; _cpd_pod=$2; _cpd_ip=$3; _cpd_node=$4; _cpd_label=$5
433+
434+
_cpd_agent=$(pod_on_node "$CILIUM_NS" k8s-app=cilium "$_cpd_node")
435+
_cpd_ovs=$(pod_on_node "$KUBEOVN_NS" app=ovs "$_cpd_node")
436+
_cpd_cni=$(pod_on_node "$KUBEOVN_NS" app=kube-ovn-cni "$_cpd_node")
437+
438+
echo "################################################################"
439+
echo "# POD $_cpd_ns/$_cpd_pod podIP=${_cpd_ip:-<none>} node=$_cpd_node ($_cpd_label)"
440+
echo "# node-global captures are in node-$_cpd_node.txt"
441+
echo "################################################################"
442+
443+
echo
444+
echo "=== pod Ready conditions + recent probe events ==="
445+
kubectl get pod -n "$_cpd_ns" "$_cpd_pod" \
446+
-o jsonpath='{range .status.conditions[*]}{.type}={.status} reason={.reason}: {.message}{"\n"}{end}' 2>&1 || true
447+
kubectl get events -n "$_cpd_ns" --field-selector "involvedObject.name=$_cpd_pod" \
448+
-o jsonpath='{range .items[*]}{.lastTimestamp}{" "}{.reason}{": "}{.message}{"\n"}{end}' 2>&1 || true
449+
450+
if [ -z "$_cpd_ip" ]; then
451+
echo
452+
echo "=== pod has no podIP; IP-specific route/conntrack capture skipped ==="
453+
echo "Node-global Cilium/OVN state and pod allocation events remain captured."
454+
fi
455+
456+
if [ -n "$_cpd_ip" ] && [ -n "$_cpd_agent" ]; then
457+
echo
458+
echo "=== cilium-dbg bpf ct list global | grep $_cpd_ip (node=$_cpd_node agent=$_cpd_agent) ==="
459+
timeout 25 kubectl exec -n "$CILIUM_NS" "$_cpd_agent" -c cilium-agent -- \
460+
sh -c "cilium-dbg bpf ct list global 2>/dev/null | grep -F '$_cpd_ip' || echo 'no CT entries for $_cpd_ip'" 2>&1 || true
461+
fi
462+
463+
if [ -n "$_cpd_ip" ] && [ -n "$_cpd_cni" ]; then
464+
echo
465+
echo "=== host netns: ip route get $_cpd_ip (via kube-ovn cni-server) ==="
466+
timeout 15 kubectl exec -n "$KUBEOVN_NS" "$_cpd_cni" -c cni-server -- \
467+
ip route get "$_cpd_ip" 2>&1 || true
468+
469+
echo
470+
echo "=== host netns: kernel conntrack for $_cpd_ip ==="
471+
# Under enable-host-legacy-routing the host->local-pod path is in the
472+
# kernel netfilter conntrack table, NOT cilium BPF -- this is the
473+
# authoritative table for the transient. Prefer the conntrack CLI; fall
474+
# back to /proc/net/nf_conntrack when it is absent from the image.
475+
timeout 15 kubectl exec -n "$KUBEOVN_NS" "$_cpd_cni" -c cni-server -- \
476+
sh -c "if command -v conntrack >/dev/null 2>&1; then conntrack -L 2>/dev/null | grep -F '$_cpd_ip' || echo 'no conntrack entries for $_cpd_ip'; else grep -F '$_cpd_ip' /proc/net/nf_conntrack 2>/dev/null || echo 'no conntrack CLI; no /proc/net/nf_conntrack match for $_cpd_ip'; fi" 2>&1 || true
477+
fi
478+
479+
if [ -n "$central" ]; then
480+
echo
481+
echo "=== OVN Port_Binding / Logical_Switch_Port for $_cpd_pod.$_cpd_ns ==="
482+
# kube-ovn names the OVN logical port <pod>.<namespace>.
483+
timeout 20 kubectl exec -n "$KUBEOVN_NS" "$central" -c ovn-central -- \
484+
ovn-sbctl --no-leader-only find port_binding "logical_port=$_cpd_pod.$_cpd_ns" 2>&1 || true
485+
timeout 20 kubectl exec -n "$KUBEOVN_NS" "$central" -c ovn-central -- \
486+
ovn-nbctl --no-leader-only find logical_switch_port "name=$_cpd_pod.$_cpd_ns" 2>&1 || true
487+
fi
488+
489+
if [ -n "$_cpd_ovs" ]; then
490+
echo
491+
echo "=== OVS interface ovn-installed flag + ts for iface-id=$_cpd_pod.$_cpd_ns (ovs pod $_cpd_ovs) ==="
492+
# kube-ovn stamps external_ids:ovn-installed (+ -ts) on the pod's OVS
493+
# interface once ovn-controller reports the port programmed -- this is
494+
# the barrier the CNI waits on. The OVS interface name is opaque, so
495+
# look it up by iface-id (<pod>.<ns>), then read the flag + timestamp.
496+
# An ovn-installed-ts set early (before physical_flow_output installed
497+
# the local-delivery flow, see node-$node.txt) is the I-P-lag signature.
498+
_cpd_ovsif=$(timeout 20 kubectl exec -n "$KUBEOVN_NS" "$_cpd_ovs" -c openvswitch -- \
499+
ovs-vsctl --no-heading --columns=name find interface "external_ids:iface-id=$_cpd_pod.$_cpd_ns" 2>/dev/null \
500+
| head -n 1 | tr -d '" ')
501+
if [ -n "$_cpd_ovsif" ]; then
502+
echo "ovs interface = $_cpd_ovsif"
503+
timeout 20 kubectl exec -n "$KUBEOVN_NS" "$_cpd_ovs" -c openvswitch -- \
504+
ovs-vsctl get interface "$_cpd_ovsif" external_ids:ovn-installed external_ids:ovn-installed-ts 2>&1 || true
505+
else
506+
echo "no OVS interface with external_ids:iface-id=$_cpd_pod.$_cpd_ns"
507+
fi
508+
fi
509+
}
510+
511+
# capture_pod_reference <ns> <pod> <node> <outfile> -- the in-run healthy-pod
512+
# control for #3615's address-attribution gap: this collector otherwise
513+
# inspects NotReady pods only, so there is no same-window baseline for what a
514+
# normal route/conntrack looks like on the SAME node. Picks one Ready, Running,
515+
# non-hostNetwork pod on <node> other than <ns>/<pod> and runs the identical
516+
# per-pod capture against it, appended to <outfile>. hostNetwork is excluded
517+
# by pod_first_ready (see its doc comment): a hostNetwork pod's podIP is the
518+
# node's own address, so it would "legitimately" show the same local-route
519+
# fingerprint as the bug this control exists to rule out. Honest about the
520+
# negative: when no eligible Ready pod exists on the node, that absence is
521+
# recorded too, not left silent.
522+
capture_pod_reference() {
523+
_cpr_ns=$1; _cpr_pod=$2; _cpr_node=$3; _cpr_of=$4
524+
525+
_cpr_ref=$(kubectl get pods -A --field-selector "spec.nodeName=$_cpr_node" \
526+
-o jsonpath='{range .items[*]}{.metadata.namespace}{"|"}{.metadata.name}{"|"}{.status.podIP}{"|"}{.status.conditions[?(@.type=="Ready")].status}{"|"}{.status.phase}{"|"}{.spec.hostNetwork}{"\n"}{end}' \
527+
2>/dev/null | awk -F'|' -v sns="$_cpr_ns" -v spod="$_cpr_pod" '$1 == sns && $2 == spod { next } { print }' \
528+
| pod_first_ready)
529+
530+
if [ -z "$_cpr_ref" ]; then
531+
{
532+
echo
533+
echo "################################################################"
534+
echo "# REFERENCE (Ready) pod on node=$_cpr_node: none found -- no in-run baseline available"
535+
echo "################################################################"
536+
} >> "$_cpr_of" 2>&1 || true
537+
return 0
538+
fi
539+
540+
_cpr_refns=$(printf '%s' "$_cpr_ref" | cut -d'|' -f1)
541+
_cpr_refpod=$(printf '%s' "$_cpr_ref" | cut -d'|' -f2)
542+
_cpr_refip=$(printf '%s' "$_cpr_ref" | cut -d'|' -f3)
543+
{
544+
echo
545+
capture_pod_dataplane "$_cpr_refns" "$_cpr_refpod" "$_cpr_refip" "$_cpr_node" "REFERENCE (Ready), for comparison with $_cpr_ns/$_cpr_pod"
546+
} >> "$_cpr_of" 2>&1 || true
547+
}
548+
377549
if [ -z "$affected" ]; then
378550
log "no scheduled NotReady pods -- skipping host->pod capture; checking LoadBalancers independently"
379551
else
@@ -410,84 +582,10 @@ else
410582
# ip rule, ovn0 addr) -- captured once per node.
411583
capture_node "$node"
412584

413-
# Pod-specific state.
585+
# Pod-specific state, plus the in-run healthy-pod reference for #3615.
414586
pf="$OUT/pod-$ns-$pod.txt"
415-
agent=$(pod_on_node "$CILIUM_NS" k8s-app=cilium "$node")
416-
ovs=$(pod_on_node "$KUBEOVN_NS" app=ovs "$node")
417-
{
418-
echo "################################################################"
419-
echo "# POD $ns/$pod podIP=${podip:-<none>} node=$node (Ready=$_ready)"
420-
echo "# node-global captures are in node-$node.txt"
421-
echo "################################################################"
422-
423-
echo
424-
echo "=== pod Ready conditions + recent probe events ==="
425-
kubectl get pod -n "$ns" "$pod" \
426-
-o jsonpath='{range .status.conditions[*]}{.type}={.status} reason={.reason}: {.message}{"\n"}{end}' 2>&1 || true
427-
kubectl get events -n "$ns" --field-selector "involvedObject.name=$pod" \
428-
-o jsonpath='{range .items[*]}{.lastTimestamp}{" "}{.reason}{": "}{.message}{"\n"}{end}' 2>&1 || true
429-
430-
if [ -z "$podip" ]; then
431-
echo
432-
echo "=== pod has no podIP; IP-specific route/conntrack capture skipped ==="
433-
echo "Node-global Cilium/OVN state and pod allocation events remain captured."
434-
fi
435-
436-
if [ -n "$podip" ] && [ -n "$agent" ]; then
437-
echo
438-
echo "=== cilium-dbg bpf ct list global | grep $podip (node=$node agent=$agent) ==="
439-
timeout 25 kubectl exec -n "$CILIUM_NS" "$agent" -c cilium-agent -- \
440-
sh -c "cilium-dbg bpf ct list global 2>/dev/null | grep -F '$podip' || echo 'no CT entries for $podip'" 2>&1 || true
441-
fi
442-
443-
cni=$(pod_on_node "$KUBEOVN_NS" app=kube-ovn-cni "$node")
444-
if [ -n "$podip" ] && [ -n "$cni" ]; then
445-
echo
446-
echo "=== host netns: ip route get $podip (via kube-ovn cni-server) ==="
447-
timeout 15 kubectl exec -n "$KUBEOVN_NS" "$cni" -c cni-server -- \
448-
ip route get "$podip" 2>&1 || true
449-
450-
echo
451-
echo "=== host netns: kernel conntrack for $podip ==="
452-
# Under enable-host-legacy-routing the host->local-pod path is in the
453-
# kernel netfilter conntrack table, NOT cilium BPF -- this is the
454-
# authoritative table for the transient. Prefer the conntrack CLI; fall
455-
# back to /proc/net/nf_conntrack when it is absent from the image.
456-
timeout 15 kubectl exec -n "$KUBEOVN_NS" "$cni" -c cni-server -- \
457-
sh -c "if command -v conntrack >/dev/null 2>&1; then conntrack -L 2>/dev/null | grep -F '$podip' || echo 'no conntrack entries for $podip'; else grep -F '$podip' /proc/net/nf_conntrack 2>/dev/null || echo 'no conntrack CLI; no /proc/net/nf_conntrack match for $podip'; fi" 2>&1 || true
458-
fi
459-
460-
if [ -n "$central" ]; then
461-
echo
462-
echo "=== OVN Port_Binding / Logical_Switch_Port for $pod.$ns ==="
463-
# kube-ovn names the OVN logical port <pod>.<namespace>.
464-
timeout 20 kubectl exec -n "$KUBEOVN_NS" "$central" -c ovn-central -- \
465-
ovn-sbctl --no-leader-only find port_binding "logical_port=$pod.$ns" 2>&1 || true
466-
timeout 20 kubectl exec -n "$KUBEOVN_NS" "$central" -c ovn-central -- \
467-
ovn-nbctl --no-leader-only find logical_switch_port "name=$pod.$ns" 2>&1 || true
468-
fi
469-
470-
if [ -n "$ovs" ]; then
471-
echo
472-
echo "=== OVS interface ovn-installed flag + ts for iface-id=$pod.$ns (ovs pod $ovs) ==="
473-
# kube-ovn stamps external_ids:ovn-installed (+ -ts) on the pod's OVS
474-
# interface once ovn-controller reports the port programmed -- this is
475-
# the barrier the CNI waits on. The OVS interface name is opaque, so
476-
# look it up by iface-id (<pod>.<ns>), then read the flag + timestamp.
477-
# An ovn-installed-ts set early (before physical_flow_output installed
478-
# the local-delivery flow, see node-$node.txt) is the I-P-lag signature.
479-
ovsif=$(timeout 20 kubectl exec -n "$KUBEOVN_NS" "$ovs" -c openvswitch -- \
480-
ovs-vsctl --no-heading --columns=name find interface "external_ids:iface-id=$pod.$ns" 2>/dev/null \
481-
| head -n 1 | tr -d '" ')
482-
if [ -n "$ovsif" ]; then
483-
echo "ovs interface = $ovsif"
484-
timeout 20 kubectl exec -n "$KUBEOVN_NS" "$ovs" -c openvswitch -- \
485-
ovs-vsctl get interface "$ovsif" external_ids:ovn-installed external_ids:ovn-installed-ts 2>&1 || true
486-
else
487-
echo "no OVS interface with external_ids:iface-id=$pod.$ns"
488-
fi
489-
fi
490-
} > "$pf" 2>&1 || true
587+
capture_pod_dataplane "$ns" "$pod" "$podip" "$node" "NotReady, Ready=$_ready" > "$pf" 2>&1 || true
588+
capture_pod_reference "$ns" "$pod" "$node" "$pf"
491589
done
492590
log "host->pod data-plane capture complete"
493591
}

0 commit comments

Comments
 (0)