|
35 | 35 | # kernel-path misforward / missing route / wrong rule / unresolved neighbor |
36 | 36 | # -- the actual mechanism behind host->local-pod "connection refused" on the |
37 | 37 | # legacy-routing path (the crux of the transient). |
| 38 | +# - Address attribution for a podIP that turns out to be a LOCAL address of |
| 39 | +# the host netns instead of reachable via ovn0 (a route to nowhere-in- |
| 40 | +# particular is not enough to name the owner): the same host netns above |
| 41 | +# also gets `ip -o addr show` (every interface) and `ip route show table |
| 42 | +# local`, and one Ready, non-hostNetwork pod on the same node gets the |
| 43 | +# identical per-pod capture below as an in-run healthy-pod control, since |
| 44 | +# this collector otherwise inspects NotReady pods only and had no |
| 45 | +# same-window baseline. hostNetwork is excluded because its podIP IS the |
| 46 | +# node's own address, so it would show the same local-route fingerprint |
| 47 | +# as the bug for an unrelated reason. |
38 | 48 | # - OVS/OVN on the node: `ovs-ofctl dump-flows br-int`, and the OVN |
39 | 49 | # Port_Binding / Logical_Switch_Port for the pod (LSP name = <pod>.<ns> in |
40 | 50 | # kube-ovn) plus a bounded `ovn-sbctl lflow-list`. |
@@ -211,6 +221,24 @@ pod_filter_affected() { |
211 | 221 | awk -F'|' '$4 != "" && $5 != "True" && $6 != "Succeeded" && $6 != "Failed"' |
212 | 222 | } |
213 | 223 |
|
| 224 | +# pod_first_ready: stdin = `namespace|name|podIP|ready|phase|hostNetwork` rows, |
| 225 | +# already scoped to one node by the caller's kubectl field-selector and with |
| 226 | +# the affected pod itself excluded. Emits `namespace|name|podIP` for the first |
| 227 | +# pod that is Ready=True, Running, and NOT hostNetwork -- the in-run |
| 228 | +# healthy-pod control for that node (see the address-attribution addition in |
| 229 | +# the header block) -- then stops. hostNetwork pods are excluded because their |
| 230 | +# podIP IS the node's own address: cilium-agent / kube-ovn-cni / ovs-ovn run |
| 231 | +# hostNetwork on every node, so without this exclusion the "healthy" reference |
| 232 | +# could legitimately resolve `ip route get <ip>` to `local <ip> dev lo table |
| 233 | +# local` -- the exact fingerprint #3615 is trying to distinguish from the |
| 234 | +# actual bug -- and silently defeat the whole point of the control. A blank |
| 235 | +# hostNetwork column counts as NOT hostNetwork (the API omits the field when |
| 236 | +# false); only the literal string "true" excludes. Empty output means no |
| 237 | +# eligible Ready pod exists on the node. |
| 238 | +pod_first_ready() { |
| 239 | + awk -F'|' '$4 == "True" && $5 == "Running" && $6 != "true" { print $1 "|" $2 "|" $3; exit }' |
| 240 | +} |
| 241 | + |
214 | 242 | # Sourcing guard: hack/capture-dataplane.bats sets E2E_CAPTURE_DATAPLANE_LIB and |
215 | 243 | # sources this file purely to reach the helpers above; return before touching $1 |
216 | 244 | # or running any capture so the unit test never needs a cluster. The script's |
@@ -367,13 +395,157 @@ capture_node() { |
367 | 395 | echo "=== host netns: ip addr show ovn0 ===" |
368 | 396 | timeout 15 kubectl exec -n "$KUBEOVN_NS" "$cni" -c cni-server -- \ |
369 | 397 | ip addr show ovn0 2>&1 || true |
| 398 | + |
| 399 | + echo |
| 400 | + echo "=== host netns: ip -o addr show, all interfaces (address attribution) ===" |
| 401 | + # ovn0 alone cannot name the owner when a podIP turns out to be a LOCAL |
| 402 | + # address instead: the interface actually holding it might be anything on |
| 403 | + # the host (see the header block). This is the same info scoped to every |
| 404 | + # interface, not just ovn0. |
| 405 | + timeout 15 kubectl exec -n "$KUBEOVN_NS" "$cni" -c cni-server -- \ |
| 406 | + ip -o addr show 2>&1 || true |
| 407 | + |
| 408 | + echo |
| 409 | + echo "=== host netns: ip route show table local (address attribution) ===" |
| 410 | + # A podIP resolving to `local <ip> dev lo table local` here is the |
| 411 | + # fingerprint from the header block: it wins over the ovn0 route in |
| 412 | + # `main` because table local is consulted first (see `ip rule` above). |
| 413 | + timeout 15 kubectl exec -n "$KUBEOVN_NS" "$cni" -c cni-server -- \ |
| 414 | + ip route show table local 2>&1 || true |
370 | 415 | else |
371 | 416 | echo |
372 | 417 | echo "(no kube-ovn-cni pod found on node $node -- host netns capture skipped)" |
373 | 418 | fi |
374 | 419 | } >> "$nf" 2>&1 || true |
375 | 420 | } |
376 | 421 |
|
| 422 | +# capture_pod_dataplane <ns> <pod> <podip> <node> <label> -- the pod-specific |
| 423 | +# captures (Ready conditions/events, cilium CT, host route, kernel conntrack, |
| 424 | +# OVN port binding, OVS ovn-installed) for ONE pod on <node>. <label> is |
| 425 | +# stamped in the section header: "NotReady" for an affected pod, "REFERENCE |
| 426 | +# (Ready)" for the in-run healthy-pod control added for the address- |
| 427 | +# attribution gap in the header block. Writes to stdout; the caller redirects |
| 428 | +# into the pod's output file. $central (the ovn-central pod, or empty) is read |
| 429 | +# from the caller's scope rather than threaded as a parameter, matching how |
| 430 | +# the rest of this script shares node-global lookups. |
| 431 | +capture_pod_dataplane() { |
| 432 | + _cpd_ns=$1; _cpd_pod=$2; _cpd_ip=$3; _cpd_node=$4; _cpd_label=$5 |
| 433 | + |
| 434 | + _cpd_agent=$(pod_on_node "$CILIUM_NS" k8s-app=cilium "$_cpd_node") |
| 435 | + _cpd_ovs=$(pod_on_node "$KUBEOVN_NS" app=ovs "$_cpd_node") |
| 436 | + _cpd_cni=$(pod_on_node "$KUBEOVN_NS" app=kube-ovn-cni "$_cpd_node") |
| 437 | + |
| 438 | + echo "################################################################" |
| 439 | + echo "# POD $_cpd_ns/$_cpd_pod podIP=${_cpd_ip:-<none>} node=$_cpd_node ($_cpd_label)" |
| 440 | + echo "# node-global captures are in node-$_cpd_node.txt" |
| 441 | + echo "################################################################" |
| 442 | + |
| 443 | + echo |
| 444 | + echo "=== pod Ready conditions + recent probe events ===" |
| 445 | + kubectl get pod -n "$_cpd_ns" "$_cpd_pod" \ |
| 446 | + -o jsonpath='{range .status.conditions[*]}{.type}={.status} reason={.reason}: {.message}{"\n"}{end}' 2>&1 || true |
| 447 | + kubectl get events -n "$_cpd_ns" --field-selector "involvedObject.name=$_cpd_pod" \ |
| 448 | + -o jsonpath='{range .items[*]}{.lastTimestamp}{" "}{.reason}{": "}{.message}{"\n"}{end}' 2>&1 || true |
| 449 | + |
| 450 | + if [ -z "$_cpd_ip" ]; then |
| 451 | + echo |
| 452 | + echo "=== pod has no podIP; IP-specific route/conntrack capture skipped ===" |
| 453 | + echo "Node-global Cilium/OVN state and pod allocation events remain captured." |
| 454 | + fi |
| 455 | + |
| 456 | + if [ -n "$_cpd_ip" ] && [ -n "$_cpd_agent" ]; then |
| 457 | + echo |
| 458 | + echo "=== cilium-dbg bpf ct list global | grep $_cpd_ip (node=$_cpd_node agent=$_cpd_agent) ===" |
| 459 | + timeout 25 kubectl exec -n "$CILIUM_NS" "$_cpd_agent" -c cilium-agent -- \ |
| 460 | + sh -c "cilium-dbg bpf ct list global 2>/dev/null | grep -F '$_cpd_ip' || echo 'no CT entries for $_cpd_ip'" 2>&1 || true |
| 461 | + fi |
| 462 | + |
| 463 | + if [ -n "$_cpd_ip" ] && [ -n "$_cpd_cni" ]; then |
| 464 | + echo |
| 465 | + echo "=== host netns: ip route get $_cpd_ip (via kube-ovn cni-server) ===" |
| 466 | + timeout 15 kubectl exec -n "$KUBEOVN_NS" "$_cpd_cni" -c cni-server -- \ |
| 467 | + ip route get "$_cpd_ip" 2>&1 || true |
| 468 | + |
| 469 | + echo |
| 470 | + echo "=== host netns: kernel conntrack for $_cpd_ip ===" |
| 471 | + # Under enable-host-legacy-routing the host->local-pod path is in the |
| 472 | + # kernel netfilter conntrack table, NOT cilium BPF -- this is the |
| 473 | + # authoritative table for the transient. Prefer the conntrack CLI; fall |
| 474 | + # back to /proc/net/nf_conntrack when it is absent from the image. |
| 475 | + timeout 15 kubectl exec -n "$KUBEOVN_NS" "$_cpd_cni" -c cni-server -- \ |
| 476 | + sh -c "if command -v conntrack >/dev/null 2>&1; then conntrack -L 2>/dev/null | grep -F '$_cpd_ip' || echo 'no conntrack entries for $_cpd_ip'; else grep -F '$_cpd_ip' /proc/net/nf_conntrack 2>/dev/null || echo 'no conntrack CLI; no /proc/net/nf_conntrack match for $_cpd_ip'; fi" 2>&1 || true |
| 477 | + fi |
| 478 | + |
| 479 | + if [ -n "$central" ]; then |
| 480 | + echo |
| 481 | + echo "=== OVN Port_Binding / Logical_Switch_Port for $_cpd_pod.$_cpd_ns ===" |
| 482 | + # kube-ovn names the OVN logical port <pod>.<namespace>. |
| 483 | + timeout 20 kubectl exec -n "$KUBEOVN_NS" "$central" -c ovn-central -- \ |
| 484 | + ovn-sbctl --no-leader-only find port_binding "logical_port=$_cpd_pod.$_cpd_ns" 2>&1 || true |
| 485 | + timeout 20 kubectl exec -n "$KUBEOVN_NS" "$central" -c ovn-central -- \ |
| 486 | + ovn-nbctl --no-leader-only find logical_switch_port "name=$_cpd_pod.$_cpd_ns" 2>&1 || true |
| 487 | + fi |
| 488 | + |
| 489 | + if [ -n "$_cpd_ovs" ]; then |
| 490 | + echo |
| 491 | + echo "=== OVS interface ovn-installed flag + ts for iface-id=$_cpd_pod.$_cpd_ns (ovs pod $_cpd_ovs) ===" |
| 492 | + # kube-ovn stamps external_ids:ovn-installed (+ -ts) on the pod's OVS |
| 493 | + # interface once ovn-controller reports the port programmed -- this is |
| 494 | + # the barrier the CNI waits on. The OVS interface name is opaque, so |
| 495 | + # look it up by iface-id (<pod>.<ns>), then read the flag + timestamp. |
| 496 | + # An ovn-installed-ts set early (before physical_flow_output installed |
| 497 | + # the local-delivery flow, see node-$node.txt) is the I-P-lag signature. |
| 498 | + _cpd_ovsif=$(timeout 20 kubectl exec -n "$KUBEOVN_NS" "$_cpd_ovs" -c openvswitch -- \ |
| 499 | + ovs-vsctl --no-heading --columns=name find interface "external_ids:iface-id=$_cpd_pod.$_cpd_ns" 2>/dev/null \ |
| 500 | + | head -n 1 | tr -d '" ') |
| 501 | + if [ -n "$_cpd_ovsif" ]; then |
| 502 | + echo "ovs interface = $_cpd_ovsif" |
| 503 | + timeout 20 kubectl exec -n "$KUBEOVN_NS" "$_cpd_ovs" -c openvswitch -- \ |
| 504 | + ovs-vsctl get interface "$_cpd_ovsif" external_ids:ovn-installed external_ids:ovn-installed-ts 2>&1 || true |
| 505 | + else |
| 506 | + echo "no OVS interface with external_ids:iface-id=$_cpd_pod.$_cpd_ns" |
| 507 | + fi |
| 508 | + fi |
| 509 | +} |
| 510 | + |
| 511 | +# capture_pod_reference <ns> <pod> <node> <outfile> -- the in-run healthy-pod |
| 512 | +# control for #3615's address-attribution gap: this collector otherwise |
| 513 | +# inspects NotReady pods only, so there is no same-window baseline for what a |
| 514 | +# normal route/conntrack looks like on the SAME node. Picks one Ready, Running, |
| 515 | +# non-hostNetwork pod on <node> other than <ns>/<pod> and runs the identical |
| 516 | +# per-pod capture against it, appended to <outfile>. hostNetwork is excluded |
| 517 | +# by pod_first_ready (see its doc comment): a hostNetwork pod's podIP is the |
| 518 | +# node's own address, so it would "legitimately" show the same local-route |
| 519 | +# fingerprint as the bug this control exists to rule out. Honest about the |
| 520 | +# negative: when no eligible Ready pod exists on the node, that absence is |
| 521 | +# recorded too, not left silent. |
| 522 | +capture_pod_reference() { |
| 523 | + _cpr_ns=$1; _cpr_pod=$2; _cpr_node=$3; _cpr_of=$4 |
| 524 | + |
| 525 | + _cpr_ref=$(kubectl get pods -A --field-selector "spec.nodeName=$_cpr_node" \ |
| 526 | + -o jsonpath='{range .items[*]}{.metadata.namespace}{"|"}{.metadata.name}{"|"}{.status.podIP}{"|"}{.status.conditions[?(@.type=="Ready")].status}{"|"}{.status.phase}{"|"}{.spec.hostNetwork}{"\n"}{end}' \ |
| 527 | + 2>/dev/null | awk -F'|' -v sns="$_cpr_ns" -v spod="$_cpr_pod" '$1 == sns && $2 == spod { next } { print }' \ |
| 528 | + | pod_first_ready) |
| 529 | + |
| 530 | + if [ -z "$_cpr_ref" ]; then |
| 531 | + { |
| 532 | + echo |
| 533 | + echo "################################################################" |
| 534 | + echo "# REFERENCE (Ready) pod on node=$_cpr_node: none found -- no in-run baseline available" |
| 535 | + echo "################################################################" |
| 536 | + } >> "$_cpr_of" 2>&1 || true |
| 537 | + return 0 |
| 538 | + fi |
| 539 | + |
| 540 | + _cpr_refns=$(printf '%s' "$_cpr_ref" | cut -d'|' -f1) |
| 541 | + _cpr_refpod=$(printf '%s' "$_cpr_ref" | cut -d'|' -f2) |
| 542 | + _cpr_refip=$(printf '%s' "$_cpr_ref" | cut -d'|' -f3) |
| 543 | + { |
| 544 | + echo |
| 545 | + capture_pod_dataplane "$_cpr_refns" "$_cpr_refpod" "$_cpr_refip" "$_cpr_node" "REFERENCE (Ready), for comparison with $_cpr_ns/$_cpr_pod" |
| 546 | + } >> "$_cpr_of" 2>&1 || true |
| 547 | +} |
| 548 | + |
377 | 549 | if [ -z "$affected" ]; then |
378 | 550 | log "no scheduled NotReady pods -- skipping host->pod capture; checking LoadBalancers independently" |
379 | 551 | else |
@@ -410,84 +582,10 @@ else |
410 | 582 | # ip rule, ovn0 addr) -- captured once per node. |
411 | 583 | capture_node "$node" |
412 | 584 |
|
413 | | - # Pod-specific state. |
| 585 | + # Pod-specific state, plus the in-run healthy-pod reference for #3615. |
414 | 586 | pf="$OUT/pod-$ns-$pod.txt" |
415 | | - agent=$(pod_on_node "$CILIUM_NS" k8s-app=cilium "$node") |
416 | | - ovs=$(pod_on_node "$KUBEOVN_NS" app=ovs "$node") |
417 | | - { |
418 | | - echo "################################################################" |
419 | | - echo "# POD $ns/$pod podIP=${podip:-<none>} node=$node (Ready=$_ready)" |
420 | | - echo "# node-global captures are in node-$node.txt" |
421 | | - echo "################################################################" |
422 | | - |
423 | | - echo |
424 | | - echo "=== pod Ready conditions + recent probe events ===" |
425 | | - kubectl get pod -n "$ns" "$pod" \ |
426 | | - -o jsonpath='{range .status.conditions[*]}{.type}={.status} reason={.reason}: {.message}{"\n"}{end}' 2>&1 || true |
427 | | - kubectl get events -n "$ns" --field-selector "involvedObject.name=$pod" \ |
428 | | - -o jsonpath='{range .items[*]}{.lastTimestamp}{" "}{.reason}{": "}{.message}{"\n"}{end}' 2>&1 || true |
429 | | - |
430 | | - if [ -z "$podip" ]; then |
431 | | - echo |
432 | | - echo "=== pod has no podIP; IP-specific route/conntrack capture skipped ===" |
433 | | - echo "Node-global Cilium/OVN state and pod allocation events remain captured." |
434 | | - fi |
435 | | - |
436 | | - if [ -n "$podip" ] && [ -n "$agent" ]; then |
437 | | - echo |
438 | | - echo "=== cilium-dbg bpf ct list global | grep $podip (node=$node agent=$agent) ===" |
439 | | - timeout 25 kubectl exec -n "$CILIUM_NS" "$agent" -c cilium-agent -- \ |
440 | | - sh -c "cilium-dbg bpf ct list global 2>/dev/null | grep -F '$podip' || echo 'no CT entries for $podip'" 2>&1 || true |
441 | | - fi |
442 | | - |
443 | | - cni=$(pod_on_node "$KUBEOVN_NS" app=kube-ovn-cni "$node") |
444 | | - if [ -n "$podip" ] && [ -n "$cni" ]; then |
445 | | - echo |
446 | | - echo "=== host netns: ip route get $podip (via kube-ovn cni-server) ===" |
447 | | - timeout 15 kubectl exec -n "$KUBEOVN_NS" "$cni" -c cni-server -- \ |
448 | | - ip route get "$podip" 2>&1 || true |
449 | | - |
450 | | - echo |
451 | | - echo "=== host netns: kernel conntrack for $podip ===" |
452 | | - # Under enable-host-legacy-routing the host->local-pod path is in the |
453 | | - # kernel netfilter conntrack table, NOT cilium BPF -- this is the |
454 | | - # authoritative table for the transient. Prefer the conntrack CLI; fall |
455 | | - # back to /proc/net/nf_conntrack when it is absent from the image. |
456 | | - timeout 15 kubectl exec -n "$KUBEOVN_NS" "$cni" -c cni-server -- \ |
457 | | - sh -c "if command -v conntrack >/dev/null 2>&1; then conntrack -L 2>/dev/null | grep -F '$podip' || echo 'no conntrack entries for $podip'; else grep -F '$podip' /proc/net/nf_conntrack 2>/dev/null || echo 'no conntrack CLI; no /proc/net/nf_conntrack match for $podip'; fi" 2>&1 || true |
458 | | - fi |
459 | | - |
460 | | - if [ -n "$central" ]; then |
461 | | - echo |
462 | | - echo "=== OVN Port_Binding / Logical_Switch_Port for $pod.$ns ===" |
463 | | - # kube-ovn names the OVN logical port <pod>.<namespace>. |
464 | | - timeout 20 kubectl exec -n "$KUBEOVN_NS" "$central" -c ovn-central -- \ |
465 | | - ovn-sbctl --no-leader-only find port_binding "logical_port=$pod.$ns" 2>&1 || true |
466 | | - timeout 20 kubectl exec -n "$KUBEOVN_NS" "$central" -c ovn-central -- \ |
467 | | - ovn-nbctl --no-leader-only find logical_switch_port "name=$pod.$ns" 2>&1 || true |
468 | | - fi |
469 | | - |
470 | | - if [ -n "$ovs" ]; then |
471 | | - echo |
472 | | - echo "=== OVS interface ovn-installed flag + ts for iface-id=$pod.$ns (ovs pod $ovs) ===" |
473 | | - # kube-ovn stamps external_ids:ovn-installed (+ -ts) on the pod's OVS |
474 | | - # interface once ovn-controller reports the port programmed -- this is |
475 | | - # the barrier the CNI waits on. The OVS interface name is opaque, so |
476 | | - # look it up by iface-id (<pod>.<ns>), then read the flag + timestamp. |
477 | | - # An ovn-installed-ts set early (before physical_flow_output installed |
478 | | - # the local-delivery flow, see node-$node.txt) is the I-P-lag signature. |
479 | | - ovsif=$(timeout 20 kubectl exec -n "$KUBEOVN_NS" "$ovs" -c openvswitch -- \ |
480 | | - ovs-vsctl --no-heading --columns=name find interface "external_ids:iface-id=$pod.$ns" 2>/dev/null \ |
481 | | - | head -n 1 | tr -d '" ') |
482 | | - if [ -n "$ovsif" ]; then |
483 | | - echo "ovs interface = $ovsif" |
484 | | - timeout 20 kubectl exec -n "$KUBEOVN_NS" "$ovs" -c openvswitch -- \ |
485 | | - ovs-vsctl get interface "$ovsif" external_ids:ovn-installed external_ids:ovn-installed-ts 2>&1 || true |
486 | | - else |
487 | | - echo "no OVS interface with external_ids:iface-id=$pod.$ns" |
488 | | - fi |
489 | | - fi |
490 | | - } > "$pf" 2>&1 || true |
| 587 | + capture_pod_dataplane "$ns" "$pod" "$podip" "$node" "NotReady, Ready=$_ready" > "$pf" 2>&1 || true |
| 588 | + capture_pod_reference "$ns" "$pod" "$node" "$pf" |
491 | 589 | done |
492 | 590 | log "host->pod data-plane capture complete" |
493 | 591 | } |
|
0 commit comments