Skip to content

Commit 0e138c4

Browse files
test(site-router): add the two-VM e2e suite
Most of what site-router promises only shows on a booted gateway: the tunnel coming up through the LoadBalancer, the return route reaching new pods, the MSS clamp, and the negative cases, a source outside the declared networks and the management API over the tunnel, being dropped. The suite boots the app next to a plain VM that stands in for the remote site and drives both through a probe pod. On failure it collects the dataplane and the gateways' serial consoles, and e2e selection runs it when a change touches site-router. Assisted-by: LLM Signed-off-by: Myasnikov Daniil <[email protected]>
1 parent 6c0858f commit 0e138c4

12 files changed

Lines changed: 2260 additions & 41 deletions

‎hack/capture-dataplane.bats‎

Lines changed: 74 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,8 @@
1212
# - lb_announcer_node -- the speaker node of the most recent announce;
1313
# - lb_capture_decision -- capture when probes failed, skip when one
1414
# succeeded, unknown when none ran or none could.
15+
# - lb_probe_blocker -- the reasons a probe cannot be attempted at all,
16+
# decided from the Service row alone.
1517
# - pod_filter_affected -- retain scheduled NotReady pods even without IPs,
1618
# and only from whole rows.
1719
# - pod_first_ready -- pick the healthy-pod baseline for one node out
@@ -227,10 +229,10 @@ EOF
227229

228230
@test "lb_filter_services keeps only LoadBalancer rows that have an ingress IP" {
229231
rows="$(printf '%s\n' \
230-
'tenant|app|LoadBalancer|192.0.2.50|80|31000|Cluster' \
231-
'kube-system|kube-dns|ClusterIP||53||Cluster' \
232-
'tenant|pending|LoadBalancer||443|31443|Local' \
233-
'tenant|db|LoadBalancer|192.0.2.51|5432|31543|Local')"
232+
'tenant|app|LoadBalancer|192.0.2.50|80|TCP|31000|Cluster' \
233+
'kube-system|kube-dns|ClusterIP||53|UDP||Cluster' \
234+
'tenant|pending|LoadBalancer||443|TCP|31443|Local' \
235+
'tenant|db|LoadBalancer|192.0.2.51|5432|TCP|31543|Local')"
234236

235237
out="$(printf '%s\n' "$rows" | lb_filter_services)"
236238

@@ -249,10 +251,10 @@ EOF
249251
# behind still satisfies a filter that only looks at values: what remains of
250252
# `192.0.2.53` here is `192.0.2.5`, an address the cluster never had, and the
251253
# row carrying it would be probed and written up as a Service. The jsonpath
252-
# emits a fixed seven fields per Service, so a shorter row is a fragment by
254+
# emits a fixed eight fields per Service, so a shorter row is a fragment by
253255
# construction rather than by guess.
254256
rows="$(printf '%s\n' \
255-
'tenant|app|LoadBalancer|192.0.2.50|80|31000|Cluster' \
257+
'tenant|app|LoadBalancer|192.0.2.50|80|TCP|31000|Cluster' \
256258
'tenant|db|LoadBalancer|192.0.2.5')"
257259

258260
out="$(printf '%s\n' "$rows" | lb_filter_services)"
@@ -740,7 +742,7 @@ STUB
740742
#!/bin/sh
741743
case "$*" in
742744
*'get svc'*)
743-
echo 'tenant|web|LoadBalancer|192.0.2.10|80|30080|Cluster'
745+
echo 'tenant|web|LoadBalancer|192.0.2.10|80|TCP|30080|Cluster'
744746
exit 1 ;;
745747
esac
746748
exit 0
@@ -772,7 +774,7 @@ STUB
772774
for a in "$@"; do
773775
case $a in
774776
pods) exit 0 ;;
775-
svc) echo 'tenant|web|LoadBalancer|192.0.2.10|80|30080|Cluster'; exit 0 ;;
777+
svc) echo 'tenant|web|LoadBalancer|192.0.2.10|80|TCP|30080|Cluster'; exit 0 ;;
776778
# Complete rows AND a bound's status: the read answered in part.
777779
endpointslices) echo '10.0.0.1|node-a|tenant-test|wedged|true'; exit 124 ;;
778780
esac
@@ -1315,7 +1317,7 @@ STUB
13151317
for a in "$@"; do
13161318
case $a in
13171319
pods) echo 'tenant-test|wedged|10.0.0.1|node-a|False|Running||eol'; exit 0 ;;
1318-
svc) echo 'tenant|web|LoadBalancer|192.0.2.10|80|30080|Cluster'; exit 0 ;;
1320+
svc) echo 'tenant|web|LoadBalancer|192.0.2.10|80|TCP|30080|Cluster'; exit 0 ;;
13191321
endpointslices) echo '10.0.0.1|node-a|tenant-test|wedged|true'; exit 0 ;;
13201322
esac
13211323
done
@@ -1365,7 +1367,7 @@ STUB
13651367
for a in "$@"; do
13661368
case $a in
13671369
pods) echo 'tenant-test|wedged|10.0.0.1|node-a|False|Running||eol'; exit 0 ;;
1368-
svc) echo 'tenant|web|LoadBalancer|192.0.2.10|80|30080|Cluster'; exit 0 ;;
1370+
svc) echo 'tenant|web|LoadBalancer|192.0.2.10|80|TCP|30080|Cluster'; exit 0 ;;
13691371
endpointslices) echo '10.0.0.1|node-a|tenant-test|wedged|true'; exit 0 ;;
13701372
esac
13711373
done
@@ -1460,7 +1462,7 @@ STUB
14601462
for a in "$@"; do
14611463
case $a in
14621464
pods) exit 0 ;;
1463-
svc) echo 'tenant|web|LoadBalancer|192.0.2.10|80|30080|Cluster'; exit 0 ;;
1465+
svc) echo 'tenant|web|LoadBalancer|192.0.2.10|80|TCP|30080|Cluster'; exit 0 ;;
14641466
endpointslices) echo '10.0.0.1|node-a|tenant-test|wedged|true'; exit 0 ;;
14651467
esac
14661468
done
@@ -1501,7 +1503,7 @@ STUB
15011503
for a in "$@"; do
15021504
case $a in
15031505
pods) exit 0 ;;
1504-
svc) echo 'tenant|web|LoadBalancer|192.0.2.10|80|30080|Cluster'; exit 0 ;;
1506+
svc) echo 'tenant|web|LoadBalancer|192.0.2.10|80|TCP|30080|Cluster'; exit 0 ;;
15051507
endpointslices) echo 'Error from server (Forbidden): endpointslices is forbidden' >&2; exit 1 ;;
15061508
esac
15071509
done
@@ -1535,7 +1537,7 @@ STUB
15351537
for a in "$@"; do
15361538
case $a in
15371539
pods) exit 0 ;;
1538-
svc) echo 'tenant|web|LoadBalancer|192.0.2.10|80|30080|Cluster'; exit 0 ;;
1540+
svc) echo 'tenant|web|LoadBalancer|192.0.2.10|80|TCP|30080|Cluster'; exit 0 ;;
15391541
esac
15401542
done
15411543
# The endpointslice reads fall through to here: an empty list, answered.
@@ -1575,7 +1577,7 @@ STUB
15751577
for a in "$@"; do
15761578
case $a in
15771579
pods) exit 0 ;;
1578-
svc) echo 'tenant|web|LoadBalancer|192.0.2.10|80|30080|Cluster'; exit 0 ;;
1580+
svc) echo 'tenant|web|LoadBalancer|192.0.2.10|80|TCP|30080|Cluster'; exit 0 ;;
15791581
endpointslices) echo '10.0.0.1|node-a|tenant-test|wedged|true'; exit 0 ;;
15801582
esac
15811583
done
@@ -1631,7 +1633,7 @@ STUB
16311633
for a in "$@"; do
16321634
case $a in
16331635
pods) exit 0 ;;
1634-
svc) echo 'tenant|web|LoadBalancer|192.0.2.10|80|30080|Cluster'; exit 0 ;;
1636+
svc) echo 'tenant|web|LoadBalancer|192.0.2.10|80|TCP|30080|Cluster'; exit 0 ;;
16351637
# No ready endpoint: the Service is enumerated, its backend is not.
16361638
endpointslices) exit 0 ;;
16371639
# The announcer, so the probe has somewhere to run from and the capture is
@@ -1686,7 +1688,7 @@ STUB
16861688
for a in "$@"; do
16871689
case $a in
16881690
pods) exit 0 ;;
1689-
svc) echo 'tenant|web|LoadBalancer|192.0.2.10|80|30080|Cluster'; exit 0 ;;
1691+
svc) echo 'tenant|web|LoadBalancer|192.0.2.10|80|TCP|30080|Cluster'; exit 0 ;;
16901692
endpointslices) echo '10.0.0.1|node-a|tenant-test|wedged|true'; exit 0 ;;
16911693
esac
16921694
done
@@ -1719,7 +1721,7 @@ STUB
17191721
for a in "$@"; do
17201722
case $a in
17211723
pods) exit 0 ;;
1722-
svc) echo 'tenant|web|LoadBalancer|192.0.2.10|80|30080|Cluster'; exit 0 ;;
1724+
svc) echo 'tenant|web|LoadBalancer|192.0.2.10|80|TCP|30080|Cluster'; exit 0 ;;
17231725
endpointslices) echo '10.0.0.1|node-a|tenant-test|wedged|true'; exit 0 ;;
17241726
esac
17251727
done
@@ -1767,7 +1769,7 @@ STUB
17671769
for a in "$@"; do
17681770
case $a in
17691771
pods) exit 0 ;;
1770-
svc) echo 'tenant|web|LoadBalancer|192.0.2.10|80|30080|Cluster'; exit 0 ;;
1772+
svc) echo 'tenant|web|LoadBalancer|192.0.2.10|80|TCP|30080|Cluster'; exit 0 ;;
17711773
endpointslices) echo '10.0.0.1|node-a|tenant-test|wedged|true'; exit 0 ;;
17721774
esac
17731775
done
@@ -1806,7 +1808,7 @@ STUB
18061808
for a in "$@"; do
18071809
case $a in
18081810
pods) exit 0 ;;
1809-
svc) echo 'tenant|web|LoadBalancer|192.0.2.10|80|30080|Cluster'; exit 0 ;;
1811+
svc) echo 'tenant|web|LoadBalancer|192.0.2.10|80|TCP|30080|Cluster'; exit 0 ;;
18101812
esac
18111813
done
18121814
# Everything else answers empty: no endpointslices, so no endpoint node, and no
@@ -1836,7 +1838,7 @@ STUB
18361838
for a in "$@"; do
18371839
case $a in
18381840
pods) exit 0 ;;
1839-
svc) echo 'tenant|web|LoadBalancer|192.0.2.10||30080|Cluster'; exit 0 ;;
1841+
svc) echo 'tenant|web|LoadBalancer|192.0.2.10||TCP|30080|Cluster'; exit 0 ;;
18401842
endpointslices) echo '10.0.0.1|node-a|tenant-test|wedged|true'; exit 0 ;;
18411843
esac
18421844
done
@@ -1858,6 +1860,58 @@ STUB
18581860
rm -rf "$d"
18591861
}
18601862

1863+
@test "lb_probe_blocker names each reason a probe cannot be attempted" {
1864+
[ -z "$(lb_probe_blocker node-a 443 TCP)" ]
1865+
# An absent protocol is the shape a cut read leaves; the Service object always
1866+
# carries one, so it must not become a fourth verdict of its own.
1867+
[ -z "$(lb_probe_blocker node-a 443 '')" ]
1868+
case "$(lb_probe_blocker '' 443 TCP)" in *"nowhere to probe from"*) ;; *) false ;; esac
1869+
case "$(lb_probe_blocker node-a 0 TCP)" in *"no port to probe"*) ;; *) false ;; esac
1870+
case "$(lb_probe_blocker node-a 500 UDP)" in *"TCP connect"*) ;; *) false ;; esac
1871+
case "$(lb_probe_blocker node-a 132 SCTP)" in *"TCP connect"*) ;; *) false ;; esac
1872+
}
1873+
1874+
@test "a UDP LoadBalancer is recorded as unprobed rather than unreachable" {
1875+
# The probe is a TCP connect, so against a UDP Service it fails for a reason
1876+
# that says nothing about the datapath. site-router's tunnel LB is 500/4500
1877+
# UDP and took the heavy capture on every failed run, with the conntrack dump
1878+
# in the artifact showing the probe as `tcp SYN_SENT dport=500 [UNREPLIED]`.
1879+
d=$(mktemp -d)
1880+
mkdir -p "$d/bin"
1881+
cat >"$d/bin/kubectl" <<'STUB'
1882+
#!/bin/sh
1883+
for a in "$@"; do
1884+
case $a in
1885+
pods) exit 0 ;;
1886+
svc) echo 'tenant|tunnel|LoadBalancer|192.0.2.10|500|UDP|30500|Local'; exit 0 ;;
1887+
endpointslices) echo '10.0.0.1|node-a|tenant-test|gw|true'; exit 0 ;;
1888+
esac
1889+
done
1890+
case "$*" in
1891+
*'app=kube-ovn-cni'*) echo 'cni-abc'; exit 0 ;;
1892+
esac
1893+
exit 0
1894+
STUB
1895+
chmod +x "$d/bin/kubectl"
1896+
PATH="$d/bin:$PATH" timeout 60 "$SCRIPT" "$d/out" >"$d/log" 2>&1 || true
1897+
[ -f "$d/out/lb-tenant-tunnel.txt" ]
1898+
if grep -qi 'unreachable' "$d/out/lb-tenant-tunnel.txt"; then
1899+
echo "a UDP LB was written up as unreachable on the strength of a TCP probe:"
1900+
cat "$d/out/lb-tenant-tunnel.txt"
1901+
exit 1
1902+
fi
1903+
if grep -q -- '-- reachable, skipped' "$d/out/lb-tenant-tunnel.txt"; then
1904+
echo "a UDP LB was recorded as reachable without a probe behind it:"
1905+
cat "$d/out/lb-tenant-tunnel.txt"
1906+
exit 1
1907+
fi
1908+
grep -q 'TCP connect' "$d/out/lb-tenant-tunnel.txt"
1909+
# The protocol belongs in the header too: a reader who sees "unprobed" has to
1910+
# be able to tell an unprobed UDP service from a broken lookup.
1911+
grep -q 'port=500/UDP' "$d/out/lb-tenant-tunnel.txt"
1912+
rm -rf "$d"
1913+
}
1914+
18611915
@test "lb_capture_decision keeps failures visible when one probe could not run" {
18621916
# An unrun probe adds no reason to doubt the ones that failed, so the mix must
18631917
# not read as reachable. Collapsing it toward skip is what stamps the artifact

‎hack/e2e-capture-dataplane.sh‎

Lines changed: 53 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -88,7 +88,9 @@
8888
# - Probe the LB IP:port a few times from a host netns (the kube-ovn
8989
# cni-server, so the probe traverses the same host-sourced datapath the
9090
# flake breaks). Only an LB whose every probe FAILS gets the heavy capture
91-
# below, and that same probe is the traffic the tcpdumps observe.
91+
# below, and that same probe is the traffic the tcpdumps observe. The probe
92+
# is a TCP connect, so a Service whose first port is not TCP is unprobed
93+
# rather than probed with something that cannot answer for it.
9294
# - For a FAILING LB, on BOTH the announcer node and the endpoint node
9395
# (reusing pod_on_node), labelled by node + role (ANNOUNCER vs ENDPOINT):
9496
# * `cilium-dbg bpf lb list` grepped for the LB IP + nodePort -- did the
@@ -132,16 +134,21 @@ set -u
132134
# any runtime state. #
133135
# --------------------------------------------------------------------------- #
134136

135-
# lb_filter_services: stdin = `ns|name|type|lbip|port|nodePort|extPolicy` rows
136-
# (one Service per line, as emitted by the kubectl jsonpath in main). Emits only
137-
# the rows that are type=LoadBalancer AND carry a status ingress IP -- i.e. the
138-
# Services that actually have an external datapath to characterise.
137+
# lb_filter_services: stdin = `ns|name|type|lbip|port|protocol|nodePort|extPolicy`
138+
# rows (one Service per line, as emitted by the kubectl jsonpath in main). Emits
139+
# only the rows that are type=LoadBalancer AND carry a status ingress IP -- i.e.
140+
# the Services that actually have an external datapath to characterise.
139141
#
140-
# NF == 7 first, because a truncated list is exactly what a bounded or
142+
# The protocol sits before the two display-only columns rather than at the end,
143+
# because it decides whether a probe runs at all and the field-count check below
144+
# cannot see a cut inside the LAST value. Keeping a decision column away from
145+
# that blind spot is the whole reason for the position.
146+
#
147+
# NF == 8 first, because a truncated list is exactly what a bounded or
141148
# interrupted read leaves behind and a fragment passes a value-only test: an IP
142149
# cut mid-octet still looks like an IP, and the row it sits in would be probed
143150
# and written up as a Service that has no such address. The jsonpath emits a
144-
# fixed seven fields per record, so anything shorter is a cut rather than a
151+
# fixed eight fields per record, so anything shorter is a cut rather than a
145152
# Service. Dropping it loses nothing a reader needs -- the read's own note
146153
# already says the list did not finish.
147154
#
@@ -164,7 +171,37 @@ set -u
164171
# marker after it can. Either way the read's own cut-off note is what tells
165172
# the reader the list ended early.
166173
lb_filter_services() {
167-
awk -F'|' 'NF == 7 && $3 == "LoadBalancer" && $4 != "" { print }'
174+
awk -F'|' 'NF == 8 && $3 == "LoadBalancer" && $4 != "" { print }'
175+
}
176+
177+
# lb_probe_blocker <probenode> <port> <protocol>: emits the reason no probe can
178+
# be attempted against this Service, or nothing when one should run. Kept here
179+
# rather than inline at the call site so each reason is testable, and because the
180+
# set of them is what decides whether an LB is written up as unreachable or as
181+
# unprobed -- the one distinction this section exists to keep honest.
182+
#
183+
# A non-TCP port is in that set. The probe is a TCP connect (nc -z), so against a
184+
# UDP Service every attempt fails for a reason that says nothing about the
185+
# datapath, and the old code wrote that down as "UNREACHABLE -- capturing
186+
# announcer/endpoint datapath". cozystack/cozystack#3426's tunnel LB (500/UDP,
187+
# 4500/UDP) took the heavy capture on every failed run and the conntrack dump in
188+
# the artifact shows what was actually measured: `tcp SYN_SENT dport=500
189+
# [UNREPLIED]`. Probing UDP is not the alternative -- an unanswered UDP datagram
190+
# and a delivered one are the same observation without an application reply, so
191+
# there is nothing to tell them apart from here. Unprobed is the honest verdict.
192+
#
193+
# An empty protocol defaults to TCP rather than to a third outcome: the Service
194+
# object always carries one, so the only way to arrive here without it is a read
195+
# that stopped short, and that read files its own note.
196+
lb_probe_blocker() {
197+
_pb_node=$1; _pb_port=$2; _pb_proto=${3:-TCP}
198+
if [ -z "$_pb_node" ]; then
199+
echo "nowhere to probe from -- neither an announcer nor an endpoint node is known"
200+
elif [ "$_pb_port" = "0" ] || [ -z "$_pb_port" ]; then
201+
echo "the Service names no port to probe"
202+
elif [ "$_pb_proto" != "TCP" ]; then
203+
echo "its first port is $_pb_proto, and the probe here is a TCP connect -- nothing it could observe would be about this datapath"
204+
fi
168205
}
169206

170207
# lb_first_ready_endpoint: stdin = `ip|node|targetNs|targetName|ready` rows (one
@@ -1166,7 +1203,7 @@ capture_lb_node() {
11661203
capture_lb_datapath() {
11671204
# shellcheck disable=SC2086 # empty DP_LIST_BOUND must vanish, not become ""
11681205
_raw=$($DP_LIST_BOUND kubectl get svc -A \
1169-
-o jsonpath='{range .items[*]}{.metadata.namespace}{"|"}{.metadata.name}{"|"}{.spec.type}{"|"}{.status.loadBalancer.ingress[0].ip}{"|"}{.spec.ports[0].port}{"|"}{.spec.ports[0].nodePort}{"|"}{.spec.externalTrafficPolicy}{"\n"}{end}' \
1206+
-o jsonpath='{range .items[*]}{.metadata.namespace}{"|"}{.metadata.name}{"|"}{.spec.type}{"|"}{.status.loadBalancer.ingress[0].ip}{"|"}{.spec.ports[0].port}{"|"}{.spec.ports[0].protocol}{"|"}{.spec.ports[0].nodePort}{"|"}{.spec.externalTrafficPolicy}{"\n"}{end}' \
11701207
2>"${DP_ERR:-/dev/null}")
11711208
_raw_rc=$?
11721209
# Unconditional, like every other read's note: a list can fail AFTER emitting
@@ -1232,9 +1269,10 @@ capture_lb_datapath() {
12321269
# captured. The gate lives on the capture branch (lb_budget_ok), not here.
12331270
_captured=0
12341271
printf '%s\n' "$_lbs" | {
1235-
while IFS='|' read -r _ns _name _type _lbip _port _np _etp; do
1272+
while IFS='|' read -r _ns _name _type _lbip _port _proto _np _etp; do
12361273
[ -n "$_lbip" ] || continue
12371274
_lbport="${_port:-0}"
1275+
_lbproto="${_proto:-TCP}"
12381276

12391277
# EndpointSlice backend for this Service (first ready, addressed endpoint).
12401278
# The status is kept beside the value, as the per-node pod lookups do:
@@ -1297,7 +1335,7 @@ capture_lb_datapath() {
12971335
_of="$OUT/lb-$_ns-$_name.txt"
12981336
{
12991337
echo "################################################################"
1300-
echo "# LB $_ns/$_name ip=$_lbip port=$_lbport nodePort=${_np:-<none>} etp=${_etp:-<default>}"
1338+
echo "# LB $_ns/$_name ip=$_lbip port=$_lbport/$_lbproto nodePort=${_np:-<none>} etp=${_etp:-<default>}"
13011339
echo "# backend: ip=${_epip:-$_ep_absent} node=${_epnode:-$_ep_absent} pod=${_eptns:-$_ep_absent}/${_eptname:-$_ep_absent} targetPort=${_eptport:-$_eptport_absent}"
13021340
echo "# announcer node: ${_annode:-<unknown>}"
13031341
echo "################################################################"
@@ -1318,13 +1356,12 @@ capture_lb_datapath() {
13181356
# unattempted probe is what this section must never write down -- an image
13191357
# without nc/curl/wget alone would stamp every LB in the cluster reachable
13201358
# and skip the entire heavy capture, which reads as a healthy datapath.
1359+
# lb_probe_blocker holds the reasons that are decidable before a probe
1360+
# runs; the two below need the probe's own outcome set to tell apart.
13211361
_probenode="${_annode:-$_epnode}"
13221362
_decision=unknown
1323-
if [ -z "$_probenode" ]; then
1324-
_why="nowhere to probe from -- neither an announcer nor an endpoint node is known"
1325-
elif [ "$_lbport" = "0" ]; then
1326-
_why="the Service names no port to probe"
1327-
else
1363+
_why=$(lb_probe_blocker "$_probenode" "$_lbport" "$_lbproto")
1364+
if [ -z "$_why" ]; then
13281365
# Held in a variable rather than piped straight in, because the SHAPE of
13291366
# the outcome set is the only place the two routes to an unknown verdict
13301367
# stay apart, and the reason line has to name a cause that was actually

0 commit comments

Comments
 (0)