@@ -2142,7 +2142,7 @@ exit 0
21422142STUB
21432143 chmod +x " $d /bin/kubectl"
21442144 dp_cut_timeout " $d /bin"
2145- PATH=" $d /bin:$PATH " DP_CUT=' *bpf lb list*' timeout 90 " $SCRIPT " " $d /out" > " $d /log" 2>&1
2145+ timeout 90 env PATH=" $d /bin:$PATH " DP_CUT=' *bpf lb list*' " $SCRIPT " " $d /out" > " $d /log" 2>&1
21462146 f=" $d /out/node-node-a.txt"
21472147 # The marker sits under the service map's own header, before the next block.
21482148 block=$( awk ' /^=== cilium-dbg bpf lb list/ { on = 1; next } on && /^=== / { exit } on { print }' " $f " )
@@ -2223,11 +2223,232 @@ STUB
22232223 grep -q ' ^=== pod Ready conditions' " $f "
22242224 case " $block " in
22252225 * " (reading this pod's Ready conditions was cut off at the deadline: the wall-clock budget of its section ran out)" * ) : ;;
2226- * ) echo " the read the deadline cut does not name the budget:" ; printf ' %s\n' " $block " ; exit 1 ;;
2226+ * ) echo " the read the deadline cut does not name itself and the budget:" ; printf ' %s\n' " $block " ; exit 1 ;;
22272227 esac
22282228 case " $block " in
22292229 * ' cut off by its own' * ) echo " the block blames a bound of its own:" ; printf ' %s\n' " $block " ; exit 1 ;;
22302230 esac
2231+ # One line per cut: the conditions read is cut once, so the block says so
2232+ # once.
2233+ n=$( printf ' %s\n' " $block " | grep -c ' cut off' )
2234+ [ " $n " -eq 1 ] || { echo " the block reports one cut $n times:" ; printf ' %s\n' " $block " ; exit 1; }
2235+ rm -rf " $d "
2236+ }
2237+
2238+ @test " a read cut off by its own bound is reported once in its block" {
2239+ # The read's own bound fires; dp_run marks the cut in the block, and the
2240+ # read must not add a second line saying the same thing.
2241+ d=$( mktemp -d)
2242+ mkdir -p " $d /bin"
2243+ cat > " $d /bin/kubectl" << 'STUB '
2244+ #!/bin/sh
2245+ case "$*" in
2246+ 'get pods -A '*) echo 'tenant-test|wedged|10.0.0.1|node-a|False|Running||eol'; exit 0 ;;
2247+ esac
2248+ exit 0
2249+ STUB
2250+ chmod +x " $d /bin/kubectl"
2251+ dp_cut_timeout " $d /bin"
2252+ timeout 90 env PATH=" $d /bin:$PATH " DP_CUT=' *get pod -n tenant-test wedged *' " $SCRIPT " " $d /out" > " $d /log" 2>&1
2253+ f=" $d /out/pod-tenant-test_wedged.txt"
2254+ block=$( awk ' /^=== pod Ready conditions/ { on = 1; next } on && /^=== / { exit } on { print }' " $f " )
2255+ # Positive control: the block exists and carries the cut.
2256+ grep -q ' ^=== pod Ready conditions' " $f "
2257+ case " $block " in
2258+ * " (reading this pod's Ready conditions was cut off by its own 20s timeout)" * ) : ;;
2259+ * ) echo " the cut read is not marked by name:" ; printf ' %s\n' " $block " ; exit 1 ;;
2260+ esac
2261+ n=$( printf ' %s\n' " $block " | grep -c ' cut off' )
2262+ [ " $n " -eq 1 ] || { echo " the block reports one cut $n times:" ; printf ' %s\n' " $block " ; exit 1; }
2263+ rm -rf " $d "
2264+ }
2265+
2266+ @test " a cut events read is named as the events read" {
2267+ # Two reads share the Ready block, so a cut line has to say which one it is:
2268+ # conditions answered and events cut reads the same as conditions cut
2269+ # mid-output otherwise.
2270+ d=$( mktemp -d)
2271+ mkdir -p " $d /bin"
2272+ cat > " $d /bin/kubectl" << 'STUB '
2273+ #!/bin/sh
2274+ case "$*" in
2275+ 'get pods -A '*) echo 'tenant-test|wedged|10.0.0.1|node-a|False|Running||eol'; exit 0 ;;
2276+ 'get pod -n tenant-test wedged '*) echo 'Ready=False reason=ContainersNotReady: x'; exit 0 ;;
2277+ esac
2278+ exit 0
2279+ STUB
2280+ chmod +x " $d /bin/kubectl"
2281+ dp_cut_timeout " $d /bin"
2282+ timeout 90 env PATH=" $d /bin:$PATH " DP_CUT=' *get events -n tenant-test *' " $SCRIPT " " $d /out" > " $d /log" 2>&1
2283+ f=" $d /out/pod-tenant-test_wedged.txt"
2284+ block=$( awk ' /^=== pod Ready conditions/ { on = 1; next } on && /^=== / { exit } on { print }' " $f " )
2285+ # Positive control: the conditions read answered.
2286+ case " $block " in * ' Ready=False reason=ContainersNotReady' * ) : ;; * ) echo " no conditions:" ; printf ' %s\n' " $block " ; exit 1 ;; esac
2287+ case " $block " in
2288+ * " (reading this pod's events was cut off by its own 20s timeout)" * ) : ;;
2289+ * ) echo " the cut events read is not named:" ; printf ' %s\n' " $block " ; exit 1 ;;
2290+ esac
2291+ n=$( printf ' %s\n' " $block " | grep -c ' cut off' )
2292+ [ " $n " -eq 1 ] || { echo " the block reports one cut $n times:" ; printf ' %s\n' " $block " ; exit 1; }
2293+ rm -rf " $d "
2294+ }
2295+
2296+ @test " a pod read kubectl refused keeps kubectl's message in its block, once" {
2297+ # A refusal from the apiserver is not a cut: the block names the read and
2298+ # carries kubectl's own words, once.
2299+ d=$( mktemp -d)
2300+ mkdir -p " $d /bin"
2301+ cat > " $d /bin/kubectl" << 'STUB '
2302+ #!/bin/sh
2303+ case "$*" in
2304+ 'get pods -A '*) echo 'tenant-test|wedged|10.0.0.1|node-a|False|Running||eol'; exit 0 ;;
2305+ 'get pod -n tenant-test wedged '*) echo 'Error from server (Forbidden): pods "wedged" is forbidden' >&2; exit 1 ;;
2306+ esac
2307+ exit 0
2308+ STUB
2309+ chmod +x " $d /bin/kubectl"
2310+ timeout 90 env PATH=" $d /bin:$PATH " " $SCRIPT " " $d /out" > " $d /log" 2>&1
2311+ f=" $d /out/pod-tenant-test_wedged.txt"
2312+ block=$( awk ' /^=== pod Ready conditions/ { on = 1; next } on && /^=== / { exit } on { print }' " $f " )
2313+ case " $block " in
2314+ * " (reading this pod's Ready conditions failed: kubectl exited 1)" * ) : ;;
2315+ * ) echo " the refused read is not named:" ; printf ' %s\n' " $block " ; exit 1 ;;
2316+ esac
2317+ n=$( printf ' %s\n' " $block " | grep -c ' Forbidden' )
2318+ [ " $n " -eq 1 ] || { echo " kubectl's message appears $n times:" ; printf ' %s\n' " $block " ; exit 1; }
2319+ rm -rf " $d "
2320+ }
2321+
2322+ @test " a pod read kubectl refused keeps all of kubectl's stderr in its block" {
2323+ # A degraded apiserver throttles clients, and kubectl prints its throttling
2324+ # lines ahead of the error that matters. The block keeps kubectl's stderr as
2325+ # it came, not a flattened, shortened copy that can end before the reason.
2326+ d=$( mktemp -d)
2327+ mkdir -p " $d /bin"
2328+ cat > " $d /bin/kubectl" << 'STUB '
2329+ #!/bin/sh
2330+ case "$*" in
2331+ 'get pods -A '*) echo 'tenant-test|wedged|10.0.0.1|node-a|False|Running||eol'; exit 0 ;;
2332+ 'get pod -n tenant-test wedged '*)
2333+ for i in 1 2 3; do
2334+ echo 'I1001 12:00:00.000000 1 request.go:697] Waited for 1.19s due to client-side throttling, not priority and fairness, request: GET:https://192.0.2.1:6443/api/v1/namespaces/tenant-test/pods/wedged' >&2
2335+ done
2336+ echo 'Error from server (Forbidden): pods "wedged" is forbidden: THE-REAL-REASON' >&2
2337+ exit 1 ;;
2338+ esac
2339+ exit 0
2340+ STUB
2341+ chmod +x " $d /bin/kubectl"
2342+ timeout 90 env PATH=" $d /bin:$PATH " " $SCRIPT " " $d /out" > " $d /log" 2>&1
2343+ f=" $d /out/pod-tenant-test_wedged.txt"
2344+ block=$( awk ' /^=== pod Ready conditions/ { on = 1; next } on && /^=== / { exit } on { print }' " $f " )
2345+ # Positive control: the read failed and said so by name.
2346+ case " $block " in
2347+ * " (reading this pod's Ready conditions failed: kubectl exited 1)" * ) : ;;
2348+ * ) echo " the refused read is not named:" ; printf ' %s\n' " $block " ; exit 1 ;;
2349+ esac
2350+ n=$( printf ' %s\n' " $block " | grep -c ' THE-REAL-REASON' )
2351+ [ " $n " -eq 1 ] || { echo " kubectl's reason appears $n times:" ; printf ' %s\n' " $block " ; exit 1; }
2352+ rm -rf " $d "
2353+ }
2354+
2355+ @test " a pod read that answered with a warning keeps the warning in its block" {
2356+ # A read can succeed and still write to stderr -- a deprecation notice, a
2357+ # partial-result warning. That warning is part of what the read returned and
2358+ # stays beside it, with no failure line, since nothing failed.
2359+ d=$( mktemp -d)
2360+ mkdir -p " $d /bin"
2361+ cat > " $d /bin/kubectl" << 'STUB '
2362+ #!/bin/sh
2363+ case "$*" in
2364+ 'get pods -A '*) echo 'tenant-test|wedged|10.0.0.1|node-a|False|Running||eol'; exit 0 ;;
2365+ 'get pod -n tenant-test wedged '*)
2366+ echo 'Ready=False reason=ContainersNotReady: x'
2367+ echo 'Warning: THE-WARNING' >&2
2368+ exit 0 ;;
2369+ esac
2370+ exit 0
2371+ STUB
2372+ chmod +x " $d /bin/kubectl"
2373+ timeout 90 env PATH=" $d /bin:$PATH " " $SCRIPT " " $d /out" > " $d /log" 2>&1
2374+ f=" $d /out/pod-tenant-test_wedged.txt"
2375+ block=$( awk ' /^=== pod Ready conditions/ { on = 1; next } on && /^=== / { exit } on { print }' " $f " )
2376+ # Positive control: the read answered.
2377+ case " $block " in * ' Ready=False reason=ContainersNotReady' * ) : ;; * ) echo " no conditions:" ; printf ' %s\n' " $block " ; exit 1 ;; esac
2378+ n=$( printf ' %s\n' " $block " | grep -c ' THE-WARNING' )
2379+ [ " $n " -eq 1 ] || { echo " the warning appears $n times:" ; printf ' %s\n' " $block " ; exit 1; }
2380+ case " $block " in
2381+ * " (reading this pod's Ready conditions" * ) echo " a read that answered got a failure line:" ; printf ' %s\n' " $block " ; exit 1 ;;
2382+ esac
2383+ rm -rf " $d "
2384+ }
2385+
2386+ @test " a pod read kubectl refused keeps kubectl's message even without a scratch file" {
2387+ # The reads send stderr to a scratch file, which mktemp may fail to create.
2388+ # kubectl's message must then still reach the block, once.
2389+ d=$( mktemp -d)
2390+ mkdir -p " $d /bin"
2391+ cat > " $d /bin/kubectl" << 'STUB '
2392+ #!/bin/sh
2393+ case "$*" in
2394+ 'get pods -A '*) echo 'tenant-test|wedged|10.0.0.1|node-a|False|Running||eol'; exit 0 ;;
2395+ 'get pod -n tenant-test wedged '*) echo 'Error from server (Forbidden): pods "wedged" is forbidden' >&2; exit 1 ;;
2396+ esac
2397+ exit 0
2398+ STUB
2399+ chmod +x " $d /bin/kubectl"
2400+ timeout 90 env TMPDIR=" $d /missing" PATH=" $d /bin:$PATH " " $SCRIPT " " $d /out" > " $d /log" 2>&1
2401+ f=" $d /out/pod-tenant-test_wedged.txt"
2402+ block=$( awk ' /^=== pod Ready conditions/ { on = 1; next } on && /^=== / { exit } on { print }' " $f " )
2403+ # Positive control: the read failed and said so by name.
2404+ case " $block " in
2405+ * " (reading this pod's Ready conditions failed: kubectl exited 1" * ) : ;;
2406+ * ) echo " the refused read is not named:" ; printf ' %s\n' " $block " ; exit 1 ;;
2407+ esac
2408+ n=$( printf ' %s\n' " $block " | grep -c ' Forbidden' )
2409+ [ " $n " -eq 1 ] || { echo " kubectl's message appears $n times:" ; printf ' %s\n' " $block " ; exit 1; }
2410+ rm -rf " $d "
2411+ }
2412+
2413+ @test " an OVS interface lookup the deadline cut names the budget on both paths" {
2414+ # The per-pod ovn-installed lookup and the LB endpoint lookup both read their
2415+ # own status. A cut at the deadline has to name the budget there too, not
2416+ # report the raw status of the call.
2417+ d=$( mktemp -d)
2418+ mkdir -p " $d /bin"
2419+ cat > " $d /bin/kubectl" << 'STUB '
2420+ #!/bin/sh
2421+ case "$*" in
2422+ 'get pods -A '*) [ -n "${STUB_NO_POD:-}" ] && exit 0; echo 'tenant-test|wedged|10.0.0.1|node-a|False|Running||eol'; exit 0 ;;
2423+ 'get svc -A '*) [ -n "${STUB_NO_POD:-}" ] || exit 0; echo 'tenant|web|LoadBalancer|192.0.2.10|80|30080|Cluster'; exit 0 ;;
2424+ *endpointslices*) echo '10.0.0.1|node-a|tenant-test|wedged|true'; exit 0 ;;
2425+ *'app=kube-ovn-cni'*) [ -n "${STUB_NO_POD:-}" ] || exit 0; echo 'cni-abc'; exit 0 ;;
2426+ *'app=ovs'*) echo 'ovs-xyz'; exit 0 ;;
2427+ *'nc -z'*) echo fail; exit 0 ;;
2428+ *'find interface'*) sleep 300 ;;
2429+ esac
2430+ exit 0
2431+ STUB
2432+ chmod +x " $d /bin/kubectl"
2433+ real_timeout=$( command -v timeout)
2434+ dp_cut_timeout " $d /bin"
2435+ # Per-pod path: the pod section's 3s run into the lookup.
2436+ PATH=" $d /bin:$PATH " DP_REAL_TIMEOUT=" $real_timeout " DP_PASS=" $( printf ' %s\n' ' *kubectl get pods -A *' ' *kubectl get svc -A *' ) " \
2437+ COZY_DATAPLANE_BUDGET=6 COZY_DATAPLANE_LB_RESERVE=3 \
2438+ " $real_timeout " 60 " $SCRIPT " " $d /pod" > " $d /log" 2>&1
2439+ f=" $d /pod/pod-tenant-test_wedged.txt"
2440+ grep -q ' the OVS interface lookup for iface-id=wedged.tenant-test was cut off at the deadline: the pod section ran out of its share' " $f "
2441+ # LB path: the LB section's 4s run into the endpoint lookup.
2442+ STUB_NO_POD=1 PATH=" $d /bin:$PATH " DP_REAL_TIMEOUT=" $real_timeout " DP_PASS=" $( printf ' %s\n' ' *kubectl get pods -A *' ' *kubectl get svc -A *' ) " \
2443+ COZY_DATAPLANE_BUDGET=4 COZY_DATAPLANE_LB_RESERVE=4 \
2444+ " $real_timeout " 60 " $SCRIPT " " $d /lb" >> " $d /log" 2>&1
2445+ g=" $d /lb/lb-tenant_web.txt"
2446+ grep -q ' the OVS interface lookup for iface-id=wedged.tenant-test on node node-a was cut off at the deadline: the LoadBalancer section ran out of its share' " $g "
2447+ if grep -q ' the lookup exited' " $f " " $g " ; then
2448+ echo " a deadline cut was reported as a raw status:"
2449+ grep ' the lookup exited' " $f " " $g "
2450+ exit 1
2451+ fi
22312452 rm -rf " $d "
22322453}
22332454
@@ -3238,7 +3459,7 @@ exit 0
32383459STUB
32393460 chmod +x " $d /bin/kubectl"
32403461 dp_cut_timeout " $d /bin"
3241- PATH=" $d /bin:$PATH " DP_CUT=' *app=ovs*' timeout 90 " $SCRIPT " " $d /cut" > " $d /log" 2>&1
3462+ timeout 90 env PATH=" $d /bin:$PATH " DP_CUT=' *app=ovs*' " $SCRIPT " " $d /cut" > " $d /log" 2>&1
32423463 f=" $d /cut/lb-tenant_web.txt"
32433464 # Positive control: the endpoint side ran on the fallback.
32443465 grep -q ' iface=genev_sys_6081' " $d /cut/lb-tenant_web.tcpdump-endpoint.txt"
0 commit comments