Skip to content

Commit 0cc3749

Browse files
committed
fix(e2e): report a dataplane pod read once and by name in its block
The Ready-conditions and events reads share one block and added a line of their own when they failed, on top of what their stderr put into the block. Once dp_run marked every cut and refusal there, a cut read carried two lines for one event, and before the budget status existed the second named a timeout the first said the deadline had chosen. The two reads now send stderr to the scratch file and, after their output, write kubectl's stderr into the block whole, without dp_run's marker, plus one line naming the read when it failed. kubectl's message is kept as it came, since an error can follow lines of throttling notices on a degraded apiserver. Without a scratch file their stderr goes into the block directly. The probe comments also say what the "spent" outcome now covers. Assisted-by: LLM Signed-off-by: Aleksei Sviridkin <[email protected]>
1 parent 0b958b1 commit 0cc3749

2 files changed

Lines changed: 263 additions & 18 deletions

File tree

‎hack/capture-dataplane.bats‎

Lines changed: 224 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2142,7 +2142,7 @@ exit 0
21422142
STUB
21432143
chmod +x "$d/bin/kubectl"
21442144
dp_cut_timeout "$d/bin"
2145-
PATH="$d/bin:$PATH" DP_CUT='*bpf lb list*' timeout 90 "$SCRIPT" "$d/out" >"$d/log" 2>&1
2145+
timeout 90 env PATH="$d/bin:$PATH" DP_CUT='*bpf lb list*' "$SCRIPT" "$d/out" >"$d/log" 2>&1
21462146
f="$d/out/node-node-a.txt"
21472147
# The marker sits under the service map's own header, before the next block.
21482148
block=$(awk '/^=== cilium-dbg bpf lb list/ { on = 1; next } on && /^=== / { exit } on { print }' "$f")
@@ -2223,11 +2223,232 @@ STUB
22232223
grep -q '^=== pod Ready conditions' "$f"
22242224
case "$block" in
22252225
*"(reading this pod's Ready conditions was cut off at the deadline: the wall-clock budget of its section ran out)"*) : ;;
2226-
*) echo "the read the deadline cut does not name the budget:"; printf '%s\n' "$block"; exit 1 ;;
2226+
*) echo "the read the deadline cut does not name itself and the budget:"; printf '%s\n' "$block"; exit 1 ;;
22272227
esac
22282228
case "$block" in
22292229
*'cut off by its own'*) echo "the block blames a bound of its own:"; printf '%s\n' "$block"; exit 1 ;;
22302230
esac
2231+
# One line per cut: the conditions read is cut once, so the block says so
2232+
# once.
2233+
n=$(printf '%s\n' "$block" | grep -c 'cut off')
2234+
[ "$n" -eq 1 ] || { echo "the block reports one cut $n times:"; printf '%s\n' "$block"; exit 1; }
2235+
rm -rf "$d"
2236+
}
2237+
2238+
@test "a read cut off by its own bound is reported once in its block" {
2239+
# The read's own bound fires; dp_run marks the cut in the block, and the
2240+
# read must not add a second line saying the same thing.
2241+
d=$(mktemp -d)
2242+
mkdir -p "$d/bin"
2243+
cat >"$d/bin/kubectl" <<'STUB'
2244+
#!/bin/sh
2245+
case "$*" in
2246+
'get pods -A '*) echo 'tenant-test|wedged|10.0.0.1|node-a|False|Running||eol'; exit 0 ;;
2247+
esac
2248+
exit 0
2249+
STUB
2250+
chmod +x "$d/bin/kubectl"
2251+
dp_cut_timeout "$d/bin"
2252+
timeout 90 env PATH="$d/bin:$PATH" DP_CUT='*get pod -n tenant-test wedged *' "$SCRIPT" "$d/out" >"$d/log" 2>&1
2253+
f="$d/out/pod-tenant-test_wedged.txt"
2254+
block=$(awk '/^=== pod Ready conditions/ { on = 1; next } on && /^=== / { exit } on { print }' "$f")
2255+
# Positive control: the block exists and carries the cut.
2256+
grep -q '^=== pod Ready conditions' "$f"
2257+
case "$block" in
2258+
*"(reading this pod's Ready conditions was cut off by its own 20s timeout)"*) : ;;
2259+
*) echo "the cut read is not marked by name:"; printf '%s\n' "$block"; exit 1 ;;
2260+
esac
2261+
n=$(printf '%s\n' "$block" | grep -c 'cut off')
2262+
[ "$n" -eq 1 ] || { echo "the block reports one cut $n times:"; printf '%s\n' "$block"; exit 1; }
2263+
rm -rf "$d"
2264+
}
2265+
2266+
@test "a cut events read is named as the events read" {
2267+
# Two reads share the Ready block, so a cut line has to say which one it is:
2268+
# conditions answered and events cut reads the same as conditions cut
2269+
# mid-output otherwise.
2270+
d=$(mktemp -d)
2271+
mkdir -p "$d/bin"
2272+
cat >"$d/bin/kubectl" <<'STUB'
2273+
#!/bin/sh
2274+
case "$*" in
2275+
'get pods -A '*) echo 'tenant-test|wedged|10.0.0.1|node-a|False|Running||eol'; exit 0 ;;
2276+
'get pod -n tenant-test wedged '*) echo 'Ready=False reason=ContainersNotReady: x'; exit 0 ;;
2277+
esac
2278+
exit 0
2279+
STUB
2280+
chmod +x "$d/bin/kubectl"
2281+
dp_cut_timeout "$d/bin"
2282+
timeout 90 env PATH="$d/bin:$PATH" DP_CUT='*get events -n tenant-test *' "$SCRIPT" "$d/out" >"$d/log" 2>&1
2283+
f="$d/out/pod-tenant-test_wedged.txt"
2284+
block=$(awk '/^=== pod Ready conditions/ { on = 1; next } on && /^=== / { exit } on { print }' "$f")
2285+
# Positive control: the conditions read answered.
2286+
case "$block" in *'Ready=False reason=ContainersNotReady'*) : ;; *) echo "no conditions:"; printf '%s\n' "$block"; exit 1 ;; esac
2287+
case "$block" in
2288+
*"(reading this pod's events was cut off by its own 20s timeout)"*) : ;;
2289+
*) echo "the cut events read is not named:"; printf '%s\n' "$block"; exit 1 ;;
2290+
esac
2291+
n=$(printf '%s\n' "$block" | grep -c 'cut off')
2292+
[ "$n" -eq 1 ] || { echo "the block reports one cut $n times:"; printf '%s\n' "$block"; exit 1; }
2293+
rm -rf "$d"
2294+
}
2295+
2296+
@test "a pod read kubectl refused keeps kubectl's message in its block, once" {
2297+
# A refusal from the apiserver is not a cut: the block names the read and
2298+
# carries kubectl's own words, once.
2299+
d=$(mktemp -d)
2300+
mkdir -p "$d/bin"
2301+
cat >"$d/bin/kubectl" <<'STUB'
2302+
#!/bin/sh
2303+
case "$*" in
2304+
'get pods -A '*) echo 'tenant-test|wedged|10.0.0.1|node-a|False|Running||eol'; exit 0 ;;
2305+
'get pod -n tenant-test wedged '*) echo 'Error from server (Forbidden): pods "wedged" is forbidden' >&2; exit 1 ;;
2306+
esac
2307+
exit 0
2308+
STUB
2309+
chmod +x "$d/bin/kubectl"
2310+
timeout 90 env PATH="$d/bin:$PATH" "$SCRIPT" "$d/out" >"$d/log" 2>&1
2311+
f="$d/out/pod-tenant-test_wedged.txt"
2312+
block=$(awk '/^=== pod Ready conditions/ { on = 1; next } on && /^=== / { exit } on { print }' "$f")
2313+
case "$block" in
2314+
*"(reading this pod's Ready conditions failed: kubectl exited 1)"*) : ;;
2315+
*) echo "the refused read is not named:"; printf '%s\n' "$block"; exit 1 ;;
2316+
esac
2317+
n=$(printf '%s\n' "$block" | grep -c 'Forbidden')
2318+
[ "$n" -eq 1 ] || { echo "kubectl's message appears $n times:"; printf '%s\n' "$block"; exit 1; }
2319+
rm -rf "$d"
2320+
}
2321+
2322+
@test "a pod read kubectl refused keeps all of kubectl's stderr in its block" {
2323+
# A degraded apiserver throttles clients, and kubectl prints its throttling
2324+
# lines ahead of the error that matters. The block keeps kubectl's stderr as
2325+
# it came, not a flattened, shortened copy that can end before the reason.
2326+
d=$(mktemp -d)
2327+
mkdir -p "$d/bin"
2328+
cat >"$d/bin/kubectl" <<'STUB'
2329+
#!/bin/sh
2330+
case "$*" in
2331+
'get pods -A '*) echo 'tenant-test|wedged|10.0.0.1|node-a|False|Running||eol'; exit 0 ;;
2332+
'get pod -n tenant-test wedged '*)
2333+
for i in 1 2 3; do
2334+
echo 'I1001 12:00:00.000000 1 request.go:697] Waited for 1.19s due to client-side throttling, not priority and fairness, request: GET:https://192.0.2.1:6443/api/v1/namespaces/tenant-test/pods/wedged' >&2
2335+
done
2336+
echo 'Error from server (Forbidden): pods "wedged" is forbidden: THE-REAL-REASON' >&2
2337+
exit 1 ;;
2338+
esac
2339+
exit 0
2340+
STUB
2341+
chmod +x "$d/bin/kubectl"
2342+
timeout 90 env PATH="$d/bin:$PATH" "$SCRIPT" "$d/out" >"$d/log" 2>&1
2343+
f="$d/out/pod-tenant-test_wedged.txt"
2344+
block=$(awk '/^=== pod Ready conditions/ { on = 1; next } on && /^=== / { exit } on { print }' "$f")
2345+
# Positive control: the read failed and said so by name.
2346+
case "$block" in
2347+
*"(reading this pod's Ready conditions failed: kubectl exited 1)"*) : ;;
2348+
*) echo "the refused read is not named:"; printf '%s\n' "$block"; exit 1 ;;
2349+
esac
2350+
n=$(printf '%s\n' "$block" | grep -c 'THE-REAL-REASON')
2351+
[ "$n" -eq 1 ] || { echo "kubectl's reason appears $n times:"; printf '%s\n' "$block"; exit 1; }
2352+
rm -rf "$d"
2353+
}
2354+
2355+
@test "a pod read that answered with a warning keeps the warning in its block" {
2356+
# A read can succeed and still write to stderr -- a deprecation notice, a
2357+
# partial-result warning. That warning is part of what the read returned and
2358+
# stays beside it, with no failure line, since nothing failed.
2359+
d=$(mktemp -d)
2360+
mkdir -p "$d/bin"
2361+
cat >"$d/bin/kubectl" <<'STUB'
2362+
#!/bin/sh
2363+
case "$*" in
2364+
'get pods -A '*) echo 'tenant-test|wedged|10.0.0.1|node-a|False|Running||eol'; exit 0 ;;
2365+
'get pod -n tenant-test wedged '*)
2366+
echo 'Ready=False reason=ContainersNotReady: x'
2367+
echo 'Warning: THE-WARNING' >&2
2368+
exit 0 ;;
2369+
esac
2370+
exit 0
2371+
STUB
2372+
chmod +x "$d/bin/kubectl"
2373+
timeout 90 env PATH="$d/bin:$PATH" "$SCRIPT" "$d/out" >"$d/log" 2>&1
2374+
f="$d/out/pod-tenant-test_wedged.txt"
2375+
block=$(awk '/^=== pod Ready conditions/ { on = 1; next } on && /^=== / { exit } on { print }' "$f")
2376+
# Positive control: the read answered.
2377+
case "$block" in *'Ready=False reason=ContainersNotReady'*) : ;; *) echo "no conditions:"; printf '%s\n' "$block"; exit 1 ;; esac
2378+
n=$(printf '%s\n' "$block" | grep -c 'THE-WARNING')
2379+
[ "$n" -eq 1 ] || { echo "the warning appears $n times:"; printf '%s\n' "$block"; exit 1; }
2380+
case "$block" in
2381+
*"(reading this pod's Ready conditions"*) echo "a read that answered got a failure line:"; printf '%s\n' "$block"; exit 1 ;;
2382+
esac
2383+
rm -rf "$d"
2384+
}
2385+
2386+
@test "a pod read kubectl refused keeps kubectl's message even without a scratch file" {
2387+
# The reads send stderr to a scratch file, which mktemp may fail to create.
2388+
# kubectl's message must then still reach the block, once.
2389+
d=$(mktemp -d)
2390+
mkdir -p "$d/bin"
2391+
cat >"$d/bin/kubectl" <<'STUB'
2392+
#!/bin/sh
2393+
case "$*" in
2394+
'get pods -A '*) echo 'tenant-test|wedged|10.0.0.1|node-a|False|Running||eol'; exit 0 ;;
2395+
'get pod -n tenant-test wedged '*) echo 'Error from server (Forbidden): pods "wedged" is forbidden' >&2; exit 1 ;;
2396+
esac
2397+
exit 0
2398+
STUB
2399+
chmod +x "$d/bin/kubectl"
2400+
timeout 90 env TMPDIR="$d/missing" PATH="$d/bin:$PATH" "$SCRIPT" "$d/out" >"$d/log" 2>&1
2401+
f="$d/out/pod-tenant-test_wedged.txt"
2402+
block=$(awk '/^=== pod Ready conditions/ { on = 1; next } on && /^=== / { exit } on { print }' "$f")
2403+
# Positive control: the read failed and said so by name.
2404+
case "$block" in
2405+
*"(reading this pod's Ready conditions failed: kubectl exited 1"*) : ;;
2406+
*) echo "the refused read is not named:"; printf '%s\n' "$block"; exit 1 ;;
2407+
esac
2408+
n=$(printf '%s\n' "$block" | grep -c 'Forbidden')
2409+
[ "$n" -eq 1 ] || { echo "kubectl's message appears $n times:"; printf '%s\n' "$block"; exit 1; }
2410+
rm -rf "$d"
2411+
}
2412+
2413+
@test "an OVS interface lookup the deadline cut names the budget on both paths" {
2414+
# The per-pod ovn-installed lookup and the LB endpoint lookup both read their
2415+
# own status. A cut at the deadline has to name the budget there too, not
2416+
# report the raw status of the call.
2417+
d=$(mktemp -d)
2418+
mkdir -p "$d/bin"
2419+
cat >"$d/bin/kubectl" <<'STUB'
2420+
#!/bin/sh
2421+
case "$*" in
2422+
'get pods -A '*) [ -n "${STUB_NO_POD:-}" ] && exit 0; echo 'tenant-test|wedged|10.0.0.1|node-a|False|Running||eol'; exit 0 ;;
2423+
'get svc -A '*) [ -n "${STUB_NO_POD:-}" ] || exit 0; echo 'tenant|web|LoadBalancer|192.0.2.10|80|30080|Cluster'; exit 0 ;;
2424+
*endpointslices*) echo '10.0.0.1|node-a|tenant-test|wedged|true'; exit 0 ;;
2425+
*'app=kube-ovn-cni'*) [ -n "${STUB_NO_POD:-}" ] || exit 0; echo 'cni-abc'; exit 0 ;;
2426+
*'app=ovs'*) echo 'ovs-xyz'; exit 0 ;;
2427+
*'nc -z'*) echo fail; exit 0 ;;
2428+
*'find interface'*) sleep 300 ;;
2429+
esac
2430+
exit 0
2431+
STUB
2432+
chmod +x "$d/bin/kubectl"
2433+
real_timeout=$(command -v timeout)
2434+
dp_cut_timeout "$d/bin"
2435+
# Per-pod path: the pod section's 3s run into the lookup.
2436+
PATH="$d/bin:$PATH" DP_REAL_TIMEOUT="$real_timeout" DP_PASS="$(printf '%s\n' '*kubectl get pods -A *' '*kubectl get svc -A *')" \
2437+
COZY_DATAPLANE_BUDGET=6 COZY_DATAPLANE_LB_RESERVE=3 \
2438+
"$real_timeout" 60 "$SCRIPT" "$d/pod" >"$d/log" 2>&1
2439+
f="$d/pod/pod-tenant-test_wedged.txt"
2440+
grep -q 'the OVS interface lookup for iface-id=wedged.tenant-test was cut off at the deadline: the pod section ran out of its share' "$f"
2441+
# LB path: the LB section's 4s run into the endpoint lookup.
2442+
STUB_NO_POD=1 PATH="$d/bin:$PATH" DP_REAL_TIMEOUT="$real_timeout" DP_PASS="$(printf '%s\n' '*kubectl get pods -A *' '*kubectl get svc -A *')" \
2443+
COZY_DATAPLANE_BUDGET=4 COZY_DATAPLANE_LB_RESERVE=4 \
2444+
"$real_timeout" 60 "$SCRIPT" "$d/lb" >>"$d/log" 2>&1
2445+
g="$d/lb/lb-tenant_web.txt"
2446+
grep -q 'the OVS interface lookup for iface-id=wedged.tenant-test on node node-a was cut off at the deadline: the LoadBalancer section ran out of its share' "$g"
2447+
if grep -q 'the lookup exited' "$f" "$g"; then
2448+
echo "a deadline cut was reported as a raw status:"
2449+
grep 'the lookup exited' "$f" "$g"
2450+
exit 1
2451+
fi
22312452
rm -rf "$d"
22322453
}
22332454

@@ -3238,7 +3459,7 @@ exit 0
32383459
STUB
32393460
chmod +x "$d/bin/kubectl"
32403461
dp_cut_timeout "$d/bin"
3241-
PATH="$d/bin:$PATH" DP_CUT='*app=ovs*' timeout 90 "$SCRIPT" "$d/cut" >"$d/log" 2>&1
3462+
timeout 90 env PATH="$d/bin:$PATH" DP_CUT='*app=ovs*' "$SCRIPT" "$d/cut" >"$d/log" 2>&1
32423463
f="$d/cut/lb-tenant_web.txt"
32433464
# Positive control: the endpoint side ran on the fallback.
32443465
grep -q 'iface=genev_sys_6081' "$d/cut/lb-tenant_web.tcpdump-endpoint.txt"

‎hack/e2e-capture-dataplane.sh‎

Lines changed: 39 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -225,8 +225,8 @@ lb_announcer_node() {
225225
}
226226

227227
# lb_capture_decision: stdin = one probe outcome token per line ("ok" / "fail" /
228-
# "unknown" / "spent"; the last two mean the probe could not be run at all, and
229-
# "spent" that the wall-clock budget refused it, so both weigh the same here).
228+
# "unknown" / "spent"; the last two mean the probe reached no verdict, and
229+
# "spent" that the wall-clock budget stopped it, so both weigh the same here).
230230
# Emits the gate decision for the heavy per-node capture:
231231
# - "capture" when at least one probe ran, none succeeded, and at least one
232232
# genuinely failed (the LB IP is unreachable -- the symptom we want
@@ -607,6 +607,11 @@ dp_new_run() {
607607
# Empty if mktemp fails; the notes then omit kubectl's message rather than
608608
# inventing one.
609609
DP_ERR=$(mktemp "${TMPDIR:-/tmp}/dataplane-read.XXXXXX" 2>/dev/null) || DP_ERR=""
610+
# Where a read that writes into its own block sends stderr: the scratch file
611+
# when there is one, the block itself when there is not. The reads open it for
612+
# append and empty the scratch file first themselves: a truncating open of
613+
# /dev/stdout reopens the block's file on Linux and would cut it to nothing.
614+
DP_BLOCK_ERR=${DP_ERR:-/dev/stdout}
610615
if [ -s "$NOTES" ]; then
611616
printf -- '--- new capture run ---\n' >> "$NOTES" 2>/dev/null || true
612617
fi
@@ -931,6 +936,27 @@ capture_node() {
931936
} >> "$_cn_nf" 2>&1 || true
932937
}
933938

939+
# dp_block_outcome <status> <what> <secs> -- what a read writing into a shared
940+
# block adds after its output, from its stderr in DP_ERR. kubectl's own stderr
941+
# goes into the block as it came, whole: a degraded apiserver throttles
942+
# clients, and the error that matters can follow lines of throttling notices.
943+
# The two reads of the Ready block share it, so a failure then gets one line
944+
# that names the read. dp_run's own marker sits in DP_ERR too, unnamed, and is
945+
# left out, so a cut is reported once.
946+
#
947+
# Without a scratch file the reads send stderr into the block instead
948+
# (DP_BLOCK_ERR), so kubectl's words survive; dp_run's marker then sits
949+
# beside the named line, which is the cost of having nowhere else to put it.
950+
dp_block_outcome() {
951+
if [ -n "$DP_ERR" ] && [ -s "$DP_ERR" ]; then
952+
grep -v -e '^(cut off ' -e '^(not started: ' "$DP_ERR" 2>/dev/null || true
953+
fi
954+
if [ "$1" -ne 0 ]; then
955+
echo "($2 $(dp_read_outcome "$1" "$3" "$DP_TIMEOUT"))"
956+
fi
957+
return 0
958+
}
959+
934960
# capture_pod_dataplane <ns> <pod> <podip> <node> <label> [scope] -- the
935961
# pod-specific captures for ONE pod on <node>.
936962
#
@@ -989,13 +1015,15 @@ capture_pod_dataplane() {
9891015
echo
9901016
echo "=== pod Ready conditions + recent probe events ==="
9911017
_cpd_b=$(dp_clip "$DP_READ_TIMEOUT")
1018+
[ -z "$DP_ERR" ] || : > "$DP_ERR"
9921019
dp_run "$_cpd_b" kubectl get pod -n "$_cpd_ns" "$_cpd_pod" \
993-
-o jsonpath='{range .status.conditions[*]}{.type}={.status} reason={.reason}: {.message}{"\n"}{end}' 2>&1 \
994-
|| echo "(reading this pod's Ready conditions $(dp_read_outcome "$?" "$_cpd_b" "$DP_TIMEOUT"))"
1020+
-o jsonpath='{range .status.conditions[*]}{.type}={.status} reason={.reason}: {.message}{"\n"}{end}' 2>>"$DP_BLOCK_ERR"
1021+
dp_block_outcome "$?" "reading this pod's Ready conditions" "$_cpd_b"
9951022
_cpd_b=$(dp_clip "$DP_READ_TIMEOUT")
1023+
[ -z "$DP_ERR" ] || : > "$DP_ERR"
9961024
dp_run "$_cpd_b" kubectl get events -n "$_cpd_ns" --field-selector "involvedObject.name=$_cpd_pod" \
997-
-o jsonpath='{range .items[*]}{.lastTimestamp}{" "}{.reason}{": "}{.message}{"\n"}{end}' 2>&1 \
998-
|| echo "(reading this pod's events $(dp_read_outcome "$?" "$_cpd_b" "$DP_TIMEOUT"))"
1025+
-o jsonpath='{range .items[*]}{.lastTimestamp}{" "}{.reason}{": "}{.message}{"\n"}{end}' 2>>"$DP_BLOCK_ERR"
1026+
dp_block_outcome "$?" "reading this pod's events" "$_cpd_b"
9991027
fi
10001028

10011029
if [ -z "$_cpd_ip" ]; then
@@ -1263,13 +1291,9 @@ host_http_probe() {
12631291
# capture-or-skip this token changes nothing. It exists for the caller's
12641292
# reason line, which is the only place the distinction reaches a reader.
12651293
#
1266-
# A third token, "spent", marks a probe the wall-clock budget refused: dp_run
1267-
# declined the lookup or the exec. Neither the lookup nor the node is the
1268-
# cause then, and the caller says so. An exec cut at the deadline itself
1269-
# leaves no token, but the probes after it are refused and leave this one.
1270-
# A lookup cut by a bound the deadline had shortened counts the same way: it
1271-
# says nothing about the cni-server -- the rule the pod_on_node memo applies
1272-
# to such a cutoff.
1294+
# A third token, "spent", marks a probe the wall-clock budget stopped: dp_run
1295+
# refused the lookup or the exec, or cut one of them at the deadline. Neither
1296+
# the lookup nor the node is the cause then, and the caller says so.
12731297
if [ "$_hp_rc" -eq "$DP_RC_SPENT" ] || [ "$_hp_rc" -eq "$DP_RC_BUDGET_CUT" ]; then
12741298
echo spent
12751299
return 0
@@ -1586,8 +1610,8 @@ capture_lb_datapath() {
15861610
# the outcome set is the only place the two routes to an unknown verdict
15871611
# stay apart, and the reason line has to name a cause that was actually
15881612
# observed. host_http_probe emits an explicit unknown only when the
1589-
# cni-server lookup did not answer; everything else that stops a probe
1590-
# emits nothing.
1613+
# cni-server lookup did not answer, and "spent" only when the budget
1614+
# stopped the probe; everything else that stops a probe emits nothing.
15911615
_outcomes=$( { host_http_probe "$_probenode" "$_lbip" "$_lbport"
15921616
host_http_probe "$_probenode" "$_lbip" "$_lbport"
15931617
host_http_probe "$_probenode" "$_lbip" "$_lbport"; } )

0 commit comments

Comments
 (0)