Skip to content

Commit 08e3836

Browse files
committed
fix(e2e): compare LINSTOR pool datasets, not free bytes, at cleanup
For a node with only thick ZFS pools, `linstor sp l` reports the free capacity cached at the satellite's last report rather than asking the satellite, and that report can land while `zfs destroy` is still freeing blocks in the background. A cleanup that had removed every volume it created could then wait out its whole budget against a figure that never moved. That residue has reached about 0.9 GiB, with no known upper bound, against 1 GiB for the smallest volume a suite is known to have leaked, so a byte tolerance has no safe value between the two. Record the datasets under each node's pool before the suite, read with `zfs list` on that node's satellite, and wait until no dataset outside that set remains. A leaked volume is then reported by name and node, and a destroy still freeing space in the background does not count. Assisted-by: LLM Signed-off-by: Aleksei Sviridkin <[email protected]>
1 parent 3303cc5 commit 08e3836

2 files changed

Lines changed: 250 additions & 89 deletions

File tree

‎hack/e2e-chainsaw/_lib/run-kubernetes.sh‎

Lines changed: 78 additions & 59 deletions
Original file line numberDiff line numberDiff line change
@@ -451,7 +451,10 @@ cozy_wait_schedulable_node() {
451451
# default threshold covers that with margin. That makes it a precondition of
452452
# the replicated suite rather than a cleanup criterion: volumes earlier suites
453453
# legitimately keep can hold a node below it after this suite reclaimed
454-
# everything it used, which only the pre-suite baseline can tell apart.
454+
# everything it used, which only the pre-suite baseline can tell apart. It reads
455+
# the free capacity LINSTOR caches for thick pools (see
456+
# _cozy_linstor_pool_datasets), so it can understate a pool whose last destroy
457+
# was still freeing blocks when the satellite reported.
455458
cozy_wait_linstor_pool_free() {
456459
_min_free_gib="${1:-90}"
457460
_timeout="${2:-300}"
@@ -501,8 +504,41 @@ _cozy_linstor_pool_baseline_file() {
501504
printf '%s\n' "${COZY_LINSTOR_POOL_BASELINE_FILE:-_out/e2e-kubernetes-linstor-pool-baseline}"
502505
}
503506

504-
# Capture the per-node free-capacity baseline after stale tenant cleanup and
505-
# before creating this suite's workers. Chainsaw executes `try` and `finally` in
507+
# Print every ZFS dataset under each node's LINSTOR storage pool as
508+
# `node dataset` rows, the pool's own root included, so a node whose pool holds
509+
# no volume still shows up as read. The names come from `zfs list` on the
510+
# satellite rather than from LINSTOR's free capacity: on a node with only thick
511+
# pools `linstor sp l` serves the figure cached at the satellite's last report,
512+
# which can be taken while `zfs destroy` is still freeing blocks in the
513+
# background, so a pool can read hundreds of MiB short of its baseline until
514+
# some later report with nothing left on it. A destroyed dataset's name is gone
515+
# by the time `zfs destroy` returns. The zpool is read off the pool's own
516+
# property because it differs by lane:
517+
# `data` on QEMU nodes, `data-srvN` in container mode, where every node's pool
518+
# is imported into one shared kernel.
519+
_cozy_linstor_pool_datasets() {
520+
_pools=$(kubectl -n cozy-linstor exec deploy/linstor-controller -- sh -c '
521+
linstor --machine-readable sp l 2>/dev/null |
522+
jq -r "first | .[] | select(.provider_kind | test(\"^ZFS\")) | \"\(.node_name) \(.props[\"StorDriver/ZPool\"] // .props[\"StorDriver/ZPoolThin\"] // \"\")\""
523+
' 2>/dev/null) || return 1
524+
[ -n "$_pools" ] || return 1
525+
_satellites=$(kubectl -n cozy-linstor get pods -l app.kubernetes.io/component=linstor-satellite \
526+
-o jsonpath='{range .items[*]}{.spec.nodeName} {.metadata.name}{"\n"}{end}' 2>/dev/null) || return 1
527+
while read -r _node _zpool; do
528+
[ -n "$_node" ] && [ -n "$_zpool" ] || return 1
529+
_pod=$(printf '%s\n' "$_satellites" | awk -v node="$_node" '$1 == node { print $2; exit }')
530+
[ -n "$_pod" ] || return 1
531+
_datasets=$(kubectl -n cozy-linstor exec "$_pod" --container=linstor-satellite -- \
532+
zfs list -H -o name -t filesystem,volume,snapshot -r "$_zpool" </dev/null 2>/dev/null) || return 1
533+
[ -n "$_datasets" ] || return 1
534+
printf '%s\n' "$_datasets" | awk -v node="$_node" '{ print node " " $0 }'
535+
done <<EOF
536+
$_pools
537+
EOF
538+
}
539+
540+
# Capture the per-node dataset baseline after stale tenant cleanup and before
541+
# creating this suite's workers. Chainsaw executes `try` and `finally` in
506542
# separate shells, so the small state file is the hand-off between them.
507543
cozy_capture_linstor_pool_baseline() {
508544
_baseline_file=$(_cozy_linstor_pool_baseline_file)
@@ -511,62 +547,45 @@ cozy_capture_linstor_pool_baseline() {
511547
mkdir -p "$_baseline_dir"
512548
: >"$_baseline_file"
513549

514-
_baseline=$(kubectl -n cozy-linstor exec deploy/linstor-controller -- sh -c '
515-
linstor --machine-readable sp l 2>/dev/null |
516-
jq -r "first | .[] | select((.provider_kind | test(\"^ZFS\")) and .free_capacity != null) | \"\(.node_name):\(.free_capacity)\"" |
517-
sort
518-
' 2>/dev/null) || _baseline=""
550+
_baseline=$(_cozy_linstor_pool_datasets) || _baseline=""
519551
if [ -z "$_baseline" ]; then
520-
echo "» ERROR: could not record the LINSTOR pool baseline; physical ZFS reclamation would be unverifiable" >&2
552+
echo "» ERROR: could not record the LINSTOR pool baseline; reclamation of this suite's volumes would be unverifiable" >&2
521553
return 1
522554
fi
523-
_baseline_nodes=$(printf '%s\n' "$_baseline" | awk -F: 'NF == 2 { print $1 }' | sort | tr '\n' ' ')
555+
_baseline_nodes=$(printf '%s\n' "$_baseline" | awk '{ print $1 }' | sort -u | tr '\n' ' ')
524556
if [ "$_baseline_nodes" != "srv1 srv2 srv3 " ] \
525-
|| ! cozy_linstor_pools_at_baseline "$_baseline" "$_baseline" 0; then
526-
echo "» ERROR: LINSTOR pool baseline is incomplete or malformed (expected numeric rows for srv1, srv2 and srv3): ${_baseline:-<empty>}" >&2
557+
|| printf '%s\n' "$_baseline" | awk 'NF != 2 { bad = 1 } END { exit !bad }'; then
558+
echo "» ERROR: LINSTOR pool baseline is incomplete or malformed (expected 'node dataset' rows for srv1, srv2 and srv3): ${_baseline}" >&2
527559
return 1
528560
fi
529561
printf '%s\n' "$_baseline" >"$_baseline_file"
530562
echo "» LINSTOR pool baseline recorded:"
531-
printf '%s\n' "$_baseline" | sed 's/^/ baseline-free-kib: /'
563+
printf '%s\n' "$_baseline" | sed 's/^/ baseline-dataset: /'
532564
}
533565

534-
# Pure comparison used by the pool reclamation loop. Both captures contain
535-
# `node:free_capacity_kib` rows. Every node present in the baseline must still be
536-
# present and may fall short only by the small caller-provided metadata tolerance.
537-
# The 512 MiB default covers the ~0.27 GiB free-capacity drift measured between
538-
# two otherwise-clean container suites, while still rejecting the smallest known
539-
# leaked test volume (the 1 GiB ClickHouse keeper PVC). In the replicated lane
540-
# the drift from a baseline to its cleanup has not been measured; its pools are
541-
# thick ZFS as well, and the cleanups of the two kubernetes suites in the
542-
# 2026-09-30 nightly left srv2 about 1 MiB apart.
543-
cozy_linstor_pools_at_baseline() {
544-
_baseline_rows="$1"
545-
_current_rows="$2"
546-
_tolerance_kib="${3:-524288}"
547-
[ -n "$_baseline_rows" ] && [ -n "$_current_rows" ] || return 1
548-
case "$_tolerance_kib" in
549-
'' | *[!0-9]*) return 1 ;;
550-
esac
551-
552-
while IFS=: read -r _baseline_node _baseline_kib; do
553-
[ -n "$_baseline_node" ] && [ -n "$_baseline_kib" ] || return 1
554-
_current_kib=$(printf '%s\n' "$_current_rows" | awk -F: -v node="$_baseline_node" '$1 == node { print $2; exit }')
555-
[ -n "$_current_kib" ] || return 1
556-
case "$_baseline_kib" in '' | *[!0-9]*) return 1 ;; esac
557-
case "$_current_kib" in '' | *[!0-9]*) return 1 ;; esac
558-
if [ "$(( _current_kib + _tolerance_kib ))" -lt "$_baseline_kib" ]; then
559-
return 1
560-
fi
561-
done <<EOF
562-
$_baseline_rows
563-
EOF
564-
return 0
566+
# Pure comparison of two `node dataset` listings. Prints `new <node> <dataset>`
567+
# for each dataset the current listing has and the baseline does not, and
568+
# `unread <node>` for each baseline node the current listing lacks, and fails
569+
# when it printed anything. A baseline dataset that is gone is not a finding:
570+
# the suite may have reclaimed something an earlier one left.
571+
cozy_linstor_pool_new_datasets() {
572+
[ -n "$1" ] && [ -n "$2" ] || return 1
573+
{
574+
printf '%s\n' "$1"
575+
printf '%s\n' '--'
576+
printf '%s\n' "$2"
577+
} | awk '
578+
!current && $0 == "--" { current = 1; next }
579+
!current { seen[$0] = 1; base[$1] = 1; next }
580+
{ read_nodes[$1] = 1; if (!($0 in seen)) { print "new " $0; bad = 1 } }
581+
END {
582+
for (node in base) if (!(node in read_nodes)) { print "unread " node; bad = 1 }
583+
exit bad
584+
}'
565585
}
566586

567587
cozy_wait_linstor_pool_baseline() {
568588
_timeout="${1:-300}"
569-
_tolerance_kib="${2:-524288}"
570589
_baseline_file=$(_cozy_linstor_pool_baseline_file)
571590
# Returns 2 here and 1 on a timeout, so the caller can tell an unchecked
572591
# reclamation from a failed one.
@@ -577,20 +596,20 @@ cozy_wait_linstor_pool_baseline() {
577596
_baseline=$(cat "$_baseline_file")
578597
_deadline=$(( $(date +%s) + _timeout ))
579598
_current=""
599+
_residue=""
580600
while :; do
581-
_current=$(kubectl -n cozy-linstor exec deploy/linstor-controller -- sh -c '
582-
linstor --machine-readable sp l 2>/dev/null |
583-
jq -r "first | .[] | select((.provider_kind | test(\"^ZFS\")) and .free_capacity != null) | \"\(.node_name):\(.free_capacity)\"" |
584-
sort
585-
' 2>/dev/null) || _current=""
586-
if cozy_linstor_pools_at_baseline "$_baseline" "$_current" "$_tolerance_kib"; then
587-
echo "» LINSTOR pools returned to their pre-suite baseline"
601+
_current=$(_cozy_linstor_pool_datasets) || _current=""
602+
if [ -n "$_current" ] && _residue=$(cozy_linstor_pool_new_datasets "$_baseline" "$_current"); then
603+
echo "» LINSTOR pools hold no dataset beyond their pre-suite baseline"
588604
return 0
589605
fi
590606
if [ "$(date +%s)" -ge "$_deadline" ]; then
591607
echo "» ERROR: LINSTOR pools did not return to their pre-suite baseline within ${_timeout}s; a worker or CDI scratch volume may still occupy ZFS space" >&2
592-
printf '%s\n' "$_baseline" | sed 's/^/ baseline-free-kib: /' >&2
593-
printf '%s\n' "${_current:-<the LINSTOR pool probe returned nothing>}" | sed 's/^/ current-free-kib: /' >&2
608+
if [ -z "$_current" ]; then
609+
echo " linstor-pool: the dataset probe returned nothing" >&2
610+
else
611+
printf '%s\n' "$_residue" | sed 's/^/ linstor-pool: /' >&2
612+
fi
594613
return 1
595614
fi
596615
sleep 5
@@ -660,13 +679,13 @@ cozy_cleanup() {
660679
echo "» ERROR: cleanup has no test name; scoped tenant drain cannot be verified" >&2
661680
cleanup_failed=1
662681
fi
663-
# Wait for this suite's saved per-node baseline: that catches delayed physical
664-
# teardown, DRBD's included, without requiring volumes the suite did not
665-
# create to vanish.
682+
# Wait for this suite's saved per-node baseline: that catches a volume LINSTOR
683+
# has not destroyed yet, or never will, without requiring volumes the suite
684+
# did not create to vanish. Blocks a destroy is still freeing do not count.
666685
local pool_rc=0
667686
cozy_wait_linstor_pool_baseline 300 || pool_rc=$?
668687
if [ "$pool_rc" -eq 1 ]; then
669-
echo "» ERROR: LINSTOR capacity did not return to the pre-suite level" >&2
688+
echo "» ERROR: LINSTOR pools did not return to the pre-suite level" >&2
670689
fi
671690
[ "$pool_rc" -eq 0 ] || cleanup_failed=1
672691
return "$cleanup_failed"
@@ -5082,7 +5101,7 @@ run_kubernetes_test() {
50825101
return 1
50835102
fi
50845103
# Save the post-stale-cleanup state for the separate Chainsaw `finally` shell;
5085-
# cleanup later verifies that this suite returned every node to these figures.
5104+
# cleanup later verifies that this suite left no dataset outside this listing.
50865105
if ! cozy_capture_linstor_pool_baseline; then
50875106
echo "cannot verify LINSTOR reclamation without a complete baseline" >&2
50885107
return 1

0 commit comments

Comments
 (0)