Skip to content

Commit cb7a579

Browse files
committed
fix(e2e): compare LINSTOR pool datasets, not free bytes, at cleanup
For a node with only thick ZFS pools, `linstor sp l` reports the free capacity cached at the satellite's last report rather than asking the satellite, and that report can land while `zfs destroy` is still freeing blocks in the background. A cleanup that had removed every volume it created could then wait out its whole budget against a figure that never moved. That residue has reached about 0.9 GiB, with no known upper bound, against 1 GiB for the smallest volume a suite is known to have leaked, so a byte tolerance has no safe value between the two. Record the datasets under each node's pool before the suite, read with `zfs list` on that node's satellite, and wait until no dataset outside that set remains. A leaked volume is then reported by name and node, and a destroy still freeing space in the background does not count. Assisted-by: LLM Signed-off-by: Aleksei Sviridkin <[email protected]>
1 parent 3303cc5 commit cb7a579

2 files changed

Lines changed: 340 additions & 98 deletions

File tree

‎hack/e2e-chainsaw/_lib/run-kubernetes.sh‎

Lines changed: 80 additions & 59 deletions
Original file line numberDiff line numberDiff line change
@@ -451,7 +451,10 @@ cozy_wait_schedulable_node() {
451451
# default threshold covers that with margin. That makes it a precondition of
452452
# the replicated suite rather than a cleanup criterion: volumes earlier suites
453453
# legitimately keep can hold a node below it after this suite reclaimed
454-
# everything it used, which only the pre-suite baseline can tell apart.
454+
# everything it used, which only the pre-suite baseline can tell apart. It reads
455+
# the free capacity LINSTOR caches for thick pools (see
456+
# _cozy_linstor_pool_datasets), so it can understate a pool whose last destroy
457+
# was still freeing blocks when the satellite reported.
455458
cozy_wait_linstor_pool_free() {
456459
_min_free_gib="${1:-90}"
457460
_timeout="${2:-300}"
@@ -501,8 +504,43 @@ _cozy_linstor_pool_baseline_file() {
501504
printf '%s\n' "${COZY_LINSTOR_POOL_BASELINE_FILE:-_out/e2e-kubernetes-linstor-pool-baseline}"
502505
}
503506

504-
# Capture the per-node free-capacity baseline after stale tenant cleanup and
505-
# before creating this suite's workers. Chainsaw executes `try` and `finally` in
507+
# Print every ZFS dataset under each node's LINSTOR storage pool as
508+
# `node dataset` rows, the pool's own root included, so a node whose pool holds
509+
# no volume still shows up as read. The names come from `zfs list` on the
510+
# satellite rather than from LINSTOR's free capacity: on a node with only thick
511+
# pools `linstor sp l` serves the figure cached at the satellite's last report,
512+
# which can be taken while `zfs destroy` is still freeing blocks in the
513+
# background, so a pool can read hundreds of MiB short of its baseline until
514+
# some later report with nothing left on it. A destroyed dataset's name is gone
515+
# by the time `zfs destroy` returns. The zpool is read off the pool's
516+
# `StorDriver/StorPoolName` property, under which LINSTOR stores the zpool of
517+
# both ZFS drivers, because it differs by lane: `data` on QEMU nodes,
518+
# `data-srvN` in container mode, where every node's pool is imported into one
519+
# shared kernel. Every read carries its own ceiling, so a wedged controller or
520+
# satellite fails the poll within 35s instead of holding the caller for good.
521+
_cozy_linstor_pool_datasets() {
522+
_pools=$(timeout -k 5 30 kubectl -n cozy-linstor exec deploy/linstor-controller -- sh -c '
523+
linstor --machine-readable sp l 2>/dev/null |
524+
jq -r "first | .[] | select(.provider_kind | test(\"^ZFS\")) | \"\(.node_name) \(.props[\"StorDriver/StorPoolName\"] // \"\")\""
525+
' 2>/dev/null) || return 1
526+
[ -n "$_pools" ] || return 1
527+
_satellites=$(timeout -k 5 30 kubectl -n cozy-linstor get pods -l app.kubernetes.io/component=linstor-satellite \
528+
-o jsonpath='{range .items[*]}{.spec.nodeName} {.metadata.name}{"\n"}{end}' 2>/dev/null) || return 1
529+
while read -r _node _zpool; do
530+
[ -n "$_node" ] && [ -n "$_zpool" ] || return 1
531+
_pod=$(printf '%s\n' "$_satellites" | awk -v node="$_node" '$1 == node { print $2; exit }')
532+
[ -n "$_pod" ] || return 1
533+
_datasets=$(timeout -k 5 30 kubectl -n cozy-linstor exec "$_pod" --container=linstor-satellite -- \
534+
zfs list -H -o name -t filesystem,volume,snapshot -r "$_zpool" </dev/null 2>/dev/null) || return 1
535+
[ -n "$_datasets" ] || return 1
536+
printf '%s\n' "$_datasets" | awk -v node="$_node" '{ print node " " $0 }'
537+
done <<EOF
538+
$_pools
539+
EOF
540+
}
541+
542+
# Capture the per-node dataset baseline after stale tenant cleanup and before
543+
# creating this suite's workers. Chainsaw executes `try` and `finally` in
506544
# separate shells, so the small state file is the hand-off between them.
507545
cozy_capture_linstor_pool_baseline() {
508546
_baseline_file=$(_cozy_linstor_pool_baseline_file)
@@ -511,62 +549,45 @@ cozy_capture_linstor_pool_baseline() {
511549
mkdir -p "$_baseline_dir"
512550
: >"$_baseline_file"
513551

514-
_baseline=$(kubectl -n cozy-linstor exec deploy/linstor-controller -- sh -c '
515-
linstor --machine-readable sp l 2>/dev/null |
516-
jq -r "first | .[] | select((.provider_kind | test(\"^ZFS\")) and .free_capacity != null) | \"\(.node_name):\(.free_capacity)\"" |
517-
sort
518-
' 2>/dev/null) || _baseline=""
552+
_baseline=$(_cozy_linstor_pool_datasets) || _baseline=""
519553
if [ -z "$_baseline" ]; then
520-
echo "» ERROR: could not record the LINSTOR pool baseline; physical ZFS reclamation would be unverifiable" >&2
554+
echo "» ERROR: could not record the LINSTOR pool baseline; reclamation of this suite's volumes would be unverifiable" >&2
521555
return 1
522556
fi
523-
_baseline_nodes=$(printf '%s\n' "$_baseline" | awk -F: 'NF == 2 { print $1 }' | sort | tr '\n' ' ')
557+
_baseline_nodes=$(printf '%s\n' "$_baseline" | awk '{ print $1 }' | sort -u | tr '\n' ' ')
524558
if [ "$_baseline_nodes" != "srv1 srv2 srv3 " ] \
525-
|| ! cozy_linstor_pools_at_baseline "$_baseline" "$_baseline" 0; then
526-
echo "» ERROR: LINSTOR pool baseline is incomplete or malformed (expected numeric rows for srv1, srv2 and srv3): ${_baseline:-<empty>}" >&2
559+
|| printf '%s\n' "$_baseline" | awk 'NF != 2 { bad = 1 } END { exit !bad }'; then
560+
echo "» ERROR: LINSTOR pool baseline is incomplete or malformed (expected 'node dataset' rows for srv1, srv2 and srv3): ${_baseline}" >&2
527561
return 1
528562
fi
529563
printf '%s\n' "$_baseline" >"$_baseline_file"
530564
echo "» LINSTOR pool baseline recorded:"
531-
printf '%s\n' "$_baseline" | sed 's/^/ baseline-free-kib: /'
565+
printf '%s\n' "$_baseline" | sed 's/^/ baseline-dataset: /'
532566
}
533567

534-
# Pure comparison used by the pool reclamation loop. Both captures contain
535-
# `node:free_capacity_kib` rows. Every node present in the baseline must still be
536-
# present and may fall short only by the small caller-provided metadata tolerance.
537-
# The 512 MiB default covers the ~0.27 GiB free-capacity drift measured between
538-
# two otherwise-clean container suites, while still rejecting the smallest known
539-
# leaked test volume (the 1 GiB ClickHouse keeper PVC). In the replicated lane
540-
# the drift from a baseline to its cleanup has not been measured; its pools are
541-
# thick ZFS as well, and the cleanups of the two kubernetes suites in the
542-
# 2026-09-30 nightly left srv2 about 1 MiB apart.
543-
cozy_linstor_pools_at_baseline() {
544-
_baseline_rows="$1"
545-
_current_rows="$2"
546-
_tolerance_kib="${3:-524288}"
547-
[ -n "$_baseline_rows" ] && [ -n "$_current_rows" ] || return 1
548-
case "$_tolerance_kib" in
549-
'' | *[!0-9]*) return 1 ;;
550-
esac
551-
552-
while IFS=: read -r _baseline_node _baseline_kib; do
553-
[ -n "$_baseline_node" ] && [ -n "$_baseline_kib" ] || return 1
554-
_current_kib=$(printf '%s\n' "$_current_rows" | awk -F: -v node="$_baseline_node" '$1 == node { print $2; exit }')
555-
[ -n "$_current_kib" ] || return 1
556-
case "$_baseline_kib" in '' | *[!0-9]*) return 1 ;; esac
557-
case "$_current_kib" in '' | *[!0-9]*) return 1 ;; esac
558-
if [ "$(( _current_kib + _tolerance_kib ))" -lt "$_baseline_kib" ]; then
559-
return 1
560-
fi
561-
done <<EOF
562-
$_baseline_rows
563-
EOF
564-
return 0
568+
# Pure comparison of two `node dataset` listings. Prints `new <node> <dataset>`
569+
# for each dataset the current listing has and the baseline does not, and
570+
# `unread <node>` for each baseline node the current listing lacks, and fails
571+
# when it printed anything. A baseline dataset that is gone is not a finding:
572+
# the suite may have reclaimed something an earlier one left.
573+
cozy_linstor_pool_new_datasets() {
574+
[ -n "$1" ] && [ -n "$2" ] || return 1
575+
{
576+
printf '%s\n' "$1"
577+
printf '%s\n' '--'
578+
printf '%s\n' "$2"
579+
} | awk '
580+
!current && $0 == "--" { current = 1; next }
581+
!current { seen[$0] = 1; base[$1] = 1; next }
582+
{ read_nodes[$1] = 1; if (!($0 in seen)) { print "new " $0; bad = 1 } }
583+
END {
584+
for (node in base) if (!(node in read_nodes)) { print "unread " node; bad = 1 }
585+
exit bad
586+
}'
565587
}
566588

567589
cozy_wait_linstor_pool_baseline() {
568590
_timeout="${1:-300}"
569-
_tolerance_kib="${2:-524288}"
570591
_baseline_file=$(_cozy_linstor_pool_baseline_file)
571592
# Returns 2 here and 1 on a timeout, so the caller can tell an unchecked
572593
# reclamation from a failed one.
@@ -577,20 +598,20 @@ cozy_wait_linstor_pool_baseline() {
577598
_baseline=$(cat "$_baseline_file")
578599
_deadline=$(( $(date +%s) + _timeout ))
579600
_current=""
601+
_residue=""
580602
while :; do
581-
_current=$(kubectl -n cozy-linstor exec deploy/linstor-controller -- sh -c '
582-
linstor --machine-readable sp l 2>/dev/null |
583-
jq -r "first | .[] | select((.provider_kind | test(\"^ZFS\")) and .free_capacity != null) | \"\(.node_name):\(.free_capacity)\"" |
584-
sort
585-
' 2>/dev/null) || _current=""
586-
if cozy_linstor_pools_at_baseline "$_baseline" "$_current" "$_tolerance_kib"; then
587-
echo "» LINSTOR pools returned to their pre-suite baseline"
603+
_current=$(_cozy_linstor_pool_datasets) || _current=""
604+
if [ -n "$_current" ] && _residue=$(cozy_linstor_pool_new_datasets "$_baseline" "$_current"); then
605+
echo "» LINSTOR pools hold no dataset beyond their pre-suite baseline"
588606
return 0
589607
fi
590608
if [ "$(date +%s)" -ge "$_deadline" ]; then
591609
echo "» ERROR: LINSTOR pools did not return to their pre-suite baseline within ${_timeout}s; a worker or CDI scratch volume may still occupy ZFS space" >&2
592-
printf '%s\n' "$_baseline" | sed 's/^/ baseline-free-kib: /' >&2
593-
printf '%s\n' "${_current:-<the LINSTOR pool probe returned nothing>}" | sed 's/^/ current-free-kib: /' >&2
610+
if [ -z "$_current" ]; then
611+
echo " linstor-pool: the dataset probe returned nothing" >&2
612+
else
613+
printf '%s\n' "$_residue" | sed 's/^/ linstor-pool: /' >&2
614+
fi
594615
return 1
595616
fi
596617
sleep 5
@@ -660,13 +681,13 @@ cozy_cleanup() {
660681
echo "» ERROR: cleanup has no test name; scoped tenant drain cannot be verified" >&2
661682
cleanup_failed=1
662683
fi
663-
# Wait for this suite's saved per-node baseline: that catches delayed physical
664-
# teardown, DRBD's included, without requiring volumes the suite did not
665-
# create to vanish.
684+
# Wait for this suite's saved per-node baseline: that catches a volume LINSTOR
685+
# has not destroyed yet, or never will, without requiring volumes the suite
686+
# did not create to vanish. Blocks a destroy is still freeing do not count.
666687
local pool_rc=0
667688
cozy_wait_linstor_pool_baseline 300 || pool_rc=$?
668689
if [ "$pool_rc" -eq 1 ]; then
669-
echo "» ERROR: LINSTOR capacity did not return to the pre-suite level" >&2
690+
echo "» ERROR: LINSTOR pools did not return to the pre-suite level" >&2
670691
fi
671692
[ "$pool_rc" -eq 0 ] || cleanup_failed=1
672693
return "$cleanup_failed"
@@ -5082,7 +5103,7 @@ run_kubernetes_test() {
50825103
return 1
50835104
fi
50845105
# Save the post-stale-cleanup state for the separate Chainsaw `finally` shell;
5085-
# cleanup later verifies that this suite returned every node to these figures.
5106+
# cleanup later verifies that this suite left no dataset outside this listing.
50865107
if ! cozy_capture_linstor_pool_baseline; then
50875108
echo "cannot verify LINSTOR reclamation without a complete baseline" >&2
50885109
return 1

0 commit comments

Comments
 (0)