Skip to content

Commit 7011510

Browse files
committed
fix(fluxcd): keep the hook waiting on a release it cannot read
The pre-delete hook read a release's state once, so a single failed read was taken as the release being handled: its fence came off before helm-controller's uninstall, and the report counted it as released cleanly. A state read is now retried within the release's share of the window, a release whose state still cannot be read keeps its fence for the wait like one helm-controller has not handled, and the end of the wait reads each again before naming what the release was left to. A release still suspended after the sweep is named as suspended but not by flux-plunger, which also holds when no manager is recorded for the field at all. Assisted-by: LLM Signed-off-by: Alexey Artamonov <[email protected]>
1 parent 568e6f5 commit 7011510

2 files changed

Lines changed: 66 additions & 19 deletions

File tree

‎hack/flux-plunger-fence-hook_test.bats‎

Lines changed: 42 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -118,7 +118,8 @@ fences_left() {
118118
# for a reason other than pods (wait), the listing
119119
# (list), a listing after the first (relist), the
120120
# pod listing (pods), a release's ownership read
121-
# (<ns>@<name>.ownedread), unsuspend
121+
# (<ns>@<name>.ownedread), a release's state read
122+
# (<ns>@<name>.stateread), unsuspend
122123
# (<ns>@<name>.apply), unsuspend racing another
123124
# write (<ns>@<name>.applyconflict) or finalizer patch
124125
# (<ns>@<name>.patch);
@@ -269,6 +270,7 @@ if [ "$verb" = get ]; then
269270
case "$tmpl" in *'ne .manager "flux-plunger"'*) [ -e "$rel.coowned" ] && echo other ;; esac
270271
exit 0 ;;
271272
*deletionTimestamp*)
273+
if take_fail "$rel.stateread"; then echo "fake: the state read timed out" >&2; exit 1; fi
272274
[ -e "$rel.deleting" ] || exit 0
273275
case "$tmpl" in *"$SUSPENDED_DELETING"*) if [ -e "$rel.suspended" ]; then printf 'suspended'; exit 0; fi ;; esac
274276
case "$tmpl" in *"$UNHANDLED"*) ;; *) exit 0 ;; esac
@@ -492,7 +494,7 @@ run_hook() {
492494

493495
[ "$status" -eq 0 ]
494496
[ "$(fences_left "$STATE")" -eq 0 ]
495-
grep -q -x "released $FENCE from tenant-x/app-x without an uninstall: another manager holds its suspension" "$tmp/out"
497+
grep -q -x "released $FENCE from tenant-x/app-x without an uninstall: it is still suspended, not by flux-plunger" "$tmp/out"
496498
grep -q -x 'released 1 of 1 fenced HelmReleases, 1 without an uninstall' "$tmp/out"
497499
rm -rf "$tmp"
498500
}
@@ -573,6 +575,44 @@ run_hook() {
573575
rm -rf "$tmp"
574576
}
575577

578+
@test "a state read that fails once does not end the wait for helm-controller" {
579+
tmp=$(mktemp -d)
580+
prepare "$tmp"
581+
add_release "$STATE" tenant-a app-a "$FENCE"
582+
touch "$STATE/[email protected]"
583+
echo yes > "$STATE/[email protected]"
584+
echo 3 > "$STATE/[email protected]"
585+
echo 1 > "$STATE/[email protected]"
586+
587+
run_hook "$tmp"
588+
589+
[ "$status" -eq 0 ]
590+
[ "$(fences_left "$STATE")" -eq 0 ]
591+
[ "$(cat "$STATE/[email protected]")" = "$(cat "$STATE/[email protected]")" ]
592+
grep -q -x "released $FENCE from tenant-a/app-a" "$tmp/out"
593+
grep -q -x 'released 1 of 1 fenced HelmReleases, 0 without an uninstall' "$tmp/out"
594+
rm -rf "$tmp"
595+
}
596+
597+
@test "a release whose state can never be read is still released inside the deadline, and says so" {
598+
tmp=$(mktemp -d)
599+
prepare "$tmp"
600+
add_release "$STATE" tenant-a app-a "$FENCE"
601+
touch "$STATE/[email protected]"
602+
echo yes > "$STATE/[email protected]"
603+
echo 9999 > "$STATE/[email protected]"
604+
605+
run_hook "$tmp"
606+
607+
[ "$status" -eq 0 ]
608+
[ "$(fences_left "$STATE")" -eq 0 ]
609+
grep -q -x "released $FENCE from tenant-a/app-a; whether helm-controller had handled it could not be read" "$tmp/out"
610+
# It kept the fence for the whole wait, as one helm-controller had not handled.
611+
[ "$(cat "$CLOCK")" -ge $((WINDOW - 30)) ]
612+
[ "$(cat "$CLOCK")" -lt "$DEADLINE" ]
613+
rm -rf "$tmp"
614+
}
615+
576616
@test "a release helm-controller handles while the hook waits keeps its fence until the uninstall has run" {
577617
tmp=$(mktemp -d)
578618
prepare "$tmp"

‎packages/system/flux-plunger/templates/uninstall-fence-cleanup.yaml‎

Lines changed: 24 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -347,12 +347,13 @@ spec:
347347
348348
# read_state <ns> <name>: how helm-controller stands with a release
349349
# being deleted. "suspended": it skips the uninstall of a suspended
350-
# release, and the suspension is another manager's. "unhandled":
350+
# release, and the suspension is not flux-plunger's. "unhandled":
351351
# its own finalizer no longer holds the release and it has not
352352
# observed the latest generation, so the fence is all that keeps
353-
# the release until it runs the uninstall. Empty otherwise.
353+
# the release until it runs the uninstall. Empty otherwise. Fails
354+
# when the release cannot be read.
354355
read_state() {
355-
kubectl --namespace "$1" get "$hr" "$2" --ignore-not-found --output go-template='{{`{{if .metadata.deletionTimestamp}}{{if .spec.suspend}}suspended{{else}}{{$held := false}}{{range .metadata.finalizers}}{{if eq . "finalizers.fluxcd.io"}}{{$held = true}}{{end}}{{end}}{{if and (not $held) (ne (printf "%v" .status.observedGeneration) (printf "%v" .metadata.generation))}}unhandled{{end}}{{end}}{{end}}`}}' || echo unknown
356+
kubectl --namespace "$1" get "$hr" "$2" --ignore-not-found --output go-template='{{`{{if .metadata.deletionTimestamp}}{{if .spec.suspend}}suspended{{else}}{{$held := false}}{{range .metadata.finalizers}}{{if eq . "finalizers.fluxcd.io"}}{{$held = true}}{{end}}{{end}}{{if and (not $held) (ne (printf "%v" .status.observedGeneration) (printf "%v" .metadata.generation))}}unhandled{{end}}{{end}}{{end}}`}}'
356357
}
357358
358359
# report_release <ref> <state>: remove the fence and name what that
@@ -371,7 +372,7 @@ spec:
371372
echo "released $fence from $1 without an uninstall: helm-controller had not handled it" ;;
372373
suspended)
373374
unhandled=$((unhandled + 1))
374-
echo "released $fence from $1 without an uninstall: another manager holds its suspension" ;;
375+
echo "released $fence from $1 without an uninstall: it is still suspended, not by flux-plunger" ;;
375376
unknown)
376377
echo "released $fence from $1; whether helm-controller had handled it could not be read" ;;
377378
*)
@@ -408,11 +409,15 @@ spec:
408409
echo "NOT released $ref: flux-plunger's suspension could not be lifted" >&2
409410
continue ;;
410411
esac
411-
state=$(read_state "$hns" "$name")
412-
if [ "$state" = unhandled ]; then
413-
pending="$pending $ref"
414-
continue
415-
fi
412+
# A release whose state cannot be read waits like one helm-controller
413+
# has not handled: unfencing it on a failed read could delete it
414+
# before its uninstall.
415+
state=$(retry read_state "$hns" "$name") || state=unknown
416+
case "$state" in
417+
unhandled|unknown)
418+
pending="$pending $ref"
419+
continue ;;
420+
esac
416421
report_release "$ref" "$state"
417422
done
418423
# The last fences go well before the window closes, so their
@@ -422,19 +427,21 @@ spec:
422427
sleep 2
423428
still=
424429
for ref in $pending; do
425-
state=$(read_state "${ref%%/*}" "${ref#*/}")
426-
if [ "$state" = unhandled ]; then
427-
still="$still $ref"
428-
else
429-
begin_release
430-
report_release "$ref" "$state"
431-
fi
430+
state=$(read_state "${ref%%/*}" "${ref#*/}") || state=unknown
431+
case "$state" in
432+
unhandled|unknown)
433+
still="$still $ref" ;;
434+
*)
435+
begin_release
436+
report_release "$ref" "$state" ;;
437+
esac
432438
done
433439
pending=$still
434440
done
435441
for ref in $pending; do
436442
begin_release
437-
report_release "$ref" unhandled
443+
state=$(read_state "${ref%%/*}" "${ref#*/}") || state=unknown
444+
report_release "$ref" "$state"
438445
done
439446
stop_at=$retry_until
440447

0 commit comments

Comments
 (0)