Skip to content

Commit f27b694

Browse files
committed
fix(platform): keep the legacy etcd-operator stopped after etcd adoption
Migration 50 scales the legacy etcd-operator to 0, adopts every legacy etcd.aenix.io cluster with etcd-migrate and then scales the operator back to 1. At that point it is still the legacy operator, and it has nothing left to manage. Until a tenant's etcd HelmRelease reconciles the new chart, Flux can upgrade it with the old one, which re-creates the legacy EtcdCluster that etcd-migrate just orphan-deleted. The legacy operator then reconciles it, re-creates the legacy StatefulSet and rewrites the legacy etcd-headless Service, dropping the owner references etcd-migrate pointed at the adopted EtcdMembers. When the new chart prunes the legacy EtcdCluster, the garbage collector deletes the Service with it, and the adopted members' etcd-<i>.etcd-headless names stop resolving. After a successful adoption the operator now stays at 0 until the etcd-operator HelmRelease replaces the Deployment with the v1alpha2 operator later in the same upgrade. A failed or interrupted adoption still restores it through the EXIT trap. Assisted-by: LLM Signed-off-by: Yan Bondarenko <[email protected]>
1 parent 091b854 commit f27b694

2 files changed

Lines changed: 44 additions & 16 deletions

File tree

‎hack/migration-50-etcd-adopt.bats‎

Lines changed: 26 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -15,8 +15,9 @@
1515
# never skipped — and the script fails loudly if the target is unresolvable;
1616
# 2. the staged credentials Secret is tenant-invisible (managed-by set,
1717
# tenantresource stripped); and
18-
# 3. the operator is scaled to 0 BEFORE etcd-migrate --apply and back to 1
19-
# AFTER, with --apply always carrying the S3 backup destination.
18+
# 3. the operator is scaled to 0 BEFORE etcd-migrate --apply, with --apply
19+
# always carrying the S3 backup destination, and stays at 0 after a
20+
# successful adoption. Only a failed adoption restores it to 1.
2021
#
2122
# cozytest.sh's awk parser recognizes only @test blocks and a bare `}` on its
2223
# own line; there is no bats `run`/`$status`/`setup`. Assertions are direct
@@ -105,17 +106,35 @@ lineno() {
105106
[ "$(jq -r '.data.AWS_ACCESS_KEY_ID' "$staged")" = "QUtJQUVYQU1QTEU=" ]
106107
[ "$(jq -r '.metadata.namespace // "absent"' "$staged")" = "absent" ]
107108

108-
# Order: stage creds -> scale operator down -> --apply -> scale up -> stamp.
109+
# Steps run in this order, stage creds, scale the operator down, --apply, stamp.
109110
s_stage=$(lineno "STAGE tenant-foo")
110111
s_down=$(lineno "SCALE 0")
111112
s_apply=$(lineno "ETCD-MIGRATE --apply")
112-
s_up=$(lineno "SCALE 1")
113113
s_stamp=$(lineno "STAMP")
114-
[ -n "$s_stage" ] && [ -n "$s_down" ] && [ -n "$s_apply" ] && [ -n "$s_up" ] && [ -n "$s_stamp" ]
114+
[ -n "$s_stage" ] && [ -n "$s_down" ] && [ -n "$s_apply" ] && [ -n "$s_stamp" ]
115115
[ "$s_stage" -lt "$s_down" ]
116116
[ "$s_down" -lt "$s_apply" ]
117-
[ "$s_apply" -lt "$s_up" ]
118-
[ "$s_up" -lt "$s_stamp" ]
117+
[ "$s_apply" -lt "$s_stamp" ]
118+
rm -rf "$WORK"
119+
}
120+
121+
@test "successful adoption leaves the legacy operator at 0 for the v1alpha2 operator to replace" {
122+
prep
123+
rc=0
124+
bash "$MIG" >"$WORK/out" 2>&1 || rc=$?
125+
cat "$WORK/out"
126+
cat "$FAKE_CMDLOG"
127+
[ "$rc" -eq 0 ]
128+
# A legacy operator running after adoption reconciles the legacy EtcdCluster
129+
# that Flux re-creates from a not yet upgraded etcd chart, re-creates its
130+
# StatefulSet and takes the legacy headless Service back from the adopted
131+
# members, which the garbage collector then deletes with that EtcdCluster.
132+
grep -qF -- "SCALE 0" "$FAKE_CMDLOG"
133+
if grep -qF -- "SCALE 1" "$FAKE_CMDLOG"; then
134+
echo "FAIL: the legacy operator was scaled back up after a successful adoption"
135+
return 1
136+
fi
137+
grep -qF -- "STAMP" "$FAKE_CMDLOG"
119138
rm -rf "$WORK"
120139
}
121140

‎packages/core/platform/images/migrations/migrations/50‎

Lines changed: 18 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -691,7 +691,7 @@ etcd-migrate \
691691
--skip-controller-check || true
692692

693693
if [ "$DRY_RUN" = "1" ]; then
694-
echo "[dry-run] would scale ${ETCD_OPERATOR_NS}/${ETCD_OPERATOR_DEPLOY} to 0, run etcd-migrate --apply ${ETCD_MIGRATE_BACKUP_ARGS:-<no clusters>}, scale back to 1, then stamp version 51"
694+
echo "[dry-run] would scale ${ETCD_OPERATOR_NS}/${ETCD_OPERATOR_DEPLOY} to 0, run etcd-migrate --apply ${ETCD_MIGRATE_BACKUP_ARGS:-<no clusters>}, leave it at 0 for the etcd-operator HelmRelease to replace, then stamp version 51"
695695
exit 0
696696
fi
697697

@@ -714,9 +714,10 @@ if [ "$LEGACY_COUNT" -gt 0 ]; then
714714
# etcd is reconciled. With `set -euo pipefail`, a non-zero exit from
715715
# etcd-migrate --apply would abort the script before the scale-back, stranding
716716
# the operator at 0 platform-wide until a human intervenes. Arm an EXIT trap
717-
# the moment we scale down so the operator is restored on ANY exit (success,
718-
# failure, or signal); the Job then retries adoption (idempotent) from a
719-
# healthy operator instead of a frozen one.
717+
# the moment we scale down so the operator is restored when adoption fails or
718+
# the script is interrupted. The Job then retries adoption (idempotent) from a
719+
# healthy operator instead of a frozen one. A successful adoption disarms the
720+
# trap and leaves the legacy operator at 0, see below.
720721
restore_etcd_operator() {
721722
echo "Restoring ${ETCD_OPERATOR_NS}/${ETCD_OPERATOR_DEPLOY} to 1 replica..."
722723
kubectl -n "$ETCD_OPERATOR_NS" scale deploy "$ETCD_OPERATOR_DEPLOY" --replicas=1 >/dev/null 2>&1 || true
@@ -738,11 +739,19 @@ if [ "$LEGACY_COUNT" -gt 0 ]; then
738739
--agent-image="${ETCD_OPERATOR_IMAGE}" \
739740
${ETCD_MIGRATE_BACKUP_ARGS}
740741

741-
echo "Scaling ${ETCD_OPERATOR_NS}/${ETCD_OPERATOR_DEPLOY} back to 1..."
742-
kubectl -n "$ETCD_OPERATOR_NS" scale deploy "$ETCD_OPERATOR_DEPLOY" --replicas=1
743-
kubectl -n "$ETCD_OPERATOR_NS" rollout status deploy "$ETCD_OPERATOR_DEPLOY" --timeout=180s || true
744-
# Clean scale-back done; disarm the safety net so the version stamp below is
745-
# not shadowed by a redundant restore.
742+
# Adoption succeeded, so the legacy operator stays at 0. Every legacy cluster
743+
# is now a v1alpha2 EtcdCluster whose members only the v1alpha2 operator
744+
# manages, and that operator replaces this Deployment when the etcd-operator
745+
# HelmRelease reconciles later in the same upgrade. Bringing the legacy
746+
# operator back undoes the adoption. Until a tenant's etcd HelmRelease picks
747+
# up the new chart, Flux re-creates the legacy EtcdCluster from the old one,
748+
# and a running legacy operator reconciles it, re-creates the StatefulSet that
749+
# etcd-migrate orphan-deleted and takes the legacy headless Service back from
750+
# the adopted members. The new chart then prunes that EtcdCluster, the garbage
751+
# collector deletes the Service with it, and the members'
752+
# etcd-<i>.etcd-headless names stop resolving. Only a failed or interrupted
753+
# adoption restores the operator, through the EXIT trap above.
754+
echo "Leaving ${ETCD_OPERATOR_NS}/${ETCD_OPERATOR_DEPLOY} at 0 replicas, the etcd-operator HelmRelease replaces it with the v1alpha2 operator."
746755
trap - EXIT
747756
fi
748757

0 commit comments

Comments
 (0)