Skip to content

Commit 6248d45

Browse files
committed
fix(platform): keep the legacy etcd-operator stopped after etcd adoption
Migration 50 scales the legacy etcd-operator to 0, adopts every legacy etcd.aenix.io cluster with etcd-migrate and then scales the operator back to 1. At that point it is still the legacy operator, and it has nothing left to manage. Until a tenant's etcd HelmRelease reconciles the new chart, Flux can upgrade it with the old one, which re-creates the legacy EtcdCluster that etcd-migrate just orphan-deleted. The legacy operator then reconciles it, re-creates the legacy StatefulSet and rewrites the legacy etcd-headless Service, dropping the owner references etcd-migrate pointed at the adopted EtcdMembers. When the new chart prunes the legacy EtcdCluster, the garbage collector deletes the Service with it, and the adopted members' etcd-<i>.etcd-headless names stop resolving. After a successful adoption the operator now stays at 0 until the etcd-operator HelmRelease replaces the Deployment with the v1alpha2 operator later in the same upgrade. A failed or interrupted adoption still restores it through the EXIT trap. Assisted-by: LLM Signed-off-by: Yan Bondarenko <[email protected]>
1 parent 091b854 commit 6248d45

2 files changed

Lines changed: 41 additions & 18 deletions

File tree

‎hack/migration-50-etcd-adopt.bats‎

Lines changed: 21 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -15,8 +15,9 @@
1515
# never skipped — and the script fails loudly if the target is unresolvable;
1616
# 2. the staged credentials Secret is tenant-invisible (managed-by set,
1717
# tenantresource stripped); and
18-
# 3. the operator is scaled to 0 BEFORE etcd-migrate --apply and back to 1
19-
# AFTER, with --apply always carrying the S3 backup destination.
18+
# 3. the operator is scaled to 0 BEFORE etcd-migrate --apply, with --apply
19+
# always carrying the S3 backup destination, and stays at 0 after a
20+
# successful adoption. Only a failed adoption restores it to 1.
2021
#
2122
# cozytest.sh's awk parser recognizes only @test blocks and a bare `}` on its
2223
# own line; there is no bats `run`/`$status`/`setup`. Assertions are direct
@@ -105,17 +106,30 @@ lineno() {
105106
[ "$(jq -r '.data.AWS_ACCESS_KEY_ID' "$staged")" = "QUtJQUVYQU1QTEU=" ]
106107
[ "$(jq -r '.metadata.namespace // "absent"' "$staged")" = "absent" ]
107108

108-
# Order: stage creds -> scale operator down -> --apply -> scale up -> stamp.
109+
# Steps run in this order, stage creds, scale the operator down, --apply, stamp.
109110
s_stage=$(lineno "STAGE tenant-foo")
110111
s_down=$(lineno "SCALE 0")
111112
s_apply=$(lineno "ETCD-MIGRATE --apply")
112-
s_up=$(lineno "SCALE 1")
113113
s_stamp=$(lineno "STAMP")
114-
[ -n "$s_stage" ] && [ -n "$s_down" ] && [ -n "$s_apply" ] && [ -n "$s_up" ] && [ -n "$s_stamp" ]
114+
[ -n "$s_stage" ] && [ -n "$s_down" ] && [ -n "$s_apply" ] && [ -n "$s_stamp" ]
115115
[ "$s_stage" -lt "$s_down" ]
116116
[ "$s_down" -lt "$s_apply" ]
117-
[ "$s_apply" -lt "$s_up" ]
118-
[ "$s_up" -lt "$s_stamp" ]
117+
[ "$s_apply" -lt "$s_stamp" ]
118+
rm -rf "$WORK"
119+
}
120+
121+
@test "successful adoption leaves the legacy operator at 0 for the v1alpha2 operator to replace" {
122+
prep
123+
rc=0
124+
bash "$MIG" >"$WORK/out" 2>&1 || rc=$?
125+
cat "$WORK/out"
126+
cat "$FAKE_CMDLOG"
127+
[ "$rc" -eq 0 ]
128+
# A legacy operator back at 1 takes etcd-headless away from the adopted
129+
# members, see the success path of migration 50.
130+
grep -qF -- "SCALE 0" "$FAKE_CMDLOG"
131+
if grep -qF -- "SCALE 1" "$FAKE_CMDLOG"; then echo "FAIL: the legacy operator was scaled back up after a successful adoption"; false; fi
132+
grep -qF -- "STAMP" "$FAKE_CMDLOG"
119133
rm -rf "$WORK"
120134
}
121135

‎packages/core/platform/images/migrations/migrations/50‎

Lines changed: 20 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -691,7 +691,7 @@ etcd-migrate \
691691
--skip-controller-check || true
692692

693693
if [ "$DRY_RUN" = "1" ]; then
694-
echo "[dry-run] would scale ${ETCD_OPERATOR_NS}/${ETCD_OPERATOR_DEPLOY} to 0, run etcd-migrate --apply ${ETCD_MIGRATE_BACKUP_ARGS:-<no clusters>}, scale back to 1, then stamp version 51"
694+
echo "[dry-run] would scale ${ETCD_OPERATOR_NS}/${ETCD_OPERATOR_DEPLOY} to 0, run etcd-migrate --apply ${ETCD_MIGRATE_BACKUP_ARGS:-<no clusters>}, leave it at 0 for the etcd-operator HelmRelease to replace, then stamp version 51"
695695
exit 0
696696
fi
697697

@@ -712,11 +712,12 @@ if [ "$LEGACY_COUNT" -gt 0 ]; then
712712

713713
# The etcd-operator is cluster-wide: while it sits at 0 replicas NO tenant's
714714
# etcd is reconciled. With `set -euo pipefail`, a non-zero exit from
715-
# etcd-migrate --apply would abort the script before the scale-back, stranding
716-
# the operator at 0 platform-wide until a human intervenes. Arm an EXIT trap
717-
# the moment we scale down so the operator is restored on ANY exit (success,
718-
# failure, or signal); the Job then retries adoption (idempotent) from a
719-
# healthy operator instead of a frozen one.
715+
# etcd-migrate --apply would abort the script and strand the operator at 0
716+
# platform-wide until a human intervenes. Arm an EXIT trap the moment we scale
717+
# down so the operator is restored when adoption fails or the script is
718+
# interrupted. The Job then retries adoption (idempotent) from a healthy
719+
# operator instead of a frozen one. A successful adoption disarms the trap and
720+
# leaves the legacy operator at 0, see below.
720721
restore_etcd_operator() {
721722
echo "Restoring ${ETCD_OPERATOR_NS}/${ETCD_OPERATOR_DEPLOY} to 1 replica..."
722723
kubectl -n "$ETCD_OPERATOR_NS" scale deploy "$ETCD_OPERATOR_DEPLOY" --replicas=1 >/dev/null 2>&1 || true
@@ -738,11 +739,19 @@ if [ "$LEGACY_COUNT" -gt 0 ]; then
738739
--agent-image="${ETCD_OPERATOR_IMAGE}" \
739740
${ETCD_MIGRATE_BACKUP_ARGS}
740741

741-
echo "Scaling ${ETCD_OPERATOR_NS}/${ETCD_OPERATOR_DEPLOY} back to 1..."
742-
kubectl -n "$ETCD_OPERATOR_NS" scale deploy "$ETCD_OPERATOR_DEPLOY" --replicas=1
743-
kubectl -n "$ETCD_OPERATOR_NS" rollout status deploy "$ETCD_OPERATOR_DEPLOY" --timeout=180s || true
744-
# Clean scale-back done; disarm the safety net so the version stamp below is
745-
# not shadowed by a redundant restore.
742+
# Adoption succeeded, so the legacy operator stays at 0. Every legacy cluster
743+
# is now a v1alpha2 EtcdCluster that only the v1alpha2 operator manages, and
744+
# the etcd-operator HelmRelease replaces this Deployment with that operator
745+
# later in the same upgrade. A legacy operator running in between undoes part
746+
# of the adoption. Until a tenant's etcd HelmRelease picks up the new chart,
747+
# Flux re-creates the legacy EtcdCluster from the old one, and the legacy
748+
# operator reconciles it, re-creates the StatefulSet that etcd-migrate
749+
# orphan-deleted and takes the legacy headless Service back from the adopted
750+
# members. The new chart then prunes that EtcdCluster, the garbage collector
751+
# deletes the Service with it, and the members' etcd-<i>.etcd-headless names
752+
# stop resolving. Only a failed or interrupted adoption restores the operator,
753+
# through the EXIT trap above.
754+
echo "Leaving ${ETCD_OPERATOR_NS}/${ETCD_OPERATOR_DEPLOY} at 0 replicas, the etcd-operator HelmRelease replaces it with the v1alpha2 operator."
746755
trap - EXIT
747756
fi
748757

0 commit comments

Comments
 (0)