Skip to content

Commit 33322b8

Browse files
[ci] Align upgrade test with real user flow
Four fixes surfaced by manually running the upgrade on a long-lived cluster (dev10 → v1.3.0-rc.1): 1. Use `cozystack` as the helm release name (not `installer`) to match the install command documented on the website. 2. Pull the published OCI chart (oci://ghcr.io/cozystack/cozystack/cozy-installer --version X.Y.Z) instead of a local `packages/core/installer` path, so the release pipeline actually exercises the OCI artifact end users consume. 3. Compare pre/post snapshots in Phase 4 validation (HR-not-Ready, CrashLoopBackOff pods, unbound PVs) so pre-existing noise on non-pristine clusters doesn't mask real upgrade regressions. 4. New `Verify migration state after upgrade` test — asserts the cozystack-version configmap equals the chart's migrations.targetVersion, doesn't regress, and the migration job completed when a version bump was needed. Co-Authored-By: Claude Opus 4.7 <[email protected]> Signed-off-by: Myasnikov Daniil <[email protected]>
1 parent 7f3b23f commit 33322b8

1 file changed

Lines changed: 90 additions & 47 deletions

File tree

‎hack/e2e-upgrade-cozystack.bats‎

Lines changed: 90 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -64,26 +64,16 @@ find_previous_release() {
6464
echo "$PREV_RELEASE" > "$STATE_DIR/prev-release"
6565
}
6666

67-
@test "Extract and install previous version of Cozystack" {
67+
@test "Install previous version of Cozystack from OCI" {
6868
PREV_RELEASE=$(cat "$STATE_DIR/prev-release")
69+
local prev_version="${PREV_RELEASE#v}"
6970

70-
# Extract installer chart from the previous release tag
71-
local prev_installer="$STATE_DIR/prev-installer"
72-
rm -rf "$prev_installer"
73-
mkdir -p "$prev_installer/templates"
71+
echo "Installing Cozystack ${PREV_RELEASE} from oci://ghcr.io/cozystack/cozystack/cozy-installer"
7472

75-
git show "${PREV_RELEASE}:packages/core/installer/Chart.yaml" \
76-
> "$prev_installer/Chart.yaml"
77-
git show "${PREV_RELEASE}:packages/core/installer/values.yaml" \
78-
> "$prev_installer/values.yaml"
79-
git show "${PREV_RELEASE}:packages/core/installer/templates/cozystack-operator.yaml" \
80-
> "$prev_installer/templates/cozystack-operator.yaml"
81-
82-
echo "Operator image (previous): $(yq '.cozystackOperator.image' "$prev_installer/values.yaml")"
83-
84-
# Install previous version via Helm (same mechanism users use)
85-
helm upgrade installer "$prev_installer" \
86-
--install \
73+
# Install the same way end users do — pull the published OCI chart
74+
helm upgrade --install cozystack \
75+
oci://ghcr.io/cozystack/cozystack/cozy-installer \
76+
--version "${prev_version}" \
8777
--namespace cozy-system \
8878
--create-namespace \
8979
--wait \
@@ -338,6 +328,20 @@ EOF
338328
kubectl get hr -A --no-headers | wc -l > "$STATE_DIR/pre-hr-count"
339329
echo "Pre-upgrade HelmRelease count: $(cat "$STATE_DIR/pre-hr-count")"
340330

331+
# Baseline sets for post-upgrade delta comparison. In the CI sandbox these
332+
# are empty (clean start); on long-lived clusters they capture pre-existing
333+
# noise so we only fail on upgrade-caused regressions.
334+
kubectl get hr -A --no-headers 2>/dev/null \
335+
| awk '$4 != "True" {print $1"/"$2}' | sort > "$STATE_DIR/pre-hr-not-ready"
336+
kubectl get pods -A --no-headers 2>/dev/null \
337+
| awk '/CrashLoopBackOff/ {print $1"/"$2}' | sort > "$STATE_DIR/pre-crashloop-pods"
338+
kubectl get pv --no-headers 2>/dev/null \
339+
| awk '$5 != "Bound" {print $1}' | sort > "$STATE_DIR/pre-unbound-pvs"
340+
341+
# Pre-upgrade cozystack migration version (for post-upgrade assertion)
342+
kubectl get cm cozystack-version -n cozy-system -o jsonpath='{.data.version}' \
343+
> "$STATE_DIR/pre-cozystack-version" 2>/dev/null || echo "0" > "$STATE_DIR/pre-cozystack-version"
344+
341345
# Snapshot pod state (for debugging if upgrade breaks things)
342346
kubectl get pods -A --no-headers > "$STATE_DIR/pre-pods"
343347
echo "Pre-upgrade pod count: $(wc -l < "$STATE_DIR/pre-pods")"
@@ -349,11 +353,17 @@ EOF
349353

350354
@test "Upgrade Cozystack to current version" {
351355
PREV_RELEASE=$(cat "$STATE_DIR/prev-release")
352-
echo "Upgrading from $PREV_RELEASE to current version"
353-
echo "Operator image (current): $(yq '.cozystackOperator.image' packages/core/installer/values.yaml)"
356+
local current_tag="${UPGRADE_TARGET_TAG:-}"
357+
if [ -z "$current_tag" ]; then
358+
current_tag=$(git describe --tags --exact-match --match 'v*' 2>/dev/null)
359+
fi
360+
local current_version="${current_tag#v}"
361+
echo "Upgrading from $PREV_RELEASE to $current_tag (oci://ghcr.io/cozystack/cozystack/cozy-installer:${current_version})"
354362

355-
helm upgrade installer packages/core/installer \
356-
--install \
363+
# Pull the published OCI chart — same mechanism users run during upgrades
364+
helm upgrade cozystack \
365+
oci://ghcr.io/cozystack/cozystack/cozy-installer \
366+
--version "${current_version}" \
357367
--namespace cozy-system \
358368
--wait \
359369
--timeout 2m
@@ -370,64 +380,97 @@ EOF
370380
# Phase 4: Post-upgrade validation
371381
# ===================================================================
372382

373-
@test "Wait for all HelmReleases to reconcile after upgrade" {
383+
@test "Wait for HelmReleases to reconcile after upgrade (no new failures)" {
374384
# Give Flux time to detect the new package source and start reconciling
375385
sleep 10
376386

377-
# Wait for all HelmReleases to become Ready (generous timeout for full reconciliation)
378387
local max_wait=900 # 15 minutes
379388
local interval=10
380389
local elapsed=0
390+
local pre_not_ready="$STATE_DIR/pre-hr-not-ready"
381391

392+
# Wait until no HRs are newly not-ready (i.e., not present in the pre-upgrade baseline)
382393
while [ $elapsed -lt $max_wait ]; do
383-
local not_ready
384-
not_ready=$(kubectl get hr -A --no-headers 2>/dev/null | grep -v " True " | wc -l)
385-
if [ "$not_ready" -eq 0 ]; then
386-
echo "All HelmReleases are Ready after ${elapsed}s"
394+
local new_not_ready
395+
new_not_ready=$(comm -23 \
396+
<(kubectl get hr -A --no-headers 2>/dev/null | awk '$4 != "True" {print $1"/"$2}' | sort) \
397+
"$pre_not_ready")
398+
if [ -z "$new_not_ready" ]; then
399+
echo "No HelmReleases newly not-ready after ${elapsed}s"
387400
break
388401
fi
389-
echo "Waiting for $not_ready HelmReleases to reconcile (${elapsed}s / ${max_wait}s)..."
402+
local count
403+
count=$(printf '%s\n' "$new_not_ready" | grep -c . || true)
404+
echo "Waiting for $count newly not-ready HelmReleases (${elapsed}s / ${max_wait}s)..."
390405
sleep "$interval"
391406
elapsed=$((elapsed + interval))
392407
done
393408

394-
# Final check: fail if any HR is still not ready
395-
if kubectl get hr -A --no-headers | grep -v " True "; then
396-
echo "HelmReleases still not ready after ${max_wait}s:" >&2
409+
# Final check: fail only on HRs that weren't already failing pre-upgrade
410+
local new_not_ready
411+
new_not_ready=$(comm -23 \
412+
<(kubectl get hr -A --no-headers 2>/dev/null | awk '$4 != "True" {print $1"/"$2}' | sort) \
413+
"$pre_not_ready")
414+
if [ -n "$new_not_ready" ]; then
415+
echo "HelmReleases newly not-ready after upgrade:" >&2
416+
echo "$new_not_ready" >&2
397417
kubectl get hr -A >&2
398418
exit 1
399419
fi
400420
}
401421

402-
@test "Verify no CrashLoopBackOff pods after upgrade" {
422+
@test "Verify migration state after upgrade" {
423+
local expected actual previous
424+
# Target version from the upgraded platform chart
425+
expected=$(yq '.migrations.targetVersion' packages/core/platform/values.yaml)
426+
actual=$(kubectl get cm cozystack-version -n cozy-system -o jsonpath='{.data.version}')
427+
previous=$(cat "$STATE_DIR/pre-cozystack-version")
428+
429+
echo "cozystack-version: pre=$previous post=$actual chart_target=$expected"
430+
431+
# Version must match what the new platform chart expects (catches silent skips / wrong image)
432+
[ "$actual" = "$expected" ]
433+
# Version must not regress
434+
[ "$actual" -ge "$previous" ]
435+
436+
# If a migration was required, verify the Job completed successfully
437+
if [ "$previous" -lt "$expected" ]; then
438+
kubectl wait job/cozystack-migration-hook -n cozy-system \
439+
--for=condition=Complete --timeout=5m
440+
fi
441+
}
442+
443+
@test "Verify no new CrashLoopBackOff pods after upgrade" {
403444
# Allow a brief settling period for pods to restart
404445
sleep 10
405446

406-
local crashloop_pods
407-
crashloop_pods=$(kubectl get pods -A --no-headers 2>/dev/null \
408-
| grep -i "CrashLoopBackOff" || true)
447+
local new_crashloop
448+
new_crashloop=$(comm -23 \
449+
<(kubectl get pods -A --no-headers 2>/dev/null | awk '/CrashLoopBackOff/ {print $1"/"$2}' | sort) \
450+
"$STATE_DIR/pre-crashloop-pods")
409451

410-
if [ -n "$crashloop_pods" ]; then
411-
echo "CrashLoopBackOff pods detected after upgrade:" >&2
412-
echo "$crashloop_pods" >&2
452+
if [ -n "$new_crashloop" ]; then
453+
echo "New CrashLoopBackOff pods detected after upgrade:" >&2
454+
echo "$new_crashloop" >&2
413455
exit 1
414456
fi
415457

416-
echo "No CrashLoopBackOff pods found"
458+
echo "No new CrashLoopBackOff pods"
417459
}
418460

419-
@test "Verify all PersistentVolumes are Bound after upgrade" {
420-
local unbound_pvs
421-
unbound_pvs=$(kubectl get pv --no-headers 2>/dev/null \
422-
| grep -v "Bound" || true)
461+
@test "Verify no new unbound PersistentVolumes after upgrade" {
462+
local new_unbound
463+
new_unbound=$(comm -23 \
464+
<(kubectl get pv --no-headers 2>/dev/null | awk '$5 != "Bound" {print $1}' | sort) \
465+
"$STATE_DIR/pre-unbound-pvs")
423466

424-
if [ -n "$unbound_pvs" ]; then
425-
echo "Unbound PersistentVolumes detected after upgrade:" >&2
426-
echo "$unbound_pvs" >&2
467+
if [ -n "$new_unbound" ]; then
468+
echo "New unbound PersistentVolumes detected after upgrade:" >&2
469+
echo "$new_unbound" >&2
427470
exit 1
428471
fi
429472

430-
echo "All PersistentVolumes are Bound"
473+
echo "No new unbound PersistentVolumes"
431474
}
432475

433476
@test "Verify PostgreSQL data survived upgrade" {

0 commit comments

Comments
 (0)