|
| 1 | +#!/bin/bash |
| 2 | +# FoundationDB backup/restore round-trip through the platform cozy-default |
| 3 | +# BackupClass, run from the foundationdb-3-backup-roundtrip Test: |
| 4 | +# |
| 5 | +# 1. a source and an empty target FoundationDB come up, and a sentinel key |
| 6 | +# is written to the source; |
| 7 | +# 2. two consecutive BackupJobs both reach Succeeded, each with its own |
| 8 | +# blob-store directory (the second stops the first's backup); |
| 9 | +# 3. a RestoreJob of the second backup into the target reaches Succeeded |
| 10 | +# and the sentinel reads back byte for byte; |
| 11 | +# 4. deleting the first Backup removes its directory and leaves the |
| 12 | +# second's in place. |
| 13 | +# |
| 14 | +# Usage: backup-roundtrip.sh run|cleanup (NAMESPACE must be set) |
| 15 | +# |
| 16 | +# `cleanup` is idempotent and is what the Test's finally step calls: deleting |
| 17 | +# the Backups runs the driver's delete Jobs, so the bucket is left as found. |
| 18 | +set -euo pipefail |
| 19 | + |
| 20 | +: "${NAMESPACE:?NAMESPACE must be set}" |
| 21 | +cd "$(dirname "${BASH_SOURCE[0]}")/../../.." |
| 22 | + |
| 23 | +log_substep() { echo " -> $*"; } |
| 24 | +log_success() { echo " OK $*"; } |
| 25 | +log_error() { echo " ERROR $*" >&2; } |
| 26 | +# shellcheck source=examples/backups/_lib/wait-helpers.sh |
| 27 | +. examples/backups/_lib/wait-helpers.sh |
| 28 | + |
| 29 | +SRC=fdb-rt-src |
| 30 | +DST=fdb-rt-copy |
| 31 | +BJ1=fdb-rt-bj1 |
| 32 | +BJ2=fdb-rt-bj2 |
| 33 | +RJ=fdb-rt-restore |
| 34 | +SENTINEL_KEY=/e2e/fdb-roundtrip |
| 35 | +OWNED_BY=backups.cozystack.io/owned-by.BackupJobName |
| 36 | + |
| 37 | +apply_app() { |
| 38 | + kubectl -n "$NAMESPACE" apply -f - <<EOF |
| 39 | +apiVersion: apps.cozystack.io/v1alpha1 |
| 40 | +kind: FoundationDB |
| 41 | +metadata: |
| 42 | + name: $1 |
| 43 | +spec: |
| 44 | + cluster: |
| 45 | + version: "7.3.63" |
| 46 | + processCounts: |
| 47 | + storage: 1 |
| 48 | + stateless: -1 |
| 49 | + cluster_controller: 1 |
| 50 | + redundancyMode: "single" |
| 51 | + storageEngine: "ssd-2" |
| 52 | + faultDomain: |
| 53 | + key: "foundationdb.org/none" |
| 54 | + valueFrom: "\$FDB_ZONE_ID" |
| 55 | + storage: |
| 56 | + size: "1Gi" |
| 57 | + storageClass: "" |
| 58 | + resourcesPreset: "small" |
| 59 | + monitoring: |
| 60 | + enabled: false |
| 61 | + imageType: "unified" |
| 62 | + automaticReplacements: true |
| 63 | +EOF |
| 64 | +} |
| 65 | + |
| 66 | +apply_backupjob() { |
| 67 | + kubectl -n "$NAMESPACE" apply -f - <<EOF |
| 68 | +apiVersion: backups.cozystack.io/v1alpha1 |
| 69 | +kind: BackupJob |
| 70 | +metadata: |
| 71 | + name: $1 |
| 72 | +spec: |
| 73 | + applicationRef: |
| 74 | + apiGroup: apps.cozystack.io |
| 75 | + kind: FoundationDB |
| 76 | + name: $SRC |
| 77 | + backupClassName: cozy-default |
| 78 | +EOF |
| 79 | +} |
| 80 | + |
| 81 | +fdbcli_exec() { |
| 82 | + local pod |
| 83 | + pod=$(kubectl -n "$NAMESPACE" get pods \ |
| 84 | + -l "foundationdb.org/fdb-cluster-name=foundationdb-$1,foundationdb.org/fdb-process-class=cluster_controller" \ |
| 85 | + --field-selector=status.phase=Running -o jsonpath='{.items[0].metadata.name}') |
| 86 | + [[ -n "$pod" ]] || { log_error "no running cluster_controller pod for foundationdb-$1"; return 1; } |
| 87 | + timeout -k 5 60 kubectl -n "$NAMESPACE" exec "$pod" -c foundationdb -- \ |
| 88 | + fdbcli --cluster-file=/var/dynamic-conf/fdb.cluster --exec "$2" |
| 89 | +} |
| 90 | + |
| 91 | +backup_uri() { |
| 92 | + kubectl -n "$NAMESPACE" get backup "$1" -o json | jq -r '.spec.driverMetadata["apps.foundationdb.org/storage-uri"] // ""' |
| 93 | +} |
| 94 | + |
| 95 | +# fdbbackup describe exits 0 for a directory holding a backup and prints |
| 96 | +# "Backup does not exist" for one that is gone; any other failure is neither |
| 97 | +# answer and fails the run. It runs in the second backup's agent pod, which |
| 98 | +# carries the platform credentials and CA. |
| 99 | +describe_dir() { |
| 100 | + local deploy out rc=0 |
| 101 | + deploy=$(kubectl -n "$NAMESPACE" get deploy -o name | grep -- "^deployment.apps/$BJ2-.*-backup-agents$" | head -n1) |
| 102 | + [[ -n "$deploy" ]] || { log_error "no backup agents Deployment for $BJ2"; return 2; } |
| 103 | + out=$(timeout -k 5 120 kubectl -n "$NAMESPACE" exec "$deploy" -c foundationdb -- fdbbackup describe -d "$1" 2>&1) || rc=$? |
| 104 | + if [[ $rc -eq 0 ]]; then |
| 105 | + echo present |
| 106 | + elif grep -q 'Backup does not exist' <<<"$out"; then |
| 107 | + echo absent |
| 108 | + else |
| 109 | + log_error "fdbbackup describe failed for $1: $out" |
| 110 | + return 2 |
| 111 | + fi |
| 112 | +} |
| 113 | + |
| 114 | +run() { |
| 115 | + local value backup1 backup2 uri1 uri2 state1 state2 got |
| 116 | + value="roundtrip-$(date -u +%Y%m%dT%H%M%SZ)" |
| 117 | + |
| 118 | + apply_app "$SRC" |
| 119 | + apply_app "$DST" |
| 120 | + # 660s: above the 600s install timeout cozystack-api gives these releases |
| 121 | + # (no helm-install-timeout override on foundationdb-rd), per the HR-Ready |
| 122 | + # sizing in docs/agents/e2e-testing.md. |
| 123 | + wait_hr_ready "foundationdb-$SRC" 660 |
| 124 | + wait_hr_ready "foundationdb-$DST" 660 |
| 125 | + wait_for_field foundationdbclusters.apps.foundationdb.org "foundationdb-$SRC" '{.status.health.available}' true "$NAMESPACE" 600 |
| 126 | + wait_for_field foundationdbclusters.apps.foundationdb.org "foundationdb-$DST" '{.status.health.available}' true "$NAMESPACE" 600 |
| 127 | + |
| 128 | + fdbcli_exec "$SRC" "writemode on; set $SENTINEL_KEY $value" |
| 129 | + |
| 130 | + # A BackupJob succeeds once its own directory is restorable; 900s covers |
| 131 | + # agent start, the first snapshot and the driver's 60s status refresh. |
| 132 | + apply_backupjob "$BJ1" |
| 133 | + wait_for_field backupjob "$BJ1" '{.status.phase}' Succeeded "$NAMESPACE" 900 Failed |
| 134 | + apply_backupjob "$BJ2" |
| 135 | + wait_for_field backupjob "$BJ2" '{.status.phase}' Succeeded "$NAMESPACE" 900 Failed |
| 136 | + |
| 137 | + backup1=$(kubectl -n "$NAMESPACE" get backupjob "$BJ1" -o jsonpath='{.status.backupRef.name}') |
| 138 | + backup2=$(kubectl -n "$NAMESPACE" get backupjob "$BJ2" -o jsonpath='{.status.backupRef.name}') |
| 139 | + uri1=$(backup_uri "$backup1") |
| 140 | + uri2=$(backup_uri "$backup2") |
| 141 | + [[ "$uri1" == *"/$NAMESPACE/$BJ1?"* ]] || { log_error "first backup does not name its own directory: $uri1"; return 1; } |
| 142 | + [[ "$uri2" == *"/$NAMESPACE/$BJ2?"* ]] || { log_error "second backup does not name its own directory: $uri2"; return 1; } |
| 143 | + |
| 144 | + kubectl -n "$NAMESPACE" apply -f - <<EOF |
| 145 | +apiVersion: backups.cozystack.io/v1alpha1 |
| 146 | +kind: RestoreJob |
| 147 | +metadata: |
| 148 | + name: $RJ |
| 149 | +spec: |
| 150 | + backupRef: |
| 151 | + name: $backup2 |
| 152 | + targetApplicationRef: |
| 153 | + apiGroup: apps.cozystack.io |
| 154 | + kind: FoundationDB |
| 155 | + name: $DST |
| 156 | +EOF |
| 157 | + wait_for_field restorejob "$RJ" '{.status.phase}' Succeeded "$NAMESPACE" 900 Failed |
| 158 | + got=$(fdbcli_exec "$DST" "get $SENTINEL_KEY") |
| 159 | + if ! grep -qF "is \`$value'" <<<"$got"; then |
| 160 | + log_error "sentinel did not survive the restore: $got" |
| 161 | + return 1 |
| 162 | + fi |
| 163 | + log_success "sentinel restored into $DST" |
| 164 | + |
| 165 | + kubectl -n "$NAMESPACE" delete backup "$backup1" --wait=false |
| 166 | + wait_deleted backup "$backup1" 600 |
| 167 | + state1=$(describe_dir "$uri1") |
| 168 | + state2=$(describe_dir "$uri2") |
| 169 | + [[ "$state1" == absent ]] || { log_error "deleting the first Backup left its directory: $uri1"; return 1; } |
| 170 | + [[ "$state2" == present ]] || { log_error "deleting the first Backup removed the second's directory: $uri2"; return 1; } |
| 171 | + log_success "FoundationDB round-trip verified" |
| 172 | +} |
| 173 | + |
| 174 | +cleanup() { |
| 175 | + kubectl -n "$NAMESPACE" delete restorejob "$RJ" --ignore-not-found --wait=false |
| 176 | + # Each Backup's finalizer stops its operator CR and deletes its directory. |
| 177 | + local bj b |
| 178 | + for bj in "$BJ1" "$BJ2"; do |
| 179 | + b=$(kubectl -n "$NAMESPACE" get backupjob "$bj" --ignore-not-found -o jsonpath='{.status.backupRef.name}') |
| 180 | + if [[ -n "$b" ]]; then |
| 181 | + kubectl -n "$NAMESPACE" delete backup "$b" --ignore-not-found --wait=false |
| 182 | + wait_deleted backup "$b" 600 |
| 183 | + fi |
| 184 | + done |
| 185 | + kubectl -n "$NAMESPACE" delete backupjob "$BJ1" "$BJ2" --ignore-not-found --wait=false |
| 186 | + kubectl -n "$NAMESPACE" delete foundationdbbackups.apps.foundationdb.org -l "$OWNED_BY in ($BJ1,$BJ2)" --ignore-not-found |
| 187 | + kubectl -n "$NAMESPACE" delete foundationdbrestores.apps.foundationdb.org -l "$OWNED_BY=$RJ" --ignore-not-found |
| 188 | + kubectl -n "$NAMESPACE" delete foundationdbs.apps.cozystack.io "$SRC" "$DST" --ignore-not-found --wait=false |
| 189 | + wait_deleted hr "foundationdb-$SRC" 600 |
| 190 | + wait_deleted hr "foundationdb-$DST" 600 |
| 191 | +} |
| 192 | + |
| 193 | +case "${1:-}" in |
| 194 | +run) run ;; |
| 195 | +cleanup) cleanup ;; |
| 196 | +*) echo "usage: $0 run|cleanup" >&2; exit 2 ;; |
| 197 | +esac |
0 commit comments