Skip to content

Commit 3dca7c0

Browse files
committed
test(e2e): cover FoundationDB backups on cozy-default, round-trip behind a gate
foundationdb-2-backup-contracts runs in CI: a BackupJob on cozy-default must yield an operator CR naming the job's own directory and an "@host" account, at the SNI-capable agent version, with the credentials in spec.podTemplateSpec; the agent pod must get them, and the operator pod its projected credentials and CA. The sandbox has no S3 endpoint the backup can reach, so it stops at the wiring and tears the CR down. foundationdb-3-backup-roundtrip drives backup-roundtrip.sh: two consecutive BackupJobs in their own directories, a restore of the second into an empty copy with the sentinel intact, and deleting the first Backup removing only its directory. Like the etcd round-trip it is gated behind FDB_E2E_S3_ROUNDTRIP=1: fdbbackup and fdbrestore run inside the cluster-wide operator pod, whose credentials and CA come from the package values, so a test cannot point them at the in-cluster SeaweedFS endpoint, and the external one is a placeholder in kind. Assisted-by: LLM Signed-off-by: Andrey Kolkov <[email protected]>
1 parent 70c4d8a commit 3dca7c0

4 files changed

Lines changed: 446 additions & 0 deletions

File tree

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
apiVersion: backups.cozystack.io/v1alpha1
2+
kind: BackupJob
3+
metadata:
4+
name: fdb-contract
5+
spec:
6+
applicationRef:
7+
apiGroup: apps.cozystack.io
8+
kind: FoundationDB
9+
name: backup-contract
10+
backupClassName: cozy-default
Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
apiVersion: apps.cozystack.io/v1alpha1
2+
kind: FoundationDB
3+
metadata:
4+
name: backup-contract
5+
spec:
6+
cluster:
7+
version: "7.3.63"
8+
processCounts:
9+
storage: 1
10+
stateless: -1
11+
cluster_controller: 1
12+
redundancyMode: "single"
13+
storageEngine: "ssd-2"
14+
faultDomain:
15+
key: "foundationdb.org/none"
16+
valueFrom: "$FDB_ZONE_ID"
17+
storage:
18+
size: "1Gi"
19+
storageClass: ""
20+
resourcesPreset: "small"
21+
monitoring:
22+
enabled: false
23+
customParameters:
24+
- "knob_disable_posix_kernel_aio=1"
25+
imageType: "unified"
26+
automaticReplacements: true
Lines changed: 197 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,197 @@
1+
#!/bin/bash
2+
# FoundationDB backup/restore round-trip through the platform cozy-default
3+
# BackupClass, run from the foundationdb-3-backup-roundtrip Test:
4+
#
5+
# 1. a source and an empty target FoundationDB come up, and a sentinel key
6+
# is written to the source;
7+
# 2. two consecutive BackupJobs both reach Succeeded, each with its own
8+
# blob-store directory (the second stops the first's backup);
9+
# 3. a RestoreJob of the second backup into the target reaches Succeeded
10+
# and the sentinel reads back byte for byte;
11+
# 4. deleting the first Backup removes its directory and leaves the
12+
# second's in place.
13+
#
14+
# Usage: backup-roundtrip.sh run|cleanup (NAMESPACE must be set)
15+
#
16+
# `cleanup` is idempotent and is what the Test's finally step calls: deleting
17+
# the Backups runs the driver's delete Jobs, so the bucket is left as found.
18+
set -euo pipefail
19+
20+
: "${NAMESPACE:?NAMESPACE must be set}"
21+
cd "$(dirname "${BASH_SOURCE[0]}")/../../.."
22+
23+
log_substep() { echo " -> $*"; }
24+
log_success() { echo " OK $*"; }
25+
log_error() { echo " ERROR $*" >&2; }
26+
# shellcheck source=examples/backups/_lib/wait-helpers.sh
27+
. examples/backups/_lib/wait-helpers.sh
28+
29+
SRC=fdb-rt-src
30+
DST=fdb-rt-copy
31+
BJ1=fdb-rt-bj1
32+
BJ2=fdb-rt-bj2
33+
RJ=fdb-rt-restore
34+
SENTINEL_KEY=/e2e/fdb-roundtrip
35+
OWNED_BY=backups.cozystack.io/owned-by.BackupJobName
36+
37+
apply_app() {
38+
kubectl -n "$NAMESPACE" apply -f - <<EOF
39+
apiVersion: apps.cozystack.io/v1alpha1
40+
kind: FoundationDB
41+
metadata:
42+
name: $1
43+
spec:
44+
cluster:
45+
version: "7.3.63"
46+
processCounts:
47+
storage: 1
48+
stateless: -1
49+
cluster_controller: 1
50+
redundancyMode: "single"
51+
storageEngine: "ssd-2"
52+
faultDomain:
53+
key: "foundationdb.org/none"
54+
valueFrom: "\$FDB_ZONE_ID"
55+
storage:
56+
size: "1Gi"
57+
storageClass: ""
58+
resourcesPreset: "small"
59+
monitoring:
60+
enabled: false
61+
imageType: "unified"
62+
automaticReplacements: true
63+
EOF
64+
}
65+
66+
apply_backupjob() {
67+
kubectl -n "$NAMESPACE" apply -f - <<EOF
68+
apiVersion: backups.cozystack.io/v1alpha1
69+
kind: BackupJob
70+
metadata:
71+
name: $1
72+
spec:
73+
applicationRef:
74+
apiGroup: apps.cozystack.io
75+
kind: FoundationDB
76+
name: $SRC
77+
backupClassName: cozy-default
78+
EOF
79+
}
80+
81+
fdbcli_exec() {
82+
local pod
83+
pod=$(kubectl -n "$NAMESPACE" get pods \
84+
-l "foundationdb.org/fdb-cluster-name=foundationdb-$1,foundationdb.org/fdb-process-class=cluster_controller" \
85+
--field-selector=status.phase=Running -o jsonpath='{.items[0].metadata.name}')
86+
[[ -n "$pod" ]] || { log_error "no running cluster_controller pod for foundationdb-$1"; return 1; }
87+
timeout -k 5 60 kubectl -n "$NAMESPACE" exec "$pod" -c foundationdb -- \
88+
fdbcli --cluster-file=/var/dynamic-conf/fdb.cluster --exec "$2"
89+
}
90+
91+
backup_uri() {
92+
kubectl -n "$NAMESPACE" get backup "$1" -o json | jq -r '.spec.driverMetadata["apps.foundationdb.org/storage-uri"] // ""'
93+
}
94+
95+
# fdbbackup describe exits 0 for a directory holding a backup and prints
96+
# "Backup does not exist" for one that is gone; any other failure is neither
97+
# answer and fails the run. It runs in the second backup's agent pod, which
98+
# carries the platform credentials and CA.
99+
describe_dir() {
100+
local deploy out rc=0
101+
deploy=$(kubectl -n "$NAMESPACE" get deploy -o name | grep -- "^deployment.apps/$BJ2-.*-backup-agents$" | head -n1)
102+
[[ -n "$deploy" ]] || { log_error "no backup agents Deployment for $BJ2"; return 2; }
103+
out=$(timeout -k 5 120 kubectl -n "$NAMESPACE" exec "$deploy" -c foundationdb -- fdbbackup describe -d "$1" 2>&1) || rc=$?
104+
if [[ $rc -eq 0 ]]; then
105+
echo present
106+
elif grep -q 'Backup does not exist' <<<"$out"; then
107+
echo absent
108+
else
109+
log_error "fdbbackup describe failed for $1: $out"
110+
return 2
111+
fi
112+
}
113+
114+
run() {
115+
local value backup1 backup2 uri1 uri2 state1 state2 got
116+
value="roundtrip-$(date -u +%Y%m%dT%H%M%SZ)"
117+
118+
apply_app "$SRC"
119+
apply_app "$DST"
120+
# 660s: above the 600s install timeout cozystack-api gives these releases
121+
# (no helm-install-timeout override on foundationdb-rd), per the HR-Ready
122+
# sizing in docs/agents/e2e-testing.md.
123+
wait_hr_ready "foundationdb-$SRC" 660
124+
wait_hr_ready "foundationdb-$DST" 660
125+
wait_for_field foundationdbclusters.apps.foundationdb.org "foundationdb-$SRC" '{.status.health.available}' true "$NAMESPACE" 600
126+
wait_for_field foundationdbclusters.apps.foundationdb.org "foundationdb-$DST" '{.status.health.available}' true "$NAMESPACE" 600
127+
128+
fdbcli_exec "$SRC" "writemode on; set $SENTINEL_KEY $value"
129+
130+
# A BackupJob succeeds once its own directory is restorable; 900s covers
131+
# agent start, the first snapshot and the driver's 60s status refresh.
132+
apply_backupjob "$BJ1"
133+
wait_for_field backupjob "$BJ1" '{.status.phase}' Succeeded "$NAMESPACE" 900 Failed
134+
apply_backupjob "$BJ2"
135+
wait_for_field backupjob "$BJ2" '{.status.phase}' Succeeded "$NAMESPACE" 900 Failed
136+
137+
backup1=$(kubectl -n "$NAMESPACE" get backupjob "$BJ1" -o jsonpath='{.status.backupRef.name}')
138+
backup2=$(kubectl -n "$NAMESPACE" get backupjob "$BJ2" -o jsonpath='{.status.backupRef.name}')
139+
uri1=$(backup_uri "$backup1")
140+
uri2=$(backup_uri "$backup2")
141+
[[ "$uri1" == *"/$NAMESPACE/$BJ1?"* ]] || { log_error "first backup does not name its own directory: $uri1"; return 1; }
142+
[[ "$uri2" == *"/$NAMESPACE/$BJ2?"* ]] || { log_error "second backup does not name its own directory: $uri2"; return 1; }
143+
144+
kubectl -n "$NAMESPACE" apply -f - <<EOF
145+
apiVersion: backups.cozystack.io/v1alpha1
146+
kind: RestoreJob
147+
metadata:
148+
name: $RJ
149+
spec:
150+
backupRef:
151+
name: $backup2
152+
targetApplicationRef:
153+
apiGroup: apps.cozystack.io
154+
kind: FoundationDB
155+
name: $DST
156+
EOF
157+
wait_for_field restorejob "$RJ" '{.status.phase}' Succeeded "$NAMESPACE" 900 Failed
158+
got=$(fdbcli_exec "$DST" "get $SENTINEL_KEY")
159+
if ! grep -qF "is \`$value'" <<<"$got"; then
160+
log_error "sentinel did not survive the restore: $got"
161+
return 1
162+
fi
163+
log_success "sentinel restored into $DST"
164+
165+
kubectl -n "$NAMESPACE" delete backup "$backup1" --wait=false
166+
wait_deleted backup "$backup1" 600
167+
state1=$(describe_dir "$uri1")
168+
state2=$(describe_dir "$uri2")
169+
[[ "$state1" == absent ]] || { log_error "deleting the first Backup left its directory: $uri1"; return 1; }
170+
[[ "$state2" == present ]] || { log_error "deleting the first Backup removed the second's directory: $uri2"; return 1; }
171+
log_success "FoundationDB round-trip verified"
172+
}
173+
174+
cleanup() {
175+
kubectl -n "$NAMESPACE" delete restorejob "$RJ" --ignore-not-found --wait=false
176+
# Each Backup's finalizer stops its operator CR and deletes its directory.
177+
local bj b
178+
for bj in "$BJ1" "$BJ2"; do
179+
b=$(kubectl -n "$NAMESPACE" get backupjob "$bj" --ignore-not-found -o jsonpath='{.status.backupRef.name}')
180+
if [[ -n "$b" ]]; then
181+
kubectl -n "$NAMESPACE" delete backup "$b" --ignore-not-found --wait=false
182+
wait_deleted backup "$b" 600
183+
fi
184+
done
185+
kubectl -n "$NAMESPACE" delete backupjob "$BJ1" "$BJ2" --ignore-not-found --wait=false
186+
kubectl -n "$NAMESPACE" delete foundationdbbackups.apps.foundationdb.org -l "$OWNED_BY in ($BJ1,$BJ2)" --ignore-not-found
187+
kubectl -n "$NAMESPACE" delete foundationdbrestores.apps.foundationdb.org -l "$OWNED_BY=$RJ" --ignore-not-found
188+
kubectl -n "$NAMESPACE" delete foundationdbs.apps.cozystack.io "$SRC" "$DST" --ignore-not-found --wait=false
189+
wait_deleted hr "foundationdb-$SRC" 600
190+
wait_deleted hr "foundationdb-$DST" 600
191+
}
192+
193+
case "${1:-}" in
194+
run) run ;;
195+
cleanup) cleanup ;;
196+
*) echo "usage: $0 run|cleanup" >&2; exit 2 ;;
197+
esac

0 commit comments

Comments
 (0)