Skip to content

Commit c60dd31

Browse files
committed
fix(backupstrategy-controller): reclaim Bucket backups, bound the mirror Job, wire its e2e
Address the round-2 review on the S3 Bucket backup strategy. - e2e wiring: the backup round-trip lived in hack/e2e-chainsaw/bucket-backup/, which no Test Impact Analysis rule selected, so a scoped edit to the harness ran the bucket app-create suite instead and the round-trip never ran under TIA. Route examples/backups/<app>/ to an "<app>-backup" suite when one exists (select-e2e.sh), map bucket-backup to the bucket-application source in both mapping tables so it round-trips (select-install.sh + src_to_suites), and pin it with a bats case. - reclaim on delete: deleting a Backup left its mirrored objects orphaned in cozy-backups. Add an `s3-mirror --mode=delete` that purges a repo prefix, a cleanupBucketBackup finalizer branch that runs it as a one-shot Job and waits (failing open on a terminating namespace / missing strategy / unprojectable credentials, with the skip-artifact-cleanup escape hatch), mirroring Redis. - mirror Job deadline: the mirror Pod set no activeDeadlineSeconds, so a wedged copy ran forever. Set one and promote it to the Job in buildJobStrategyBatchJob so the backoffLimit retries share one ceiling (no-op for drivers without it). - empty-source restore: a restore-as-copy whose source lists zero objects now fails instead of reporting success having restored nothing; the refusal no longer names the CLI-only --allow-empty-source flag. - TLS: --ca-file now appends to the system roots instead of replacing them, so the one transport serving both endpoints does not strip public trust from the side that relies on it. - tests: cover buildBucketMirrorPod (args, delete flag, deadline, env) and the s3-mirror purge; docs: the in-place restore note no longer claims byte-for-byte. Assisted-By: LLM Signed-off-by: Andrey Kolkov <[email protected]>
1 parent 5f163c4 commit c60dd31

10 files changed

Lines changed: 459 additions & 41 deletions

File tree

‎examples/backups/bucket/90-scenario.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ A tenant stores objects in an `apps.cozystack.io/Bucket` and enables backups by
44

55
## In-place restore
66

7-
After an accidental deletion the tenant creates a `RestoreJob` with `backupRef` set and no `targetApplicationRef` (`35-restorejob-in-place.yaml`). The driver provisions a read-write `BucketAccess` on the source bucket and mirrors the chosen backup snapshot back over the live contents, deleting objects the snapshot does not contain, so the bucket ends up byte-for-byte as it was at backup time. This is destructive to anything written after the backup — take a fresh backup first if the current state matters.
7+
After an accidental deletion the tenant creates a `RestoreJob` with `backupRef` set and no `targetApplicationRef` (`35-restorejob-in-place.yaml`). The driver provisions a read-write `BucketAccess` on the source bucket and mirrors the chosen backup snapshot back over the live contents, deleting objects the snapshot does not contain, so the bucket's objects and their content type are restored to the snapshot (object versions and tags are not carried). This is destructive to anything written after the backup — take a fresh backup first if the current state matters.
88

99
## To-copy restore
1010

‎hack/select-e2e.sh‎

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -246,6 +246,9 @@ src_to_suites() {
246246
# roundtrip, so an edit to packages/apps/kafka (or kafka-operator, which
247247
# reaches this source) selects both.
248248
kafka-application) echo "kafka kafka-metadata" ;;
249+
# The Bucket app source owns both the app-create suite and the backup
250+
# round-trip suite, so a bucket-app change exercises both.
251+
bucket-application) echo "bucket bucket-backup" ;;
249252
*-application) echo "${1%-application}" ;;
250253
*) echo "$1" ;;
251254
esac
@@ -409,8 +412,17 @@ while IFS= read -r file || [ -n "$file" ]; do
409412
# selected_apps would empty the final intersection and trip the
410413
# full-suite safety net).
411414
app=$(echo "$file" | sed -nE 's,^examples/backups/([^/]+)/.*,\1,p')
412-
if echo "$all_apps" | grep -Fxq "$app"; then
413-
selected_apps="$selected_apps $app"
415+
# The round-trip Test conventionally lives in its app's own suite dir, but
416+
# an app that already ships a non-backup suite (the Bucket app-create test
417+
# holds the "bucket" suite) keeps its round-trip in "<app>-backup"; prefer
418+
# that when it exists so the harness edit selects the suite that exercises
419+
# the backup path rather than the app-create one.
420+
suite="$app"
421+
if echo "$all_apps" | grep -Fxq "${app}-backup"; then
422+
suite="${app}-backup"
423+
fi
424+
if echo "$all_apps" | grep -Fxq "$suite"; then
425+
selected_apps="$selected_apps $suite"
414426
trigger_any=1
415427
fi
416428
continue ;;

‎hack/select-e2e_test.bats‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -274,6 +274,15 @@ assert_full_suite() {
274274
[ "$output" = "postgres" ]
275275
}
276276

277+
@test "backup example harness edit prefers the app's -backup suite when present" {
278+
tmp=$(mktemp -d)
279+
trap 'rm -rf "$tmp"' EXIT
280+
cp -r packages/core/platform/sources "$tmp/sources"
281+
echo "examples/backups/bucket/run-all.sh" > "$tmp/diff"
282+
output=$(hack/select-e2e.sh "$tmp/diff" "$tmp/sources")
283+
[ "$output" = "bucket-backup" ]
284+
}
285+
277286
@test "backup example without a matching suite selects nothing" {
278287
tmp=$(mktemp -d)
279288
trap 'rm -rf "$tmp"' EXIT

‎hack/select-install.sh‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,10 @@ suite_to_source() {
7878
vminstance) echo cozystack.vm-instance-application ; return ;;
7979
securitygroup) echo cozystack.securitygroup-controller ; return ;;
8080
kafka-metadata) echo cozystack.kafka-application ; return ;;
81+
# The Bucket backup round-trip installs the Bucket app (it provisions its own
82+
# buckets through the example scripts), so it rides the same source as the
83+
# bucket app-create suite; select-e2e.sh maps that source back to both.
84+
bucket-backup) echo cozystack.bucket-application ; return ;;
8185
esac
8286
for cand in "cozystack.$1-application" "cozystack.$1"; do
8387
if echo "$NODES" | grep -Fxq "$cand"; then echo "$cand"; return; fi

‎internal/backupcontroller/backup_controller.go‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -185,6 +185,14 @@ func (r *BackupReconciler) cleanupOnDelete(ctx context.Context, backup *backupsv
185185
// deletes the object and WAITS for that delete before the Backup is
186186
// removed, so no object is orphaned (see cleanupKafkaBackup).
187187
return r.cleanupKafkaBackup(ctx, backup)
188+
case strategyv1alpha1.BucketStrategyKind:
189+
// Like Rabbitmq/Redis, the Bucket driver OWNS its artifact - the mirrored
190+
// objects under the repo prefix, which no engine retention prunes - so
191+
// this branch deletes them and WAITS for that delete before the Backup is
192+
// removed. Without it a retention-pruned Plan leaves orphaned copies in
193+
// cozy-backups with nothing left that knows their prefix (see
194+
// cleanupBucketBackup).
195+
return r.cleanupBucketBackup(ctx, backup)
188196
case strategyv1alpha1.VeleroStrategyKind:
189197
return ctrl.Result{}, r.cleanupVeleroBackup(ctx, backup)
190198
default:

‎internal/backupcontroller/bucketstrategy_controller.go‎

Lines changed: 196 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ import (
1717
"k8s.io/apimachinery/pkg/types"
1818
ctrl "sigs.k8s.io/controller-runtime"
1919
"sigs.k8s.io/controller-runtime/pkg/client"
20+
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
2021

2122
strategyv1alpha1 "github.com/cozystack/cozystack/api/backups/strategy/v1alpha1"
2223
backupsv1alpha1 "github.com/cozystack/cozystack/api/backups/v1alpha1"
@@ -36,6 +37,7 @@ const (
3637
bucketLabelMode = "bucket.strategy.backups.cozystack.io/mode"
3738
bucketModeBackup = "backup"
3839
bucketModeRestore = "restore"
40+
bucketModeCleanup = "cleanup"
3941

4042
// Driver-metadata key prefix used to round-trip BackupClassStrategy
4143
// parameters through the Backup artifact, mirroring the Job driver.
@@ -44,6 +46,14 @@ const (
4446
// bucketPollInterval matches the Job / Altinity cadence.
4547
bucketPollInterval = 5 * time.Second
4648

49+
// bucketMirrorDeadlineSeconds bounds a single mirror Job run. Without it a
50+
// wedged copy (a hung S3 connection, a never-completing list) keeps the
51+
// mirror Pod Running forever and the BackupJob polling it never terminates.
52+
// Generous so a large legitimate bucket still finishes; the point is a
53+
// finite ceiling, not a tight SLA. ensureJobStrategy*Job promotes it to the
54+
// Job so retries share the ceiling rather than each getting a fresh one.
55+
bucketMirrorDeadlineSeconds int64 = 2 * 60 * 60
56+
4757
// bucketFieldManager owns the COSI BucketAccess objects the driver
4858
// applies via server-side apply.
4959
bucketFieldManager = "cozystack-bucket-backup-driver"
@@ -845,11 +855,194 @@ func buildBucketMirrorPod(mode string, tmpl strategyv1alpha1.BucketTemplate, app
845855
}
846856
}
847857

858+
deadline := bucketMirrorDeadlineSeconds
848859
return &corev1.PodTemplateSpec{
849860
Spec: corev1.PodSpec{
850-
RestartPolicy: corev1.RestartPolicyNever,
851-
Containers: []corev1.Container{container},
852-
Volumes: volumes,
861+
RestartPolicy: corev1.RestartPolicyNever,
862+
ActiveDeadlineSeconds: &deadline,
863+
Containers: []corev1.Container{container},
864+
Volumes: volumes,
865+
},
866+
}
867+
}
868+
869+
// buildBucketCleanupPod builds the one-shot Pod that reclaims a backup's repo
870+
// objects (`s3-mirror --mode=delete`) when its Backup is deleted. Only the repo
871+
// side matters to a purge, so no BucketAccess or app credentials are wired.
872+
func buildBucketCleanupPod(snapshot *bucketBackupSnapshot, image string) *corev1.PodTemplateSpec {
873+
args := []string{
874+
"s3-mirror",
875+
"--mode=delete",
876+
"--repo-endpoint=" + snapshot.RepoEndpoint,
877+
"--repo-bucket=" + snapshot.RepoBucket,
878+
"--repo-prefix=" + snapshot.RepoPrefix,
879+
}
880+
if snapshot.RepoRegion != "" {
881+
args = append(args, "--repo-region="+snapshot.RepoRegion)
882+
}
883+
884+
secretKeySecret := snapshot.RepoCredentialsSecretKeySecret
885+
if secretKeySecret == "" {
886+
secretKeySecret = snapshot.RepoCredentialsSecret
887+
}
888+
env := []corev1.EnvVar{
889+
{
890+
Name: "REPO_ACCESS_KEY",
891+
ValueFrom: &corev1.EnvVarSource{SecretKeyRef: &corev1.SecretKeySelector{
892+
LocalObjectReference: corev1.LocalObjectReference{Name: snapshot.RepoCredentialsSecret},
893+
Key: snapshot.RepoCredentialsAccessKeyKey,
894+
}},
853895
},
896+
{
897+
Name: "REPO_SECRET_KEY",
898+
ValueFrom: &corev1.EnvVarSource{SecretKeyRef: &corev1.SecretKeySelector{
899+
LocalObjectReference: corev1.LocalObjectReference{Name: secretKeySecret},
900+
Key: snapshot.RepoCredentialsSecretKeyKey,
901+
}},
902+
},
903+
}
904+
905+
container := corev1.Container{Name: "cleanup", Image: image, Args: args, Env: env}
906+
907+
var volumes []corev1.Volume
908+
if snapshot.RepoCACertSecret != "" {
909+
container.Args = append(container.Args, "--ca-file="+bucketCAMountPath+"/"+bucketCAFileName)
910+
container.VolumeMounts = append(container.VolumeMounts, corev1.VolumeMount{
911+
Name: "s3-ca",
912+
MountPath: bucketCAMountPath,
913+
ReadOnly: true,
914+
})
915+
volumes = append(volumes, corev1.Volume{
916+
Name: "s3-ca",
917+
VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{
918+
SecretName: snapshot.RepoCACertSecret,
919+
Items: []corev1.KeyToPath{{Key: snapshot.RepoCACertKey, Path: bucketCAFileName}},
920+
}},
921+
})
922+
} else if snapshot.InsecureSkipVerify {
923+
container.Args = append(container.Args, "--insecure")
924+
}
925+
926+
deadline := bucketMirrorDeadlineSeconds
927+
return &corev1.PodTemplateSpec{
928+
Spec: corev1.PodSpec{
929+
RestartPolicy: corev1.RestartPolicyNever,
930+
ActiveDeadlineSeconds: &deadline,
931+
Containers: []corev1.Container{container},
932+
Volumes: volumes,
933+
},
934+
}
935+
}
936+
937+
// cleanupBucketBackup deletes a Backup's mirrored repo objects when the Backup
938+
// is removed. Like the Redis / Rabbitmq drivers the Bucket driver OWNS its
939+
// artifact (the objects under the repo prefix; no engine retention prunes them),
940+
// so it runs a one-shot delete Job and WAITS for it before the finalizer is
941+
// removed, leaving nothing orphaned in cozy-backups. It fails open - releasing
942+
// the Backup and leaving the objects - when the delete cannot proceed (namespace
943+
// terminating, strategy gone, credentials unprojectable) rather than wedging the
944+
// Backup Terminating forever; `backups.cozystack.io/skip-artifact-cleanup` is the
945+
// operator escape hatch.
946+
func (r *BackupReconciler) cleanupBucketBackup(ctx context.Context, backup *backupsv1alpha1.Backup) (ctrl.Result, error) {
947+
logger := getLogger(ctx)
948+
jobName := backup.Name + "-cleanup"
949+
950+
if backup.Annotations[redisSkipArtifactCleanupAnnotation] == "true" {
951+
existing := &batchv1.Job{}
952+
if err := r.Get(ctx, types.NamespacedName{Namespace: backup.Namespace, Name: jobName}, existing); err == nil {
953+
_ = r.deleteBucketCleanupJob(ctx, existing)
954+
}
955+
logger.Debug("skipping Bucket artifact cleanup per annotation; objects left in the repo bucket", "backup", backup.Name)
956+
return ctrl.Result{}, nil
957+
}
958+
959+
snapshot, err := unmarshalBucketSnapshot(backup.Status.UnderlyingResources)
960+
if err != nil {
961+
return ctrl.Result{}, fmt.Errorf("decode Backup snapshot: %w", err)
962+
}
963+
if snapshot == nil || snapshot.RepoBucket == "" || snapshot.RepoPrefix == "" {
964+
// A Backup written before the snapshot recorded repo coordinates names
965+
// nothing to delete.
966+
return ctrl.Result{}, nil
967+
}
968+
969+
strategy := &strategyv1alpha1.Bucket{}
970+
if err := r.Get(ctx, client.ObjectKey{Name: backup.Spec.StrategyRef.Name}, strategy); err != nil {
971+
if apierrors.IsNotFound(err) {
972+
return r.releaseBucketCleanup(ctx, backup, snapshot.RepoPrefix, "strategy CR is gone"), nil
973+
}
974+
return ctrl.Result{}, err
975+
}
976+
977+
job := &batchv1.Job{}
978+
err = r.Get(ctx, types.NamespacedName{Namespace: backup.Namespace, Name: jobName}, job)
979+
switch {
980+
case apierrors.IsNotFound(err):
981+
terminating, nsErr := r.namespaceTerminating(ctx, backup.Namespace)
982+
if nsErr != nil {
983+
return ctrl.Result{}, nsErr
984+
}
985+
if terminating {
986+
return r.releaseBucketCleanup(ctx, backup, snapshot.RepoPrefix, "namespace is terminating"), nil
987+
}
988+
if perr := ProjectBackupCredentials(ctx, r.Client, r.CredentialsConfig, backup.Namespace); perr != nil {
989+
return r.releaseBucketCleanup(ctx, backup, snapshot.RepoPrefix, fmt.Sprintf("cannot project credentials: %v", perr)), nil
990+
}
991+
pod := buildBucketCleanupPod(snapshot, strategy.Spec.Template.Image)
992+
desired := buildJobStrategyBatchJob(backup.Namespace, jobName, map[string]string{bucketLabelMode: bucketModeCleanup}, pod)
993+
if cerr := controllerutil.SetControllerReference(backup, desired, r.Scheme); cerr != nil {
994+
return ctrl.Result{}, fmt.Errorf("set controller reference on cleanup Job: %w", cerr)
995+
}
996+
if cerr := r.Create(ctx, desired); cerr != nil && !apierrors.IsAlreadyExists(cerr) {
997+
// Forbidden (namespace went Terminating after the check) or Invalid
998+
// (a long Backup name overflows the 63-char Job name) cannot be fixed
999+
// by retrying; release. Other errors are transient - requeue.
1000+
if apierrors.IsForbidden(cerr) || apierrors.IsInvalid(cerr) {
1001+
return r.releaseBucketCleanup(ctx, backup, snapshot.RepoPrefix, fmt.Sprintf("cannot create cleanup Job: %v", cerr)), nil
1002+
}
1003+
return ctrl.Result{}, cerr
1004+
}
1005+
return ctrl.Result{RequeueAfter: bucketPollInterval}, nil
1006+
case err != nil:
1007+
return ctrl.Result{}, err
1008+
}
1009+
1010+
// Only a Job this Backup controls and stamped as a cleanup Job may drive the
1011+
// release decision below: a same-named Job belonging to someone else (or a
1012+
// stale one from a prior identically-named Backup) must not let us drop the
1013+
// finalizer without ever purging the objects.
1014+
if !metav1.IsControlledBy(job, backup) || job.Labels[bucketLabelMode] != bucketModeCleanup {
1015+
return ctrl.Result{}, fmt.Errorf("Job %s/%s exists but is not this Backup's cleanup Job; refusing to act on it", backup.Namespace, jobName)
1016+
}
1017+
1018+
if !job.DeletionTimestamp.IsZero() {
1019+
// A prior failed attempt is being collected; wait for it to go, then the
1020+
// NotFound branch recreates a fresh one.
1021+
return ctrl.Result{RequeueAfter: bucketPollInterval}, nil
1022+
}
1023+
switch jobConditionState(job) {
1024+
case batchv1.JobComplete:
1025+
_ = r.deleteBucketCleanupJob(ctx, job)
1026+
logger.Debug("Bucket backup objects deleted", "backup", backup.Name, "prefix", snapshot.RepoPrefix)
1027+
return ctrl.Result{}, nil
1028+
case batchv1.JobFailed:
1029+
_ = r.deleteBucketCleanupJob(ctx, job)
1030+
logger.Debug("Bucket cleanup Job failed; retrying", "backup", backup.Name)
1031+
return ctrl.Result{RequeueAfter: bucketPollInterval}, nil
1032+
default:
1033+
return ctrl.Result{RequeueAfter: bucketPollInterval}, nil
1034+
}
1035+
}
1036+
1037+
func (r *BackupReconciler) deleteBucketCleanupJob(ctx context.Context, job *batchv1.Job) error {
1038+
policy := metav1.DeletePropagationBackground
1039+
return client.IgnoreNotFound(r.Delete(ctx, job, &client.DeleteOptions{PropagationPolicy: &policy}))
1040+
}
1041+
1042+
func (r *BackupReconciler) releaseBucketCleanup(ctx context.Context, backup *backupsv1alpha1.Backup, prefix, reason string) ctrl.Result {
1043+
getLogger(ctx).Info("releasing Backup without deleting its repo objects", "backup", backup.Name, "prefix", prefix, "reason", reason)
1044+
if r.Recorder != nil {
1045+
r.Recorder.Eventf(backup, corev1.EventTypeWarning, "ArtifactNotDeleted", "left objects under %s in the repo bucket: %s", prefix, reason)
8541046
}
1047+
return ctrl.Result{}
8551048
}

0 commit comments

Comments
 (0)