Skip to content

Commit 9a57150

Browse files
committed
fix(backups): retry instead of failing when an existing Backup cannot be read
When Create collides with a Backup an earlier reconcile already made and the read that follows fails, the drivers failed the BackupJob, although the backup and its Backup both exist. Return a distinct error instead, which the drivers hand back to the controller so the reconcile is retried with backoff and then finds the Backup. minio.New already refuses an endpoint with a path prefix and accepts a trailing slash, so drop the duplicate check in the shared client; the CNPG probe keeps its own check because it parses the URL itself and would otherwise drop the path before minio sees it. Say in the sizeBytes API doc what the value covers, such as the base backup without the shared WAL archive for CNPG, and that strategies which cannot attribute stored bytes to one backup leave it unset. Assisted-by: LLM Signed-off-by: Andrey Kolkov <[email protected]>
1 parent aa81013 commit 9a57150

8 files changed

Lines changed: 116 additions & 32 deletions

File tree

‎api/backups/v1alpha1/backup_types.go‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -37,9 +37,14 @@ type BackupArtifact struct {
3737
// For example: s3://bucket/prefix/file.tar.gz
3838
URI string `json:"uri"`
3939

40-
// SizeBytes is the size of the artifact in bytes, if known. Drivers
41-
// leave it unset rather than writing 0 when they cannot measure the
42-
// stored size, so an absent value means "unknown", not "empty".
40+
// SizeBytes is the stored size in bytes of the files this backup owns,
41+
// if known: the dump of a MariaDB backup, for example, or the base backup
42+
// of a CNPG one, which leaves out the WAL archive the server's backups
43+
// share and a point-in-time restore also reads. Strategies that cannot
44+
// attribute stored bytes to a single backup, such as Velero's, whose
45+
// repository deduplicates across backups, leave it unset. Drivers never
46+
// write 0 for a size they cannot measure, so an absent value means
47+
// "unknown", not "empty".
4348
// +optional
4449
SizeBytes int64 `json:"sizeBytes,omitempty"`
4550

‎internal/backupcontroller/artifact_size.go‎

Lines changed: 20 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -6,14 +6,14 @@ import (
66
"crypto/x509"
77
"errors"
88
"fmt"
9-
"strings"
109
"time"
1110

1211
"github.com/minio/minio-go/v7"
1312
"github.com/minio/minio-go/v7/pkg/credentials"
1413
corev1 "k8s.io/api/core/v1"
1514
apierrors "k8s.io/apimachinery/pkg/api/errors"
1615
"k8s.io/apimachinery/pkg/types"
16+
ctrl "sigs.k8s.io/controller-runtime"
1717

1818
backupsv1alpha1 "github.com/cozystack/cozystack/api/backups/v1alpha1"
1919
)
@@ -38,7 +38,7 @@ type s3ObjectLister interface {
3838
// s3Connection is what a driver resolves from its rendered strategy to
3939
// read the bucket with the same identity the backup was written with.
4040
type s3Connection struct {
41-
// Endpoint is host[:port], without a scheme or a path.
41+
// Endpoint is host[:port], without a scheme.
4242
Endpoint string
4343
Secure bool
4444
Region string
@@ -61,14 +61,6 @@ type minioObjectLister struct {
6161
}
6262

6363
func newMinioObjectLister(conn s3Connection) (s3ObjectLister, error) {
64-
// An endpoint behind a path prefix cannot be addressed: minio-go has no
65-
// such option, and the SigV4 signature covers the request path, so the
66-
// prefix cannot be added below the client either. A bare trailing slash
67-
// is not a prefix, and the operators accept it.
68-
endpoint := strings.TrimSuffix(conn.Endpoint, "/")
69-
if strings.Contains(endpoint, "/") {
70-
return nil, fmt.Errorf("endpoint %q has a path, which the S3 client cannot address", conn.Endpoint)
71-
}
7264
opts := &minio.Options{
7365
Creds: credentials.NewStaticV4(conn.AccessKey, conn.SecretKey, conn.SessionToken),
7466
Region: conn.Region,
@@ -92,7 +84,9 @@ func newMinioObjectLister(conn s3Connection) (s3ObjectLister, error) {
9284
tr.TLSClientConfig.RootCAs = pool
9385
opts.Transport = tr
9486
}
95-
client, err := minio.New(endpoint, opts)
87+
// minio.New refuses an endpoint with a path prefix, which it cannot
88+
// address, and accepts a bare trailing slash.
89+
client, err := minio.New(conn.Endpoint, opts)
9690
if err != nil {
9791
return nil, err
9892
}
@@ -121,6 +115,21 @@ func collectObjects(ctx context.Context, objects <-chan minio.ObjectInfo) ([]s3O
121115
return out, nil
122116
}
123117

118+
// errBackupNotReadBack reports that Create collided with an existing Backup
119+
// that could then not be read. The backup itself succeeded, so callers
120+
// retry the reconcile instead of failing the BackupJob.
121+
var errBackupNotReadBack = errors.New("the Backup already exists but could not be read back")
122+
123+
// failBackupArtifact handles an error from creating the Backup of a
124+
// completed backup: errBackupNotReadBack is retried through the controller's
125+
// backoff, anything else fails the BackupJob.
126+
func (r *BackupJobReconciler) failBackupArtifact(ctx context.Context, j *backupsv1alpha1.BackupJob, err error) (ctrl.Result, error) {
127+
if errors.Is(err, errBackupNotReadBack) {
128+
return ctrl.Result{}, err
129+
}
130+
return r.markBackupJobFailed(ctx, j, fmt.Sprintf("failed to create Backup artifact: %v", err))
131+
}
132+
124133
// existingBackup returns the Backup a previous reconcile already created
125134
// for the job, so a retry skips the size probe whose result it would
126135
// discard. It reads past the cache: a retry typically follows a Create made

‎internal/backupcontroller/artifact_size_test.go‎

Lines changed: 41 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ package backupcontroller
44
import (
55
"context"
66
"errors"
7+
"fmt"
78
"testing"
89
"time"
910

@@ -86,17 +87,6 @@ func TestCollectObjects(t *testing.T) {
8687
}
8788
}
8889

89-
func TestNewMinioObjectLister_RejectsEndpointPath(t *testing.T) {
90-
if _, err := newMinioObjectLister(s3Connection{Endpoint: "s3.example.org/prefix", Secure: true}); err == nil {
91-
t.Error("an endpoint with a path was accepted")
92-
}
93-
for _, ok := range []string{"s3.example.org:9000", "s3.example.org/"} {
94-
if _, err := newMinioObjectLister(s3Connection{Endpoint: ok, Secure: true}); err != nil {
95-
t.Errorf("endpoint %q rejected: %v", ok, err)
96-
}
97-
}
98-
}
99-
10090
// TestExistingBackup_ReadsPastTheCache pins that the retry check sees a
10191
// Backup the cache does not show yet: the previous reconcile created it
10292
// moments earlier.
@@ -181,3 +171,43 @@ func withShortProbeTimeout(t *testing.T, fn func()) {
181171
t.Fatal("the probe did not give up at its timeout")
182172
}
183173
}
174+
175+
// failingReads wraps a fake client whose every Get fails, like an apiserver
176+
// that times out.
177+
func failingReads(t *testing.T, c client.Client) client.Client {
178+
t.Helper()
179+
return interceptor.NewClient(c.(client.WithWatch), interceptor.Funcs{
180+
Get: func(context.Context, client.WithWatch, client.ObjectKey, client.Object, ...client.GetOption) error {
181+
return errors.New("etcdserver: request timed out")
182+
},
183+
})
184+
}
185+
186+
func TestFailBackupArtifact(t *testing.T) {
187+
newJob := func() *backupsv1alpha1.BackupJob {
188+
return &backupsv1alpha1.BackupJob{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj"}}
189+
}
190+
191+
t.Run("a Backup that exists but cannot be read is retried", func(t *testing.T) {
192+
j := newJob()
193+
r := &BackupJobReconciler{Client: newMariaDBStrategyTestClient(t, j)}
194+
_, err := r.failBackupArtifact(context.Background(), j, fmt.Errorf("%w: timeout", errBackupNotReadBack))
195+
if !errors.Is(err, errBackupNotReadBack) {
196+
t.Errorf("err = %v, want it returned so the reconcile is retried", err)
197+
}
198+
if j.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed {
199+
t.Error("the BackupJob was failed although its Backup exists")
200+
}
201+
})
202+
203+
t.Run("any other error fails the BackupJob", func(t *testing.T) {
204+
j := newJob()
205+
r := &BackupJobReconciler{Client: newMariaDBStrategyTestClient(t, j)}
206+
if _, err := r.failBackupArtifact(context.Background(), j, errors.New("admission webhook denied")); err != nil {
207+
t.Fatalf("failBackupArtifact: %v", err)
208+
}
209+
if j.Status.Phase != backupsv1alpha1.BackupJobPhaseFailed {
210+
t.Errorf("phase = %q, want Failed", j.Status.Phase)
211+
}
212+
})
213+
}

‎internal/backupcontroller/cnpg_artifact_size_test.go‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -334,3 +334,21 @@ func TestCreateCNPGBackupArtifact_AlreadyExistsRefetchReadsPastTheCache(t *testi
334334
t.Fatalf("createCNPGBackupArtifact = %v, %v; want the Backup that already exists", got, err)
335335
}
336336
}
337+
338+
func TestCreateCNPGBackupArtifact_UnreadableExistingBackupIsRetried(t *testing.T) {
339+
apiGroup := backupsv1alpha1.DefaultApplicationAPIGroup
340+
j := &backupsv1alpha1.BackupJob{
341+
ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj-unreadable"},
342+
Spec: backupsv1alpha1.BackupJobSpec{
343+
ApplicationRef: corev1.TypedLocalObjectReference{APIGroup: &apiGroup, Kind: postgresAppKind, Name: "pg"},
344+
},
345+
}
346+
backup := &backupsv1alpha1.Backup{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj-unreadable"}}
347+
store := newCNPGStrategyTestClient(t, backup)
348+
r := &BackupJobReconciler{Client: staleCache(t, store), APIReader: failingReads(t, store)}
349+
rendered := &strategyv1alpha1.CNPGTemplate{BarmanObjectStore: strategyv1alpha1.BarmanObjectStoreTemplate{DestinationPath: "s3://b/"}}
350+
_, err := r.createCNPGBackupArtifact(context.Background(), j, &ResolvedBackupConfig{}, &cnpgtypes.Backup{ObjectMeta: metav1.ObjectMeta{Name: "cnpg-bk"}}, "postgres-pg", "srv", "postgresql", rendered, newPostgresApp("pg", "tenant"))
351+
if !errors.Is(err, errBackupNotReadBack) {
352+
t.Errorf("err = %v, want errBackupNotReadBack", err)
353+
}
354+
}

‎internal/backupcontroller/cnpgstrategy_controller.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -435,7 +435,7 @@ func (r *BackupJobReconciler) reconcileCNPG(ctx context.Context, j *backupsv1alp
435435
}
436436
artifact, err := r.createCNPGBackupArtifact(ctx, j, resolved, cnpgBackup, clusterName, serverName, attachment.flavor, rendered, app)
437437
if err != nil {
438-
return r.markBackupJobFailed(ctx, j, fmt.Sprintf("failed to create Backup artifact: %v", err))
438+
return r.failBackupArtifact(ctx, j, err)
439439
}
440440
now := metav1.Now()
441441
j.Status.BackupRef = &corev1.LocalObjectReference{Name: artifact.Name}
@@ -877,7 +877,7 @@ func (r *BackupJobReconciler) createCNPGBackupArtifact(
877877
}
878878
existing := &backupsv1alpha1.Backup{}
879879
if getErr := r.apiReader().Get(ctx, types.NamespacedName{Namespace: backup.Namespace, Name: backup.Name}, existing); getErr != nil {
880-
return nil, getErr
880+
return nil, fmt.Errorf("%w: %v", errBackupNotReadBack, getErr)
881881
}
882882
return existing, nil
883883
}

‎internal/backupcontroller/mariadb_artifact_size_test.go‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -359,3 +359,20 @@ func TestCreateMariaDBBackupArtifact_AlreadyExistsRefetchReadsPastTheCache(t *te
359359
t.Fatalf("createMariaDBBackupArtifact = %v, %v; want the Backup that already exists", got, err)
360360
}
361361
}
362+
363+
func TestCreateMariaDBBackupArtifact_UnreadableExistingBackupIsRetried(t *testing.T) {
364+
apps := mariadbapp.GroupName
365+
job := &backupsv1alpha1.BackupJob{
366+
ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj-unreadable"},
367+
Spec: backupsv1alpha1.BackupJobSpec{
368+
ApplicationRef: corev1.TypedLocalObjectReference{Kind: "MariaDB", Name: "src", APIGroup: &apps},
369+
},
370+
}
371+
backup := &backupsv1alpha1.Backup{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj-unreadable"}}
372+
store := newMariaDBStrategyTestClient(t, job, backup)
373+
r := &BackupJobReconciler{Client: staleCache(t, store), APIReader: failingReads(t, store), Scheme: store.Scheme()}
374+
_, err := r.createMariaDBBackupArtifact(context.Background(), job, &ResolvedBackupConfig{}, &mariadbtypes.Backup{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "op"}}, &strategyv1alpha1.MariaDBTemplate{})
375+
if !errors.Is(err, errBackupNotReadBack) {
376+
t.Errorf("err = %v, want errBackupNotReadBack", err)
377+
}
378+
}

‎internal/backupcontroller/mariadbstrategy_controller.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -214,7 +214,7 @@ func (r *BackupJobReconciler) reconcileMariaDB(ctx context.Context, j *backupsv1
214214
}
215215
artifact, err := r.createMariaDBBackupArtifact(ctx, j, resolved, mdbBackup, rendered)
216216
if err != nil {
217-
return r.markBackupJobFailed(ctx, j, fmt.Sprintf("failed to create Backup artifact: %v", err))
217+
return r.failBackupArtifact(ctx, j, err)
218218
}
219219
now := metav1.Now()
220220
j.Status.BackupRef = &corev1.LocalObjectReference{Name: artifact.Name}
@@ -422,7 +422,7 @@ func (r *BackupJobReconciler) createMariaDBBackupArtifact(
422422
}
423423
existing := &backupsv1alpha1.Backup{}
424424
if getErr := r.apiReader().Get(ctx, types.NamespacedName{Namespace: backup.Namespace, Name: backup.Name}, existing); getErr != nil {
425-
return nil, getErr
425+
return nil, fmt.Errorf("%w: %v", errBackupNotReadBack, getErr)
426426
}
427427
return existing, nil
428428
}

‎packages/system/backup-controller/definitions/backups.cozystack.io_backups.yaml‎

Lines changed: 8 additions & 3 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)