Skip to content

Commit aa81013

Browse files
committed
fix(backups): do not fail a backup on a transient retry-check read
The check for a Backup an earlier reconcile already created returned its read error, and the drivers turn any error from that path into a terminally Failed BackupJob. An apiserver timeout right after the operator backup completed therefore lost a backup whose data was safely in S3. The check only saves a probe, and the Create that follows already handles an existing Backup, so log the error and go on to Create. A trailing slash on the S3 endpoint is not a path prefix; the operators accept it, so do not refuse it. Pin the uncached AlreadyExists refetch, the upper slack of the MariaDB window and the probe timeout with tests. Assisted-by: LLM Signed-off-by: Andrey Kolkov <[email protected]>
1 parent 27ce981 commit aa81013

6 files changed

Lines changed: 190 additions & 22 deletions

File tree

‎internal/backupcontroller/artifact_size.go‎

Lines changed: 17 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -22,8 +22,8 @@ import (
2222
// makes to measure Backup.status.artifact.sizeBytes. The probe runs inside
2323
// the reconcile that creates the Backup, and the BackupJob controller has a
2424
// single worker, so an unreachable endpoint stalls every other BackupJob for
25-
// this long.
26-
const artifactSizeProbeTimeout = 10 * time.Second
25+
// this long. A variable so tests can shorten it.
26+
var artifactSizeProbeTimeout = 10 * time.Second
2727

2828
type s3ObjectInfo struct {
2929
Key string
@@ -63,8 +63,10 @@ type minioObjectLister struct {
6363
func newMinioObjectLister(conn s3Connection) (s3ObjectLister, error) {
6464
// An endpoint behind a path prefix cannot be addressed: minio-go has no
6565
// such option, and the SigV4 signature covers the request path, so the
66-
// prefix cannot be added below the client either.
67-
if strings.Contains(conn.Endpoint, "/") {
66+
// prefix cannot be added below the client either. A bare trailing slash
67+
// is not a prefix, and the operators accept it.
68+
endpoint := strings.TrimSuffix(conn.Endpoint, "/")
69+
if strings.Contains(endpoint, "/") {
6870
return nil, fmt.Errorf("endpoint %q has a path, which the S3 client cannot address", conn.Endpoint)
6971
}
7072
opts := &minio.Options{
@@ -90,7 +92,7 @@ func newMinioObjectLister(conn s3Connection) (s3ObjectLister, error) {
9092
tr.TLSClientConfig.RootCAs = pool
9193
opts.Transport = tr
9294
}
93-
client, err := minio.New(conn.Endpoint, opts)
95+
client, err := minio.New(endpoint, opts)
9496
if err != nil {
9597
return nil, err
9698
}
@@ -121,18 +123,20 @@ func collectObjects(ctx context.Context, objects <-chan minio.ObjectInfo) ([]s3O
121123

122124
// existingBackup returns the Backup a previous reconcile already created
123125
// for the job, so a retry skips the size probe whose result it would
124-
// discard. Returns (nil, nil) when there is none. It reads past the cache:
125-
// a retry typically follows a Create made moments earlier, which the cache
126-
// may not show yet.
127-
func (r *BackupJobReconciler) existingBackup(ctx context.Context, namespace, name string) (*backupsv1alpha1.Backup, error) {
126+
// discard. It reads past the cache: a retry typically follows a Create made
127+
// moments earlier, which the cache may not show yet. It returns nil both
128+
// when there is no Backup and when the read fails: it only saves a probe,
129+
// and the Create that follows handles an existing Backup on its own, so a
130+
// transient read error must not fail a backup that succeeded.
131+
func (r *BackupJobReconciler) existingBackup(ctx context.Context, namespace, name string) *backupsv1alpha1.Backup {
128132
existing := &backupsv1alpha1.Backup{}
129133
if err := r.apiReader().Get(ctx, types.NamespacedName{Namespace: namespace, Name: name}, existing); err != nil {
130-
if apierrors.IsNotFound(err) {
131-
return nil, nil
134+
if !apierrors.IsNotFound(err) {
135+
getLogger(ctx).Info("cannot check for an existing Backup; creating it", "backup", name, "error", err)
132136
}
133-
return nil, err
137+
return nil
134138
}
135-
return existing, nil
139+
return existing
136140
}
137141

138142
func (r *BackupJobReconciler) secretValue(ctx context.Context, namespace, name, key string) (string, error) {

‎internal/backupcontroller/artifact_size_test.go‎

Lines changed: 80 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -5,12 +5,15 @@ import (
55
"context"
66
"errors"
77
"testing"
8+
"time"
89

910
"github.com/minio/minio-go/v7"
1011

1112
corev1 "k8s.io/api/core/v1"
13+
apierrors "k8s.io/apimachinery/pkg/api/errors"
1214
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
1315
"sigs.k8s.io/controller-runtime/pkg/client"
16+
"sigs.k8s.io/controller-runtime/pkg/client/interceptor"
1417

1518
backupsv1alpha1 "github.com/cozystack/cozystack/api/backups/v1alpha1"
1619
)
@@ -87,8 +90,10 @@ func TestNewMinioObjectLister_RejectsEndpointPath(t *testing.T) {
8790
if _, err := newMinioObjectLister(s3Connection{Endpoint: "s3.example.org/prefix", Secure: true}); err == nil {
8891
t.Error("an endpoint with a path was accepted")
8992
}
90-
if _, err := newMinioObjectLister(s3Connection{Endpoint: "s3.example.org:9000", Secure: true}); err != nil {
91-
t.Errorf("host:port endpoint rejected: %v", err)
93+
for _, ok := range []string{"s3.example.org:9000", "s3.example.org/"} {
94+
if _, err := newMinioObjectLister(s3Connection{Endpoint: ok, Secure: true}); err != nil {
95+
t.Errorf("endpoint %q rejected: %v", ok, err)
96+
}
9297
}
9398
}
9499

@@ -101,8 +106,78 @@ func TestExistingBackup_ReadsPastTheCache(t *testing.T) {
101106
Client: newMariaDBStrategyTestClient(t),
102107
APIReader: newMariaDBStrategyTestClient(t, backup),
103108
}
104-
got, err := r.existingBackup(context.Background(), "tenant", "bj")
105-
if err != nil || got == nil {
106-
t.Fatalf("existingBackup = %v, %v; want the Backup the API server holds", got, err)
109+
if got := r.existingBackup(context.Background(), "tenant", "bj"); got == nil {
110+
t.Fatal("existingBackup = nil; want the Backup the API server holds")
111+
}
112+
}
113+
114+
// staleCache wraps a fake client so that Get reports NotFound for Backups,
115+
// the way an informer cache does right after another writer created one.
116+
func staleCache(t *testing.T, c client.Client) client.Client {
117+
t.Helper()
118+
return interceptor.NewClient(c.(client.WithWatch), interceptor.Funcs{
119+
Get: func(ctx context.Context, cl client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error {
120+
if _, ok := obj.(*backupsv1alpha1.Backup); ok {
121+
return apierrors.NewNotFound(backupsv1alpha1.GroupVersion.WithResource("backups").GroupResource(), key.Name)
122+
}
123+
return cl.Get(ctx, key, obj, opts...)
124+
},
125+
})
126+
}
127+
128+
// notFoundOnce wraps a fake client whose first Backup Get reports NotFound:
129+
// the retry check runs before the Backup exists, the refetch after Create
130+
// collides with it.
131+
func notFoundOnce(t *testing.T, c client.Client) client.Client {
132+
t.Helper()
133+
missed := false
134+
return interceptor.NewClient(c.(client.WithWatch), interceptor.Funcs{
135+
Get: func(ctx context.Context, cl client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error {
136+
if _, ok := obj.(*backupsv1alpha1.Backup); ok && !missed {
137+
missed = true
138+
return apierrors.NewNotFound(backupsv1alpha1.GroupVersion.WithResource("backups").GroupResource(), key.Name)
139+
}
140+
return cl.Get(ctx, key, obj, opts...)
141+
},
142+
})
143+
}
144+
145+
func TestExistingBackup_ReadErrorIsNotFatal(t *testing.T) {
146+
failing := interceptor.NewClient(newMariaDBStrategyTestClient(t).(client.WithWatch), interceptor.Funcs{
147+
Get: func(context.Context, client.WithWatch, client.ObjectKey, client.Object, ...client.GetOption) error {
148+
return errors.New("etcdserver: request timed out")
149+
},
150+
})
151+
r := &BackupJobReconciler{Client: newMariaDBStrategyTestClient(t), APIReader: failing}
152+
if got := r.existingBackup(context.Background(), "tenant", "bj"); got != nil {
153+
t.Errorf("existingBackup = %#v, want nil on a read error", got)
154+
}
155+
}
156+
157+
// blockingLister answers nothing until the probe's context is done, the way
158+
// a black-holed endpoint does.
159+
type blockingLister struct{}
160+
161+
func (blockingLister) ListObjects(ctx context.Context, _, _ string, _ bool) ([]s3ObjectInfo, error) {
162+
<-ctx.Done()
163+
return nil, ctx.Err()
164+
}
165+
166+
// withShortProbeTimeout shortens the probe timeout and fails the test if fn
167+
// is not back well before the default timeout would have expired.
168+
func withShortProbeTimeout(t *testing.T, fn func()) {
169+
t.Helper()
170+
saved := artifactSizeProbeTimeout
171+
artifactSizeProbeTimeout = 50 * time.Millisecond
172+
defer func() { artifactSizeProbeTimeout = saved }()
173+
done := make(chan struct{})
174+
go func() {
175+
fn()
176+
close(done)
177+
}()
178+
select {
179+
case <-done:
180+
case <-time.After(2 * time.Second):
181+
t.Fatal("the probe did not give up at its timeout")
107182
}
108183
}

‎internal/backupcontroller/cnpg_artifact_size_test.go‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,16 @@ func TestCNPGArtifactSizeBytes(t *testing.T) {
8383
}
8484
})
8585

86+
t.Run("a hanging endpoint is given up at the probe timeout", func(t *testing.T) {
87+
r := reconcilerWithLister(t, &fakeS3Lister{}, s3CredsSecret("tenant", "creds"))
88+
r.newS3ObjectLister = func(s3Connection) (s3ObjectLister, error) { return blockingLister{}, nil }
89+
withShortProbeTimeout(t, func() {
90+
if got := r.cnpgArtifactSizeBytes(context.Background(), "tenant", newSizedBarmanObjectStore(), "srv", "20261021T100000"); got != 0 {
91+
t.Errorf("size = %d, want 0", got)
92+
}
93+
})
94+
})
95+
8696
t.Run("an empty endpointURL means AWS", func(t *testing.T) {
8797
store := newSizedBarmanObjectStore()
8898
store.EndpointURL = ""
@@ -306,3 +316,21 @@ func TestCreateCNPGBackupArtifact_ExistingBackupSkipsProbe(t *testing.T) {
306316
t.Errorf("S3 listed %d times for a Backup that already exists", l.calls)
307317
}
308318
}
319+
320+
func TestCreateCNPGBackupArtifact_AlreadyExistsRefetchReadsPastTheCache(t *testing.T) {
321+
apiGroup := backupsv1alpha1.DefaultApplicationAPIGroup
322+
j := &backupsv1alpha1.BackupJob{
323+
ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj-race"},
324+
Spec: backupsv1alpha1.BackupJobSpec{
325+
ApplicationRef: corev1.TypedLocalObjectReference{APIGroup: &apiGroup, Kind: postgresAppKind, Name: "pg"},
326+
},
327+
}
328+
backup := &backupsv1alpha1.Backup{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj-race"}}
329+
store := newCNPGStrategyTestClient(t, backup)
330+
r := &BackupJobReconciler{Client: staleCache(t, store), APIReader: notFoundOnce(t, store)}
331+
rendered := &strategyv1alpha1.CNPGTemplate{BarmanObjectStore: strategyv1alpha1.BarmanObjectStoreTemplate{DestinationPath: "s3://b/"}}
332+
got, err := r.createCNPGBackupArtifact(context.Background(), j, &ResolvedBackupConfig{}, &cnpgtypes.Backup{ObjectMeta: metav1.ObjectMeta{Name: "cnpg-bk"}}, "postgres-pg", "srv", "postgresql", rendered, newPostgresApp("pg", "tenant"))
333+
if err != nil || got == nil || got.Name != "bj-race" {
334+
t.Fatalf("createCNPGBackupArtifact = %v, %v; want the Backup that already exists", got, err)
335+
}
336+
}

‎internal/backupcontroller/cnpgstrategy_controller.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -815,8 +815,8 @@ func (r *BackupJobReconciler) createCNPGBackupArtifact(
815815
rendered *strategyv1alpha1.CNPGTemplate,
816816
sourceApp *postgresapp.Postgres,
817817
) (*backupsv1alpha1.Backup, error) {
818-
if existing, err := r.existingBackup(ctx, j.Namespace, j.Name); err != nil || existing != nil {
819-
return existing, err
818+
if existing := r.existingBackup(ctx, j.Namespace, j.Name); existing != nil {
819+
return existing, nil
820820
}
821821
takenAt := metav1.Now()
822822
if cnpgBackup.Status.StartedAt != nil && !cnpgBackup.Status.StartedAt.IsZero() {

‎internal/backupcontroller/mariadb_artifact_size_test.go‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,8 @@ import (
1313

1414
corev1 "k8s.io/api/core/v1"
1515
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
16+
"sigs.k8s.io/controller-runtime/pkg/client"
17+
"sigs.k8s.io/controller-runtime/pkg/client/interceptor"
1618

1719
strategyv1alpha1 "github.com/cozystack/cozystack/api/backups/strategy/v1alpha1"
1820
backupsv1alpha1 "github.com/cozystack/cozystack/api/backups/v1alpha1"
@@ -151,6 +153,25 @@ func TestMariaDBArtifactSizeBytes(t *testing.T) {
151153
}
152154
})
153155

156+
t.Run("a dump uploaded just after Complete is inside the slack", func(t *testing.T) {
157+
l := &fakeS3Lister{objects: []s3ObjectInfo{
158+
{Key: dumpKey(prefix, completed.Add(30*time.Second), "gzip"), Size: 5, LastModified: completed.Add(30 * time.Second)},
159+
}}
160+
if got := reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds")).mariadbArtifactSizeBytes(context.Background(), "tenant", s3, mdbBackup, completed); got != 5 {
161+
t.Errorf("size = %d, want 5", got)
162+
}
163+
})
164+
165+
t.Run("a hanging endpoint is given up at the probe timeout", func(t *testing.T) {
166+
r := reconcilerWithLister(t, &fakeS3Lister{}, s3CredsSecret("tenant", "creds"))
167+
r.newS3ObjectLister = func(s3Connection) (s3ObjectLister, error) { return blockingLister{}, nil }
168+
withShortProbeTimeout(t, func() {
169+
if got := r.mariadbArtifactSizeBytes(context.Background(), "tenant", s3, mdbBackup, completed); got != 0 {
170+
t.Errorf("size = %d, want 0", got)
171+
}
172+
})
173+
})
174+
154175
t.Run("two dumps in the window leave size unset", func(t *testing.T) {
155176
l := &fakeS3Lister{objects: []s3ObjectInfo{
156177
{Key: dumpKey(prefix, created.Add(time.Minute), "gzip"), Size: 1, LastModified: created.Add(time.Minute)},
@@ -298,3 +319,43 @@ func TestCreateMariaDBBackupArtifact_ExistingBackupSkipsProbe(t *testing.T) {
298319
t.Errorf("Artifact = %#v, want the existing one", got.Status.Artifact)
299320
}
300321
}
322+
323+
func TestCreateMariaDBBackupArtifact_TransientRetryReadStillCreates(t *testing.T) {
324+
apps := mariadbapp.GroupName
325+
job := &backupsv1alpha1.BackupJob{
326+
ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj-transient"},
327+
Spec: backupsv1alpha1.BackupJobSpec{
328+
ApplicationRef: corev1.TypedLocalObjectReference{Kind: "MariaDB", Name: "src", APIGroup: &apps},
329+
},
330+
}
331+
r := reconcilerWithLister(t, &fakeS3Lister{}, job)
332+
r.APIReader = interceptor.NewClient(newMariaDBStrategyTestClient(t).(client.WithWatch), interceptor.Funcs{
333+
Get: func(context.Context, client.WithWatch, client.ObjectKey, client.Object, ...client.GetOption) error {
334+
return errors.New("etcdserver: request timed out")
335+
},
336+
})
337+
got, err := r.createMariaDBBackupArtifact(context.Background(), job, &ResolvedBackupConfig{}, &mariadbtypes.Backup{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "op"}}, newRenderedMariaDBTemplate())
338+
if err != nil || got == nil || got.Name != "bj-transient" {
339+
t.Fatalf("createMariaDBBackupArtifact = %v, %v; a transient read must not fail the backup", got, err)
340+
}
341+
if err := r.Get(context.Background(), client.ObjectKeyFromObject(got), &backupsv1alpha1.Backup{}); err != nil {
342+
t.Errorf("the Backup was not created: %v", err)
343+
}
344+
}
345+
346+
func TestCreateMariaDBBackupArtifact_AlreadyExistsRefetchReadsPastTheCache(t *testing.T) {
347+
apps := mariadbapp.GroupName
348+
job := &backupsv1alpha1.BackupJob{
349+
ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj-race"},
350+
Spec: backupsv1alpha1.BackupJobSpec{
351+
ApplicationRef: corev1.TypedLocalObjectReference{Kind: "MariaDB", Name: "src", APIGroup: &apps},
352+
},
353+
}
354+
backup := &backupsv1alpha1.Backup{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj-race"}}
355+
store := newMariaDBStrategyTestClient(t, job, backup)
356+
r := &BackupJobReconciler{Client: staleCache(t, store), APIReader: notFoundOnce(t, store), Scheme: store.Scheme()}
357+
got, err := r.createMariaDBBackupArtifact(context.Background(), job, &ResolvedBackupConfig{}, &mariadbtypes.Backup{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "op"}}, &strategyv1alpha1.MariaDBTemplate{})
358+
if err != nil || got == nil || got.Name != "bj-race" {
359+
t.Fatalf("createMariaDBBackupArtifact = %v, %v; want the Backup that already exists", got, err)
360+
}
361+
}

‎internal/backupcontroller/mariadbstrategy_controller.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -365,8 +365,8 @@ func (r *BackupJobReconciler) createMariaDBBackupArtifact(
365365
mdbBackup *mariadbtypes.Backup,
366366
rendered *strategyv1alpha1.MariaDBTemplate,
367367
) (*backupsv1alpha1.Backup, error) {
368-
if existing, err := r.existingBackup(ctx, j.Namespace, j.Name); err != nil || existing != nil {
369-
return existing, err
368+
if existing := r.existingBackup(ctx, j.Namespace, j.Name); existing != nil {
369+
return existing, nil
370370
}
371371
takenAt := metav1.Now()
372372
if cond := apimeta.FindStatusCondition(mdbBackup.Status.Conditions, mariadbtypes.ConditionTypeComplete); cond != nil && !cond.LastTransitionTime.IsZero() {

0 commit comments

Comments
 (0)