Skip to content

Commit 59f0013

Browse files
fix(backups): attribute a MariaDB dump only when no other run can own it
Every backup of a MariaDB application writes its dump under one shared prefix, so the probe took the dump whose file-name timestamp (the backup pod's clock) and LastModified (the object store's clock) both fall in the operator Backup's lifetime, with a minute of slack for skew. That keeps a neighbour's dump out only while this run's own dump is in the window. With the pod clock more than the slack behind, the own dump is named before the window opens, and a previous run that finished within the minute before this one was created becomes the single match: a 123-byte dump was reported as 666 bytes. A pod clock ahead or an object store clock behind does the same with the next run, and an own dump that is gone by the time it is looked up, as it can be for a restore long after the backup, leaves a neighbour as the single match with no skew at all. A dump is now attributed only when ownership is proven: - it is the one dump that either clock places in the window. A dump that one clock places there and the other does not may be this run's own with a skewed clock, so it leaves any other match unproven, the way a second dump in the window already did, and is not taken itself; - both of its clocks place it in the window; - neither clock places it in the window of another operator Backup that writes under the same bucket and prefix and has completed. This rejects a neighbour's dump when this run's own is missing, which the clocks alone cannot tell. Operator Backups still running are not counted as neighbours: their window has no end, and one stuck short of Complete (a Job whose pod never starts stays JobRunning for good) would otherwise keep every later run of the application from being sized. A dump such a run writes inside this run's window still competes with this run's own. Narrower slack does not prove ownership: a neighbour that finished within the narrower slack still matches, and a smaller skew pushes the own dump out. The Job's start and completion times are stamped by the control plane, like the operator Backup's own, so they narrow the window by seconds without removing the dependence on the two clocks a dump carries. What is left: a neighbour's dump is taken only when this run's own is missing from the listing or misplaced on both clocks and the neighbour's operator Backup no longer exists or has not completed. Every other unproven case leaves the size unset, as two runs that overlap or follow each other within the slack already did. Assisted-by: LLM Signed-off-by: Myasnikov Daniil <[email protected]>
1 parent 560c44f commit 59f0013

3 files changed

Lines changed: 250 additions & 23 deletions

File tree

‎internal/backupcontroller/artifact_size.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,8 @@ type s3Connection struct {
6060
// so that the restore path, which has a reconciler of its own, can read
6161
// the same objects.
6262
type objectStore struct {
63-
// reader must reach Secrets; the manager's client reads them uncached.
63+
// reader reads Secrets, which the manager's client fetches uncached,
64+
// and lists operator Backups, which it serves from the cache.
6465
reader client.Reader
6566
// newLister overrides the S3 client; nil means a real minio client.
6667
newLister func(conn s3Connection) (s3ObjectLister, error)

‎internal/backupcontroller/mariadb_artifact_size.go‎

Lines changed: 96 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,10 @@ import (
88
"strings"
99
"time"
1010

11+
apimeta "k8s.io/apimachinery/pkg/api/meta"
12+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
13+
"sigs.k8s.io/controller-runtime/pkg/client"
14+
1115
"github.com/cozystack/cozystack/internal/backupcontroller/mariadbtypes"
1216
)
1317

@@ -60,6 +64,10 @@ func findMariaDBDump(ctx context.Context, store objectStore, mdbBackup *mariadbt
6064
if s3 == nil {
6165
return mariadbDump{}, errors.New("the operator Backup does not store to S3")
6266
}
67+
others, err := finishedMariaDBRuns(ctx, store.reader, mdbBackup)
68+
if err != nil {
69+
return mariadbDump{}, fmt.Errorf("list operator Backups: %w", err)
70+
}
6371
conn, err := mariadbS3Connection(ctx, store, mdbBackup.Namespace, s3)
6472
if err != nil {
6573
return mariadbDump{}, fmt.Errorf("resolve S3 credentials: %w", err)
@@ -76,35 +84,104 @@ func findMariaDBDump(ctx context.Context, store objectStore, mdbBackup *mariadbt
7684
if err != nil {
7785
return mariadbDump{}, fmt.Errorf("list s3://%s/%s: %w", s3.Bucket, prefix, err)
7886
}
79-
return matchMariaDBDump(objects, prefix, mdbBackup.CreationTimestamp.Time, completedAt)
87+
own := newMariaDBRun(mdbBackup, mdbBackup.CreationTimestamp.Time, completedAt)
88+
return matchMariaDBDump(objects, prefix, own, others)
89+
}
90+
91+
// mariadbRun is the span in which an operator Backup can have written its
92+
// dump: its lifetime, widened by mariadbSizeProbeClockSkew on each side.
93+
type mariadbRun struct {
94+
name string
95+
lower, upper time.Time
96+
}
97+
98+
func newMariaDBRun(b *mariadbtypes.Backup, created, completed time.Time) mariadbRun {
99+
return mariadbRun{
100+
name: b.Namespace + "/" + b.Name,
101+
lower: created.Add(-mariadbSizeProbeClockSkew),
102+
upper: completed.Add(mariadbSizeProbeClockSkew),
103+
}
104+
}
105+
106+
func (r mariadbRun) placesOnBothClocks(d mariadbDump) bool {
107+
return inWindow(d.Timestamp, r.lower, r.upper) && inWindow(d.LastModified, r.lower, r.upper)
80108
}
81109

82-
// matchMariaDBDump picks the dump of the run that lived from created to
83-
// completed. The prefix is shared by every backup of the application, so
84-
// the dump is identified by both the timestamp in its file name and its
85-
// LastModified falling inside that lifetime: the two come from different
86-
// clocks, so a skewed pod clock alone cannot pull a neighbouring run's dump
87-
// into the window.
88-
func matchMariaDBDump(objects []s3ObjectInfo, prefix string, created, completed time.Time) (mariadbDump, error) {
89-
lower := created.Add(-mariadbSizeProbeClockSkew)
90-
upper := completed.Add(mariadbSizeProbeClockSkew)
91-
var matched []mariadbDump
110+
func (r mariadbRun) placesOnEitherClock(d mariadbDump) bool {
111+
return inWindow(d.Timestamp, r.lower, r.upper) || inWindow(d.LastModified, r.lower, r.upper)
112+
}
113+
114+
// matchMariaDBDump picks the dump of run own among objects, which share a
115+
// prefix with every other run of the application. A dump carries two
116+
// clocks, the backup pod's in its file name and the object store's in its
117+
// LastModified, and either can be skewed. A dump is taken only when it is
118+
// the one dump either clock places in own's window, both clocks place it
119+
// there, and neither places it in the window of another finished run: a
120+
// dump in the window on one clock only may be this run's own, misplaced by
121+
// a skewed clock, which leaves any other candidate unproven. A neighbour's
122+
// dump is still taken if this run's own is missing from the listing, or
123+
// misplaced on both clocks, and the neighbour's operator Backup is gone or
124+
// has not completed.
125+
func matchMariaDBDump(objects []s3ObjectInfo, prefix string, own mariadbRun, others []mariadbRun) (mariadbDump, error) {
126+
var near []mariadbDump
92127
for _, obj := range objects {
93128
name := strings.TrimPrefix(obj.Key, prefix)
94129
ts, ok := parseMariaDBLogicalBackupFile(name)
95-
if !ok || !inWindow(ts, lower, upper) || !inWindow(obj.LastModified, lower, upper) {
130+
if !ok {
96131
continue
97132
}
98-
matched = append(matched, mariadbDump{Key: obj.Key, Name: name, Timestamp: ts, Size: obj.Size, LastModified: obj.LastModified})
133+
d := mariadbDump{Key: obj.Key, Name: name, Timestamp: ts, Size: obj.Size, LastModified: obj.LastModified}
134+
if own.placesOnEitherClock(d) {
135+
near = append(near, d)
136+
}
99137
}
100-
switch len(matched) {
101-
case 0:
138+
if len(near) == 0 {
102139
return mariadbDump{}, errMariaDBDumpNotFound
103-
case 1:
104-
return matched[0], nil
105-
default:
106-
return mariadbDump{}, fmt.Errorf("%w: %d dumps fall in its lifetime", errMariaDBDumpAmbiguous, len(matched))
107140
}
141+
if len(near) > 1 {
142+
return mariadbDump{}, fmt.Errorf("%w: %d dumps fall in its lifetime on at least one clock", errMariaDBDumpAmbiguous, len(near))
143+
}
144+
d := near[0]
145+
if !own.placesOnBothClocks(d) {
146+
return mariadbDump{}, fmt.Errorf("%w: %s falls in its lifetime on one clock only", errMariaDBDumpAmbiguous, d.Name)
147+
}
148+
for _, other := range others {
149+
if other.placesOnEitherClock(d) {
150+
return mariadbDump{}, fmt.Errorf("%w: %s also falls in the lifetime of operator Backup %s", errMariaDBDumpAmbiguous, d.Name, other.name)
151+
}
152+
}
153+
return d, nil
154+
}
155+
156+
// finishedMariaDBRuns returns the runs of the other operator Backups that
157+
// write under the same bucket and prefix as b and have completed, whether
158+
// their Job succeeded or failed. One still running is left out: its window
159+
// has no end yet, and an operator Backup stuck short of Complete, as one
160+
// whose Job pod never starts is, would otherwise contest every later run of
161+
// the application for good. The endpoint is not compared, since one has
162+
// several spellings, and a false neighbour only leaves a size unset.
163+
func finishedMariaDBRuns(ctx context.Context, reader client.Reader, b *mariadbtypes.Backup) ([]mariadbRun, error) {
164+
list := &mariadbtypes.BackupList{}
165+
if err := reader.List(ctx, list); err != nil {
166+
return nil, err
167+
}
168+
var runs []mariadbRun
169+
for i := range list.Items {
170+
other := &list.Items[i]
171+
if other.Namespace == b.Namespace && other.Name == b.Name {
172+
continue
173+
}
174+
s3 := other.Spec.Storage.S3
175+
if s3 == nil || s3.Bucket != b.Spec.Storage.S3.Bucket || mariadbOperatorS3Prefix(s3.Prefix) != mariadbOperatorS3Prefix(b.Spec.Storage.S3.Prefix) {
176+
continue
177+
}
178+
cond := apimeta.FindStatusCondition(other.Status.Conditions, mariadbtypes.ConditionTypeComplete)
179+
if cond == nil || cond.Status != metav1.ConditionTrue {
180+
continue
181+
}
182+
runs = append(runs, newMariaDBRun(other, other.CreationTimestamp.Time, cond.LastTransitionTime.Time))
183+
}
184+
return runs, nil
108185
}
109186

110187
// mariadbS3Connection resolves the Secrets and TLS settings of the operator

‎internal/backupcontroller/mariadb_artifact_size_test.go‎

Lines changed: 152 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -239,8 +239,9 @@ func TestMatchMariaDBDump(t *testing.T) {
239239
completed := created.Add(3 * time.Minute)
240240
const prefix = "tenant/db/"
241241
own := s3ObjectInfo{Key: dumpKey(prefix, created.Add(time.Minute), "gzip"), Size: 4242, LastModified: created.Add(2 * time.Minute)}
242+
run := newMariaDBRun(&mariadbtypes.Backup{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "op-bk"}}, created, completed)
242243

243-
got, err := matchMariaDBDump([]s3ObjectInfo{own}, prefix, created, completed)
244+
got, err := matchMariaDBDump([]s3ObjectInfo{own}, prefix, run, nil)
244245
if err != nil {
245246
t.Fatalf("matchMariaDBDump: %v", err)
246247
}
@@ -255,11 +256,11 @@ func TestMatchMariaDBDump(t *testing.T) {
255256
t.Errorf("matchMariaDBDump = %+v, want %+v", got, want)
256257
}
257258

258-
if _, err := matchMariaDBDump(nil, prefix, created, completed); !errors.Is(err, errMariaDBDumpNotFound) {
259+
if _, err := matchMariaDBDump(nil, prefix, run, nil); !errors.Is(err, errMariaDBDumpNotFound) {
259260
t.Errorf("no dump: err = %v, want errMariaDBDumpNotFound", err)
260261
}
261262
second := s3ObjectInfo{Key: dumpKey(prefix, created.Add(2*time.Minute), "gzip"), Size: 1, LastModified: created.Add(2 * time.Minute)}
262-
if _, err := matchMariaDBDump([]s3ObjectInfo{own, second}, prefix, created, completed); !errors.Is(err, errMariaDBDumpAmbiguous) {
263+
if _, err := matchMariaDBDump([]s3ObjectInfo{own, second}, prefix, run, nil); !errors.Is(err, errMariaDBDumpAmbiguous) {
263264
t.Errorf("two dumps: err = %v, want errMariaDBDumpAmbiguous", err)
264265
}
265266
}
@@ -303,6 +304,154 @@ func TestFindMariaDBDump(t *testing.T) {
303304
}
304305
}
305306

307+
// TestFindMariaDBDump_Ownership drives runs of one application that share
308+
// a prefix. This run lives from t0 to t0+40s, so its window with the
309+
// one-minute slack is [t0-1m, t0+1m40s]; its own dump is 123 bytes and a
310+
// neighbour's is 666 or 777. A neighbour's dump must never be returned.
311+
func TestFindMariaDBDump_Ownership(t *testing.T) {
312+
t0 := time.Date(2026, 10, 21, 10, 0, 0, 0, time.UTC)
313+
s3 := newRenderedMariaDBTemplate().Storage.S3
314+
s3.Prefix = "tenant/db"
315+
const prefix = "tenant/db/"
316+
run := func(name string, created, completed time.Time) *mariadbtypes.Backup {
317+
b := operatorBackupStoringTo(t, s3, created)
318+
b.Name = name
319+
b.Status.Conditions = []metav1.Condition{mariadbBackupSucceeded(metav1.NewTime(completed))}
320+
return b
321+
}
322+
dump := func(named, uploaded time.Time, size int64) s3ObjectInfo {
323+
return s3ObjectInfo{Key: dumpKey(prefix, named, "gzip"), Size: size, LastModified: uploaded}
324+
}
325+
own := run("op-bk", t0, t0.Add(40*time.Second))
326+
find := func(t *testing.T, objects []s3ObjectInfo, operatorBackups ...client.Object) (mariadbDump, error) {
327+
t.Helper()
328+
store := objectStore{
329+
reader: newMariaDBStrategyTestClient(t, append(operatorBackups, own, s3CredsSecret("tenant", "creds"))...),
330+
newLister: func(s3Connection) (s3ObjectLister, error) {
331+
return &fakeS3Lister{objects: objects}, nil
332+
},
333+
}
334+
return findMariaDBDump(context.Background(), store, own, t0.Add(40*time.Second))
335+
}
336+
wantAmbiguous := func(t *testing.T, got mariadbDump, err error) {
337+
t.Helper()
338+
if !errors.Is(err, errMariaDBDumpAmbiguous) {
339+
t.Errorf("findMariaDBDump = %+v, %v; want errMariaDBDumpAmbiguous", got, err)
340+
}
341+
}
342+
wantOwn := func(t *testing.T, got mariadbDump, err error) {
343+
t.Helper()
344+
if err != nil || got.Size != 123 {
345+
t.Errorf("findMariaDBDump = %+v, %v; want this run's 123-byte dump", got, err)
346+
}
347+
}
348+
349+
t.Run("a pod clock behind does not hand this run the previous run's dump", func(t *testing.T) {
350+
// The previous run finished 20s before this one was created and its
351+
// operator Backup is gone. This run's pod clock is 5m behind, so its
352+
// own dump is named outside the window and uploaded inside it.
353+
got, err := find(t, []s3ObjectInfo{
354+
dump(t0.Add(-30*time.Second), t0.Add(-20*time.Second), 666),
355+
dump(t0.Add(5*time.Second-5*time.Minute), t0.Add(30*time.Second), 123),
356+
})
357+
wantAmbiguous(t, got, err)
358+
})
359+
360+
t.Run("a pod clock ahead does not hand this run the next run's dump", func(t *testing.T) {
361+
// The next run started 5s after this one completed and has uploaded
362+
// its dump by the time this one is probed; it has not completed yet.
363+
next := operatorBackupStoringTo(t, s3, t0.Add(45*time.Second))
364+
next.Name = "op-next"
365+
next.Status.Conditions = []metav1.Condition{{
366+
Type: mariadbtypes.ConditionTypeComplete, Status: metav1.ConditionFalse,
367+
Reason: mariadbtypes.ConditionReasonJobRunning, Message: "Running", LastTransitionTime: metav1.NewTime(t0.Add(45 * time.Second)),
368+
}}
369+
got, err := find(t, []s3ObjectInfo{
370+
dump(t0.Add(5*time.Second+5*time.Minute), t0.Add(30*time.Second), 123),
371+
dump(t0.Add(50*time.Second), t0.Add(80*time.Second), 777),
372+
}, next)
373+
wantAmbiguous(t, got, err)
374+
})
375+
376+
t.Run("an object store clock behind does not hand this run the next run's dump", func(t *testing.T) {
377+
// The store clock is 2m behind: this run's upload is stamped before
378+
// the window opens, the next run's is pulled back into it.
379+
got, err := find(t, []s3ObjectInfo{
380+
dump(t0.Add(5*time.Second), t0.Add(30*time.Second-2*time.Minute), 123),
381+
dump(t0.Add(50*time.Second), t0.Add(3*time.Minute-2*time.Minute), 777),
382+
})
383+
wantAmbiguous(t, got, err)
384+
})
385+
386+
t.Run("a dump another finished run wrote is not taken when this run's own is gone", func(t *testing.T) {
387+
// This run's dump was pruned; the next run's falls in this run's
388+
// window on both clocks, and in its own.
389+
next := run("op-next", t0.Add(45*time.Second), t0.Add(85*time.Second))
390+
got, err := find(t, []s3ObjectInfo{
391+
dump(t0.Add(50*time.Second), t0.Add(75*time.Second), 777),
392+
}, next)
393+
wantAmbiguous(t, got, err)
394+
})
395+
396+
t.Run("a dump the window holds on one clock only is not taken", func(t *testing.T) {
397+
// This run's dump is gone; the previous run's operator Backup is too,
398+
// and its dump, started 2m before this run, finished uploading 20s
399+
// before it.
400+
got, err := find(t, []s3ObjectInfo{
401+
dump(t0.Add(-2*time.Minute), t0.Add(-20*time.Second), 666),
402+
})
403+
if err == nil {
404+
t.Errorf("findMariaDBDump = %+v; want no dump attributed", got)
405+
}
406+
})
407+
408+
t.Run("a run still in progress does not contest this run's dump", func(t *testing.T) {
409+
// A concurrent run created 30s before this one has not uploaded yet.
410+
running := operatorBackupStoringTo(t, s3, t0.Add(-30*time.Second))
411+
running.Name = "op-running"
412+
running.Status.Conditions = []metav1.Condition{{
413+
Type: mariadbtypes.ConditionTypeComplete, Status: metav1.ConditionFalse,
414+
Reason: mariadbtypes.ConditionReasonJobRunning, Message: "Running", LastTransitionTime: metav1.NewTime(t0.Add(-30 * time.Second)),
415+
}}
416+
got, err := find(t, []s3ObjectInfo{dump(t0.Add(5*time.Second), t0.Add(30*time.Second), 123)}, running)
417+
wantOwn(t, got, err)
418+
})
419+
420+
t.Run("runs minutes apart each keep their own dump", func(t *testing.T) {
421+
previous := run("op-previous", t0.Add(-4*time.Minute), t0.Add(-3*time.Minute))
422+
next := run("op-next", t0.Add(3*time.Minute), t0.Add(4*time.Minute))
423+
got, err := find(t, []s3ObjectInfo{
424+
dump(t0.Add(-230*time.Second), t0.Add(-190*time.Second), 666),
425+
dump(t0.Add(5*time.Second), t0.Add(30*time.Second), 123),
426+
dump(t0.Add(185*time.Second), t0.Add(220*time.Second), 777),
427+
}, previous, next)
428+
wantOwn(t, got, err)
429+
})
430+
431+
t.Run("a run that writes elsewhere does not contest this run's dump", func(t *testing.T) {
432+
other := newRenderedMariaDBTemplate().Storage.S3
433+
other.Prefix = "tenant/other"
434+
elsewhere := operatorBackupStoringTo(t, other, t0)
435+
elsewhere.Name = "op-elsewhere"
436+
elsewhere.Status.Conditions = []metav1.Condition{mariadbBackupSucceeded(metav1.NewTime(t0.Add(40 * time.Second)))}
437+
got, err := find(t, []s3ObjectInfo{dump(t0.Add(5*time.Second), t0.Add(30*time.Second), 123)}, elsewhere)
438+
wantOwn(t, got, err)
439+
})
440+
441+
t.Run("a run stuck short of Complete does not contest later runs", func(t *testing.T) {
442+
// A backup Job whose pod never starts leaves its operator Backup
443+
// JobRunning for good.
444+
stuck := operatorBackupStoringTo(t, s3, t0.Add(-3*time.Hour))
445+
stuck.Name = "op-stuck"
446+
stuck.Status.Conditions = []metav1.Condition{{
447+
Type: mariadbtypes.ConditionTypeComplete, Status: metav1.ConditionFalse,
448+
Reason: mariadbtypes.ConditionReasonJobRunning, Message: "Running", LastTransitionTime: metav1.NewTime(t0.Add(-3 * time.Hour)),
449+
}}
450+
got, err := find(t, []s3ObjectInfo{dump(t0.Add(5*time.Second), t0.Add(30*time.Second), 123)}, stuck)
451+
wantOwn(t, got, err)
452+
})
453+
}
454+
306455
// TestMariaDBArtifactSizeBytes_MinioClient drives the real minio client
307456
// against a stub ListObjectsV2 endpoint, so the Secret wiring, prefix and
308457
// response parsing are exercised together.

0 commit comments

Comments
 (0)