Skip to content

Commit c4fbd5f

Browse files
fix(backups): hold a BackupJob while its application is being restored
Nothing on the backup path looks at restores. A Plan creates its BackupJobs on schedule, and the BackupJob reconciler dispatches them to their driver whatever else is happening to the application. For CNPG that is not merely a backup of a half-restored state: every BackupJob reconcile re-applies the Cluster's barman plugin, and on a cluster under an in-place restore it can land after the chart has re-rendered the Cluster with its new serverName and put it back on the source's. The restored cluster then archives onto the source's WAL prefix and its recovery fails with "WAL archive check failed ... Expected empty archive". This was reproduced with a backup triggered a few seconds after an in-place restore, and an hourly plan overlaps a restore of up to thirty minutes about half the time. A BackupJob that has not started yet now waits while a non-terminal RestoreJob in its namespace writes into the same application, resolved as the drivers resolve it: targetApplicationRef, with any empty field taken from the Backup's applicationRef. It stays Pending with a Ready=False RestoreInProgress condition naming the restore and is requeued, rather than failed, so a scheduled run still happens once the restore ends. A run that has already started is left alone: its driver owns state the gate would strand. Assisted-by: LLM Signed-off-by: Mattia Eleuteri <[email protected]>
1 parent 9c94337 commit c4fbd5f

3 files changed

Lines changed: 372 additions & 0 deletions

File tree

‎internal/backupcontroller/backupjob_controller.go‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -130,6 +130,20 @@ func (r *BackupJobReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
130130
return r.markBackupJobFailed(ctx, j, fmt.Sprintf("strategy Kind %q is not supported by this controller (supported: %s)", strategyRef.Kind, strings.Join(supportedBackupStrategyKinds(), ", ")))
131131
}
132132

133+
// Hold a run that has not started while a restore writes into its
134+
// application. Only before StartedAt: every driver stamps it on its
135+
// first pass, and a run already moving data is not interrupted here.
136+
if j.Status.StartedAt == nil {
137+
restore, err := activeRestoreTargeting(ctx, r.Client, j.Namespace, normalizedAppRef)
138+
if err != nil {
139+
logger.Error(err, "failed to check for a restore in progress")
140+
return ctrl.Result{}, err
141+
}
142+
if restore != "" {
143+
return r.holdForRestore(ctx, j, restore)
144+
}
145+
}
146+
133147
// Now project the platform-managed S3 credentials into the tenant
134148
// namespace so default Strategy CRs can reference a deterministic
135149
// Secret name. The projection is idempotent and silently skipped on
Lines changed: 147 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,147 @@
1+
// SPDX-License-Identifier: Apache-2.0
2+
package backupcontroller
3+
4+
import (
5+
"context"
6+
"encoding/json"
7+
"fmt"
8+
"sort"
9+
"time"
10+
11+
corev1 "k8s.io/api/core/v1"
12+
"k8s.io/apimachinery/pkg/api/meta"
13+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
14+
ctrl "sigs.k8s.io/controller-runtime"
15+
"sigs.k8s.io/controller-runtime/pkg/client"
16+
17+
backupsv1alpha1 "github.com/cozystack/cozystack/api/backups/v1alpha1"
18+
)
19+
20+
// RestoreInProgressRequeue is how often a BackupJob held by a restore checks
21+
// again. Restores run for minutes, so there is nothing to gain from polling
22+
// faster.
23+
const RestoreInProgressRequeue = 30 * time.Second
24+
25+
// ConditionReasonRestoreInProgress is the Ready=False reason of a BackupJob
26+
// held back because its application is being restored.
27+
const ConditionReasonRestoreInProgress = "RestoreInProgress"
28+
29+
// activeRestoreTargeting returns the name of a non-terminal RestoreJob in
30+
// namespace that writes into app, or "" when there is none.
31+
//
32+
// A backup must not start against an application a restore is rewriting. It
33+
// would capture a half-restored state at best, and on the CNPG path it does
34+
// worse: every BackupJob reconcile re-applies the Cluster's barman plugin, and
35+
// one landing between the restore re-rendering the Cluster with its new
36+
// serverName and the plugin being attached can put the restored cluster back
37+
// on the source's WAL prefix, where its recovery fails with "Expected empty
38+
// archive". Plan-created runs are the ones that hit this, since nothing else
39+
// looks at restores before creating a BackupJob.
40+
//
41+
// A RestoreJob whose options send it to another namespace does not write here.
42+
// When the target cannot be resolved because the Backup is gone, the restore
43+
// is skipped: it fails on its own, and holding backups on a guess would stall
44+
// them until it does.
45+
func activeRestoreTargeting(ctx context.Context, c client.Client, namespace string, app corev1.TypedLocalObjectReference) (string, error) {
46+
app = NormalizeApplicationRef(app)
47+
list := &backupsv1alpha1.RestoreJobList{}
48+
if err := c.List(ctx, list, client.InNamespace(namespace)); err != nil {
49+
return "", fmt.Errorf("list RestoreJobs: %w", err)
50+
}
51+
var holding []string
52+
for i := range list.Items {
53+
rj := &list.Items[i]
54+
if rj.Status.Phase == backupsv1alpha1.RestoreJobPhaseSucceeded || rj.Status.Phase == backupsv1alpha1.RestoreJobPhaseFailed {
55+
continue
56+
}
57+
if restoresIntoAnotherNamespace(rj) {
58+
continue
59+
}
60+
target, ok, err := restoreJobTarget(ctx, c, rj)
61+
if err != nil {
62+
return "", err
63+
}
64+
if ok && sameApplication(target, app) {
65+
holding = append(holding, rj.Name)
66+
}
67+
}
68+
if len(holding) == 0 {
69+
return "", nil
70+
}
71+
sort.Strings(holding)
72+
return holding[0], nil
73+
}
74+
75+
// restoreJobTarget resolves the application a RestoreJob writes into: its
76+
// targetApplicationRef, with any field it leaves empty taken from the Backup's
77+
// applicationRef, as the drivers resolve it.
78+
func restoreJobTarget(ctx context.Context, c client.Client, rj *backupsv1alpha1.RestoreJob) (corev1.TypedLocalObjectReference, bool, error) {
79+
t := rj.Spec.TargetApplicationRef
80+
if t != nil && t.Name != "" && t.Kind != "" {
81+
return NormalizeApplicationRef(*t), true, nil
82+
}
83+
backup := &backupsv1alpha1.Backup{}
84+
if err := c.Get(ctx, client.ObjectKey{Namespace: rj.Namespace, Name: rj.Spec.BackupRef.Name}, backup); err != nil {
85+
if client.IgnoreNotFound(err) == nil {
86+
return corev1.TypedLocalObjectReference{}, false, nil
87+
}
88+
return corev1.TypedLocalObjectReference{}, false, fmt.Errorf("get Backup %s of RestoreJob %s: %w", rj.Spec.BackupRef.Name, rj.Name, err)
89+
}
90+
ref := backup.Spec.ApplicationRef
91+
if t != nil {
92+
if t.Name != "" {
93+
ref.Name = t.Name
94+
}
95+
if t.Kind != "" {
96+
ref.Kind = t.Kind
97+
}
98+
if t.APIGroup != nil {
99+
ref.APIGroup = t.APIGroup
100+
}
101+
}
102+
return NormalizeApplicationRef(ref), true, nil
103+
}
104+
105+
func restoresIntoAnotherNamespace(rj *backupsv1alpha1.RestoreJob) bool {
106+
if rj.Spec.Options == nil || len(rj.Spec.Options.Raw) == 0 {
107+
return false
108+
}
109+
var opts struct {
110+
TargetNamespace string `json:"targetNamespace"`
111+
}
112+
if err := json.Unmarshal(rj.Spec.Options.Raw, &opts); err != nil {
113+
return false
114+
}
115+
return opts.TargetNamespace != "" && opts.TargetNamespace != rj.Namespace
116+
}
117+
118+
func sameApplication(a, b corev1.TypedLocalObjectReference) bool {
119+
return a.Kind == b.Kind && a.Name == b.Name && ptrString(a.APIGroup) == ptrString(b.APIGroup)
120+
}
121+
122+
func ptrString(s *string) string {
123+
if s == nil {
124+
return ""
125+
}
126+
return *s
127+
}
128+
129+
// holdForRestore keeps a BackupJob that has not started yet waiting while a
130+
// restore writes into its application. It stays non-terminal, so a scheduled
131+
// run still happens once the restore ends, only later than its slot.
132+
func (r *BackupJobReconciler) holdForRestore(ctx context.Context, j *backupsv1alpha1.BackupJob, restore string) (ctrl.Result, error) {
133+
logger := getLogger(ctx)
134+
if j.Status.Phase == backupsv1alpha1.BackupJobPhaseEmpty {
135+
j.Status.Phase = backupsv1alpha1.BackupJobPhasePending
136+
}
137+
meta.SetStatusCondition(&j.Status.Conditions, metav1.Condition{
138+
Type: "Ready",
139+
Status: metav1.ConditionFalse,
140+
Reason: ConditionReasonRestoreInProgress,
141+
Message: fmt.Sprintf("waiting for RestoreJob %q to finish: application %s/%s is being restored", restore, j.Spec.ApplicationRef.Kind, j.Spec.ApplicationRef.Name),
142+
})
143+
if err := r.Status().Update(ctx, j); err != nil {
144+
logger.Error(err, "failed to update BackupJob status to RestoreInProgress")
145+
}
146+
return ctrl.Result{RequeueAfter: RestoreInProgressRequeue}, nil
147+
}
Lines changed: 211 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,211 @@
1+
// SPDX-License-Identifier: Apache-2.0
2+
package backupcontroller
3+
4+
import (
5+
"context"
6+
"testing"
7+
8+
corev1 "k8s.io/api/core/v1"
9+
"k8s.io/apimachinery/pkg/api/meta"
10+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
11+
"k8s.io/apimachinery/pkg/runtime"
12+
"k8s.io/apimachinery/pkg/types"
13+
"k8s.io/client-go/kubernetes/scheme"
14+
"sigs.k8s.io/controller-runtime/pkg/client"
15+
clientfake "sigs.k8s.io/controller-runtime/pkg/client/fake"
16+
"sigs.k8s.io/controller-runtime/pkg/reconcile"
17+
18+
strategyv1alpha1 "github.com/cozystack/cozystack/api/backups/strategy/v1alpha1"
19+
backupsv1alpha1 "github.com/cozystack/cozystack/api/backups/v1alpha1"
20+
)
21+
22+
const lockNS = "tenant-acme"
23+
24+
func lockTestClient(t *testing.T, objs ...client.Object) client.Client {
25+
t.Helper()
26+
s := runtime.NewScheme()
27+
_ = scheme.AddToScheme(s)
28+
_ = backupsv1alpha1.AddToScheme(s)
29+
return clientfake.NewClientBuilder().
30+
WithScheme(s).
31+
WithObjects(objs...).
32+
WithStatusSubresource(&backupsv1alpha1.BackupJob{}, &backupsv1alpha1.RestoreJob{}).
33+
Build()
34+
}
35+
36+
func pgRef(name string) corev1.TypedLocalObjectReference {
37+
return corev1.TypedLocalObjectReference{APIGroup: new("apps.cozystack.io"), Kind: "Postgres", Name: name}
38+
}
39+
40+
func lockBackup(name string, app corev1.TypedLocalObjectReference) *backupsv1alpha1.Backup {
41+
return &backupsv1alpha1.Backup{
42+
ObjectMeta: metav1.ObjectMeta{Namespace: lockNS, Name: name},
43+
Spec: backupsv1alpha1.BackupSpec{ApplicationRef: app},
44+
}
45+
}
46+
47+
func lockRestore(name, backup string, phase backupsv1alpha1.RestoreJobPhase, target *corev1.TypedLocalObjectReference, options string) *backupsv1alpha1.RestoreJob {
48+
rj := &backupsv1alpha1.RestoreJob{
49+
ObjectMeta: metav1.ObjectMeta{Namespace: lockNS, Name: name},
50+
Spec: backupsv1alpha1.RestoreJobSpec{
51+
BackupRef: corev1.LocalObjectReference{Name: backup},
52+
TargetApplicationRef: target,
53+
},
54+
Status: backupsv1alpha1.RestoreJobStatus{Phase: phase},
55+
}
56+
if options != "" {
57+
rj.Spec.Options = &runtime.RawExtension{Raw: []byte(options)}
58+
}
59+
return rj
60+
}
61+
62+
func TestActiveRestoreTargeting(t *testing.T) {
63+
other := pgRef("other")
64+
anotherKind := corev1.TypedLocalObjectReference{APIGroup: new("apps.cozystack.io"), Kind: "MariaDB", Name: "pg"}
65+
for _, tc := range []struct {
66+
name string
67+
objects []client.Object
68+
want string
69+
}{
70+
{"no restore", nil, ""},
71+
{"in-place restore of the same application", []client.Object{
72+
lockBackup("pg-1", pgRef("pg")), lockRestore("r1", "pg-1", backupsv1alpha1.RestoreJobPhaseRunning, nil, ""),
73+
}, "r1"},
74+
{"restore not picked up yet", []client.Object{
75+
lockBackup("pg-1", pgRef("pg")), lockRestore("r1", "pg-1", "", nil, ""),
76+
}, "r1"},
77+
{"to-copy restore into this application", []client.Object{
78+
lockBackup("other-1", other), lockRestore("r1", "other-1", backupsv1alpha1.RestoreJobPhaseRunning, &corev1.TypedLocalObjectReference{Kind: "Postgres", Name: "pg"}, ""),
79+
}, "r1"},
80+
{"to-copy restore out of this application", []client.Object{
81+
lockBackup("pg-1", pgRef("pg")), lockRestore("r1", "pg-1", backupsv1alpha1.RestoreJobPhaseRunning, &other, ""),
82+
}, ""},
83+
{"finished restores", []client.Object{
84+
lockBackup("pg-1", pgRef("pg")),
85+
lockRestore("r1", "pg-1", backupsv1alpha1.RestoreJobPhaseSucceeded, nil, ""),
86+
lockRestore("r2", "pg-1", backupsv1alpha1.RestoreJobPhaseFailed, nil, ""),
87+
}, ""},
88+
{"same name, another kind", []client.Object{
89+
lockBackup("m-1", anotherKind), lockRestore("r1", "m-1", backupsv1alpha1.RestoreJobPhaseRunning, nil, ""),
90+
}, ""},
91+
{"restore sent to another namespace", []client.Object{
92+
lockBackup("pg-1", pgRef("pg")), lockRestore("r1", "pg-1", backupsv1alpha1.RestoreJobPhaseRunning, nil, `{"targetNamespace":"tenant-other"}`),
93+
}, ""},
94+
{"backup gone", []client.Object{
95+
lockRestore("r1", "pg-1", backupsv1alpha1.RestoreJobPhaseRunning, nil, ""),
96+
}, ""},
97+
} {
98+
t.Run(tc.name, func(t *testing.T) {
99+
c := lockTestClient(t, tc.objects...)
100+
got, err := activeRestoreTargeting(context.Background(), c, lockNS, pgRef("pg"))
101+
if err != nil {
102+
t.Fatalf("activeRestoreTargeting: %v", err)
103+
}
104+
if got != tc.want {
105+
t.Errorf("got %q, want %q", got, tc.want)
106+
}
107+
})
108+
}
109+
}
110+
111+
func lockBackupClass() *backupsv1alpha1.BackupClass {
112+
return &backupsv1alpha1.BackupClass{
113+
ObjectMeta: metav1.ObjectMeta{Name: "cozy-default"},
114+
Spec: backupsv1alpha1.BackupClassSpec{
115+
Strategies: []backupsv1alpha1.BackupClassStrategy{{
116+
Application: backupsv1alpha1.ApplicationSelector{APIGroup: new("apps.cozystack.io"), Kind: "Postgres"},
117+
StrategyRef: corev1.TypedLocalObjectReference{
118+
APIGroup: new(strategyv1alpha1.GroupVersion.Group), Kind: strategyv1alpha1.CNPGStrategyKind, Name: "cozy-default-cnpg",
119+
},
120+
}},
121+
},
122+
}
123+
}
124+
125+
func lockBackupJob(started bool) *backupsv1alpha1.BackupJob {
126+
j := &backupsv1alpha1.BackupJob{
127+
ObjectMeta: metav1.ObjectMeta{Namespace: lockNS, Name: "nightly-29842560"},
128+
Spec: backupsv1alpha1.BackupJobSpec{BackupClassName: "cozy-default", ApplicationRef: pgRef("pg")},
129+
}
130+
if started {
131+
now := metav1.Now()
132+
j.Status.StartedAt = &now
133+
j.Status.Phase = backupsv1alpha1.BackupJobPhaseRunning
134+
}
135+
return j
136+
}
137+
138+
// TestReconcile_HoldsABackupJobWhileItsApplicationIsRestored is the
139+
// regression: a Plan-created BackupJob for a Postgres application under an
140+
// in-place restore reached the CNPG driver, which re-applied the Cluster's
141+
// barman plugin and put the restored cluster back on the source's WAL
142+
// prefix. The job must wait, non-terminal, without reaching projection or
143+
// dispatch; and once the restore is over, it must go ahead.
144+
func TestReconcile_HoldsABackupJobWhileItsApplicationIsRestored(t *testing.T) {
145+
restore := lockRestore("restore-pg", "pg-1", backupsv1alpha1.RestoreJobPhaseRunning, nil, "")
146+
c := lockTestClient(t, lockBackupJob(false), lockBackupClass(), lockBackup("pg-1", pgRef("pg")), restore, flatSourceSecret())
147+
r := &BackupJobReconciler{Client: c, CredentialsConfig: defaultCfg()}
148+
key := types.NamespacedName{Namespace: lockNS, Name: "nightly-29842560"}
149+
150+
res, err := r.Reconcile(context.Background(), reconcile.Request{NamespacedName: key})
151+
if err != nil {
152+
t.Fatalf("Reconcile: %v", err)
153+
}
154+
if res.RequeueAfter != RestoreInProgressRequeue {
155+
t.Errorf("RequeueAfter = %v, want %v", res.RequeueAfter, RestoreInProgressRequeue)
156+
}
157+
got := &backupsv1alpha1.BackupJob{}
158+
if err := c.Get(context.Background(), key, got); err != nil {
159+
t.Fatal(err)
160+
}
161+
if got.Status.Phase != backupsv1alpha1.BackupJobPhasePending {
162+
t.Errorf("Phase = %q, want Pending", got.Status.Phase)
163+
}
164+
if got.Status.StartedAt != nil {
165+
t.Error("a held BackupJob must not be marked started")
166+
}
167+
cond := meta.FindStatusCondition(got.Status.Conditions, "Ready")
168+
if cond == nil || cond.Reason != ConditionReasonRestoreInProgress {
169+
t.Fatalf("Ready condition = %+v, want reason %s", cond, ConditionReasonRestoreInProgress)
170+
}
171+
secret := &corev1.Secret{}
172+
if err := c.Get(context.Background(), types.NamespacedName{Namespace: lockNS, Name: "cozy-backups-creds"}, secret); err == nil {
173+
t.Error("a held BackupJob went on to project credentials")
174+
}
175+
176+
// The restore ends: the job is no longer held.
177+
fresh := &backupsv1alpha1.RestoreJob{}
178+
if err := c.Get(context.Background(), types.NamespacedName{Namespace: lockNS, Name: "restore-pg"}, fresh); err != nil {
179+
t.Fatal(err)
180+
}
181+
fresh.Status.Phase = backupsv1alpha1.RestoreJobPhaseSucceeded
182+
if err := c.Status().Update(context.Background(), fresh); err != nil {
183+
t.Fatal(err)
184+
}
185+
if held, err := activeRestoreTargeting(context.Background(), c, lockNS, pgRef("pg")); err != nil || held != "" {
186+
t.Errorf("after the restore ended: held by %q (%v), want none", held, err)
187+
}
188+
}
189+
190+
// TestReconcile_DoesNotHoldARunningBackupJob: the gate is on starting. A run
191+
// already moving data is left to its driver, whose own state it would
192+
// otherwise strand.
193+
func TestReconcile_DoesNotHoldARunningBackupJob(t *testing.T) {
194+
restore := lockRestore("restore-pg", "pg-1", backupsv1alpha1.RestoreJobPhaseRunning, nil, "")
195+
// No source Secret: projection fails transiently, which stops the
196+
// reconcile before dispatch and shows the gate was passed.
197+
c := lockTestClient(t, lockBackupJob(true), lockBackupClass(), lockBackup("pg-1", pgRef("pg")), restore)
198+
r := &BackupJobReconciler{Client: c, CredentialsConfig: defaultCfg()}
199+
key := types.NamespacedName{Namespace: lockNS, Name: "nightly-29842560"}
200+
201+
if _, err := r.Reconcile(context.Background(), reconcile.Request{NamespacedName: key}); err != nil {
202+
t.Fatalf("Reconcile: %v", err)
203+
}
204+
got := &backupsv1alpha1.BackupJob{}
205+
if err := c.Get(context.Background(), key, got); err != nil {
206+
t.Fatal(err)
207+
}
208+
if cond := meta.FindStatusCondition(got.Status.Conditions, "Ready"); cond != nil && cond.Reason == ConditionReasonRestoreInProgress {
209+
t.Error("a BackupJob already started was held")
210+
}
211+
}

0 commit comments

Comments
 (0)