Skip to content

Commit af8cf74

Browse files
fix(backups): hold a BackupJob while its application is being restored
Nothing on the backup path looks at restores. A Plan creates its BackupJobs on schedule, and the BackupJob reconciler dispatches them to their driver whatever else is happening to the application. For CNPG that is not merely a backup of a half-restored state: every BackupJob reconcile re-applies the Cluster's barman plugin, and on a cluster under an in-place restore it can land after the chart has re-rendered the Cluster with its new serverName and put it back on the source's. The restored cluster then archives onto the source's WAL prefix and its recovery fails with "WAL archive check failed ... Expected empty archive". This was reproduced with a backup triggered a few seconds after an in-place restore, and an hourly plan overlaps a restore of up to thirty minutes about half the time. A BackupJob that has not started yet now waits while a non-terminal RestoreJob in its namespace writes into the same application, resolved as the drivers resolve it: the Backup's applicationRef, with the target's kind and name when a targetApplicationRef is set, and the target's API group only when it names one. A RestoreJob whose Backup is gone holds nothing, since the restore controller fails it on its next pass. options.targetNamespace is honoured only for a Backup taken by the Velero strategy, the one driver that reads it; the CNPG and Job-based drivers ignore the option and restore in place, so their restores hold whatever it says. The held job stays Pending with a Ready=False RestoreInProgress condition naming the restore and is requeued, rather than failed, so a scheduled run still happens once the restore ends. A failed write of that status is returned, so the manager retries it. When the restore ends the condition is removed before the job starts, since it would otherwise read "waiting" until the driver next writes Ready, which for a healthy run is when the backup completes. A run that has already started is left alone: its driver owns state the gate would strand. Assisted-by: LLM Signed-off-by: Mattia Eleuteri <[email protected]>
1 parent 9c94337 commit af8cf74

3 files changed

Lines changed: 517 additions & 0 deletions

File tree

‎internal/backupcontroller/backupjob_controller.go‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -130,6 +130,23 @@ func (r *BackupJobReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
130130
return r.markBackupJobFailed(ctx, j, fmt.Sprintf("strategy Kind %q is not supported by this controller (supported: %s)", strategyRef.Kind, strings.Join(supportedBackupStrategyKinds(), ", ")))
131131
}
132132

133+
// Hold a run that has not started while a restore writes into its
134+
// application. Only before StartedAt: every driver stamps it on its
135+
// first pass, and a run already moving data is not interrupted here.
136+
if j.Status.StartedAt == nil {
137+
restore, err := activeRestoreTargeting(ctx, r.Client, j.Namespace, normalizedAppRef)
138+
if err != nil {
139+
logger.Error(err, "failed to check for a restore in progress")
140+
return ctrl.Result{}, err
141+
}
142+
if restore != "" {
143+
return r.holdForRestore(ctx, j, restore)
144+
}
145+
if err := r.releaseRestoreHold(ctx, j); err != nil {
146+
return ctrl.Result{}, err
147+
}
148+
}
149+
133150
// Now project the platform-managed S3 credentials into the tenant
134151
// namespace so default Strategy CRs can reference a deterministic
135152
// Secret name. The projection is idempotent and silently skipped on
Lines changed: 168 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,168 @@
1+
// SPDX-License-Identifier: Apache-2.0
2+
package backupcontroller
3+
4+
import (
5+
"context"
6+
"encoding/json"
7+
"fmt"
8+
"sort"
9+
"time"
10+
11+
corev1 "k8s.io/api/core/v1"
12+
"k8s.io/apimachinery/pkg/api/meta"
13+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
14+
ctrl "sigs.k8s.io/controller-runtime"
15+
"sigs.k8s.io/controller-runtime/pkg/client"
16+
17+
strategyv1alpha1 "github.com/cozystack/cozystack/api/backups/strategy/v1alpha1"
18+
backupsv1alpha1 "github.com/cozystack/cozystack/api/backups/v1alpha1"
19+
)
20+
21+
// RestoreInProgressRequeue is how often a BackupJob held by a restore checks
22+
// again. Restores run for minutes, so there is nothing to gain from polling
23+
// faster.
24+
const RestoreInProgressRequeue = 30 * time.Second
25+
26+
// ConditionReasonRestoreInProgress is the Ready=False reason of a BackupJob
27+
// held back because its application is being restored.
28+
const ConditionReasonRestoreInProgress = "RestoreInProgress"
29+
30+
// activeRestoreTargeting returns the name of a non-terminal RestoreJob in
31+
// namespace that writes into app, or "" when there is none.
32+
//
33+
// A backup must not start against an application a restore is rewriting. It
34+
// would capture a half-restored state at best, and on the CNPG path it does
35+
// worse: every BackupJob reconcile re-applies the Cluster's barman plugin, and
36+
// one landing between the restore re-rendering the Cluster with its new
37+
// serverName and the plugin being attached can put the restored cluster back
38+
// on the source's WAL prefix, where its recovery fails with "Expected empty
39+
// archive". Plan-created runs are the ones that hit this, since nothing else
40+
// looks at restores before creating a BackupJob.
41+
func activeRestoreTargeting(ctx context.Context, c client.Client, namespace string, app corev1.TypedLocalObjectReference) (string, error) {
42+
app = NormalizeApplicationRef(app)
43+
list := &backupsv1alpha1.RestoreJobList{}
44+
if err := c.List(ctx, list, client.InNamespace(namespace)); err != nil {
45+
return "", fmt.Errorf("list RestoreJobs: %w", err)
46+
}
47+
var holding []string
48+
for i := range list.Items {
49+
rj := &list.Items[i]
50+
if rj.Status.Phase == backupsv1alpha1.RestoreJobPhaseSucceeded || rj.Status.Phase == backupsv1alpha1.RestoreJobPhaseFailed {
51+
continue
52+
}
53+
target, ok, err := restoreJobTarget(ctx, c, rj)
54+
if err != nil {
55+
return "", err
56+
}
57+
if ok && sameApplication(target, app) {
58+
holding = append(holding, rj.Name)
59+
}
60+
}
61+
if len(holding) == 0 {
62+
return "", nil
63+
}
64+
sort.Strings(holding)
65+
return holding[0], nil
66+
}
67+
68+
// restoreJobTarget resolves the application a RestoreJob writes into in its
69+
// own namespace, as the drivers resolve it (resolveCNPGRestoreTarget and the
70+
// Job-based paths): the Backup's applicationRef, with the target's kind and
71+
// name when a target is set — the CRD requires both — and the target's API
72+
// group only when it names one. A target without an API group therefore stays
73+
// in the Backup's group, which is not always the default one.
74+
//
75+
// It reports false for a restore that writes nothing here. One whose Backup is
76+
// gone is failed by the restore controller on its next pass. A Velero restore
77+
// whose options name another namespace writes there: only the Velero driver
78+
// reads options.targetNamespace, so the option is looked at only when the
79+
// Backup's strategy is Velero — the CNPG and Job-based drivers ignore it and
80+
// restore in place.
81+
func restoreJobTarget(ctx context.Context, c client.Client, rj *backupsv1alpha1.RestoreJob) (corev1.TypedLocalObjectReference, bool, error) {
82+
backup := &backupsv1alpha1.Backup{}
83+
if err := c.Get(ctx, client.ObjectKey{Namespace: rj.Namespace, Name: rj.Spec.BackupRef.Name}, backup); err != nil {
84+
if client.IgnoreNotFound(err) == nil {
85+
return corev1.TypedLocalObjectReference{}, false, nil
86+
}
87+
return corev1.TypedLocalObjectReference{}, false, fmt.Errorf("get Backup %s of RestoreJob %s: %w", rj.Spec.BackupRef.Name, rj.Name, err)
88+
}
89+
if backup.Spec.StrategyRef.Kind == strategyv1alpha1.VeleroStrategyKind && restoresIntoAnotherNamespace(rj, backup.Namespace) {
90+
return corev1.TypedLocalObjectReference{}, false, nil
91+
}
92+
ref := backup.Spec.ApplicationRef
93+
if t := rj.Spec.TargetApplicationRef; t != nil {
94+
ref.Kind, ref.Name = t.Kind, t.Name
95+
if t.APIGroup != nil {
96+
ref.APIGroup = t.APIGroup
97+
}
98+
}
99+
return NormalizeApplicationRef(ref), true, nil
100+
}
101+
102+
// restoresIntoAnotherNamespace reports whether a Velero RestoreJob's
103+
// options.targetNamespace sends it away from namespace, compared as the Velero
104+
// driver compares it. Options that do not parse are read as none.
105+
func restoresIntoAnotherNamespace(rj *backupsv1alpha1.RestoreJob, namespace string) bool {
106+
if rj.Spec.Options == nil || len(rj.Spec.Options.Raw) == 0 {
107+
return false
108+
}
109+
var opts struct {
110+
TargetNamespace string `json:"targetNamespace"`
111+
}
112+
if err := json.Unmarshal(rj.Spec.Options.Raw, &opts); err != nil {
113+
return false
114+
}
115+
return opts.TargetNamespace != "" && opts.TargetNamespace != namespace
116+
}
117+
118+
// sameApplication expects both references normalized: an API group left
119+
// unset compares equal only once it has been defaulted on both sides.
120+
func sameApplication(a, b corev1.TypedLocalObjectReference) bool {
121+
return a.Kind == b.Kind && a.Name == b.Name && ptrString(a.APIGroup) == ptrString(b.APIGroup)
122+
}
123+
124+
func ptrString(s *string) string {
125+
if s == nil {
126+
return ""
127+
}
128+
return *s
129+
}
130+
131+
// holdForRestore keeps a BackupJob that has not started yet waiting while a
132+
// restore writes into its application. It stays non-terminal, so a scheduled
133+
// run still happens once the restore ends, only later than its slot.
134+
//
135+
// A failed status write is returned so the manager retries it with backoff.
136+
// The job is held either way, since nothing past the gate ran, but a hold
137+
// nobody can see reads as a run stuck for no reason.
138+
func (r *BackupJobReconciler) holdForRestore(ctx context.Context, j *backupsv1alpha1.BackupJob, restore string) (ctrl.Result, error) {
139+
if j.Status.Phase == backupsv1alpha1.BackupJobPhaseEmpty {
140+
j.Status.Phase = backupsv1alpha1.BackupJobPhasePending
141+
}
142+
meta.SetStatusCondition(&j.Status.Conditions, metav1.Condition{
143+
Type: "Ready",
144+
Status: metav1.ConditionFalse,
145+
Reason: ConditionReasonRestoreInProgress,
146+
Message: fmt.Sprintf("waiting for RestoreJob %q to finish: application %s/%s is being restored", restore, j.Spec.ApplicationRef.Kind, j.Spec.ApplicationRef.Name),
147+
})
148+
if err := r.Status().Update(ctx, j); err != nil {
149+
return ctrl.Result{}, fmt.Errorf("record RestoreInProgress on BackupJob %s: %w", j.Name, err)
150+
}
151+
return ctrl.Result{RequeueAfter: RestoreInProgressRequeue}, nil
152+
}
153+
154+
// releaseRestoreHold removes the condition holdForRestore left, once the
155+
// restore has ended and the job goes on to start. Left in place it would read
156+
// "waiting for RestoreJob" until the driver next writes Ready, which for a
157+
// healthy run is when the backup completes.
158+
func (r *BackupJobReconciler) releaseRestoreHold(ctx context.Context, j *backupsv1alpha1.BackupJob) error {
159+
cond := meta.FindStatusCondition(j.Status.Conditions, "Ready")
160+
if cond == nil || cond.Reason != ConditionReasonRestoreInProgress {
161+
return nil
162+
}
163+
meta.RemoveStatusCondition(&j.Status.Conditions, "Ready")
164+
if err := r.Status().Update(ctx, j); err != nil {
165+
return fmt.Errorf("clear RestoreInProgress on BackupJob %s: %w", j.Name, err)
166+
}
167+
return nil
168+
}

0 commit comments

Comments
 (0)