Skip to content

Commit e406818

Browse files
committed
fix(backups): refuse a restore into an application another restore holds
Nothing stopped two RestoreJobs from rewriting one application at once. Each restores the target from its own Backup, so the application ends up on whichever finishes last, and the CNPG driver's purge decision cannot tell which of the two produced the live Cluster: the second job could skip its purge and report Succeeded on the first job's data. A RestoreJob that has not started now fails with reason ConcurrentRestore when another unfinished RestoreJob in the namespace writes into the same application and goes first. A started restore goes before one that has not; between two that have not, the older one goes first, by creation time and then name, so a running restore is never failed by a newer one. The refused job fails instead of waiting, so the application is not restored a second time later, after the first restore, by a job its author may no longer expect to run. The gate lists RestoreJobs through the manager's API reader rather than the informer cache. Creation times have one-second precision, so two jobs created in the same second are ordered by name, and a job reconciled before the cache held its peer's StartedAt would see that peer as not started and proceed alongside it. The controller runs one worker, so a peer's StartedAt written by an earlier reconcile is on the API server by the time the next job reads it; SetupWithManager now states that. The target is resolved as the BackupJob hold resolves it, so a Velero restore into another namespace does not collide with one in place. Refs #3322 Assisted-by: LLM Signed-off-by: Andrey Kolkov <[email protected]>
1 parent f4a6eb8 commit e406818

3 files changed

Lines changed: 332 additions & 0 deletions

File tree

‎internal/backupcontroller/restorejob_controller.go‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ import (
1919
ctrl "sigs.k8s.io/controller-runtime"
2020
"sigs.k8s.io/controller-runtime/pkg/client"
2121
"sigs.k8s.io/controller-runtime/pkg/client/apiutil"
22+
"sigs.k8s.io/controller-runtime/pkg/controller"
2223
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
2324
"sigs.k8s.io/controller-runtime/pkg/log"
2425

@@ -49,6 +50,10 @@ type RestoreJobReconciler struct {
4950
// (missing pods/log RBAC) branch - is unit-testable without a live cluster.
5051
readPodLog func(ctx context.Context, namespace, podName, container string) (string, error)
5152
CredentialsConfig BackupCredentialsConfig
53+
// APIReader reads RestoreJobs past the informer cache for the
54+
// concurrent-restore gate (restoreAhead). Wired in SetupWithManager; nil
55+
// falls back to Client.
56+
APIReader client.Reader
5257
}
5358

5459
func (r *RestoreJobReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
@@ -133,6 +138,23 @@ func (r *RestoreJobReconciler) Reconcile(ctx context.Context, req ctrl.Request)
133138
}
134139
}
135140

141+
// Only a job that has not started yet is refused, so a restore already
142+
// rewriting the application is never failed by one created after it.
143+
if !restoreJobStarted(restoreJob) {
144+
reader := r.APIReader
145+
if reader == nil {
146+
reader = r.Client
147+
}
148+
ahead, err := restoreAhead(ctx, reader, r.Client, restoreJob)
149+
if err != nil {
150+
return ctrl.Result{}, err
151+
}
152+
if ahead != "" {
153+
return r.markRestoreJobFailedReason(ctx, restoreJob, ConditionReasonConcurrentRestore, fmt.Sprintf(
154+
"RestoreJob %s is already restoring the same application; wait for it to finish, then create a new RestoreJob", ahead))
155+
}
156+
}
157+
136158
// Step 3: Project the platform-managed S3 credentials into the tenant
137159
// namespace so default Strategy CRs (re-rendered at restore time) can
138160
// reference a deterministic Secret name. Idempotent / no-op when
@@ -189,8 +211,13 @@ func (r *RestoreJobReconciler) SetupWithManager(mgr ctrl.Manager) error {
189211
if r.RESTMapper, err = apiutil.NewDynamicRESTMapper(cfg, h); err != nil {
190212
return err
191213
}
214+
r.APIReader = mgr.GetAPIReader()
192215
return ctrl.NewControllerManagedBy(mgr).
193216
For(&backupsv1alpha1.RestoreJob{}).
217+
// One worker: restoreAhead lets exactly one of two concurrent restores
218+
// through only because a peer's StartedAt, written by an earlier
219+
// reconcile, is already on the API server when the next one reads.
220+
WithOptions(controller.Options{MaxConcurrentReconciles: 1}).
194221
Complete(r)
195222
}
196223

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
// SPDX-License-Identifier: Apache-2.0
2+
package backupcontroller
3+
4+
import (
5+
"context"
6+
"fmt"
7+
"sort"
8+
9+
"sigs.k8s.io/controller-runtime/pkg/client"
10+
11+
backupsv1alpha1 "github.com/cozystack/cozystack/api/backups/v1alpha1"
12+
)
13+
14+
// ConditionReasonConcurrentRestore is the Failed reason of a RestoreJob that
15+
// was refused because another restore into the same application goes first.
16+
const ConditionReasonConcurrentRestore = "ConcurrentRestore"
17+
18+
// restoreAhead returns the name of a non-terminal RestoreJob in rj's namespace
19+
// that writes into the same application as rj and goes before it, or "" when
20+
// there is none.
21+
//
22+
// Two restores into one application must not interleave: each rewrites the
23+
// target from its own Backup, and the CNPG driver cannot tell which of them
24+
// produced the live Cluster when both run. The one that goes second fails
25+
// instead of waiting, so the application is never restored a second time,
26+
// later, by a job its author may no longer expect to run.
27+
//
28+
// A restore that has started goes before one that has not; between two that
29+
// have not, the older goes first, by creation time and then by name. Creation
30+
// times have one-second precision, so two jobs created in the same second are
31+
// ordered by name. The order is the same from either side.
32+
//
33+
// The list comes from reader, which must not be the informer cache: a job can
34+
// be reconciled before the cache holds a peer's StartedAt, see that peer as
35+
// not started, and proceed alongside it. Backups are read through c.
36+
func restoreAhead(ctx context.Context, reader client.Reader, c client.Client, rj *backupsv1alpha1.RestoreJob) (string, error) {
37+
target, ok, err := restoreJobTarget(ctx, c, rj)
38+
if err != nil || !ok {
39+
return "", err
40+
}
41+
list := &backupsv1alpha1.RestoreJobList{}
42+
if err := reader.List(ctx, list, client.InNamespace(rj.Namespace)); err != nil {
43+
return "", fmt.Errorf("list RestoreJobs: %w", err)
44+
}
45+
var ahead []string
46+
for i := range list.Items {
47+
other := &list.Items[i]
48+
if other.Name == rj.Name {
49+
continue
50+
}
51+
if other.Status.Phase == backupsv1alpha1.RestoreJobPhaseSucceeded || other.Status.Phase == backupsv1alpha1.RestoreJobPhaseFailed {
52+
continue
53+
}
54+
if !other.DeletionTimestamp.IsZero() || !restoreGoesFirst(other, rj) {
55+
continue
56+
}
57+
otherTarget, ok, err := restoreJobTarget(ctx, c, other)
58+
if err != nil {
59+
return "", err
60+
}
61+
if ok && sameApplication(otherTarget, target) {
62+
ahead = append(ahead, other.Name)
63+
}
64+
}
65+
if len(ahead) == 0 {
66+
return "", nil
67+
}
68+
sort.Strings(ahead)
69+
return ahead[0], nil
70+
}
71+
72+
func restoreJobStarted(rj *backupsv1alpha1.RestoreJob) bool {
73+
return rj.Status.StartedAt != nil || rj.Status.Phase == backupsv1alpha1.RestoreJobPhaseRunning
74+
}
75+
76+
// restoreGoesFirst reports whether a runs before b; see restoreAhead.
77+
func restoreGoesFirst(a, b *backupsv1alpha1.RestoreJob) bool {
78+
if sa, sb := restoreJobStarted(a), restoreJobStarted(b); sa != sb {
79+
return sa
80+
}
81+
if !a.CreationTimestamp.Equal(&b.CreationTimestamp) {
82+
return a.CreationTimestamp.Before(&b.CreationTimestamp)
83+
}
84+
return a.Name < b.Name
85+
}
Lines changed: 220 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,220 @@
1+
// SPDX-License-Identifier: Apache-2.0
2+
package backupcontroller
3+
4+
import (
5+
"context"
6+
"strings"
7+
"testing"
8+
"time"
9+
10+
corev1 "k8s.io/api/core/v1"
11+
"k8s.io/apimachinery/pkg/api/meta"
12+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
13+
"k8s.io/apimachinery/pkg/types"
14+
"k8s.io/client-go/tools/record"
15+
"sigs.k8s.io/controller-runtime/pkg/client"
16+
"sigs.k8s.io/controller-runtime/pkg/reconcile"
17+
18+
strategyv1alpha1 "github.com/cozystack/cozystack/api/backups/strategy/v1alpha1"
19+
backupsv1alpha1 "github.com/cozystack/cozystack/api/backups/v1alpha1"
20+
)
21+
22+
func TestRestoreAhead(t *testing.T) {
23+
t0 := metav1.NewTime(time.Date(2026, 10, 1, 12, 0, 0, 0, time.UTC))
24+
t1 := metav1.NewTime(t0.Add(time.Minute))
25+
started := metav1.NewTime(t0.Add(30 * time.Second))
26+
other := pgRef("other")
27+
28+
restore := func(name string, created metav1.Time, phase backupsv1alpha1.RestoreJobPhase, target *corev1.TypedLocalObjectReference) *backupsv1alpha1.RestoreJob {
29+
rj := lockRestore(name, "bk", phase, target, "")
30+
rj.CreationTimestamp = created
31+
return rj
32+
}
33+
34+
for _, tc := range []struct {
35+
name string
36+
self *backupsv1alpha1.RestoreJob
37+
others []client.Object
38+
want string
39+
}{
40+
{
41+
name: "no other restore",
42+
self: restore("second", t1, "", nil),
43+
want: "",
44+
},
45+
{
46+
name: "an older restore that has not started goes first",
47+
self: restore("second", t1, "", nil),
48+
others: []client.Object{restore("first", t0, backupsv1alpha1.RestoreJobPhasePending, nil)},
49+
want: "first",
50+
},
51+
{
52+
name: "a newer restore that has not started does not go first",
53+
self: restore("first", t0, "", nil),
54+
others: []client.Object{restore("second", t1, "", nil)},
55+
want: "",
56+
},
57+
{
58+
name: "a running restore goes first even when created later",
59+
self: restore("first", t0, "", nil),
60+
others: []client.Object{restore("second", t1, backupsv1alpha1.RestoreJobPhaseRunning, nil)},
61+
want: "second",
62+
},
63+
{
64+
name: "a restore with StartedAt goes first even when created later",
65+
self: restore("first", t0, "", nil),
66+
others: []client.Object{func() client.Object {
67+
rj := restore("second", t1, "", nil)
68+
rj.Status.StartedAt = &started
69+
return rj
70+
}()},
71+
want: "second",
72+
},
73+
{
74+
name: "created at the same time: the lower name goes first",
75+
self: restore("b", t0, "", nil),
76+
others: []client.Object{restore("a", t0, "", nil)},
77+
want: "a",
78+
},
79+
{
80+
name: "created at the same time: a higher name does not go first",
81+
self: restore("a", t0, "", nil),
82+
others: []client.Object{restore("b", t0, "", nil)},
83+
want: "",
84+
},
85+
{
86+
name: "finished restores do not hold",
87+
self: restore("third", t1, "", nil),
88+
others: []client.Object{
89+
restore("done", t0, backupsv1alpha1.RestoreJobPhaseSucceeded, nil),
90+
restore("failed", t0, backupsv1alpha1.RestoreJobPhaseFailed, nil),
91+
},
92+
want: "",
93+
},
94+
{
95+
name: "a restore being deleted does not hold",
96+
self: restore("second", t1, "", nil),
97+
others: []client.Object{func() client.Object {
98+
rj := restore("first", t0, backupsv1alpha1.RestoreJobPhaseRunning, nil)
99+
rj.DeletionTimestamp = &t1
100+
rj.Finalizers = []string{restoreJobFinalizer}
101+
return rj
102+
}()},
103+
want: "",
104+
},
105+
{
106+
name: "a restore into another application does not hold",
107+
self: restore("second", t1, "", nil),
108+
others: []client.Object{restore("first", t0, backupsv1alpha1.RestoreJobPhaseRunning, &other)},
109+
want: "",
110+
},
111+
{
112+
name: "a restore retargeted onto this job's application holds",
113+
self: restore("second", t1, "", &other),
114+
others: []client.Object{restore("first", t0, backupsv1alpha1.RestoreJobPhaseRunning, &other)},
115+
want: "first",
116+
},
117+
{
118+
name: "the lowest name is reported when several hold",
119+
self: restore("third", t1, "", nil),
120+
others: []client.Object{restore("zeta", t0, backupsv1alpha1.RestoreJobPhaseRunning, nil), restore("alpha", t0, backupsv1alpha1.RestoreJobPhaseRunning, nil)},
121+
want: "alpha",
122+
},
123+
} {
124+
t.Run(tc.name, func(t *testing.T) {
125+
objs := append([]client.Object{lockBackup("bk", pgRef("pg")), tc.self}, tc.others...)
126+
c := lockTestClient(t, objs...)
127+
got, err := restoreAhead(context.Background(), c, c, tc.self)
128+
if err != nil {
129+
t.Fatalf("restoreAhead: %v", err)
130+
}
131+
if got != tc.want {
132+
t.Fatalf("restoreAhead = %q, want %q", got, tc.want)
133+
}
134+
})
135+
}
136+
}
137+
138+
func TestRestoreAhead_VeleroIntoAnotherNamespaceIsNotHeld(t *testing.T) {
139+
t0 := metav1.NewTime(time.Date(2026, 10, 1, 12, 0, 0, 0, time.UTC))
140+
vm := corev1.TypedLocalObjectReference{APIGroup: new("apps.cozystack.io"), Kind: "VMInstance", Name: "vm"}
141+
first := lockRestore("first", "bk", backupsv1alpha1.RestoreJobPhaseRunning, nil, "")
142+
first.CreationTimestamp = t0
143+
second := lockRestore("second", "bk", "", nil, `{"targetNamespace":"tenant-copy"}`)
144+
second.CreationTimestamp = metav1.NewTime(t0.Add(time.Minute))
145+
c := lockTestClient(t, lockBackupBy("bk", vm, strategyv1alpha1.VeleroStrategyKind), first, second)
146+
got, err := restoreAhead(context.Background(), c, c, second)
147+
if err != nil {
148+
t.Fatalf("restoreAhead: %v", err)
149+
}
150+
if got != "" {
151+
t.Fatalf("restoreAhead = %q, want none: the job writes into another namespace", got)
152+
}
153+
}
154+
155+
func TestReconcileRestore_ConcurrentRestoreIsRefused(t *testing.T) {
156+
t0 := metav1.NewTime(time.Date(2026, 10, 1, 12, 0, 0, 0, time.UTC))
157+
first := lockRestore("first", "bk", backupsv1alpha1.RestoreJobPhaseRunning, nil, "")
158+
first.CreationTimestamp = t0
159+
first.Status.StartedAt = &t0
160+
second := lockRestore("second", "bk", "", nil, "")
161+
second.CreationTimestamp = metav1.NewTime(t0.Add(time.Minute))
162+
c := lockTestClient(t, lockBackup("bk", pgRef("pg")), first, second)
163+
r := &RestoreJobReconciler{Client: c, Recorder: record.NewFakeRecorder(10)}
164+
165+
if _, err := r.Reconcile(context.Background(), reconcile.Request{NamespacedName: types.NamespacedName{Namespace: lockNS, Name: "second"}}); err != nil {
166+
t.Fatalf("Reconcile: %v", err)
167+
}
168+
got := &backupsv1alpha1.RestoreJob{}
169+
if err := c.Get(context.Background(), types.NamespacedName{Namespace: lockNS, Name: "second"}, got); err != nil {
170+
t.Fatalf("get RestoreJob: %v", err)
171+
}
172+
if got.Status.Phase != backupsv1alpha1.RestoreJobPhaseFailed {
173+
t.Fatalf("Phase = %q, want Failed", got.Status.Phase)
174+
}
175+
ready := meta.FindStatusCondition(got.Status.Conditions, "Ready")
176+
if ready == nil || ready.Reason != ConditionReasonConcurrentRestore {
177+
t.Fatalf("Ready condition = %+v, want reason %s", ready, ConditionReasonConcurrentRestore)
178+
}
179+
if !strings.Contains(got.Status.Message, "first") {
180+
t.Fatalf("message %q does not name the restore that goes first", got.Status.Message)
181+
}
182+
183+
running := &backupsv1alpha1.RestoreJob{}
184+
if err := c.Get(context.Background(), types.NamespacedName{Namespace: lockNS, Name: "first"}, running); err != nil {
185+
t.Fatalf("get RestoreJob: %v", err)
186+
}
187+
if running.Status.Phase != backupsv1alpha1.RestoreJobPhaseRunning {
188+
t.Fatalf("the running restore changed phase to %q", running.Status.Phase)
189+
}
190+
}
191+
192+
// The informer cache can lag a peer's status: here it still holds zeta as not
193+
// started, while the API server already has its StartedAt. Read from the cache,
194+
// alpha would see zeta behind it by name and proceed alongside it.
195+
func TestReconcileRestore_ConcurrentRestoreReadsPastTheCache(t *testing.T) {
196+
t0 := metav1.NewTime(time.Date(2026, 10, 1, 12, 0, 0, 0, time.UTC))
197+
staleZeta := lockRestore("zeta", "bk", "", nil, "")
198+
staleZeta.CreationTimestamp = t0
199+
alpha := lockRestore("alpha", "bk", "", nil, "")
200+
alpha.CreationTimestamp = t0
201+
startedZeta := staleZeta.DeepCopy()
202+
startedZeta.Status.Phase = backupsv1alpha1.RestoreJobPhaseRunning
203+
startedZeta.Status.StartedAt = &t0
204+
205+
cached := lockTestClient(t, lockBackup("bk", pgRef("pg")), staleZeta, alpha)
206+
live := lockTestClient(t, lockBackup("bk", pgRef("pg")), startedZeta, alpha.DeepCopy())
207+
r := &RestoreJobReconciler{Client: cached, APIReader: live, Recorder: record.NewFakeRecorder(10)}
208+
209+
if _, err := r.Reconcile(context.Background(), reconcile.Request{NamespacedName: types.NamespacedName{Namespace: lockNS, Name: "alpha"}}); err != nil {
210+
t.Fatalf("Reconcile: %v", err)
211+
}
212+
got := &backupsv1alpha1.RestoreJob{}
213+
if err := cached.Get(context.Background(), types.NamespacedName{Namespace: lockNS, Name: "alpha"}, got); err != nil {
214+
t.Fatalf("get RestoreJob: %v", err)
215+
}
216+
ready := meta.FindStatusCondition(got.Status.Conditions, "Ready")
217+
if got.Status.Phase != backupsv1alpha1.RestoreJobPhaseFailed || ready == nil || ready.Reason != ConditionReasonConcurrentRestore {
218+
t.Fatalf("alpha: phase %q, Ready %+v; want Failed with reason %s, since zeta has started", got.Status.Phase, ready, ConditionReasonConcurrentRestore)
219+
}
220+
}

0 commit comments

Comments
 (0)