|
| 1 | +// SPDX-License-Identifier: Apache-2.0 |
| 2 | +package backupcontroller |
| 3 | + |
| 4 | +import ( |
| 5 | + "context" |
| 6 | + "strings" |
| 7 | + "testing" |
| 8 | + "time" |
| 9 | + |
| 10 | + corev1 "k8s.io/api/core/v1" |
| 11 | + "k8s.io/apimachinery/pkg/api/meta" |
| 12 | + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" |
| 13 | + "k8s.io/apimachinery/pkg/types" |
| 14 | + "k8s.io/client-go/tools/record" |
| 15 | + "sigs.k8s.io/controller-runtime/pkg/client" |
| 16 | + "sigs.k8s.io/controller-runtime/pkg/reconcile" |
| 17 | + |
| 18 | + strategyv1alpha1 "github.com/cozystack/cozystack/api/backups/strategy/v1alpha1" |
| 19 | + backupsv1alpha1 "github.com/cozystack/cozystack/api/backups/v1alpha1" |
| 20 | +) |
| 21 | + |
| 22 | +func TestRestoreAhead(t *testing.T) { |
| 23 | + t0 := metav1.NewTime(time.Date(2026, 10, 1, 12, 0, 0, 0, time.UTC)) |
| 24 | + t1 := metav1.NewTime(t0.Add(time.Minute)) |
| 25 | + started := metav1.NewTime(t0.Add(30 * time.Second)) |
| 26 | + other := pgRef("other") |
| 27 | + |
| 28 | + restore := func(name string, created metav1.Time, phase backupsv1alpha1.RestoreJobPhase, target *corev1.TypedLocalObjectReference) *backupsv1alpha1.RestoreJob { |
| 29 | + rj := lockRestore(name, "bk", phase, target, "") |
| 30 | + rj.CreationTimestamp = created |
| 31 | + return rj |
| 32 | + } |
| 33 | + |
| 34 | + for _, tc := range []struct { |
| 35 | + name string |
| 36 | + self *backupsv1alpha1.RestoreJob |
| 37 | + others []client.Object |
| 38 | + want string |
| 39 | + }{ |
| 40 | + { |
| 41 | + name: "no other restore", |
| 42 | + self: restore("second", t1, "", nil), |
| 43 | + want: "", |
| 44 | + }, |
| 45 | + { |
| 46 | + name: "an older restore that has not started goes first", |
| 47 | + self: restore("second", t1, "", nil), |
| 48 | + others: []client.Object{restore("first", t0, backupsv1alpha1.RestoreJobPhasePending, nil)}, |
| 49 | + want: "first", |
| 50 | + }, |
| 51 | + { |
| 52 | + name: "a newer restore that has not started does not go first", |
| 53 | + self: restore("first", t0, "", nil), |
| 54 | + others: []client.Object{restore("second", t1, "", nil)}, |
| 55 | + want: "", |
| 56 | + }, |
| 57 | + { |
| 58 | + name: "a running restore goes first even when created later", |
| 59 | + self: restore("first", t0, "", nil), |
| 60 | + others: []client.Object{restore("second", t1, backupsv1alpha1.RestoreJobPhaseRunning, nil)}, |
| 61 | + want: "second", |
| 62 | + }, |
| 63 | + { |
| 64 | + name: "a restore with StartedAt goes first even when created later", |
| 65 | + self: restore("first", t0, "", nil), |
| 66 | + others: []client.Object{func() client.Object { |
| 67 | + rj := restore("second", t1, "", nil) |
| 68 | + rj.Status.StartedAt = &started |
| 69 | + return rj |
| 70 | + }()}, |
| 71 | + want: "second", |
| 72 | + }, |
| 73 | + { |
| 74 | + name: "created at the same time: the lower name goes first", |
| 75 | + self: restore("b", t0, "", nil), |
| 76 | + others: []client.Object{restore("a", t0, "", nil)}, |
| 77 | + want: "a", |
| 78 | + }, |
| 79 | + { |
| 80 | + name: "created at the same time: a higher name does not go first", |
| 81 | + self: restore("a", t0, "", nil), |
| 82 | + others: []client.Object{restore("b", t0, "", nil)}, |
| 83 | + want: "", |
| 84 | + }, |
| 85 | + { |
| 86 | + name: "finished restores do not hold", |
| 87 | + self: restore("third", t1, "", nil), |
| 88 | + others: []client.Object{ |
| 89 | + restore("done", t0, backupsv1alpha1.RestoreJobPhaseSucceeded, nil), |
| 90 | + restore("failed", t0, backupsv1alpha1.RestoreJobPhaseFailed, nil), |
| 91 | + }, |
| 92 | + want: "", |
| 93 | + }, |
| 94 | + { |
| 95 | + name: "a restore being deleted does not hold", |
| 96 | + self: restore("second", t1, "", nil), |
| 97 | + others: []client.Object{func() client.Object { |
| 98 | + rj := restore("first", t0, backupsv1alpha1.RestoreJobPhaseRunning, nil) |
| 99 | + rj.DeletionTimestamp = &t1 |
| 100 | + rj.Finalizers = []string{restoreJobFinalizer} |
| 101 | + return rj |
| 102 | + }()}, |
| 103 | + want: "", |
| 104 | + }, |
| 105 | + { |
| 106 | + name: "a restore into another application does not hold", |
| 107 | + self: restore("second", t1, "", nil), |
| 108 | + others: []client.Object{restore("first", t0, backupsv1alpha1.RestoreJobPhaseRunning, &other)}, |
| 109 | + want: "", |
| 110 | + }, |
| 111 | + { |
| 112 | + name: "a restore retargeted onto this job's application holds", |
| 113 | + self: restore("second", t1, "", &other), |
| 114 | + others: []client.Object{restore("first", t0, backupsv1alpha1.RestoreJobPhaseRunning, &other)}, |
| 115 | + want: "first", |
| 116 | + }, |
| 117 | + { |
| 118 | + name: "the lowest name is reported when several hold", |
| 119 | + self: restore("third", t1, "", nil), |
| 120 | + others: []client.Object{restore("zeta", t0, backupsv1alpha1.RestoreJobPhaseRunning, nil), restore("alpha", t0, backupsv1alpha1.RestoreJobPhaseRunning, nil)}, |
| 121 | + want: "alpha", |
| 122 | + }, |
| 123 | + } { |
| 124 | + t.Run(tc.name, func(t *testing.T) { |
| 125 | + objs := append([]client.Object{lockBackup("bk", pgRef("pg")), tc.self}, tc.others...) |
| 126 | + c := lockTestClient(t, objs...) |
| 127 | + got, err := restoreAhead(context.Background(), c, c, tc.self) |
| 128 | + if err != nil { |
| 129 | + t.Fatalf("restoreAhead: %v", err) |
| 130 | + } |
| 131 | + if got != tc.want { |
| 132 | + t.Fatalf("restoreAhead = %q, want %q", got, tc.want) |
| 133 | + } |
| 134 | + }) |
| 135 | + } |
| 136 | +} |
| 137 | + |
| 138 | +func TestRestoreAhead_VeleroIntoAnotherNamespaceIsNotHeld(t *testing.T) { |
| 139 | + t0 := metav1.NewTime(time.Date(2026, 10, 1, 12, 0, 0, 0, time.UTC)) |
| 140 | + vm := corev1.TypedLocalObjectReference{APIGroup: new("apps.cozystack.io"), Kind: "VMInstance", Name: "vm"} |
| 141 | + first := lockRestore("first", "bk", backupsv1alpha1.RestoreJobPhaseRunning, nil, "") |
| 142 | + first.CreationTimestamp = t0 |
| 143 | + second := lockRestore("second", "bk", "", nil, `{"targetNamespace":"tenant-copy"}`) |
| 144 | + second.CreationTimestamp = metav1.NewTime(t0.Add(time.Minute)) |
| 145 | + c := lockTestClient(t, lockBackupBy("bk", vm, strategyv1alpha1.VeleroStrategyKind), first, second) |
| 146 | + got, err := restoreAhead(context.Background(), c, c, second) |
| 147 | + if err != nil { |
| 148 | + t.Fatalf("restoreAhead: %v", err) |
| 149 | + } |
| 150 | + if got != "" { |
| 151 | + t.Fatalf("restoreAhead = %q, want none: the job writes into another namespace", got) |
| 152 | + } |
| 153 | +} |
| 154 | + |
| 155 | +func TestReconcileRestore_ConcurrentRestoreIsRefused(t *testing.T) { |
| 156 | + t0 := metav1.NewTime(time.Date(2026, 10, 1, 12, 0, 0, 0, time.UTC)) |
| 157 | + first := lockRestore("first", "bk", backupsv1alpha1.RestoreJobPhaseRunning, nil, "") |
| 158 | + first.CreationTimestamp = t0 |
| 159 | + first.Status.StartedAt = &t0 |
| 160 | + second := lockRestore("second", "bk", "", nil, "") |
| 161 | + second.CreationTimestamp = metav1.NewTime(t0.Add(time.Minute)) |
| 162 | + c := lockTestClient(t, lockBackup("bk", pgRef("pg")), first, second) |
| 163 | + r := &RestoreJobReconciler{Client: c, Recorder: record.NewFakeRecorder(10)} |
| 164 | + |
| 165 | + if _, err := r.Reconcile(context.Background(), reconcile.Request{NamespacedName: types.NamespacedName{Namespace: lockNS, Name: "second"}}); err != nil { |
| 166 | + t.Fatalf("Reconcile: %v", err) |
| 167 | + } |
| 168 | + got := &backupsv1alpha1.RestoreJob{} |
| 169 | + if err := c.Get(context.Background(), types.NamespacedName{Namespace: lockNS, Name: "second"}, got); err != nil { |
| 170 | + t.Fatalf("get RestoreJob: %v", err) |
| 171 | + } |
| 172 | + if got.Status.Phase != backupsv1alpha1.RestoreJobPhaseFailed { |
| 173 | + t.Fatalf("Phase = %q, want Failed", got.Status.Phase) |
| 174 | + } |
| 175 | + ready := meta.FindStatusCondition(got.Status.Conditions, "Ready") |
| 176 | + if ready == nil || ready.Reason != ConditionReasonConcurrentRestore { |
| 177 | + t.Fatalf("Ready condition = %+v, want reason %s", ready, ConditionReasonConcurrentRestore) |
| 178 | + } |
| 179 | + if !strings.Contains(got.Status.Message, "first") { |
| 180 | + t.Fatalf("message %q does not name the restore that goes first", got.Status.Message) |
| 181 | + } |
| 182 | + |
| 183 | + running := &backupsv1alpha1.RestoreJob{} |
| 184 | + if err := c.Get(context.Background(), types.NamespacedName{Namespace: lockNS, Name: "first"}, running); err != nil { |
| 185 | + t.Fatalf("get RestoreJob: %v", err) |
| 186 | + } |
| 187 | + if running.Status.Phase != backupsv1alpha1.RestoreJobPhaseRunning { |
| 188 | + t.Fatalf("the running restore changed phase to %q", running.Status.Phase) |
| 189 | + } |
| 190 | +} |
| 191 | + |
| 192 | +// The informer cache can lag a peer's status: here it still holds zeta as not |
| 193 | +// started, while the API server already has its StartedAt. Read from the cache, |
| 194 | +// alpha would see zeta behind it by name and proceed alongside it. |
| 195 | +func TestReconcileRestore_ConcurrentRestoreReadsPastTheCache(t *testing.T) { |
| 196 | + t0 := metav1.NewTime(time.Date(2026, 10, 1, 12, 0, 0, 0, time.UTC)) |
| 197 | + staleZeta := lockRestore("zeta", "bk", "", nil, "") |
| 198 | + staleZeta.CreationTimestamp = t0 |
| 199 | + alpha := lockRestore("alpha", "bk", "", nil, "") |
| 200 | + alpha.CreationTimestamp = t0 |
| 201 | + startedZeta := staleZeta.DeepCopy() |
| 202 | + startedZeta.Status.Phase = backupsv1alpha1.RestoreJobPhaseRunning |
| 203 | + startedZeta.Status.StartedAt = &t0 |
| 204 | + |
| 205 | + cached := lockTestClient(t, lockBackup("bk", pgRef("pg")), staleZeta, alpha) |
| 206 | + live := lockTestClient(t, lockBackup("bk", pgRef("pg")), startedZeta, alpha.DeepCopy()) |
| 207 | + r := &RestoreJobReconciler{Client: cached, APIReader: live, Recorder: record.NewFakeRecorder(10)} |
| 208 | + |
| 209 | + if _, err := r.Reconcile(context.Background(), reconcile.Request{NamespacedName: types.NamespacedName{Namespace: lockNS, Name: "alpha"}}); err != nil { |
| 210 | + t.Fatalf("Reconcile: %v", err) |
| 211 | + } |
| 212 | + got := &backupsv1alpha1.RestoreJob{} |
| 213 | + if err := cached.Get(context.Background(), types.NamespacedName{Namespace: lockNS, Name: "alpha"}, got); err != nil { |
| 214 | + t.Fatalf("get RestoreJob: %v", err) |
| 215 | + } |
| 216 | + ready := meta.FindStatusCondition(got.Status.Conditions, "Ready") |
| 217 | + if got.Status.Phase != backupsv1alpha1.RestoreJobPhaseFailed || ready == nil || ready.Reason != ConditionReasonConcurrentRestore { |
| 218 | + t.Fatalf("alpha: phase %q, Ready %+v; want Failed with reason %s, since zeta has started", got.Status.Phase, ready, ConditionReasonConcurrentRestore) |
| 219 | + } |
| 220 | +} |
0 commit comments