Skip to content

Commit ac50346

Browse files
committed
fix(clickhouse): take no restore Job's result that this RestoreJob did not run
The restore path took any Job standing under a RestoreJob's Job name as its own, so a RestoreJob re-created under a used name met the completed Job of the earlier one before the garbage collector removed it and was marked Succeeded although nothing was restored for it. The restore path now refuses such a Job the way the backup path does: the Job of an earlier RestoreJob of the same name is waited out, with the Ready condition naming it, and any other Job holding the name fails the RestoreJob. Assisted-by: LLM Signed-off-by: Alexey Artamonov <[email protected]>
1 parent 5932f62 commit ac50346

2 files changed

Lines changed: 122 additions & 11 deletions

File tree

‎internal/backupcontroller/altinitystrategy_controller.go‎

Lines changed: 25 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ import (
1818
"k8s.io/apimachinery/pkg/api/resource"
1919
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
2020
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
21+
"k8s.io/apimachinery/pkg/runtime"
2122
"k8s.io/apimachinery/pkg/runtime/schema"
2223
"k8s.io/apimachinery/pkg/types"
2324
ctrl "sigs.k8s.io/controller-runtime"
@@ -390,7 +391,7 @@ func (r *BackupJobReconciler) ensureAltinityJob(
390391
existing := &batchv1.Job{}
391392
err := r.Get(ctx, types.NamespacedName{Namespace: namespace, Name: name}, existing)
392393
if err == nil {
393-
return existing, r.checkAltinityJobController(existing, owner)
394+
return existing, checkAltinityJobController(r.Scheme, existing, owner)
394395
}
395396
if !apierrors.IsNotFound(err) {
396397
return nil, err
@@ -409,26 +410,27 @@ func (r *BackupJobReconciler) ensureAltinityJob(
409410
if err := r.Get(ctx, types.NamespacedName{Namespace: namespace, Name: name}, existing); err != nil {
410411
return nil, err
411412
}
412-
return existing, r.checkAltinityJobController(existing, owner)
413+
return existing, checkAltinityJobController(r.Scheme, existing, owner)
413414
}
414415
return nil, err
415416
}
416417
return desired, nil
417418
}
418419

419-
// errAltinityPredecessorJob marks a backup Job left under the Job name by an
420-
// earlier BackupJob of the same name. Its owner is gone, so the garbage
421-
// collector removes it, and the BackupJob waits for that instead of failing.
420+
// errAltinityPredecessorJob marks a Job left under the Job name by an earlier
421+
// BackupJob or RestoreJob of the same name. Its owner is gone, so the garbage
422+
// collector removes it, and the new owner waits for that instead of failing.
422423
var errAltinityPredecessorJob = errors.New("waiting for the garbage collector to remove a Job of an earlier owner of the same name")
423424

424425
// checkAltinityJobController refuses a Job under owner's Job name that owner
425-
// does not control. A BackupJob re-created under a used name meets the Job of
426-
// the earlier one, and that Job's completion says nothing about this run.
427-
func (r *BackupJobReconciler) checkAltinityJobController(job *batchv1.Job, owner client.Object) error {
426+
// does not control. A BackupJob or RestoreJob re-created under a used name
427+
// meets the Job of the earlier one, and that Job's completion says nothing
428+
// about this run.
429+
func checkAltinityJobController(scheme *runtime.Scheme, job *batchv1.Job, owner client.Object) error {
428430
if metav1.IsControlledBy(job, owner) {
429431
return nil
430432
}
431-
gvk, err := apiutil.GVKForObject(owner, r.Scheme)
433+
gvk, err := apiutil.GVKForObject(owner, scheme)
432434
if err != nil {
433435
return err
434436
}
@@ -456,7 +458,7 @@ func (r *RestoreJobReconciler) ensureAltinityRestoreJob(
456458
existing := &batchv1.Job{}
457459
err := r.Get(ctx, types.NamespacedName{Namespace: namespace, Name: name}, existing)
458460
if err == nil {
459-
return existing, nil
461+
return existing, checkAltinityJobController(r.Scheme, existing, owner)
460462
}
461463
if !apierrors.IsNotFound(err) {
462464
return nil, err
@@ -474,7 +476,7 @@ func (r *RestoreJobReconciler) ensureAltinityRestoreJob(
474476
if err := r.Get(ctx, types.NamespacedName{Namespace: namespace, Name: name}, existing); err != nil {
475477
return nil, err
476478
}
477-
return existing, nil
479+
return existing, checkAltinityJobController(r.Scheme, existing, owner)
478480
}
479481
return nil, err
480482
}
@@ -777,6 +779,18 @@ func (r *RestoreJobReconciler) reconcileAltinityRestore(ctx context.Context, res
777779
},
778780
rendered,
779781
)
782+
if errors.Is(err, errAltinityPredecessorJob) {
783+
apimeta.SetStatusCondition(&restoreJob.Status.Conditions, metav1.Condition{
784+
Type: "Ready",
785+
Status: metav1.ConditionFalse,
786+
Reason: "PredecessorJobPending",
787+
Message: err.Error(),
788+
})
789+
if uerr := r.Status().Update(ctx, restoreJob); uerr != nil {
790+
return ctrl.Result{}, uerr
791+
}
792+
return ctrl.Result{RequeueAfter: altinityPollInterval}, nil
793+
}
780794
if err != nil {
781795
return r.markRestoreJobFailed(ctx, restoreJob, fmt.Sprintf("failed to ensure batch/v1.Job: %v", err))
782796
}

‎internal/backupcontroller/altinitystrategy_controller_test.go‎

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1243,3 +1243,100 @@ func TestReconcileAltinity_DoesNotReuseAJobItDoesNotControl(t *testing.T) {
12431243
}
12441244
}
12451245
}
1246+
1247+
// TestReconcileAltinityRestore_DoesNotReuseAJobItDoesNotControl is the restore
1248+
// side of TestReconcileAltinity_DoesNotReuseAJobItDoesNotControl: taking
1249+
// another Job's completion would report a restore that never ran.
1250+
func TestReconcileAltinityRestore_DoesNotReuseAJobItDoesNotControl(t *testing.T) {
1251+
restoreRef := func(name, uid string, gv schema.GroupVersion, kind string) []metav1.OwnerReference {
1252+
owner := &backupsv1alpha1.RestoreJob{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: "tenant-test", UID: types.UID(uid)}}
1253+
return []metav1.OwnerReference{*metav1.NewControllerRef(owner, gv.WithKind(kind))}
1254+
}
1255+
other := schema.GroupVersion{Group: "example.com", Version: "v1alpha1"}
1256+
cases := []struct {
1257+
name string
1258+
owners []metav1.OwnerReference
1259+
wantFailed bool
1260+
}{
1261+
{name: "an earlier RestoreJob of the same name controls it", owners: restoreRef("rj", "predecessor-uid", backupsv1alpha1.GroupVersion, "RestoreJob")},
1262+
{name: "nothing controls it", wantFailed: true},
1263+
{name: "a RestoreJob of another name controls it", wantFailed: true, owners: restoreRef("other-rj", "other-uid", backupsv1alpha1.GroupVersion, "RestoreJob")},
1264+
{name: "a BackupJob controls it", wantFailed: true, owners: restoreRef("rj", "predecessor-uid", backupsv1alpha1.GroupVersion, "BackupJob")},
1265+
{name: "a RestoreJob of another API group controls it", wantFailed: true, owners: restoreRef("rj", "predecessor-uid", other, "RestoreJob")},
1266+
}
1267+
for _, tc := range cases {
1268+
for _, raced := range []bool{false, true} {
1269+
t.Run(fmt.Sprintf("%s/raced=%t", tc.name, raced), func(t *testing.T) {
1270+
targetApp := newAltinityClickHouseApp("ch-test", "tenant-test")
1271+
strategy := newAltinityArchiveStrategy("clickhouse-strategy")
1272+
now := metav1.Now()
1273+
backup := &backupsv1alpha1.Backup{
1274+
ObjectMeta: metav1.ObjectMeta{Name: "ch-backup", Namespace: "tenant-test"},
1275+
Spec: backupsv1alpha1.BackupSpec{
1276+
ApplicationRef: newAltinityClickHouseRef("ch-test"),
1277+
StrategyRef: corev1.TypedLocalObjectReference{
1278+
APIGroup: new(strategyv1alpha1.GroupVersion.Group),
1279+
Kind: strategyv1alpha1.AltinityStrategyKind,
1280+
Name: "clickhouse-strategy",
1281+
},
1282+
TakenAt: now,
1283+
},
1284+
}
1285+
restoreJob := &backupsv1alpha1.RestoreJob{
1286+
ObjectMeta: metav1.ObjectMeta{Name: "rj", Namespace: "tenant-test", UID: "this-uid"},
1287+
Spec: backupsv1alpha1.RestoreJobSpec{BackupRef: corev1.LocalObjectReference{Name: backup.Name}},
1288+
Status: backupsv1alpha1.RestoreJobStatus{StartedAt: &now, Phase: backupsv1alpha1.RestoreJobPhaseRunning},
1289+
}
1290+
job := &batchv1.Job{
1291+
ObjectMeta: metav1.ObjectMeta{Name: jobNameForRestoreJob(restoreJob), Namespace: "tenant-test", OwnerReferences: tc.owners},
1292+
Status: batchv1.JobStatus{Conditions: []batchv1.JobCondition{
1293+
{Type: batchv1.JobComplete, Status: corev1.ConditionTrue},
1294+
}},
1295+
}
1296+
builder := clientfake.NewClientBuilder().WithObjects(strategy, backup, restoreJob, job)
1297+
if raced {
1298+
hidden := false
1299+
builder = builder.WithInterceptorFuncs(interceptor.Funcs{
1300+
Get: func(ctx context.Context, c client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error {
1301+
if _, isJob := obj.(*batchv1.Job); isJob && !hidden {
1302+
hidden = true
1303+
return apierrors.NewNotFound(batchv1.Resource("jobs"), key.Name)
1304+
}
1305+
return c.Get(ctx, key, obj, opts...)
1306+
},
1307+
})
1308+
}
1309+
_, rr, _ := newAltinityTestEnv(t, targetApp, builder)
1310+
ctx := context.Background()
1311+
1312+
res, err := rr.reconcileAltinityRestore(ctx, restoreJob, backup)
1313+
if err != nil {
1314+
t.Fatalf("reconcileAltinityRestore: %v", err)
1315+
}
1316+
1317+
got := &backupsv1alpha1.RestoreJob{}
1318+
if err := rr.Get(ctx, client.ObjectKeyFromObject(restoreJob), got); err != nil {
1319+
t.Fatalf("get RestoreJob: %v", err)
1320+
}
1321+
if tc.wantFailed {
1322+
if got.Status.Phase != backupsv1alpha1.RestoreJobPhaseFailed {
1323+
t.Fatalf("phase = %q, want Failed", got.Status.Phase)
1324+
}
1325+
if !strings.Contains(got.Status.Message, job.Name) {
1326+
t.Errorf("message %q does not name Job %s", got.Status.Message, job.Name)
1327+
}
1328+
return
1329+
}
1330+
if got.Status.Phase != backupsv1alpha1.RestoreJobPhaseRunning {
1331+
t.Errorf("phase = %q, want Running while the earlier Job awaits garbage collection", got.Status.Phase)
1332+
}
1333+
if res.RequeueAfter <= 0 {
1334+
t.Errorf("no requeue while the earlier Job awaits garbage collection: %+v", res)
1335+
}
1336+
if c := meta.FindStatusCondition(got.Status.Conditions, "Ready"); c == nil || !strings.Contains(c.Message, job.Name) {
1337+
t.Errorf("Ready condition %+v does not say which Job it waits for", c)
1338+
}
1339+
})
1340+
}
1341+
}
1342+
}

0 commit comments

Comments
 (0)