Skip to content

Commit d0404f8

Browse files
committed
fix(clickhouse): leave a Job the cleanup did not create alone
The cleanup Job is found by name alone, and <backup>-cleanup is also the name of the ClickHouse chart's own cleanup hook when a BackupJob is named after the release. The skip annotation and the stale-Job check then deleted that hook, and its completion could pass for the archive delete. Only a Job carrying the cleanup mode label is treated as the controller's own; any other Job of that name releases the Backup with a Warning Event instead. Assisted-by: LLM Signed-off-by: Alexey Artamonov <[email protected]>
1 parent 5d44733 commit d0404f8

2 files changed

Lines changed: 48 additions & 2 deletions

File tree

‎internal/backupcontroller/altinitystrategy_archive_test.go‎

Lines changed: 34 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -583,8 +583,7 @@ func TestCleanupAltinityBackup_ApplicationKindNotServedReleases(t *testing.T) {
583583
func TestCleanupAltinityBackup_SkipAnnotationReapsInflightJob(t *testing.T) {
584584
backup := altinityCleanupBackup("ch-bk")
585585
backup.Annotations = map[string]string{"backups.cozystack.io/skip-artifact-cleanup": "true"}
586-
running := &batchv1.Job{ObjectMeta: metav1.ObjectMeta{Name: "ch-bk-cleanup", Namespace: "tenant-test"}}
587-
c, s := newAltinityCleanupClient(append(altinityCleanupFixtures(backup), running)...)
586+
c, s := newAltinityCleanupClient(append(altinityCleanupFixtures(backup), ownCleanupJob(backup))...)
588587
r := &BackupReconciler{Client: c, Scheme: s, Recorder: record.NewFakeRecorder(10)}
589588
if _, err := r.cleanupOnDelete(context.Background(), backup); err != nil {
590589
t.Fatalf("cleanupOnDelete: %v", err)
@@ -594,6 +593,39 @@ func TestCleanupAltinityBackup_SkipAnnotationReapsInflightJob(t *testing.T) {
594593
}
595594
}
596595

596+
// A Job of the same name this controller did not create (the ClickHouse
597+
// chart's own cleanup hook is <release>-cleanup) is never deleted or read as
598+
// the outcome of the archive delete.
599+
func TestCleanupAltinityBackup_LeavesForeignJobOfTheSameName(t *testing.T) {
600+
for name, skip := range map[string]bool{"cleanup": false, "skip annotation": true} {
601+
t.Run(name, func(t *testing.T) {
602+
backup := altinityCleanupBackup("ch-bk")
603+
if skip {
604+
backup.Annotations = map[string]string{"backups.cozystack.io/skip-artifact-cleanup": "true"}
605+
}
606+
foreign := ownCleanupJob(backup)
607+
foreign.Labels = map[string]string{"app.kubernetes.io/name": "clickhouse"}
608+
foreign.Status.Conditions = []batchv1.JobCondition{{Type: batchv1.JobComplete, Status: corev1.ConditionTrue}}
609+
c, s := newAltinityCleanupClient(append(altinityCleanupFixtures(backup), foreign)...)
610+
rec := record.NewFakeRecorder(10)
611+
r := &BackupReconciler{Client: c, Scheme: s, Recorder: rec}
612+
res, err := r.cleanupOnDelete(context.Background(), backup)
613+
if err != nil {
614+
t.Fatalf("cleanupOnDelete: %v", err)
615+
}
616+
if left := listJobs(t, c); len(left) != 1 {
617+
t.Fatalf("the foreign Job must be left alone, %d jobs", len(left))
618+
}
619+
if res.RequeueAfter != 0 || res.Requeue {
620+
t.Errorf("expected the Backup released, got %+v", res)
621+
}
622+
if !skip && len(rec.Events) == 0 {
623+
t.Error("a release because of a foreign Job must surface as a Warning Event")
624+
}
625+
})
626+
}
627+
}
628+
597629
func dropObject(objs []client.Object, like client.Object) []client.Object {
598630
out := objs[:0:0]
599631
for _, o := range objs {

‎internal/backupcontroller/altinitystrategy_controller.go‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -786,6 +786,9 @@ func (r *BackupReconciler) cleanupAltinityBackup(ctx context.Context, backup *ba
786786
err := r.Get(ctx, jobKey, existing)
787787
switch {
788788
case err == nil:
789+
if !isAltinityCleanupJob(existing) {
790+
break
791+
}
789792
if derr := r.deleteAltinityCleanupJob(ctx, existing); derr != nil {
790793
return ctrl.Result{}, derr
791794
}
@@ -862,6 +865,10 @@ func (r *BackupReconciler) cleanupAltinityBackup(ctx context.Context, backup *ba
862865
return ctrl.Result{}, err
863866
}
864867

868+
if !isAltinityCleanupJob(job) {
869+
return r.releaseAltinityCleanup(ctx, backup, archive,
870+
fmt.Sprintf("Job %s, which this controller did not create, holds the cleanup Job name", job.Name)), nil
871+
}
865872
if !job.DeletionTimestamp.IsZero() {
866873
return ctrl.Result{RequeueAfter: altinityPollInterval}, nil
867874
}
@@ -924,6 +931,13 @@ func (r *BackupReconciler) altinityAppOnSystemBucket(ctx context.Context, backup
924931
return v, true, nil
925932
}
926933

934+
// isAltinityCleanupJob tells a cleanup Job this controller created from
935+
// another Job of the same name, such as the ClickHouse chart's own
936+
// <release>-cleanup hook.
937+
func isAltinityCleanupJob(job *batchv1.Job) bool {
938+
return job.Labels[altinityLabelMode] == altinityModeCleanup
939+
}
940+
927941
func altinityCleanupJobArchive(job *batchv1.Job) string {
928942
for _, c := range job.Spec.Template.Spec.Containers {
929943
for _, e := range c.Env {

0 commit comments

Comments
 (0)