Skip to content

Commit 9f51b9d

Browse files
committed
fix(mariadb): report the operator Job's own failure on JobFailed
The operator's JobFailed condition message is only "Failed", and with the operator's default restartPolicy OnFailure the Job controller deletes the pod once the backoff limit is spent, before the operator sets JobFailed. Pointing the reader at the Job's logs therefore sent them to logs that no longer exist. The Job, which the operator names after the Backup or Restore, keeps its Failed condition with the reason and message, so the BackupJob or RestoreJob now carries those instead; a Job that cannot be read is named in the message rather than dropped. Assisted-by: LLM Signed-off-by: Alexey Artamonov <[email protected]>
1 parent 7e1fc71 commit 9f51b9d

2 files changed

Lines changed: 94 additions & 16 deletions

File tree

‎internal/backupcontroller/mariadbstrategy_controller.go‎

Lines changed: 21 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import (
77
"fmt"
88
"time"
99

10+
batchv1 "k8s.io/api/batch/v1"
1011
corev1 "k8s.io/api/core/v1"
1112
apierrors "k8s.io/apimachinery/pkg/api/errors"
1213
apimeta "k8s.io/apimachinery/pkg/api/meta"
@@ -205,7 +206,7 @@ func (r *BackupJobReconciler) reconcileMariaDB(ctx context.Context, j *backupsv1
205206

206207
switch {
207208
case cond != nil && cond.Reason == mariadbtypes.ConditionReasonJobFailed:
208-
return r.markBackupJobFailed(ctx, j, mariadbJobFailedMessage("Backup", mdbBackup.Namespace, mdbBackup.Name, cond))
209+
return r.markBackupJobFailed(ctx, j, mariadbJobFailedMessage(ctx, r.Client, "Backup", mdbBackup.Namespace, mdbBackup.Name, cond))
209210

210211
case cond != nil && cond.Status == metav1.ConditionTrue && cond.Reason == mariadbtypes.ConditionReasonJobComplete:
211212
// Backup completed successfully. Materialise the Cozystack artifact
@@ -252,12 +253,24 @@ func (r *BackupJobReconciler) reconcileMariaDB(ctx context.Context, j *backupsv1
252253
}
253254
}
254255

255-
// mariadbJobFailedMessage names the operator object whose Job failed. The
256-
// operator's own condition message is only "Failed"; the cause is in the
257-
// logs of that object's Job.
258-
func mariadbJobFailedMessage(kind, namespace, name string, cond *metav1.Condition) string {
259-
return fmt.Sprintf("k8s.mariadb.com/%s %s/%s failed (Complete=%s/%s); the cause is in the logs of its Job",
260-
kind, namespace, name, cond.Status, cond.Reason)
256+
// mariadbJobFailedMessage names the operator object whose Job failed and
257+
// carries that Job's own Failed condition, because the operator's condition
258+
// message is only "Failed". The operator names the Job after the object.
259+
func mariadbJobFailedMessage(ctx context.Context, c client.Reader, kind, namespace, name string, cond *metav1.Condition) string {
260+
message := fmt.Sprintf("k8s.mariadb.com/%s %s/%s failed (Complete=%s/%s)", kind, namespace, name, cond.Status, cond.Reason)
261+
job := &batchv1.Job{}
262+
if err := c.Get(ctx, types.NamespacedName{Namespace: namespace, Name: name}, job); err != nil {
263+
if apierrors.IsNotFound(err) {
264+
return fmt.Sprintf("%s; Job %s/%s not found", message, namespace, name)
265+
}
266+
return fmt.Sprintf("%s; reading Job %s/%s: %v", message, namespace, name, err)
267+
}
268+
for _, jc := range job.Status.Conditions {
269+
if jc.Type == batchv1.JobFailed && jc.Status == corev1.ConditionTrue {
270+
return fmt.Sprintf("%s; Job %s/%s: %s: %s", message, namespace, name, jc.Reason, jc.Message)
271+
}
272+
}
273+
return fmt.Sprintf("%s; Job %s/%s reports no Failed condition", message, namespace, name)
261274
}
262275

263276
// mariadbBackupDeadlineExceeded reports whether enough wall-clock time has
@@ -572,7 +585,7 @@ func (r *RestoreJobReconciler) reconcileMariaDBRestore(ctx context.Context, rest
572585
cond := apimeta.FindStatusCondition(mdbRestore.Status.Conditions, mariadbtypes.ConditionTypeComplete)
573586
switch {
574587
case cond != nil && cond.Reason == mariadbtypes.ConditionReasonJobFailed:
575-
return r.markRestoreJobFailed(ctx, restoreJob, mariadbJobFailedMessage("Restore", mdbRestore.Namespace, mdbRestore.Name, cond))
588+
return r.markRestoreJobFailed(ctx, restoreJob, mariadbJobFailedMessage(ctx, r.Client, "Restore", mdbRestore.Namespace, mdbRestore.Name, cond))
576589

577590
case cond != nil && cond.Status == metav1.ConditionTrue && cond.Reason == mariadbtypes.ConditionReasonJobComplete:
578591
now := metav1.Now()

‎internal/backupcontroller/mariadbstrategy_controller_test.go‎

Lines changed: 73 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -4,19 +4,23 @@ package backupcontroller
44
import (
55
"context"
66
"encoding/json"
7+
"errors"
78
"strings"
89
"testing"
910
"time"
1011

12+
batchv1 "k8s.io/api/batch/v1"
1113
corev1 "k8s.io/api/core/v1"
1214
apiequality "k8s.io/apimachinery/pkg/api/equality"
15+
apierrors "k8s.io/apimachinery/pkg/api/errors"
1316
apimeta "k8s.io/apimachinery/pkg/api/meta"
1417
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
1518
"k8s.io/apimachinery/pkg/runtime"
1619
"k8s.io/client-go/kubernetes/scheme"
1720
"k8s.io/client-go/tools/record"
1821
"sigs.k8s.io/controller-runtime/pkg/client"
1922
clientfake "sigs.k8s.io/controller-runtime/pkg/client/fake"
23+
"sigs.k8s.io/controller-runtime/pkg/client/interceptor"
2024

2125
strategyv1alpha1 "github.com/cozystack/cozystack/api/backups/strategy/v1alpha1"
2226
backupsv1alpha1 "github.com/cozystack/cozystack/api/backups/v1alpha1"
@@ -877,15 +881,18 @@ var mariadbCompleteOutcomeCases = []struct {
877881
reason string
878882
message string
879883
expired bool
884+
failedJob bool
880885
wantFailed bool
881886
wantDone bool
882887
wantMessage []string
883888
}{
884889
{name: "Complete=True/JobComplete succeeds", status: metav1.ConditionTrue, reason: mariadbtypes.ConditionReasonJobComplete, message: "Success", wantDone: true},
885-
{name: "Complete=True/JobFailed fails", status: metav1.ConditionTrue, reason: mariadbtypes.ConditionReasonJobFailed, message: "Failed", wantFailed: true,
886-
wantMessage: []string{"tenant/op"}},
887-
{name: "Complete=False/JobFailed fails", status: metav1.ConditionFalse, reason: mariadbtypes.ConditionReasonJobFailed, message: "Failed", wantFailed: true,
888-
wantMessage: []string{"tenant/op"}},
890+
{name: "Complete=True/JobFailed fails with the Job's failure", status: metav1.ConditionTrue, reason: mariadbtypes.ConditionReasonJobFailed, message: "Failed", failedJob: true, wantFailed: true,
891+
wantMessage: []string{"tenant/op", "BackoffLimitExceeded", "Job has reached the specified backoff limit"}},
892+
{name: "Complete=False/JobFailed fails with the Job's failure", status: metav1.ConditionFalse, reason: mariadbtypes.ConditionReasonJobFailed, message: "Failed", failedJob: true, wantFailed: true,
893+
wantMessage: []string{"tenant/op", "BackoffLimitExceeded", "Job has reached the specified backoff limit"}},
894+
{name: "Complete=True/JobFailed with its Job gone fails", status: metav1.ConditionTrue, reason: mariadbtypes.ConditionReasonJobFailed, message: "Failed", wantFailed: true,
895+
wantMessage: []string{"tenant/op", "Job tenant/op not found"}},
889896
{name: "Complete=True with an unknown reason keeps waiting", status: metav1.ConditionTrue, reason: "SomethingElse", message: "Something"},
890897
{name: "Complete=False/JobComplete keeps waiting", status: metav1.ConditionFalse, reason: mariadbtypes.ConditionReasonJobComplete, message: "Success"},
891898
{name: "Complete=False/JobRunning keeps waiting", status: metav1.ConditionFalse, reason: mariadbtypes.ConditionReasonJobRunning, message: "Running"},
@@ -920,8 +927,29 @@ func mariadbOutcomeStartedAt(expired bool) *metav1.Time {
920927
return &past
921928
}
922929

930+
// mariadbOutcomeObjects adds the operator's Job, named after the operator
931+
// Backup or Restore, failed the way the Job controller fails it once the
932+
// backoff limit is spent.
933+
func mariadbOutcomeObjects(failedJob bool, objs ...client.Object) []client.Object {
934+
if !failedJob {
935+
return objs
936+
}
937+
return append(objs, &batchv1.Job{
938+
ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "op"},
939+
Status: batchv1.JobStatus{Conditions: []batchv1.JobCondition{{
940+
Type: batchv1.JobFailed,
941+
Status: corev1.ConditionTrue,
942+
Reason: batchv1.JobReasonBackoffLimitExceeded,
943+
Message: "Job has reached the specified backoff limit",
944+
}}},
945+
})
946+
}
947+
923948
func assertMariaDBFailureMessage(t *testing.T, message string, want []string) {
924949
t.Helper()
950+
if strings.Contains(message, "logs") {
951+
t.Errorf("failure message must not send the reader to Job logs, which are gone with the deleted pod: %q", message)
952+
}
925953
for _, w := range want {
926954
if !strings.Contains(message, w) {
927955
t.Errorf("failure message must contain %q, got %q", w, message)
@@ -961,8 +989,8 @@ func TestReconcileMariaDBBackup_CompleteConditionOutcome(t *testing.T) {
961989
},
962990
Status: mariadbtypes.BackupStatus{Conditions: mariadbCompleteConditions(tc.noCondition, tc.status, tc.reason, tc.message)},
963991
}
964-
c := newMariaDBStrategyTestClient(t, seed, newMariaDBApp(seed.Spec.ApplicationRef.Name, "tenant"),
965-
strategy, operatorMariaDB, operatorBackup)
992+
c := newMariaDBStrategyTestClient(t, mariadbOutcomeObjects(tc.failedJob, seed, newMariaDBApp(seed.Spec.ApplicationRef.Name, "tenant"),
993+
strategy, operatorMariaDB, operatorBackup)...)
966994
r := &BackupJobReconciler{Client: c, Scheme: c.Scheme(), Recorder: record.NewFakeRecorder(10)}
967995

968996
bj := &backupsv1alpha1.BackupJob{}
@@ -1013,6 +1041,43 @@ func TestReconcileMariaDBBackup_CompleteConditionOutcome(t *testing.T) {
10131041
}
10141042
}
10151043

1044+
func TestMariaDBJobFailedMessage_JobUnreadable(t *testing.T) {
1045+
cond := &metav1.Condition{Type: mariadbtypes.ConditionTypeComplete, Status: metav1.ConditionTrue, Reason: mariadbtypes.ConditionReasonJobFailed}
1046+
readErr := apierrors.NewForbidden(batchv1.Resource("jobs"), "op", errors.New("rbac"))
1047+
c := interceptor.NewClient(newMariaDBStrategyTestClient(t).(client.WithWatch), interceptor.Funcs{
1048+
Get: func(ctx context.Context, cl client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error {
1049+
if _, ok := obj.(*batchv1.Job); ok {
1050+
return readErr
1051+
}
1052+
return cl.Get(ctx, key, obj, opts...)
1053+
},
1054+
})
1055+
1056+
got := mariadbJobFailedMessage(context.Background(), c, "Backup", "tenant", "op", cond)
1057+
for _, want := range []string{"k8s.mariadb.com/Backup tenant/op failed (Complete=True/JobFailed)", "reading Job tenant/op", readErr.Error()} {
1058+
if !strings.Contains(got, want) {
1059+
t.Errorf("message must contain %q, got %q", want, got)
1060+
}
1061+
}
1062+
}
1063+
1064+
func TestMariaDBJobFailedMessage_JobWithoutFailedCondition(t *testing.T) {
1065+
cond := &metav1.Condition{Type: mariadbtypes.ConditionTypeComplete, Status: metav1.ConditionTrue, Reason: mariadbtypes.ConditionReasonJobFailed}
1066+
c := newMariaDBStrategyTestClient(t, &batchv1.Job{
1067+
ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "op"},
1068+
Status: batchv1.JobStatus{Conditions: []batchv1.JobCondition{
1069+
{Type: batchv1.JobFailureTarget, Status: corev1.ConditionTrue, Reason: batchv1.JobReasonPodFailurePolicy},
1070+
{Type: batchv1.JobFailed, Status: corev1.ConditionFalse, Reason: batchv1.JobReasonBackoffLimitExceeded},
1071+
}},
1072+
})
1073+
1074+
got := mariadbJobFailedMessage(context.Background(), c, "Restore", "tenant", "op", cond)
1075+
if !strings.Contains(got, "Job tenant/op reports no Failed condition") ||
1076+
strings.Contains(got, batchv1.JobReasonBackoffLimitExceeded) || strings.Contains(got, batchv1.JobReasonPodFailurePolicy) {
1077+
t.Errorf("a Failed condition that is not True must not be reported as the cause, got %q", got)
1078+
}
1079+
}
1080+
10161081
func TestReconcileMariaDBRestore_CompleteConditionOutcome(t *testing.T) {
10171082
apps := mariadbapp.GroupName
10181083

@@ -1044,9 +1109,9 @@ func TestReconcileMariaDBRestore_CompleteConditionOutcome(t *testing.T) {
10441109
},
10451110
Status: mariadbtypes.RestoreStatus{Conditions: mariadbCompleteConditions(tc.noCondition, tc.status, tc.reason, tc.message)},
10461111
}
1047-
c := newMariaDBStrategyTestClient(t, cozyBackup, seed, operatorRestore,
1112+
c := newMariaDBStrategyTestClient(t, mariadbOutcomeObjects(tc.failedJob, cozyBackup, seed, operatorRestore,
10481113
&mariadbtypes.Backup{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "src-op-bk"}},
1049-
&mariadbtypes.MariaDB{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: mariadbNameForApp("target")}})
1114+
&mariadbtypes.MariaDB{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: mariadbNameForApp("target")}})...)
10501115
r := &RestoreJobReconciler{Client: c, Recorder: record.NewFakeRecorder(10)}
10511116

10521117
rj := &backupsv1alpha1.RestoreJob{}

0 commit comments

Comments
 (0)