Skip to content

Commit 69cea9c

Browse files
scooby87scooby87
committed
fix(mariadb): keep the operator's message in deadline failures
The operator sets Complete=False with reason Failed and its own cause in the message ("Error creating Job", "Backup not complete") when a run fails before or outside its Job. Such a run waits out the deadline, and the deadline failure kept only the status and reason, so the BackupJob or RestoreJob ended without the one line that says why. Assisted-by: LLM Signed-off-by: Alexey Artamonov <[email protected]>
1 parent 9f51b9d commit 69cea9c

2 files changed

Lines changed: 33 additions & 10 deletions

File tree

‎internal/backupcontroller/mariadbstrategy_controller.go‎

Lines changed: 16 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -242,12 +242,8 @@ func (r *BackupJobReconciler) reconcileMariaDB(ctx context.Context, j *backupsv1
242242
// Backup eventually fails the BackupJob instead of pinning it Running
243243
// forever.
244244
if mariadbBackupDeadlineExceeded(j.Status.StartedAt) {
245-
detail := "no Complete condition observed"
246-
if cond != nil {
247-
detail = fmt.Sprintf("Complete=%s/%s", cond.Status, cond.Reason)
248-
}
249245
return r.markBackupJobFailed(ctx, j, fmt.Sprintf(
250-
"k8s.mariadb.com Backup did not complete within %s (%s)", mariadbDefaultBackupDeadline, detail))
246+
"k8s.mariadb.com Backup did not complete within %s (%s)", mariadbDefaultBackupDeadline, mariadbCompleteDetail(cond)))
251247
}
252248
return ctrl.Result{RequeueAfter: mariadbPollInterval}, nil
253249
}
@@ -273,6 +269,20 @@ func mariadbJobFailedMessage(ctx context.Context, c client.Reader, kind, namespa
273269
return fmt.Sprintf("%s; Job %s/%s reports no Failed condition", message, namespace, name)
274270
}
275271

272+
// mariadbCompleteDetail describes the last Complete condition for a deadline
273+
// failure. The message matters for reason Failed, which the operator sets
274+
// with its own cause ("Error creating Job", "Backup not complete") rather
275+
// than from a Job.
276+
func mariadbCompleteDetail(cond *metav1.Condition) string {
277+
if cond == nil {
278+
return "no Complete condition observed"
279+
}
280+
if cond.Message == "" {
281+
return fmt.Sprintf("Complete=%s/%s", cond.Status, cond.Reason)
282+
}
283+
return fmt.Sprintf("Complete=%s/%s: %s", cond.Status, cond.Reason, cond.Message)
284+
}
285+
276286
// mariadbBackupDeadlineExceeded reports whether enough wall-clock time has
277287
// elapsed since the BackupJob started that we should give up on a stuck
278288
// operator Backup. Returns false when StartedAt is nil so the very first
@@ -605,13 +615,9 @@ func (r *RestoreJobReconciler) reconcileMariaDBRestore(ctx context.Context, rest
605615
default:
606616
deadline := options.effectiveRestoreDeadline()
607617
if restoreJob.Status.StartedAt != nil && time.Since(restoreJob.Status.StartedAt.Time) > deadline {
608-
detail := "no Complete condition observed"
609-
if cond != nil {
610-
detail = fmt.Sprintf("Complete=%s/%s", cond.Status, cond.Reason)
611-
}
612618
return r.markRestoreJobFailed(ctx, restoreJob, fmt.Sprintf(
613619
"k8s.mariadb.com/Restore did not complete within %s (%s; override via spec.options.restoreTimeoutSeconds)",
614-
deadline, detail))
620+
deadline, mariadbCompleteDetail(cond)))
615621
}
616622
return ctrl.Result{RequeueAfter: mariadbPollInterval}, nil
617623
}

‎internal/backupcontroller/mariadbstrategy_controller_test.go‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -901,6 +901,8 @@ var mariadbCompleteOutcomeCases = []struct {
901901
wantMessage: []string{"no Complete condition observed"}},
902902
{name: "Complete=True with an unknown reason past the deadline fails", status: metav1.ConditionTrue, reason: "SomethingElse", message: "Something", expired: true, wantFailed: true,
903903
wantMessage: []string{"Complete=True/SomethingElse"}},
904+
{name: "Complete=False/Failed past the deadline carries the operator's message", status: metav1.ConditionFalse, reason: "Failed", message: "Error creating Job", expired: true, wantFailed: true,
905+
wantMessage: []string{"Complete=False/Failed: Error creating Job"}},
904906
}
905907

906908
func mariadbCompleteConditions(noCondition bool, status metav1.ConditionStatus, reason, message string) []metav1.Condition {
@@ -1078,6 +1080,21 @@ func TestMariaDBJobFailedMessage_JobWithoutFailedCondition(t *testing.T) {
10781080
}
10791081
}
10801082

1083+
func TestMariaDBCompleteDetail(t *testing.T) {
1084+
for _, tc := range []struct {
1085+
cond *metav1.Condition
1086+
want string
1087+
}{
1088+
{nil, "no Complete condition observed"},
1089+
{&metav1.Condition{Status: metav1.ConditionFalse, Reason: "Failed", Message: "Backup not complete"}, "Complete=False/Failed: Backup not complete"},
1090+
{&metav1.Condition{Status: metav1.ConditionFalse, Reason: "Failed"}, "Complete=False/Failed"},
1091+
} {
1092+
if got := mariadbCompleteDetail(tc.cond); got != tc.want {
1093+
t.Errorf("mariadbCompleteDetail(%+v) = %q, want %q", tc.cond, got, tc.want)
1094+
}
1095+
}
1096+
}
1097+
10811098
func TestReconcileMariaDBRestore_CompleteConditionOutcome(t *testing.T) {
10821099
apps := mariadbapp.GroupName
10831100

0 commit comments

Comments
 (0)