@@ -4,19 +4,23 @@ package backupcontroller
44import (
55 "context"
66 "encoding/json"
7+ "errors"
78 "strings"
89 "testing"
910 "time"
1011
12+ batchv1 "k8s.io/api/batch/v1"
1113 corev1 "k8s.io/api/core/v1"
1214 apiequality "k8s.io/apimachinery/pkg/api/equality"
15+ apierrors "k8s.io/apimachinery/pkg/api/errors"
1316 apimeta "k8s.io/apimachinery/pkg/api/meta"
1417 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
1518 "k8s.io/apimachinery/pkg/runtime"
1619 "k8s.io/client-go/kubernetes/scheme"
1720 "k8s.io/client-go/tools/record"
1821 "sigs.k8s.io/controller-runtime/pkg/client"
1922 clientfake "sigs.k8s.io/controller-runtime/pkg/client/fake"
23+ "sigs.k8s.io/controller-runtime/pkg/client/interceptor"
2024
2125 strategyv1alpha1 "github.com/cozystack/cozystack/api/backups/strategy/v1alpha1"
2226 backupsv1alpha1 "github.com/cozystack/cozystack/api/backups/v1alpha1"
@@ -877,15 +881,18 @@ var mariadbCompleteOutcomeCases = []struct {
877881 reason string
878882 message string
879883 expired bool
884+ failedJob bool
880885 wantFailed bool
881886 wantDone bool
882887 wantMessage []string
883888}{
884889 {name : "Complete=True/JobComplete succeeds" , status : metav1 .ConditionTrue , reason : mariadbtypes .ConditionReasonJobComplete , message : "Success" , wantDone : true },
885- {name : "Complete=True/JobFailed fails" , status : metav1 .ConditionTrue , reason : mariadbtypes .ConditionReasonJobFailed , message : "Failed" , wantFailed : true ,
886- wantMessage : []string {"tenant/op" }},
887- {name : "Complete=False/JobFailed fails" , status : metav1 .ConditionFalse , reason : mariadbtypes .ConditionReasonJobFailed , message : "Failed" , wantFailed : true ,
888- wantMessage : []string {"tenant/op" }},
890+ {name : "Complete=True/JobFailed fails with the Job's failure" , status : metav1 .ConditionTrue , reason : mariadbtypes .ConditionReasonJobFailed , message : "Failed" , failedJob : true , wantFailed : true ,
891+ wantMessage : []string {"tenant/op" , "BackoffLimitExceeded" , "Job has reached the specified backoff limit" }},
892+ {name : "Complete=False/JobFailed fails with the Job's failure" , status : metav1 .ConditionFalse , reason : mariadbtypes .ConditionReasonJobFailed , message : "Failed" , failedJob : true , wantFailed : true ,
893+ wantMessage : []string {"tenant/op" , "BackoffLimitExceeded" , "Job has reached the specified backoff limit" }},
894+ {name : "Complete=True/JobFailed with its Job gone fails" , status : metav1 .ConditionTrue , reason : mariadbtypes .ConditionReasonJobFailed , message : "Failed" , wantFailed : true ,
895+ wantMessage : []string {"tenant/op" , "Job tenant/op not found" }},
889896 {name : "Complete=True with an unknown reason keeps waiting" , status : metav1 .ConditionTrue , reason : "SomethingElse" , message : "Something" },
890897 {name : "Complete=False/JobComplete keeps waiting" , status : metav1 .ConditionFalse , reason : mariadbtypes .ConditionReasonJobComplete , message : "Success" },
891898 {name : "Complete=False/JobRunning keeps waiting" , status : metav1 .ConditionFalse , reason : mariadbtypes .ConditionReasonJobRunning , message : "Running" },
@@ -920,8 +927,29 @@ func mariadbOutcomeStartedAt(expired bool) *metav1.Time {
920927 return & past
921928}
922929
930+ // mariadbOutcomeObjects adds the operator's Job, named after the operator
931+ // Backup or Restore, failed the way the Job controller fails it once the
932+ // backoff limit is spent.
933+ func mariadbOutcomeObjects (failedJob bool , objs ... client.Object ) []client.Object {
934+ if ! failedJob {
935+ return objs
936+ }
937+ return append (objs , & batchv1.Job {
938+ ObjectMeta : metav1.ObjectMeta {Namespace : "tenant" , Name : "op" },
939+ Status : batchv1.JobStatus {Conditions : []batchv1.JobCondition {{
940+ Type : batchv1 .JobFailed ,
941+ Status : corev1 .ConditionTrue ,
942+ Reason : batchv1 .JobReasonBackoffLimitExceeded ,
943+ Message : "Job has reached the specified backoff limit" ,
944+ }}},
945+ })
946+ }
947+
923948func assertMariaDBFailureMessage (t * testing.T , message string , want []string ) {
924949 t .Helper ()
950+ if strings .Contains (message , "logs" ) {
951+ t .Errorf ("failure message must not send the reader to Job logs, which are gone with the deleted pod: %q" , message )
952+ }
925953 for _ , w := range want {
926954 if ! strings .Contains (message , w ) {
927955 t .Errorf ("failure message must contain %q, got %q" , w , message )
@@ -961,8 +989,8 @@ func TestReconcileMariaDBBackup_CompleteConditionOutcome(t *testing.T) {
961989 },
962990 Status : mariadbtypes.BackupStatus {Conditions : mariadbCompleteConditions (tc .noCondition , tc .status , tc .reason , tc .message )},
963991 }
964- c := newMariaDBStrategyTestClient (t , seed , newMariaDBApp (seed .Spec .ApplicationRef .Name , "tenant" ),
965- strategy , operatorMariaDB , operatorBackup )
992+ c := newMariaDBStrategyTestClient (t , mariadbOutcomeObjects ( tc . failedJob , seed , newMariaDBApp (seed .Spec .ApplicationRef .Name , "tenant" ),
993+ strategy , operatorMariaDB , operatorBackup )... )
966994 r := & BackupJobReconciler {Client : c , Scheme : c .Scheme (), Recorder : record .NewFakeRecorder (10 )}
967995
968996 bj := & backupsv1alpha1.BackupJob {}
@@ -1013,6 +1041,43 @@ func TestReconcileMariaDBBackup_CompleteConditionOutcome(t *testing.T) {
10131041 }
10141042}
10151043
1044+ func TestMariaDBJobFailedMessage_JobUnreadable (t * testing.T ) {
1045+ cond := & metav1.Condition {Type : mariadbtypes .ConditionTypeComplete , Status : metav1 .ConditionTrue , Reason : mariadbtypes .ConditionReasonJobFailed }
1046+ readErr := apierrors .NewForbidden (batchv1 .Resource ("jobs" ), "op" , errors .New ("rbac" ))
1047+ c := interceptor .NewClient (newMariaDBStrategyTestClient (t ).(client.WithWatch ), interceptor.Funcs {
1048+ Get : func (ctx context.Context , cl client.WithWatch , key client.ObjectKey , obj client.Object , opts ... client.GetOption ) error {
1049+ if _ , ok := obj .(* batchv1.Job ); ok {
1050+ return readErr
1051+ }
1052+ return cl .Get (ctx , key , obj , opts ... )
1053+ },
1054+ })
1055+
1056+ got := mariadbJobFailedMessage (context .Background (), c , "Backup" , "tenant" , "op" , cond )
1057+ for _ , want := range []string {"k8s.mariadb.com/Backup tenant/op failed (Complete=True/JobFailed)" , "reading Job tenant/op" , readErr .Error ()} {
1058+ if ! strings .Contains (got , want ) {
1059+ t .Errorf ("message must contain %q, got %q" , want , got )
1060+ }
1061+ }
1062+ }
1063+
1064+ func TestMariaDBJobFailedMessage_JobWithoutFailedCondition (t * testing.T ) {
1065+ cond := & metav1.Condition {Type : mariadbtypes .ConditionTypeComplete , Status : metav1 .ConditionTrue , Reason : mariadbtypes .ConditionReasonJobFailed }
1066+ c := newMariaDBStrategyTestClient (t , & batchv1.Job {
1067+ ObjectMeta : metav1.ObjectMeta {Namespace : "tenant" , Name : "op" },
1068+ Status : batchv1.JobStatus {Conditions : []batchv1.JobCondition {
1069+ {Type : batchv1 .JobFailureTarget , Status : corev1 .ConditionTrue , Reason : batchv1 .JobReasonPodFailurePolicy },
1070+ {Type : batchv1 .JobFailed , Status : corev1 .ConditionFalse , Reason : batchv1 .JobReasonBackoffLimitExceeded },
1071+ }},
1072+ })
1073+
1074+ got := mariadbJobFailedMessage (context .Background (), c , "Restore" , "tenant" , "op" , cond )
1075+ if ! strings .Contains (got , "Job tenant/op reports no Failed condition" ) ||
1076+ strings .Contains (got , batchv1 .JobReasonBackoffLimitExceeded ) || strings .Contains (got , batchv1 .JobReasonPodFailurePolicy ) {
1077+ t .Errorf ("a Failed condition that is not True must not be reported as the cause, got %q" , got )
1078+ }
1079+ }
1080+
10161081func TestReconcileMariaDBRestore_CompleteConditionOutcome (t * testing.T ) {
10171082 apps := mariadbapp .GroupName
10181083
@@ -1044,9 +1109,9 @@ func TestReconcileMariaDBRestore_CompleteConditionOutcome(t *testing.T) {
10441109 },
10451110 Status : mariadbtypes.RestoreStatus {Conditions : mariadbCompleteConditions (tc .noCondition , tc .status , tc .reason , tc .message )},
10461111 }
1047- c := newMariaDBStrategyTestClient (t , cozyBackup , seed , operatorRestore ,
1112+ c := newMariaDBStrategyTestClient (t , mariadbOutcomeObjects ( tc . failedJob , cozyBackup , seed , operatorRestore ,
10481113 & mariadbtypes.Backup {ObjectMeta : metav1.ObjectMeta {Namespace : "tenant" , Name : "src-op-bk" }},
1049- & mariadbtypes.MariaDB {ObjectMeta : metav1.ObjectMeta {Namespace : "tenant" , Name : mariadbNameForApp ("target" )}})
1114+ & mariadbtypes.MariaDB {ObjectMeta : metav1.ObjectMeta {Namespace : "tenant" , Name : mariadbNameForApp ("target" )}})... )
10501115 r := & RestoreJobReconciler {Client : c , Recorder : record .NewFakeRecorder (10 )}
10511116
10521117 rj := & backupsv1alpha1.RestoreJob {}
0 commit comments