Skip to content
This repository was archived by the owner on Dec 21, 2023. It is now read-only.

Commit 49ca272

Browse files
bacherflbacherfl
authored
fix(lighthouse-service): Added error logs for failing monitoring configuration (#5088) (#5220)
Signed-off-by: Florian Bacher <[email protected]>
1 parent a5aecd4 commit 49ca272

File tree

7 files changed

+237
-67
lines changed

7 files changed

+237
-67
lines changed

‎lighthouse-service/event_handler/common.go‎

Lines changed: 39 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -3,14 +3,16 @@ package event_handler
33
import (
44
"context"
55
"errors"
6+
cloudevents "github.com/cloudevents/sdk-go/v2"
67
keptnapimodels "github.com/keptn/go-utils/pkg/api/models"
8+
"gopkg.in/yaml.v3"
9+
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
10+
"k8s.io/client-go/kubernetes"
11+
"k8s.io/client-go/rest"
712
"net/url"
813
"os"
914
"strings"
10-
11-
cloudevents "github.com/cloudevents/sdk-go/v2"
12-
"gopkg.in/yaml.v3"
13-
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
15+
"sync"
1416

1517
utils "github.com/keptn/go-utils/pkg/api/utils"
1618
keptn "github.com/keptn/go-utils/pkg/lib"
@@ -179,7 +181,7 @@ type K8sSLIProviderConfig struct{}
179181

180182
// GetDefaultSLIProvider godoc
181183
func (K8sSLIProviderConfig) GetDefaultSLIProvider() (string, error) {
182-
kubeAPI, err := getKubeAPI()
184+
kubeAPI, err := GetConfig().GetKubeAPI()
183185
if err != nil {
184186
return "", err
185187
}
@@ -197,7 +199,7 @@ func (K8sSLIProviderConfig) GetDefaultSLIProvider() (string, error) {
197199

198200
// GetSLIProvider godoc
199201
func (K8sSLIProviderConfig) GetSLIProvider(project string) (string, error) {
200-
kubeAPI, err := getKubeAPI()
202+
kubeAPI, err := GetConfig().GetKubeAPI()
201203
if err != nil {
202204
return "", err
203205
}
@@ -212,3 +214,34 @@ func (K8sSLIProviderConfig) GetSLIProvider(project string) (string, error) {
212214

213215
return sliProvider, nil
214216
}
217+
218+
type Config struct {
219+
GetKubeAPI KubeAPIConfigFunc
220+
}
221+
222+
var config *Config
223+
var configOnce sync.Once
224+
225+
func GetConfig() *Config {
226+
configOnce.Do(func() {
227+
config = &Config{GetKubeAPI: getInClusterKubeClient}
228+
})
229+
return config
230+
}
231+
232+
type KubeAPIConfigFunc func() (kubernetes.Interface, error)
233+
234+
func getInClusterKubeClient() (kubernetes.Interface, error) {
235+
var config *rest.Config
236+
config, err := rest.InClusterConfig()
237+
238+
if err != nil {
239+
return nil, err
240+
}
241+
242+
kubeAPI, err := kubernetes.NewForConfig(config)
243+
if err != nil {
244+
return nil, err
245+
}
246+
return kubeAPI, nil
247+
}

‎lighthouse-service/event_handler/configure_monitoring_handler.go‎

Lines changed: 45 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -2,25 +2,54 @@ package event_handler
22

33
import (
44
"context"
5+
"github.com/sirupsen/logrus"
6+
k8serrors "k8s.io/apimachinery/pkg/api/errors"
57
"os"
68

79
cloudevents "github.com/cloudevents/sdk-go/v2"
10+
keptnevents "github.com/keptn/go-utils/pkg/lib"
811
v1 "k8s.io/api/core/v1"
912
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
1013
"k8s.io/client-go/kubernetes"
11-
"k8s.io/client-go/rest"
12-
13-
keptnevents "github.com/keptn/go-utils/pkg/lib"
14-
keptnv2 "github.com/keptn/go-utils/pkg/lib/v0_2_0"
1514
)
1615

1716
type ConfigureMonitoringHandler struct {
18-
Event cloudevents.Event
19-
KeptnHandler *keptnv2.Keptn
17+
Event cloudevents.Event
18+
Logger *logrus.Logger
19+
K8sClient kubernetes.Interface
2020
}
2121

2222
var namespace = os.Getenv("POD_NAMESPACE")
2323

24+
type ConfigureMonitoringHandlerOption func(cmh *ConfigureMonitoringHandler)
25+
26+
func WithK8sClient(k8sClient kubernetes.Interface) ConfigureMonitoringHandlerOption {
27+
return func(cmh *ConfigureMonitoringHandler) {
28+
cmh.K8sClient = k8sClient
29+
}
30+
}
31+
32+
func NewConfigureMonitoringHandler(event cloudevents.Event, logger *logrus.Logger, opts ...ConfigureMonitoringHandlerOption) (*ConfigureMonitoringHandler, error) {
33+
cmh := &ConfigureMonitoringHandler{
34+
Event: event,
35+
Logger: logger,
36+
}
37+
38+
for _, opt := range opts {
39+
opt(cmh)
40+
}
41+
42+
if cmh.K8sClient == nil {
43+
defaultK8sClient, err := GetConfig().GetKubeAPI()
44+
if err != nil {
45+
return nil, err
46+
}
47+
cmh.K8sClient = defaultK8sClient
48+
}
49+
50+
return cmh, nil
51+
}
52+
2453
func (eh *ConfigureMonitoringHandler) HandleEvent() error {
2554

2655
var keptnContext string
@@ -29,48 +58,28 @@ func (eh *ConfigureMonitoringHandler) HandleEvent() error {
2958
e := &keptnevents.ConfigureMonitoringEventData{}
3059
err := eh.Event.DataAs(e)
3160
if err != nil {
32-
eh.KeptnHandler.Logger.Error("Could not parse event payload: " + err.Error())
61+
eh.Logger.Error("Could not parse event payload: " + err.Error())
3362
return err
3463
}
3564

36-
configMap := eh.getSLISourceConfigMap(e)
65+
configMap := getSLISourceConfigMap(e)
3766

38-
kubeAPI, err := getKubeAPI()
39-
if err != nil {
40-
return err
41-
}
67+
_, err = eh.K8sClient.CoreV1().ConfigMaps(namespace).Create(context.TODO(), configMap, metav1.CreateOptions{})
4268

43-
if err != nil {
44-
eh.KeptnHandler.Logger.Error("Could not create Kube API")
45-
return err
46-
}
47-
_, err = kubeAPI.CoreV1().ConfigMaps(namespace).Create(context.TODO(), configMap, metav1.CreateOptions{})
48-
49-
if err != nil {
50-
_, err = kubeAPI.CoreV1().ConfigMaps(namespace).Update(context.TODO(), configMap, metav1.UpdateOptions{})
69+
if err != nil && k8serrors.IsAlreadyExists(err) {
70+
_, err = eh.K8sClient.CoreV1().ConfigMaps(namespace).Update(context.TODO(), configMap, metav1.UpdateOptions{})
5171
if err != nil {
72+
eh.Logger.WithError(err).Error("could not update sli-provider ConfigMap")
5273
return err
5374
}
75+
} else if err != nil {
76+
eh.Logger.WithError(err).Error("could not create sli-provider ConfigMap")
77+
return err
5478
}
5579
return nil
5680
}
5781

58-
func getKubeAPI() (*kubernetes.Clientset, error) {
59-
var config *rest.Config
60-
config, err := rest.InClusterConfig()
61-
62-
if err != nil {
63-
return nil, err
64-
}
65-
66-
kubeAPI, err := kubernetes.NewForConfig(config)
67-
if err != nil {
68-
return nil, err
69-
}
70-
return kubeAPI, nil
71-
}
72-
73-
func (eh *ConfigureMonitoringHandler) getSLISourceConfigMap(e *keptnevents.ConfigureMonitoringEventData) *v1.ConfigMap {
82+
func getSLISourceConfigMap(e *keptnevents.ConfigureMonitoringEventData) *v1.ConfigMap {
7483
configMap := &v1.ConfigMap{
7584
ObjectMeta: metav1.ObjectMeta{
7685
Name: "lighthouse-config-" + e.Project,

‎lighthouse-service/event_handler/configure_monitoring_handler_test.go‎

Lines changed: 131 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,14 @@
11
package event_handler
22

33
import (
4+
"errors"
5+
keptnv2 "github.com/keptn/go-utils/pkg/lib/v0_2_0"
6+
"github.com/sirupsen/logrus"
7+
"github.com/sirupsen/logrus/hooks/test"
8+
"github.com/stretchr/testify/require"
9+
"k8s.io/apimachinery/pkg/runtime"
10+
"k8s.io/client-go/kubernetes/fake"
11+
k8stesting "k8s.io/client-go/testing"
412
"reflect"
513
"testing"
614

@@ -9,29 +17,20 @@ import (
917
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
1018

1119
keptn "github.com/keptn/go-utils/pkg/lib"
12-
keptncommon "github.com/keptn/go-utils/pkg/lib/keptn"
20+
keptnevents "github.com/keptn/go-utils/pkg/lib"
1321
)
1422

1523
func TestConfigureMonitoringHandler_getSLISourceConfigMap(t *testing.T) {
16-
type fields struct {
17-
Logger *keptncommon.Logger
18-
Event cloudevents.Event
19-
}
2024
type args struct {
2125
e *keptn.ConfigureMonitoringEventData
2226
}
2327
tests := []struct {
24-
name string
25-
fields fields
26-
args args
27-
want *v1.ConfigMap
28+
name string
29+
args args
30+
want *v1.ConfigMap
2831
}{
2932
{
3033
name: "configure for prometheus monitoring",
31-
fields: fields{
32-
Logger: nil,
33-
Event: cloudevents.Event{},
34-
},
3534
args: args{
3635
e: &keptn.ConfigureMonitoringEventData{
3736
Type: "prometheus",
@@ -51,10 +50,6 @@ func TestConfigureMonitoringHandler_getSLISourceConfigMap(t *testing.T) {
5150
},
5251
{
5352
name: "configure for dynatrace monitoring",
54-
fields: fields{
55-
Logger: nil,
56-
Event: cloudevents.Event{},
57-
},
5853
args: args{
5954
e: &keptn.ConfigureMonitoringEventData{
6055
Type: "dynatrace",
@@ -78,13 +73,127 @@ func TestConfigureMonitoringHandler_getSLISourceConfigMap(t *testing.T) {
7873

7974
for _, tt := range tests {
8075
t.Run(tt.name, func(t *testing.T) {
81-
eh := &ConfigureMonitoringHandler{
82-
KeptnHandler: nil,
83-
Event: tt.fields.Event,
84-
}
85-
if got := eh.getSLISourceConfigMap(tt.args.e); !reflect.DeepEqual(got, tt.want) {
76+
if got := getSLISourceConfigMap(tt.args.e); !reflect.DeepEqual(got, tt.want) {
8677
t.Errorf("getSLISourceConfigMap() = %v, want %v", got, tt.want)
8778
}
8879
})
8980
}
9081
}
82+
83+
func TestConfigureMonitoringHandler_HandleEvent_ConfigMapDoesntExistYet(t *testing.T) {
84+
ce := cloudevents.NewEvent()
85+
86+
configureMonitoringData := &keptnevents.ConfigureMonitoringEventData{
87+
Project: "my-project",
88+
Service: "my-service",
89+
Type: "my-sli-provider",
90+
}
91+
ce.SetType(keptnv2.GetTriggeredEventType(keptnv2.ConfigureMonitoringTaskName))
92+
ce.SetData(cloudevents.ApplicationJSON, configureMonitoringData)
93+
94+
logger, _ := test.NewNullLogger()
95+
96+
fakeK8sClient := fake.NewSimpleClientset()
97+
98+
handler, err := NewConfigureMonitoringHandler(ce, logger, WithK8sClient(fakeK8sClient))
99+
require.Nil(t, err)
100+
101+
err = handler.HandleEvent()
102+
require.Nil(t, err)
103+
require.Len(t, fakeK8sClient.Actions(), 1)
104+
require.Equal(t, "create", fakeK8sClient.Actions()[0].GetVerb())
105+
}
106+
107+
func TestConfigureMonitoringHandler_HandleEvent_ConfigMapDoesntExistYetAndCreateFails(t *testing.T) {
108+
ce := cloudevents.NewEvent()
109+
110+
configureMonitoringData := &keptnevents.ConfigureMonitoringEventData{
111+
Project: "my-project",
112+
Service: "my-service",
113+
Type: "my-sli-provider",
114+
}
115+
ce.SetType(keptnv2.GetTriggeredEventType(keptnv2.ConfigureMonitoringTaskName))
116+
ce.SetData(cloudevents.ApplicationJSON, configureMonitoringData)
117+
118+
logger, hook := test.NewNullLogger()
119+
120+
fakeK8sClient := fake.NewSimpleClientset()
121+
122+
fakeK8sClient.PrependReactor("create", "configmaps", func(action k8stesting.Action) (handled bool, ret runtime.Object, err error) {
123+
return true, nil, errors.New("oops")
124+
})
125+
126+
handler, err := NewConfigureMonitoringHandler(ce, logger, WithK8sClient(fakeK8sClient))
127+
require.Nil(t, err)
128+
129+
err = handler.HandleEvent()
130+
require.NotNil(t, err)
131+
require.Len(t, fakeK8sClient.Actions(), 1)
132+
require.Equal(t, "create", fakeK8sClient.Actions()[0].GetVerb())
133+
134+
require.NotNil(t, hook)
135+
require.NotEmpty(t, hook.Entries)
136+
require.Contains(t, hook.LastEntry().Message, "could not create")
137+
require.Equal(t, logrus.ErrorLevel, hook.LastEntry().Level)
138+
}
139+
140+
func TestConfigureMonitoringHandler_HandleEvent_ConfigMapAlreadyExists(t *testing.T) {
141+
ce := cloudevents.NewEvent()
142+
143+
configureMonitoringData := &keptnevents.ConfigureMonitoringEventData{
144+
Project: "my-project",
145+
Service: "my-service",
146+
Type: "my-sli-provider",
147+
}
148+
ce.SetType(keptnv2.GetTriggeredEventType(keptnv2.ConfigureMonitoringTaskName))
149+
ce.SetData(cloudevents.ApplicationJSON, configureMonitoringData)
150+
151+
logger, _ := test.NewNullLogger()
152+
153+
// initialize the fake k8s client with an already existing configmap for the project
154+
fakeK8sClient := fake.NewSimpleClientset(getSLISourceConfigMap(configureMonitoringData))
155+
156+
handler, err := NewConfigureMonitoringHandler(ce, logger, WithK8sClient(fakeK8sClient))
157+
require.Nil(t, err)
158+
159+
err = handler.HandleEvent()
160+
require.Nil(t, err)
161+
require.Len(t, fakeK8sClient.Actions(), 2)
162+
require.Equal(t, "create", fakeK8sClient.Actions()[0].GetVerb())
163+
require.Equal(t, "update", fakeK8sClient.Actions()[1].GetVerb())
164+
}
165+
166+
func TestConfigureMonitoringHandler_HandleEvent_ConfigMapAlreadyExistsUpdateFails(t *testing.T) {
167+
ce := cloudevents.NewEvent()
168+
169+
configureMonitoringData := &keptnevents.ConfigureMonitoringEventData{
170+
Project: "my-project",
171+
Service: "my-service",
172+
Type: "my-sli-provider",
173+
}
174+
ce.SetType(keptnv2.GetTriggeredEventType(keptnv2.ConfigureMonitoringTaskName))
175+
ce.SetData(cloudevents.ApplicationJSON, configureMonitoringData)
176+
177+
logger, hook := test.NewNullLogger()
178+
179+
// initialize the fake k8s client with an already existing configmap for the project
180+
fakeK8sClient := fake.NewSimpleClientset(getSLISourceConfigMap(configureMonitoringData))
181+
182+
fakeK8sClient.PrependReactor("update", "configmaps", func(action k8stesting.Action) (handled bool, ret runtime.Object, err error) {
183+
return true, nil, errors.New("oops")
184+
})
185+
186+
handler, err := NewConfigureMonitoringHandler(ce, logger, WithK8sClient(fakeK8sClient))
187+
require.Nil(t, err)
188+
189+
err = handler.HandleEvent()
190+
require.NotNil(t, err)
191+
require.Len(t, fakeK8sClient.Actions(), 2)
192+
require.Equal(t, "create", fakeK8sClient.Actions()[0].GetVerb())
193+
require.Equal(t, "update", fakeK8sClient.Actions()[1].GetVerb())
194+
195+
require.NotNil(t, hook)
196+
require.NotEmpty(t, hook.Entries)
197+
require.Contains(t, hook.LastEntry().Message, "could not update")
198+
require.Equal(t, logrus.ErrorLevel, hook.LastEntry().Level)
199+
}

0 commit comments

Comments
 (0)