Skip to content

Commit 861e6c4

Browse files
authored
feature/add-client-etcd-monitoring (#613)
<!-- This is an auto-generated comment: release notes by coderabbit.ai --> ## Summary by CodeRabbit - **New Features** - Enhanced etcd monitoring with new metrics exposure, pod scraping configuration, and comprehensive alert rules for proactive observability. - Introduced a new `VMPodScrape` resource for improved pod metrics collection. - Added a new PrometheusRule configuration for monitoring etcd clusters with various alert conditions. - **Chores** - Upgraded the etcd release from version 2.4.0 to 2.5.0. - Consolidated and renamed monitoring dashboard references for better consistency. <!-- end of auto-generated comment: release notes by coderabbit.ai -->
1 parent 835ee11 commit 861e6c4

9 files changed

Lines changed: 3766 additions & 1608 deletions

File tree

‎dashboards/control-plane/kube-etcd.json‎

Lines changed: 3611 additions & 0 deletions
Large diffs are not rendered by default.

‎dashboards/control-plane/kube-etcd3.json‎

Lines changed: 0 additions & 1602 deletions
This file was deleted.

‎hack/download-dashboards.sh‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ fix_d8() {
2121
}
2222

2323
swap_pvc_overview() {
24-
jq '(.panels[] | select(.title=="PVC Detailed") | .panels[] | select(.title=="Overview")) as $a | del(.panels[] | select(.title=="PVC Detailed").panels[] | select(.title=="Overview")) | ( (.panels[] | select(.title=="PVC Detailed"))) as $b | del( .panels[] | select(.title=="PVC Detailed")) | (.panels[.panels|length]=($a|.gridPos.y=$b.gridPos.y)) | (.panels[.panels|length]=($b|.gridPos.y=$a.gridPos.y))'
24+
jq '(.panels[] | select(.title=="PVC Detailed") | .panels[] | select(.title=="Overview")) as $a | del(.panels[] | select(.title=="PVC Detailed").panels[] | select(.title=="Overview")) | ( (.panels[] | select(.title=="PVC Detailed"))) as $b | del( .panels[] | select(.title=="PVC Detailed")) | (.panels[.panels|length]=($a|.gridPos.y=$b.gridPos.y)) | (.panels[.panels|length]=($b|.gridPos.y=$a.gridPos.y))'
2525
}
2626

2727
deprectaed_remove_faq() {
@@ -68,7 +68,7 @@ modules/402-ingress-nginx/monitoring/grafana-dashboards/ingress-nginx/namespace/
6868
modules/402-ingress-nginx/monitoring/grafana-dashboards/ingress-nginx/vhost/vhost_detail.json
6969
modules/402-ingress-nginx/monitoring/grafana-dashboards/ingress-nginx/vhost/vhosts.json
7070
modules/340-monitoring-kubernetes-control-plane/monitoring/grafana-dashboards/kubernetes-cluster/control-plane-status.json
71-
modules/340-monitoring-kubernetes-control-plane/monitoring/grafana-dashboards/kubernetes-cluster/kube-etcd3.json #TODO
71+
modules/340-monitoring-kubernetes-control-plane/monitoring/grafana-dashboards/kubernetes-cluster/kube-etcd.json #TODO
7272
modules/340-monitoring-kubernetes-control-plane/monitoring/grafana-dashboards/kubernetes-cluster/deprecated-resources.json
7373
modules/340-monitoring-kubernetes/monitoring/grafana-dashboards//kubernetes-cluster/nodes/ntp.json #TODO
7474
modules/340-monitoring-kubernetes/monitoring/grafana-dashboards//kubernetes-cluster/nodes/nodes.json
@@ -109,4 +109,3 @@ done <<\EOT
109109
https://raw.githubusercontent.com/dotdc/grafana-dashboards-kubernetes/master/dashboards/k8s-views-namespaces.json
110110
https://raw.githubusercontent.com/dotdc/grafana-dashboards-kubernetes/master/dashboards/k8s-views-pods.json
111111
EOT
112-

‎packages/extra/etcd/Chart.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,4 +3,4 @@ name: etcd
33
description: Storage for Kubernetes clusters
44
icon: /logos/etcd.svg
55
type: application
6-
version: 2.4.0
6+
version: 2.5.0

‎packages/extra/etcd/templates/etcd-cluster.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,12 @@ spec:
4040
labels:
4141
cozystack.io/service: etcd
4242
spec:
43+
containers:
44+
- name: etcd
45+
ports:
46+
- name: metrics
47+
containerPort: 2381
48+
protocol: TCP
4349
topologySpreadConstraints:
4450
- maxSkew: 1
4551
topologyKey: "kubernetes.io/hostname"
Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
apiVersion: operator.victoriametrics.com/v1beta1
2+
kind: VMPodScrape
3+
metadata:
4+
name: etcd-pod-scrape
5+
spec:
6+
podMetricsEndpoints:
7+
- port: metrics
8+
scheme: http
9+
selector:
10+
matchLabels:
11+
app.kubernetes.io/name: etcd
Lines changed: 132 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,132 @@
1+
apiVersion: monitoring.coreos.com/v1
2+
kind: PrometheusRule
3+
metadata:
4+
name: etcd-rules
5+
spec:
6+
groups:
7+
- name: etcd
8+
rules:
9+
- alert: etcdInsufficientMembers
10+
annotations:
11+
summary: "etcd cluster '{{`{{ $labels.job }}`}}': insufficient members '{{`{{ $value }}`}}'."
12+
expr: |
13+
sum(up{job=~".*etcd.*"} == bool 1) by (job) < ((count(up{job=~".*etcd.*"}) by (job) + 1) / 2)
14+
for: 3m
15+
labels:
16+
severity: critical
17+
18+
- alert: etcdNoLeader
19+
annotations:
20+
summary: "etcd cluster '{{`{{ $labels.job }}`}}': member '{{`{{ $labels.instance }}`}}' has no leader."
21+
expr: |
22+
etcd_server_has_leader{job=~".*etcd.*"} == 0
23+
for: 1m
24+
labels:
25+
severity: critical
26+
27+
- alert: etcdHighNumberOfLeaderChanges
28+
annotations:
29+
summary: "etcd cluster '{{`{{ $labels.job }}`}}': instance '{{`{{ $labels.instance }}`}}' has seen '{{`{{ $value }}`}}' leader changes within the last hour."
30+
expr: |
31+
rate(etcd_server_leader_changes_seen_total{job=~".*etcd.*"}[15m]) > 3
32+
for: 15m
33+
labels:
34+
severity: warning
35+
36+
- alert: etcdHighNumberOfFailedGRPCRequests
37+
annotations:
38+
summary: "etcd cluster '{{`{{ $labels.job }}`}}': '{{`{{ $value }}`}}' of requests for '{{`{{ $labels.grpc_method }}`}}' failed on etcd instance '{{`{{ $labels.instance }}`}}'."
39+
expr: |
40+
100 * sum(rate(grpc_server_handled_total{job=~".*etcd.*", grpc_code!="OK"}[5m])) BY (job, instance, grpc_service, grpc_method)
41+
/
42+
sum(rate(grpc_server_handled_total{job=~".*etcd.*"}[5m])) BY (job, instance, grpc_service, grpc_method)
43+
> 1
44+
for: 10m
45+
labels:
46+
severity: warning
47+
48+
- alert: etcdHighNumberOfFailedGRPCRequests
49+
annotations:
50+
summary: "etcd cluster '{{`{{ $labels.job }}`}}': '{{`{{ $value }}`}}' of requests for '{{`{{ $labels.grpc_method }}`}}' failed on etcd instance '{{`{{ $labels.instance }}`}}'."
51+
expr: |
52+
100 * sum(rate(grpc_server_handled_total{job=~".*etcd.*", grpc_code!="OK"}[5m])) BY (job, instance, grpc_service, grpc_method)
53+
/
54+
sum(rate(grpc_server_handled_total{job=~".*etcd.*"}[5m])) BY (job, instance, grpc_service, grpc_method)
55+
> 5
56+
for: 5m
57+
labels:
58+
severity: critical
59+
60+
- alert: etcdGRPCRequestsSlow
61+
annotations:
62+
summary: "etcd cluster '{{`{{ $labels.job }}`}}': gRPC requests to '{{`{{ $labels.grpc_method }}`}}' are taking '{{`{{ $value }}`}}' on etcd instance '{{`{{ $labels.instance }}`}}'."
63+
expr: |
64+
histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{job=~".*etcd.*", grpc_type="unary"}[5m])) by (job, instance, grpc_service, grpc_method, le))
65+
> 0.15
66+
for: 10m
67+
labels:
68+
severity: critical
69+
70+
- alert: etcdMemberCommunicationSlow
71+
annotations:
72+
summary: "etcd cluster '{{`{{ $labels.job }}`}}': member communication with '{{`{{ $labels.To }}`}}' is taking '{{`{{ $value }}`}}' on etcd instance '{{`{{ $labels.instance }}`}}'."
73+
expr: |
74+
histogram_quantile(0.99, rate(etcd_network_peer_round_trip_time_seconds_bucket{job=~".*etcd.*"}[5m]))
75+
> 0.15
76+
for: 10m
77+
labels:
78+
severity: warning
79+
80+
- alert: etcdHighNumberOfFailedProposals
81+
annotations:
82+
summary: "etcd cluster '{{`{{ $labels.job }}`}}': '{{`{{ $value }}`}}' proposal failures within the last hour on etcd instance '{{`{{ $labels.instance }}`}}'."
83+
expr: |
84+
rate(etcd_server_proposals_failed_total{job=~".*etcd.*"}[15m]) > 5
85+
for: 15m
86+
labels:
87+
severity: warning
88+
89+
- alert: etcdHighNumberOfFailedHTTPRequests
90+
annotations:
91+
summary: "'{{`{{ $value }}`}}' of requests for '{{`{{ $labels.method }}`}}' failed on etcd instance '{{`{{ $labels.instance }}`}}'."
92+
expr: |
93+
sum(rate(etcd_http_failed_total{job=~".*etcd.*", code!="404"}[5m])) BY (method) / sum(rate(etcd_http_received_total{job=~".*etcd.*"}[5m])) BY (method) > 0.01
94+
for: 10m
95+
labels:
96+
severity: warning
97+
98+
- alert: etcdHighNumberOfFailedHTTPRequests
99+
annotations:
100+
summary: "'{{`{{ $value }}`}}' of requests for '{{`{{ $labels.method }}`}}' failed on etcd instance '{{`{{ $labels.instance }}`}}'."
101+
expr: |
102+
sum(rate(etcd_http_failed_total{job=~".*etcd.*", code!="404"}[5m])) BY (method) / sum(rate(etcd_http_received_total{job=~".*etcd.*"}[5m])) BY (method) > 0.05
103+
for: 10m
104+
labels:
105+
severity: critical
106+
107+
- alert: etcdHTTPRequestsSlow
108+
annotations:
109+
summary: "etcd instance '{{`{{ $labels.instance }}`}}' HTTP requests to '{{`{{ $labels.method }}`}}' are slow."
110+
expr: |
111+
histogram_quantile(0.99, rate(etcd_http_successful_duration_seconds_bucket[5m]))
112+
> 0.15
113+
for: 10m
114+
labels:
115+
severity: warning
116+
117+
- alert: etcdMembersDown
118+
annotations:
119+
summary: "etcd cluster '{{`{{ $labels.job }}`}}' members are down."
120+
description: 'etcd cluster "{{`{{ $labels.job }}`}}": members are down {{`{{ $value }}`}}.'
121+
expr: |
122+
max without (endpoint) (
123+
sum without (instance, pod) (up{job=~".*etcd.*"} == bool 0)
124+
or
125+
count without (To) (
126+
sum without (instance, pod) (rate(etcd_network_peer_sent_failures_total{job=~".*etcd.*"}[120s])) > 0.01
127+
)
128+
)
129+
> 0
130+
for: 10m
131+
labels:
132+
severity: critical

‎packages/extra/monitoring/dashboards.list‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,5 +30,5 @@ main/nodes
3030
control-plane/control-plane-status
3131
control-plane/deprecated-resources
3232
control-plane/dns-coredns
33-
control-plane/kube-etcd3
33+
control-plane/kube-etcd
3434
kubevirt/kubevirt-control-plane

‎packages/extra/versions_map‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,8 @@ etcd 2.0.1 6fc1cc7d
55
etcd 2.1.0 2b00fcf8
66
etcd 2.2.0 5ca8823
77
etcd 2.3.0 b908400d
8-
etcd 2.4.0 HEAD
8+
etcd 2.4.0 cb7b8158
9+
etcd 2.5.0 HEAD
910
ingress 1.0.0 f642698
1011
ingress 1.1.0 838bee5d
1112
ingress 1.2.0 ced8e5b

0 commit comments

Comments
 (0)