Skip to content

Commit 38e4089

Browse files
yankawaimyasnikovdaniil
yankawai
authored and
myasnikovdaniil
committed
fix(etcd): stop counting canceled watch streams as failed gRPC requests
The tenant etcd copy of etcdHighNumberOfFailedGRPCRequests counted every code other than OK. A Watch stream is counted once, when it ends, and the streams of a kube-apiserver that stops end with Canceled. So whenever an API server on that etcd goes away, the Watch method of every member reads as 100 percent failed and the critical rule fires on all of them. The rules now count the codes the etcd mixin counts since etcd-io/etcd#13127, the same list the platform copy in monitoring-agents already uses. The grouping by method stays, so the alert labels do not change. Assisted-by: LLM Signed-off-by: Yan Bondarenko <[email protected]> (cherry picked from commit 1ca7c0d) Signed-off-by: cozystack-ci[bot] <274107086+cozystack-ci[bot]@users.noreply.github.com>
1 parent ec30722 commit 38e4089

3 files changed

Lines changed: 129 additions & 2 deletions

File tree

‎hack/etcd-grpc-alerts.bats‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
#!/usr/bin/env bats
2+
3+
load test_helper
4+
5+
REPO_ROOT="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")/.." && pwd)"
6+
VMALERT_TOOL_IMAGE="victoriametrics/vmalert-tool:v1.136.0@sha256:417346d7b6fdb66d8e3984887c2ad114535ec7d983eeef53381b56c776f333f1"
7+
8+
@test "tenant etcd failed gRPC request alerts pass their vmalert-tool cases" {
9+
docker info >/dev/null 2>&1 || {
10+
echo "docker is required to run the cases in $VMALERT_TOOL_IMAGE" >&2
11+
return 1
12+
}
13+
work=$(mktemp -d)
14+
helm template etcd "$REPO_ROOT/packages/extra/etcd" --namespace tenant-test \
15+
--set '_cluster.cluster-domain=cozy.local' \
16+
--show-only templates/prometheus-rules.yaml > "$work/rendered.yaml"
17+
yq '.spec' "$work/rendered.yaml" > "$work/rules.yaml"
18+
cp "$REPO_ROOT/hack/testdata/etcd-grpc-alerts/unittest.yaml" "$work/unittest.yaml"
19+
rc=0
20+
docker run --rm --network none -v "$work:/w:ro" "$VMALERT_TOOL_IMAGE" \
21+
unittest --files /w/unittest.yaml > "$work/out" 2>&1 || rc=$?
22+
cat "$work/out"
23+
passed=$(grep -cx 'SUCCESS' "$work/out" || true)
24+
rm -rf "$work"
25+
[ "$rc" -eq 0 ]
26+
[ "$passed" -eq 1 ]
27+
}

‎hack/testdata/etcd-grpc-alerts/unittest.yaml‎

Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,100 @@
1+
rule_files:
2+
- rules.yaml
3+
4+
evaluation_interval: 15s
5+
6+
tests:
7+
- name: watch streams of a stopped API server end with Canceled
8+
interval: 30s
9+
input_series:
10+
- series: 'grpc_server_handled_total{job="tenant-test/etcd-pod-scrape", instance="10.0.0.1:2381", grpc_service="etcdserverpb.Watch", grpc_method="Watch", grpc_type="bidi_stream", grpc_code="Canceled"}'
11+
values: '0+19x80'
12+
- series: 'grpc_server_handled_total{job="tenant-test/etcd-pod-scrape", instance="10.0.0.1:2381", grpc_service="etcdserverpb.Watch", grpc_method="Watch", grpc_type="bidi_stream", grpc_code="Unavailable"}'
13+
values: '2x80'
14+
- series: 'grpc_server_handled_total{job="tenant-test/etcd-pod-scrape", instance="10.0.0.1:2381", grpc_service="etcdserverpb.KV", grpc_method="Range", grpc_type="unary", grpc_code="OK"}'
15+
values: '0+30x80'
16+
alert_rule_test:
17+
- eval_time: 6m
18+
groupname: etcd
19+
alertname: etcdHighNumberOfFailedGRPCRequests
20+
- eval_time: 12m
21+
groupname: etcd
22+
alertname: etcdHighNumberOfFailedGRPCRequests
23+
- eval_time: 30m
24+
groupname: etcd
25+
alertname: etcdHighNumberOfFailedGRPCRequests
26+
27+
- name: range calls fail with Unavailable
28+
interval: 30s
29+
input_series:
30+
- series: 'grpc_server_handled_total{job="tenant-test/etcd-pod-scrape", instance="10.0.0.1:2381", grpc_service="etcdserverpb.KV", grpc_method="Range", grpc_type="unary", grpc_code="OK"}'
31+
values: '0+3x80'
32+
- series: 'grpc_server_handled_total{job="tenant-test/etcd-pod-scrape", instance="10.0.0.1:2381", grpc_service="etcdserverpb.KV", grpc_method="Range", grpc_type="unary", grpc_code="Unavailable"}'
33+
values: '0+1x80'
34+
alert_rule_test:
35+
- eval_time: 4m
36+
groupname: etcd
37+
alertname: etcdHighNumberOfFailedGRPCRequests
38+
- eval_time: 7m
39+
groupname: etcd
40+
alertname: etcdHighNumberOfFailedGRPCRequests
41+
exp_alerts:
42+
- exp_labels:
43+
job: tenant-test/etcd-pod-scrape
44+
instance: 10.0.0.1:2381
45+
grpc_service: etcdserverpb.KV
46+
grpc_method: Range
47+
severity: critical
48+
exp_annotations:
49+
summary: "etcd cluster 'tenant-test/etcd-pod-scrape': '25' of requests for 'Range' failed on etcd instance '10.0.0.1:2381'."
50+
- eval_time: 12m
51+
groupname: etcd
52+
alertname: etcdHighNumberOfFailedGRPCRequests
53+
exp_alerts:
54+
- exp_labels:
55+
job: tenant-test/etcd-pod-scrape
56+
instance: 10.0.0.1:2381
57+
grpc_service: etcdserverpb.KV
58+
grpc_method: Range
59+
severity: warning
60+
exp_annotations:
61+
summary: "etcd cluster 'tenant-test/etcd-pod-scrape': '25' of requests for 'Range' failed on etcd instance '10.0.0.1:2381'."
62+
- exp_labels:
63+
job: tenant-test/etcd-pod-scrape
64+
instance: 10.0.0.1:2381
65+
grpc_service: etcdserverpb.KV
66+
grpc_method: Range
67+
severity: critical
68+
exp_annotations:
69+
summary: "etcd cluster 'tenant-test/etcd-pod-scrape': '25' of requests for 'Range' failed on etcd instance '10.0.0.1:2381'."
70+
71+
- name: canceled watch streams next to failing range calls
72+
interval: 30s
73+
input_series:
74+
- series: 'grpc_server_handled_total{job="tenant-test/etcd-pod-scrape", instance="10.0.0.1:2381", grpc_service="etcdserverpb.Watch", grpc_method="Watch", grpc_type="bidi_stream", grpc_code="Canceled"}'
75+
values: '0+19x80'
76+
- series: 'grpc_server_handled_total{job="tenant-test/etcd-pod-scrape", instance="10.0.0.1:2381", grpc_service="etcdserverpb.KV", grpc_method="Range", grpc_type="unary", grpc_code="OK"}'
77+
values: '0+3x80'
78+
- series: 'grpc_server_handled_total{job="tenant-test/etcd-pod-scrape", instance="10.0.0.1:2381", grpc_service="etcdserverpb.KV", grpc_method="Range", grpc_type="unary", grpc_code="Unavailable"}'
79+
values: '0+1x80'
80+
alert_rule_test:
81+
- eval_time: 12m
82+
groupname: etcd
83+
alertname: etcdHighNumberOfFailedGRPCRequests
84+
exp_alerts:
85+
- exp_labels:
86+
job: tenant-test/etcd-pod-scrape
87+
instance: 10.0.0.1:2381
88+
grpc_service: etcdserverpb.KV
89+
grpc_method: Range
90+
severity: warning
91+
exp_annotations:
92+
summary: "etcd cluster 'tenant-test/etcd-pod-scrape': '25' of requests for 'Range' failed on etcd instance '10.0.0.1:2381'."
93+
- exp_labels:
94+
job: tenant-test/etcd-pod-scrape
95+
instance: 10.0.0.1:2381
96+
grpc_service: etcdserverpb.KV
97+
grpc_method: Range
98+
severity: critical
99+
exp_annotations:
100+
summary: "etcd cluster 'tenant-test/etcd-pod-scrape': '25' of requests for 'Range' failed on etcd instance '10.0.0.1:2381'."

‎packages/extra/etcd/templates/prometheus-rules.yaml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ spec:
3737
annotations:
3838
summary: "etcd cluster '{{`{{ $labels.job }}`}}': '{{`{{ $value }}`}}' of requests for '{{`{{ $labels.grpc_method }}`}}' failed on etcd instance '{{`{{ $labels.instance }}`}}'."
3939
expr: |
40-
100 * sum(rate(grpc_server_handled_total{job=~".*etcd.*", grpc_code!="OK"}[5m])) BY (job, instance, grpc_service, grpc_method)
40+
100 * sum(rate(grpc_server_handled_total{job=~".*etcd.*", grpc_code=~"Unknown|FailedPrecondition|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded"}[5m])) BY (job, instance, grpc_service, grpc_method)
4141
/
4242
sum(rate(grpc_server_handled_total{job=~".*etcd.*"}[5m])) BY (job, instance, grpc_service, grpc_method)
4343
> 1
@@ -49,7 +49,7 @@ spec:
4949
annotations:
5050
summary: "etcd cluster '{{`{{ $labels.job }}`}}': '{{`{{ $value }}`}}' of requests for '{{`{{ $labels.grpc_method }}`}}' failed on etcd instance '{{`{{ $labels.instance }}`}}'."
5151
expr: |
52-
100 * sum(rate(grpc_server_handled_total{job=~".*etcd.*", grpc_code!="OK"}[5m])) BY (job, instance, grpc_service, grpc_method)
52+
100 * sum(rate(grpc_server_handled_total{job=~".*etcd.*", grpc_code=~"Unknown|FailedPrecondition|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded"}[5m])) BY (job, instance, grpc_service, grpc_method)
5353
/
5454
sum(rate(grpc_server_handled_total{job=~".*etcd.*"}[5m])) BY (job, instance, grpc_service, grpc_method)
5555
> 5

0 commit comments

Comments
 (0)