Skip to content

Commit 4eaca42

Browse files
authored
fix node-exporter alerts (#389)
to show node hostname instead of ip address Signed-off-by: Andrei Kvapil <[email protected]> Signed-off-by: Andrei Kvapil <[email protected]>
1 parent b605c85 commit 4eaca42

1 file changed

Lines changed: 49 additions & 49 deletions

File tree

‎packages/system/monitoring/alerts/node-exporter.yaml‎

Lines changed: 49 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ spec:
1010
- alert: NodeFilesystemSpaceFillingUp
1111
annotations:
1212
description: Filesystem on {{ $labels.device }}, mounted on {{ $labels.mountpoint
13-
}}, at {{ $labels.instance }} has only {{ printf "%.2f" $value }}% available
13+
}}, at {{ $labels.node }} has only {{ printf "%.2f" $value }}% available
1414
space left and is filling up.
1515
runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodefilesystemspacefillingup
1616
summary: Filesystem is predicted to run out of space within the next 24 hours.
@@ -25,12 +25,12 @@ spec:
2525
for: 1h
2626
labels:
2727
severity: warning
28-
exported_instance: '{{ $labels.instance }}/{{ $labels.device }}'
28+
exported_instance: '{{ $labels.node }}/{{ $labels.device }}'
2929
service: node-exporter
3030
- alert: NodeFilesystemSpaceFillingUp
3131
annotations:
3232
description: Filesystem on {{ $labels.device }}, mounted on {{ $labels.mountpoint
33-
}}, at {{ $labels.instance }} has only {{ printf "%.2f" $value }}% available
33+
}}, at {{ $labels.node }} has only {{ printf "%.2f" $value }}% available
3434
space left and is filling up fast.
3535
runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodefilesystemspacefillingup
3636
summary: Filesystem is predicted to run out of space within the next 4 hours.
@@ -45,12 +45,12 @@ spec:
4545
for: 1h
4646
labels:
4747
severity: critical
48-
exported_instance: '{{ $labels.instance }}/{{ $labels.device }}'
48+
exported_instance: '{{ $labels.node }}/{{ $labels.device }}'
4949
service: node-exporter
5050
- alert: NodeFilesystemAlmostOutOfSpace
5151
annotations:
5252
description: Filesystem on {{ $labels.device }}, mounted on {{ $labels.mountpoint
53-
}}, at {{ $labels.instance }} has only {{ printf "%.2f" $value }}% available
53+
}}, at {{ $labels.node }} has only {{ printf "%.2f" $value }}% available
5454
space left.
5555
runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodefilesystemalmostoutofspace
5656
summary: Filesystem has less than 5% space left.
@@ -63,12 +63,12 @@ spec:
6363
for: 30m
6464
labels:
6565
severity: warning
66-
exported_instance: '{{ $labels.instance }}/{{ $labels.device }}'
66+
exported_instance: '{{ $labels.node }}/{{ $labels.device }}'
6767
service: node-exporter
6868
- alert: NodeFilesystemAlmostOutOfSpace
6969
annotations:
7070
description: Filesystem on {{ $labels.device }}, mounted on {{ $labels.mountpoint
71-
}}, at {{ $labels.instance }} has only {{ printf "%.2f" $value }}% available
71+
}}, at {{ $labels.node }} has only {{ printf "%.2f" $value }}% available
7272
space left.
7373
runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodefilesystemalmostoutofspace
7474
summary: Filesystem has less than 3% space left.
@@ -81,12 +81,12 @@ spec:
8181
for: 30m
8282
labels:
8383
severity: critical
84-
exported_instance: '{{ $labels.instance }}/{{ $labels.device }}'
84+
exported_instance: '{{ $labels.node }}/{{ $labels.device }}'
8585
service: node-exporter
8686
- alert: NodeFilesystemFilesFillingUp
8787
annotations:
8888
description: Filesystem on {{ $labels.device }}, mounted on {{ $labels.mountpoint
89-
}}, at {{ $labels.instance }} has only {{ printf "%.2f" $value }}% available
89+
}}, at {{ $labels.node }} has only {{ printf "%.2f" $value }}% available
9090
inodes left and is filling up.
9191
runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodefilesystemfilesfillingup
9292
summary: Filesystem is predicted to run out of inodes within the next 24 hours.
@@ -101,12 +101,12 @@ spec:
101101
for: 1h
102102
labels:
103103
severity: warning
104-
exported_instance: '{{ $labels.instance }}/{{ $labels.device }}'
104+
exported_instance: '{{ $labels.node }}/{{ $labels.device }}'
105105
service: node-exporter
106106
- alert: NodeFilesystemFilesFillingUp
107107
annotations:
108108
description: Filesystem on {{ $labels.device }}, mounted on {{ $labels.mountpoint
109-
}}, at {{ $labels.instance }} has only {{ printf "%.2f" $value }}% available
109+
}}, at {{ $labels.node }} has only {{ printf "%.2f" $value }}% available
110110
inodes left and is filling up fast.
111111
runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodefilesystemfilesfillingup
112112
summary: Filesystem is predicted to run out of inodes within the next 4 hours.
@@ -121,12 +121,12 @@ spec:
121121
for: 1h
122122
labels:
123123
severity: critical
124-
exported_instance: '{{ $labels.instance }}/{{ $labels.device }}'
124+
exported_instance: '{{ $labels.node }}/{{ $labels.device }}'
125125
service: node-exporter
126126
- alert: NodeFilesystemAlmostOutOfFiles
127127
annotations:
128128
description: Filesystem on {{ $labels.device }}, mounted on {{ $labels.mountpoint
129-
}}, at {{ $labels.instance }} has only {{ printf "%.2f" $value }}% available
129+
}}, at {{ $labels.node }} has only {{ printf "%.2f" $value }}% available
130130
inodes left.
131131
runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodefilesystemalmostoutoffiles
132132
summary: Filesystem has less than 5% inodes left.
@@ -139,12 +139,12 @@ spec:
139139
for: 1h
140140
labels:
141141
severity: warning
142-
exported_instance: '{{ $labels.instance }}/{{ $labels.device }}'
142+
exported_instance: '{{ $labels.node }}/{{ $labels.device }}'
143143
service: node-exporter
144144
- alert: NodeFilesystemAlmostOutOfFiles
145145
annotations:
146146
description: Filesystem on {{ $labels.device }}, mounted on {{ $labels.mountpoint
147-
}}, at {{ $labels.instance }} has only {{ printf "%.2f" $value }}% available
147+
}}, at {{ $labels.node }} has only {{ printf "%.2f" $value }}% available
148148
inodes left.
149149
runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodefilesystemalmostoutoffiles
150150
summary: Filesystem has less than 3% inodes left.
@@ -157,11 +157,11 @@ spec:
157157
for: 1h
158158
labels:
159159
severity: critical
160-
exported_instance: '{{ $labels.instance }}/{{ $labels.device }}'
160+
exported_instance: '{{ $labels.node }}/{{ $labels.device }}'
161161
service: node-exporter
162162
- alert: NodeNetworkReceiveErrs
163163
annotations:
164-
description: '{{ $labels.instance }} interface {{ $labels.device }} has encountered
164+
description: '{{ $labels.node }} interface {{ $labels.device }} has encountered
165165
{{ printf "%.0f" $value }} receive errors in the last two minutes.'
166166
runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodenetworkreceiveerrs
167167
summary: Network interface is reporting many receive errors.
@@ -170,11 +170,11 @@ spec:
170170
for: 1h
171171
labels:
172172
severity: warning
173-
exported_instance: '{{ $labels.instance }}/{{ $labels.device }}'
173+
exported_instance: '{{ $labels.node }}/{{ $labels.device }}'
174174
service: node-exporter
175175
- alert: NodeNetworkTransmitErrs
176176
annotations:
177-
description: '{{ $labels.instance }} interface {{ $labels.device }} has encountered
177+
description: '{{ $labels.node }} interface {{ $labels.device }} has encountered
178178
{{ printf "%.0f" $value }} transmit errors in the last two minutes.'
179179
runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodenetworktransmiterrs
180180
summary: Network interface is reporting many transmit errors.
@@ -183,7 +183,7 @@ spec:
183183
for: 1h
184184
labels:
185185
severity: warning
186-
exported_instance: '{{ $labels.instance }}/{{ $labels.device }}'
186+
exported_instance: '{{ $labels.node }}/{{ $labels.device }}'
187187
service: node-exporter
188188
- alert: NodeHighNumberConntrackEntriesUsed
189189
annotations:
@@ -194,22 +194,22 @@ spec:
194194
> 0.75
195195
labels:
196196
severity: warning
197-
exported_instance: '{{ $labels.instance }}'
197+
exported_instance: '{{ $labels.node }}'
198198
service: node-exporter
199199
- alert: NodeTextFileCollectorScrapeError
200200
annotations:
201-
description: Node Exporter text file collector on {{ $labels.instance }} failed
201+
description: Node Exporter text file collector on {{ $labels.node }} failed
202202
to scrape.
203203
runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodetextfilecollectorscrapeerror
204204
summary: Node Exporter text file collector failed to scrape.
205205
expr: node_textfile_scrape_error{job="node-exporter"} == 1
206206
labels:
207207
severity: warning
208-
exported_instance: '{{ $labels.instance }}'
208+
exported_instance: '{{ $labels.node }}'
209209
service: node-exporter
210210
- alert: NodeClockSkewDetected
211211
annotations:
212-
description: Clock at {{ $labels.instance }} is out of sync by more than 0.05s.
212+
description: Clock at {{ $labels.node }} is out of sync by more than 0.05s.
213213
Ensure NTP is configured correctly on this host.
214214
runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodeclockskewdetected
215215
summary: Clock skew detected.
@@ -228,11 +228,11 @@ spec:
228228
for: 10m
229229
labels:
230230
severity: warning
231-
exported_instance: '{{ $labels.instance }}'
231+
exported_instance: '{{ $labels.node }}'
232232
service: node-exporter
233233
- alert: NodeClockNotSynchronising
234234
annotations:
235-
description: Clock at {{ $labels.instance }} is not synchronising. Ensure
235+
description: Clock at {{ $labels.node }} is not synchronising. Ensure
236236
NTP is configured on this host.
237237
runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodeclocknotsynchronising
238238
summary: Clock not synchronising.
@@ -243,11 +243,11 @@ spec:
243243
for: 10m
244244
labels:
245245
severity: warning
246-
exported_instance: '{{ $labels.instance }}'
246+
exported_instance: '{{ $labels.node }}'
247247
service: node-exporter
248248
- alert: NodeRAIDDegraded
249249
annotations:
250-
description: RAID array '{{ $labels.device }}' at {{ $labels.instance }} is
250+
description: RAID array '{{ $labels.device }}' at {{ $labels.node }} is
251251
in degraded state due to one or more disks failures. Number of spare drives
252252
is insufficient to fix issue automatically.
253253
runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/noderaiddegraded
@@ -258,23 +258,23 @@ spec:
258258
for: 15m
259259
labels:
260260
severity: critical
261-
exported_instance: '{{ $labels.instance }}/{{ $labels.device }}'
261+
exported_instance: '{{ $labels.node }}/{{ $labels.device }}'
262262
service: node-exporter
263263
- alert: NodeRAIDDiskFailure
264264
annotations:
265-
description: At least one device in RAID array at {{ $labels.instance }} failed.
265+
description: At least one device in RAID array at {{ $labels.node }} failed.
266266
Array '{{ $labels.device }}' needs attention and possibly a disk swap.
267267
runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/noderaiddiskfailure
268268
summary: Failed device in RAID array.
269269
expr: node_md_disks{state="failed",job="node-exporter",device=~"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)"}
270270
> 0
271271
labels:
272272
severity: warning
273-
exported_instance: '{{ $labels.instance }}/{{ $labels.device }}'
273+
exported_instance: '{{ $labels.node }}/{{ $labels.device }}'
274274
service: node-exporter
275275
- alert: NodeFileDescriptorLimit
276276
annotations:
277-
description: File descriptors limit at {{ $labels.instance }} is currently
277+
description: File descriptors limit at {{ $labels.node }} is currently
278278
at {{ printf "%.2f" $value }}%.
279279
runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodefiledescriptorlimit
280280
summary: Kernel is predicted to exhaust file descriptors limit soon.
@@ -285,11 +285,11 @@ spec:
285285
for: 15m
286286
labels:
287287
severity: warning
288-
exported_instance: '{{ $labels.instance }}'
288+
exported_instance: '{{ $labels.node }}'
289289
service: node-exporter
290290
- alert: NodeFileDescriptorLimit
291291
annotations:
292-
description: File descriptors limit at {{ $labels.instance }} is currently
292+
description: File descriptors limit at {{ $labels.node }} is currently
293293
at {{ printf "%.2f" $value }}%.
294294
runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodefiledescriptorlimit
295295
summary: Kernel is predicted to exhaust file descriptors limit soon.
@@ -300,25 +300,25 @@ spec:
300300
for: 15m
301301
labels:
302302
severity: critical
303-
exported_instance: '{{ $labels.instance }}'
303+
exported_instance: '{{ $labels.node }}'
304304
service: node-exporter
305305
- alert: NodeCPUHighUsage
306306
annotations:
307307
description: |
308-
CPU usage at {{ $labels.instance }} has been above 90% for the last 15 minutes, is currently at {{ printf "%.2f" $value }}%.
308+
CPU usage at {{ $labels.node }} has been above 90% for the last 15 minutes, is currently at {{ printf "%.2f" $value }}%.
309309
runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodecpuhighusage
310310
summary: High CPU usage.
311311
expr: sum without(mode) (avg without (cpu) (rate(node_cpu_seconds_total{job="node-exporter",
312312
mode!="idle"}[2m]))) * 100 > 90
313313
for: 15m
314314
labels:
315315
severity: informational
316-
exported_instance: '{{ $labels.instance }}'
316+
exported_instance: '{{ $labels.node }}'
317317
service: node-exporter
318318
- alert: NodeSystemSaturation
319319
annotations:
320320
description: |
321-
System load per core at {{ $labels.instance }} has been above 2 for the last 15 minutes, is currently at {{ printf "%.2f" $value }}.
321+
System load per core at {{ $labels.node }} has been above 2 for the last 15 minutes, is currently at {{ printf "%.2f" $value }}.
322322
This might indicate this instance resources saturation and can cause it becoming unresponsive.
323323
runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodesystemsaturation
324324
summary: System saturated, load per core is very high.
@@ -328,38 +328,38 @@ spec:
328328
for: 15m
329329
labels:
330330
severity: warning
331-
exported_instance: '{{ $labels.instance }}'
331+
exported_instance: '{{ $labels.node }}'
332332
service: node-exporter
333333
- alert: NodeMemoryMajorPagesFaults
334334
annotations:
335335
description: |
336-
Memory major pages are occurring at very high rate at {{ $labels.instance }}, 500 major page faults per second for the last 15 minutes, is currently at {{ printf "%.2f" $value }}.
336+
Memory major pages are occurring at very high rate at {{ $labels.node }}, 500 major page faults per second for the last 15 minutes, is currently at {{ printf "%.2f" $value }}.
337337
Please check that there is enough memory available at this instance.
338338
runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodememorymajorpagesfaults
339339
summary: Memory major page faults are occurring at very high rate.
340340
expr: rate(node_vmstat_pgmajfault{job="node-exporter"}[5m]) > 500
341341
for: 15m
342342
labels:
343343
severity: warning
344-
exported_instance: '{{ $labels.instance }}'
344+
exported_instance: '{{ $labels.node }}'
345345
service: node-exporter
346346
- alert: NodeMemoryHighUtilization
347347
annotations:
348348
description: |
349-
Memory is filling up at {{ $labels.instance }}, has been above 90% for the last 15 minutes, is currently at {{ printf "%.2f" $value }}%.
349+
Memory is filling up at {{ $labels.node }}, has been above 90% for the last 15 minutes, is currently at {{ printf "%.2f" $value }}%.
350350
runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodememoryhighutilization
351351
summary: Host is running out of memory.
352352
expr: 100 - (node_memory_MemAvailable_bytes{job="node-exporter"} / node_memory_MemTotal_bytes{job="node-exporter"}
353353
* 100) > 90
354354
for: 15m
355355
labels:
356356
severity: warning
357-
exported_instance: '{{ $labels.instance }}'
357+
exported_instance: '{{ $labels.node }}'
358358
service: node-exporter
359359
- alert: NodeDiskIOSaturation
360360
annotations:
361361
description: |
362-
Disk IO queue (aqu-sq) is high on {{ $labels.device }} at {{ $labels.instance }}, has been above 10 for the last 30 minutes, is currently at {{ printf "%.2f" $value }}.
362+
Disk IO queue (aqu-sq) is high on {{ $labels.device }} at {{ $labels.node }}, has been above 10 for the last 30 minutes, is currently at {{ printf "%.2f" $value }}.
363363
This symptom might indicate disk saturation.
364364
runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodediskiosaturation
365365
summary: Disk IO queue is high.
@@ -368,29 +368,29 @@ spec:
368368
for: 30m
369369
labels:
370370
severity: warning
371-
exported_instance: '{{ $labels.instance }}/{{ $labels.device }}'
371+
exported_instance: '{{ $labels.node }}/{{ $labels.device }}'
372372
service: node-exporter
373373
- alert: NodeSystemdServiceFailed
374374
annotations:
375375
description: Systemd service {{ $labels.name }} has entered failed state at
376-
{{ $labels.instance }}
376+
{{ $labels.node }}
377377
runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodesystemdservicefailed
378378
summary: Systemd service has entered failed state.
379379
expr: node_systemd_unit_state{job="node-exporter", state="failed"} == 1
380380
for: 5m
381381
labels:
382382
severity: warning
383-
exported_instance: '{{ $labels.instance }}/{{ $labels.name }}'
383+
exported_instance: '{{ $labels.node }}/{{ $labels.name }}'
384384
service: node-exporter
385385
- alert: NodeBondingDegraded
386386
annotations:
387-
description: Bonding interface {{ $labels.master }} on {{ $labels.instance
387+
description: Bonding interface {{ $labels.master }} on {{ $labels.node
388388
}} is in degraded state due to one or more slave failures.
389389
runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodebondingdegraded
390390
summary: Bonding interface is degraded
391391
expr: (node_bonding_slaves - node_bonding_active) != 0
392392
for: 5m
393393
labels:
394394
severity: warning
395-
exported_instance: '{{ $labels.instance }}/{{ $labels.master }}'
395+
exported_instance: '{{ $labels.node }}/{{ $labels.master }}'
396396
service: node-exporter

0 commit comments

Comments
 (0)